1#[cfg(feature = "unwind")]
2use crate::unwind_winx64::create_unwind_info_from_insts;
3use crate::{
4 codegen_error,
5 common_decl::*,
6 emitter_x64::*,
7 location::{Location as AbstractLocation, Reg},
8 machine::*,
9 output_reporter::ChunkedOutputReporter,
10 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
11 x64_decl::{ArgumentRegisterAllocator, GPR, X64Register, XMM},
12};
13use dynasmrt::{DynasmError, VecAssembler, x64::X64Relocation};
14use fixedbitset::FixedBitSet;
15#[cfg(feature = "unwind")]
16use gimli::{X86_64, write::CallFrameInstruction};
17use std::{
18 collections::HashMap,
19 ops::{Deref, DerefMut},
20};
21use wasmer_compiler::{
22 types::{
23 address_map::InstructionAddressMap,
24 function::FunctionBody,
25 relocation::{Relocation, RelocationKind, RelocationTarget},
26 section::{CustomSection, CustomSectionProtection, SectionBody},
27 },
28 wasmparser::MemArg,
29};
30use wasmer_types::{
31 CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode,
32 TrapInformation, Type, VMOffsets,
33 target::{CallingConvention, CpuFeature, Target},
34 vmctx_offset,
35};
36
37type Assembler = VecAssembler<X64Relocation>;
38
39pub struct AssemblerX64 {
40 pub inner: Assembler,
42 pub simd_arch: CpuFeature,
45 pub target: Option<Target>,
47}
48
49impl AssemblerX64 {
50 fn new(baseaddr: usize, target: Option<Target>) -> Result<Self, CompileError> {
51 let simd_arch = target.as_ref().map_or_else(
52 || Ok(CpuFeature::SSE42),
53 |target| {
54 if target.cpu_features().contains(CpuFeature::AVX) {
55 Ok(CpuFeature::AVX)
56 } else if target.cpu_features().contains(CpuFeature::SSE42) {
57 Ok(CpuFeature::SSE42)
58 } else {
59 Err(CompileError::UnsupportedTarget(
60 "x86_64 without AVX or SSE 4.2, use -m avx to enable".to_string(),
61 ))
62 }
63 },
64 )?;
65
66 Ok(Self {
67 inner: Assembler::new(baseaddr),
68 simd_arch,
69 target,
70 })
71 }
72
73 fn finalize(self) -> Result<Vec<u8>, DynasmError> {
74 self.inner.finalize()
75 }
76}
77
78impl Deref for AssemblerX64 {
79 type Target = Assembler;
80
81 fn deref(&self) -> &Self::Target {
82 &self.inner
83 }
84}
85
86impl DerefMut for AssemblerX64 {
87 fn deref_mut(&mut self) -> &mut Self::Target {
88 &mut self.inner
89 }
90}
91
92type Location = AbstractLocation<GPR, XMM>;
93
94pub struct MachineX86_64 {
95 assembler: AssemblerX64,
96 used_gprs: FixedBitSet,
97 used_simd: FixedBitSet,
98 trap_table: TrapTable,
99 instructions_address_map: Vec<InstructionAddressMap>,
103 src_loc: u32,
105 unwind_ops: Vec<(usize, UnwindOps<GPR, XMM>)>,
107}
108
109pub(crate) const X86_64_RETURN_VALUE_REGISTERS: [GPR; 2] = [GPR::RAX, GPR::RDX];
112
113const DIV_REM_REGISTERS: [GPR; 2] = [GPR::RAX, GPR::RDX];
115
116impl MachineX86_64 {
117 pub fn new(target: Option<Target>) -> Result<Self, CompileError> {
118 let assembler = AssemblerX64::new(0, target)?;
119 Ok(MachineX86_64 {
120 assembler,
121 used_gprs: FixedBitSet::with_capacity(16),
122 used_simd: FixedBitSet::with_capacity(16),
123 trap_table: TrapTable::default(),
124 instructions_address_map: vec![],
125 src_loc: 0,
126 unwind_ops: vec![],
127 })
128 }
129 pub fn emit_relaxed_binop(
130 &mut self,
131 op: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
132 sz: Size,
133 src: Location,
134 dst: Location,
135 ) -> Result<(), CompileError> {
136 enum RelaxMode {
137 Direct,
138 SrcToGPR,
139 DstToGPR,
140 BothToGPR,
141 }
142 let mode = match (src, dst) {
143 (Location::GPR(_), Location::GPR(_))
144 if std::ptr::eq(op as *const u8, AssemblerX64::emit_imul as *const u8) =>
145 {
146 RelaxMode::Direct
147 }
148 _ if std::ptr::eq(op as *const u8, AssemblerX64::emit_imul as *const u8) => {
149 RelaxMode::BothToGPR
150 }
151
152 (Location::Memory(_, _), Location::Memory(_, _)) => RelaxMode::SrcToGPR,
153 (Location::Imm64(_), Location::Imm64(_)) | (Location::Imm64(_), Location::Imm32(_)) => {
154 RelaxMode::BothToGPR
155 }
156 (_, Location::Imm32(_)) | (_, Location::Imm64(_)) => RelaxMode::DstToGPR,
157 (Location::Imm64(_), Location::Memory(_, _)) => RelaxMode::SrcToGPR,
158 (Location::Imm64(_), Location::GPR(_))
159 if (op as *const u8 != AssemblerX64::emit_mov as *const u8) =>
160 {
161 RelaxMode::SrcToGPR
162 }
163 (_, Location::SIMD(_)) => RelaxMode::SrcToGPR,
164 _ => RelaxMode::Direct,
165 };
166
167 match mode {
168 RelaxMode::SrcToGPR => {
169 let temp = self.acquire_temp_gpr().ok_or_else(|| {
170 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
171 })?;
172 self.move_location(sz, src, Location::GPR(temp))?;
173 op(&mut self.assembler, sz, Location::GPR(temp), dst)?;
174 self.release_gpr(temp);
175 }
176 RelaxMode::DstToGPR => {
177 let temp = self.acquire_temp_gpr().ok_or_else(|| {
178 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
179 })?;
180 self.move_location(sz, dst, Location::GPR(temp))?;
181 op(&mut self.assembler, sz, src, Location::GPR(temp))?;
182 self.release_gpr(temp);
183 }
184 RelaxMode::BothToGPR => {
185 let temp_src = self.acquire_temp_gpr().ok_or_else(|| {
186 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
187 })?;
188 let temp_dst = self.acquire_temp_gpr().ok_or_else(|| {
189 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
190 })?;
191 self.move_location(sz, src, Location::GPR(temp_src))?;
192 self.move_location(sz, dst, Location::GPR(temp_dst))?;
193 op(
194 &mut self.assembler,
195 sz,
196 Location::GPR(temp_src),
197 Location::GPR(temp_dst),
198 )?;
199 match dst {
200 Location::Memory(_, _) | Location::GPR(_) => {
201 self.move_location(sz, Location::GPR(temp_dst), dst)?;
202 }
203 _ => {}
204 }
205 self.release_gpr(temp_dst);
206 self.release_gpr(temp_src);
207 }
208 RelaxMode::Direct => {
209 op(&mut self.assembler, sz, src, dst)?;
210 }
211 }
212 Ok(())
213 }
214 pub fn emit_relaxed_zx_sx(
215 &mut self,
216 op: fn(&mut AssemblerX64, Size, Location, Size, Location) -> Result<(), CompileError>,
217 sz_src: Size,
218 src: Location,
219 sz_dst: Size,
220 dst: Location,
221 ) -> Result<(), CompileError> {
222 match src {
223 Location::Imm32(_) | Location::Imm64(_) => {
224 let tmp_src = self.acquire_temp_gpr().ok_or_else(|| {
225 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
226 })?;
227 self.assembler
228 .emit_mov(Size::S64, src, Location::GPR(tmp_src))?;
229 let src = Location::GPR(tmp_src);
230
231 match dst {
232 Location::Imm32(_) | Location::Imm64(_) => unreachable!(),
233 Location::Memory(_, _) => {
234 let tmp_dst = self.acquire_temp_gpr().ok_or_else(|| {
235 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
236 })?;
237 op(
238 &mut self.assembler,
239 sz_src,
240 src,
241 sz_dst,
242 Location::GPR(tmp_dst),
243 )?;
244 self.move_location(Size::S64, Location::GPR(tmp_dst), dst)?;
245
246 self.release_gpr(tmp_dst);
247 }
248 Location::GPR(_) => {
249 op(&mut self.assembler, sz_src, src, sz_dst, dst)?;
250 }
251 _ => {
252 codegen_error!("singlepass emit_relaxed_zx_sx unreachable");
253 }
254 };
255
256 self.release_gpr(tmp_src);
257 }
258 Location::GPR(_) | Location::Memory(_, _) => {
259 match dst {
260 Location::Imm32(_) | Location::Imm64(_) => unreachable!(),
261 Location::Memory(_, _) => {
262 let tmp_dst = self.acquire_temp_gpr().ok_or_else(|| {
263 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
264 })?;
265 op(
266 &mut self.assembler,
267 sz_src,
268 src,
269 sz_dst,
270 Location::GPR(tmp_dst),
271 )?;
272 self.move_location(Size::S64, Location::GPR(tmp_dst), dst)?;
273
274 self.release_gpr(tmp_dst);
275 }
276 Location::GPR(_) => {
277 op(&mut self.assembler, sz_src, src, sz_dst, dst)?;
278 }
279 _ => {
280 codegen_error!("singlepass emit_relaxed_zx_sx unreachable");
281 }
282 };
283 }
284 _ => {
285 codegen_error!("singlepass emit_relaxed_zx_sx unreachable");
286 }
287 }
288 Ok(())
289 }
290 fn emit_binop_i32(
292 &mut self,
293 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
294 loc_a: Location,
295 loc_b: Location,
296 ret: Location,
297 ) -> Result<(), CompileError> {
298 if loc_a != ret {
299 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
300 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
301 })?;
302 self.emit_relaxed_mov(Size::S32, loc_a, Location::GPR(tmp))?;
303 self.emit_relaxed_binop(f, Size::S32, loc_b, Location::GPR(tmp))?;
304 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp), ret)?;
305 self.release_gpr(tmp);
306 } else {
307 self.emit_relaxed_binop(f, Size::S32, loc_b, ret)?;
308 }
309 Ok(())
310 }
311 fn emit_binop_i64(
313 &mut self,
314 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
315 loc_a: Location,
316 loc_b: Location,
317 ret: Location,
318 ) -> Result<(), CompileError> {
319 if loc_a != ret {
320 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
321 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
322 })?;
323 self.emit_relaxed_mov(Size::S64, loc_a, Location::GPR(tmp))?;
324 self.emit_relaxed_binop(f, Size::S64, loc_b, Location::GPR(tmp))?;
325 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp), ret)?;
326 self.release_gpr(tmp);
327 } else {
328 self.emit_relaxed_binop(f, Size::S64, loc_b, ret)?;
329 }
330 Ok(())
331 }
332 fn emit_cmpop_i64_dynamic_b(
334 &mut self,
335 c: Condition,
336 loc_a: Location,
337 loc_b: Location,
338 ret: Location,
339 ) -> Result<(), CompileError> {
340 match ret {
341 Location::GPR(x) => {
342 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
343 self.assembler.emit_set(c, x)?;
344 self.assembler
345 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x))?;
346 }
347 Location::Memory(_, _) => {
348 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
349 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
350 })?;
351 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
352 self.assembler.emit_set(c, tmp)?;
353 self.assembler
354 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp))?;
355 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
356 self.release_gpr(tmp);
357 }
358 _ => {
359 codegen_error!("singlepass emit_cmpop_i64_dynamic_b unreachable");
360 }
361 }
362 Ok(())
363 }
364 fn emit_shift_i64(
366 &mut self,
367 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
368 loc_a: Location,
369 loc_b: Location,
370 ret: Location,
371 ) -> Result<(), CompileError> {
372 self.assembler
373 .emit_mov(Size::S64, loc_b, Location::GPR(GPR::RCX))?;
374
375 if loc_a != ret {
376 self.emit_relaxed_mov(Size::S64, loc_a, ret)?;
377 }
378
379 f(&mut self.assembler, Size::S64, Location::GPR(GPR::RCX), ret)
380 }
381 fn emit_relaxed_xdiv(
383 &mut self,
384 op: fn(&mut AssemblerX64, Size, Location) -> Result<(), CompileError>,
385 sz: Size,
386 loc: Location,
387 integer_division_by_zero: Label,
388 ) -> Result<usize, CompileError> {
389 self.emit_relaxed_cmp(sz, Location::Imm32(0), loc)?;
390 self.assembler
391 .emit_jmp(Condition::Equal, integer_division_by_zero)?;
392
393 match loc {
394 Location::Imm64(_) | Location::Imm32(_) => {
395 self.move_location(sz, loc, Location::GPR(GPR::RCX))?; let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
397 op(&mut self.assembler, sz, Location::GPR(GPR::RCX))?;
398 self.mark_instruction_address_end(offset);
399 Ok(offset)
400 }
401 _ => {
402 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
403 op(&mut self.assembler, sz, loc)?;
404 self.mark_instruction_address_end(offset);
405 Ok(offset)
406 }
407 }
408 }
409 fn emit_cmpop_i32_dynamic_b(
411 &mut self,
412 c: Condition,
413 loc_a: Location,
414 loc_b: Location,
415 ret: Location,
416 ) -> Result<(), CompileError> {
417 match ret {
418 Location::GPR(x) => {
419 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
420 self.assembler.emit_set(c, x)?;
421 self.assembler
422 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x))?;
423 }
424 Location::Memory(_, _) => {
425 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
426 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
427 })?;
428 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
429 self.assembler.emit_set(c, tmp)?;
430 self.assembler
431 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp))?;
432 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
433 self.release_gpr(tmp);
434 }
435 _ => {
436 codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
437 }
438 }
439 Ok(())
440 }
441 fn emit_shift_i32(
443 &mut self,
444 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
445 loc_a: Location,
446 loc_b: Location,
447 ret: Location,
448 ) -> Result<(), CompileError> {
449 self.assembler
450 .emit_mov(Size::S32, loc_b, Location::GPR(GPR::RCX))?;
451
452 if loc_a != ret {
453 self.emit_relaxed_mov(Size::S32, loc_a, ret)?;
454 }
455
456 f(&mut self.assembler, Size::S32, Location::GPR(GPR::RCX), ret)
457 }
458
459 #[allow(clippy::too_many_arguments)]
460 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
461 &mut self,
462 addr: Location,
463 memarg: &MemArg,
464 check_alignment: bool,
465 value_size: usize,
466 need_check: bool,
467 imported_memories: bool,
468 offset: i32,
469 heap_access_oob: Label,
470 unaligned_atomic: Label,
471 cb: F,
472 ) -> Result<(), CompileError> {
473 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
479 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
480 })?;
481 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
482 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
483 })?;
484
485 let base_loc = if imported_memories {
487 self.emit_relaxed_binop(
489 AssemblerX64::emit_mov,
490 Size::S64,
491 Location::Memory(self.get_vmctx_reg(), offset),
492 Location::GPR(tmp2),
493 )?;
494 Location::Memory(tmp2, 0)
495 } else {
496 Location::Memory(self.get_vmctx_reg(), offset)
497 };
498
499 self.assembler
501 .emit_mov(Size::S64, base_loc, Location::GPR(tmp2))?;
502
503 self.assembler
507 .emit_mov(Size::S32, addr, Location::GPR(tmp_addr))?;
508
509 if memarg.offset != 0 {
511 self.assembler.emit_add(
512 Size::S32,
513 Location::Imm32(memarg.offset as u32),
514 Location::GPR(tmp_addr),
515 )?;
516
517 self.assembler.emit_jmp(Condition::Carry, heap_access_oob)?;
519 }
520
521 if need_check {
522 let bound_loc = if imported_memories {
523 self.emit_relaxed_binop(
525 AssemblerX64::emit_mov,
526 Size::S64,
527 Location::Memory(self.get_vmctx_reg(), offset),
528 Location::GPR(tmp2),
529 )?;
530 Location::Memory(tmp2, 8)
531 } else {
532 Location::Memory(self.get_vmctx_reg(), offset + 8)
533 };
534 self.assembler
535 .emit_mov(Size::S64, bound_loc, Location::GPR(tmp2))?;
536
537 self.assembler.emit_lea(
539 Size::S64,
540 Location::Memory(tmp2, -(value_size as i32)),
541 Location::GPR(tmp2),
542 )?;
543 self.emit_relaxed_cmp(Size::S64, Location::GPR(tmp2), Location::GPR(tmp_addr))?;
545
546 self.assembler.emit_jmp(Condition::Above, heap_access_oob)?;
548 }
549 let base_loc = if imported_memories {
551 self.emit_relaxed_binop(
553 AssemblerX64::emit_mov,
554 Size::S64,
555 Location::Memory(self.get_vmctx_reg(), offset),
556 Location::GPR(tmp2),
557 )?;
558 Location::Memory(tmp2, 0)
559 } else {
560 Location::Memory(self.get_vmctx_reg(), offset)
561 };
562 self.assembler
564 .emit_add(Size::S64, base_loc, Location::GPR(tmp_addr))?;
565
566 self.release_gpr(tmp2);
567
568 let align = value_size as u32;
569 if check_alignment && align != 1 {
570 let tmp_aligncheck = self.acquire_temp_gpr().ok_or_else(|| {
571 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
572 })?;
573 self.assembler.emit_mov(
574 Size::S32,
575 Location::GPR(tmp_addr),
576 Location::GPR(tmp_aligncheck),
577 )?;
578 self.assembler.emit_and(
579 Size::S64,
580 Location::Imm32(align - 1),
581 Location::GPR(tmp_aligncheck),
582 )?;
583 self.assembler
584 .emit_jmp(Condition::NotEqual, unaligned_atomic)?;
585 self.release_gpr(tmp_aligncheck);
586 }
587 let begin = self.assembler.get_offset().0;
588 cb(self, tmp_addr)?;
589 let end = self.assembler.get_offset().0;
590 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
591
592 self.release_gpr(tmp_addr);
593 Ok(())
594 }
595
596 #[allow(clippy::too_many_arguments)]
597 fn emit_compare_and_swap<F: FnOnce(&mut Self, GPR, GPR) -> Result<(), CompileError>>(
598 &mut self,
599 loc: Location,
600 target: Location,
601 ret: Location,
602 memarg: &MemArg,
603 value_size: usize,
604 memory_sz: Size,
605 stack_sz: Size,
606 need_check: bool,
607 imported_memories: bool,
608 offset: i32,
609 heap_access_oob: Label,
610 unaligned_atomic: Label,
611 cb: F,
612 ) -> Result<(), CompileError> {
613 if memory_sz > stack_sz {
614 codegen_error!("singlepass emit_compare_and_swap unreachable");
615 }
616
617 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
618 let value = if loc == Location::GPR(GPR::R14) {
619 GPR::R13
620 } else {
621 GPR::R14
622 };
623 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
624
625 self.move_location(stack_sz, loc, Location::GPR(value))?;
626
627 let retry = self.assembler.get_label();
628 self.emit_label(retry)?;
629
630 self.memory_op(
631 target,
632 memarg,
633 true,
634 value_size,
635 need_check,
636 imported_memories,
637 offset,
638 heap_access_oob,
639 unaligned_atomic,
640 |this, addr| {
641 this.load_address(memory_sz, Location::GPR(compare), Location::Memory(addr, 0))?;
642 this.move_location(stack_sz, Location::GPR(compare), ret)?;
643 cb(this, compare, value)?;
644 this.assembler.emit_lock_cmpxchg(
645 memory_sz,
646 Location::GPR(value),
647 Location::Memory(addr, 0),
648 )
649 },
650 )?;
651
652 self.assembler.emit_jmp(Condition::NotEqual, retry)?;
653
654 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
655 self.release_gpr(compare);
656 Ok(())
657 }
658
659 #[allow(clippy::too_many_arguments)]
661 fn emit_f32_int_conv_check(
662 &mut self,
663 reg: XMM,
664 lower_bound: f32,
665 upper_bound: f32,
666 underflow_label: Label,
667 overflow_label: Label,
668 nan_label: Label,
669 succeed_label: Label,
670 ) -> Result<(), CompileError> {
671 let lower_bound = f32::to_bits(lower_bound);
672 let upper_bound = f32::to_bits(upper_bound);
673
674 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
675 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
676 })?;
677 let tmp_x = self.acquire_temp_simd().ok_or_else(|| {
678 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
679 })?;
680
681 self.move_location(Size::S32, Location::Imm32(lower_bound), Location::GPR(tmp))?;
683 self.move_location(Size::S32, Location::GPR(tmp), Location::SIMD(tmp_x))?;
684 self.assembler
685 .emit_vcmpless(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
686 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
687 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
688 self.assembler
689 .emit_jmp(Condition::NotEqual, underflow_label)?;
690
691 self.move_location(Size::S32, Location::Imm32(upper_bound), Location::GPR(tmp))?;
693 self.move_location(Size::S32, Location::GPR(tmp), Location::SIMD(tmp_x))?;
694 self.assembler
695 .emit_vcmpgess(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
696 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
697 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
698 self.assembler
699 .emit_jmp(Condition::NotEqual, overflow_label)?;
700
701 self.assembler
703 .emit_vcmpeqss(reg, XMMOrMemory::XMM(reg), tmp_x)?;
704 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
705 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
706 self.assembler.emit_jmp(Condition::Equal, nan_label)?;
707
708 self.assembler.emit_jmp(Condition::None, succeed_label)?;
709
710 self.release_simd(tmp_x);
711 self.release_gpr(tmp);
712 Ok(())
713 }
714
715 fn emit_f32_int_conv_check_trap(
717 &mut self,
718 reg: XMM,
719 lower_bound: f32,
720 upper_bound: f32,
721 ) -> Result<(), CompileError> {
722 let trap_overflow = self.assembler.get_label();
723 let trap_badconv = self.assembler.get_label();
724 let end = self.assembler.get_label();
725
726 self.emit_f32_int_conv_check(
727 reg,
728 lower_bound,
729 upper_bound,
730 trap_overflow,
731 trap_overflow,
732 trap_badconv,
733 end,
734 )?;
735
736 self.emit_label(trap_overflow)?;
737
738 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
739
740 self.emit_label(trap_badconv)?;
741
742 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
743
744 self.emit_label(end)?;
745 Ok(())
746 }
747 #[allow(clippy::too_many_arguments)]
748 fn emit_f32_int_conv_check_sat<
749 F1: FnOnce(&mut Self) -> Result<(), CompileError>,
750 F2: FnOnce(&mut Self) -> Result<(), CompileError>,
751 F3: FnOnce(&mut Self) -> Result<(), CompileError>,
752 F4: FnOnce(&mut Self) -> Result<(), CompileError>,
753 >(
754 &mut self,
755 reg: XMM,
756 lower_bound: f32,
757 upper_bound: f32,
758 underflow_cb: F1,
759 overflow_cb: F2,
760 nan_cb: Option<F3>,
761 convert_cb: F4,
762 ) -> Result<(), CompileError> {
763 let underflow = self.assembler.get_label();
768 let overflow = self.assembler.get_label();
769 let nan = if nan_cb.is_some() {
770 self.assembler.get_label()
771 } else {
772 underflow
773 };
774 let convert = self.assembler.get_label();
775 let end = self.assembler.get_label();
776
777 self.emit_f32_int_conv_check(
778 reg,
779 lower_bound,
780 upper_bound,
781 underflow,
782 overflow,
783 nan,
784 convert,
785 )?;
786
787 self.emit_label(underflow)?;
788 underflow_cb(self)?;
789 self.assembler.emit_jmp(Condition::None, end)?;
790
791 self.emit_label(overflow)?;
792 overflow_cb(self)?;
793 self.assembler.emit_jmp(Condition::None, end)?;
794
795 if let Some(cb) = nan_cb {
796 self.emit_label(nan)?;
797 cb(self)?;
798 self.assembler.emit_jmp(Condition::None, end)?;
799 }
800
801 self.emit_label(convert)?;
802 convert_cb(self)?;
803 self.emit_label(end)
804 }
805 #[allow(clippy::too_many_arguments)]
807 fn emit_f64_int_conv_check(
808 &mut self,
809 reg: XMM,
810 lower_bound: f64,
811 upper_bound: f64,
812 underflow_label: Label,
813 overflow_label: Label,
814 nan_label: Label,
815 succeed_label: Label,
816 ) -> Result<(), CompileError> {
817 let lower_bound = f64::to_bits(lower_bound);
818 let upper_bound = f64::to_bits(upper_bound);
819
820 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
821 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
822 })?;
823 let tmp_x = self.acquire_temp_simd().ok_or_else(|| {
824 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
825 })?;
826
827 self.move_location(Size::S64, Location::Imm64(lower_bound), Location::GPR(tmp))?;
829 self.move_location(Size::S64, Location::GPR(tmp), Location::SIMD(tmp_x))?;
830 self.assembler
831 .emit_vcmplesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
832 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
833 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
834 self.assembler
835 .emit_jmp(Condition::NotEqual, underflow_label)?;
836
837 self.move_location(Size::S64, Location::Imm64(upper_bound), Location::GPR(tmp))?;
839 self.move_location(Size::S64, Location::GPR(tmp), Location::SIMD(tmp_x))?;
840 self.assembler
841 .emit_vcmpgesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
842 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
843 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
844 self.assembler
845 .emit_jmp(Condition::NotEqual, overflow_label)?;
846
847 self.assembler
849 .emit_vcmpeqsd(reg, XMMOrMemory::XMM(reg), tmp_x)?;
850 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
851 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
852 self.assembler.emit_jmp(Condition::Equal, nan_label)?;
853
854 self.assembler.emit_jmp(Condition::None, succeed_label)?;
855
856 self.release_simd(tmp_x);
857 self.release_gpr(tmp);
858 Ok(())
859 }
860 fn emit_f64_int_conv_check_trap(
862 &mut self,
863 reg: XMM,
864 lower_bound: f64,
865 upper_bound: f64,
866 ) -> Result<(), CompileError> {
867 let trap_overflow = self.assembler.get_label();
868 let trap_badconv = self.assembler.get_label();
869 let end = self.assembler.get_label();
870
871 self.emit_f64_int_conv_check(
872 reg,
873 lower_bound,
874 upper_bound,
875 trap_overflow,
876 trap_overflow,
877 trap_badconv,
878 end,
879 )?;
880
881 self.emit_label(trap_overflow)?;
882 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
883
884 self.emit_label(trap_badconv)?;
885 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
886
887 self.emit_label(end)
888 }
889 #[allow(clippy::too_many_arguments)]
890 fn emit_f64_int_conv_check_sat<
891 F1: FnOnce(&mut Self) -> Result<(), CompileError>,
892 F2: FnOnce(&mut Self) -> Result<(), CompileError>,
893 F3: FnOnce(&mut Self) -> Result<(), CompileError>,
894 F4: FnOnce(&mut Self) -> Result<(), CompileError>,
895 >(
896 &mut self,
897 reg: XMM,
898 lower_bound: f64,
899 upper_bound: f64,
900 underflow_cb: F1,
901 overflow_cb: F2,
902 nan_cb: Option<F3>,
903 convert_cb: F4,
904 ) -> Result<(), CompileError> {
905 let underflow = self.assembler.get_label();
910 let overflow = self.assembler.get_label();
911 let nan = if nan_cb.is_some() {
912 self.assembler.get_label()
913 } else {
914 underflow
915 };
916 let convert = self.assembler.get_label();
917 let end = self.assembler.get_label();
918
919 self.emit_f64_int_conv_check(
920 reg,
921 lower_bound,
922 upper_bound,
923 underflow,
924 overflow,
925 nan,
926 convert,
927 )?;
928
929 self.emit_label(underflow)?;
930 underflow_cb(self)?;
931 self.assembler.emit_jmp(Condition::None, end)?;
932
933 self.emit_label(overflow)?;
934 overflow_cb(self)?;
935 self.assembler.emit_jmp(Condition::None, end)?;
936
937 if let Some(cb) = nan_cb {
938 self.emit_label(nan)?;
939 cb(self)?;
940 self.assembler.emit_jmp(Condition::None, end)?;
941 }
942
943 self.emit_label(convert)?;
944 convert_cb(self)?;
945 self.emit_label(end)
946 }
947 fn emit_relaxed_avx(
949 &mut self,
950 op: fn(&mut AssemblerX64, XMM, XMMOrMemory, XMM) -> Result<(), CompileError>,
951 src1: Location,
952 src2: Location,
953 dst: Location,
954 ) -> Result<(), CompileError> {
955 self.emit_relaxed_avx_base(
956 |this, src1, src2, dst| op(&mut this.assembler, src1, src2, dst),
957 src1,
958 src2,
959 dst,
960 )
961 }
962
963 fn emit_relaxed_avx_base<
965 F: FnOnce(&mut Self, XMM, XMMOrMemory, XMM) -> Result<(), CompileError>,
966 >(
967 &mut self,
968 op: F,
969 src1: Location,
970 src2: Location,
971 dst: Location,
972 ) -> Result<(), CompileError> {
973 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
974 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
975 })?;
976 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
977 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
978 })?;
979 let tmp3 = self.acquire_temp_simd().ok_or_else(|| {
980 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
981 })?;
982 let tmpg = self.acquire_temp_gpr().ok_or_else(|| {
983 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
984 })?;
985
986 let src1 = match src1 {
987 Location::SIMD(x) => x,
988 Location::GPR(_) | Location::Memory(_, _) => {
989 self.assembler
990 .emit_mov(Size::S64, src1, Location::SIMD(tmp1))?;
991 tmp1
992 }
993 Location::Imm32(_) => {
994 self.assembler
995 .emit_mov(Size::S32, src1, Location::GPR(tmpg))?;
996 self.move_location(Size::S32, Location::GPR(tmpg), Location::SIMD(tmp1))?;
997 tmp1
998 }
999 Location::Imm64(_) => {
1000 self.assembler
1001 .emit_mov(Size::S64, src1, Location::GPR(tmpg))?;
1002 self.move_location(Size::S64, Location::GPR(tmpg), Location::SIMD(tmp1))?;
1003 tmp1
1004 }
1005 _ => {
1006 codegen_error!("singlepass emit_relaxed_avx_base unreachable")
1007 }
1008 };
1009
1010 let src2 = match src2 {
1011 Location::SIMD(x) => XMMOrMemory::XMM(x),
1012 Location::Memory(base, disp) => XMMOrMemory::Memory(base, disp),
1013 Location::GPR(_) => {
1014 self.assembler
1015 .emit_mov(Size::S64, src2, Location::SIMD(tmp2))?;
1016 XMMOrMemory::XMM(tmp2)
1017 }
1018 Location::Imm32(_) => {
1019 self.assembler
1020 .emit_mov(Size::S32, src2, Location::GPR(tmpg))?;
1021 self.move_location(Size::S32, Location::GPR(tmpg), Location::SIMD(tmp2))?;
1022 XMMOrMemory::XMM(tmp2)
1023 }
1024 Location::Imm64(_) => {
1025 self.assembler
1026 .emit_mov(Size::S64, src2, Location::GPR(tmpg))?;
1027 self.move_location(Size::S64, Location::GPR(tmpg), Location::SIMD(tmp2))?;
1028 XMMOrMemory::XMM(tmp2)
1029 }
1030 _ => {
1031 codegen_error!("singlepass emit_relaxed_avx_base unreachable")
1032 }
1033 };
1034
1035 match dst {
1036 Location::SIMD(x) => {
1037 op(self, src1, src2, x)?;
1038 }
1039 Location::Memory(_, _) | Location::GPR(_) => {
1040 op(self, src1, src2, tmp3)?;
1041 self.assembler
1042 .emit_mov(Size::S64, Location::SIMD(tmp3), dst)?;
1043 }
1044 _ => {
1045 codegen_error!("singlepass emit_relaxed_avx_base unreachable")
1046 }
1047 }
1048
1049 self.release_gpr(tmpg);
1050 self.release_simd(tmp3);
1051 self.release_simd(tmp2);
1052 self.release_simd(tmp1);
1053 Ok(())
1054 }
1055
1056 fn convert_i64_f64_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1057 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1058 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1059 })?;
1060 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1061 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1062 })?;
1063
1064 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1065 self.emit_f64_int_conv_check_sat(
1066 tmp_in,
1067 GEF64_LT_U64_MIN,
1068 LEF64_GT_U64_MAX,
1069 |this| {
1070 this.assembler
1071 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1072 },
1073 |this| {
1074 this.assembler.emit_mov(
1075 Size::S64,
1076 Location::Imm64(u64::MAX),
1077 Location::GPR(tmp_out),
1078 )
1079 },
1080 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1081 |this| {
1082 if this.assembler.arch_has_itruncf() {
1083 this.assembler.arch_emit_i64_trunc_uf64(tmp_in, tmp_out)
1084 } else {
1085 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
1086 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1087 })?;
1088 let tmp_x1 = this.acquire_temp_simd().ok_or_else(|| {
1089 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1090 })?;
1091 let tmp_x2 = this.acquire_temp_simd().ok_or_else(|| {
1092 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1093 })?;
1094
1095 this.assembler.emit_mov(
1096 Size::S64,
1097 Location::Imm64(4890909195324358656u64),
1098 Location::GPR(tmp),
1099 )?; this.assembler.emit_mov(
1101 Size::S64,
1102 Location::GPR(tmp),
1103 Location::SIMD(tmp_x1),
1104 )?;
1105 this.assembler.emit_mov(
1106 Size::S64,
1107 Location::SIMD(tmp_in),
1108 Location::SIMD(tmp_x2),
1109 )?;
1110 this.assembler
1111 .emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1112 this.assembler
1113 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1114 this.assembler.emit_mov(
1115 Size::S64,
1116 Location::Imm64(0x8000000000000000u64),
1117 Location::GPR(tmp),
1118 )?;
1119 this.assembler.emit_xor(
1120 Size::S64,
1121 Location::GPR(tmp_out),
1122 Location::GPR(tmp),
1123 )?;
1124 this.assembler
1125 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1126 this.assembler
1127 .emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1128 this.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1129
1130 this.release_simd(tmp_x2);
1131 this.release_simd(tmp_x1);
1132 this.release_gpr(tmp);
1133 Ok(())
1134 }
1135 },
1136 )?;
1137
1138 self.assembler
1139 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1140 self.release_simd(tmp_in);
1141 self.release_gpr(tmp_out);
1142 Ok(())
1143 }
1144 fn convert_i64_f64_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1145 if self.assembler.arch_has_itruncf() {
1146 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1147 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1148 })?;
1149 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1150 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1151 })?;
1152 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1153 self.assembler.arch_emit_i64_trunc_uf64(tmp_in, tmp_out)?;
1154 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1155 self.release_simd(tmp_in);
1156 self.release_gpr(tmp_out);
1157 } else {
1158 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1159 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1160 })?;
1161 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1162 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1163 })?; self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1166 self.emit_f64_int_conv_check_trap(tmp_in, GEF64_LT_U64_MIN, LEF64_GT_U64_MAX)?;
1167
1168 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1169 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1170 })?; let tmp_x1 = self.acquire_temp_simd().ok_or_else(|| {
1172 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1173 })?; let tmp_x2 = self.acquire_temp_simd().ok_or_else(|| {
1175 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1176 })?; self.move_location(
1179 Size::S64,
1180 Location::Imm64(4890909195324358656u64),
1181 Location::GPR(tmp),
1182 )?; self.move_location(Size::S64, Location::GPR(tmp), Location::SIMD(tmp_x1))?;
1184 self.move_location(Size::S64, Location::SIMD(tmp_in), Location::SIMD(tmp_x2))?;
1185 self.assembler
1186 .emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1187 self.assembler
1188 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1189 self.move_location(
1190 Size::S64,
1191 Location::Imm64(0x8000000000000000u64),
1192 Location::GPR(tmp),
1193 )?;
1194 self.assembler
1195 .emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp))?;
1196 self.assembler
1197 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1198 self.assembler
1199 .emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1200 self.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1201 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1202
1203 self.release_simd(tmp_x2);
1204 self.release_simd(tmp_x1);
1205 self.release_gpr(tmp);
1206 self.release_simd(tmp_in);
1207 self.release_gpr(tmp_out);
1208 }
1209 Ok(())
1210 }
1211 fn convert_i64_f64_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1212 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1213 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1214 })?;
1215 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1216 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1217 })?;
1218
1219 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1220 self.emit_f64_int_conv_check_sat(
1221 tmp_in,
1222 GEF64_LT_I64_MIN,
1223 LEF64_GT_I64_MAX,
1224 |this| {
1225 this.assembler.emit_mov(
1226 Size::S64,
1227 Location::Imm64(i64::MIN as u64),
1228 Location::GPR(tmp_out),
1229 )
1230 },
1231 |this| {
1232 this.assembler.emit_mov(
1233 Size::S64,
1234 Location::Imm64(i64::MAX as u64),
1235 Location::GPR(tmp_out),
1236 )
1237 },
1238 Some(|this: &mut Self| {
1239 this.assembler
1240 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1241 }),
1242 |this| {
1243 if this.assembler.arch_has_itruncf() {
1244 this.assembler.arch_emit_i64_trunc_sf64(tmp_in, tmp_out)
1245 } else {
1246 this.assembler
1247 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1248 }
1249 },
1250 )?;
1251
1252 self.assembler
1253 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1254 self.release_simd(tmp_in);
1255 self.release_gpr(tmp_out);
1256 Ok(())
1257 }
1258 fn convert_i64_f64_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1259 if self.assembler.arch_has_itruncf() {
1260 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1261 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1262 })?;
1263 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1264 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1265 })?;
1266 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1267 self.assembler.arch_emit_i64_trunc_sf64(tmp_in, tmp_out)?;
1268 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1269 self.release_simd(tmp_in);
1270 self.release_gpr(tmp_out);
1271 } else {
1272 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1273 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1274 })?;
1275 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1276 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1277 })?;
1278
1279 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1280 self.emit_f64_int_conv_check_trap(tmp_in, GEF64_LT_I64_MIN, LEF64_GT_I64_MAX)?;
1281
1282 self.assembler
1283 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1284 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1285
1286 self.release_simd(tmp_in);
1287 self.release_gpr(tmp_out);
1288 }
1289 Ok(())
1290 }
1291 fn convert_i32_f64_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1292 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1293 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1294 })?;
1295 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1296 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1297 })?;
1298
1299 let real_in = match loc {
1300 Location::Imm32(_) | Location::Imm64(_) => {
1301 self.move_location(Size::S64, loc, Location::GPR(tmp_out))?;
1302 self.move_location(Size::S64, Location::GPR(tmp_out), Location::SIMD(tmp_in))?;
1303 tmp_in
1304 }
1305 Location::SIMD(x) => x,
1306 _ => {
1307 self.move_location(Size::S64, loc, Location::SIMD(tmp_in))?;
1308 tmp_in
1309 }
1310 };
1311
1312 self.emit_f64_int_conv_check_sat(
1313 real_in,
1314 GEF64_LT_I32_MIN,
1315 LEF64_GT_I32_MAX,
1316 |this| {
1317 this.assembler.emit_mov(
1318 Size::S32,
1319 Location::Imm32(i32::MIN as u32),
1320 Location::GPR(tmp_out),
1321 )
1322 },
1323 |this| {
1324 this.assembler.emit_mov(
1325 Size::S32,
1326 Location::Imm32(i32::MAX as u32),
1327 Location::GPR(tmp_out),
1328 )
1329 },
1330 Some(|this: &mut Self| {
1331 this.assembler
1332 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1333 }),
1334 |this| {
1335 if this.assembler.arch_has_itruncf() {
1336 this.assembler.arch_emit_i32_trunc_sf64(tmp_in, tmp_out)
1337 } else {
1338 this.assembler
1339 .emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out)
1340 }
1341 },
1342 )?;
1343
1344 self.assembler
1345 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1346 self.release_simd(tmp_in);
1347 self.release_gpr(tmp_out);
1348 Ok(())
1349 }
1350 fn convert_i32_f64_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1351 if self.assembler.arch_has_itruncf() {
1352 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1353 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1354 })?;
1355 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1356 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1357 })?;
1358 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1359 self.assembler.arch_emit_i32_trunc_sf64(tmp_in, tmp_out)?;
1360 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1361 self.release_simd(tmp_in);
1362 self.release_gpr(tmp_out);
1363 } else {
1364 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1365 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1366 })?;
1367 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1368 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1369 })?;
1370
1371 let real_in = match loc {
1372 Location::Imm32(_) | Location::Imm64(_) => {
1373 self.move_location(Size::S64, loc, Location::GPR(tmp_out))?;
1374 self.move_location(Size::S64, Location::GPR(tmp_out), Location::SIMD(tmp_in))?;
1375 tmp_in
1376 }
1377 Location::SIMD(x) => x,
1378 _ => {
1379 self.move_location(Size::S64, loc, Location::SIMD(tmp_in))?;
1380 tmp_in
1381 }
1382 };
1383
1384 self.emit_f64_int_conv_check_trap(real_in, GEF64_LT_I32_MIN, LEF64_GT_I32_MAX)?;
1385
1386 self.assembler
1387 .emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out)?;
1388 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1389
1390 self.release_simd(tmp_in);
1391 self.release_gpr(tmp_out);
1392 }
1393 Ok(())
1394 }
1395 fn convert_i32_f64_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1396 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1397 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1398 })?;
1399 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1400 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1401 })?;
1402
1403 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1404 self.emit_f64_int_conv_check_sat(
1405 tmp_in,
1406 GEF64_LT_U32_MIN,
1407 LEF64_GT_U32_MAX,
1408 |this| {
1409 this.assembler
1410 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1411 },
1412 |this| {
1413 this.assembler.emit_mov(
1414 Size::S32,
1415 Location::Imm32(u32::MAX),
1416 Location::GPR(tmp_out),
1417 )
1418 },
1419 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1420 |this| {
1421 if this.assembler.arch_has_itruncf() {
1422 this.assembler.arch_emit_i32_trunc_uf64(tmp_in, tmp_out)
1423 } else {
1424 this.assembler
1425 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1426 }
1427 },
1428 )?;
1429
1430 self.assembler
1431 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1432 self.release_simd(tmp_in);
1433 self.release_gpr(tmp_out);
1434 Ok(())
1435 }
1436 fn convert_i32_f64_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1437 if self.assembler.arch_has_itruncf() {
1438 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1439 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1440 })?;
1441 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1442 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1443 })?;
1444 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1445 self.assembler.arch_emit_i32_trunc_uf64(tmp_in, tmp_out)?;
1446 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1447 self.release_simd(tmp_in);
1448 self.release_gpr(tmp_out);
1449 } else {
1450 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1451 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1452 })?;
1453 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1454 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1455 })?;
1456
1457 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1458 self.emit_f64_int_conv_check_trap(tmp_in, GEF64_LT_U32_MIN, LEF64_GT_U32_MAX)?;
1459
1460 self.assembler
1461 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1462 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1463
1464 self.release_simd(tmp_in);
1465 self.release_gpr(tmp_out);
1466 }
1467 Ok(())
1468 }
1469 fn convert_i64_f32_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1470 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1471 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1472 })?;
1473 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1474 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1475 })?;
1476
1477 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1478 self.emit_f32_int_conv_check_sat(
1479 tmp_in,
1480 GEF32_LT_U64_MIN,
1481 LEF32_GT_U64_MAX,
1482 |this| {
1483 this.assembler
1484 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1485 },
1486 |this| {
1487 this.assembler.emit_mov(
1488 Size::S64,
1489 Location::Imm64(u64::MAX),
1490 Location::GPR(tmp_out),
1491 )
1492 },
1493 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1494 |this| {
1495 if this.assembler.arch_has_itruncf() {
1496 this.assembler.arch_emit_i64_trunc_uf32(tmp_in, tmp_out)
1497 } else {
1498 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
1499 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1500 })?;
1501 let tmp_x1 = this.acquire_temp_simd().ok_or_else(|| {
1502 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1503 })?;
1504 let tmp_x2 = this.acquire_temp_simd().ok_or_else(|| {
1505 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1506 })?;
1507
1508 this.assembler.emit_mov(
1509 Size::S32,
1510 Location::Imm32(1593835520u32),
1511 Location::GPR(tmp),
1512 )?; this.assembler.emit_mov(
1514 Size::S32,
1515 Location::GPR(tmp),
1516 Location::SIMD(tmp_x1),
1517 )?;
1518 this.assembler.emit_mov(
1519 Size::S32,
1520 Location::SIMD(tmp_in),
1521 Location::SIMD(tmp_x2),
1522 )?;
1523 this.assembler
1524 .emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1525 this.assembler
1526 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1527 this.assembler.emit_mov(
1528 Size::S64,
1529 Location::Imm64(0x8000000000000000u64),
1530 Location::GPR(tmp),
1531 )?;
1532 this.assembler.emit_xor(
1533 Size::S64,
1534 Location::GPR(tmp_out),
1535 Location::GPR(tmp),
1536 )?;
1537 this.assembler
1538 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1539 this.assembler
1540 .emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1541 this.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1542
1543 this.release_simd(tmp_x2);
1544 this.release_simd(tmp_x1);
1545 this.release_gpr(tmp);
1546 Ok(())
1547 }
1548 },
1549 )?;
1550
1551 self.assembler
1552 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1553 self.release_simd(tmp_in);
1554 self.release_gpr(tmp_out);
1555 Ok(())
1556 }
1557 fn convert_i64_f32_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1558 if self.assembler.arch_has_itruncf() {
1559 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1560 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1561 })?;
1562 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1563 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1564 })?;
1565 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1566 self.assembler.arch_emit_i64_trunc_uf32(tmp_in, tmp_out)?;
1567 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1568 self.release_simd(tmp_in);
1569 self.release_gpr(tmp_out);
1570 } else {
1571 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1572 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1573 })?;
1574 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1575 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1576 })?; self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1579 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_U64_MIN, LEF32_GT_U64_MAX)?;
1580
1581 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1582 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1583 })?; let tmp_x1 = self.acquire_temp_simd().ok_or_else(|| {
1585 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1586 })?; let tmp_x2 = self.acquire_temp_simd().ok_or_else(|| {
1588 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1589 })?; self.move_location(
1592 Size::S32,
1593 Location::Imm32(1593835520u32),
1594 Location::GPR(tmp),
1595 )?; self.move_location(Size::S32, Location::GPR(tmp), Location::SIMD(tmp_x1))?;
1597 self.move_location(Size::S32, Location::SIMD(tmp_in), Location::SIMD(tmp_x2))?;
1598 self.assembler
1599 .emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1600 self.assembler
1601 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1602 self.move_location(
1603 Size::S64,
1604 Location::Imm64(0x8000000000000000u64),
1605 Location::GPR(tmp),
1606 )?;
1607 self.assembler
1608 .emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp))?;
1609 self.assembler
1610 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1611 self.assembler
1612 .emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1613 self.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1614 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1615
1616 self.release_simd(tmp_x2);
1617 self.release_simd(tmp_x1);
1618 self.release_gpr(tmp);
1619 self.release_simd(tmp_in);
1620 self.release_gpr(tmp_out);
1621 }
1622 Ok(())
1623 }
1624 fn convert_i64_f32_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1625 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1626 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1627 })?;
1628 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1629 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1630 })?;
1631
1632 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1633 self.emit_f32_int_conv_check_sat(
1634 tmp_in,
1635 GEF32_LT_I64_MIN,
1636 LEF32_GT_I64_MAX,
1637 |this| {
1638 this.assembler.emit_mov(
1639 Size::S64,
1640 Location::Imm64(i64::MIN as u64),
1641 Location::GPR(tmp_out),
1642 )
1643 },
1644 |this| {
1645 this.assembler.emit_mov(
1646 Size::S64,
1647 Location::Imm64(i64::MAX as u64),
1648 Location::GPR(tmp_out),
1649 )
1650 },
1651 Some(|this: &mut Self| {
1652 this.assembler
1653 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1654 }),
1655 |this| {
1656 if this.assembler.arch_has_itruncf() {
1657 this.assembler.arch_emit_i64_trunc_sf32(tmp_in, tmp_out)
1658 } else {
1659 this.assembler
1660 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1661 }
1662 },
1663 )?;
1664
1665 self.assembler
1666 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1667 self.release_simd(tmp_in);
1668 self.release_gpr(tmp_out);
1669 Ok(())
1670 }
1671 fn convert_i64_f32_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1672 if self.assembler.arch_has_itruncf() {
1673 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1674 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1675 })?;
1676 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1677 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1678 })?;
1679 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1680 self.assembler.arch_emit_i64_trunc_sf32(tmp_in, tmp_out)?;
1681 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1682 self.release_simd(tmp_in);
1683 self.release_gpr(tmp_out);
1684 } else {
1685 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1686 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1687 })?;
1688 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1689 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1690 })?;
1691
1692 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1693 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_I64_MIN, LEF32_GT_I64_MAX)?;
1694 self.assembler
1695 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1696 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1697
1698 self.release_simd(tmp_in);
1699 self.release_gpr(tmp_out);
1700 }
1701 Ok(())
1702 }
1703 fn convert_i32_f32_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1704 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1705 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1706 })?;
1707 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1708 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1709 })?;
1710
1711 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1712 self.emit_f32_int_conv_check_sat(
1713 tmp_in,
1714 GEF32_LT_I32_MIN,
1715 LEF32_GT_I32_MAX,
1716 |this| {
1717 this.assembler.emit_mov(
1718 Size::S32,
1719 Location::Imm32(i32::MIN as u32),
1720 Location::GPR(tmp_out),
1721 )
1722 },
1723 |this| {
1724 this.assembler.emit_mov(
1725 Size::S32,
1726 Location::Imm32(i32::MAX as u32),
1727 Location::GPR(tmp_out),
1728 )
1729 },
1730 Some(|this: &mut Self| {
1731 this.assembler
1732 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1733 }),
1734 |this| {
1735 if this.assembler.arch_has_itruncf() {
1736 this.assembler.arch_emit_i32_trunc_sf32(tmp_in, tmp_out)
1737 } else {
1738 this.assembler
1739 .emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out)
1740 }
1741 },
1742 )?;
1743
1744 self.assembler
1745 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1746 self.release_simd(tmp_in);
1747 self.release_gpr(tmp_out);
1748 Ok(())
1749 }
1750 fn convert_i32_f32_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1751 if self.assembler.arch_has_itruncf() {
1752 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1753 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1754 })?;
1755 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1756 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1757 })?;
1758 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1759 self.assembler.arch_emit_i32_trunc_sf32(tmp_in, tmp_out)?;
1760 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1761 self.release_simd(tmp_in);
1762 self.release_gpr(tmp_out);
1763 } else {
1764 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1765 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1766 })?;
1767 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1768 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1769 })?;
1770
1771 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1772 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_I32_MIN, LEF32_GT_I32_MAX)?;
1773
1774 self.assembler
1775 .emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1776 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1777
1778 self.release_simd(tmp_in);
1779 self.release_gpr(tmp_out);
1780 }
1781 Ok(())
1782 }
1783 fn convert_i32_f32_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1784 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1785 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1786 })?;
1787 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1788 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1789 })?;
1790 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1791 self.emit_f32_int_conv_check_sat(
1792 tmp_in,
1793 GEF32_LT_U32_MIN,
1794 LEF32_GT_U32_MAX,
1795 |this| {
1796 this.assembler
1797 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1798 },
1799 |this| {
1800 this.assembler.emit_mov(
1801 Size::S32,
1802 Location::Imm32(u32::MAX),
1803 Location::GPR(tmp_out),
1804 )
1805 },
1806 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1807 |this| {
1808 if this.assembler.arch_has_itruncf() {
1809 this.assembler.arch_emit_i32_trunc_uf32(tmp_in, tmp_out)
1810 } else {
1811 this.assembler
1812 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1813 }
1814 },
1815 )?;
1816
1817 self.assembler
1818 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1819 self.release_simd(tmp_in);
1820 self.release_gpr(tmp_out);
1821 Ok(())
1822 }
1823 fn convert_i32_f32_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1824 if self.assembler.arch_has_itruncf() {
1825 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1826 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1827 })?;
1828 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1829 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1830 })?;
1831 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1832 self.assembler.arch_emit_i32_trunc_uf32(tmp_in, tmp_out)?;
1833 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1834 self.release_simd(tmp_in);
1835 self.release_gpr(tmp_out);
1836 } else {
1837 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1838 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1839 })?;
1840 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1841 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1842 })?;
1843 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1844 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_U32_MIN, LEF32_GT_U32_MAX)?;
1845
1846 self.assembler
1847 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1848 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1849
1850 self.release_simd(tmp_in);
1851 self.release_gpr(tmp_out);
1852 }
1853 Ok(())
1854 }
1855
1856 fn emit_relaxed_atomic_xchg(
1857 &mut self,
1858 sz: Size,
1859 src: Location,
1860 dst: Location,
1861 ) -> Result<(), CompileError> {
1862 self.emit_relaxed_binop(AssemblerX64::emit_xchg, sz, src, dst)
1863 }
1864
1865 fn used_gprs_contains(&self, r: &GPR) -> bool {
1866 self.used_gprs.contains(r.into_index())
1867 }
1868 fn used_simd_contains(&self, r: &XMM) -> bool {
1869 self.used_simd.contains(r.into_index())
1870 }
1871 fn used_gprs_insert(&mut self, r: GPR) {
1872 self.used_gprs.insert(r.into_index());
1873 }
1874 fn used_simd_insert(&mut self, r: XMM) {
1875 self.used_simd.insert(r.into_index());
1876 }
1877 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1878 let ret = self.used_gprs_contains(r);
1879 self.used_gprs.set(r.into_index(), false);
1880 ret
1881 }
1882 fn used_simd_remove(&mut self, r: &XMM) -> bool {
1883 let ret = self.used_simd_contains(r);
1884 self.used_simd.set(r.into_index(), false);
1885 ret
1886 }
1887 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, XMM>) -> Result<(), CompileError> {
1888 self.unwind_ops.push((self.get_offset().0, op));
1889 Ok(())
1890 }
1891 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1892 let v = trap as u8;
1893 self.assembler.emit_ud1_payload(v)
1894 }
1895
1896 fn location_xor(
1898 &mut self,
1899 size: Size,
1900 source: Location,
1901 dest: Location,
1902 _flags: bool,
1903 ) -> Result<(), CompileError> {
1904 self.assembler.emit_xor(size, source, dest)
1905 }
1906 fn location_or(
1907 &mut self,
1908 size: Size,
1909 source: Location,
1910 dest: Location,
1911 _flags: bool,
1912 ) -> Result<(), CompileError> {
1913 self.assembler.emit_or(size, source, dest)
1914 }
1915 fn load_address(
1916 &mut self,
1917 size: Size,
1918 reg: Location,
1919 mem: Location,
1920 ) -> Result<(), CompileError> {
1921 match reg {
1922 Location::GPR(_) => {
1923 match mem {
1924 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
1925 if size < Size::S32 {
1927 self.assembler.emit_xor(Size::S32, reg, reg)?;
1928 }
1929 self.assembler.emit_mov(size, mem, reg)?;
1930 }
1931 _ => codegen_error!("singlepass load_address unreachable"),
1932 }
1933 }
1934 _ => codegen_error!("singlepass load_address unreachable"),
1935 }
1936 Ok(())
1937 }
1938
1939 fn location_neg(
1940 &mut self,
1941 size_val: Size, signed: bool,
1943 source: Location,
1944 size_op: Size,
1945 dest: Location,
1946 ) -> Result<(), CompileError> {
1947 self.move_location_extend(size_val, signed, source, size_op, dest)?;
1948 self.assembler.emit_neg(size_val, dest)
1949 }
1950
1951 fn emit_relaxed_zero_extension(
1952 &mut self,
1953 sz_src: Size,
1954 src: Location,
1955 sz_dst: Size,
1956 dst: Location,
1957 ) -> Result<(), CompileError> {
1958 if (sz_src == Size::S32 || sz_src == Size::S64) && sz_dst == Size::S64 {
1959 self.emit_relaxed_binop(AssemblerX64::emit_mov, sz_src, src, dst)
1960 } else {
1961 self.emit_relaxed_zx_sx(AssemblerX64::emit_movzx, sz_src, src, sz_dst, dst)
1962 }
1963 }
1964}
1965
1966impl Machine for MachineX86_64 {
1967 type GPR = GPR;
1968 type SIMD = XMM;
1969
1970 const STACK_ALIGNMENT: usize = 8;
1972
1973 fn assembler_get_offset(&self) -> Offset {
1974 self.assembler.get_offset()
1975 }
1976
1977 fn get_vmctx_reg(&self) -> GPR {
1978 GPR::R15
1979 }
1980
1981 fn get_used_gprs(&self) -> Vec<GPR> {
1982 GPR::iterator()
1983 .filter(|x| self.used_gprs.contains(x.into_index()))
1984 .cloned()
1985 .collect()
1986 }
1987
1988 fn get_used_simd(&self) -> Vec<XMM> {
1989 XMM::iterator()
1990 .filter(|x| self.used_simd.contains(x.into_index()))
1991 .cloned()
1992 .collect()
1993 }
1994
1995 fn pick_gpr(&self) -> Option<GPR> {
1996 use GPR::*;
1997 static REGS: &[GPR] = &[R8, R9, R10, R11];
1998 for r in REGS {
1999 if !self.used_gprs_contains(r) {
2000 return Some(*r);
2001 }
2002 }
2003 None
2004 }
2005
2006 fn pick_temp_gpr(&self) -> Option<GPR> {
2007 use GPR::*;
2008 static REGS: &[GPR] = &[RAX, RCX, RDX, RDI, RSI];
2009 for r in REGS {
2010 if !self.used_gprs_contains(r) {
2011 return Some(*r);
2012 }
2013 }
2014 None
2015 }
2016
2017 fn acquire_temp_gpr(&mut self) -> Option<GPR> {
2018 let gpr = self.pick_temp_gpr();
2019 if let Some(x) = gpr {
2020 self.used_gprs_insert(x);
2021 }
2022 gpr
2023 }
2024
2025 fn release_gpr(&mut self, gpr: GPR) {
2026 assert!(self.used_gprs_remove(&gpr));
2027 }
2028
2029 fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
2030 assert!(!self.used_gprs_contains(&gpr));
2031 self.used_gprs_insert(gpr);
2032 gpr
2033 }
2034
2035 fn reserve_gpr(&mut self, gpr: GPR) {
2036 self.used_gprs_insert(gpr);
2037 }
2038
2039 fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
2040 for r in used_gprs.iter() {
2041 self.assembler.emit_push(Size::S64, Location::GPR(*r))?;
2042 }
2043 Ok(used_gprs.len() * 8)
2044 }
2045
2046 fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
2047 for r in used_gprs.iter().rev() {
2048 self.assembler.emit_pop(Size::S64, Location::GPR(*r))?;
2049 }
2050 Ok(())
2051 }
2052
2053 fn pick_simd(&self) -> Option<XMM> {
2054 use XMM::*;
2055 static REGS: &[XMM] = &[XMM3, XMM4, XMM5, XMM6, XMM7];
2056 for r in REGS {
2057 if !self.used_simd_contains(r) {
2058 return Some(*r);
2059 }
2060 }
2061 None
2062 }
2063
2064 fn pick_temp_simd(&self) -> Option<XMM> {
2065 use XMM::*;
2066 static REGS: &[XMM] = &[XMM0, XMM1, XMM2];
2067 for r in REGS {
2068 if !self.used_simd_contains(r) {
2069 return Some(*r);
2070 }
2071 }
2072 None
2073 }
2074
2075 fn acquire_temp_simd(&mut self) -> Option<XMM> {
2076 let simd = self.pick_temp_simd();
2077 if let Some(x) = simd {
2078 self.used_simd_insert(x);
2079 }
2080 simd
2081 }
2082
2083 fn reserve_simd(&mut self, simd: XMM) {
2084 self.used_simd_insert(simd);
2085 }
2086
2087 fn release_simd(&mut self, simd: XMM) {
2088 assert!(self.used_simd_remove(&simd));
2089 }
2090
2091 fn push_used_simd(&mut self, used_xmms: &[XMM]) -> Result<usize, CompileError> {
2092 self.extend_stack((used_xmms.len() * 8) as u32)?;
2093
2094 for (i, r) in used_xmms.iter().enumerate() {
2095 self.move_location(
2096 Size::S64,
2097 Location::SIMD(*r),
2098 Location::Memory(GPR::RSP, (i * 8) as i32),
2099 )?;
2100 }
2101
2102 Ok(used_xmms.len() * 8)
2103 }
2104
2105 fn pop_used_simd(&mut self, used_xmms: &[XMM]) -> Result<(), CompileError> {
2106 for (i, r) in used_xmms.iter().enumerate() {
2107 self.move_location(
2108 Size::S64,
2109 Location::Memory(GPR::RSP, (i * 8) as i32),
2110 Location::SIMD(*r),
2111 )?;
2112 }
2113 self.assembler.emit_add(
2114 Size::S64,
2115 Location::Imm32((used_xmms.len() * 8) as u32),
2116 Location::GPR(GPR::RSP),
2117 )
2118 }
2119
2120 fn set_srcloc(&mut self, offset: u32) {
2121 self.src_loc = offset;
2122 }
2123
2124 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
2125 for i in begin..end {
2126 self.trap_table.offset_to_code.insert(i, code);
2127 }
2128 self.mark_instruction_address_end(begin);
2129 }
2130
2131 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
2132 let offset = self.assembler.get_offset().0;
2133 self.trap_table.offset_to_code.insert(offset, code);
2134 self.mark_instruction_address_end(offset);
2135 }
2136
2137 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
2138 let offset = self.assembler.get_offset().0;
2139 self.trap_table.offset_to_code.insert(offset, code);
2140 offset
2141 }
2142
2143 fn mark_instruction_address_end(&mut self, begin: usize) {
2144 self.instructions_address_map.push(InstructionAddressMap {
2145 srcloc: SourceLoc::new(self.src_loc),
2146 code_offset: begin,
2147 code_len: self.assembler.get_offset().0 - begin,
2148 });
2149 }
2150
2151 fn insert_stackoverflow(&mut self) {
2152 let offset = 0;
2153 self.trap_table
2154 .offset_to_code
2155 .insert(offset, TrapCode::StackOverflow);
2156 self.mark_instruction_address_end(offset);
2157 }
2158
2159 fn collect_trap_information(&self) -> Vec<TrapInformation> {
2160 self.trap_table
2161 .offset_to_code
2162 .clone()
2163 .into_iter()
2164 .map(|(offset, code)| TrapInformation {
2165 code_offset: offset as u32,
2166 trap_code: code,
2167 })
2168 .collect()
2169 }
2170
2171 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
2172 self.instructions_address_map.clone()
2173 }
2174
2175 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
2176 Location::Memory(GPR::RBP, -stack_offset)
2177 }
2178
2179 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2180 self.assembler.emit_sub(
2181 Size::S64,
2182 Location::Imm32(delta_stack_offset),
2183 Location::GPR(GPR::RSP),
2184 )
2185 }
2186
2187 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2188 self.assembler.emit_add(
2189 Size::S64,
2190 Location::Imm32(delta_stack_offset),
2191 Location::GPR(GPR::RSP),
2192 )
2193 }
2194
2195 fn move_location_for_native(
2196 &mut self,
2197 _size: Size,
2198 loc: Location,
2199 dest: Location,
2200 ) -> Result<(), CompileError> {
2201 match loc {
2202 Location::Imm64(_) | Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2203 let tmp = self.pick_temp_gpr();
2204 if let Some(x) = tmp {
2205 self.assembler.emit_mov(Size::S64, loc, Location::GPR(x))?;
2206 self.assembler.emit_mov(Size::S64, Location::GPR(x), dest)
2207 } else {
2208 self.assembler
2209 .emit_mov(Size::S64, Location::GPR(GPR::RAX), dest)?;
2210 self.assembler
2211 .emit_mov(Size::S64, loc, Location::GPR(GPR::RAX))?;
2212 self.assembler
2213 .emit_xchg(Size::S64, Location::GPR(GPR::RAX), dest)
2214 }
2215 }
2216 _ => self.assembler.emit_mov(Size::S64, loc, dest),
2217 }
2218 }
2219
2220 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
2221 self.assembler.emit_mov(size, Location::Imm32(0), location)
2222 }
2223
2224 fn local_pointer(&self) -> GPR {
2225 GPR::RBP
2226 }
2227
2228 fn is_local_on_stack(&self, idx: usize) -> bool {
2229 idx > 3
2230 }
2231
2232 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
2233 match idx {
2235 0 => Location::GPR(GPR::R12),
2236 1 => Location::GPR(GPR::R13),
2237 2 => Location::GPR(GPR::R14),
2238 3 => Location::GPR(GPR::RBX),
2239 _ => Location::Memory(GPR::RBP, -(((idx - 3) * 8 + callee_saved_regs_size) as i32)),
2240 }
2241 }
2242
2243 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
2244 self.assembler.emit_mov(
2245 Size::S64,
2246 location,
2247 Location::Memory(GPR::RBP, -stack_offset),
2248 )?;
2249 match location {
2250 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2251 reg: UnwindRegister::GPR(x),
2252 bp_neg_offset: stack_offset,
2253 }),
2254 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2255 reg: UnwindRegister::FPR(x),
2256 bp_neg_offset: stack_offset,
2257 }),
2258 _ => Ok(()),
2259 }
2260 }
2261
2262 fn get_param_registers(&self) -> &'static [Self::GPR] {
2263 &[GPR::RDI, GPR::RSI, GPR::RDX, GPR::RCX, GPR::R8, GPR::R9]
2264 }
2265
2266 fn get_param_location(
2267 &self,
2268 idx: usize,
2269 _sz: Size,
2270 stack_location: &mut usize,
2271 _calling_convention: CallingConvention,
2272 ) -> Location {
2273 self.get_param_registers().get(idx).map_or_else(
2274 || {
2275 let loc = Location::Memory(GPR::RSP, *stack_location as i32);
2276 *stack_location += 8;
2277 loc
2278 },
2279 |reg| Location::GPR(*reg),
2280 )
2281 }
2282
2283 fn get_call_param_location(
2284 &self,
2285 return_slots: usize,
2286 idx: usize,
2287 _sz: Size,
2288 _stack_location: &mut usize,
2289 _calling_convention: CallingConvention,
2290 ) -> Location {
2291 let register_params = self.get_param_registers();
2292 let return_values_memory_size =
2293 8 * return_slots.saturating_sub(X86_64_RETURN_VALUE_REGISTERS.len());
2294 register_params.get(idx).map_or_else(
2295 || {
2296 Location::Memory(
2297 GPR::RBP,
2298 (16 + return_values_memory_size + (idx - register_params.len()) * 8) as i32,
2299 )
2300 },
2301 |reg| Location::GPR(*reg),
2302 )
2303 }
2304
2305 fn get_simple_param_location(&self, idx: usize) -> Self::GPR {
2306 self.get_param_registers()[idx]
2307 }
2308
2309 fn adjust_gpr_param_location(
2310 &mut self,
2311 _register: Self::GPR,
2312 _size: Size,
2313 ) -> Result<(), CompileError> {
2314 Ok(())
2315 }
2316
2317 fn get_return_value_location(&self, idx: usize, stack_location: &mut usize) -> Location {
2318 X86_64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2319 || {
2320 let loc = Location::Memory(GPR::RSP, *stack_location as i32);
2321 *stack_location += 8;
2322 loc
2323 },
2324 |reg| Location::GPR(*reg),
2325 )
2326 }
2327
2328 fn get_call_return_value_location(&self, idx: usize) -> Location {
2329 X86_64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2330 || {
2331 Location::Memory(
2332 GPR::RBP,
2333 (16 + (idx - X86_64_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
2334 )
2335 },
2336 |reg| Location::GPR(*reg),
2337 )
2338 }
2339
2340 fn move_location(
2341 &mut self,
2342 size: Size,
2343 source: Location,
2344 dest: Location,
2345 ) -> Result<(), CompileError> {
2346 match source {
2347 Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
2348 Location::Memory(_, _) => match dest {
2349 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2350 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2351 let tmp = self.pick_temp_gpr().ok_or_else(|| {
2352 CompileError::Codegen("singlepass can't pick a temp gpr".to_owned())
2353 })?;
2354 self.assembler.emit_mov(size, source, Location::GPR(tmp))?;
2355 self.assembler.emit_mov(size, Location::GPR(tmp), dest)
2356 }
2357 _ => codegen_error!("singlepass move_location unreachable"),
2358 },
2359 Location::Memory2(_, _, _, _) => match dest {
2360 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2361 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2362 let tmp = self.pick_temp_gpr().ok_or_else(|| {
2363 CompileError::Codegen("singlepass can't pick a temp gpr".to_owned())
2364 })?;
2365 self.assembler.emit_mov(size, source, Location::GPR(tmp))?;
2366 self.assembler.emit_mov(size, Location::GPR(tmp), dest)
2367 }
2368 _ => codegen_error!("singlepass move_location unreachable"),
2369 },
2370 Location::Imm8(_) | Location::Imm32(_) | Location::Imm64(_) => match dest {
2371 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2372 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2373 let tmp = self.pick_temp_gpr().ok_or_else(|| {
2374 CompileError::Codegen("singlepass can't pick a temp gpr".to_owned())
2375 })?;
2376 self.assembler.emit_mov(size, source, Location::GPR(tmp))?;
2377 self.assembler.emit_mov(size, Location::GPR(tmp), dest)
2378 }
2379 _ => codegen_error!("singlepass move_location unreachable"),
2380 },
2381 Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2382 _ => codegen_error!("singlepass move_location unreachable"),
2383 }
2384 }
2385
2386 fn move_location_extend(
2387 &mut self,
2388 size_val: Size,
2389 signed: bool,
2390 source: Location,
2391 size_op: Size,
2392 dest: Location,
2393 ) -> Result<(), CompileError> {
2394 let dst = match dest {
2395 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2396 Location::GPR(self.acquire_temp_gpr().ok_or_else(|| {
2397 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2398 })?)
2399 }
2400 Location::GPR(_) | Location::SIMD(_) => dest,
2401 _ => codegen_error!("singlepass move_location_extend unreachable"),
2402 };
2403 match source {
2404 Location::GPR(_)
2405 | Location::Memory(_, _)
2406 | Location::Memory2(_, _, _, _)
2407 | Location::Imm32(_)
2408 | Location::Imm64(_) => match size_val {
2409 Size::S32 | Size::S64 => self.assembler.emit_mov(size_val, source, dst),
2410 Size::S16 | Size::S8 => {
2411 if signed {
2412 self.assembler.emit_movsx(size_val, source, size_op, dst)
2413 } else {
2414 self.assembler.emit_movzx(size_val, source, size_op, dst)
2415 }
2416 }
2417 },
2418 _ => panic!(
2419 "unimplemented move_location_extend({size_val:?}, {signed}, {source:?}, {size_op:?}, {dest:?}"
2420 ),
2421 }?;
2422 if dst != dest {
2423 self.assembler.emit_mov(size_op, dst, dest)?;
2424 match dst {
2425 Location::GPR(x) => self.release_gpr(x),
2426 _ => codegen_error!("singlepass move_location_extend unreachable"),
2427 };
2428 }
2429 Ok(())
2430 }
2431
2432 fn init_stack_loc(
2433 &mut self,
2434 init_stack_loc_cnt: u64,
2435 last_stack_loc: Location,
2436 ) -> Result<(), CompileError> {
2437 self.assembler.emit_mov(
2439 Size::S64,
2440 Location::Imm64(init_stack_loc_cnt),
2441 Location::GPR(GPR::RCX),
2442 )?;
2443 self.assembler
2444 .emit_xor(Size::S64, Location::GPR(GPR::RAX), Location::GPR(GPR::RAX))?;
2445 self.assembler
2446 .emit_lea(Size::S64, last_stack_loc, Location::GPR(GPR::RDI))?;
2447 self.assembler.emit_rep_stosq()
2448 }
2449
2450 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2451 self.assembler.emit_lea(
2452 Size::S64,
2453 Location::Memory(GPR::RBP, -saved_area_offset),
2454 Location::GPR(GPR::RSP),
2455 )
2456 }
2457
2458 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2459 self.assembler.emit_pop(Size::S64, location)
2460 }
2461
2462 fn assembler_finalize(
2463 self,
2464 assembly_comments: HashMap<usize, AssemblyComment>,
2465 ) -> Result<FinalizedAssembly, CompileError> {
2466 Ok(FinalizedAssembly {
2467 body: self.assembler.finalize().map_err(|e| {
2468 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2469 })?,
2470 assembly_comments,
2471 })
2472 }
2473
2474 fn get_offset(&self) -> Offset {
2475 self.assembler.get_offset()
2476 }
2477
2478 fn finalize_function(&mut self) -> Result<(), CompileError> {
2479 self.assembler.finalize_function()?;
2480 Ok(())
2481 }
2482
2483 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2484 self.emit_push(Size::S64, Location::GPR(GPR::RBP))?;
2485 self.emit_unwind_op(UnwindOps::PushFP { up_to_sp: 16 })?;
2486 self.move_location(Size::S64, Location::GPR(GPR::RSP), Location::GPR(GPR::RBP))?;
2487 self.emit_unwind_op(UnwindOps::DefineNewFrame)
2488 }
2489
2490 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2491 self.move_location(Size::S64, Location::GPR(GPR::RBP), Location::GPR(GPR::RSP))?;
2492 self.emit_pop(Size::S64, Location::GPR(GPR::RBP))
2493 }
2494
2495 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2496 self.move_location(
2497 Size::S64,
2498 Location::GPR(GPR::RAX),
2499 Location::SIMD(XMM::XMM0),
2500 )
2501 }
2502
2503 fn canonicalize_nan(
2504 &mut self,
2505 sz: Size,
2506 input: Location,
2507 output: Location,
2508 ) -> Result<(), CompileError> {
2509 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
2510 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
2511 })?;
2512 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
2513 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
2514 })?;
2515 let tmp3 = self.acquire_temp_simd().ok_or_else(|| {
2516 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
2517 })?;
2518
2519 self.emit_relaxed_mov(sz, input, Location::SIMD(tmp1))?;
2520 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
2521 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2522 })?;
2523
2524 match sz {
2525 Size::S32 => {
2526 self.assembler
2527 .emit_vcmpunordss(tmp1, XMMOrMemory::XMM(tmp1), tmp2)?;
2528 self.move_location(
2529 Size::S32,
2530 Location::Imm32(0x7FC0_0000), Location::GPR(tmpg1),
2532 )?;
2533 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp3))?;
2534 self.assembler
2535 .emit_vblendvps(tmp2, XMMOrMemory::XMM(tmp3), tmp1, tmp1)?;
2536 }
2537 Size::S64 => {
2538 self.assembler
2539 .emit_vcmpunordsd(tmp1, XMMOrMemory::XMM(tmp1), tmp2)?;
2540 self.move_location(
2541 Size::S64,
2542 Location::Imm64(0x7FF8_0000_0000_0000), Location::GPR(tmpg1),
2544 )?;
2545 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp3))?;
2546 self.assembler
2547 .emit_vblendvpd(tmp2, XMMOrMemory::XMM(tmp3), tmp1, tmp1)?;
2548 }
2549 _ => codegen_error!("singlepass canonicalize_nan unreachable"),
2550 }
2551
2552 self.emit_relaxed_mov(sz, Location::SIMD(tmp1), output)?;
2553
2554 self.release_gpr(tmpg1);
2555 self.release_simd(tmp3);
2556 self.release_simd(tmp2);
2557 self.release_simd(tmp1);
2558 Ok(())
2559 }
2560
2561 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2562 let v = trap as u8;
2572 let offset = self.assembler.get_offset().0;
2575 self.assembler.emit_ud1_payload(v)?;
2576 self.mark_instruction_address_end(offset);
2577 Ok(())
2578 }
2579
2580 fn get_label(&mut self) -> Label {
2581 self.assembler.new_dynamic_label()
2582 }
2583
2584 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2585 self.assembler.emit_label(label)
2586 }
2587
2588 fn get_gpr_for_call(&self) -> GPR {
2589 GPR::RAX
2590 }
2591
2592 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2593 self.assembler.emit_call_register(reg)
2594 }
2595
2596 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2597 self.assembler.emit_call_label(label)
2598 }
2599
2600 fn arch_emit_indirect_call_with_trampoline(
2601 &mut self,
2602 location: Location,
2603 ) -> Result<(), CompileError> {
2604 self.assembler
2605 .arch_emit_indirect_call_with_trampoline(location)
2606 }
2607
2608 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2609 self.assembler.emit_bkpt()
2610 }
2611
2612 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2613 self.assembler.emit_call_location(location)
2614 }
2615
2616 fn location_add(
2617 &mut self,
2618 size: Size,
2619 source: Location,
2620 dest: Location,
2621 _flags: bool,
2622 ) -> Result<(), CompileError> {
2623 self.assembler.emit_add(size, source, dest)
2624 }
2625
2626 fn location_cmp(
2627 &mut self,
2628 size: Size,
2629 source: Location,
2630 dest: Location,
2631 ) -> Result<(), CompileError> {
2632 self.emit_relaxed_cmp(size, source, dest)
2633 }
2634
2635 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2636 self.assembler.emit_jmp(Condition::None, label)
2637 }
2638
2639 fn jmp_on_condition(
2640 &mut self,
2641 cond: UnsignedCondition,
2642 size: Size,
2643 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2644 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2645 label: Label,
2646 ) -> Result<(), CompileError> {
2647 self.emit_relaxed_cmp(size, loc_b, loc_a)?;
2648 let cond = match cond {
2649 UnsignedCondition::Equal => Condition::Equal,
2650 UnsignedCondition::NotEqual => Condition::NotEqual,
2651 UnsignedCondition::Above => Condition::Above,
2652 UnsignedCondition::AboveEqual => Condition::AboveEqual,
2653 UnsignedCondition::Below => Condition::Below,
2654 UnsignedCondition::BelowEqual => Condition::BelowEqual,
2655 };
2656 self.assembler.emit_jmp(cond, label)
2657 }
2658
2659 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2660 let tmp1 = self
2661 .pick_temp_gpr()
2662 .ok_or_else(|| CompileError::Codegen("singlepass can't pick a temp gpr".to_owned()))?;
2663 self.reserve_gpr(tmp1);
2664 let tmp2 = self
2665 .pick_temp_gpr()
2666 .ok_or_else(|| CompileError::Codegen("singlepass can't pick a temp gpr".to_owned()))?;
2667 self.reserve_gpr(tmp2);
2668
2669 self.assembler.emit_lea_label(label, Location::GPR(tmp1))?;
2670 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2671
2672 let instr_size = self.assembler.get_jmp_instr_size();
2673 self.assembler
2674 .emit_imul_imm32_gpr64(instr_size as _, tmp2)?;
2675 self.assembler
2676 .emit_add(Size::S64, Location::GPR(tmp1), Location::GPR(tmp2))?;
2677 self.assembler.emit_jmp_location(Location::GPR(tmp2))?;
2678 self.release_gpr(tmp2);
2679 self.release_gpr(tmp1);
2680 Ok(())
2681 }
2682
2683 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2684 match self.assembler.get_offset().0 % 16 {
2688 0 => {}
2689 x => {
2690 self.assembler.emit_nop_n(16 - x)?;
2691 }
2692 }
2693 assert_eq!(self.assembler.get_offset().0 % 16, 0);
2694 Ok(())
2695 }
2696
2697 fn emit_ret(&mut self) -> Result<(), CompileError> {
2698 self.assembler.emit_ret()
2699 }
2700
2701 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2702 self.assembler.emit_push(size, loc)
2703 }
2704
2705 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2706 self.assembler.emit_pop(size, loc)
2707 }
2708
2709 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2710 Ok(())
2712 }
2713
2714 fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2715 match size {
2716 Size::S64 => self.assembler.emit_imul_imm32_gpr64(imm32, gpr),
2717 _ => {
2718 codegen_error!("singlepass emit_imul_imm32 unreachable");
2719 }
2720 }
2721 }
2722
2723 fn emit_relaxed_mov(
2724 &mut self,
2725 sz: Size,
2726 src: Location,
2727 dst: Location,
2728 ) -> Result<(), CompileError> {
2729 self.emit_relaxed_binop(AssemblerX64::emit_mov, sz, src, dst)
2730 }
2731
2732 fn emit_relaxed_cmp(
2733 &mut self,
2734 sz: Size,
2735 src: Location,
2736 dst: Location,
2737 ) -> Result<(), CompileError> {
2738 self.emit_relaxed_binop(AssemblerX64::emit_cmp, sz, src, dst)
2739 }
2740
2741 fn emit_relaxed_sign_extension(
2742 &mut self,
2743 sz_src: Size,
2744 src: Location,
2745 sz_dst: Size,
2746 dst: Location,
2747 ) -> Result<(), CompileError> {
2748 self.emit_relaxed_zx_sx(AssemblerX64::emit_movsx, sz_src, src, sz_dst, dst)
2749 }
2750
2751 fn emit_binop_add32(
2752 &mut self,
2753 loc_a: Location,
2754 loc_b: Location,
2755 ret: Location,
2756 ) -> Result<(), CompileError> {
2757 self.emit_binop_i32(AssemblerX64::emit_add, loc_a, loc_b, ret)
2758 }
2759
2760 fn emit_binop_sub32(
2761 &mut self,
2762 loc_a: Location,
2763 loc_b: Location,
2764 ret: Location,
2765 ) -> Result<(), CompileError> {
2766 self.emit_binop_i32(AssemblerX64::emit_sub, loc_a, loc_b, ret)
2767 }
2768
2769 fn emit_binop_mul32(
2770 &mut self,
2771 loc_a: Location,
2772 loc_b: Location,
2773 ret: Location,
2774 ) -> Result<(), CompileError> {
2775 self.emit_binop_i32(AssemblerX64::emit_imul, loc_a, loc_b, ret)
2776 }
2777
2778 fn emit_binop_udiv32(
2779 &mut self,
2780 loc_a: Location,
2781 loc_b: Location,
2782 ret: Location,
2783 integer_division_by_zero: Label,
2784 ) -> Result<usize, CompileError> {
2785 for reg in DIV_REM_REGISTERS {
2786 self.reserve_unused_temp_gpr(reg);
2787 }
2788 self.assembler
2789 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2790 self.assembler
2791 .emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
2792 let offset = self.emit_relaxed_xdiv(
2793 AssemblerX64::emit_div,
2794 Size::S32,
2795 loc_b,
2796 integer_division_by_zero,
2797 )?;
2798 self.assembler
2799 .emit_mov(Size::S32, Location::GPR(GPR::RAX), ret)?;
2800 for reg in DIV_REM_REGISTERS {
2801 self.release_gpr(reg);
2802 }
2803 Ok(offset)
2804 }
2805
2806 fn emit_binop_sdiv32(
2807 &mut self,
2808 loc_a: Location,
2809 loc_b: Location,
2810 ret: Location,
2811 integer_division_by_zero: Label,
2812 _integer_overflow: Label,
2813 ) -> Result<usize, CompileError> {
2814 for reg in DIV_REM_REGISTERS {
2815 self.reserve_unused_temp_gpr(reg);
2816 }
2817 self.assembler
2818 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2819 self.assembler.emit_cdq()?;
2820 let offset = self.emit_relaxed_xdiv(
2821 AssemblerX64::emit_idiv,
2822 Size::S32,
2823 loc_b,
2824 integer_division_by_zero,
2825 )?;
2826 self.assembler
2827 .emit_mov(Size::S32, Location::GPR(GPR::RAX), ret)?;
2828 for reg in DIV_REM_REGISTERS {
2829 self.release_gpr(reg);
2830 }
2831 Ok(offset)
2832 }
2833
2834 fn emit_binop_urem32(
2835 &mut self,
2836 loc_a: Location,
2837 loc_b: Location,
2838 ret: Location,
2839 integer_division_by_zero: Label,
2840 ) -> Result<usize, CompileError> {
2841 for reg in DIV_REM_REGISTERS {
2842 self.reserve_unused_temp_gpr(reg);
2843 }
2844 self.assembler
2845 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2846 self.assembler
2847 .emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
2848 let offset = self.emit_relaxed_xdiv(
2849 AssemblerX64::emit_div,
2850 Size::S32,
2851 loc_b,
2852 integer_division_by_zero,
2853 )?;
2854 self.assembler
2855 .emit_mov(Size::S32, Location::GPR(GPR::RDX), ret)?;
2856 for reg in DIV_REM_REGISTERS {
2857 self.release_gpr(reg);
2858 }
2859 Ok(offset)
2860 }
2861
2862 fn emit_binop_srem32(
2863 &mut self,
2864 loc_a: Location,
2865 loc_b: Location,
2866 ret: Location,
2867 integer_division_by_zero: Label,
2868 ) -> Result<usize, CompileError> {
2869 for reg in DIV_REM_REGISTERS {
2870 self.reserve_unused_temp_gpr(reg);
2871 }
2872 let normal_path = self.assembler.get_label();
2873 let end = self.assembler.get_label();
2874
2875 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0x80000000), loc_a)?;
2876 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
2877 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0xffffffff), loc_b)?;
2878 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
2879 self.move_location(Size::S32, Location::Imm32(0), ret)?;
2880 self.assembler.emit_jmp(Condition::None, end)?;
2881
2882 self.emit_label(normal_path)?;
2883 self.assembler
2884 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2885 self.assembler.emit_cdq()?;
2886 let offset = self.emit_relaxed_xdiv(
2887 AssemblerX64::emit_idiv,
2888 Size::S32,
2889 loc_b,
2890 integer_division_by_zero,
2891 )?;
2892 self.assembler
2893 .emit_mov(Size::S32, Location::GPR(GPR::RDX), ret)?;
2894
2895 self.emit_label(end)?;
2896 for reg in DIV_REM_REGISTERS {
2897 self.release_gpr(reg);
2898 }
2899 Ok(offset)
2900 }
2901
2902 fn emit_binop_and32(
2903 &mut self,
2904 loc_a: Location,
2905 loc_b: Location,
2906 ret: Location,
2907 ) -> Result<(), CompileError> {
2908 self.emit_binop_i32(AssemblerX64::emit_and, loc_a, loc_b, ret)
2909 }
2910
2911 fn emit_binop_or32(
2912 &mut self,
2913 loc_a: Location,
2914 loc_b: Location,
2915 ret: Location,
2916 ) -> Result<(), CompileError> {
2917 self.emit_binop_i32(AssemblerX64::emit_or, loc_a, loc_b, ret)
2918 }
2919
2920 fn emit_binop_xor32(
2921 &mut self,
2922 loc_a: Location,
2923 loc_b: Location,
2924 ret: Location,
2925 ) -> Result<(), CompileError> {
2926 self.emit_binop_i32(AssemblerX64::emit_xor, loc_a, loc_b, ret)
2927 }
2928
2929 fn i32_cmp_ge_s(
2930 &mut self,
2931 loc_a: Location,
2932 loc_b: Location,
2933 ret: Location,
2934 ) -> Result<(), CompileError> {
2935 self.emit_cmpop_i32_dynamic_b(Condition::GreaterEqual, loc_a, loc_b, ret)
2936 }
2937
2938 fn i32_cmp_gt_s(
2939 &mut self,
2940 loc_a: Location,
2941 loc_b: Location,
2942 ret: Location,
2943 ) -> Result<(), CompileError> {
2944 self.emit_cmpop_i32_dynamic_b(Condition::Greater, loc_a, loc_b, ret)
2945 }
2946
2947 fn i32_cmp_le_s(
2948 &mut self,
2949 loc_a: Location,
2950 loc_b: Location,
2951 ret: Location,
2952 ) -> Result<(), CompileError> {
2953 self.emit_cmpop_i32_dynamic_b(Condition::LessEqual, loc_a, loc_b, ret)
2954 }
2955
2956 fn i32_cmp_lt_s(
2957 &mut self,
2958 loc_a: Location,
2959 loc_b: Location,
2960 ret: Location,
2961 ) -> Result<(), CompileError> {
2962 self.emit_cmpop_i32_dynamic_b(Condition::Less, loc_a, loc_b, ret)
2963 }
2964
2965 fn i32_cmp_ge_u(
2966 &mut self,
2967 loc_a: Location,
2968 loc_b: Location,
2969 ret: Location,
2970 ) -> Result<(), CompileError> {
2971 self.emit_cmpop_i32_dynamic_b(Condition::AboveEqual, loc_a, loc_b, ret)
2972 }
2973
2974 fn i32_cmp_gt_u(
2975 &mut self,
2976 loc_a: Location,
2977 loc_b: Location,
2978 ret: Location,
2979 ) -> Result<(), CompileError> {
2980 self.emit_cmpop_i32_dynamic_b(Condition::Above, loc_a, loc_b, ret)
2981 }
2982
2983 fn i32_cmp_le_u(
2984 &mut self,
2985 loc_a: Location,
2986 loc_b: Location,
2987 ret: Location,
2988 ) -> Result<(), CompileError> {
2989 self.emit_cmpop_i32_dynamic_b(Condition::BelowEqual, loc_a, loc_b, ret)
2990 }
2991
2992 fn i32_cmp_lt_u(
2993 &mut self,
2994 loc_a: Location,
2995 loc_b: Location,
2996 ret: Location,
2997 ) -> Result<(), CompileError> {
2998 self.emit_cmpop_i32_dynamic_b(Condition::Below, loc_a, loc_b, ret)
2999 }
3000
3001 fn i32_cmp_ne(
3002 &mut self,
3003 loc_a: Location,
3004 loc_b: Location,
3005 ret: Location,
3006 ) -> Result<(), CompileError> {
3007 self.emit_cmpop_i32_dynamic_b(Condition::NotEqual, loc_a, loc_b, ret)
3008 }
3009
3010 fn i32_cmp_eq(
3011 &mut self,
3012 loc_a: Location,
3013 loc_b: Location,
3014 ret: Location,
3015 ) -> Result<(), CompileError> {
3016 self.emit_cmpop_i32_dynamic_b(Condition::Equal, loc_a, loc_b, ret)
3017 }
3018
3019 fn i32_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3020 let src = match loc {
3021 Location::Imm32(_) | Location::Memory(_, _) => {
3022 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3023 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3024 })?;
3025 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
3026 tmp
3027 }
3028 Location::GPR(reg) => reg,
3029 _ => {
3030 codegen_error!("singlepass i32_clz unreachable");
3031 }
3032 };
3033 let dst = match ret {
3034 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
3035 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3036 })?,
3037 Location::GPR(reg) => reg,
3038 _ => {
3039 codegen_error!("singlepass i32_clz unreachable");
3040 }
3041 };
3042
3043 if self.assembler.arch_has_xzcnt() {
3044 self.assembler
3045 .arch_emit_lzcnt(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3046 } else {
3047 let zero_path = self.assembler.get_label();
3048 let end = self.assembler.get_label();
3049
3050 self.assembler.emit_test_gpr_64(src)?;
3051 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
3052 self.assembler
3053 .emit_bsr(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3054 self.assembler
3055 .emit_xor(Size::S32, Location::Imm32(31), Location::GPR(dst))?;
3056 self.assembler.emit_jmp(Condition::None, end)?;
3057 self.emit_label(zero_path)?;
3058 self.move_location(Size::S32, Location::Imm32(32), Location::GPR(dst))?;
3059 self.emit_label(end)?;
3060 }
3061 match loc {
3062 Location::Imm32(_) | Location::Memory(_, _) => {
3063 self.release_gpr(src);
3064 }
3065 _ => {}
3066 };
3067 if let Location::Memory(_, _) = ret {
3068 self.move_location(Size::S32, Location::GPR(dst), ret)?;
3069 self.release_gpr(dst);
3070 };
3071 Ok(())
3072 }
3073
3074 fn i32_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3075 let src = match loc {
3076 Location::Imm32(_) | Location::Memory(_, _) => {
3077 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3078 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3079 })?;
3080 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
3081 tmp
3082 }
3083 Location::GPR(reg) => reg,
3084 _ => {
3085 codegen_error!("singlepass i32_ctz unreachable");
3086 }
3087 };
3088 let dst = match ret {
3089 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
3090 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3091 })?,
3092 Location::GPR(reg) => reg,
3093 _ => {
3094 codegen_error!("singlepass i32_ctz unreachable");
3095 }
3096 };
3097
3098 if self.assembler.arch_has_xzcnt() {
3099 self.assembler
3100 .arch_emit_tzcnt(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3101 } else {
3102 let zero_path = self.assembler.get_label();
3103 let end = self.assembler.get_label();
3104
3105 self.assembler.emit_test_gpr_64(src)?;
3106 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
3107 self.assembler
3108 .emit_bsf(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3109 self.assembler.emit_jmp(Condition::None, end)?;
3110 self.emit_label(zero_path)?;
3111 self.move_location(Size::S32, Location::Imm32(32), Location::GPR(dst))?;
3112 self.emit_label(end)?;
3113 }
3114
3115 match loc {
3116 Location::Imm32(_) | Location::Memory(_, _) => {
3117 self.release_gpr(src);
3118 }
3119 _ => {}
3120 };
3121 if let Location::Memory(_, _) = ret {
3122 self.move_location(Size::S32, Location::GPR(dst), ret)?;
3123 self.release_gpr(dst);
3124 };
3125 Ok(())
3126 }
3127
3128 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3129 match loc {
3130 Location::Imm32(_) => {
3131 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3132 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3133 })?;
3134 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
3135 if let Location::Memory(_, _) = ret {
3136 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3137 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3138 })?;
3139 self.assembler.emit_popcnt(
3140 Size::S32,
3141 Location::GPR(tmp),
3142 Location::GPR(out_tmp),
3143 )?;
3144 self.move_location(Size::S32, Location::GPR(out_tmp), ret)?;
3145 self.release_gpr(out_tmp);
3146 } else {
3147 self.assembler
3148 .emit_popcnt(Size::S32, Location::GPR(tmp), ret)?;
3149 }
3150 self.release_gpr(tmp);
3151 }
3152 Location::Memory(_, _) | Location::GPR(_) => {
3153 if let Location::Memory(_, _) = ret {
3154 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3155 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3156 })?;
3157 self.assembler
3158 .emit_popcnt(Size::S32, loc, Location::GPR(out_tmp))?;
3159 self.move_location(Size::S32, Location::GPR(out_tmp), ret)?;
3160 self.release_gpr(out_tmp);
3161 } else {
3162 self.assembler.emit_popcnt(Size::S32, loc, ret)?;
3163 }
3164 }
3165 _ => {
3166 codegen_error!("singlepass i32_popcnt unreachable");
3167 }
3168 }
3169 Ok(())
3170 }
3171
3172 fn i32_shl(
3173 &mut self,
3174 loc_a: Location,
3175 loc_b: Location,
3176 ret: Location,
3177 ) -> Result<(), CompileError> {
3178 self.emit_shift_i32(AssemblerX64::emit_shl, loc_a, loc_b, ret)
3179 }
3180
3181 fn i32_shr(
3182 &mut self,
3183 loc_a: Location,
3184 loc_b: Location,
3185 ret: Location,
3186 ) -> Result<(), CompileError> {
3187 self.emit_shift_i32(AssemblerX64::emit_shr, loc_a, loc_b, ret)
3188 }
3189
3190 fn i32_sar(
3191 &mut self,
3192 loc_a: Location,
3193 loc_b: Location,
3194 ret: Location,
3195 ) -> Result<(), CompileError> {
3196 self.emit_shift_i32(AssemblerX64::emit_sar, loc_a, loc_b, ret)
3197 }
3198
3199 fn i32_rol(
3200 &mut self,
3201 loc_a: Location,
3202 loc_b: Location,
3203 ret: Location,
3204 ) -> Result<(), CompileError> {
3205 self.emit_shift_i32(AssemblerX64::emit_rol, loc_a, loc_b, ret)
3206 }
3207
3208 fn i32_ror(
3209 &mut self,
3210 loc_a: Location,
3211 loc_b: Location,
3212 ret: Location,
3213 ) -> Result<(), CompileError> {
3214 self.emit_shift_i32(AssemblerX64::emit_ror, loc_a, loc_b, ret)
3215 }
3216
3217 fn i32_load(
3218 &mut self,
3219 addr: Location,
3220 memarg: &MemArg,
3221 ret: Location,
3222 need_check: bool,
3223 imported_memories: bool,
3224 offset: i32,
3225 heap_access_oob: Label,
3226 unaligned_atomic: Label,
3227 ) -> Result<(), CompileError> {
3228 self.memory_op(
3229 addr,
3230 memarg,
3231 false,
3232 4,
3233 need_check,
3234 imported_memories,
3235 offset,
3236 heap_access_oob,
3237 unaligned_atomic,
3238 |this, addr| {
3239 this.emit_relaxed_binop(
3240 AssemblerX64::emit_mov,
3241 Size::S32,
3242 Location::Memory(addr, 0),
3243 ret,
3244 )
3245 },
3246 )
3247 }
3248
3249 fn i32_load_8u(
3250 &mut self,
3251 addr: Location,
3252 memarg: &MemArg,
3253 ret: Location,
3254 need_check: bool,
3255 imported_memories: bool,
3256 offset: i32,
3257 heap_access_oob: Label,
3258 unaligned_atomic: Label,
3259 ) -> Result<(), CompileError> {
3260 self.memory_op(
3261 addr,
3262 memarg,
3263 false,
3264 1,
3265 need_check,
3266 imported_memories,
3267 offset,
3268 heap_access_oob,
3269 unaligned_atomic,
3270 |this, addr| {
3271 this.emit_relaxed_zx_sx(
3272 AssemblerX64::emit_movzx,
3273 Size::S8,
3274 Location::Memory(addr, 0),
3275 Size::S32,
3276 ret,
3277 )
3278 },
3279 )
3280 }
3281
3282 fn i32_load_8s(
3283 &mut self,
3284 addr: Location,
3285 memarg: &MemArg,
3286 ret: Location,
3287 need_check: bool,
3288 imported_memories: bool,
3289 offset: i32,
3290 heap_access_oob: Label,
3291 unaligned_atomic: Label,
3292 ) -> Result<(), CompileError> {
3293 self.memory_op(
3294 addr,
3295 memarg,
3296 false,
3297 1,
3298 need_check,
3299 imported_memories,
3300 offset,
3301 heap_access_oob,
3302 unaligned_atomic,
3303 |this, addr| {
3304 this.emit_relaxed_zx_sx(
3305 AssemblerX64::emit_movsx,
3306 Size::S8,
3307 Location::Memory(addr, 0),
3308 Size::S32,
3309 ret,
3310 )
3311 },
3312 )
3313 }
3314
3315 fn i32_load_16u(
3316 &mut self,
3317 addr: Location,
3318 memarg: &MemArg,
3319 ret: Location,
3320 need_check: bool,
3321 imported_memories: bool,
3322 offset: i32,
3323 heap_access_oob: Label,
3324 unaligned_atomic: Label,
3325 ) -> Result<(), CompileError> {
3326 self.memory_op(
3327 addr,
3328 memarg,
3329 false,
3330 2,
3331 need_check,
3332 imported_memories,
3333 offset,
3334 heap_access_oob,
3335 unaligned_atomic,
3336 |this, addr| {
3337 this.emit_relaxed_zx_sx(
3338 AssemblerX64::emit_movzx,
3339 Size::S16,
3340 Location::Memory(addr, 0),
3341 Size::S32,
3342 ret,
3343 )
3344 },
3345 )
3346 }
3347
3348 fn i32_load_16s(
3349 &mut self,
3350 addr: Location,
3351 memarg: &MemArg,
3352 ret: Location,
3353 need_check: bool,
3354 imported_memories: bool,
3355 offset: i32,
3356 heap_access_oob: Label,
3357 unaligned_atomic: Label,
3358 ) -> Result<(), CompileError> {
3359 self.memory_op(
3360 addr,
3361 memarg,
3362 false,
3363 2,
3364 need_check,
3365 imported_memories,
3366 offset,
3367 heap_access_oob,
3368 unaligned_atomic,
3369 |this, addr| {
3370 this.emit_relaxed_zx_sx(
3371 AssemblerX64::emit_movsx,
3372 Size::S16,
3373 Location::Memory(addr, 0),
3374 Size::S32,
3375 ret,
3376 )
3377 },
3378 )
3379 }
3380
3381 fn i32_atomic_load(
3382 &mut self,
3383 addr: Location,
3384 memarg: &MemArg,
3385 ret: Location,
3386 need_check: bool,
3387 imported_memories: bool,
3388 offset: i32,
3389 heap_access_oob: Label,
3390 unaligned_atomic: Label,
3391 ) -> Result<(), CompileError> {
3392 self.memory_op(
3393 addr,
3394 memarg,
3395 true,
3396 4,
3397 need_check,
3398 imported_memories,
3399 offset,
3400 heap_access_oob,
3401 unaligned_atomic,
3402 |this, addr| this.emit_relaxed_mov(Size::S32, Location::Memory(addr, 0), ret),
3403 )
3404 }
3405
3406 fn i32_atomic_load_8u(
3407 &mut self,
3408 addr: Location,
3409 memarg: &MemArg,
3410 ret: Location,
3411 need_check: bool,
3412 imported_memories: bool,
3413 offset: i32,
3414 heap_access_oob: Label,
3415 unaligned_atomic: Label,
3416 ) -> Result<(), CompileError> {
3417 self.memory_op(
3418 addr,
3419 memarg,
3420 true,
3421 1,
3422 need_check,
3423 imported_memories,
3424 offset,
3425 heap_access_oob,
3426 unaligned_atomic,
3427 |this, addr| {
3428 this.emit_relaxed_zero_extension(
3429 Size::S8,
3430 Location::Memory(addr, 0),
3431 Size::S32,
3432 ret,
3433 )
3434 },
3435 )
3436 }
3437
3438 fn i32_atomic_load_16u(
3439 &mut self,
3440 addr: Location,
3441 memarg: &MemArg,
3442 ret: Location,
3443 need_check: bool,
3444 imported_memories: bool,
3445 offset: i32,
3446 heap_access_oob: Label,
3447 unaligned_atomic: Label,
3448 ) -> Result<(), CompileError> {
3449 self.memory_op(
3450 addr,
3451 memarg,
3452 true,
3453 2,
3454 need_check,
3455 imported_memories,
3456 offset,
3457 heap_access_oob,
3458 unaligned_atomic,
3459 |this, addr| {
3460 this.emit_relaxed_zero_extension(
3461 Size::S16,
3462 Location::Memory(addr, 0),
3463 Size::S32,
3464 ret,
3465 )
3466 },
3467 )
3468 }
3469
3470 fn i32_save(
3471 &mut self,
3472 target_value: Location,
3473 memarg: &MemArg,
3474 target_addr: Location,
3475 need_check: bool,
3476 imported_memories: bool,
3477 offset: i32,
3478 heap_access_oob: Label,
3479 unaligned_atomic: Label,
3480 ) -> Result<(), CompileError> {
3481 self.memory_op(
3482 target_addr,
3483 memarg,
3484 false,
3485 4,
3486 need_check,
3487 imported_memories,
3488 offset,
3489 heap_access_oob,
3490 unaligned_atomic,
3491 |this, addr| {
3492 this.emit_relaxed_binop(
3493 AssemblerX64::emit_mov,
3494 Size::S32,
3495 target_value,
3496 Location::Memory(addr, 0),
3497 )
3498 },
3499 )
3500 }
3501
3502 fn i32_save_8(
3503 &mut self,
3504 target_value: Location,
3505 memarg: &MemArg,
3506 target_addr: Location,
3507 need_check: bool,
3508 imported_memories: bool,
3509 offset: i32,
3510 heap_access_oob: Label,
3511 unaligned_atomic: Label,
3512 ) -> Result<(), CompileError> {
3513 self.memory_op(
3514 target_addr,
3515 memarg,
3516 false,
3517 1,
3518 need_check,
3519 imported_memories,
3520 offset,
3521 heap_access_oob,
3522 unaligned_atomic,
3523 |this, addr| {
3524 this.emit_relaxed_binop(
3525 AssemblerX64::emit_mov,
3526 Size::S8,
3527 target_value,
3528 Location::Memory(addr, 0),
3529 )
3530 },
3531 )
3532 }
3533
3534 fn i32_save_16(
3535 &mut self,
3536 target_value: Location,
3537 memarg: &MemArg,
3538 target_addr: Location,
3539 need_check: bool,
3540 imported_memories: bool,
3541 offset: i32,
3542 heap_access_oob: Label,
3543 unaligned_atomic: Label,
3544 ) -> Result<(), CompileError> {
3545 self.memory_op(
3546 target_addr,
3547 memarg,
3548 false,
3549 2,
3550 need_check,
3551 imported_memories,
3552 offset,
3553 heap_access_oob,
3554 unaligned_atomic,
3555 |this, addr| {
3556 this.emit_relaxed_binop(
3557 AssemblerX64::emit_mov,
3558 Size::S16,
3559 target_value,
3560 Location::Memory(addr, 0),
3561 )
3562 },
3563 )
3564 }
3565
3566 fn i32_atomic_save(
3567 &mut self,
3568 value: Location,
3569 memarg: &MemArg,
3570 target_addr: Location,
3571 need_check: bool,
3572 imported_memories: bool,
3573 offset: i32,
3574 heap_access_oob: Label,
3575 unaligned_atomic: Label,
3576 ) -> Result<(), CompileError> {
3577 self.memory_op(
3578 target_addr,
3579 memarg,
3580 true,
3581 4,
3582 need_check,
3583 imported_memories,
3584 offset,
3585 heap_access_oob,
3586 unaligned_atomic,
3587 |this, addr| {
3588 this.emit_relaxed_binop(
3589 AssemblerX64::emit_mov,
3590 Size::S32,
3591 value,
3592 Location::Memory(addr, 0),
3593 )
3594 },
3595 )
3596 }
3597
3598 fn i32_atomic_save_8(
3599 &mut self,
3600 value: Location,
3601 memarg: &MemArg,
3602 target_addr: Location,
3603 need_check: bool,
3604 imported_memories: bool,
3605 offset: i32,
3606 heap_access_oob: Label,
3607 unaligned_atomic: Label,
3608 ) -> Result<(), CompileError> {
3609 self.memory_op(
3610 target_addr,
3611 memarg,
3612 true,
3613 1,
3614 need_check,
3615 imported_memories,
3616 offset,
3617 heap_access_oob,
3618 unaligned_atomic,
3619 |this, addr| {
3620 this.emit_relaxed_binop(
3621 AssemblerX64::emit_mov,
3622 Size::S8,
3623 value,
3624 Location::Memory(addr, 0),
3625 )
3626 },
3627 )
3628 }
3629
3630 fn i32_atomic_save_16(
3631 &mut self,
3632 value: Location,
3633 memarg: &MemArg,
3634 target_addr: Location,
3635 need_check: bool,
3636 imported_memories: bool,
3637 offset: i32,
3638 heap_access_oob: Label,
3639 unaligned_atomic: Label,
3640 ) -> Result<(), CompileError> {
3641 self.memory_op(
3642 target_addr,
3643 memarg,
3644 true,
3645 2,
3646 need_check,
3647 imported_memories,
3648 offset,
3649 heap_access_oob,
3650 unaligned_atomic,
3651 |this, addr| {
3652 this.emit_relaxed_binop(
3653 AssemblerX64::emit_mov,
3654 Size::S16,
3655 value,
3656 Location::Memory(addr, 0),
3657 )
3658 },
3659 )
3660 }
3661
3662 fn i32_atomic_add(
3663 &mut self,
3664 loc: Location,
3665 target: Location,
3666 memarg: &MemArg,
3667 ret: Location,
3668 need_check: bool,
3669 imported_memories: bool,
3670 offset: i32,
3671 heap_access_oob: Label,
3672 unaligned_atomic: Label,
3673 ) -> Result<(), CompileError> {
3674 let value = self.acquire_temp_gpr().ok_or_else(|| {
3675 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3676 })?;
3677 self.move_location(Size::S32, loc, Location::GPR(value))?;
3678 self.memory_op(
3679 target,
3680 memarg,
3681 true,
3682 4,
3683 need_check,
3684 imported_memories,
3685 offset,
3686 heap_access_oob,
3687 unaligned_atomic,
3688 |this, addr| {
3689 this.assembler.emit_lock_xadd(
3690 Size::S32,
3691 Location::GPR(value),
3692 Location::Memory(addr, 0),
3693 )
3694 },
3695 )?;
3696 self.move_location(Size::S32, Location::GPR(value), ret)?;
3697 self.release_gpr(value);
3698 Ok(())
3699 }
3700
3701 fn i32_atomic_add_8u(
3702 &mut self,
3703 loc: Location,
3704 target: Location,
3705 memarg: &MemArg,
3706 ret: Location,
3707 need_check: bool,
3708 imported_memories: bool,
3709 offset: i32,
3710 heap_access_oob: Label,
3711 unaligned_atomic: Label,
3712 ) -> Result<(), CompileError> {
3713 let value = self.acquire_temp_gpr().ok_or_else(|| {
3714 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3715 })?;
3716 self.move_location_extend(Size::S8, false, loc, Size::S32, Location::GPR(value))?;
3717 self.memory_op(
3718 target,
3719 memarg,
3720 true,
3721 1,
3722 need_check,
3723 imported_memories,
3724 offset,
3725 heap_access_oob,
3726 unaligned_atomic,
3727 |this, addr| {
3728 this.assembler.emit_lock_xadd(
3729 Size::S8,
3730 Location::GPR(value),
3731 Location::Memory(addr, 0),
3732 )
3733 },
3734 )?;
3735 self.move_location(Size::S32, Location::GPR(value), ret)?;
3736 self.release_gpr(value);
3737 Ok(())
3738 }
3739
3740 fn i32_atomic_add_16u(
3741 &mut self,
3742 loc: Location,
3743 target: Location,
3744 memarg: &MemArg,
3745 ret: Location,
3746 need_check: bool,
3747 imported_memories: bool,
3748 offset: i32,
3749 heap_access_oob: Label,
3750 unaligned_atomic: Label,
3751 ) -> Result<(), CompileError> {
3752 let value = self.acquire_temp_gpr().ok_or_else(|| {
3753 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3754 })?;
3755 self.move_location_extend(Size::S16, false, loc, Size::S32, Location::GPR(value))?;
3756 self.memory_op(
3757 target,
3758 memarg,
3759 true,
3760 2,
3761 need_check,
3762 imported_memories,
3763 offset,
3764 heap_access_oob,
3765 unaligned_atomic,
3766 |this, addr| {
3767 this.assembler.emit_lock_xadd(
3768 Size::S16,
3769 Location::GPR(value),
3770 Location::Memory(addr, 0),
3771 )
3772 },
3773 )?;
3774 self.move_location(Size::S32, Location::GPR(value), ret)?;
3775 self.release_gpr(value);
3776 Ok(())
3777 }
3778
3779 fn i32_atomic_sub(
3780 &mut self,
3781 loc: Location,
3782 target: Location,
3783 memarg: &MemArg,
3784 ret: Location,
3785 need_check: bool,
3786 imported_memories: bool,
3787 offset: i32,
3788 heap_access_oob: Label,
3789 unaligned_atomic: Label,
3790 ) -> Result<(), CompileError> {
3791 let value = self.acquire_temp_gpr().ok_or_else(|| {
3792 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3793 })?;
3794 self.location_neg(Size::S32, false, loc, Size::S32, Location::GPR(value))?;
3795 self.memory_op(
3796 target,
3797 memarg,
3798 true,
3799 4,
3800 need_check,
3801 imported_memories,
3802 offset,
3803 heap_access_oob,
3804 unaligned_atomic,
3805 |this, addr| {
3806 this.assembler.emit_lock_xadd(
3807 Size::S32,
3808 Location::GPR(value),
3809 Location::Memory(addr, 0),
3810 )
3811 },
3812 )?;
3813 self.move_location(Size::S32, Location::GPR(value), ret)?;
3814 self.release_gpr(value);
3815 Ok(())
3816 }
3817
3818 fn i32_atomic_sub_8u(
3819 &mut self,
3820 loc: Location,
3821 target: Location,
3822 memarg: &MemArg,
3823 ret: Location,
3824 need_check: bool,
3825 imported_memories: bool,
3826 offset: i32,
3827 heap_access_oob: Label,
3828 unaligned_atomic: Label,
3829 ) -> Result<(), CompileError> {
3830 let value = self.acquire_temp_gpr().ok_or_else(|| {
3831 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3832 })?;
3833 self.location_neg(Size::S8, false, loc, Size::S32, Location::GPR(value))?;
3834 self.memory_op(
3835 target,
3836 memarg,
3837 true,
3838 1,
3839 need_check,
3840 imported_memories,
3841 offset,
3842 heap_access_oob,
3843 unaligned_atomic,
3844 |this, addr| {
3845 this.assembler.emit_lock_xadd(
3846 Size::S8,
3847 Location::GPR(value),
3848 Location::Memory(addr, 0),
3849 )
3850 },
3851 )?;
3852 self.move_location(Size::S32, Location::GPR(value), ret)?;
3853 self.release_gpr(value);
3854 Ok(())
3855 }
3856
3857 fn i32_atomic_sub_16u(
3858 &mut self,
3859 loc: Location,
3860 target: Location,
3861 memarg: &MemArg,
3862 ret: Location,
3863 need_check: bool,
3864 imported_memories: bool,
3865 offset: i32,
3866 heap_access_oob: Label,
3867 unaligned_atomic: Label,
3868 ) -> Result<(), CompileError> {
3869 let value = self.acquire_temp_gpr().ok_or_else(|| {
3870 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3871 })?;
3872 self.location_neg(Size::S16, false, loc, Size::S32, Location::GPR(value))?;
3873 self.memory_op(
3874 target,
3875 memarg,
3876 true,
3877 2,
3878 need_check,
3879 imported_memories,
3880 offset,
3881 heap_access_oob,
3882 unaligned_atomic,
3883 |this, addr| {
3884 this.assembler.emit_lock_xadd(
3885 Size::S16,
3886 Location::GPR(value),
3887 Location::Memory(addr, 0),
3888 )
3889 },
3890 )?;
3891 self.move_location(Size::S32, Location::GPR(value), ret)?;
3892 self.release_gpr(value);
3893 Ok(())
3894 }
3895
3896 fn i32_atomic_and(
3897 &mut self,
3898 loc: Location,
3899 target: Location,
3900 memarg: &MemArg,
3901 ret: Location,
3902 need_check: bool,
3903 imported_memories: bool,
3904 offset: i32,
3905 heap_access_oob: Label,
3906 unaligned_atomic: Label,
3907 ) -> Result<(), CompileError> {
3908 self.emit_compare_and_swap(
3909 loc,
3910 target,
3911 ret,
3912 memarg,
3913 4,
3914 Size::S32,
3915 Size::S32,
3916 need_check,
3917 imported_memories,
3918 offset,
3919 heap_access_oob,
3920 unaligned_atomic,
3921 |this, src, dst| {
3922 this.assembler
3923 .emit_and(Size::S32, Location::GPR(src), Location::GPR(dst))
3924 },
3925 )
3926 }
3927
3928 fn i32_atomic_and_8u(
3929 &mut self,
3930 loc: Location,
3931 target: Location,
3932 memarg: &MemArg,
3933 ret: Location,
3934 need_check: bool,
3935 imported_memories: bool,
3936 offset: i32,
3937 heap_access_oob: Label,
3938 unaligned_atomic: Label,
3939 ) -> Result<(), CompileError> {
3940 self.emit_compare_and_swap(
3941 loc,
3942 target,
3943 ret,
3944 memarg,
3945 1,
3946 Size::S8,
3947 Size::S32,
3948 need_check,
3949 imported_memories,
3950 offset,
3951 heap_access_oob,
3952 unaligned_atomic,
3953 |this, src, dst| {
3954 this.assembler
3955 .emit_and(Size::S32, Location::GPR(src), Location::GPR(dst))
3956 },
3957 )
3958 }
3959
3960 fn i32_atomic_and_16u(
3961 &mut self,
3962 loc: Location,
3963 target: Location,
3964 memarg: &MemArg,
3965 ret: Location,
3966 need_check: bool,
3967 imported_memories: bool,
3968 offset: i32,
3969 heap_access_oob: Label,
3970 unaligned_atomic: Label,
3971 ) -> Result<(), CompileError> {
3972 self.emit_compare_and_swap(
3973 loc,
3974 target,
3975 ret,
3976 memarg,
3977 2,
3978 Size::S16,
3979 Size::S32,
3980 need_check,
3981 imported_memories,
3982 offset,
3983 heap_access_oob,
3984 unaligned_atomic,
3985 |this, src, dst| {
3986 this.assembler
3987 .emit_and(Size::S32, Location::GPR(src), Location::GPR(dst))
3988 },
3989 )
3990 }
3991
3992 fn i32_atomic_or(
3993 &mut self,
3994 loc: Location,
3995 target: Location,
3996 memarg: &MemArg,
3997 ret: Location,
3998 need_check: bool,
3999 imported_memories: bool,
4000 offset: i32,
4001 heap_access_oob: Label,
4002 unaligned_atomic: Label,
4003 ) -> Result<(), CompileError> {
4004 self.emit_compare_and_swap(
4005 loc,
4006 target,
4007 ret,
4008 memarg,
4009 4,
4010 Size::S32,
4011 Size::S32,
4012 need_check,
4013 imported_memories,
4014 offset,
4015 heap_access_oob,
4016 unaligned_atomic,
4017 |this, src, dst| {
4018 this.assembler
4019 .emit_or(Size::S32, Location::GPR(src), Location::GPR(dst))
4020 },
4021 )
4022 }
4023
4024 fn i32_atomic_or_8u(
4025 &mut self,
4026 loc: Location,
4027 target: Location,
4028 memarg: &MemArg,
4029 ret: Location,
4030 need_check: bool,
4031 imported_memories: bool,
4032 offset: i32,
4033 heap_access_oob: Label,
4034 unaligned_atomic: Label,
4035 ) -> Result<(), CompileError> {
4036 self.emit_compare_and_swap(
4037 loc,
4038 target,
4039 ret,
4040 memarg,
4041 1,
4042 Size::S8,
4043 Size::S32,
4044 need_check,
4045 imported_memories,
4046 offset,
4047 heap_access_oob,
4048 unaligned_atomic,
4049 |this, src, dst| {
4050 this.assembler
4051 .emit_or(Size::S32, Location::GPR(src), Location::GPR(dst))
4052 },
4053 )
4054 }
4055
4056 fn i32_atomic_or_16u(
4057 &mut self,
4058 loc: Location,
4059 target: Location,
4060 memarg: &MemArg,
4061 ret: Location,
4062 need_check: bool,
4063 imported_memories: bool,
4064 offset: i32,
4065 heap_access_oob: Label,
4066 unaligned_atomic: Label,
4067 ) -> Result<(), CompileError> {
4068 self.emit_compare_and_swap(
4069 loc,
4070 target,
4071 ret,
4072 memarg,
4073 2,
4074 Size::S16,
4075 Size::S32,
4076 need_check,
4077 imported_memories,
4078 offset,
4079 heap_access_oob,
4080 unaligned_atomic,
4081 |this, src, dst| {
4082 this.assembler
4083 .emit_or(Size::S32, Location::GPR(src), Location::GPR(dst))
4084 },
4085 )
4086 }
4087
4088 fn i32_atomic_xor(
4089 &mut self,
4090 loc: Location,
4091 target: Location,
4092 memarg: &MemArg,
4093 ret: Location,
4094 need_check: bool,
4095 imported_memories: bool,
4096 offset: i32,
4097 heap_access_oob: Label,
4098 unaligned_atomic: Label,
4099 ) -> Result<(), CompileError> {
4100 self.emit_compare_and_swap(
4101 loc,
4102 target,
4103 ret,
4104 memarg,
4105 4,
4106 Size::S32,
4107 Size::S32,
4108 need_check,
4109 imported_memories,
4110 offset,
4111 heap_access_oob,
4112 unaligned_atomic,
4113 |this, src, dst| {
4114 this.assembler
4115 .emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst))
4116 },
4117 )
4118 }
4119
4120 fn i32_atomic_xor_8u(
4121 &mut self,
4122 loc: Location,
4123 target: Location,
4124 memarg: &MemArg,
4125 ret: Location,
4126 need_check: bool,
4127 imported_memories: bool,
4128 offset: i32,
4129 heap_access_oob: Label,
4130 unaligned_atomic: Label,
4131 ) -> Result<(), CompileError> {
4132 self.emit_compare_and_swap(
4133 loc,
4134 target,
4135 ret,
4136 memarg,
4137 1,
4138 Size::S8,
4139 Size::S32,
4140 need_check,
4141 imported_memories,
4142 offset,
4143 heap_access_oob,
4144 unaligned_atomic,
4145 |this, src, dst| {
4146 this.assembler
4147 .emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst))
4148 },
4149 )
4150 }
4151
4152 fn i32_atomic_xor_16u(
4153 &mut self,
4154 loc: Location,
4155 target: Location,
4156 memarg: &MemArg,
4157 ret: Location,
4158 need_check: bool,
4159 imported_memories: bool,
4160 offset: i32,
4161 heap_access_oob: Label,
4162 unaligned_atomic: Label,
4163 ) -> Result<(), CompileError> {
4164 self.emit_compare_and_swap(
4165 loc,
4166 target,
4167 ret,
4168 memarg,
4169 2,
4170 Size::S16,
4171 Size::S32,
4172 need_check,
4173 imported_memories,
4174 offset,
4175 heap_access_oob,
4176 unaligned_atomic,
4177 |this, src, dst| {
4178 this.assembler
4179 .emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst))
4180 },
4181 )
4182 }
4183
4184 fn i32_atomic_xchg(
4185 &mut self,
4186 loc: Location,
4187 target: Location,
4188 memarg: &MemArg,
4189 ret: Location,
4190 need_check: bool,
4191 imported_memories: bool,
4192 offset: i32,
4193 heap_access_oob: Label,
4194 unaligned_atomic: Label,
4195 ) -> Result<(), CompileError> {
4196 let value = self.acquire_temp_gpr().ok_or_else(|| {
4197 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4198 })?;
4199 self.move_location(Size::S32, loc, Location::GPR(value))?;
4200 self.memory_op(
4201 target,
4202 memarg,
4203 true,
4204 4,
4205 need_check,
4206 imported_memories,
4207 offset,
4208 heap_access_oob,
4209 unaligned_atomic,
4210 |this, addr| {
4211 this.assembler
4212 .emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0))
4213 },
4214 )?;
4215 self.move_location(Size::S32, Location::GPR(value), ret)?;
4216 self.release_gpr(value);
4217 Ok(())
4218 }
4219
4220 fn i32_atomic_xchg_8u(
4221 &mut self,
4222 loc: Location,
4223 target: Location,
4224 memarg: &MemArg,
4225 ret: Location,
4226 need_check: bool,
4227 imported_memories: bool,
4228 offset: i32,
4229 heap_access_oob: Label,
4230 unaligned_atomic: Label,
4231 ) -> Result<(), CompileError> {
4232 let value = self.acquire_temp_gpr().ok_or_else(|| {
4233 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4234 })?;
4235 self.assembler
4236 .emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value))?;
4237 self.memory_op(
4238 target,
4239 memarg,
4240 true,
4241 1,
4242 need_check,
4243 imported_memories,
4244 offset,
4245 heap_access_oob,
4246 unaligned_atomic,
4247 |this, addr| {
4248 this.assembler
4249 .emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0))
4250 },
4251 )?;
4252 self.move_location(Size::S32, Location::GPR(value), ret)?;
4253 self.release_gpr(value);
4254 Ok(())
4255 }
4256
4257 fn i32_atomic_xchg_16u(
4258 &mut self,
4259 loc: Location,
4260 target: Location,
4261 memarg: &MemArg,
4262 ret: Location,
4263 need_check: bool,
4264 imported_memories: bool,
4265 offset: i32,
4266 heap_access_oob: Label,
4267 unaligned_atomic: Label,
4268 ) -> Result<(), CompileError> {
4269 let value = self.acquire_temp_gpr().ok_or_else(|| {
4270 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4271 })?;
4272 self.assembler
4273 .emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value))?;
4274 self.memory_op(
4275 target,
4276 memarg,
4277 true,
4278 2,
4279 need_check,
4280 imported_memories,
4281 offset,
4282 heap_access_oob,
4283 unaligned_atomic,
4284 |this, addr| {
4285 this.assembler
4286 .emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0))
4287 },
4288 )?;
4289 self.move_location(Size::S32, Location::GPR(value), ret)?;
4290 self.release_gpr(value);
4291 Ok(())
4292 }
4293
4294 fn i32_atomic_cmpxchg(
4295 &mut self,
4296 new: Location,
4297 cmp: Location,
4298 target: Location,
4299 memarg: &MemArg,
4300 ret: Location,
4301 need_check: bool,
4302 imported_memories: bool,
4303 offset: i32,
4304 heap_access_oob: Label,
4305 unaligned_atomic: Label,
4306 ) -> Result<(), CompileError> {
4307 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
4308 let value = if cmp == Location::GPR(GPR::R14) {
4309 if new == Location::GPR(GPR::R13) {
4310 GPR::R12
4311 } else {
4312 GPR::R13
4313 }
4314 } else {
4315 GPR::R14
4316 };
4317 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
4318 self.assembler
4319 .emit_mov(Size::S32, cmp, Location::GPR(compare))?;
4320 self.assembler
4321 .emit_mov(Size::S32, new, Location::GPR(value))?;
4322
4323 self.memory_op(
4324 target,
4325 memarg,
4326 true,
4327 4,
4328 need_check,
4329 imported_memories,
4330 offset,
4331 heap_access_oob,
4332 unaligned_atomic,
4333 |this, addr| {
4334 this.assembler.emit_lock_cmpxchg(
4335 Size::S32,
4336 Location::GPR(value),
4337 Location::Memory(addr, 0),
4338 )?;
4339 this.assembler
4340 .emit_mov(Size::S32, Location::GPR(compare), ret)
4341 },
4342 )?;
4343 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
4344 self.release_gpr(compare);
4345 Ok(())
4346 }
4347
4348 fn i32_atomic_cmpxchg_8u(
4349 &mut self,
4350 new: Location,
4351 cmp: Location,
4352 target: Location,
4353 memarg: &MemArg,
4354 ret: Location,
4355 need_check: bool,
4356 imported_memories: bool,
4357 offset: i32,
4358 heap_access_oob: Label,
4359 unaligned_atomic: Label,
4360 ) -> Result<(), CompileError> {
4361 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
4362 let value = if cmp == Location::GPR(GPR::R14) {
4363 if new == Location::GPR(GPR::R13) {
4364 GPR::R12
4365 } else {
4366 GPR::R13
4367 }
4368 } else {
4369 GPR::R14
4370 };
4371 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
4372 self.assembler
4373 .emit_mov(Size::S32, cmp, Location::GPR(compare))?;
4374 self.assembler
4375 .emit_mov(Size::S32, new, Location::GPR(value))?;
4376
4377 self.memory_op(
4378 target,
4379 memarg,
4380 true,
4381 1,
4382 need_check,
4383 imported_memories,
4384 offset,
4385 heap_access_oob,
4386 unaligned_atomic,
4387 |this, addr| {
4388 this.assembler.emit_lock_cmpxchg(
4389 Size::S8,
4390 Location::GPR(value),
4391 Location::Memory(addr, 0),
4392 )?;
4393 this.assembler
4394 .emit_movzx(Size::S8, Location::GPR(compare), Size::S32, ret)
4395 },
4396 )?;
4397 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
4398 self.release_gpr(compare);
4399 Ok(())
4400 }
4401
4402 fn i32_atomic_cmpxchg_16u(
4403 &mut self,
4404 new: Location,
4405 cmp: Location,
4406 target: Location,
4407 memarg: &MemArg,
4408 ret: Location,
4409 need_check: bool,
4410 imported_memories: bool,
4411 offset: i32,
4412 heap_access_oob: Label,
4413 unaligned_atomic: Label,
4414 ) -> Result<(), CompileError> {
4415 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
4416 let value = if cmp == Location::GPR(GPR::R14) {
4417 if new == Location::GPR(GPR::R13) {
4418 GPR::R12
4419 } else {
4420 GPR::R13
4421 }
4422 } else {
4423 GPR::R14
4424 };
4425 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
4426 self.assembler
4427 .emit_mov(Size::S32, cmp, Location::GPR(compare))?;
4428 self.assembler
4429 .emit_mov(Size::S32, new, Location::GPR(value))?;
4430
4431 self.memory_op(
4432 target,
4433 memarg,
4434 true,
4435 2,
4436 need_check,
4437 imported_memories,
4438 offset,
4439 heap_access_oob,
4440 unaligned_atomic,
4441 |this, addr| {
4442 this.assembler.emit_lock_cmpxchg(
4443 Size::S16,
4444 Location::GPR(value),
4445 Location::Memory(addr, 0),
4446 )?;
4447 this.assembler
4448 .emit_movzx(Size::S16, Location::GPR(compare), Size::S32, ret)
4449 },
4450 )?;
4451 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
4452 self.release_gpr(compare);
4453 Ok(())
4454 }
4455
4456 fn emit_call_with_reloc(
4457 &mut self,
4458 reloc_target: RelocationTarget,
4459 ) -> Result<Vec<Relocation>, CompileError> {
4460 let mut relocations = vec![];
4461 let next = self.get_label();
4462 let reloc_at = self.assembler.get_offset().0 + 1; self.assembler.emit_call_label(next)?;
4464 self.emit_label(next)?;
4465 relocations.push(Relocation {
4466 kind: RelocationKind::X86CallPCRel4,
4467 reloc_target,
4468 offset: reloc_at as u32,
4469 addend: -4,
4470 });
4471 Ok(relocations)
4472 }
4473
4474 fn emit_binop_add64(
4475 &mut self,
4476 loc_a: Location,
4477 loc_b: Location,
4478 ret: Location,
4479 ) -> Result<(), CompileError> {
4480 self.emit_binop_i64(AssemblerX64::emit_add, loc_a, loc_b, ret)
4481 }
4482
4483 fn emit_binop_sub64(
4484 &mut self,
4485 loc_a: Location,
4486 loc_b: Location,
4487 ret: Location,
4488 ) -> Result<(), CompileError> {
4489 self.emit_binop_i64(AssemblerX64::emit_sub, loc_a, loc_b, ret)
4490 }
4491
4492 fn emit_binop_mul64(
4493 &mut self,
4494 loc_a: Location,
4495 loc_b: Location,
4496 ret: Location,
4497 ) -> Result<(), CompileError> {
4498 self.emit_binop_i64(AssemblerX64::emit_imul, loc_a, loc_b, ret)
4499 }
4500
4501 fn emit_binop_udiv64(
4502 &mut self,
4503 loc_a: Location,
4504 loc_b: Location,
4505 ret: Location,
4506 integer_division_by_zero: Label,
4507 ) -> Result<usize, CompileError> {
4508 for reg in DIV_REM_REGISTERS {
4509 self.reserve_unused_temp_gpr(reg);
4510 }
4511 self.assembler
4512 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4513 self.assembler
4514 .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
4515 let offset = self.emit_relaxed_xdiv(
4516 AssemblerX64::emit_div,
4517 Size::S64,
4518 loc_b,
4519 integer_division_by_zero,
4520 )?;
4521 self.assembler
4522 .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret)?;
4523 for reg in DIV_REM_REGISTERS {
4524 self.release_gpr(reg);
4525 }
4526 Ok(offset)
4527 }
4528
4529 fn emit_binop_sdiv64(
4530 &mut self,
4531 loc_a: Location,
4532 loc_b: Location,
4533 ret: Location,
4534 integer_division_by_zero: Label,
4535 _integer_overflow: Label,
4536 ) -> Result<usize, CompileError> {
4537 for reg in DIV_REM_REGISTERS {
4538 self.reserve_unused_temp_gpr(reg);
4539 }
4540 self.assembler
4541 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4542 self.assembler.emit_cqo()?;
4543 let offset = self.emit_relaxed_xdiv(
4544 AssemblerX64::emit_idiv,
4545 Size::S64,
4546 loc_b,
4547 integer_division_by_zero,
4548 )?;
4549 self.assembler
4550 .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret)?;
4551 for reg in DIV_REM_REGISTERS {
4552 self.release_gpr(reg);
4553 }
4554 Ok(offset)
4555 }
4556
4557 fn emit_binop_urem64(
4558 &mut self,
4559 loc_a: Location,
4560 loc_b: Location,
4561 ret: Location,
4562 integer_division_by_zero: Label,
4563 ) -> Result<usize, CompileError> {
4564 for reg in DIV_REM_REGISTERS {
4565 self.reserve_unused_temp_gpr(reg);
4566 }
4567 self.assembler
4568 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4569 self.assembler
4570 .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
4571 let offset = self.emit_relaxed_xdiv(
4572 AssemblerX64::emit_div,
4573 Size::S64,
4574 loc_b,
4575 integer_division_by_zero,
4576 )?;
4577 self.assembler
4578 .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret)?;
4579 for reg in DIV_REM_REGISTERS {
4580 self.release_gpr(reg);
4581 }
4582 Ok(offset)
4583 }
4584
4585 fn emit_binop_srem64(
4586 &mut self,
4587 loc_a: Location,
4588 loc_b: Location,
4589 ret: Location,
4590 integer_division_by_zero: Label,
4591 ) -> Result<usize, CompileError> {
4592 for reg in DIV_REM_REGISTERS {
4593 self.reserve_unused_temp_gpr(reg);
4594 }
4595 let normal_path = self.assembler.get_label();
4596 let end = self.assembler.get_label();
4597
4598 self.emit_relaxed_cmp(Size::S64, Location::Imm64(0x8000000000000000u64), loc_a)?;
4599 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
4600 self.emit_relaxed_cmp(Size::S64, Location::Imm64(0xffffffffffffffffu64), loc_b)?;
4601 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
4602 self.move_location(Size::S64, Location::Imm64(0), ret)?;
4603 self.assembler.emit_jmp(Condition::None, end)?;
4604
4605 self.emit_label(normal_path)?;
4606 self.assembler
4607 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4608 self.assembler.emit_cqo()?;
4609 let offset = self.emit_relaxed_xdiv(
4610 AssemblerX64::emit_idiv,
4611 Size::S64,
4612 loc_b,
4613 integer_division_by_zero,
4614 )?;
4615 self.assembler
4616 .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret)?;
4617
4618 self.emit_label(end)?;
4619 for reg in DIV_REM_REGISTERS {
4620 self.release_gpr(reg);
4621 }
4622 Ok(offset)
4623 }
4624
4625 fn emit_binop_and64(
4626 &mut self,
4627 loc_a: Location,
4628 loc_b: Location,
4629 ret: Location,
4630 ) -> Result<(), CompileError> {
4631 self.emit_binop_i64(AssemblerX64::emit_and, loc_a, loc_b, ret)
4632 }
4633
4634 fn emit_binop_or64(
4635 &mut self,
4636 loc_a: Location,
4637 loc_b: Location,
4638 ret: Location,
4639 ) -> Result<(), CompileError> {
4640 self.emit_binop_i64(AssemblerX64::emit_or, loc_a, loc_b, ret)
4641 }
4642
4643 fn emit_binop_xor64(
4644 &mut self,
4645 loc_a: Location,
4646 loc_b: Location,
4647 ret: Location,
4648 ) -> Result<(), CompileError> {
4649 self.emit_binop_i64(AssemblerX64::emit_xor, loc_a, loc_b, ret)
4650 }
4651
4652 fn i64_cmp_ge_s(
4653 &mut self,
4654 loc_a: Location,
4655 loc_b: Location,
4656 ret: Location,
4657 ) -> Result<(), CompileError> {
4658 self.emit_cmpop_i64_dynamic_b(Condition::GreaterEqual, loc_a, loc_b, ret)
4659 }
4660
4661 fn i64_cmp_gt_s(
4662 &mut self,
4663 loc_a: Location,
4664 loc_b: Location,
4665 ret: Location,
4666 ) -> Result<(), CompileError> {
4667 self.emit_cmpop_i64_dynamic_b(Condition::Greater, loc_a, loc_b, ret)
4668 }
4669
4670 fn i64_cmp_le_s(
4671 &mut self,
4672 loc_a: Location,
4673 loc_b: Location,
4674 ret: Location,
4675 ) -> Result<(), CompileError> {
4676 self.emit_cmpop_i64_dynamic_b(Condition::LessEqual, loc_a, loc_b, ret)
4677 }
4678
4679 fn i64_cmp_lt_s(
4680 &mut self,
4681 loc_a: Location,
4682 loc_b: Location,
4683 ret: Location,
4684 ) -> Result<(), CompileError> {
4685 self.emit_cmpop_i64_dynamic_b(Condition::Less, loc_a, loc_b, ret)
4686 }
4687
4688 fn i64_cmp_ge_u(
4689 &mut self,
4690 loc_a: Location,
4691 loc_b: Location,
4692 ret: Location,
4693 ) -> Result<(), CompileError> {
4694 self.emit_cmpop_i64_dynamic_b(Condition::AboveEqual, loc_a, loc_b, ret)
4695 }
4696
4697 fn i64_cmp_gt_u(
4698 &mut self,
4699 loc_a: Location,
4700 loc_b: Location,
4701 ret: Location,
4702 ) -> Result<(), CompileError> {
4703 self.emit_cmpop_i64_dynamic_b(Condition::Above, loc_a, loc_b, ret)
4704 }
4705
4706 fn i64_cmp_le_u(
4707 &mut self,
4708 loc_a: Location,
4709 loc_b: Location,
4710 ret: Location,
4711 ) -> Result<(), CompileError> {
4712 self.emit_cmpop_i64_dynamic_b(Condition::BelowEqual, loc_a, loc_b, ret)
4713 }
4714
4715 fn i64_cmp_lt_u(
4716 &mut self,
4717 loc_a: Location,
4718 loc_b: Location,
4719 ret: Location,
4720 ) -> Result<(), CompileError> {
4721 self.emit_cmpop_i64_dynamic_b(Condition::Below, loc_a, loc_b, ret)
4722 }
4723
4724 fn i64_cmp_ne(
4725 &mut self,
4726 loc_a: Location,
4727 loc_b: Location,
4728 ret: Location,
4729 ) -> Result<(), CompileError> {
4730 self.emit_cmpop_i64_dynamic_b(Condition::NotEqual, loc_a, loc_b, ret)
4731 }
4732
4733 fn i64_cmp_eq(
4734 &mut self,
4735 loc_a: Location,
4736 loc_b: Location,
4737 ret: Location,
4738 ) -> Result<(), CompileError> {
4739 self.emit_cmpop_i64_dynamic_b(Condition::Equal, loc_a, loc_b, ret)
4740 }
4741
4742 fn i64_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4743 let src = match loc {
4744 Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => {
4745 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4746 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4747 })?;
4748 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
4749 tmp
4750 }
4751 Location::GPR(reg) => reg,
4752 _ => {
4753 codegen_error!("singlepass i64_clz unreachable");
4754 }
4755 };
4756 let dst = match ret {
4757 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
4758 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4759 })?,
4760 Location::GPR(reg) => reg,
4761 _ => {
4762 codegen_error!("singlepass i64_clz unreachable");
4763 }
4764 };
4765
4766 if self.assembler.arch_has_xzcnt() {
4767 self.assembler
4768 .arch_emit_lzcnt(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4769 } else {
4770 let zero_path = self.assembler.get_label();
4771 let end = self.assembler.get_label();
4772
4773 self.assembler.emit_test_gpr_64(src)?;
4774 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
4775 self.assembler
4776 .emit_bsr(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4777 self.assembler
4778 .emit_xor(Size::S64, Location::Imm32(63), Location::GPR(dst))?;
4779 self.assembler.emit_jmp(Condition::None, end)?;
4780 self.emit_label(zero_path)?;
4781 self.move_location(Size::S64, Location::Imm32(64), Location::GPR(dst))?;
4782 self.emit_label(end)?;
4783 }
4784 match loc {
4785 Location::Imm64(_) | Location::Memory(_, _) => {
4786 self.release_gpr(src);
4787 }
4788 _ => {}
4789 };
4790 if let Location::Memory(_, _) = ret {
4791 self.move_location(Size::S64, Location::GPR(dst), ret)?;
4792 self.release_gpr(dst);
4793 };
4794 Ok(())
4795 }
4796
4797 fn i64_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4798 let src = match loc {
4799 Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => {
4800 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4801 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4802 })?;
4803 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
4804 tmp
4805 }
4806 Location::GPR(reg) => reg,
4807 _ => {
4808 codegen_error!("singlepass i64_ctz unreachable");
4809 }
4810 };
4811 let dst = match ret {
4812 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
4813 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4814 })?,
4815 Location::GPR(reg) => reg,
4816 _ => {
4817 codegen_error!("singlepass i64_ctz unreachable");
4818 }
4819 };
4820
4821 if self.assembler.arch_has_xzcnt() {
4822 self.assembler
4823 .arch_emit_tzcnt(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4824 } else {
4825 let zero_path = self.assembler.get_label();
4826 let end = self.assembler.get_label();
4827
4828 self.assembler.emit_test_gpr_64(src)?;
4829 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
4830 self.assembler
4831 .emit_bsf(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4832 self.assembler.emit_jmp(Condition::None, end)?;
4833 self.emit_label(zero_path)?;
4834 self.move_location(Size::S64, Location::Imm64(64), Location::GPR(dst))?;
4835 self.emit_label(end)?;
4836 }
4837
4838 match loc {
4839 Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => {
4840 self.release_gpr(src);
4841 }
4842 _ => {}
4843 };
4844 if let Location::Memory(_, _) = ret {
4845 self.move_location(Size::S64, Location::GPR(dst), ret)?;
4846 self.release_gpr(dst);
4847 };
4848 Ok(())
4849 }
4850
4851 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4852 match loc {
4853 Location::Imm64(_) | Location::Imm32(_) => {
4854 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4855 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4856 })?;
4857 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
4858 if let Location::Memory(_, _) = ret {
4859 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4860 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4861 })?;
4862 self.assembler.emit_popcnt(
4863 Size::S64,
4864 Location::GPR(tmp),
4865 Location::GPR(out_tmp),
4866 )?;
4867 self.move_location(Size::S64, Location::GPR(out_tmp), ret)?;
4868 self.release_gpr(out_tmp);
4869 } else {
4870 self.assembler
4871 .emit_popcnt(Size::S64, Location::GPR(tmp), ret)?;
4872 }
4873 self.release_gpr(tmp);
4874 }
4875 Location::Memory(_, _) | Location::GPR(_) => {
4876 if let Location::Memory(_, _) = ret {
4877 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4878 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4879 })?;
4880 self.assembler
4881 .emit_popcnt(Size::S64, loc, Location::GPR(out_tmp))?;
4882 self.move_location(Size::S64, Location::GPR(out_tmp), ret)?;
4883 self.release_gpr(out_tmp);
4884 } else {
4885 self.assembler.emit_popcnt(Size::S64, loc, ret)?;
4886 }
4887 }
4888 _ => {
4889 codegen_error!("singlepass i64_popcnt unreachable");
4890 }
4891 }
4892 Ok(())
4893 }
4894
4895 fn i64_shl(
4896 &mut self,
4897 loc_a: Location,
4898 loc_b: Location,
4899 ret: Location,
4900 ) -> Result<(), CompileError> {
4901 self.emit_shift_i64(AssemblerX64::emit_shl, loc_a, loc_b, ret)
4902 }
4903
4904 fn i64_shr(
4905 &mut self,
4906 loc_a: Location,
4907 loc_b: Location,
4908 ret: Location,
4909 ) -> Result<(), CompileError> {
4910 self.emit_shift_i64(AssemblerX64::emit_shr, loc_a, loc_b, ret)
4911 }
4912
4913 fn i64_sar(
4914 &mut self,
4915 loc_a: Location,
4916 loc_b: Location,
4917 ret: Location,
4918 ) -> Result<(), CompileError> {
4919 self.emit_shift_i64(AssemblerX64::emit_sar, loc_a, loc_b, ret)
4920 }
4921
4922 fn i64_rol(
4923 &mut self,
4924 loc_a: Location,
4925 loc_b: Location,
4926 ret: Location,
4927 ) -> Result<(), CompileError> {
4928 self.emit_shift_i64(AssemblerX64::emit_rol, loc_a, loc_b, ret)
4929 }
4930
4931 fn i64_ror(
4932 &mut self,
4933 loc_a: Location,
4934 loc_b: Location,
4935 ret: Location,
4936 ) -> Result<(), CompileError> {
4937 self.emit_shift_i64(AssemblerX64::emit_ror, loc_a, loc_b, ret)
4938 }
4939
4940 fn i64_load(
4941 &mut self,
4942 addr: Location,
4943 memarg: &MemArg,
4944 ret: Location,
4945 need_check: bool,
4946 imported_memories: bool,
4947 offset: i32,
4948 heap_access_oob: Label,
4949 unaligned_atomic: Label,
4950 ) -> Result<(), CompileError> {
4951 self.memory_op(
4952 addr,
4953 memarg,
4954 false,
4955 8,
4956 need_check,
4957 imported_memories,
4958 offset,
4959 heap_access_oob,
4960 unaligned_atomic,
4961 |this, addr| {
4962 this.emit_relaxed_binop(
4963 AssemblerX64::emit_mov,
4964 Size::S64,
4965 Location::Memory(addr, 0),
4966 ret,
4967 )
4968 },
4969 )
4970 }
4971
4972 fn i64_load_8u(
4973 &mut self,
4974 addr: Location,
4975 memarg: &MemArg,
4976 ret: Location,
4977 need_check: bool,
4978 imported_memories: bool,
4979 offset: i32,
4980 heap_access_oob: Label,
4981 unaligned_atomic: Label,
4982 ) -> Result<(), CompileError> {
4983 self.memory_op(
4984 addr,
4985 memarg,
4986 false,
4987 1,
4988 need_check,
4989 imported_memories,
4990 offset,
4991 heap_access_oob,
4992 unaligned_atomic,
4993 |this, addr| {
4994 this.emit_relaxed_zx_sx(
4995 AssemblerX64::emit_movzx,
4996 Size::S8,
4997 Location::Memory(addr, 0),
4998 Size::S64,
4999 ret,
5000 )
5001 },
5002 )
5003 }
5004
5005 fn i64_load_8s(
5006 &mut self,
5007 addr: Location,
5008 memarg: &MemArg,
5009 ret: Location,
5010 need_check: bool,
5011 imported_memories: bool,
5012 offset: i32,
5013 heap_access_oob: Label,
5014 unaligned_atomic: Label,
5015 ) -> Result<(), CompileError> {
5016 self.memory_op(
5017 addr,
5018 memarg,
5019 false,
5020 1,
5021 need_check,
5022 imported_memories,
5023 offset,
5024 heap_access_oob,
5025 unaligned_atomic,
5026 |this, addr| {
5027 this.emit_relaxed_zx_sx(
5028 AssemblerX64::emit_movsx,
5029 Size::S8,
5030 Location::Memory(addr, 0),
5031 Size::S64,
5032 ret,
5033 )
5034 },
5035 )
5036 }
5037
5038 fn i64_load_16u(
5039 &mut self,
5040 addr: Location,
5041 memarg: &MemArg,
5042 ret: Location,
5043 need_check: bool,
5044 imported_memories: bool,
5045 offset: i32,
5046 heap_access_oob: Label,
5047 unaligned_atomic: Label,
5048 ) -> Result<(), CompileError> {
5049 self.memory_op(
5050 addr,
5051 memarg,
5052 false,
5053 2,
5054 need_check,
5055 imported_memories,
5056 offset,
5057 heap_access_oob,
5058 unaligned_atomic,
5059 |this, addr| {
5060 this.emit_relaxed_zx_sx(
5061 AssemblerX64::emit_movzx,
5062 Size::S16,
5063 Location::Memory(addr, 0),
5064 Size::S64,
5065 ret,
5066 )
5067 },
5068 )
5069 }
5070
5071 fn i64_load_16s(
5072 &mut self,
5073 addr: Location,
5074 memarg: &MemArg,
5075 ret: Location,
5076 need_check: bool,
5077 imported_memories: bool,
5078 offset: i32,
5079 heap_access_oob: Label,
5080 unaligned_atomic: Label,
5081 ) -> Result<(), CompileError> {
5082 self.memory_op(
5083 addr,
5084 memarg,
5085 false,
5086 2,
5087 need_check,
5088 imported_memories,
5089 offset,
5090 heap_access_oob,
5091 unaligned_atomic,
5092 |this, addr| {
5093 this.emit_relaxed_zx_sx(
5094 AssemblerX64::emit_movsx,
5095 Size::S16,
5096 Location::Memory(addr, 0),
5097 Size::S64,
5098 ret,
5099 )
5100 },
5101 )
5102 }
5103
5104 fn i64_load_32u(
5105 &mut self,
5106 addr: Location,
5107 memarg: &MemArg,
5108 ret: Location,
5109 need_check: bool,
5110 imported_memories: bool,
5111 offset: i32,
5112 heap_access_oob: Label,
5113 unaligned_atomic: Label,
5114 ) -> Result<(), CompileError> {
5115 self.memory_op(
5116 addr,
5117 memarg,
5118 false,
5119 4,
5120 need_check,
5121 imported_memories,
5122 offset,
5123 heap_access_oob,
5124 unaligned_atomic,
5125 |this, addr| {
5126 match ret {
5127 Location::GPR(_) => {}
5128 Location::Memory(base, offset) => {
5129 this.assembler.emit_mov(
5130 Size::S32,
5131 Location::Imm32(0),
5132 Location::Memory(base, offset + 4),
5133 )?; }
5135 _ => {
5136 codegen_error!("singlepass i64_load_32u unreachable");
5137 }
5138 }
5139 this.emit_relaxed_binop(
5140 AssemblerX64::emit_mov,
5141 Size::S32,
5142 Location::Memory(addr, 0),
5143 ret,
5144 )
5145 },
5146 )
5147 }
5148
5149 fn i64_load_32s(
5150 &mut self,
5151 addr: Location,
5152 memarg: &MemArg,
5153 ret: Location,
5154 need_check: bool,
5155 imported_memories: bool,
5156 offset: i32,
5157 heap_access_oob: Label,
5158 unaligned_atomic: Label,
5159 ) -> Result<(), CompileError> {
5160 self.memory_op(
5161 addr,
5162 memarg,
5163 false,
5164 4,
5165 need_check,
5166 imported_memories,
5167 offset,
5168 heap_access_oob,
5169 unaligned_atomic,
5170 |this, addr| {
5171 this.emit_relaxed_zx_sx(
5172 AssemblerX64::emit_movsx,
5173 Size::S32,
5174 Location::Memory(addr, 0),
5175 Size::S64,
5176 ret,
5177 )
5178 },
5179 )
5180 }
5181
5182 fn i64_atomic_load(
5183 &mut self,
5184 addr: Location,
5185 memarg: &MemArg,
5186 ret: Location,
5187 need_check: bool,
5188 imported_memories: bool,
5189 offset: i32,
5190 heap_access_oob: Label,
5191 unaligned_atomic: Label,
5192 ) -> Result<(), CompileError> {
5193 self.memory_op(
5194 addr,
5195 memarg,
5196 true,
5197 8,
5198 need_check,
5199 imported_memories,
5200 offset,
5201 heap_access_oob,
5202 unaligned_atomic,
5203 |this, addr| this.emit_relaxed_mov(Size::S64, Location::Memory(addr, 0), ret),
5204 )
5205 }
5206
5207 fn i64_atomic_load_8u(
5208 &mut self,
5209 addr: Location,
5210 memarg: &MemArg,
5211 ret: Location,
5212 need_check: bool,
5213 imported_memories: bool,
5214 offset: i32,
5215 heap_access_oob: Label,
5216 unaligned_atomic: Label,
5217 ) -> Result<(), CompileError> {
5218 self.memory_op(
5219 addr,
5220 memarg,
5221 true,
5222 1,
5223 need_check,
5224 imported_memories,
5225 offset,
5226 heap_access_oob,
5227 unaligned_atomic,
5228 |this, addr| {
5229 this.emit_relaxed_zero_extension(
5230 Size::S8,
5231 Location::Memory(addr, 0),
5232 Size::S64,
5233 ret,
5234 )
5235 },
5236 )
5237 }
5238
5239 fn i64_atomic_load_16u(
5240 &mut self,
5241 addr: Location,
5242 memarg: &MemArg,
5243 ret: Location,
5244 need_check: bool,
5245 imported_memories: bool,
5246 offset: i32,
5247 heap_access_oob: Label,
5248 unaligned_atomic: Label,
5249 ) -> Result<(), CompileError> {
5250 self.memory_op(
5251 addr,
5252 memarg,
5253 true,
5254 2,
5255 need_check,
5256 imported_memories,
5257 offset,
5258 heap_access_oob,
5259 unaligned_atomic,
5260 |this, addr| {
5261 this.emit_relaxed_zero_extension(
5262 Size::S16,
5263 Location::Memory(addr, 0),
5264 Size::S64,
5265 ret,
5266 )
5267 },
5268 )
5269 }
5270
5271 fn i64_atomic_load_32u(
5272 &mut self,
5273 addr: Location,
5274 memarg: &MemArg,
5275 ret: Location,
5276 need_check: bool,
5277 imported_memories: bool,
5278 offset: i32,
5279 heap_access_oob: Label,
5280 unaligned_atomic: Label,
5281 ) -> Result<(), CompileError> {
5282 self.memory_op(
5283 addr,
5284 memarg,
5285 true,
5286 4,
5287 need_check,
5288 imported_memories,
5289 offset,
5290 heap_access_oob,
5291 unaligned_atomic,
5292 |this, addr| {
5293 match ret {
5294 Location::GPR(_) => {}
5295 Location::Memory(base, offset) => {
5296 this.move_location(
5297 Size::S32,
5298 Location::Imm32(0),
5299 Location::Memory(base, offset + 4),
5300 )?; }
5302 _ => {
5303 codegen_error!("singlepass i64_atomic_load_32u unreachable");
5304 }
5305 }
5306 this.emit_relaxed_zero_extension(
5307 Size::S32,
5308 Location::Memory(addr, 0),
5309 Size::S64,
5310 ret,
5311 )
5312 },
5313 )
5314 }
5315
5316 fn i64_save(
5317 &mut self,
5318 target_value: Location,
5319 memarg: &MemArg,
5320 target_addr: Location,
5321 need_check: bool,
5322 imported_memories: bool,
5323 offset: i32,
5324 heap_access_oob: Label,
5325 unaligned_atomic: Label,
5326 ) -> Result<(), CompileError> {
5327 self.memory_op(
5328 target_addr,
5329 memarg,
5330 false,
5331 8,
5332 need_check,
5333 imported_memories,
5334 offset,
5335 heap_access_oob,
5336 unaligned_atomic,
5337 |this, addr| {
5338 this.emit_relaxed_binop(
5339 AssemblerX64::emit_mov,
5340 Size::S64,
5341 target_value,
5342 Location::Memory(addr, 0),
5343 )
5344 },
5345 )
5346 }
5347
5348 fn i64_save_8(
5349 &mut self,
5350 target_value: Location,
5351 memarg: &MemArg,
5352 target_addr: Location,
5353 need_check: bool,
5354 imported_memories: bool,
5355 offset: i32,
5356 heap_access_oob: Label,
5357 unaligned_atomic: Label,
5358 ) -> Result<(), CompileError> {
5359 self.memory_op(
5360 target_addr,
5361 memarg,
5362 false,
5363 1,
5364 need_check,
5365 imported_memories,
5366 offset,
5367 heap_access_oob,
5368 unaligned_atomic,
5369 |this, addr| {
5370 this.emit_relaxed_binop(
5371 AssemblerX64::emit_mov,
5372 Size::S8,
5373 target_value,
5374 Location::Memory(addr, 0),
5375 )
5376 },
5377 )
5378 }
5379
5380 fn i64_save_16(
5381 &mut self,
5382 target_value: Location,
5383 memarg: &MemArg,
5384 target_addr: Location,
5385 need_check: bool,
5386 imported_memories: bool,
5387 offset: i32,
5388 heap_access_oob: Label,
5389 unaligned_atomic: Label,
5390 ) -> Result<(), CompileError> {
5391 self.memory_op(
5392 target_addr,
5393 memarg,
5394 false,
5395 2,
5396 need_check,
5397 imported_memories,
5398 offset,
5399 heap_access_oob,
5400 unaligned_atomic,
5401 |this, addr| {
5402 this.emit_relaxed_binop(
5403 AssemblerX64::emit_mov,
5404 Size::S16,
5405 target_value,
5406 Location::Memory(addr, 0),
5407 )
5408 },
5409 )
5410 }
5411
5412 fn i64_save_32(
5413 &mut self,
5414 target_value: Location,
5415 memarg: &MemArg,
5416 target_addr: Location,
5417 need_check: bool,
5418 imported_memories: bool,
5419 offset: i32,
5420 heap_access_oob: Label,
5421 unaligned_atomic: Label,
5422 ) -> Result<(), CompileError> {
5423 self.memory_op(
5424 target_addr,
5425 memarg,
5426 false,
5427 4,
5428 need_check,
5429 imported_memories,
5430 offset,
5431 heap_access_oob,
5432 unaligned_atomic,
5433 |this, addr| {
5434 this.emit_relaxed_binop(
5435 AssemblerX64::emit_mov,
5436 Size::S32,
5437 target_value,
5438 Location::Memory(addr, 0),
5439 )
5440 },
5441 )
5442 }
5443
5444 fn i64_atomic_save(
5445 &mut self,
5446 value: Location,
5447 memarg: &MemArg,
5448 target_addr: Location,
5449 need_check: bool,
5450 imported_memories: bool,
5451 offset: i32,
5452 heap_access_oob: Label,
5453 unaligned_atomic: Label,
5454 ) -> Result<(), CompileError> {
5455 self.memory_op(
5456 target_addr,
5457 memarg,
5458 true,
5459 8,
5460 need_check,
5461 imported_memories,
5462 offset,
5463 heap_access_oob,
5464 unaligned_atomic,
5465 |this, addr| this.emit_relaxed_atomic_xchg(Size::S64, value, Location::Memory(addr, 0)),
5466 )
5467 }
5468
5469 fn i64_atomic_save_8(
5470 &mut self,
5471 value: Location,
5472 memarg: &MemArg,
5473 target_addr: Location,
5474 need_check: bool,
5475 imported_memories: bool,
5476 offset: i32,
5477 heap_access_oob: Label,
5478 unaligned_atomic: Label,
5479 ) -> Result<(), CompileError> {
5480 self.memory_op(
5481 target_addr,
5482 memarg,
5483 true,
5484 1,
5485 need_check,
5486 imported_memories,
5487 offset,
5488 heap_access_oob,
5489 unaligned_atomic,
5490 |this, addr| this.emit_relaxed_atomic_xchg(Size::S8, value, Location::Memory(addr, 0)),
5491 )
5492 }
5493
5494 fn i64_atomic_save_16(
5495 &mut self,
5496 value: Location,
5497 memarg: &MemArg,
5498 target_addr: Location,
5499 need_check: bool,
5500 imported_memories: bool,
5501 offset: i32,
5502 heap_access_oob: Label,
5503 unaligned_atomic: Label,
5504 ) -> Result<(), CompileError> {
5505 self.memory_op(
5506 target_addr,
5507 memarg,
5508 true,
5509 2,
5510 need_check,
5511 imported_memories,
5512 offset,
5513 heap_access_oob,
5514 unaligned_atomic,
5515 |this, addr| this.emit_relaxed_atomic_xchg(Size::S16, value, Location::Memory(addr, 0)),
5516 )
5517 }
5518
5519 fn i64_atomic_save_32(
5520 &mut self,
5521 value: Location,
5522 memarg: &MemArg,
5523 target_addr: Location,
5524 need_check: bool,
5525 imported_memories: bool,
5526 offset: i32,
5527 heap_access_oob: Label,
5528 unaligned_atomic: Label,
5529 ) -> Result<(), CompileError> {
5530 self.memory_op(
5531 target_addr,
5532 memarg,
5533 true,
5534 2,
5535 need_check,
5536 imported_memories,
5537 offset,
5538 heap_access_oob,
5539 unaligned_atomic,
5540 |this, addr| this.emit_relaxed_atomic_xchg(Size::S32, value, Location::Memory(addr, 0)),
5541 )
5542 }
5543
5544 fn i64_atomic_add(
5545 &mut self,
5546 loc: Location,
5547 target: Location,
5548 memarg: &MemArg,
5549 ret: Location,
5550 need_check: bool,
5551 imported_memories: bool,
5552 offset: i32,
5553 heap_access_oob: Label,
5554 unaligned_atomic: Label,
5555 ) -> Result<(), CompileError> {
5556 let value = self.acquire_temp_gpr().ok_or_else(|| {
5557 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5558 })?;
5559 self.move_location(Size::S64, loc, Location::GPR(value))?;
5560 self.memory_op(
5561 target,
5562 memarg,
5563 true,
5564 8,
5565 need_check,
5566 imported_memories,
5567 offset,
5568 heap_access_oob,
5569 unaligned_atomic,
5570 |this, addr| {
5571 this.assembler.emit_lock_xadd(
5572 Size::S64,
5573 Location::GPR(value),
5574 Location::Memory(addr, 0),
5575 )
5576 },
5577 )?;
5578 self.move_location(Size::S64, Location::GPR(value), ret)?;
5579 self.release_gpr(value);
5580 Ok(())
5581 }
5582
5583 fn i64_atomic_add_8u(
5584 &mut self,
5585 loc: Location,
5586 target: Location,
5587 memarg: &MemArg,
5588 ret: Location,
5589 need_check: bool,
5590 imported_memories: bool,
5591 offset: i32,
5592 heap_access_oob: Label,
5593 unaligned_atomic: Label,
5594 ) -> Result<(), CompileError> {
5595 let value = self.acquire_temp_gpr().ok_or_else(|| {
5596 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5597 })?;
5598 self.move_location_extend(Size::S8, false, loc, Size::S64, Location::GPR(value))?;
5599 self.memory_op(
5600 target,
5601 memarg,
5602 true,
5603 1,
5604 need_check,
5605 imported_memories,
5606 offset,
5607 heap_access_oob,
5608 unaligned_atomic,
5609 |this, addr| {
5610 this.assembler.emit_lock_xadd(
5611 Size::S8,
5612 Location::GPR(value),
5613 Location::Memory(addr, 0),
5614 )
5615 },
5616 )?;
5617 self.move_location(Size::S64, Location::GPR(value), ret)?;
5618 self.release_gpr(value);
5619 Ok(())
5620 }
5621
5622 fn i64_atomic_add_16u(
5623 &mut self,
5624 loc: Location,
5625 target: Location,
5626 memarg: &MemArg,
5627 ret: Location,
5628 need_check: bool,
5629 imported_memories: bool,
5630 offset: i32,
5631 heap_access_oob: Label,
5632 unaligned_atomic: Label,
5633 ) -> Result<(), CompileError> {
5634 let value = self.acquire_temp_gpr().ok_or_else(|| {
5635 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5636 })?;
5637 self.move_location_extend(Size::S16, false, loc, Size::S64, Location::GPR(value))?;
5638 self.memory_op(
5639 target,
5640 memarg,
5641 true,
5642 2,
5643 need_check,
5644 imported_memories,
5645 offset,
5646 heap_access_oob,
5647 unaligned_atomic,
5648 |this, addr| {
5649 this.assembler.emit_lock_xadd(
5650 Size::S16,
5651 Location::GPR(value),
5652 Location::Memory(addr, 0),
5653 )
5654 },
5655 )?;
5656 self.move_location(Size::S64, Location::GPR(value), ret)?;
5657 self.release_gpr(value);
5658 Ok(())
5659 }
5660
5661 fn i64_atomic_add_32u(
5662 &mut self,
5663 loc: Location,
5664 target: Location,
5665 memarg: &MemArg,
5666 ret: Location,
5667 need_check: bool,
5668 imported_memories: bool,
5669 offset: i32,
5670 heap_access_oob: Label,
5671 unaligned_atomic: Label,
5672 ) -> Result<(), CompileError> {
5673 let value = self.acquire_temp_gpr().ok_or_else(|| {
5674 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5675 })?;
5676 self.move_location_extend(Size::S32, false, loc, Size::S64, Location::GPR(value))?;
5677 self.memory_op(
5678 target,
5679 memarg,
5680 true,
5681 4,
5682 need_check,
5683 imported_memories,
5684 offset,
5685 heap_access_oob,
5686 unaligned_atomic,
5687 |this, addr| {
5688 this.assembler.emit_lock_xadd(
5689 Size::S32,
5690 Location::GPR(value),
5691 Location::Memory(addr, 0),
5692 )
5693 },
5694 )?;
5695 self.move_location(Size::S64, Location::GPR(value), ret)?;
5696 self.release_gpr(value);
5697 Ok(())
5698 }
5699
5700 fn i64_atomic_sub(
5701 &mut self,
5702 loc: Location,
5703 target: Location,
5704 memarg: &MemArg,
5705 ret: Location,
5706 need_check: bool,
5707 imported_memories: bool,
5708 offset: i32,
5709 heap_access_oob: Label,
5710 unaligned_atomic: Label,
5711 ) -> Result<(), CompileError> {
5712 let value = self.acquire_temp_gpr().ok_or_else(|| {
5713 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5714 })?;
5715 self.location_neg(Size::S64, false, loc, Size::S64, Location::GPR(value))?;
5716 self.memory_op(
5717 target,
5718 memarg,
5719 true,
5720 8,
5721 need_check,
5722 imported_memories,
5723 offset,
5724 heap_access_oob,
5725 unaligned_atomic,
5726 |this, addr| {
5727 this.assembler.emit_lock_xadd(
5728 Size::S64,
5729 Location::GPR(value),
5730 Location::Memory(addr, 0),
5731 )
5732 },
5733 )?;
5734 self.move_location(Size::S64, Location::GPR(value), ret)?;
5735 self.release_gpr(value);
5736 Ok(())
5737 }
5738
5739 fn i64_atomic_sub_8u(
5740 &mut self,
5741 loc: Location,
5742 target: Location,
5743 memarg: &MemArg,
5744 ret: Location,
5745 need_check: bool,
5746 imported_memories: bool,
5747 offset: i32,
5748 heap_access_oob: Label,
5749 unaligned_atomic: Label,
5750 ) -> Result<(), CompileError> {
5751 let value = self.acquire_temp_gpr().ok_or_else(|| {
5752 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5753 })?;
5754 self.location_neg(Size::S8, false, loc, Size::S64, Location::GPR(value))?;
5755 self.memory_op(
5756 target,
5757 memarg,
5758 true,
5759 1,
5760 need_check,
5761 imported_memories,
5762 offset,
5763 heap_access_oob,
5764 unaligned_atomic,
5765 |this, addr| {
5766 this.assembler.emit_lock_xadd(
5767 Size::S8,
5768 Location::GPR(value),
5769 Location::Memory(addr, 0),
5770 )
5771 },
5772 )?;
5773 self.move_location(Size::S64, Location::GPR(value), ret)?;
5774 self.release_gpr(value);
5775 Ok(())
5776 }
5777
5778 fn i64_atomic_sub_16u(
5779 &mut self,
5780 loc: Location,
5781 target: Location,
5782 memarg: &MemArg,
5783 ret: Location,
5784 need_check: bool,
5785 imported_memories: bool,
5786 offset: i32,
5787 heap_access_oob: Label,
5788 unaligned_atomic: Label,
5789 ) -> Result<(), CompileError> {
5790 let value = self.acquire_temp_gpr().ok_or_else(|| {
5791 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5792 })?;
5793 self.location_neg(Size::S16, false, loc, Size::S64, Location::GPR(value))?;
5794 self.memory_op(
5795 target,
5796 memarg,
5797 true,
5798 2,
5799 need_check,
5800 imported_memories,
5801 offset,
5802 heap_access_oob,
5803 unaligned_atomic,
5804 |this, addr| {
5805 this.assembler.emit_lock_xadd(
5806 Size::S16,
5807 Location::GPR(value),
5808 Location::Memory(addr, 0),
5809 )
5810 },
5811 )?;
5812 self.move_location(Size::S64, Location::GPR(value), ret)?;
5813 self.release_gpr(value);
5814 Ok(())
5815 }
5816
5817 fn i64_atomic_sub_32u(
5818 &mut self,
5819 loc: Location,
5820 target: Location,
5821 memarg: &MemArg,
5822 ret: Location,
5823 need_check: bool,
5824 imported_memories: bool,
5825 offset: i32,
5826 heap_access_oob: Label,
5827 unaligned_atomic: Label,
5828 ) -> Result<(), CompileError> {
5829 let value = self.acquire_temp_gpr().ok_or_else(|| {
5830 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5831 })?;
5832 self.location_neg(Size::S32, false, loc, Size::S64, Location::GPR(value))?;
5833 self.memory_op(
5834 target,
5835 memarg,
5836 true,
5837 4,
5838 need_check,
5839 imported_memories,
5840 offset,
5841 heap_access_oob,
5842 unaligned_atomic,
5843 |this, addr| {
5844 this.assembler.emit_lock_xadd(
5845 Size::S32,
5846 Location::GPR(value),
5847 Location::Memory(addr, 0),
5848 )
5849 },
5850 )?;
5851 self.move_location(Size::S64, Location::GPR(value), ret)?;
5852 self.release_gpr(value);
5853 Ok(())
5854 }
5855
5856 fn i64_atomic_and(
5857 &mut self,
5858 loc: Location,
5859 target: Location,
5860 memarg: &MemArg,
5861 ret: Location,
5862 need_check: bool,
5863 imported_memories: bool,
5864 offset: i32,
5865 heap_access_oob: Label,
5866 unaligned_atomic: Label,
5867 ) -> Result<(), CompileError> {
5868 self.emit_compare_and_swap(
5869 loc,
5870 target,
5871 ret,
5872 memarg,
5873 8,
5874 Size::S64,
5875 Size::S64,
5876 need_check,
5877 imported_memories,
5878 offset,
5879 heap_access_oob,
5880 unaligned_atomic,
5881 |this, src, dst| {
5882 this.assembler
5883 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
5884 },
5885 )
5886 }
5887
5888 fn i64_atomic_and_8u(
5889 &mut self,
5890 loc: Location,
5891 target: Location,
5892 memarg: &MemArg,
5893 ret: Location,
5894 need_check: bool,
5895 imported_memories: bool,
5896 offset: i32,
5897 heap_access_oob: Label,
5898 unaligned_atomic: Label,
5899 ) -> Result<(), CompileError> {
5900 self.emit_compare_and_swap(
5901 loc,
5902 target,
5903 ret,
5904 memarg,
5905 1,
5906 Size::S8,
5907 Size::S64,
5908 need_check,
5909 imported_memories,
5910 offset,
5911 heap_access_oob,
5912 unaligned_atomic,
5913 |this, src, dst| {
5914 this.assembler
5915 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
5916 },
5917 )
5918 }
5919
5920 fn i64_atomic_and_16u(
5921 &mut self,
5922 loc: Location,
5923 target: Location,
5924 memarg: &MemArg,
5925 ret: Location,
5926 need_check: bool,
5927 imported_memories: bool,
5928 offset: i32,
5929 heap_access_oob: Label,
5930 unaligned_atomic: Label,
5931 ) -> Result<(), CompileError> {
5932 self.emit_compare_and_swap(
5933 loc,
5934 target,
5935 ret,
5936 memarg,
5937 2,
5938 Size::S16,
5939 Size::S64,
5940 need_check,
5941 imported_memories,
5942 offset,
5943 heap_access_oob,
5944 unaligned_atomic,
5945 |this, src, dst| {
5946 this.assembler
5947 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
5948 },
5949 )
5950 }
5951
5952 fn i64_atomic_and_32u(
5953 &mut self,
5954 loc: Location,
5955 target: Location,
5956 memarg: &MemArg,
5957 ret: Location,
5958 need_check: bool,
5959 imported_memories: bool,
5960 offset: i32,
5961 heap_access_oob: Label,
5962 unaligned_atomic: Label,
5963 ) -> Result<(), CompileError> {
5964 self.emit_compare_and_swap(
5965 loc,
5966 target,
5967 ret,
5968 memarg,
5969 4,
5970 Size::S32,
5971 Size::S64,
5972 need_check,
5973 imported_memories,
5974 offset,
5975 heap_access_oob,
5976 unaligned_atomic,
5977 |this, src, dst| {
5978 this.assembler
5979 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
5980 },
5981 )
5982 }
5983
5984 fn i64_atomic_or(
5985 &mut self,
5986 loc: Location,
5987 target: Location,
5988 memarg: &MemArg,
5989 ret: Location,
5990 need_check: bool,
5991 imported_memories: bool,
5992 offset: i32,
5993 heap_access_oob: Label,
5994 unaligned_atomic: Label,
5995 ) -> Result<(), CompileError> {
5996 self.emit_compare_and_swap(
5997 loc,
5998 target,
5999 ret,
6000 memarg,
6001 8,
6002 Size::S64,
6003 Size::S64,
6004 need_check,
6005 imported_memories,
6006 offset,
6007 heap_access_oob,
6008 unaligned_atomic,
6009 |this, src, dst| {
6010 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6011 },
6012 )
6013 }
6014
6015 fn i64_atomic_or_8u(
6016 &mut self,
6017 loc: Location,
6018 target: Location,
6019 memarg: &MemArg,
6020 ret: Location,
6021 need_check: bool,
6022 imported_memories: bool,
6023 offset: i32,
6024 heap_access_oob: Label,
6025 unaligned_atomic: Label,
6026 ) -> Result<(), CompileError> {
6027 self.emit_compare_and_swap(
6028 loc,
6029 target,
6030 ret,
6031 memarg,
6032 1,
6033 Size::S8,
6034 Size::S64,
6035 need_check,
6036 imported_memories,
6037 offset,
6038 heap_access_oob,
6039 unaligned_atomic,
6040 |this, src, dst| {
6041 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6042 },
6043 )
6044 }
6045
6046 fn i64_atomic_or_16u(
6047 &mut self,
6048 loc: Location,
6049 target: Location,
6050 memarg: &MemArg,
6051 ret: Location,
6052 need_check: bool,
6053 imported_memories: bool,
6054 offset: i32,
6055 heap_access_oob: Label,
6056 unaligned_atomic: Label,
6057 ) -> Result<(), CompileError> {
6058 self.emit_compare_and_swap(
6059 loc,
6060 target,
6061 ret,
6062 memarg,
6063 2,
6064 Size::S16,
6065 Size::S64,
6066 need_check,
6067 imported_memories,
6068 offset,
6069 heap_access_oob,
6070 unaligned_atomic,
6071 |this, src, dst| {
6072 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6073 },
6074 )
6075 }
6076
6077 fn i64_atomic_or_32u(
6078 &mut self,
6079 loc: Location,
6080 target: Location,
6081 memarg: &MemArg,
6082 ret: Location,
6083 need_check: bool,
6084 imported_memories: bool,
6085 offset: i32,
6086 heap_access_oob: Label,
6087 unaligned_atomic: Label,
6088 ) -> Result<(), CompileError> {
6089 self.emit_compare_and_swap(
6090 loc,
6091 target,
6092 ret,
6093 memarg,
6094 4,
6095 Size::S32,
6096 Size::S64,
6097 need_check,
6098 imported_memories,
6099 offset,
6100 heap_access_oob,
6101 unaligned_atomic,
6102 |this, src, dst| {
6103 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6104 },
6105 )
6106 }
6107
6108 fn i64_atomic_xor(
6109 &mut self,
6110 loc: Location,
6111 target: Location,
6112 memarg: &MemArg,
6113 ret: Location,
6114 need_check: bool,
6115 imported_memories: bool,
6116 offset: i32,
6117 heap_access_oob: Label,
6118 unaligned_atomic: Label,
6119 ) -> Result<(), CompileError> {
6120 self.emit_compare_and_swap(
6121 loc,
6122 target,
6123 ret,
6124 memarg,
6125 8,
6126 Size::S64,
6127 Size::S64,
6128 need_check,
6129 imported_memories,
6130 offset,
6131 heap_access_oob,
6132 unaligned_atomic,
6133 |this, src, dst| {
6134 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6135 },
6136 )
6137 }
6138
6139 fn i64_atomic_xor_8u(
6140 &mut self,
6141 loc: Location,
6142 target: Location,
6143 memarg: &MemArg,
6144 ret: Location,
6145 need_check: bool,
6146 imported_memories: bool,
6147 offset: i32,
6148 heap_access_oob: Label,
6149 unaligned_atomic: Label,
6150 ) -> Result<(), CompileError> {
6151 self.emit_compare_and_swap(
6152 loc,
6153 target,
6154 ret,
6155 memarg,
6156 1,
6157 Size::S8,
6158 Size::S64,
6159 need_check,
6160 imported_memories,
6161 offset,
6162 heap_access_oob,
6163 unaligned_atomic,
6164 |this, src, dst| {
6165 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6166 },
6167 )
6168 }
6169
6170 fn i64_atomic_xor_16u(
6171 &mut self,
6172 loc: Location,
6173 target: Location,
6174 memarg: &MemArg,
6175 ret: Location,
6176 need_check: bool,
6177 imported_memories: bool,
6178 offset: i32,
6179 heap_access_oob: Label,
6180 unaligned_atomic: Label,
6181 ) -> Result<(), CompileError> {
6182 self.emit_compare_and_swap(
6183 loc,
6184 target,
6185 ret,
6186 memarg,
6187 2,
6188 Size::S16,
6189 Size::S64,
6190 need_check,
6191 imported_memories,
6192 offset,
6193 heap_access_oob,
6194 unaligned_atomic,
6195 |this, src, dst| {
6196 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6197 },
6198 )
6199 }
6200
6201 fn i64_atomic_xor_32u(
6202 &mut self,
6203 loc: Location,
6204 target: Location,
6205 memarg: &MemArg,
6206 ret: Location,
6207 need_check: bool,
6208 imported_memories: bool,
6209 offset: i32,
6210 heap_access_oob: Label,
6211 unaligned_atomic: Label,
6212 ) -> Result<(), CompileError> {
6213 self.emit_compare_and_swap(
6214 loc,
6215 target,
6216 ret,
6217 memarg,
6218 4,
6219 Size::S32,
6220 Size::S64,
6221 need_check,
6222 imported_memories,
6223 offset,
6224 heap_access_oob,
6225 unaligned_atomic,
6226 |this, src, dst| {
6227 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6228 },
6229 )
6230 }
6231
6232 fn i64_atomic_xchg(
6233 &mut self,
6234 loc: Location,
6235 target: Location,
6236 memarg: &MemArg,
6237 ret: Location,
6238 need_check: bool,
6239 imported_memories: bool,
6240 offset: i32,
6241 heap_access_oob: Label,
6242 unaligned_atomic: Label,
6243 ) -> Result<(), CompileError> {
6244 let value = self.acquire_temp_gpr().ok_or_else(|| {
6245 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6246 })?;
6247 self.move_location(Size::S64, loc, Location::GPR(value))?;
6248 self.memory_op(
6249 target,
6250 memarg,
6251 true,
6252 8,
6253 need_check,
6254 imported_memories,
6255 offset,
6256 heap_access_oob,
6257 unaligned_atomic,
6258 |this, addr| {
6259 this.assembler
6260 .emit_xchg(Size::S64, Location::GPR(value), Location::Memory(addr, 0))
6261 },
6262 )?;
6263 self.move_location(Size::S64, Location::GPR(value), ret)?;
6264 self.release_gpr(value);
6265 Ok(())
6266 }
6267
6268 fn i64_atomic_xchg_8u(
6269 &mut self,
6270 loc: Location,
6271 target: Location,
6272 memarg: &MemArg,
6273 ret: Location,
6274 need_check: bool,
6275 imported_memories: bool,
6276 offset: i32,
6277 heap_access_oob: Label,
6278 unaligned_atomic: Label,
6279 ) -> Result<(), CompileError> {
6280 let value = self.acquire_temp_gpr().ok_or_else(|| {
6281 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6282 })?;
6283 self.assembler
6284 .emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value))?;
6285 self.memory_op(
6286 target,
6287 memarg,
6288 true,
6289 1,
6290 need_check,
6291 imported_memories,
6292 offset,
6293 heap_access_oob,
6294 unaligned_atomic,
6295 |this, addr| {
6296 this.assembler
6297 .emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0))
6298 },
6299 )?;
6300 self.move_location(Size::S64, Location::GPR(value), ret)?;
6301 self.release_gpr(value);
6302 Ok(())
6303 }
6304
6305 fn i64_atomic_xchg_16u(
6306 &mut self,
6307 loc: Location,
6308 target: Location,
6309 memarg: &MemArg,
6310 ret: Location,
6311 need_check: bool,
6312 imported_memories: bool,
6313 offset: i32,
6314 heap_access_oob: Label,
6315 unaligned_atomic: Label,
6316 ) -> Result<(), CompileError> {
6317 let value = self.acquire_temp_gpr().ok_or_else(|| {
6318 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6319 })?;
6320 self.assembler
6321 .emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value))?;
6322 self.memory_op(
6323 target,
6324 memarg,
6325 true,
6326 2,
6327 need_check,
6328 imported_memories,
6329 offset,
6330 heap_access_oob,
6331 unaligned_atomic,
6332 |this, addr| {
6333 this.assembler
6334 .emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0))
6335 },
6336 )?;
6337 self.move_location(Size::S64, Location::GPR(value), ret)?;
6338 self.release_gpr(value);
6339 Ok(())
6340 }
6341
6342 fn i64_atomic_xchg_32u(
6343 &mut self,
6344 loc: Location,
6345 target: Location,
6346 memarg: &MemArg,
6347 ret: Location,
6348 need_check: bool,
6349 imported_memories: bool,
6350 offset: i32,
6351 heap_access_oob: Label,
6352 unaligned_atomic: Label,
6353 ) -> Result<(), CompileError> {
6354 let value = self.acquire_temp_gpr().ok_or_else(|| {
6355 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6356 })?;
6357 self.assembler
6358 .emit_movzx(Size::S32, loc, Size::S64, Location::GPR(value))?;
6359 self.memory_op(
6360 target,
6361 memarg,
6362 true,
6363 4,
6364 need_check,
6365 imported_memories,
6366 offset,
6367 heap_access_oob,
6368 unaligned_atomic,
6369 |this, addr| {
6370 this.assembler
6371 .emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0))
6372 },
6373 )?;
6374 self.move_location(Size::S64, Location::GPR(value), ret)?;
6375 self.release_gpr(value);
6376 Ok(())
6377 }
6378
6379 fn i64_atomic_cmpxchg(
6380 &mut self,
6381 new: Location,
6382 cmp: Location,
6383 target: Location,
6384 memarg: &MemArg,
6385 ret: Location,
6386 need_check: bool,
6387 imported_memories: bool,
6388 offset: i32,
6389 heap_access_oob: Label,
6390 unaligned_atomic: Label,
6391 ) -> Result<(), CompileError> {
6392 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6393 let value = if cmp == Location::GPR(GPR::R14) {
6394 if new == Location::GPR(GPR::R13) {
6395 GPR::R12
6396 } else {
6397 GPR::R13
6398 }
6399 } else {
6400 GPR::R14
6401 };
6402 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6403 self.assembler
6404 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6405 self.assembler
6406 .emit_mov(Size::S64, new, Location::GPR(value))?;
6407
6408 self.memory_op(
6409 target,
6410 memarg,
6411 true,
6412 8,
6413 need_check,
6414 imported_memories,
6415 offset,
6416 heap_access_oob,
6417 unaligned_atomic,
6418 |this, addr| {
6419 this.assembler.emit_lock_cmpxchg(
6420 Size::S64,
6421 Location::GPR(value),
6422 Location::Memory(addr, 0),
6423 )?;
6424 this.assembler
6425 .emit_mov(Size::S64, Location::GPR(compare), ret)
6426 },
6427 )?;
6428 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6429 self.release_gpr(compare);
6430 Ok(())
6431 }
6432
6433 fn i64_atomic_cmpxchg_8u(
6434 &mut self,
6435 new: Location,
6436 cmp: Location,
6437 target: Location,
6438 memarg: &MemArg,
6439 ret: Location,
6440 need_check: bool,
6441 imported_memories: bool,
6442 offset: i32,
6443 heap_access_oob: Label,
6444 unaligned_atomic: Label,
6445 ) -> Result<(), CompileError> {
6446 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6447 let value = if cmp == Location::GPR(GPR::R14) {
6448 if new == Location::GPR(GPR::R13) {
6449 GPR::R12
6450 } else {
6451 GPR::R13
6452 }
6453 } else {
6454 GPR::R14
6455 };
6456 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6457 self.assembler
6458 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6459 self.assembler
6460 .emit_mov(Size::S64, new, Location::GPR(value))?;
6461
6462 self.memory_op(
6463 target,
6464 memarg,
6465 true,
6466 1,
6467 need_check,
6468 imported_memories,
6469 offset,
6470 heap_access_oob,
6471 unaligned_atomic,
6472 |this, addr| {
6473 this.assembler.emit_lock_cmpxchg(
6474 Size::S8,
6475 Location::GPR(value),
6476 Location::Memory(addr, 0),
6477 )?;
6478 this.assembler
6479 .emit_movzx(Size::S8, Location::GPR(compare), Size::S64, ret)
6480 },
6481 )?;
6482 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6483 self.release_gpr(compare);
6484 Ok(())
6485 }
6486
6487 fn i64_atomic_cmpxchg_16u(
6488 &mut self,
6489 new: Location,
6490 cmp: Location,
6491 target: Location,
6492 memarg: &MemArg,
6493 ret: Location,
6494 need_check: bool,
6495 imported_memories: bool,
6496 offset: i32,
6497 heap_access_oob: Label,
6498 unaligned_atomic: Label,
6499 ) -> Result<(), CompileError> {
6500 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6501 let value = if cmp == Location::GPR(GPR::R14) {
6502 if new == Location::GPR(GPR::R13) {
6503 GPR::R12
6504 } else {
6505 GPR::R13
6506 }
6507 } else {
6508 GPR::R14
6509 };
6510 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6511 self.assembler
6512 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6513 self.assembler
6514 .emit_mov(Size::S64, new, Location::GPR(value))?;
6515
6516 self.memory_op(
6517 target,
6518 memarg,
6519 true,
6520 2,
6521 need_check,
6522 imported_memories,
6523 offset,
6524 heap_access_oob,
6525 unaligned_atomic,
6526 |this, addr| {
6527 this.assembler.emit_lock_cmpxchg(
6528 Size::S16,
6529 Location::GPR(value),
6530 Location::Memory(addr, 0),
6531 )?;
6532 this.assembler
6533 .emit_movzx(Size::S16, Location::GPR(compare), Size::S64, ret)
6534 },
6535 )?;
6536 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6537 self.release_gpr(compare);
6538 Ok(())
6539 }
6540
6541 fn i64_atomic_cmpxchg_32u(
6542 &mut self,
6543 new: Location,
6544 cmp: Location,
6545 target: Location,
6546 memarg: &MemArg,
6547 ret: Location,
6548 need_check: bool,
6549 imported_memories: bool,
6550 offset: i32,
6551 heap_access_oob: Label,
6552 unaligned_atomic: Label,
6553 ) -> Result<(), CompileError> {
6554 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6555 let value = if cmp == Location::GPR(GPR::R14) {
6556 if new == Location::GPR(GPR::R13) {
6557 GPR::R12
6558 } else {
6559 GPR::R13
6560 }
6561 } else {
6562 GPR::R14
6563 };
6564 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6565 self.assembler
6566 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6567 self.assembler
6568 .emit_mov(Size::S64, new, Location::GPR(value))?;
6569
6570 self.memory_op(
6571 target,
6572 memarg,
6573 true,
6574 4,
6575 need_check,
6576 imported_memories,
6577 offset,
6578 heap_access_oob,
6579 unaligned_atomic,
6580 |this, addr| {
6581 this.assembler.emit_lock_cmpxchg(
6582 Size::S32,
6583 Location::GPR(value),
6584 Location::Memory(addr, 0),
6585 )?;
6586 this.assembler
6587 .emit_mov(Size::S32, Location::GPR(compare), ret)
6588 },
6589 )?;
6590 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6591 self.release_gpr(compare);
6592 Ok(())
6593 }
6594
6595 fn f32_load(
6596 &mut self,
6597 addr: Location,
6598 memarg: &MemArg,
6599 ret: Location,
6600 need_check: bool,
6601 imported_memories: bool,
6602 offset: i32,
6603 heap_access_oob: Label,
6604 unaligned_atomic: Label,
6605 ) -> Result<(), CompileError> {
6606 self.memory_op(
6607 addr,
6608 memarg,
6609 false,
6610 4,
6611 need_check,
6612 imported_memories,
6613 offset,
6614 heap_access_oob,
6615 unaligned_atomic,
6616 |this, addr| {
6617 this.emit_relaxed_binop(
6618 AssemblerX64::emit_mov,
6619 Size::S32,
6620 Location::Memory(addr, 0),
6621 ret,
6622 )
6623 },
6624 )
6625 }
6626
6627 fn f32_save(
6628 &mut self,
6629 target_value: Location,
6630 memarg: &MemArg,
6631 target_addr: Location,
6632 canonicalize: bool,
6633 need_check: bool,
6634 imported_memories: bool,
6635 offset: i32,
6636 heap_access_oob: Label,
6637 unaligned_atomic: Label,
6638 ) -> Result<(), CompileError> {
6639 self.memory_op(
6640 target_addr,
6641 memarg,
6642 false,
6643 4,
6644 need_check,
6645 imported_memories,
6646 offset,
6647 heap_access_oob,
6648 unaligned_atomic,
6649 |this, addr| {
6650 if !canonicalize {
6651 this.emit_relaxed_binop(
6652 AssemblerX64::emit_mov,
6653 Size::S32,
6654 target_value,
6655 Location::Memory(addr, 0),
6656 )
6657 } else {
6658 this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
6659 }
6660 },
6661 )
6662 }
6663
6664 fn f64_load(
6665 &mut self,
6666 addr: Location,
6667 memarg: &MemArg,
6668 ret: Location,
6669 need_check: bool,
6670 imported_memories: bool,
6671 offset: i32,
6672 heap_access_oob: Label,
6673 unaligned_atomic: Label,
6674 ) -> Result<(), CompileError> {
6675 self.memory_op(
6676 addr,
6677 memarg,
6678 false,
6679 8,
6680 need_check,
6681 imported_memories,
6682 offset,
6683 heap_access_oob,
6684 unaligned_atomic,
6685 |this, addr| {
6686 this.emit_relaxed_binop(
6687 AssemblerX64::emit_mov,
6688 Size::S64,
6689 Location::Memory(addr, 0),
6690 ret,
6691 )
6692 },
6693 )
6694 }
6695
6696 fn f64_save(
6697 &mut self,
6698 target_value: Location,
6699 memarg: &MemArg,
6700 target_addr: Location,
6701 canonicalize: bool,
6702 need_check: bool,
6703 imported_memories: bool,
6704 offset: i32,
6705 heap_access_oob: Label,
6706 unaligned_atomic: Label,
6707 ) -> Result<(), CompileError> {
6708 self.memory_op(
6709 target_addr,
6710 memarg,
6711 false,
6712 8,
6713 need_check,
6714 imported_memories,
6715 offset,
6716 heap_access_oob,
6717 unaligned_atomic,
6718 |this, addr| {
6719 if !canonicalize {
6720 this.emit_relaxed_binop(
6721 AssemblerX64::emit_mov,
6722 Size::S64,
6723 target_value,
6724 Location::Memory(addr, 0),
6725 )
6726 } else {
6727 this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
6728 }
6729 },
6730 )
6731 }
6732
6733 fn convert_f64_i64(
6734 &mut self,
6735 loc: Location,
6736 signed: bool,
6737 ret: Location,
6738 ) -> Result<(), CompileError> {
6739 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
6740 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
6741 })?;
6742 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
6743 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6744 })?;
6745 if self.assembler.arch_has_fconverti() {
6746 self.emit_relaxed_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6747 if signed {
6748 self.assembler.arch_emit_f64_convert_si64(tmp_in, tmp_out)?;
6749 } else {
6750 self.assembler.arch_emit_f64_convert_ui64(tmp_in, tmp_out)?;
6751 }
6752 self.emit_relaxed_mov(Size::S64, Location::SIMD(tmp_out), ret)?;
6753 } else if signed {
6754 self.assembler
6755 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6756 self.assembler
6757 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6758 self.move_location(Size::S64, Location::SIMD(tmp_out), ret)?;
6759 } else {
6760 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6761 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6762 })?;
6763
6764 let do_convert = self.assembler.get_label();
6765 let end_convert = self.assembler.get_label();
6766
6767 self.assembler
6768 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6769 self.assembler.emit_test_gpr_64(tmp_in)?;
6770 self.assembler.emit_jmp(Condition::Signed, do_convert)?;
6771 self.assembler
6772 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6773 self.assembler.emit_jmp(Condition::None, end_convert)?;
6774 self.emit_label(do_convert)?;
6775 self.move_location(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp))?;
6776 self.assembler
6777 .emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp))?;
6778 self.assembler
6779 .emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in))?;
6780 self.assembler
6781 .emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in))?;
6782 self.assembler
6783 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6784 self.assembler
6785 .emit_vaddsd(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out)?;
6786 self.emit_label(end_convert)?;
6787 self.move_location(Size::S64, Location::SIMD(tmp_out), ret)?;
6788
6789 self.release_gpr(tmp);
6790 }
6791 self.release_gpr(tmp_in);
6792 self.release_simd(tmp_out);
6793 Ok(())
6794 }
6795
6796 fn convert_f64_i32(
6797 &mut self,
6798 loc: Location,
6799 signed: bool,
6800 ret: Location,
6801 ) -> Result<(), CompileError> {
6802 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
6803 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
6804 })?;
6805 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
6806 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6807 })?;
6808 if self.assembler.arch_has_fconverti() {
6809 self.emit_relaxed_mov(Size::S32, loc, Location::GPR(tmp_in))?;
6810 if signed {
6811 self.assembler.arch_emit_f64_convert_si32(tmp_in, tmp_out)?;
6812 } else {
6813 self.assembler.arch_emit_f64_convert_ui32(tmp_in, tmp_out)?;
6814 }
6815 self.emit_relaxed_mov(Size::S64, Location::SIMD(tmp_out), ret)?;
6816 } else {
6817 self.assembler
6818 .emit_mov(Size::S32, loc, Location::GPR(tmp_in))?;
6819 if signed {
6820 self.assembler
6821 .emit_vcvtsi2sd_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6822 } else {
6823 self.assembler
6824 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6825 }
6826 self.move_location(Size::S64, Location::SIMD(tmp_out), ret)?;
6827 }
6828 self.release_gpr(tmp_in);
6829 self.release_simd(tmp_out);
6830 Ok(())
6831 }
6832
6833 fn convert_f32_i64(
6834 &mut self,
6835 loc: Location,
6836 signed: bool,
6837 ret: Location,
6838 ) -> Result<(), CompileError> {
6839 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
6840 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
6841 })?;
6842 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
6843 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6844 })?;
6845 if self.assembler.arch_has_fconverti() {
6846 self.emit_relaxed_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6847 if signed {
6848 self.assembler.arch_emit_f32_convert_si64(tmp_in, tmp_out)?;
6849 } else {
6850 self.assembler.arch_emit_f32_convert_ui64(tmp_in, tmp_out)?;
6851 }
6852 self.emit_relaxed_mov(Size::S32, Location::SIMD(tmp_out), ret)?;
6853 } else if signed {
6854 self.assembler
6855 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6856 self.assembler
6857 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6858 self.move_location(Size::S32, Location::SIMD(tmp_out), ret)?;
6859 } else {
6860 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6861 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6862 })?;
6863
6864 let do_convert = self.assembler.get_label();
6865 let end_convert = self.assembler.get_label();
6866
6867 self.assembler
6868 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6869 self.assembler.emit_test_gpr_64(tmp_in)?;
6870 self.assembler.emit_jmp(Condition::Signed, do_convert)?;
6871 self.assembler
6872 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6873 self.assembler.emit_jmp(Condition::None, end_convert)?;
6874 self.emit_label(do_convert)?;
6875 self.move_location(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp))?;
6876 self.assembler
6877 .emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp))?;
6878 self.assembler
6879 .emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in))?;
6880 self.assembler
6881 .emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in))?;
6882 self.assembler
6883 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6884 self.assembler
6885 .emit_vaddss(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out)?;
6886 self.emit_label(end_convert)?;
6887 self.move_location(Size::S32, Location::SIMD(tmp_out), ret)?;
6888
6889 self.release_gpr(tmp);
6890 }
6891 self.release_gpr(tmp_in);
6892 self.release_simd(tmp_out);
6893 Ok(())
6894 }
6895
6896 fn convert_f32_i32(
6897 &mut self,
6898 loc: Location,
6899 signed: bool,
6900 ret: Location,
6901 ) -> Result<(), CompileError> {
6902 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
6903 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
6904 })?;
6905 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
6906 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6907 })?;
6908 if self.assembler.arch_has_fconverti() {
6909 self.emit_relaxed_mov(Size::S32, loc, Location::GPR(tmp_in))?;
6910 if signed {
6911 self.assembler.arch_emit_f32_convert_si32(tmp_in, tmp_out)?;
6912 } else {
6913 self.assembler.arch_emit_f32_convert_ui32(tmp_in, tmp_out)?;
6914 }
6915 self.emit_relaxed_mov(Size::S32, Location::SIMD(tmp_out), ret)?;
6916 } else {
6917 self.assembler
6918 .emit_mov(Size::S32, loc, Location::GPR(tmp_in))?;
6919 if signed {
6920 self.assembler
6921 .emit_vcvtsi2ss_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6922 } else {
6923 self.assembler
6924 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6925 }
6926 self.move_location(Size::S32, Location::SIMD(tmp_out), ret)?;
6927 }
6928 self.release_gpr(tmp_in);
6929 self.release_simd(tmp_out);
6930 Ok(())
6931 }
6932
6933 fn convert_i64_f64(
6934 &mut self,
6935 loc: Location,
6936 ret: Location,
6937 signed: bool,
6938 sat: bool,
6939 ) -> Result<(), CompileError> {
6940 match (signed, sat) {
6941 (false, true) => self.convert_i64_f64_u_s(loc, ret),
6942 (false, false) => self.convert_i64_f64_u_u(loc, ret),
6943 (true, true) => self.convert_i64_f64_s_s(loc, ret),
6944 (true, false) => self.convert_i64_f64_s_u(loc, ret),
6945 }
6946 }
6947
6948 fn convert_i32_f64(
6949 &mut self,
6950 loc: Location,
6951 ret: Location,
6952 signed: bool,
6953 sat: bool,
6954 ) -> Result<(), CompileError> {
6955 match (signed, sat) {
6956 (false, true) => self.convert_i32_f64_u_s(loc, ret),
6957 (false, false) => self.convert_i32_f64_u_u(loc, ret),
6958 (true, true) => self.convert_i32_f64_s_s(loc, ret),
6959 (true, false) => self.convert_i32_f64_s_u(loc, ret),
6960 }
6961 }
6962
6963 fn convert_i64_f32(
6964 &mut self,
6965 loc: Location,
6966 ret: Location,
6967 signed: bool,
6968 sat: bool,
6969 ) -> Result<(), CompileError> {
6970 match (signed, sat) {
6971 (false, true) => self.convert_i64_f32_u_s(loc, ret),
6972 (false, false) => self.convert_i64_f32_u_u(loc, ret),
6973 (true, true) => self.convert_i64_f32_s_s(loc, ret),
6974 (true, false) => self.convert_i64_f32_s_u(loc, ret),
6975 }
6976 }
6977
6978 fn convert_i32_f32(
6979 &mut self,
6980 loc: Location,
6981 ret: Location,
6982 signed: bool,
6983 sat: bool,
6984 ) -> Result<(), CompileError> {
6985 match (signed, sat) {
6986 (false, true) => self.convert_i32_f32_u_s(loc, ret),
6987 (false, false) => self.convert_i32_f32_u_u(loc, ret),
6988 (true, true) => self.convert_i32_f32_s_s(loc, ret),
6989 (true, false) => self.convert_i32_f32_s_u(loc, ret),
6990 }
6991 }
6992
6993 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6994 self.emit_relaxed_avx(AssemblerX64::emit_vcvtss2sd, loc, loc, ret)
6995 }
6996
6997 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6998 self.emit_relaxed_avx(AssemblerX64::emit_vcvtsd2ss, loc, loc, ret)
6999 }
7000
7001 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7002 if self.assembler.arch_has_fneg() {
7003 let tmp = self.acquire_temp_simd().ok_or_else(|| {
7004 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7005 })?;
7006 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp))?;
7007 self.assembler.arch_emit_f64_neg(tmp, tmp)?;
7008 self.emit_relaxed_mov(Size::S64, Location::SIMD(tmp), ret)?;
7009 self.release_simd(tmp);
7010 } else {
7011 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7012 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7013 })?;
7014 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7015 self.assembler.emit_btc_gpr_imm8_64(63, tmp)?;
7016 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7017 self.release_gpr(tmp);
7018 }
7019 Ok(())
7020 }
7021
7022 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7023 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7024 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7025 })?;
7026 let c = self.acquire_temp_gpr().ok_or_else(|| {
7027 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7028 })?;
7029
7030 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7031 self.move_location(
7032 Size::S64,
7033 Location::Imm64(0x7fffffffffffffffu64),
7034 Location::GPR(c),
7035 )?;
7036 self.assembler
7037 .emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp))?;
7038 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7039
7040 self.release_gpr(c);
7041 self.release_gpr(tmp);
7042 Ok(())
7043 }
7044
7045 fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7046 let c = self.acquire_temp_gpr().ok_or_else(|| {
7047 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7048 })?;
7049
7050 self.move_location(
7051 Size::S64,
7052 Location::Imm64(0x7fffffffffffffffu64),
7053 Location::GPR(c),
7054 )?;
7055 self.assembler
7056 .emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp1))?;
7057
7058 self.move_location(
7059 Size::S64,
7060 Location::Imm64(0x8000000000000000u64),
7061 Location::GPR(c),
7062 )?;
7063 self.assembler
7064 .emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp2))?;
7065
7066 self.assembler
7067 .emit_or(Size::S64, Location::GPR(tmp2), Location::GPR(tmp1))?;
7068
7069 self.release_gpr(c);
7070 Ok(())
7071 }
7072
7073 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7074 self.emit_relaxed_avx(AssemblerX64::emit_vsqrtsd, loc, loc, ret)
7075 }
7076
7077 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7078 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_trunc, loc, loc, ret)
7079 }
7080
7081 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7082 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_ceil, loc, loc, ret)
7083 }
7084
7085 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7086 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_floor, loc, loc, ret)
7087 }
7088
7089 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7090 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_nearest, loc, loc, ret)
7091 }
7092
7093 fn f64_cmp_ge(
7094 &mut self,
7095 loc_a: Location,
7096 loc_b: Location,
7097 ret: Location,
7098 ) -> Result<(), CompileError> {
7099 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgesd, loc_a, loc_b, ret)?;
7100 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7101 }
7102
7103 fn f64_cmp_gt(
7104 &mut self,
7105 loc_a: Location,
7106 loc_b: Location,
7107 ret: Location,
7108 ) -> Result<(), CompileError> {
7109 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgtsd, loc_a, loc_b, ret)?;
7110 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7111 }
7112
7113 fn f64_cmp_le(
7114 &mut self,
7115 loc_a: Location,
7116 loc_b: Location,
7117 ret: Location,
7118 ) -> Result<(), CompileError> {
7119 self.emit_relaxed_avx(AssemblerX64::emit_vcmplesd, loc_a, loc_b, ret)?;
7120 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7121 }
7122
7123 fn f64_cmp_lt(
7124 &mut self,
7125 loc_a: Location,
7126 loc_b: Location,
7127 ret: Location,
7128 ) -> Result<(), CompileError> {
7129 self.emit_relaxed_avx(AssemblerX64::emit_vcmpltsd, loc_a, loc_b, ret)?;
7130 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7131 }
7132
7133 fn f64_cmp_ne(
7134 &mut self,
7135 loc_a: Location,
7136 loc_b: Location,
7137 ret: Location,
7138 ) -> Result<(), CompileError> {
7139 self.emit_relaxed_avx(AssemblerX64::emit_vcmpneqsd, loc_a, loc_b, ret)?;
7140 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7141 }
7142
7143 fn f64_cmp_eq(
7144 &mut self,
7145 loc_a: Location,
7146 loc_b: Location,
7147 ret: Location,
7148 ) -> Result<(), CompileError> {
7149 self.emit_relaxed_avx(AssemblerX64::emit_vcmpeqsd, loc_a, loc_b, ret)?;
7150 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7151 }
7152
7153 fn f64_min(
7154 &mut self,
7155 loc_a: Location,
7156 loc_b: Location,
7157 ret: Location,
7158 ) -> Result<(), CompileError> {
7159 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7161 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7162 })?;
7163 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7164 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7165 })?;
7166 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7167 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7168 })?;
7169 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7170 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7171 })?;
7172
7173 let src1 = match loc_a {
7174 Location::SIMD(x) => x,
7175 Location::GPR(_) | Location::Memory(_, _) => {
7176 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7177 tmp1
7178 }
7179 Location::Imm32(_) => {
7180 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7181 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7182 tmp1
7183 }
7184 Location::Imm64(_) => {
7185 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7186 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7187 tmp1
7188 }
7189 _ => {
7190 codegen_error!("singlepass f64_min unreachable");
7191 }
7192 };
7193 let src2 = match loc_b {
7194 Location::SIMD(x) => x,
7195 Location::GPR(_) | Location::Memory(_, _) => {
7196 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7197 tmp2
7198 }
7199 Location::Imm32(_) => {
7200 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7201 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7202 tmp2
7203 }
7204 Location::Imm64(_) => {
7205 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7206 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7207 tmp2
7208 }
7209 _ => {
7210 codegen_error!("singlepass f64_min unreachable");
7211 }
7212 };
7213
7214 let tmp_xmm1 = XMM::XMM8;
7215 let tmp_xmm2 = XMM::XMM9;
7216 let tmp_xmm3 = XMM::XMM10;
7217
7218 self.move_location(Size::S64, Location::SIMD(src1), Location::GPR(tmpg1))?;
7219 self.move_location(Size::S64, Location::SIMD(src2), Location::GPR(tmpg2))?;
7220 self.emit_relaxed_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7221 self.assembler
7222 .emit_vminsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7223 let label1 = self.assembler.get_label();
7224 let label2 = self.assembler.get_label();
7225 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7226 self.assembler
7227 .emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7228 self.assembler.emit_jmp(Condition::None, label2)?;
7229 self.emit_label(label1)?;
7230 self.move_location(
7232 Size::S64,
7233 Location::Imm64(0x8000_0000_0000_0000), Location::GPR(tmpg1),
7235 )?;
7236 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp_xmm2))?;
7237 self.emit_label(label2)?;
7238 self.assembler
7239 .emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7240 self.assembler
7241 .emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7242 self.assembler
7243 .emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1)?;
7244 self.move_location(
7246 Size::S64,
7247 Location::Imm64(0x7FF8_0000_0000_0000), Location::GPR(tmpg1),
7249 )?;
7250 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7251 self.assembler
7252 .emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7253 match ret {
7254 Location::SIMD(x) => {
7255 self.assembler
7256 .emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7257 }
7258 Location::Memory(_, _) | Location::GPR(_) => {
7259 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7260 }
7261 _ => {
7262 codegen_error!("singlepass f64_min unreachable");
7263 }
7264 }
7265
7266 self.release_gpr(tmpg2);
7267 self.release_gpr(tmpg1);
7268 self.release_simd(tmp2);
7269 self.release_simd(tmp1);
7270 Ok(())
7271 }
7272
7273 fn f64_max(
7274 &mut self,
7275 loc_a: Location,
7276 loc_b: Location,
7277 ret: Location,
7278 ) -> Result<(), CompileError> {
7279 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7281 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7282 })?;
7283 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7284 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7285 })?;
7286 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7287 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7288 })?;
7289 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7290 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7291 })?;
7292
7293 let src1 = match loc_a {
7294 Location::SIMD(x) => x,
7295 Location::GPR(_) | Location::Memory(_, _) => {
7296 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7297 tmp1
7298 }
7299 Location::Imm32(_) => {
7300 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7301 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7302 tmp1
7303 }
7304 Location::Imm64(_) => {
7305 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7306 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7307 tmp1
7308 }
7309 _ => {
7310 codegen_error!("singlepass f64_max unreachable");
7311 }
7312 };
7313 let src2 = match loc_b {
7314 Location::SIMD(x) => x,
7315 Location::GPR(_) | Location::Memory(_, _) => {
7316 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7317 tmp2
7318 }
7319 Location::Imm32(_) => {
7320 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7321 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7322 tmp2
7323 }
7324 Location::Imm64(_) => {
7325 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7326 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7327 tmp2
7328 }
7329 _ => {
7330 codegen_error!("singlepass f64_max unreachable");
7331 }
7332 };
7333
7334 let tmp_xmm1 = XMM::XMM8;
7335 let tmp_xmm2 = XMM::XMM9;
7336 let tmp_xmm3 = XMM::XMM10;
7337
7338 self.move_location(Size::S64, Location::SIMD(src1), Location::GPR(tmpg1))?;
7339 self.move_location(Size::S64, Location::SIMD(src2), Location::GPR(tmpg2))?;
7340 self.emit_relaxed_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7341 self.assembler
7342 .emit_vmaxsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7343 let label1 = self.assembler.get_label();
7344 let label2 = self.assembler.get_label();
7345 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7346 self.assembler
7347 .emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7348 self.assembler.emit_jmp(Condition::None, label2)?;
7349 self.emit_label(label1)?;
7350 self.assembler
7351 .emit_vxorpd(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2)?;
7352 self.emit_label(label2)?;
7353 self.assembler
7354 .emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7355 self.assembler
7356 .emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7357 self.assembler
7358 .emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1)?;
7359 self.move_location(
7361 Size::S64,
7362 Location::Imm64(0x7FF8_0000_0000_0000), Location::GPR(tmpg1),
7364 )?;
7365 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7366 self.assembler
7367 .emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7368 match ret {
7369 Location::SIMD(x) => {
7370 self.assembler
7371 .emit_vmovapd(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7372 }
7373 Location::Memory(_, _) | Location::GPR(_) => {
7374 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7375 }
7376 _ => {
7377 codegen_error!("singlepass f64_max unreachable");
7378 }
7379 }
7380
7381 self.release_gpr(tmpg2);
7382 self.release_gpr(tmpg1);
7383 self.release_simd(tmp2);
7384 self.release_simd(tmp1);
7385 Ok(())
7386 }
7387
7388 fn f64_add(
7389 &mut self,
7390 loc_a: Location,
7391 loc_b: Location,
7392 ret: Location,
7393 ) -> Result<(), CompileError> {
7394 self.emit_relaxed_avx(AssemblerX64::emit_vaddsd, loc_a, loc_b, ret)
7395 }
7396
7397 fn f64_sub(
7398 &mut self,
7399 loc_a: Location,
7400 loc_b: Location,
7401 ret: Location,
7402 ) -> Result<(), CompileError> {
7403 self.emit_relaxed_avx(AssemblerX64::emit_vsubsd, loc_a, loc_b, ret)
7404 }
7405
7406 fn f64_mul(
7407 &mut self,
7408 loc_a: Location,
7409 loc_b: Location,
7410 ret: Location,
7411 ) -> Result<(), CompileError> {
7412 self.emit_relaxed_avx(AssemblerX64::emit_vmulsd, loc_a, loc_b, ret)
7413 }
7414
7415 fn f64_div(
7416 &mut self,
7417 loc_a: Location,
7418 loc_b: Location,
7419 ret: Location,
7420 ) -> Result<(), CompileError> {
7421 self.emit_relaxed_avx(AssemblerX64::emit_vdivsd, loc_a, loc_b, ret)
7422 }
7423
7424 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7425 if self.assembler.arch_has_fneg() {
7426 let tmp = self.acquire_temp_simd().ok_or_else(|| {
7427 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7428 })?;
7429 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp))?;
7430 self.assembler.arch_emit_f32_neg(tmp, tmp)?;
7431 self.emit_relaxed_mov(Size::S32, Location::SIMD(tmp), ret)?;
7432 self.release_simd(tmp);
7433 } else {
7434 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7435 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7436 })?;
7437 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
7438 self.assembler.emit_btc_gpr_imm8_32(31, tmp)?;
7439 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
7440 self.release_gpr(tmp);
7441 }
7442 Ok(())
7443 }
7444
7445 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7446 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7447 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7448 })?;
7449 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
7450 self.assembler.emit_and(
7451 Size::S32,
7452 Location::Imm32(0x7fffffffu32),
7453 Location::GPR(tmp),
7454 )?;
7455 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
7456 self.release_gpr(tmp);
7457 Ok(())
7458 }
7459
7460 fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7461 self.assembler.emit_and(
7462 Size::S32,
7463 Location::Imm32(0x7fffffffu32),
7464 Location::GPR(tmp1),
7465 )?;
7466 self.assembler.emit_and(
7467 Size::S32,
7468 Location::Imm32(0x80000000u32),
7469 Location::GPR(tmp2),
7470 )?;
7471 self.assembler
7472 .emit_or(Size::S32, Location::GPR(tmp2), Location::GPR(tmp1))
7473 }
7474
7475 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7476 self.emit_relaxed_avx(AssemblerX64::emit_vsqrtss, loc, loc, ret)
7477 }
7478
7479 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7480 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_trunc, loc, loc, ret)
7481 }
7482
7483 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7484 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_ceil, loc, loc, ret)
7485 }
7486
7487 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7488 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_floor, loc, loc, ret)
7489 }
7490
7491 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7492 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_nearest, loc, loc, ret)
7493 }
7494
7495 fn f32_cmp_ge(
7496 &mut self,
7497 loc_a: Location,
7498 loc_b: Location,
7499 ret: Location,
7500 ) -> Result<(), CompileError> {
7501 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgess, loc_a, loc_b, ret)?;
7502 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7503 }
7504
7505 fn f32_cmp_gt(
7506 &mut self,
7507 loc_a: Location,
7508 loc_b: Location,
7509 ret: Location,
7510 ) -> Result<(), CompileError> {
7511 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgtss, loc_a, loc_b, ret)?;
7512 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7513 }
7514
7515 fn f32_cmp_le(
7516 &mut self,
7517 loc_a: Location,
7518 loc_b: Location,
7519 ret: Location,
7520 ) -> Result<(), CompileError> {
7521 self.emit_relaxed_avx(AssemblerX64::emit_vcmpless, loc_a, loc_b, ret)?;
7522 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7523 }
7524
7525 fn f32_cmp_lt(
7526 &mut self,
7527 loc_a: Location,
7528 loc_b: Location,
7529 ret: Location,
7530 ) -> Result<(), CompileError> {
7531 self.emit_relaxed_avx(AssemblerX64::emit_vcmpltss, loc_a, loc_b, ret)?;
7532 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7533 }
7534
7535 fn f32_cmp_ne(
7536 &mut self,
7537 loc_a: Location,
7538 loc_b: Location,
7539 ret: Location,
7540 ) -> Result<(), CompileError> {
7541 self.emit_relaxed_avx(AssemblerX64::emit_vcmpneqss, loc_a, loc_b, ret)?;
7542 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7543 }
7544
7545 fn f32_cmp_eq(
7546 &mut self,
7547 loc_a: Location,
7548 loc_b: Location,
7549 ret: Location,
7550 ) -> Result<(), CompileError> {
7551 self.emit_relaxed_avx(AssemblerX64::emit_vcmpeqss, loc_a, loc_b, ret)?;
7552 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7553 }
7554
7555 fn f32_min(
7556 &mut self,
7557 loc_a: Location,
7558 loc_b: Location,
7559 ret: Location,
7560 ) -> Result<(), CompileError> {
7561 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7563 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7564 })?;
7565 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7566 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7567 })?;
7568 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7569 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7570 })?;
7571 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7572 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7573 })?;
7574
7575 let src1 = match loc_a {
7576 Location::SIMD(x) => x,
7577 Location::GPR(_) | Location::Memory(_, _) => {
7578 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7579 tmp1
7580 }
7581 Location::Imm32(_) => {
7582 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7583 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7584 tmp1
7585 }
7586 Location::Imm64(_) => {
7587 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7588 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7589 tmp1
7590 }
7591 _ => {
7592 codegen_error!("singlepass f32_min unreachable");
7593 }
7594 };
7595 let src2 = match loc_b {
7596 Location::SIMD(x) => x,
7597 Location::GPR(_) | Location::Memory(_, _) => {
7598 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7599 tmp2
7600 }
7601 Location::Imm32(_) => {
7602 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7603 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7604 tmp2
7605 }
7606 Location::Imm64(_) => {
7607 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7608 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7609 tmp2
7610 }
7611 _ => {
7612 codegen_error!("singlepass f32_min unreachable");
7613 }
7614 };
7615
7616 let tmp_xmm1 = XMM::XMM8;
7617 let tmp_xmm2 = XMM::XMM9;
7618 let tmp_xmm3 = XMM::XMM10;
7619
7620 self.move_location(Size::S32, Location::SIMD(src1), Location::GPR(tmpg1))?;
7621 self.move_location(Size::S32, Location::SIMD(src2), Location::GPR(tmpg2))?;
7622 self.emit_relaxed_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7623 self.assembler
7624 .emit_vminss(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7625 let label1 = self.assembler.get_label();
7626 let label2 = self.assembler.get_label();
7627 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7628 self.assembler
7629 .emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7630 self.assembler.emit_jmp(Condition::None, label2)?;
7631 self.emit_label(label1)?;
7632 self.move_location(
7634 Size::S64,
7635 Location::Imm32(0x8000_0000), Location::GPR(tmpg1),
7637 )?;
7638 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp_xmm2))?;
7639 self.emit_label(label2)?;
7640 self.assembler
7641 .emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7642 self.assembler
7643 .emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7644 self.assembler
7645 .emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1)?;
7646 self.move_location(
7648 Size::S64,
7649 Location::Imm32(0x7FC0_0000), Location::GPR(tmpg1),
7651 )?;
7652 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7653 self.assembler
7654 .emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7655 match ret {
7656 Location::SIMD(x) => {
7657 self.assembler
7658 .emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7659 }
7660 Location::Memory(_, _) | Location::GPR(_) => {
7661 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7662 }
7663 _ => {
7664 codegen_error!("singlepass f32_min unreachable");
7665 }
7666 }
7667
7668 self.release_gpr(tmpg2);
7669 self.release_gpr(tmpg1);
7670 self.release_simd(tmp2);
7671 self.release_simd(tmp1);
7672 Ok(())
7673 }
7674
7675 fn f32_max(
7676 &mut self,
7677 loc_a: Location,
7678 loc_b: Location,
7679 ret: Location,
7680 ) -> Result<(), CompileError> {
7681 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7683 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7684 })?;
7685 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7686 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7687 })?;
7688 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7689 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7690 })?;
7691 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7692 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7693 })?;
7694
7695 let src1 = match loc_a {
7696 Location::SIMD(x) => x,
7697 Location::GPR(_) | Location::Memory(_, _) => {
7698 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7699 tmp1
7700 }
7701 Location::Imm32(_) => {
7702 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7703 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7704 tmp1
7705 }
7706 Location::Imm64(_) => {
7707 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7708 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7709 tmp1
7710 }
7711 _ => {
7712 codegen_error!("singlepass f32_max unreachable");
7713 }
7714 };
7715 let src2 = match loc_b {
7716 Location::SIMD(x) => x,
7717 Location::GPR(_) | Location::Memory(_, _) => {
7718 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7719 tmp2
7720 }
7721 Location::Imm32(_) => {
7722 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7723 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7724 tmp2
7725 }
7726 Location::Imm64(_) => {
7727 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7728 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7729 tmp2
7730 }
7731 _ => {
7732 codegen_error!("singlepass f32_max unreachable");
7733 }
7734 };
7735
7736 let tmp_xmm1 = XMM::XMM8;
7737 let tmp_xmm2 = XMM::XMM9;
7738 let tmp_xmm3 = XMM::XMM10;
7739
7740 self.move_location(Size::S32, Location::SIMD(src1), Location::GPR(tmpg1))?;
7741 self.move_location(Size::S32, Location::SIMD(src2), Location::GPR(tmpg2))?;
7742 self.emit_relaxed_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7743 self.assembler
7744 .emit_vmaxss(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7745 let label1 = self.assembler.get_label();
7746 let label2 = self.assembler.get_label();
7747 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7748 self.assembler
7749 .emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7750 self.assembler.emit_jmp(Condition::None, label2)?;
7751 self.emit_label(label1)?;
7752 self.assembler
7753 .emit_vxorps(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2)?;
7754 self.emit_label(label2)?;
7755 self.assembler
7756 .emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7757 self.assembler
7758 .emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7759 self.assembler
7760 .emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1)?;
7761 self.move_location(
7763 Size::S64,
7764 Location::Imm32(0x7FC0_0000), Location::GPR(tmpg1),
7766 )?;
7767 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7768 self.assembler
7769 .emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7770 match ret {
7771 Location::SIMD(x) => {
7772 self.assembler
7773 .emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7774 }
7775 Location::Memory(_, _) | Location::GPR(_) => {
7776 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7777 }
7778 _ => {
7779 codegen_error!("singlepass f32_max unreachable");
7780 }
7781 }
7782
7783 self.release_gpr(tmpg2);
7784 self.release_gpr(tmpg1);
7785 self.release_simd(tmp2);
7786 self.release_simd(tmp1);
7787 Ok(())
7788 }
7789
7790 fn f32_add(
7791 &mut self,
7792 loc_a: Location,
7793 loc_b: Location,
7794 ret: Location,
7795 ) -> Result<(), CompileError> {
7796 self.emit_relaxed_avx(AssemblerX64::emit_vaddss, loc_a, loc_b, ret)
7797 }
7798
7799 fn f32_sub(
7800 &mut self,
7801 loc_a: Location,
7802 loc_b: Location,
7803 ret: Location,
7804 ) -> Result<(), CompileError> {
7805 self.emit_relaxed_avx(AssemblerX64::emit_vsubss, loc_a, loc_b, ret)
7806 }
7807
7808 fn f32_mul(
7809 &mut self,
7810 loc_a: Location,
7811 loc_b: Location,
7812 ret: Location,
7813 ) -> Result<(), CompileError> {
7814 self.emit_relaxed_avx(AssemblerX64::emit_vmulss, loc_a, loc_b, ret)
7815 }
7816
7817 fn f32_div(
7818 &mut self,
7819 loc_a: Location,
7820 loc_b: Location,
7821 ret: Location,
7822 ) -> Result<(), CompileError> {
7823 self.emit_relaxed_avx(AssemblerX64::emit_vdivss, loc_a, loc_b, ret)
7824 }
7825
7826 fn gen_std_trampoline(
7827 &self,
7828 sig: &FunctionType,
7829 _calling_convention: CallingConvention,
7830 progress_callback: Option<&CompilationProgressCallback>,
7831 ) -> Result<FunctionBody, CompileError> {
7832 let mut a = AssemblerX64::new(0, None)?;
7834 let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
7835
7836 let stack_params = (0..sig.params().len() + 1)
7838 .filter(|&i| self.get_param_registers().get(i).is_none())
7839 .count();
7840 let stack_return_slots = sig
7841 .results()
7842 .len()
7843 .saturating_sub(X86_64_RETURN_VALUE_REGISTERS.len());
7844
7845 let mut stack_offset = 8 * (stack_params + stack_return_slots) as u32;
7847
7848 if stack_offset % 16 != 8 {
7850 stack_offset += 8;
7851 }
7852
7853 a.emit_push(Size::S64, Location::GPR(GPR::R15))?;
7855 a.emit_push(Size::S64, Location::GPR(GPR::R14))?;
7856
7857 a.emit_sub(
7859 Size::S64,
7860 Location::Imm32(stack_offset),
7861 Location::GPR(GPR::RSP),
7862 )?;
7863
7864 a.emit_mov(
7866 Size::S64,
7867 Location::GPR(self.get_simple_param_location(1)),
7868 Location::GPR(GPR::R15),
7869 )?; a.emit_mov(
7871 Size::S64,
7872 Location::GPR(self.get_simple_param_location(2)),
7873 Location::GPR(GPR::R14),
7874 )?; {
7879 let mut n_stack_args = 0u32;
7880 for (i, _param) in sig.params().iter().enumerate() {
7881 let src_loc = Location::Memory(GPR::R14, (i * 16) as _); let dst_loc = self.get_param_registers().get(1 + i);
7883
7884 match dst_loc {
7885 Some(&gpr) => {
7886 a.emit_mov(Size::S64, src_loc, Location::GPR(gpr))?;
7887 }
7888 None => {
7889 a.emit_mov(Size::S64, src_loc, Location::GPR(GPR::RAX))?;
7892 a.emit_mov(
7893 Size::S64,
7894 Location::GPR(GPR::RAX),
7895 Location::Memory(
7896 GPR::RSP,
7897 ((n_stack_args + stack_return_slots as u32) * 8) as _,
7898 ),
7899 )?;
7900 n_stack_args += 1;
7901 }
7902 }
7903 output_reporter.check(a.get_offset().0)?;
7904 }
7905 }
7906
7907 a.emit_call_location(Location::GPR(GPR::R15))?;
7909
7910 let mut n_stack_return_slots: usize = 0;
7912 for i in 0..sig.results().len() {
7913 let src = if let Some(®) = X86_64_RETURN_VALUE_REGISTERS.get(i) {
7914 Location::GPR(reg)
7915 } else {
7916 let loc = Location::GPR(GPR::R15);
7917 a.emit_mov(
7918 Size::S64,
7919 Location::Memory(GPR::RSP, (n_stack_return_slots as u32 * 8) as _),
7920 loc,
7921 )?;
7922 n_stack_return_slots += 1;
7923 loc
7924 };
7925 a.emit_mov(Size::S64, src, Location::Memory(GPR::R14, (i * 16) as _))?;
7926 output_reporter.check(a.get_offset().0)?;
7927 }
7928
7929 a.emit_add(
7931 Size::S64,
7932 Location::Imm32(stack_offset),
7933 Location::GPR(GPR::RSP),
7934 )?;
7935
7936 a.emit_pop(Size::S64, Location::GPR(GPR::R14))?;
7938 a.emit_pop(Size::S64, Location::GPR(GPR::R15))?;
7939
7940 a.emit_ret()?;
7941
7942 let mut body = a.finalize().unwrap();
7943 body.shrink_to_fit();
7944 output_reporter.finish(body.len())?;
7945
7946 Ok(FunctionBody {
7947 body,
7948 unwind_info: None,
7949 })
7950 }
7951 fn gen_std_dynamic_import_trampoline(
7954 &self,
7955 vmoffsets: &VMOffsets,
7956 sig: &FunctionType,
7957 calling_convention: CallingConvention,
7958 progress_callback: Option<&CompilationProgressCallback>,
7959 ) -> Result<FunctionBody, CompileError> {
7960 let mut a = AssemblerX64::new(0, None)?;
7962 let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
7963
7964 let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len()) + 8; a.emit_sub(
7967 Size::S64,
7968 Location::Imm32(stack_offset as _),
7969 Location::GPR(GPR::RSP),
7970 )?;
7971
7972 if !sig.params().is_empty() {
7974 let mut argalloc = ArgumentRegisterAllocator::default();
7975 argalloc.next(Type::I64, calling_convention).unwrap(); let mut stack_param_count: usize = 0;
7978
7979 for (i, ty) in sig.params().iter().enumerate() {
7980 let source_loc = match argalloc.next(*ty, calling_convention)? {
7981 Some(X64Register::GPR(gpr)) => Location::GPR(gpr),
7982 Some(X64Register::XMM(xmm)) => Location::SIMD(xmm),
7983 None => {
7984 a.emit_mov(
7985 Size::S64,
7986 Location::Memory(
7987 GPR::RSP,
7988 (stack_offset + 8 + stack_param_count * 8) as _,
7989 ),
7990 Location::GPR(GPR::RAX),
7991 )?;
7992 stack_param_count += 1;
7993 Location::GPR(GPR::RAX)
7994 }
7995 };
7996 a.emit_mov(
7997 Size::S64,
7998 source_loc,
7999 Location::Memory(GPR::RSP, (i * 16) as _),
8000 )?;
8001
8002 a.emit_mov(
8004 Size::S64,
8005 Location::Imm32(0),
8006 Location::Memory(GPR::RSP, (i * 16 + 8) as _),
8007 )?;
8008 output_reporter.check(a.get_offset().0)?;
8009 }
8010 }
8011
8012 a.emit_mov(
8014 Size::S64,
8015 Location::Memory(
8016 GPR::RDI,
8017 vmoffsets.vmdynamicfunction_import_context_address() as i32,
8018 ),
8019 Location::GPR(GPR::RAX),
8020 )?;
8021 a.emit_mov(Size::S64, Location::GPR(GPR::RSP), Location::GPR(GPR::RSI))?;
8023
8024 a.emit_call_location(Location::GPR(GPR::RAX))?;
8026
8027 if !sig.results().is_empty() {
8029 assert_eq!(sig.results().len(), 1);
8030 a.emit_mov(
8031 Size::S64,
8032 Location::Memory(GPR::RSP, 0),
8033 Location::GPR(GPR::RAX),
8034 )?;
8035 }
8036
8037 a.emit_add(
8039 Size::S64,
8040 Location::Imm32((stack_offset) as _),
8041 Location::GPR(GPR::RSP),
8042 )?;
8043
8044 a.emit_ret()?;
8046
8047 let mut body = a.finalize().unwrap();
8048 body.shrink_to_fit();
8049 output_reporter.finish(body.len())?;
8050 Ok(FunctionBody {
8051 body,
8052 unwind_info: None,
8053 })
8054 }
8055 fn gen_import_call_trampoline(
8058 &self,
8059 vmoffsets: &VMOffsets,
8060 index: FunctionIndex,
8061 sig: &FunctionType,
8062 calling_convention: CallingConvention,
8063 progress_callback: Option<&CompilationProgressCallback>,
8064 ) -> Result<CustomSection, CompileError> {
8065 let mut a = AssemblerX64::new(0, None)?;
8067 let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
8068
8069 if sig
8079 .params()
8080 .iter()
8081 .any(|&x| x == Type::F32 || x == Type::F64)
8082 {
8083 let mut param_locations = vec![];
8084
8085 let stack_offset: i32 = if sig.params().len() > 5 {
8087 5 * 8
8088 } else {
8089 (sig.params().len() as i32) * 8
8090 };
8091 if stack_offset > 0 {
8092 a.emit_sub(
8093 Size::S64,
8094 Location::Imm32(stack_offset as u32),
8095 Location::GPR(GPR::RSP),
8096 )?;
8097 }
8098
8099 static PARAM_REGS: &[GPR] = &[GPR::RSI, GPR::RDX, GPR::RCX, GPR::R8, GPR::R9];
8101 #[allow(clippy::needless_range_loop)]
8102 for i in 0..sig.params().len() {
8103 let loc = match i {
8104 0..=4 => {
8105 let loc = Location::Memory(GPR::RSP, (i * 8) as i32);
8106 a.emit_mov(Size::S64, Location::GPR(PARAM_REGS[i]), loc)?;
8107 loc
8108 }
8109 _ => Location::Memory(GPR::RSP, stack_offset + 8 + ((i - 5) * 8) as i32),
8110 };
8111 param_locations.push(loc);
8112 output_reporter.check(a.get_offset().0)?;
8113 }
8114
8115 let mut argalloc = ArgumentRegisterAllocator::default();
8117 argalloc.next(Type::I64, calling_convention)?.unwrap(); let mut caller_stack_offset: i32 = 0;
8119 for (i, ty) in sig.params().iter().enumerate() {
8120 let prev_loc = param_locations[i];
8121 let targ = match argalloc.next(*ty, calling_convention)? {
8122 Some(X64Register::GPR(gpr)) => Location::GPR(gpr),
8123 Some(X64Register::XMM(xmm)) => Location::SIMD(xmm),
8124 None => {
8125 a.emit_mov(Size::S64, prev_loc, Location::GPR(GPR::RAX))?;
8131 a.emit_mov(
8132 Size::S64,
8133 Location::GPR(GPR::RAX),
8134 Location::Memory(GPR::RSP, stack_offset + 8 + caller_stack_offset),
8135 )?;
8136 caller_stack_offset += 8;
8137 output_reporter.check(a.get_offset().0)?;
8138 continue;
8139 }
8140 };
8141 a.emit_mov(Size::S64, prev_loc, targ)?;
8142 output_reporter.check(a.get_offset().0)?;
8143 }
8144
8145 if stack_offset > 0 {
8147 a.emit_add(
8148 Size::S64,
8149 Location::Imm32(stack_offset as u32),
8150 Location::GPR(GPR::RSP),
8151 )?;
8152 }
8153 }
8154
8155 let offset = vmoffsets.vmctx_vmfunction_import(index);
8159 let ptr_arg_offset = vmctx_offset(offset)?;
8160 let vmctx_arg_offset = vmctx_offset(offset.saturating_add(8))?;
8161
8162 a.emit_mov(
8163 Size::S64,
8164 Location::Memory(GPR::RDI, ptr_arg_offset), Location::GPR(GPR::RAX),
8166 )?;
8167 a.emit_mov(
8168 Size::S64,
8169 Location::Memory(GPR::RDI, vmctx_arg_offset), Location::GPR(GPR::RDI),
8171 )?;
8172 a.emit_host_redirection(GPR::RAX)?;
8173
8174 let mut contents = a.finalize().unwrap();
8175 contents.shrink_to_fit();
8176 output_reporter.finish(contents.len())?;
8177 let section_body = SectionBody::new_with_vec(contents);
8178
8179 Ok(CustomSection {
8180 protection: CustomSectionProtection::ReadExecute,
8181 alignment: None,
8182 bytes: section_body,
8183 relocations: vec![],
8184 })
8185 }
8186
8187 #[cfg(feature = "unwind")]
8188 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8189 let mut instructions = vec![];
8190 for &(instruction_offset, ref inst) in &self.unwind_ops {
8191 let instruction_offset = instruction_offset as u32;
8192 match *inst {
8193 UnwindOps::PushFP { up_to_sp } => {
8194 instructions.push((
8195 instruction_offset,
8196 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8197 ));
8198 instructions.push((
8199 instruction_offset,
8200 CallFrameInstruction::Offset(X86_64::RBP, -(up_to_sp as i32)),
8201 ));
8202 }
8203 UnwindOps::DefineNewFrame => {
8204 instructions.push((
8205 instruction_offset,
8206 CallFrameInstruction::CfaRegister(X86_64::RBP),
8207 ));
8208 }
8209 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8210 instruction_offset,
8211 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8212 )),
8213 UnwindOps::Push2Regs { .. } | UnwindOps::SubtractFP { .. } => unimplemented!(),
8214 }
8215 }
8216 Some(UnwindInstructions {
8217 instructions,
8218 len: code_len as u32,
8219 })
8220 }
8221 #[cfg(not(feature = "unwind"))]
8222
8223 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8224 None
8225 }
8226
8227 #[cfg(feature = "unwind")]
8228 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8229 let unwind_info = create_unwind_info_from_insts(&self.unwind_ops);
8230 if let Some(unwind) = unwind_info {
8231 let sz = unwind.emit_size();
8232 let mut tbl = vec![0; sz];
8233 unwind.emit(&mut tbl);
8234 Some(tbl)
8235 } else {
8236 None
8237 }
8238 }
8239
8240 #[cfg(not(feature = "unwind"))]
8241
8242 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8243 None
8244 }
8245}
8246
8247#[cfg(test)]
8248mod test {
8249 use super::*;
8250 use enumset::enum_set;
8251 use std::str::FromStr;
8252 use wasmer_types::target::{CpuFeature, Target, Triple};
8253
8254 fn test_move_location(machine: &mut MachineX86_64) -> Result<(), CompileError> {
8255 machine.move_location_for_native(
8256 Size::S64,
8257 Location::GPR(GPR::RAX),
8258 Location::GPR(GPR::RCX),
8259 )?;
8260 machine.move_location_for_native(
8261 Size::S64,
8262 Location::GPR(GPR::RAX),
8263 Location::Memory(GPR::RDX, 10),
8264 )?;
8265 machine.move_location_for_native(
8266 Size::S64,
8267 Location::GPR(GPR::RAX),
8268 Location::Memory(GPR::RDX, -10),
8269 )?;
8270 machine.move_location_for_native(
8271 Size::S64,
8272 Location::Memory(GPR::RDX, 10),
8273 Location::GPR(GPR::RAX),
8274 )?;
8275 machine.move_location_for_native(
8276 Size::S64,
8277 Location::Imm64(50),
8278 Location::GPR(GPR::RAX),
8279 )?;
8280 machine.move_location_for_native(
8281 Size::S64,
8282 Location::Imm32(50),
8283 Location::GPR(GPR::RAX),
8284 )?;
8285 machine.move_location_for_native(Size::S64, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8286
8287 machine.move_location_for_native(
8288 Size::S32,
8289 Location::GPR(GPR::RAX),
8290 Location::GPR(GPR::RCX),
8291 )?;
8292 machine.move_location_for_native(
8293 Size::S32,
8294 Location::GPR(GPR::RAX),
8295 Location::Memory(GPR::RDX, 10),
8296 )?;
8297 machine.move_location_for_native(
8298 Size::S32,
8299 Location::GPR(GPR::RAX),
8300 Location::Memory(GPR::RDX, -10),
8301 )?;
8302 machine.move_location_for_native(
8303 Size::S32,
8304 Location::Memory(GPR::RDX, 10),
8305 Location::GPR(GPR::RAX),
8306 )?;
8307 machine.move_location_for_native(
8308 Size::S32,
8309 Location::Imm32(50),
8310 Location::GPR(GPR::RAX),
8311 )?;
8312 machine.move_location_for_native(Size::S32, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8313
8314 machine.move_location_for_native(
8315 Size::S16,
8316 Location::GPR(GPR::RAX),
8317 Location::GPR(GPR::RCX),
8318 )?;
8319 machine.move_location_for_native(
8320 Size::S16,
8321 Location::GPR(GPR::RAX),
8322 Location::Memory(GPR::RDX, 10),
8323 )?;
8324 machine.move_location_for_native(
8325 Size::S16,
8326 Location::GPR(GPR::RAX),
8327 Location::Memory(GPR::RDX, -10),
8328 )?;
8329 machine.move_location_for_native(
8330 Size::S16,
8331 Location::Memory(GPR::RDX, 10),
8332 Location::GPR(GPR::RAX),
8333 )?;
8334 machine.move_location_for_native(Size::S16, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8335
8336 machine.move_location_for_native(
8337 Size::S8,
8338 Location::GPR(GPR::RAX),
8339 Location::GPR(GPR::RCX),
8340 )?;
8341 machine.move_location_for_native(
8342 Size::S8,
8343 Location::GPR(GPR::RAX),
8344 Location::Memory(GPR::RDX, 10),
8345 )?;
8346 machine.move_location_for_native(
8347 Size::S8,
8348 Location::GPR(GPR::RAX),
8349 Location::Memory(GPR::RDX, -10),
8350 )?;
8351 machine.move_location_for_native(
8352 Size::S8,
8353 Location::Memory(GPR::RDX, 10),
8354 Location::GPR(GPR::RAX),
8355 )?;
8356 machine.move_location_for_native(Size::S8, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8357
8358 machine.move_location_for_native(
8359 Size::S64,
8360 Location::SIMD(XMM::XMM0),
8361 Location::GPR(GPR::RAX),
8362 )?;
8363 machine.move_location_for_native(
8364 Size::S64,
8365 Location::SIMD(XMM::XMM0),
8366 Location::Memory(GPR::RDX, -10),
8367 )?;
8368 machine.move_location_for_native(
8369 Size::S64,
8370 Location::GPR(GPR::RAX),
8371 Location::SIMD(XMM::XMM0),
8372 )?;
8373 machine.move_location_for_native(
8374 Size::S64,
8375 Location::Memory(GPR::RDX, -10),
8376 Location::SIMD(XMM::XMM0),
8377 )?;
8378
8379 Ok(())
8380 }
8381
8382 fn test_move_location_extended(
8383 machine: &mut MachineX86_64,
8384 signed: bool,
8385 sized: Size,
8386 ) -> Result<(), CompileError> {
8387 machine.move_location_extend(
8388 sized,
8389 signed,
8390 Location::GPR(GPR::RAX),
8391 Size::S64,
8392 Location::GPR(GPR::RCX),
8393 )?;
8394 machine.move_location_extend(
8395 sized,
8396 signed,
8397 Location::GPR(GPR::RAX),
8398 Size::S64,
8399 Location::Memory(GPR::RCX, 10),
8400 )?;
8401 machine.move_location_extend(
8402 sized,
8403 signed,
8404 Location::Memory(GPR::RAX, 10),
8405 Size::S64,
8406 Location::GPR(GPR::RCX),
8407 )?;
8408 if sized != Size::S32 {
8409 machine.move_location_extend(
8410 sized,
8411 signed,
8412 Location::GPR(GPR::RAX),
8413 Size::S32,
8414 Location::GPR(GPR::RCX),
8415 )?;
8416 machine.move_location_extend(
8417 sized,
8418 signed,
8419 Location::GPR(GPR::RAX),
8420 Size::S32,
8421 Location::Memory(GPR::RCX, 10),
8422 )?;
8423 machine.move_location_extend(
8424 sized,
8425 signed,
8426 Location::Memory(GPR::RAX, 10),
8427 Size::S32,
8428 Location::GPR(GPR::RCX),
8429 )?;
8430 }
8431
8432 Ok(())
8433 }
8434
8435 fn test_binop_op(
8436 machine: &mut MachineX86_64,
8437 op: fn(&mut MachineX86_64, Location, Location, Location) -> Result<(), CompileError>,
8438 ) -> Result<(), CompileError> {
8439 op(
8440 machine,
8441 Location::GPR(GPR::RDX),
8442 Location::GPR(GPR::RDX),
8443 Location::GPR(GPR::RAX),
8444 )?;
8445 op(
8446 machine,
8447 Location::GPR(GPR::RDX),
8448 Location::Imm32(10),
8449 Location::GPR(GPR::RAX),
8450 )?;
8451 op(
8452 machine,
8453 Location::GPR(GPR::RAX),
8454 Location::GPR(GPR::RAX),
8455 Location::GPR(GPR::RAX),
8456 )?;
8457 op(
8458 machine,
8459 Location::Imm32(10),
8460 Location::GPR(GPR::RDX),
8461 Location::GPR(GPR::RAX),
8462 )?;
8463 op(
8464 machine,
8465 Location::GPR(GPR::RAX),
8466 Location::GPR(GPR::RDX),
8467 Location::Memory(GPR::RAX, 10),
8468 )?;
8469 op(
8470 machine,
8471 Location::GPR(GPR::RAX),
8472 Location::Memory(GPR::RDX, 16),
8473 Location::Memory(GPR::RAX, 10),
8474 )?;
8475 op(
8476 machine,
8477 Location::Memory(GPR::RAX, 0),
8478 Location::Memory(GPR::RDX, 16),
8479 Location::Memory(GPR::RAX, 10),
8480 )?;
8481
8482 Ok(())
8483 }
8484
8485 fn test_float_binop_op(
8486 machine: &mut MachineX86_64,
8487 op: fn(&mut MachineX86_64, Location, Location, Location) -> Result<(), CompileError>,
8488 ) -> Result<(), CompileError> {
8489 op(
8490 machine,
8491 Location::SIMD(XMM::XMM3),
8492 Location::SIMD(XMM::XMM2),
8493 Location::SIMD(XMM::XMM0),
8494 )?;
8495 op(
8496 machine,
8497 Location::SIMD(XMM::XMM0),
8498 Location::SIMD(XMM::XMM2),
8499 Location::SIMD(XMM::XMM0),
8500 )?;
8501 op(
8502 machine,
8503 Location::SIMD(XMM::XMM0),
8504 Location::SIMD(XMM::XMM0),
8505 Location::SIMD(XMM::XMM0),
8506 )?;
8507 op(
8508 machine,
8509 Location::Memory(GPR::RBP, 0),
8510 Location::SIMD(XMM::XMM2),
8511 Location::SIMD(XMM::XMM0),
8512 )?;
8513 op(
8514 machine,
8515 Location::Memory(GPR::RBP, 0),
8516 Location::Memory(GPR::RDX, 10),
8517 Location::SIMD(XMM::XMM0),
8518 )?;
8519 op(
8520 machine,
8521 Location::Memory(GPR::RBP, 0),
8522 Location::Memory(GPR::RDX, 16),
8523 Location::Memory(GPR::RAX, 32),
8524 )?;
8525 op(
8526 machine,
8527 Location::SIMD(XMM::XMM0),
8528 Location::Memory(GPR::RDX, 16),
8529 Location::Memory(GPR::RAX, 32),
8530 )?;
8531 op(
8532 machine,
8533 Location::SIMD(XMM::XMM0),
8534 Location::SIMD(XMM::XMM1),
8535 Location::Memory(GPR::RAX, 32),
8536 )?;
8537
8538 Ok(())
8539 }
8540
8541 fn test_float_cmp_op(
8542 machine: &mut MachineX86_64,
8543 op: fn(&mut MachineX86_64, Location, Location, Location) -> Result<(), CompileError>,
8544 ) -> Result<(), CompileError> {
8545 op(
8546 machine,
8547 Location::SIMD(XMM::XMM3),
8548 Location::SIMD(XMM::XMM2),
8549 Location::GPR(GPR::RAX),
8550 )?;
8551 op(
8552 machine,
8553 Location::SIMD(XMM::XMM0),
8554 Location::SIMD(XMM::XMM0),
8555 Location::GPR(GPR::RAX),
8556 )?;
8557 op(
8558 machine,
8559 Location::Memory(GPR::RBP, 0),
8560 Location::SIMD(XMM::XMM2),
8561 Location::GPR(GPR::RAX),
8562 )?;
8563 op(
8564 machine,
8565 Location::Memory(GPR::RBP, 0),
8566 Location::Memory(GPR::RDX, 10),
8567 Location::GPR(GPR::RAX),
8568 )?;
8569 op(
8570 machine,
8571 Location::Memory(GPR::RBP, 0),
8572 Location::Memory(GPR::RDX, 16),
8573 Location::Memory(GPR::RAX, 32),
8574 )?;
8575 op(
8576 machine,
8577 Location::SIMD(XMM::XMM0),
8578 Location::Memory(GPR::RDX, 16),
8579 Location::Memory(GPR::RAX, 32),
8580 )?;
8581 op(
8582 machine,
8583 Location::SIMD(XMM::XMM0),
8584 Location::SIMD(XMM::XMM1),
8585 Location::Memory(GPR::RAX, 32),
8586 )?;
8587
8588 Ok(())
8589 }
8590
8591 #[test]
8592 fn tests_avx() -> Result<(), CompileError> {
8593 let set = enum_set!(CpuFeature::AVX);
8594 let target = Target::new(Triple::from_str("x86_64-linux-gnu").unwrap(), set);
8595 let mut machine = MachineX86_64::new(Some(target))?;
8596
8597 test_move_location(&mut machine)?;
8598 test_move_location_extended(&mut machine, false, Size::S8)?;
8599 test_move_location_extended(&mut machine, false, Size::S16)?;
8600 test_move_location_extended(&mut machine, false, Size::S32)?;
8601 test_move_location_extended(&mut machine, true, Size::S8)?;
8602 test_move_location_extended(&mut machine, true, Size::S16)?;
8603 test_move_location_extended(&mut machine, true, Size::S32)?;
8604 test_binop_op(&mut machine, MachineX86_64::emit_binop_add32)?;
8605 test_binop_op(&mut machine, MachineX86_64::emit_binop_add64)?;
8606 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub32)?;
8607 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub64)?;
8608 test_binop_op(&mut machine, MachineX86_64::emit_binop_and32)?;
8609 test_binop_op(&mut machine, MachineX86_64::emit_binop_and64)?;
8610 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor32)?;
8611 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor64)?;
8612 test_binop_op(&mut machine, MachineX86_64::emit_binop_or32)?;
8613 test_binop_op(&mut machine, MachineX86_64::emit_binop_or64)?;
8614 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul32)?;
8615 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul64)?;
8616 test_float_binop_op(&mut machine, MachineX86_64::f32_add)?;
8617 test_float_binop_op(&mut machine, MachineX86_64::f32_sub)?;
8618 test_float_binop_op(&mut machine, MachineX86_64::f32_mul)?;
8619 test_float_binop_op(&mut machine, MachineX86_64::f32_div)?;
8620 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_eq)?;
8621 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_lt)?;
8622 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_le)?;
8623
8624 Ok(())
8625 }
8626
8627 #[test]
8628 fn tests_sse42() -> Result<(), CompileError> {
8629 let set = enum_set!(CpuFeature::SSE42);
8630 let target = Target::new(Triple::from_str("x86_64-linux-gnu").unwrap(), set);
8631 let mut machine = MachineX86_64::new(Some(target))?;
8632
8633 test_move_location(&mut machine)?;
8634 test_move_location_extended(&mut machine, false, Size::S8)?;
8635 test_move_location_extended(&mut machine, false, Size::S16)?;
8636 test_move_location_extended(&mut machine, false, Size::S32)?;
8637 test_move_location_extended(&mut machine, true, Size::S8)?;
8638 test_move_location_extended(&mut machine, true, Size::S16)?;
8639 test_move_location_extended(&mut machine, true, Size::S32)?;
8640 test_binop_op(&mut machine, MachineX86_64::emit_binop_add32)?;
8641 test_binop_op(&mut machine, MachineX86_64::emit_binop_add64)?;
8642 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub32)?;
8643 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub64)?;
8644 test_binop_op(&mut machine, MachineX86_64::emit_binop_and32)?;
8645 test_binop_op(&mut machine, MachineX86_64::emit_binop_and64)?;
8646 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor32)?;
8647 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor64)?;
8648 test_binop_op(&mut machine, MachineX86_64::emit_binop_or32)?;
8649 test_binop_op(&mut machine, MachineX86_64::emit_binop_or64)?;
8650 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul32)?;
8651 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul64)?;
8652 test_float_binop_op(&mut machine, MachineX86_64::f32_add)?;
8653 test_float_binop_op(&mut machine, MachineX86_64::f32_sub)?;
8654 test_float_binop_op(&mut machine, MachineX86_64::f32_mul)?;
8655 test_float_binop_op(&mut machine, MachineX86_64::f32_div)?;
8656 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_eq)?;
8657 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_lt)?;
8658 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_le)?;
8659
8660 Ok(())
8661 }
8662}