wasmer_compiler_singlepass/
emitter_x64.rs

1use crate::{
2    codegen_error, common_decl::Size, location::Location as AbstractLocation,
3    machine_x64::AssemblerX64,
4};
5pub use crate::{
6    location::Multiplier,
7    machine::{Label, Offset},
8    x64_decl::{GPR, XMM},
9};
10use dynasmrt::{AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi};
11use wasmer_types::{CompileError, target::CpuFeature};
12
13/// Force `dynasm!` to use the correct arch (x64) when cross-compiling.
14/// `dynasm!` proc-macro tries to auto-detect it by default by looking at the
15/// `target_arch`, but it sees the `target_arch` of the proc-macro itself, which
16/// is always equal to host, even when cross-compiling.
17macro_rules! dynasm {
18    ($a:expr ; $($tt:tt)*) => {
19        dynasm::dynasm!(
20            $a.inner
21            ; .arch x64
22            ; $($tt)*
23        )
24    };
25}
26
27pub type Location = AbstractLocation<GPR, XMM>;
28
29#[derive(Copy, Clone, Debug, Eq, PartialEq)]
30pub enum Condition {
31    None,
32    Above,
33    AboveEqual,
34    Below,
35    BelowEqual,
36    Greater,
37    GreaterEqual,
38    Less,
39    LessEqual,
40    Equal,
41    NotEqual,
42    Signed,
43    Carry,
44}
45
46#[derive(Copy, Clone, Debug, Eq, PartialEq)]
47#[allow(dead_code, clippy::upper_case_acronyms)]
48pub enum XMMOrMemory {
49    XMM(XMM),
50    Memory(GPR, i32),
51}
52
53#[derive(Copy, Clone, Debug)]
54#[allow(dead_code, clippy::upper_case_acronyms)]
55pub enum GPROrMemory {
56    GPR(GPR),
57    Memory(GPR, i32),
58}
59
60pub enum Precision {
61    Single,
62    Double,
63}
64
65#[allow(unused)]
66pub trait EmitterX64 {
67    fn get_simd_arch(&self) -> Option<&CpuFeature>;
68    fn get_label(&mut self) -> Label;
69    fn get_offset(&self) -> Offset;
70    fn get_jmp_instr_size(&self) -> u8;
71
72    fn finalize_function(&mut self) -> Result<(), CompileError> {
73        Ok(())
74    }
75
76    fn emit_u64(&mut self, x: u64) -> Result<(), CompileError>;
77    fn emit_bytes(&mut self, bytes: &[u8]) -> Result<(), CompileError>;
78
79    fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
80
81    fn emit_nop(&mut self) -> Result<(), CompileError>;
82
83    /// A high-level assembler method. Emits an instruction sequence of length `n` that is functionally
84    /// equivalent to a `nop` instruction, without guarantee about the underlying implementation.
85    fn emit_nop_n(&mut self, n: usize) -> Result<(), CompileError>;
86
87    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
88    fn emit_lea(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
89    fn emit_lea_label(&mut self, label: Label, dst: Location) -> Result<(), CompileError>;
90    fn emit_cdq(&mut self) -> Result<(), CompileError>;
91    fn emit_cqo(&mut self) -> Result<(), CompileError>;
92    fn emit_xor(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
93    fn emit_jmp(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
94    fn emit_jmp_location(&mut self, loc: Location) -> Result<(), CompileError>;
95    fn emit_set(&mut self, condition: Condition, dst: GPR) -> Result<(), CompileError>;
96    fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError>;
97    fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError>;
98    fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
99    fn emit_add(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
100    fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
101    fn emit_neg(&mut self, sz: Size, value: Location) -> Result<(), CompileError>;
102    fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
103    fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR) -> Result<(), CompileError>;
104    fn emit_div(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError>;
105    fn emit_idiv(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError>;
106    fn emit_shl(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
107    fn emit_shr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
108    fn emit_sar(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
109    fn emit_rol(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
110    fn emit_ror(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
111    fn emit_and(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
112    fn emit_test(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
113    fn emit_or(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
114    fn emit_bsr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
115    fn emit_bsf(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
116    fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
117    fn emit_movzx(
118        &mut self,
119        sz_src: Size,
120        src: Location,
121        sz_dst: Size,
122        dst: Location,
123    ) -> Result<(), CompileError>;
124    fn emit_movsx(
125        &mut self,
126        sz_src: Size,
127        src: Location,
128        sz_dst: Size,
129        dst: Location,
130    ) -> Result<(), CompileError>;
131    fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
132    fn emit_lock_xadd(
133        &mut self,
134        sz: Size,
135        src: Location,
136        dst: Location,
137    ) -> Result<(), CompileError>;
138    fn emit_lock_cmpxchg(
139        &mut self,
140        sz: Size,
141        src: Location,
142        dst: Location,
143    ) -> Result<(), CompileError>;
144    fn emit_rep_stosq(&mut self) -> Result<(), CompileError>;
145
146    fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR) -> Result<(), CompileError>;
147    fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR) -> Result<(), CompileError>;
148
149    fn emit_cmovae_gpr_32(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError>;
150    fn emit_cmovae_gpr_64(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError>;
151
152    fn emit_vmovaps(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError>;
153    fn emit_vmovapd(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError>;
154    fn emit_vxorps(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
155    fn emit_vxorpd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
156
157    fn emit_vaddss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
158    fn emit_vaddsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
159    fn emit_vsubss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
160    fn emit_vsubsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
161    fn emit_vmulss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
162    fn emit_vmulsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
163    fn emit_vdivss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
164    fn emit_vdivsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
165    fn emit_vmaxss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
166    fn emit_vmaxsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
167    fn emit_vminss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
168    fn emit_vminsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
169
170    fn emit_vcmpeqss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
171    -> Result<(), CompileError>;
172    fn emit_vcmpeqsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
173    -> Result<(), CompileError>;
174
175    fn emit_vcmpneqss(
176        &mut self,
177        src1: XMM,
178        src2: XMMOrMemory,
179        dst: XMM,
180    ) -> Result<(), CompileError>;
181    fn emit_vcmpneqsd(
182        &mut self,
183        src1: XMM,
184        src2: XMMOrMemory,
185        dst: XMM,
186    ) -> Result<(), CompileError>;
187
188    fn emit_vcmpltss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
189    -> Result<(), CompileError>;
190    fn emit_vcmpltsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
191    -> Result<(), CompileError>;
192
193    fn emit_vcmpless(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
194    -> Result<(), CompileError>;
195    fn emit_vcmplesd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
196    -> Result<(), CompileError>;
197
198    fn emit_vcmpgtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
199    -> Result<(), CompileError>;
200    fn emit_vcmpgtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
201    -> Result<(), CompileError>;
202
203    fn emit_vcmpgess(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
204    -> Result<(), CompileError>;
205    fn emit_vcmpgesd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
206    -> Result<(), CompileError>;
207
208    fn emit_vcmpunordss(
209        &mut self,
210        src1: XMM,
211        src2: XMMOrMemory,
212        dst: XMM,
213    ) -> Result<(), CompileError>;
214    fn emit_vcmpunordsd(
215        &mut self,
216        src1: XMM,
217        src2: XMMOrMemory,
218        dst: XMM,
219    ) -> Result<(), CompileError>;
220
221    fn emit_vcmpordss(
222        &mut self,
223        src1: XMM,
224        src2: XMMOrMemory,
225        dst: XMM,
226    ) -> Result<(), CompileError>;
227    fn emit_vcmpordsd(
228        &mut self,
229        src1: XMM,
230        src2: XMMOrMemory,
231        dst: XMM,
232    ) -> Result<(), CompileError>;
233
234    fn emit_vsqrtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
235    fn emit_vsqrtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
236
237    fn emit_vroundss_nearest(
238        &mut self,
239        src1: XMM,
240        src2: XMMOrMemory,
241        dst: XMM,
242    ) -> Result<(), CompileError>;
243    fn emit_vroundss_floor(
244        &mut self,
245        src1: XMM,
246        src2: XMMOrMemory,
247        dst: XMM,
248    ) -> Result<(), CompileError>;
249    fn emit_vroundss_ceil(
250        &mut self,
251        src1: XMM,
252        src2: XMMOrMemory,
253        dst: XMM,
254    ) -> Result<(), CompileError>;
255    fn emit_vroundss_trunc(
256        &mut self,
257        src1: XMM,
258        src2: XMMOrMemory,
259        dst: XMM,
260    ) -> Result<(), CompileError>;
261    fn emit_vroundsd_nearest(
262        &mut self,
263        src1: XMM,
264        src2: XMMOrMemory,
265        dst: XMM,
266    ) -> Result<(), CompileError>;
267    fn emit_vroundsd_floor(
268        &mut self,
269        src1: XMM,
270        src2: XMMOrMemory,
271        dst: XMM,
272    ) -> Result<(), CompileError>;
273    fn emit_vroundsd_ceil(
274        &mut self,
275        src1: XMM,
276        src2: XMMOrMemory,
277        dst: XMM,
278    ) -> Result<(), CompileError>;
279    fn emit_vroundsd_trunc(
280        &mut self,
281        src1: XMM,
282        src2: XMMOrMemory,
283        dst: XMM,
284    ) -> Result<(), CompileError>;
285
286    fn emit_vcvtss2sd(
287        &mut self,
288        src1: XMM,
289        src2: XMMOrMemory,
290        dst: XMM,
291    ) -> Result<(), CompileError>;
292    fn emit_vcvtsd2ss(
293        &mut self,
294        src1: XMM,
295        src2: XMMOrMemory,
296        dst: XMM,
297    ) -> Result<(), CompileError>;
298
299    fn emit_ucomiss(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
300    fn emit_ucomisd(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
301
302    fn emit_cvttss2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
303    fn emit_cvttss2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
304    fn emit_cvttsd2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
305    fn emit_cvttsd2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
306
307    fn emit_vcvtsi2ss_32(
308        &mut self,
309        src1: XMM,
310        src2: GPROrMemory,
311        dst: XMM,
312    ) -> Result<(), CompileError>;
313    fn emit_vcvtsi2ss_64(
314        &mut self,
315        src1: XMM,
316        src2: GPROrMemory,
317        dst: XMM,
318    ) -> Result<(), CompileError>;
319    fn emit_vcvtsi2sd_32(
320        &mut self,
321        src1: XMM,
322        src2: GPROrMemory,
323        dst: XMM,
324    ) -> Result<(), CompileError>;
325    fn emit_vcvtsi2sd_64(
326        &mut self,
327        src1: XMM,
328        src2: GPROrMemory,
329        dst: XMM,
330    ) -> Result<(), CompileError>;
331
332    fn emit_vblendvps(
333        &mut self,
334        src1: XMM,
335        src2: XMMOrMemory,
336        mask: XMM,
337        dst: XMM,
338    ) -> Result<(), CompileError>;
339    fn emit_vblendvpd(
340        &mut self,
341        src1: XMM,
342        src2: XMMOrMemory,
343        mask: XMM,
344        dst: XMM,
345    ) -> Result<(), CompileError>;
346
347    fn emit_test_gpr_64(&mut self, reg: GPR) -> Result<(), CompileError>;
348
349    fn emit_ud2(&mut self) -> Result<(), CompileError>;
350    fn emit_ud1_payload(&mut self, payload: u8) -> Result<(), CompileError>;
351    fn emit_ret(&mut self) -> Result<(), CompileError>;
352    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
353    fn emit_call_location(&mut self, loc: Location) -> Result<(), CompileError>;
354
355    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
356
357    fn emit_bkpt(&mut self) -> Result<(), CompileError>;
358
359    fn emit_host_redirection(&mut self, target: GPR) -> Result<(), CompileError>;
360
361    fn arch_has_itruncf(&self) -> bool {
362        false
363    }
364    fn arch_emit_i32_trunc_sf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
365        codegen_error!("singplepass arch_emit_i32_trunc_sf32 unimplemented")
366    }
367    fn arch_emit_i32_trunc_sf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
368        codegen_error!("singplepass arch_emit_i32_trunc_sf64 unimplemented")
369    }
370    fn arch_emit_i32_trunc_uf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
371        codegen_error!("singplepass arch_emit_i32_trunc_uf32 unimplemented")
372    }
373    fn arch_emit_i32_trunc_uf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
374        codegen_error!("singplepass arch_emit_i32_trunc_uf64 unimplemented")
375    }
376    fn arch_emit_i64_trunc_sf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
377        codegen_error!("singplepass arch_emit_i64_trunc_sf32 unimplemented")
378    }
379    fn arch_emit_i64_trunc_sf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
380        codegen_error!("singplepass arch_emit_i64_trunc_sf64 unimplemented")
381    }
382    fn arch_emit_i64_trunc_uf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
383        codegen_error!("singplepass arch_emit_i64_trunc_uf32 unimplemented")
384    }
385    fn arch_emit_i64_trunc_uf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
386        codegen_error!("singplepass arch_emit_i64_trunc_uf64 unimplemented")
387    }
388
389    fn arch_has_fconverti(&self) -> bool {
390        false
391    }
392    fn arch_emit_f32_convert_si32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
393        codegen_error!("singlepass arch_emit_f32_convert_si32 unimplemented")
394    }
395    fn arch_emit_f32_convert_si64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
396        codegen_error!("singlepass arch_emit_f32_convert_si64 unimplemented")
397    }
398    fn arch_emit_f32_convert_ui32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
399        codegen_error!("singlepass arch_emit_f32_convert_ui32 unimplemented")
400    }
401    fn arch_emit_f32_convert_ui64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
402        codegen_error!("singlepass arch_emit_f32_convert_ui64 unimplemented")
403    }
404    fn arch_emit_f64_convert_si32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
405        codegen_error!("singlepass arch_emit_f64_convert_si32 unimplemented")
406    }
407    fn arch_emit_f64_convert_si64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
408        codegen_error!("singlepass arch_emit_f64_convert_si64 unimplemented")
409    }
410    fn arch_emit_f64_convert_ui32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
411        codegen_error!("singlepass arch_emit_f64_convert_ui32 unimplemented")
412    }
413    fn arch_emit_f64_convert_ui64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
414        codegen_error!("singlepass arch_emit_f64_convert_ui64 unimplemented")
415    }
416
417    fn arch_has_fneg(&self) -> bool {
418        false
419    }
420    fn arch_emit_f32_neg(&mut self, _src: XMM, _dst: XMM) -> Result<(), CompileError> {
421        codegen_error!("singlepass arch_emit_f32_neg unimplemented")
422    }
423    fn arch_emit_f64_neg(&mut self, _src: XMM, _dst: XMM) -> Result<(), CompileError> {
424        codegen_error!("singlepass arch_emit_f64_neg unimplemented")
425    }
426
427    fn arch_has_xzcnt(&self) -> bool {
428        false
429    }
430    fn arch_emit_lzcnt(
431        &mut self,
432        _sz: Size,
433        _src: Location,
434        _dst: Location,
435    ) -> Result<(), CompileError> {
436        codegen_error!("singlepass arch_emit_lzcnt unimplemented")
437    }
438    fn arch_emit_tzcnt(
439        &mut self,
440        _sz: Size,
441        _src: Location,
442        _dst: Location,
443    ) -> Result<(), CompileError> {
444        codegen_error!("singlepass arch_emit_tzcnt unimplemented")
445    }
446
447    fn arch_emit_indirect_call_with_trampoline(
448        &mut self,
449        _loc: Location,
450    ) -> Result<(), CompileError> {
451        codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
452    }
453
454    // Emits entry trampoline just before the real function.
455    fn arch_emit_entry_trampoline(&mut self) -> Result<(), CompileError> {
456        Ok(())
457    }
458
459    // Byte offset from the beginning of a `mov Imm64, GPR` instruction to the imm64 value.
460    // Required to support emulation on Aarch64.
461    fn arch_mov64_imm_offset(&self) -> Result<usize, CompileError> {
462        codegen_error!("singlepass arch_mov64_imm_offset unimplemented")
463    }
464}
465
466macro_rules! unop_gpr {
467    ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => {
468        match ($sz, $loc) {
469            (Size::S32, Location::GPR(loc)) => {
470                dynasm!($assembler ; $ins Rd(loc));
471            },
472            (Size::S64, Location::GPR(loc)) => {
473                dynasm!($assembler ; $ins Rq(loc));
474            },
475            _ => $otherwise
476        }
477    };
478}
479
480macro_rules! unop_mem {
481    ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => {
482        match ($sz, $loc) {
483            (Size::S32, Location::Memory(loc, disp)) => {
484                dynasm!($assembler ; $ins DWORD [Rq(loc) + disp] );
485            },
486            (Size::S64, Location::Memory(loc, disp)) => {
487                dynasm!($assembler ; $ins QWORD [Rq(loc) + disp] );
488            },
489            _ => $otherwise
490        }
491    };
492}
493
494macro_rules! unop_gpr_or_mem {
495    ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => {
496        unop_gpr!($ins, $assembler, $sz, $loc, {
497            unop_mem!($ins, $assembler, $sz, $loc, $otherwise)
498        })
499    };
500}
501
502macro_rules! binop_imm32_gpr {
503    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
504        match ($sz, $src, $dst) {
505            (Size::S32, Location::Imm32(src), Location::GPR(dst)) => {
506                dynasm!($assembler ; $ins Rd(dst), src as i32); // IMM32_2GPR
507            },
508            (Size::S64, Location::Imm32(src), Location::GPR(dst)) => {
509                dynasm!($assembler ; $ins Rq(dst), src as i32); // IMM32_2GPR
510            },
511            _ => $otherwise
512        }
513    };
514}
515
516macro_rules! binop_imm32_mem {
517    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
518        match ($sz, $src, $dst) {
519            (Size::S32, Location::Imm32(src), Location::Memory(dst, disp)) => {
520                dynasm!($assembler ; $ins DWORD [Rq(dst) + disp], src as i32);
521            },
522            (Size::S64, Location::Imm32(src), Location::Memory(dst, disp)) => {
523                dynasm!($assembler ; $ins QWORD [Rq(dst) + disp], src as i32);
524            },
525            _ => $otherwise
526        }
527    };
528}
529
530macro_rules! binop_imm64_gpr {
531    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
532        match ($sz, $src, $dst) {
533            (Size::S64, Location::Imm64(src), Location::GPR(dst)) => {
534                dynasm!($assembler ; $ins Rq(dst), QWORD src as i64); // IMM32_2GPR
535            },
536            _ => $otherwise
537        }
538    };
539}
540
541macro_rules! binop_gpr_gpr {
542    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
543        match ($sz, $src, $dst) {
544            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
545                dynasm!($assembler ; $ins Rd(dst), Rd(src)); // GPR2GPR
546            },
547            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
548                dynasm!($assembler ; $ins Rq(dst), Rq(src)); // GPR2GPR
549            },
550            _ => $otherwise
551        }
552    };
553}
554
555macro_rules! binop_gpr_mem {
556    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
557        match ($sz, $src, $dst) {
558            (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
559                dynasm!($assembler ; $ins [Rq(dst) + disp], Rd(src)); // GPR2MEM
560            },
561            (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
562                dynasm!($assembler ; $ins [Rq(dst) + disp], Rq(src)); // GPR2MEM
563            },
564            _ => $otherwise
565        }
566    };
567}
568
569macro_rules! binop_mem_gpr {
570    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
571        match ($sz, $src, $dst) {
572            (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => {
573                dynasm!($assembler ; $ins Rd(dst), [Rq(src) + disp]); // MEM2GPR
574            },
575            (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => {
576                dynasm!($assembler ; $ins Rq(dst), [Rq(src) + disp]); // MEM2GPR
577            },
578            _ => $otherwise
579        }
580    };
581}
582
583macro_rules! binop_all_nofp {
584    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
585        binop_imm32_gpr!($ins, $assembler, $sz, $src, $dst, {
586            binop_imm32_mem!($ins, $assembler, $sz, $src, $dst, {
587                binop_gpr_gpr!($ins, $assembler, $sz, $src, $dst, {
588                    binop_gpr_mem!($ins, $assembler, $sz, $src, $dst, {
589                        binop_mem_gpr!($ins, $assembler, $sz, $src, $dst, $otherwise)
590                    })
591                })
592            })
593        })
594    };
595}
596
597macro_rules! binop_shift {
598    ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
599        match ($sz, $src, $dst) {
600            (Size::S32, Location::GPR(GPR::RCX), Location::GPR(dst)) => {
601                dynasm!($assembler ; $ins Rd(dst), cl);
602            },
603            (Size::S32, Location::GPR(GPR::RCX), Location::Memory(dst, disp)) => {
604                dynasm!($assembler ; $ins DWORD [Rq(dst) + disp], cl);
605            },
606            (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
607                dynasm!($assembler ; $ins Rd(dst), imm as i8);
608            },
609            (Size::S32, Location::Imm8(imm), Location::Memory(dst, disp)) => {
610                dynasm!($assembler ; $ins DWORD [Rq(dst) + disp], imm as i8);
611            },
612            (Size::S64, Location::GPR(GPR::RCX), Location::GPR(dst)) => {
613                dynasm!($assembler ; $ins Rq(dst), cl);
614            },
615            (Size::S64, Location::GPR(GPR::RCX), Location::Memory(dst, disp)) => {
616                dynasm!($assembler ; $ins QWORD [Rq(dst) + disp], cl);
617            },
618            (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
619                dynasm!($assembler ; $ins Rq(dst), imm as i8);
620            },
621            (Size::S64, Location::Imm8(imm), Location::Memory(dst, disp)) => {
622                dynasm!($assembler ; $ins QWORD [Rq(dst) + disp], imm as i8);
623            },
624            _ => $otherwise
625        }
626    }
627}
628
629macro_rules! jmp_op {
630    ($ins:ident, $assembler:tt, $label:ident) => {
631        dynasm!($assembler ; $ins =>$label)
632    }
633}
634
635/// Move a single or double precision XMM value to another if src and destination
636/// are not the same.
637///
638/// TODO: Can we assume data is aligned and packed? If so, this function isn't necessary
639/// TODO: as we can use [`EmitterX64::emit_vmovaps`] and [`EmitterX64::emit_vmovadp`]
640/// TODO: instead
641fn move_src_to_dst(emitter: &mut AssemblerX64, precision: Precision, src: XMM, dst: XMM) {
642    if src == dst {
643        return;
644    }
645    match precision {
646        Precision::Single => dynasm!(emitter ; movss Rx(dst), Rx(src)),
647        Precision::Double => dynasm!(emitter ; movsd Rx(dst), Rx(src)),
648    }
649}
650
651macro_rules! avx_fn {
652    ($ins:ident, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
653        match $src2 {
654            XMMOrMemory::XMM(x) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), Rx(x)),
655            XMMOrMemory::Memory(base, disp) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), [Rq(base) + disp]),
656        }
657    }
658}
659
660macro_rules! sse_fn {
661    ($ins:ident, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
662        match $src2 {
663            XMMOrMemory::XMM(x) => {
664                if x == $dst {
665                    dynasm!($emitter ; $ins Rx($dst), Rx($src1))
666                } else {
667                    move_src_to_dst($emitter, $precision, $src1, $dst);
668                    dynasm!($emitter ; $ins Rx($dst), Rx(x))
669                }
670            }
671            XMMOrMemory::Memory(base, disp) => {
672                move_src_to_dst($emitter, $precision, $src1, $dst);
673                dynasm!($emitter ; $ins Rx($dst), [Rq(base) + disp])
674            }
675        }
676    };
677    ($ins:ident, $mode:expr, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
678        match $src2 {
679            XMMOrMemory::XMM(x) => {
680                move_src_to_dst($emitter, $precision, $src1, $dst);
681                dynasm!($emitter ; $ins Rx($dst), Rx(x), $mode)
682            }
683            XMMOrMemory::Memory(base, disp) => {
684                move_src_to_dst($emitter, $precision, $src1, $dst);
685                dynasm!($emitter ; $ins Rx($dst), [Rq(base) + disp], $mode)
686            }
687        }
688    };
689}
690
691macro_rules! avx_i2f_64_fn {
692    ($ins:ident, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
693        match $src2 {
694            GPROrMemory::GPR(x) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), Rq(x)),
695            GPROrMemory::Memory(base, disp) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), QWORD [Rq(base) + disp]),
696        }
697    }
698}
699
700macro_rules! sse_i2f_64_fn {
701    ($ins:ident, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
702        match $src2 {
703            GPROrMemory::GPR(x) => {
704                move_src_to_dst($emitter, $precision, $src1, $dst);
705                dynasm!($emitter ; $ins Rx($dst), Rq(x))
706            },
707            GPROrMemory::Memory(base, disp) => {
708                move_src_to_dst($emitter, $precision, $src1, $dst);
709                dynasm!($emitter ; $ins Rx($dst), QWORD [Rq(base) + disp])
710            }
711        }
712    }
713}
714
715macro_rules! avx_i2f_32_fn {
716    ($ins:ident, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
717        match $src2 {
718            GPROrMemory::GPR(x) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), Rd(x)),
719            GPROrMemory::Memory(base, disp) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), DWORD [Rq(base) + disp]),
720        }
721    }
722}
723
724macro_rules! sse_i2f_32_fn {
725    ($ins:ident, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
726        match $src2 {
727            GPROrMemory::GPR(x) => {
728                move_src_to_dst($emitter, $precision, $src1, $dst);
729                dynasm!($emitter; $ins Rx($src1), Rd(x))
730            },
731            GPROrMemory::Memory(base, disp) => {
732                move_src_to_dst($emitter, $precision, $src1, $dst);
733                dynasm!($emitter; $ins Rx($dst), DWORD [Rq(base) + disp])
734            }
735        }
736    }
737}
738
739macro_rules! avx_round_fn {
740    ($ins:ident, $mode:expr, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
741        match $src2 {
742            XMMOrMemory::XMM(x) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), Rx(x), $mode),
743            XMMOrMemory::Memory(base, disp) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), [Rq(base) + disp], $mode),
744        }
745    }
746}
747
748macro_rules! sse_round_fn {
749    ($ins:ident, $mode:expr, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
750        match $src2 {
751            XMMOrMemory::XMM(x) => {
752                if x != $dst {
753                    move_src_to_dst($emitter, $precision, $src1, $dst);
754                }
755                dynasm!($emitter ; $ins Rx(x), Rx($dst), $mode)
756            }
757            XMMOrMemory::Memory(base, disp) => {
758                dynasm!($emitter ; $ins Rx($dst), [Rq(base) + disp], $mode)
759            },
760        }
761    }
762}
763
764impl EmitterX64 for AssemblerX64 {
765    fn get_simd_arch(&self) -> Option<&CpuFeature> {
766        Some(&self.simd_arch)
767    }
768
769    fn get_label(&mut self) -> DynamicLabel {
770        self.new_dynamic_label()
771    }
772
773    fn get_offset(&self) -> AssemblyOffset {
774        self.offset()
775    }
776
777    fn get_jmp_instr_size(&self) -> u8 {
778        5
779    }
780
781    fn finalize_function(&mut self) -> Result<(), CompileError> {
782        Ok(())
783    }
784
785    fn arch_has_xzcnt(&self) -> bool {
786        match &self.target {
787            Some(target) => {
788                target.cpu_features().contains(CpuFeature::LZCNT)
789                    && target.cpu_features().contains(CpuFeature::BMI1)
790            }
791            None => false,
792        }
793    }
794
795    fn arch_emit_lzcnt(
796        &mut self,
797        sz: Size,
798        src: Location,
799        dst: Location,
800    ) -> Result<(), CompileError> {
801        binop_gpr_gpr!(lzcnt, self, sz, src, dst, {
802            binop_mem_gpr!(lzcnt, self, sz, src, dst, {
803                codegen_error!("singlepass cannot emit lzcnt")
804            })
805        });
806        Ok(())
807    }
808
809    fn arch_emit_tzcnt(
810        &mut self,
811        sz: Size,
812        src: Location,
813        dst: Location,
814    ) -> Result<(), CompileError> {
815        binop_gpr_gpr!(tzcnt, self, sz, src, dst, {
816            binop_mem_gpr!(tzcnt, self, sz, src, dst, {
817                codegen_error!("singlepass cannot emit tzcnt")
818            })
819        });
820        Ok(())
821    }
822
823    fn emit_u64(&mut self, x: u64) -> Result<(), CompileError> {
824        self.push_u64(x);
825        Ok(())
826    }
827
828    fn emit_bytes(&mut self, bytes: &[u8]) -> Result<(), CompileError> {
829        for &b in bytes {
830            self.push(b);
831        }
832        Ok(())
833    }
834
835    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
836        dynasm!(self ; => label);
837        Ok(())
838    }
839
840    fn emit_nop(&mut self) -> Result<(), CompileError> {
841        dynasm!(self ; nop);
842        Ok(())
843    }
844
845    fn emit_nop_n(&mut self, mut n: usize) -> Result<(), CompileError> {
846        /*
847            1      90H                            NOP
848            2      66 90H                         66 NOP
849            3      0F 1F 00H                      NOP DWORD ptr [EAX]
850            4      0F 1F 40 00H                   NOP DWORD ptr [EAX + 00H]
851            5      0F 1F 44 00 00H                NOP DWORD ptr [EAX + EAX*1 + 00H]
852            6      66 0F 1F 44 00 00H             NOP DWORD ptr [AX + AX*1 + 00H]
853            7      0F 1F 80 00 00 00 00H          NOP DWORD ptr [EAX + 00000000H]
854            8      0F 1F 84 00 00 00 00 00H       NOP DWORD ptr [AX + AX*1 + 00000000H]
855            9      66 0F 1F 84 00 00 00 00 00H    NOP DWORD ptr [AX + AX*1 + 00000000H]
856        */
857        while n >= 9 {
858            n -= 9;
859            self.emit_bytes(&[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00])?;
860            // 9-byte nop
861        }
862        let seq: &[u8] = match n {
863            0 => &[],
864            1 => &[0x90],
865            2 => &[0x66, 0x90],
866            3 => &[0x0f, 0x1f, 0x00],
867            4 => &[0x0f, 0x1f, 0x40, 0x00],
868            5 => &[0x0f, 0x1f, 0x44, 0x00, 0x00],
869            6 => &[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00],
870            7 => &[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00],
871            8 => &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
872            _ => codegen_error!("singlepass emit_nop_n unreachable"),
873        };
874        self.emit_bytes(seq)
875    }
876
877    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
878        // fast path
879        if let (Location::Imm32(0), Location::GPR(x)) = (src, dst) {
880            dynasm!(self ; xor Rd(x), Rd(x));
881            return Ok(());
882        }
883
884        binop_all_nofp!(mov, self, sz, src, dst, {
885            binop_imm64_gpr!(mov, self, sz, src, dst, {
886                match (sz, src, dst) {
887                    (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
888                        dynasm!(self ; mov [Rq(dst) + disp], Rb(src));
889                    }
890                    (Size::S8, Location::Memory(src, disp), Location::GPR(dst)) => {
891                        dynasm!(self ; mov Rb(dst), [Rq(src) + disp]);
892                    }
893                    (Size::S8, Location::Imm32(src), Location::GPR(dst)) => {
894                        dynasm!(self ; mov Rb(dst), src as i8);
895                    }
896                    (Size::S8, Location::Imm64(src), Location::GPR(dst)) => {
897                        dynasm!(self ; mov Rb(dst), src as i8);
898                    }
899                    (Size::S8, Location::Imm32(src), Location::Memory(dst, disp)) => {
900                        dynasm!(self ; mov BYTE [Rq(dst) + disp], src as i8);
901                    }
902                    (Size::S8, Location::Imm64(src), Location::Memory(dst, disp)) => {
903                        dynasm!(self ; mov BYTE [Rq(dst) + disp], src as i8);
904                    }
905                    (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
906                        dynasm!(self ; mov [Rq(dst) + disp], Rw(src));
907                    }
908                    (Size::S16, Location::Memory(src, disp), Location::GPR(dst)) => {
909                        dynasm!(self ; mov Rw(dst), [Rq(src) + disp]);
910                    }
911                    (Size::S16, Location::Imm32(src), Location::GPR(dst)) => {
912                        dynasm!(self ; mov Rw(dst), src as i16);
913                    }
914                    (Size::S16, Location::Imm64(src), Location::GPR(dst)) => {
915                        dynasm!(self ; mov Rw(dst), src as i16);
916                    }
917                    (Size::S16, Location::Imm32(src), Location::Memory(dst, disp)) => {
918                        dynasm!(self ; mov WORD [Rq(dst) + disp], src as i16);
919                    }
920                    (Size::S16, Location::Imm64(src), Location::Memory(dst, disp)) => {
921                        dynasm!(self ; mov WORD [Rq(dst) + disp], src as i16);
922                    }
923                    (Size::S32, Location::Imm64(src), Location::GPR(dst)) => {
924                        dynasm!(self ; mov Rd(dst), src as i32);
925                    }
926                    (Size::S32, Location::Imm64(src), Location::Memory(dst, disp)) => {
927                        dynasm!(self ; mov DWORD [Rq(dst) + disp], src as i32);
928                    }
929                    (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
930                        dynasm!(self ; movd Rx(dst), Rd(src));
931                    }
932                    (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
933                        dynasm!(self ; movd Rd(dst), Rx(src));
934                    }
935                    (Size::S32, Location::Memory(src, disp), Location::SIMD(dst)) => {
936                        dynasm!(self ; movd Rx(dst), [Rq(src) + disp]);
937                    }
938                    (Size::S32, Location::SIMD(src), Location::Memory(dst, disp)) => {
939                        dynasm!(self ; movd [Rq(dst) + disp], Rx(src));
940                    }
941                    (Size::S64, Location::Imm64(src), Location::GPR(dst)) => {
942                        dynasm!(self ; mov Rd(dst), src as i32);
943                    }
944                    (Size::S64, Location::Imm32(src), Location::GPR(dst)) => {
945                        dynasm!(self ; mov Rd(dst), src as i32);
946                    }
947                    (Size::S64, Location::Imm8(src), Location::GPR(dst)) => {
948                        dynasm!(self ; mov Rd(dst), src as i32);
949                    }
950
951                    (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
952                        dynasm!(self ; movq Rx(dst), Rq(src));
953                    }
954                    (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
955                        dynasm!(self ; movq Rq(dst), Rx(src));
956                    }
957                    (Size::S64, Location::Memory(src, disp), Location::SIMD(dst)) => {
958                        dynasm!(self ; movq Rx(dst), [Rq(src) + disp]);
959                    }
960                    (Size::S64, Location::SIMD(src), Location::Memory(dst, disp)) => {
961                        dynasm!(self ; movq [Rq(dst) + disp], Rx(src));
962                    }
963                    (_, Location::SIMD(src), Location::SIMD(dst)) => {
964                        dynasm!(self ; movq Rx(dst), Rx(src));
965                    }
966
967                    _ => codegen_error!("singlepass can't emit MOV {:?} {:?} {:?}", sz, src, dst),
968                }
969            })
970        });
971        Ok(())
972    }
973    fn emit_lea(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
974        match (sz, src, dst) {
975            (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => {
976                dynasm!(self ; lea Rd(dst), [Rq(src) + disp]);
977            }
978            (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => {
979                dynasm!(self ; lea Rq(dst), [Rq(src) + disp]);
980            }
981            (Size::S32, Location::Memory2(src1, src2, mult, disp), Location::GPR(dst)) => {
982                match mult {
983                    Multiplier::Zero => dynasm!(self ; lea Rd(dst), [Rq(src1) + disp]),
984                    Multiplier::One => {
985                        dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) + disp])
986                    }
987                    Multiplier::Two => {
988                        dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) * 2 + disp])
989                    }
990                    Multiplier::Four => {
991                        dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) * 4 + disp])
992                    }
993                    Multiplier::Height => {
994                        dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) * 8 + disp])
995                    }
996                };
997            }
998            (Size::S64, Location::Memory2(src1, src2, mult, disp), Location::GPR(dst)) => {
999                match mult {
1000                    Multiplier::Zero => dynasm!(self ; lea Rq(dst), [Rq(src1) + disp]),
1001                    Multiplier::One => {
1002                        dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) + disp])
1003                    }
1004                    Multiplier::Two => {
1005                        dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) * 2 + disp])
1006                    }
1007                    Multiplier::Four => {
1008                        dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) * 4 + disp])
1009                    }
1010                    Multiplier::Height => {
1011                        dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) * 8 + disp])
1012                    }
1013                };
1014            }
1015            _ => codegen_error!("singlepass can't emit LEA {:?} {:?} {:?}", sz, src, dst),
1016        }
1017        Ok(())
1018    }
1019    fn emit_lea_label(&mut self, label: Label, dst: Location) -> Result<(), CompileError> {
1020        match dst {
1021            Location::GPR(x) => {
1022                dynasm!(self ; lea Rq(x), [=>label]);
1023            }
1024            _ => codegen_error!("singlepass can't emit LEA label={:?} {:?}", label, dst),
1025        }
1026        Ok(())
1027    }
1028    fn emit_cdq(&mut self) -> Result<(), CompileError> {
1029        dynasm!(self ; cdq);
1030        Ok(())
1031    }
1032    fn emit_cqo(&mut self) -> Result<(), CompileError> {
1033        dynasm!(self ; cqo);
1034        Ok(())
1035    }
1036    fn emit_xor(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1037        binop_all_nofp!(xor, self, sz, src, dst, {
1038            codegen_error!("singlepass can't emit XOR {:?} {:?} {:?}", sz, src, dst)
1039        });
1040        Ok(())
1041    }
1042    fn emit_jmp(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
1043        match condition {
1044            Condition::None => jmp_op!(jmp, self, label),
1045            Condition::Above => jmp_op!(ja, self, label),
1046            Condition::AboveEqual => jmp_op!(jae, self, label),
1047            Condition::Below => jmp_op!(jb, self, label),
1048            Condition::BelowEqual => jmp_op!(jbe, self, label),
1049            Condition::Greater => jmp_op!(jg, self, label),
1050            Condition::GreaterEqual => jmp_op!(jge, self, label),
1051            Condition::Less => jmp_op!(jl, self, label),
1052            Condition::LessEqual => jmp_op!(jle, self, label),
1053            Condition::Equal => jmp_op!(je, self, label),
1054            Condition::NotEqual => jmp_op!(jne, self, label),
1055            Condition::Signed => jmp_op!(js, self, label),
1056            Condition::Carry => jmp_op!(jc, self, label),
1057        }
1058        Ok(())
1059    }
1060    fn emit_jmp_location(&mut self, loc: Location) -> Result<(), CompileError> {
1061        match loc {
1062            Location::GPR(x) => dynasm!(self ; jmp Rq(x)),
1063            Location::Memory(base, disp) => dynasm!(self ; jmp QWORD [Rq(base) + disp]),
1064            _ => codegen_error!("singlepass can't emit JMP {:?}", loc),
1065        }
1066        Ok(())
1067    }
1068    fn emit_set(&mut self, condition: Condition, dst: GPR) -> Result<(), CompileError> {
1069        match condition {
1070            Condition::Above => dynasm!(self ; seta Rb(dst)),
1071            Condition::AboveEqual => dynasm!(self ; setae Rb(dst)),
1072            Condition::Below => dynasm!(self ; setb Rb(dst)),
1073            Condition::BelowEqual => dynasm!(self ; setbe Rb(dst)),
1074            Condition::Greater => dynasm!(self ; setg Rb(dst)),
1075            Condition::GreaterEqual => dynasm!(self ; setge Rb(dst)),
1076            Condition::Less => dynasm!(self ; setl Rb(dst)),
1077            Condition::LessEqual => dynasm!(self ; setle Rb(dst)),
1078            Condition::Equal => dynasm!(self ; sete Rb(dst)),
1079            Condition::NotEqual => dynasm!(self ; setne Rb(dst)),
1080            Condition::Signed => dynasm!(self ; sets Rb(dst)),
1081            Condition::Carry => dynasm!(self ; setc Rb(dst)),
1082            _ => codegen_error!("singlepass can't emit SET {:?} {:?}", condition, dst),
1083        }
1084        Ok(())
1085    }
1086    fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1087        match (sz, src) {
1088            (Size::S64, Location::Imm32(src)) => dynasm!(self ; push src as i32),
1089            (Size::S64, Location::GPR(src)) => dynasm!(self ; push Rq(src)),
1090            (Size::S64, Location::Memory(src, disp)) => {
1091                dynasm!(self ; push QWORD [Rq(src) + disp])
1092            }
1093            _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1094        }
1095        Ok(())
1096    }
1097    fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1098        match (sz, dst) {
1099            (Size::S64, Location::GPR(dst)) => dynasm!(self ; pop Rq(dst)),
1100            (Size::S64, Location::Memory(dst, disp)) => {
1101                dynasm!(self ; pop QWORD [Rq(dst) + disp])
1102            }
1103            _ => codegen_error!("singlepass can't emit POP {:?} {:?}", sz, dst),
1104        }
1105        Ok(())
1106    }
1107
1108    fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1109        binop_all_nofp!(cmp, self, sz, left, right, {
1110            codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right);
1111        });
1112        Ok(())
1113    }
1114
1115    fn emit_add(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1116        // Fast path
1117        if let Location::Imm32(0) = src {
1118            return Ok(());
1119        }
1120        binop_all_nofp!(add, self, sz, src, dst, {
1121            codegen_error!("singlepass can't emit ADD {:?} {:?} {:?}", sz, src, dst)
1122        });
1123        Ok(())
1124    }
1125    fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1126        // Fast path
1127        if let Location::Imm32(0) = src {
1128            return Ok(());
1129        }
1130        binop_all_nofp!(sub, self, sz, src, dst, {
1131            codegen_error!("singlepass can't emit SUB {:?} {:?} {:?}", sz, src, dst)
1132        });
1133        Ok(())
1134    }
1135    fn emit_neg(&mut self, sz: Size, value: Location) -> Result<(), CompileError> {
1136        match (sz, value) {
1137            (Size::S8, Location::GPR(value)) => dynasm!(self ; neg Rb(value)),
1138            (Size::S8, Location::Memory(value, disp)) => {
1139                dynasm!(self ; neg [Rq(value) + disp])
1140            }
1141            (Size::S16, Location::GPR(value)) => dynasm!(self ; neg Rw(value)),
1142            (Size::S16, Location::Memory(value, disp)) => {
1143                dynasm!(self ; neg [Rq(value) + disp])
1144            }
1145            (Size::S32, Location::GPR(value)) => dynasm!(self ; neg Rd(value)),
1146            (Size::S32, Location::Memory(value, disp)) => {
1147                dynasm!(self ; neg [Rq(value) + disp])
1148            }
1149            (Size::S64, Location::GPR(value)) => dynasm!(self ; neg Rq(value)),
1150            (Size::S64, Location::Memory(value, disp)) => {
1151                dynasm!(self ; neg [Rq(value) + disp])
1152            }
1153            _ => codegen_error!("singlepass can't emit NEG {:?} {:?}", sz, value),
1154        }
1155        Ok(())
1156    }
1157    fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1158        binop_gpr_gpr!(imul, self, sz, src, dst, {
1159            binop_mem_gpr!(imul, self, sz, src, dst, {
1160                codegen_error!("singlepass can't emit IMUL {:?} {:?} {:?}", sz, src, dst)
1161            })
1162        });
1163        Ok(())
1164    }
1165    fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR) -> Result<(), CompileError> {
1166        dynasm!(self ; imul Rq(dst), Rq(dst), src as i32);
1167        Ok(())
1168    }
1169    fn emit_div(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError> {
1170        unop_gpr_or_mem!(div, self, sz, divisor, {
1171            codegen_error!("singlepass can't emit DIV {:?} {:?}", sz, divisor)
1172        });
1173        Ok(())
1174    }
1175    fn emit_idiv(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError> {
1176        unop_gpr_or_mem!(idiv, self, sz, divisor, {
1177            codegen_error!("singlepass can't emit IDIV {:?} {:?}", sz, divisor)
1178        });
1179        Ok(())
1180    }
1181    fn emit_shl(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1182        binop_shift!(shl, self, sz, src, dst, {
1183            codegen_error!("singlepass can't emit SHL {:?} {:?} {:?}", sz, src, dst)
1184        });
1185        Ok(())
1186    }
1187    fn emit_shr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1188        binop_shift!(shr, self, sz, src, dst, {
1189            codegen_error!("singlepass can't emit SHR {:?} {:?} {:?}", sz, src, dst)
1190        });
1191        Ok(())
1192    }
1193    fn emit_sar(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1194        binop_shift!(sar, self, sz, src, dst, {
1195            codegen_error!("singlepass can't emit SAR {:?} {:?} {:?}", sz, src, dst)
1196        });
1197        Ok(())
1198    }
1199    fn emit_rol(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1200        binop_shift!(rol, self, sz, src, dst, {
1201            codegen_error!("singlepass can't emit ROL {:?} {:?} {:?}", sz, src, dst)
1202        });
1203        Ok(())
1204    }
1205    fn emit_ror(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1206        binop_shift!(ror, self, sz, src, dst, {
1207            codegen_error!("singlepass can't emit ROR {:?} {:?} {:?}", sz, src, dst)
1208        });
1209        Ok(())
1210    }
1211    fn emit_and(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1212        binop_all_nofp!(and, self, sz, src, dst, {
1213            codegen_error!("singlepass can't emit AND {:?} {:?} {:?}", sz, src, dst)
1214        });
1215        Ok(())
1216    }
1217    fn emit_test(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1218        binop_all_nofp!(test, self, sz, src, dst, {
1219            codegen_error!("singlepass can't emit TEST {:?} {:?} {:?}", sz, src, dst)
1220        });
1221        Ok(())
1222    }
1223    fn emit_or(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1224        binop_all_nofp!(or, self, sz, src, dst, {
1225            codegen_error!("singlepass can't emit OR {:?} {:?} {:?}", sz, src, dst)
1226        });
1227        Ok(())
1228    }
1229    fn emit_bsr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1230        binop_gpr_gpr!(bsr, self, sz, src, dst, {
1231            binop_mem_gpr!(bsr, self, sz, src, dst, {
1232                codegen_error!("singlepass can't emit BSR {:?} {:?} {:?}", sz, src, dst)
1233            })
1234        });
1235        Ok(())
1236    }
1237    fn emit_bsf(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1238        binop_gpr_gpr!(bsf, self, sz, src, dst, {
1239            binop_mem_gpr!(bsf, self, sz, src, dst, {
1240                codegen_error!("singlepass can't emit BSF {:?} {:?} {:?}", sz, src, dst)
1241            })
1242        });
1243        Ok(())
1244    }
1245    fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1246        binop_gpr_gpr!(popcnt, self, sz, src, dst, {
1247            binop_mem_gpr!(popcnt, self, sz, src, dst, {
1248                codegen_error!("singlepass can't emit POPCNT {:?} {:?} {:?}", sz, src, dst)
1249            })
1250        });
1251        Ok(())
1252    }
1253    fn emit_movzx(
1254        &mut self,
1255        sz_src: Size,
1256        src: Location,
1257        sz_dst: Size,
1258        dst: Location,
1259    ) -> Result<(), CompileError> {
1260        match (sz_src, src, sz_dst, dst) {
1261            (Size::S8, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1262                dynasm!(self ; movzx Rd(dst), Rb(src));
1263            }
1264            (Size::S16, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1265                dynasm!(self ; movzx Rd(dst), Rw(src));
1266            }
1267            (Size::S8, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1268                dynasm!(self ; movzx Rd(dst), BYTE [Rq(src) + disp]);
1269            }
1270            (Size::S16, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1271                dynasm!(self ; movzx Rd(dst), WORD [Rq(src) + disp]);
1272            }
1273            (Size::S16, Location::Imm32(imm), Size::S32, Location::GPR(dst)) => {
1274                dynasm!(self ; mov Rd(dst), imm as i32);
1275            }
1276            (Size::S8, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1277                dynasm!(self ; movzx Rq(dst), Rb(src));
1278            }
1279            (Size::S16, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1280                dynasm!(self ; movzx Rq(dst), Rw(src));
1281            }
1282            (Size::S8, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1283                dynasm!(self ; movzx Rq(dst), BYTE [Rq(src) + disp]);
1284            }
1285            (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1286                dynasm!(self ; movzx Rq(dst), WORD [Rq(src) + disp]);
1287            }
1288            (Size::S32, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1289                dynasm!(self ; mov Rd(dst), Rd(src));
1290            }
1291            (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1292                dynasm!(self ; mov Rd(dst), DWORD [Rq(src) + disp]);
1293            }
1294            (Size::S8, Location::Imm32(imm), Size::S32, Location::GPR(dst)) => {
1295                dynasm!(self ; mov Rq(dst), imm as i32);
1296            }
1297            (Size::S16, Location::Imm32(imm), Size::S64, Location::GPR(dst)) => {
1298                dynasm!(self ; mov Rq(dst), imm as i32);
1299            }
1300            (Size::S32, Location::Imm32(imm), Size::S64, Location::GPR(dst)) => {
1301                dynasm!(self ; mov Rq(dst), imm as i32);
1302            }
1303            (Size::S8, Location::Imm64(imm), Size::S64, Location::GPR(dst)) => {
1304                dynasm!(self ; mov Rq(dst), imm as i32);
1305            }
1306            (Size::S16, Location::Imm64(imm), Size::S64, Location::GPR(dst)) => {
1307                dynasm!(self ; mov Rq(dst), imm as i32);
1308            }
1309            (Size::S32, Location::Imm64(imm), Size::S64, Location::GPR(dst)) => {
1310                dynasm!(self ; mov Rq(dst), imm as i32);
1311            }
1312            _ => {
1313                codegen_error!(
1314                    "singlepass can't emit MOVZX {:?} {:?} {:?} {:?}",
1315                    sz_src,
1316                    src,
1317                    sz_dst,
1318                    dst
1319                )
1320            }
1321        }
1322        Ok(())
1323    }
1324    fn emit_movsx(
1325        &mut self,
1326        sz_src: Size,
1327        src: Location,
1328        sz_dst: Size,
1329        dst: Location,
1330    ) -> Result<(), CompileError> {
1331        match (sz_src, src, sz_dst, dst) {
1332            (Size::S8, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1333                dynasm!(self ; movsx Rd(dst), Rb(src));
1334            }
1335            (Size::S16, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1336                dynasm!(self ; movsx Rd(dst), Rw(src));
1337            }
1338            (Size::S8, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1339                dynasm!(self ; movsx Rd(dst), BYTE [Rq(src) + disp]);
1340            }
1341            (Size::S16, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1342                dynasm!(self ; movsx Rd(dst), WORD [Rq(src) + disp]);
1343            }
1344            (Size::S8, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1345                dynasm!(self ; movsx Rq(dst), Rb(src));
1346            }
1347            (Size::S16, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1348                dynasm!(self ; movsx Rq(dst), Rw(src));
1349            }
1350            (Size::S32, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1351                dynasm!(self ; movsx Rq(dst), Rd(src));
1352            }
1353            (Size::S8, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1354                dynasm!(self ; movsx Rq(dst), BYTE [Rq(src) + disp]);
1355            }
1356            (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1357                dynasm!(self ; movsx Rq(dst), WORD [Rq(src) + disp]);
1358            }
1359            (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1360                dynasm!(self ; movsx Rq(dst), DWORD [Rq(src) + disp]);
1361            }
1362            _ => {
1363                codegen_error!(
1364                    "singlepass can't emit MOVSX {:?} {:?} {:?} {:?}",
1365                    sz_src,
1366                    src,
1367                    sz_dst,
1368                    dst
1369                )
1370            }
1371        }
1372        Ok(())
1373    }
1374
1375    fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1376        match (sz, src, dst) {
1377            (Size::S8, Location::GPR(src), Location::GPR(dst)) => {
1378                dynasm!(self ; xchg Rb(dst), Rb(src));
1379            }
1380            (Size::S16, Location::GPR(src), Location::GPR(dst)) => {
1381                dynasm!(self ; xchg Rw(dst), Rw(src));
1382            }
1383            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1384                dynasm!(self ; xchg Rd(dst), Rd(src));
1385            }
1386            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1387                dynasm!(self ; xchg Rq(dst), Rq(src));
1388            }
1389            (Size::S8, Location::Memory(src, disp), Location::GPR(dst)) => {
1390                dynasm!(self ; xchg Rb(dst), [Rq(src) + disp]);
1391            }
1392            (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
1393                dynasm!(self ; xchg [Rq(dst) + disp], Rb(src));
1394            }
1395            (Size::S16, Location::Memory(src, disp), Location::GPR(dst)) => {
1396                dynasm!(self ; xchg Rw(dst), [Rq(src) + disp]);
1397            }
1398            (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
1399                dynasm!(self ; xchg [Rq(dst) + disp], Rw(src));
1400            }
1401            (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => {
1402                dynasm!(self ; xchg Rd(dst), [Rq(src) + disp]);
1403            }
1404            (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
1405                dynasm!(self ; xchg [Rq(dst) + disp], Rd(src));
1406            }
1407            (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => {
1408                dynasm!(self ; xchg Rq(dst), [Rq(src) + disp]);
1409            }
1410            (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
1411                dynasm!(self ; xchg [Rq(dst) + disp], Rq(src));
1412            }
1413            _ => codegen_error!("singlepass can't emit XCHG {:?} {:?} {:?}", sz, src, dst),
1414        }
1415        Ok(())
1416    }
1417
1418    fn emit_lock_xadd(
1419        &mut self,
1420        sz: Size,
1421        src: Location,
1422        dst: Location,
1423    ) -> Result<(), CompileError> {
1424        match (sz, src, dst) {
1425            (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
1426                dynasm!(self ; lock xadd [Rq(dst) + disp], Rb(src));
1427            }
1428            (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
1429                dynasm!(self ; lock xadd [Rq(dst) + disp], Rw(src));
1430            }
1431            (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
1432                dynasm!(self ; lock xadd [Rq(dst) + disp], Rd(src));
1433            }
1434            (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
1435                dynasm!(self ; lock xadd [Rq(dst) + disp], Rq(src));
1436            }
1437            _ => codegen_error!(
1438                "singlepass can't emit LOCK XADD {:?} {:?} {:?}",
1439                sz,
1440                src,
1441                dst
1442            ),
1443        }
1444        Ok(())
1445    }
1446
1447    fn emit_lock_cmpxchg(
1448        &mut self,
1449        sz: Size,
1450        src: Location,
1451        dst: Location,
1452    ) -> Result<(), CompileError> {
1453        match (sz, src, dst) {
1454            (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
1455                dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rb(src));
1456            }
1457            (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
1458                dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rw(src));
1459            }
1460            (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
1461                dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rd(src));
1462            }
1463            (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
1464                dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rq(src));
1465            }
1466            _ => codegen_error!(
1467                "singlepass can't emit LOCK CMPXCHG {:?} {:?} {:?}",
1468                sz,
1469                src,
1470                dst
1471            ),
1472        }
1473        Ok(())
1474    }
1475
1476    fn emit_rep_stosq(&mut self) -> Result<(), CompileError> {
1477        dynasm!(self ; rep stosq);
1478        Ok(())
1479    }
1480    fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR) -> Result<(), CompileError> {
1481        dynasm!(self ; btc Rd(dst), BYTE src as i8);
1482        Ok(())
1483    }
1484
1485    fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR) -> Result<(), CompileError> {
1486        dynasm!(self ; btc Rq(dst), BYTE src as i8);
1487        Ok(())
1488    }
1489
1490    fn emit_cmovae_gpr_32(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError> {
1491        dynasm!(self ; cmovae Rd(dst), Rd(src));
1492        Ok(())
1493    }
1494
1495    fn emit_cmovae_gpr_64(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError> {
1496        dynasm!(self ; cmovae Rq(dst), Rq(src));
1497        Ok(())
1498    }
1499
1500    fn emit_vmovaps(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError> {
1501        match (src, dst) {
1502            (XMMOrMemory::XMM(src), XMMOrMemory::XMM(dst)) => {
1503                dynasm!(self ; movaps Rx(dst), Rx(src))
1504            }
1505            (XMMOrMemory::Memory(base, disp), XMMOrMemory::XMM(dst)) => {
1506                dynasm!(self ; movaps Rx(dst), [Rq(base) + disp])
1507            }
1508            (XMMOrMemory::XMM(src), XMMOrMemory::Memory(base, disp)) => {
1509                dynasm!(self ; movaps [Rq(base) + disp], Rx(src))
1510            }
1511            _ => codegen_error!("singlepass can't emit VMOVAPS {:?} {:?}", src, dst),
1512        };
1513        Ok(())
1514    }
1515
1516    fn emit_vmovapd(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError> {
1517        match (src, dst) {
1518            (XMMOrMemory::XMM(src), XMMOrMemory::XMM(dst)) => {
1519                dynasm!(self ; movapd Rx(dst), Rx(src))
1520            }
1521            (XMMOrMemory::Memory(base, disp), XMMOrMemory::XMM(dst)) => {
1522                dynasm!(self ; movapd Rx(dst), [Rq(base) + disp])
1523            }
1524            (XMMOrMemory::XMM(src), XMMOrMemory::Memory(base, disp)) => {
1525                dynasm!(self ; movapd [Rq(base) + disp], Rx(src))
1526            }
1527            _ => codegen_error!("singlepass can't emit VMOVAPD {:?} {:?}", src, dst),
1528        };
1529        Ok(())
1530    }
1531    fn emit_vxorps(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1532        match self.get_simd_arch() {
1533            Some(CpuFeature::AVX) => avx_fn!(vxorps, self, src1, src2, dst),
1534            Some(CpuFeature::SSE42) => sse_fn!(xorps, self, Precision::Single, src1, src2, dst),
1535            _ => {}
1536        }
1537        Ok(())
1538    }
1539    fn emit_vxorpd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1540        match self.get_simd_arch() {
1541            Some(CpuFeature::AVX) => avx_fn!(vxorpd, self, src1, src2, dst),
1542            Some(CpuFeature::SSE42) => sse_fn!(xorpd, self, Precision::Double, src1, src2, dst),
1543            _ => {}
1544        }
1545        Ok(())
1546    }
1547    fn emit_vaddss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1548        match self.get_simd_arch() {
1549            Some(CpuFeature::AVX) => avx_fn!(vaddss, self, src1, src2, dst),
1550            Some(CpuFeature::SSE42) => sse_fn!(addss, self, Precision::Single, src1, src2, dst),
1551            _ => {}
1552        }
1553        Ok(())
1554    }
1555    fn emit_vaddsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1556        match self.get_simd_arch() {
1557            Some(CpuFeature::AVX) => avx_fn!(vaddsd, self, src1, src2, dst),
1558            Some(CpuFeature::SSE42) => sse_fn!(addsd, self, Precision::Double, src1, src2, dst),
1559            _ => {}
1560        }
1561        Ok(())
1562    }
1563    fn emit_vsubss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1564        match self.get_simd_arch() {
1565            Some(CpuFeature::AVX) => avx_fn!(vsubss, self, src1, src2, dst),
1566            Some(CpuFeature::SSE42) => sse_fn!(subss, self, Precision::Single, src1, src2, dst),
1567            _ => {}
1568        }
1569        Ok(())
1570    }
1571    fn emit_vsubsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1572        match self.get_simd_arch() {
1573            Some(CpuFeature::AVX) => avx_fn!(vsubsd, self, src1, src2, dst),
1574            Some(CpuFeature::SSE42) => sse_fn!(subsd, self, Precision::Double, src1, src2, dst),
1575            _ => {}
1576        }
1577        Ok(())
1578    }
1579    fn emit_vmulss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1580        match self.get_simd_arch() {
1581            Some(CpuFeature::AVX) => avx_fn!(vmulss, self, src1, src2, dst),
1582            Some(CpuFeature::SSE42) => sse_fn!(mulss, self, Precision::Single, src1, src2, dst),
1583            _ => {}
1584        }
1585        Ok(())
1586    }
1587    fn emit_vmulsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1588        match self.get_simd_arch() {
1589            Some(CpuFeature::AVX) => avx_fn!(vmulsd, self, src1, src2, dst),
1590            Some(CpuFeature::SSE42) => sse_fn!(mulsd, self, Precision::Double, src1, src2, dst),
1591            _ => {}
1592        }
1593        Ok(())
1594    }
1595    fn emit_vdivss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1596        match self.get_simd_arch() {
1597            Some(CpuFeature::AVX) => avx_fn!(vdivss, self, src1, src2, dst),
1598            Some(CpuFeature::SSE42) => sse_fn!(divss, self, Precision::Single, src1, src2, dst),
1599            _ => {}
1600        }
1601        Ok(())
1602    }
1603    fn emit_vdivsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1604        match self.get_simd_arch() {
1605            Some(CpuFeature::AVX) => avx_fn!(vdivsd, self, src1, src2, dst),
1606            Some(CpuFeature::SSE42) => sse_fn!(divsd, self, Precision::Double, src1, src2, dst),
1607            _ => {}
1608        }
1609        Ok(())
1610    }
1611    fn emit_vmaxss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1612        match self.get_simd_arch() {
1613            Some(CpuFeature::AVX) => avx_fn!(vmaxss, self, src1, src2, dst),
1614            Some(CpuFeature::SSE42) => sse_fn!(maxss, self, Precision::Single, src1, src2, dst),
1615            _ => {}
1616        }
1617        Ok(())
1618    }
1619    fn emit_vmaxsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1620        match self.get_simd_arch() {
1621            Some(CpuFeature::AVX) => avx_fn!(vmaxsd, self, src1, src2, dst),
1622            Some(CpuFeature::SSE42) => sse_fn!(maxsd, self, Precision::Double, src1, src2, dst),
1623            _ => {}
1624        }
1625        Ok(())
1626    }
1627    fn emit_vminss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1628        match self.get_simd_arch() {
1629            Some(CpuFeature::AVX) => avx_fn!(vminss, self, src1, src2, dst),
1630            Some(CpuFeature::SSE42) => sse_fn!(minss, self, Precision::Single, src1, src2, dst),
1631            _ => {}
1632        }
1633        Ok(())
1634    }
1635    fn emit_vminsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1636        match self.get_simd_arch() {
1637            Some(CpuFeature::AVX) => avx_fn!(vminsd, self, src1, src2, dst),
1638            Some(CpuFeature::SSE42) => sse_fn!(minsd, self, Precision::Double, src1, src2, dst),
1639            _ => {}
1640        }
1641        Ok(())
1642    }
1643    fn emit_vcmpeqss(
1644        &mut self,
1645        src1: XMM,
1646        src2: XMMOrMemory,
1647        dst: XMM,
1648    ) -> Result<(), CompileError> {
1649        match self.get_simd_arch() {
1650            Some(CpuFeature::AVX) => avx_fn!(vcmpeqss, self, src1, src2, dst),
1651            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 0, self, Precision::Single, src1, src2, dst),
1652            _ => {}
1653        }
1654        Ok(())
1655    }
1656    fn emit_vcmpeqsd(
1657        &mut self,
1658        src1: XMM,
1659        src2: XMMOrMemory,
1660        dst: XMM,
1661    ) -> Result<(), CompileError> {
1662        match self.get_simd_arch() {
1663            Some(CpuFeature::AVX) => avx_fn!(vcmpeqsd, self, src1, src2, dst),
1664            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 0, self, Precision::Double, src1, src2, dst),
1665            _ => {}
1666        }
1667        Ok(())
1668    }
1669    fn emit_vcmpneqss(
1670        &mut self,
1671        src1: XMM,
1672        src2: XMMOrMemory,
1673        dst: XMM,
1674    ) -> Result<(), CompileError> {
1675        match self.get_simd_arch() {
1676            Some(CpuFeature::AVX) => avx_fn!(vcmpneqss, self, src1, src2, dst),
1677            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 4, self, Precision::Single, src1, src2, dst),
1678            _ => {}
1679        }
1680        Ok(())
1681    }
1682    fn emit_vcmpneqsd(
1683        &mut self,
1684        src1: XMM,
1685        src2: XMMOrMemory,
1686        dst: XMM,
1687    ) -> Result<(), CompileError> {
1688        match self.get_simd_arch() {
1689            Some(CpuFeature::AVX) => avx_fn!(vcmpneqsd, self, src1, src2, dst),
1690            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 4, self, Precision::Double, src1, src2, dst),
1691            _ => {}
1692        }
1693        Ok(())
1694    }
1695    fn emit_vcmpltss(
1696        &mut self,
1697        src1: XMM,
1698        src2: XMMOrMemory,
1699        dst: XMM,
1700    ) -> Result<(), CompileError> {
1701        match self.get_simd_arch() {
1702            Some(CpuFeature::AVX) => avx_fn!(vcmpltss, self, src1, src2, dst),
1703            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 1, self, Precision::Single, src1, src2, dst),
1704            _ => {}
1705        }
1706        Ok(())
1707    }
1708    fn emit_vcmpltsd(
1709        &mut self,
1710        src1: XMM,
1711        src2: XMMOrMemory,
1712        dst: XMM,
1713    ) -> Result<(), CompileError> {
1714        match self.get_simd_arch() {
1715            Some(CpuFeature::AVX) => avx_fn!(vcmpltsd, self, src1, src2, dst),
1716            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 1, self, Precision::Double, src1, src2, dst),
1717            _ => {}
1718        }
1719        Ok(())
1720    }
1721    fn emit_vcmpless(
1722        &mut self,
1723        src1: XMM,
1724        src2: XMMOrMemory,
1725        dst: XMM,
1726    ) -> Result<(), CompileError> {
1727        match self.get_simd_arch() {
1728            Some(CpuFeature::AVX) => avx_fn!(vcmpless, self, src1, src2, dst),
1729            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 2, self, Precision::Single, src1, src2, dst),
1730            _ => {}
1731        }
1732        Ok(())
1733    }
1734    fn emit_vcmplesd(
1735        &mut self,
1736        src1: XMM,
1737        src2: XMMOrMemory,
1738        dst: XMM,
1739    ) -> Result<(), CompileError> {
1740        match self.get_simd_arch() {
1741            Some(CpuFeature::AVX) => avx_fn!(vcmplesd, self, src1, src2, dst),
1742            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 2, self, Precision::Double, src1, src2, dst),
1743            _ => {}
1744        }
1745        Ok(())
1746    }
1747    fn emit_vcmpgtss(
1748        &mut self,
1749        src1: XMM,
1750        src2: XMMOrMemory,
1751        dst: XMM,
1752    ) -> Result<(), CompileError> {
1753        match self.get_simd_arch() {
1754            Some(CpuFeature::AVX) => avx_fn!(vcmpgtss, self, src1, src2, dst),
1755            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 6, self, Precision::Single, src1, src2, dst),
1756            _ => {}
1757        }
1758        Ok(())
1759    }
1760    fn emit_vcmpgtsd(
1761        &mut self,
1762        src1: XMM,
1763        src2: XMMOrMemory,
1764        dst: XMM,
1765    ) -> Result<(), CompileError> {
1766        match self.get_simd_arch() {
1767            Some(CpuFeature::AVX) => avx_fn!(vcmpgtsd, self, src1, src2, dst),
1768            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 6, self, Precision::Double, src1, src2, dst),
1769            _ => {}
1770        }
1771        Ok(())
1772    }
1773    fn emit_vcmpgess(
1774        &mut self,
1775        src1: XMM,
1776        src2: XMMOrMemory,
1777        dst: XMM,
1778    ) -> Result<(), CompileError> {
1779        match self.get_simd_arch() {
1780            Some(CpuFeature::AVX) => avx_fn!(vcmpgess, self, src1, src2, dst),
1781            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 5, self, Precision::Single, src1, src2, dst),
1782            _ => {}
1783        }
1784        Ok(())
1785    }
1786    fn emit_vcmpgesd(
1787        &mut self,
1788        src1: XMM,
1789        src2: XMMOrMemory,
1790        dst: XMM,
1791    ) -> Result<(), CompileError> {
1792        match self.get_simd_arch() {
1793            Some(CpuFeature::AVX) => avx_fn!(vcmpgesd, self, src1, src2, dst),
1794            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 5, self, Precision::Double, src1, src2, dst),
1795            _ => {}
1796        }
1797        Ok(())
1798    }
1799    fn emit_vcmpunordss(
1800        &mut self,
1801        src1: XMM,
1802        src2: XMMOrMemory,
1803        dst: XMM,
1804    ) -> Result<(), CompileError> {
1805        match self.get_simd_arch() {
1806            Some(CpuFeature::AVX) => avx_fn!(vcmpunordss, self, src1, src2, dst),
1807            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 3, self, Precision::Single, src1, src2, dst),
1808            _ => {}
1809        }
1810        Ok(())
1811    }
1812    fn emit_vcmpunordsd(
1813        &mut self,
1814        src1: XMM,
1815        src2: XMMOrMemory,
1816        dst: XMM,
1817    ) -> Result<(), CompileError> {
1818        match self.get_simd_arch() {
1819            Some(CpuFeature::AVX) => avx_fn!(vcmpunordsd, self, src1, src2, dst),
1820            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 3, self, Precision::Double, src1, src2, dst),
1821            _ => {}
1822        }
1823        Ok(())
1824    }
1825    fn emit_vcmpordss(
1826        &mut self,
1827        src1: XMM,
1828        src2: XMMOrMemory,
1829        dst: XMM,
1830    ) -> Result<(), CompileError> {
1831        match self.get_simd_arch() {
1832            Some(CpuFeature::AVX) => avx_fn!(vcmpordss, self, src1, src2, dst),
1833            Some(CpuFeature::SSE42) => sse_fn!(cmpss, 7, self, Precision::Single, src1, src2, dst),
1834            _ => {}
1835        }
1836        Ok(())
1837    }
1838    fn emit_vcmpordsd(
1839        &mut self,
1840        src1: XMM,
1841        src2: XMMOrMemory,
1842        dst: XMM,
1843    ) -> Result<(), CompileError> {
1844        match self.get_simd_arch() {
1845            Some(CpuFeature::AVX) => avx_fn!(vcmpordsd, self, src1, src2, dst),
1846            Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 7, self, Precision::Double, src1, src2, dst),
1847            _ => {}
1848        }
1849        Ok(())
1850    }
1851    fn emit_vsqrtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1852        match self.get_simd_arch() {
1853            Some(CpuFeature::AVX) => avx_fn!(vsqrtss, self, src1, src2, dst),
1854            Some(CpuFeature::SSE42) => sse_fn!(sqrtss, self, Precision::Single, src1, src2, dst),
1855            _ => {}
1856        }
1857        Ok(())
1858    }
1859    fn emit_vsqrtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1860        match self.get_simd_arch() {
1861            Some(CpuFeature::AVX) => avx_fn!(vsqrtsd, self, src1, src2, dst),
1862            Some(CpuFeature::SSE42) => sse_fn!(sqrtsd, self, Precision::Double, src1, src2, dst),
1863            _ => {}
1864        }
1865        Ok(())
1866    }
1867    fn emit_vcvtss2sd(
1868        &mut self,
1869        src1: XMM,
1870        src2: XMMOrMemory,
1871        dst: XMM,
1872    ) -> Result<(), CompileError> {
1873        match self.get_simd_arch() {
1874            Some(CpuFeature::AVX) => avx_fn!(vcvtss2sd, self, src1, src2, dst),
1875            Some(CpuFeature::SSE42) => sse_fn!(cvtss2sd, self, Precision::Single, src1, src2, dst),
1876            _ => {}
1877        }
1878        Ok(())
1879    }
1880    fn emit_vcvtsd2ss(
1881        &mut self,
1882        src1: XMM,
1883        src2: XMMOrMemory,
1884        dst: XMM,
1885    ) -> Result<(), CompileError> {
1886        match self.get_simd_arch() {
1887            Some(CpuFeature::AVX) => avx_fn!(vcvtsd2ss, self, src1, src2, dst),
1888            Some(CpuFeature::SSE42) => sse_fn!(cvtsd2ss, self, Precision::Double, src1, src2, dst),
1889            _ => {}
1890        }
1891        Ok(())
1892    }
1893    fn emit_vroundss_nearest(
1894        &mut self,
1895        src1: XMM,
1896        src2: XMMOrMemory,
1897        dst: XMM,
1898    ) -> Result<(), CompileError> {
1899        match self.get_simd_arch() {
1900            Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 0, self, src1, src2, dst),
1901            Some(CpuFeature::SSE42) => {
1902                sse_round_fn!(roundss, 0, self, Precision::Single, src1, src2, dst)
1903            }
1904            _ => {}
1905        }
1906        Ok(())
1907    }
1908    fn emit_vroundsd_nearest(
1909        &mut self,
1910        src1: XMM,
1911        src2: XMMOrMemory,
1912        dst: XMM,
1913    ) -> Result<(), CompileError> {
1914        match self.get_simd_arch() {
1915            Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 0, self, src1, src2, dst),
1916            Some(CpuFeature::SSE42) => {
1917                sse_round_fn!(roundsd, 0, self, Precision::Double, src1, src2, dst)
1918            }
1919            _ => {}
1920        }
1921        Ok(())
1922    }
1923    fn emit_vroundss_floor(
1924        &mut self,
1925        src1: XMM,
1926        src2: XMMOrMemory,
1927        dst: XMM,
1928    ) -> Result<(), CompileError> {
1929        match self.get_simd_arch() {
1930            Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 1, self, src1, src2, dst),
1931            Some(CpuFeature::SSE42) => {
1932                sse_round_fn!(roundss, 1, self, Precision::Single, src1, src2, dst)
1933            }
1934            _ => {}
1935        }
1936        Ok(())
1937    }
1938    fn emit_vroundsd_floor(
1939        &mut self,
1940        src1: XMM,
1941        src2: XMMOrMemory,
1942        dst: XMM,
1943    ) -> Result<(), CompileError> {
1944        match self.get_simd_arch() {
1945            Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 1, self, src1, src2, dst),
1946            Some(CpuFeature::SSE42) => {
1947                sse_round_fn!(roundsd, 1, self, Precision::Double, src1, src2, dst)
1948            }
1949            _ => {}
1950        }
1951        Ok(())
1952    }
1953    fn emit_vroundss_ceil(
1954        &mut self,
1955        src1: XMM,
1956        src2: XMMOrMemory,
1957        dst: XMM,
1958    ) -> Result<(), CompileError> {
1959        match self.get_simd_arch() {
1960            Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 2, self, src1, src2, dst),
1961            Some(CpuFeature::SSE42) => {
1962                sse_round_fn!(roundss, 2, self, Precision::Single, src1, src2, dst)
1963            }
1964            _ => {}
1965        }
1966        Ok(())
1967    }
1968    fn emit_vroundsd_ceil(
1969        &mut self,
1970        src1: XMM,
1971        src2: XMMOrMemory,
1972        dst: XMM,
1973    ) -> Result<(), CompileError> {
1974        match self.get_simd_arch() {
1975            Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 2, self, src1, src2, dst),
1976            Some(CpuFeature::SSE42) => {
1977                sse_round_fn!(roundsd, 2, self, Precision::Double, src1, src2, dst)
1978            }
1979            _ => {}
1980        }
1981        Ok(())
1982    }
1983    fn emit_vroundss_trunc(
1984        &mut self,
1985        src1: XMM,
1986        src2: XMMOrMemory,
1987        dst: XMM,
1988    ) -> Result<(), CompileError> {
1989        match self.get_simd_arch() {
1990            Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 3, self, src1, src2, dst),
1991            Some(CpuFeature::SSE42) => {
1992                sse_round_fn!(roundss, 3, self, Precision::Single, src1, src2, dst)
1993            }
1994            _ => {}
1995        }
1996        Ok(())
1997    }
1998    fn emit_vroundsd_trunc(
1999        &mut self,
2000        src1: XMM,
2001        src2: XMMOrMemory,
2002        dst: XMM,
2003    ) -> Result<(), CompileError> {
2004        match self.get_simd_arch() {
2005            Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 3, self, src1, src2, dst),
2006            Some(CpuFeature::SSE42) => {
2007                sse_round_fn!(roundsd, 3, self, Precision::Double, src1, src2, dst)
2008            }
2009            _ => {}
2010        }
2011        Ok(())
2012    }
2013    fn emit_vcvtsi2ss_32(
2014        &mut self,
2015        src1: XMM,
2016        src2: GPROrMemory,
2017        dst: XMM,
2018    ) -> Result<(), CompileError> {
2019        match self.get_simd_arch() {
2020            Some(CpuFeature::AVX) => avx_i2f_32_fn!(vcvtsi2ss, self, src1, src2, dst),
2021            Some(CpuFeature::SSE42) => {
2022                sse_i2f_32_fn!(cvtsi2ss, self, Precision::Single, src1, src2, dst)
2023            }
2024            _ => {}
2025        }
2026        Ok(())
2027    }
2028    fn emit_vcvtsi2sd_32(
2029        &mut self,
2030        src1: XMM,
2031        src2: GPROrMemory,
2032        dst: XMM,
2033    ) -> Result<(), CompileError> {
2034        match self.get_simd_arch() {
2035            Some(CpuFeature::AVX) => avx_i2f_32_fn!(vcvtsi2sd, self, src1, src2, dst),
2036            Some(CpuFeature::SSE42) => {
2037                sse_i2f_32_fn!(cvtsi2sd, self, Precision::Double, src1, src2, dst)
2038            }
2039            _ => {}
2040        }
2041        Ok(())
2042    }
2043    fn emit_vcvtsi2ss_64(
2044        &mut self,
2045        src1: XMM,
2046        src2: GPROrMemory,
2047        dst: XMM,
2048    ) -> Result<(), CompileError> {
2049        match self.get_simd_arch() {
2050            Some(CpuFeature::AVX) => avx_i2f_64_fn!(vcvtsi2ss, self, src1, src2, dst),
2051            Some(CpuFeature::SSE42) => {
2052                sse_i2f_64_fn!(cvtsi2ss, self, Precision::Single, src1, src2, dst)
2053            }
2054            _ => {}
2055        }
2056        Ok(())
2057    }
2058    fn emit_vcvtsi2sd_64(
2059        &mut self,
2060        src1: XMM,
2061        src2: GPROrMemory,
2062        dst: XMM,
2063    ) -> Result<(), CompileError> {
2064        match self.get_simd_arch() {
2065            Some(CpuFeature::AVX) => avx_i2f_64_fn!(vcvtsi2sd, self, src1, src2, dst),
2066            Some(CpuFeature::SSE42) => {
2067                sse_i2f_64_fn!(cvtsi2sd, self, Precision::Double, src1, src2, dst)
2068            }
2069            _ => {}
2070        }
2071        Ok(())
2072    }
2073
2074    fn emit_vblendvps(
2075        &mut self,
2076        src1: XMM,
2077        src2: XMMOrMemory,
2078        mask: XMM,
2079        dst: XMM,
2080    ) -> Result<(), CompileError> {
2081        // this implementation works only for sse 4.1 and greater
2082        match self.get_simd_arch() {
2083            Some(CpuFeature::AVX) => match src2 {
2084                XMMOrMemory::XMM(src2) => {
2085                    // TODO: this argument order does not match the documentation??
2086                    dynasm!( self; vblendvps Rx(dst), Rx(mask), Rx(src2), Rx(src1))
2087                }
2088                XMMOrMemory::Memory(base, disp) => {
2089                    dynasm!( self; vblendvps Rx(dst), Rx(mask), [Rq(base) + disp], Rx(src1))
2090                }
2091            },
2092            Some(CpuFeature::SSE42) => match src2 {
2093                XMMOrMemory::XMM(src2) => {
2094                    move_src_to_dst(self, Precision::Single, src1, dst);
2095                    dynasm!( self; blendvps Rx(dst), Rx(src2))
2096                }
2097                XMMOrMemory::Memory(base, disp) => {
2098                    move_src_to_dst(self, Precision::Single, src1, dst);
2099                    dynasm!( self; blendvps Rx(dst), [Rq(base) + disp])
2100                }
2101            },
2102            _ => {}
2103        }
2104        Ok(())
2105    }
2106
2107    fn emit_vblendvpd(
2108        &mut self,
2109        src1: XMM,
2110        src2: XMMOrMemory,
2111        mask: XMM,
2112        dst: XMM,
2113    ) -> Result<(), CompileError> {
2114        // this implementation works only for sse 4.1 and greater
2115        match self.get_simd_arch() {
2116            Some(CpuFeature::AVX) => match src2 {
2117                XMMOrMemory::XMM(src2) => {
2118                    // TODO: this argument order does not match the documentation??
2119                    dynasm!( self; vblendvpd Rx(dst), Rx(mask), Rx(src2), Rx(src1))
2120                }
2121                XMMOrMemory::Memory(base, disp) => {
2122                    dynasm!( self; vblendvpd Rx(dst), Rx(mask), [Rq(base) + disp], Rx(src1))
2123                }
2124            },
2125            Some(CpuFeature::SSE42) => match src2 {
2126                XMMOrMemory::XMM(src2) => {
2127                    move_src_to_dst(self, Precision::Double, src1, dst);
2128                    dynasm!( self; blendvpd Rx(dst), Rx(src2))
2129                }
2130                XMMOrMemory::Memory(base, disp) => {
2131                    move_src_to_dst(self, Precision::Double, src1, dst);
2132                    dynasm!( self; blendvpd Rx(dst), [Rq(base) + disp])
2133                }
2134            },
2135            _ => {}
2136        }
2137        Ok(())
2138    }
2139
2140    fn emit_ucomiss(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
2141        match src {
2142            XMMOrMemory::XMM(x) => dynasm!(self ; ucomiss Rx(dst), Rx(x)),
2143            XMMOrMemory::Memory(base, disp) => {
2144                dynasm!(self ; ucomiss Rx(dst), [Rq(base) + disp])
2145            }
2146        }
2147        Ok(())
2148    }
2149
2150    fn emit_ucomisd(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
2151        match src {
2152            XMMOrMemory::XMM(x) => dynasm!(self ; ucomisd Rx(dst), Rx(x)),
2153            XMMOrMemory::Memory(base, disp) => {
2154                dynasm!(self ; ucomisd Rx(dst), [Rq(base) + disp])
2155            }
2156        }
2157        Ok(())
2158    }
2159
2160    fn emit_cvttss2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2161        match src {
2162            XMMOrMemory::XMM(x) => dynasm!(self ; cvttss2si Rd(dst), Rx(x)),
2163            XMMOrMemory::Memory(base, disp) => {
2164                dynasm!(self ; cvttss2si Rd(dst), [Rq(base) + disp])
2165            }
2166        }
2167        Ok(())
2168    }
2169
2170    fn emit_cvttss2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2171        match src {
2172            XMMOrMemory::XMM(x) => dynasm!(self ; cvttss2si Rq(dst), Rx(x)),
2173            XMMOrMemory::Memory(base, disp) => {
2174                dynasm!(self ; cvttss2si Rq(dst), [Rq(base) + disp])
2175            }
2176        }
2177        Ok(())
2178    }
2179
2180    fn emit_cvttsd2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2181        match src {
2182            XMMOrMemory::XMM(x) => dynasm!(self ; cvttsd2si Rd(dst), Rx(x)),
2183            XMMOrMemory::Memory(base, disp) => {
2184                dynasm!(self ; cvttsd2si Rd(dst), [Rq(base) + disp])
2185            }
2186        }
2187        Ok(())
2188    }
2189
2190    fn emit_cvttsd2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2191        match src {
2192            XMMOrMemory::XMM(x) => dynasm!(self ; cvttsd2si Rq(dst), Rx(x)),
2193            XMMOrMemory::Memory(base, disp) => {
2194                dynasm!(self ; cvttsd2si Rq(dst), [Rq(base) + disp])
2195            }
2196        }
2197        Ok(())
2198    }
2199
2200    fn emit_test_gpr_64(&mut self, reg: GPR) -> Result<(), CompileError> {
2201        dynasm!(self ; test Rq(reg), Rq(reg));
2202        Ok(())
2203    }
2204
2205    fn emit_ud2(&mut self) -> Result<(), CompileError> {
2206        dynasm!(self ; ud2);
2207        Ok(())
2208    }
2209    #[allow(clippy::useless_conversion)]
2210    fn emit_ud1_payload(&mut self, payload: u8) -> Result<(), CompileError> {
2211        assert!(payload & 0xf0 == 0);
2212        let reg1 = (payload >> 3) & 1;
2213        let reg2 = payload & 7;
2214
2215        dynasm!(self ; ud1 Rd(reg1), Rd(reg2));
2216        Ok(())
2217    }
2218    fn emit_ret(&mut self) -> Result<(), CompileError> {
2219        dynasm!(self ; ret);
2220        Ok(())
2221    }
2222
2223    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2224        dynasm!(self ; call =>label);
2225        Ok(())
2226    }
2227    fn emit_call_location(&mut self, loc: Location) -> Result<(), CompileError> {
2228        match loc {
2229            Location::GPR(x) => dynasm!(self ; call Rq(x)),
2230            Location::Memory(base, disp) => dynasm!(self ; call QWORD [Rq(base) + disp]),
2231            _ => codegen_error!("singlepass can't emit CALL {:?}", loc),
2232        }
2233        Ok(())
2234    }
2235
2236    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2237        dynasm!(self ; call Rq(reg));
2238        Ok(())
2239    }
2240
2241    fn emit_bkpt(&mut self) -> Result<(), CompileError> {
2242        dynasm!(self ; int3);
2243        Ok(())
2244    }
2245
2246    fn emit_host_redirection(&mut self, target: GPR) -> Result<(), CompileError> {
2247        self.emit_jmp_location(Location::GPR(target))
2248    }
2249
2250    fn arch_mov64_imm_offset(&self) -> Result<usize, CompileError> {
2251        Ok(2)
2252    }
2253}
wasmer_compiler_singlepass/emitter_x64.rs

wasmer_compiler_singlepass/
emitter_x64.rs
