1use crate::{
2 codegen_error, common_decl::Size, location::Location as AbstractLocation,
3 machine_x64::AssemblerX64,
4};
5pub use crate::{
6 location::Multiplier,
7 machine::{Label, Offset},
8 x64_decl::{GPR, XMM},
9};
10use dynasmrt::{AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi};
11use wasmer_types::{CompileError, target::CpuFeature};
12
13macro_rules! dynasm {
18 ($a:expr ; $($tt:tt)*) => {
19 dynasm::dynasm!(
20 $a.inner
21 ; .arch x64
22 ; $($tt)*
23 )
24 };
25}
26
27pub type Location = AbstractLocation<GPR, XMM>;
28
29#[derive(Copy, Clone, Debug, Eq, PartialEq)]
30pub enum Condition {
31 None,
32 Above,
33 AboveEqual,
34 Below,
35 BelowEqual,
36 Greater,
37 GreaterEqual,
38 Less,
39 LessEqual,
40 Equal,
41 NotEqual,
42 Signed,
43 Carry,
44}
45
46#[derive(Copy, Clone, Debug, Eq, PartialEq)]
47#[allow(dead_code, clippy::upper_case_acronyms)]
48pub enum XMMOrMemory {
49 XMM(XMM),
50 Memory(GPR, i32),
51}
52
53#[derive(Copy, Clone, Debug)]
54#[allow(dead_code, clippy::upper_case_acronyms)]
55pub enum GPROrMemory {
56 GPR(GPR),
57 Memory(GPR, i32),
58}
59
60pub enum Precision {
61 Single,
62 Double,
63}
64
65#[allow(unused)]
66pub trait EmitterX64 {
67 fn get_simd_arch(&self) -> Option<&CpuFeature>;
68 fn get_label(&mut self) -> Label;
69 fn get_offset(&self) -> Offset;
70 fn get_jmp_instr_size(&self) -> u8;
71
72 fn finalize_function(&mut self) -> Result<(), CompileError> {
73 Ok(())
74 }
75
76 fn emit_u64(&mut self, x: u64) -> Result<(), CompileError>;
77 fn emit_bytes(&mut self, bytes: &[u8]) -> Result<(), CompileError>;
78
79 fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
80
81 fn emit_nop(&mut self) -> Result<(), CompileError>;
82
83 fn emit_nop_n(&mut self, n: usize) -> Result<(), CompileError>;
86
87 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
88 fn emit_lea(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
89 fn emit_lea_label(&mut self, label: Label, dst: Location) -> Result<(), CompileError>;
90 fn emit_cdq(&mut self) -> Result<(), CompileError>;
91 fn emit_cqo(&mut self) -> Result<(), CompileError>;
92 fn emit_xor(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
93 fn emit_jmp(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
94 fn emit_jmp_location(&mut self, loc: Location) -> Result<(), CompileError>;
95 fn emit_set(&mut self, condition: Condition, dst: GPR) -> Result<(), CompileError>;
96 fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError>;
97 fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError>;
98 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
99 fn emit_add(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
100 fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
101 fn emit_neg(&mut self, sz: Size, value: Location) -> Result<(), CompileError>;
102 fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
103 fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR) -> Result<(), CompileError>;
104 fn emit_div(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError>;
105 fn emit_idiv(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError>;
106 fn emit_shl(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
107 fn emit_shr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
108 fn emit_sar(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
109 fn emit_rol(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
110 fn emit_ror(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
111 fn emit_and(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
112 fn emit_test(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
113 fn emit_or(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
114 fn emit_bsr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
115 fn emit_bsf(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
116 fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
117 fn emit_movzx(
118 &mut self,
119 sz_src: Size,
120 src: Location,
121 sz_dst: Size,
122 dst: Location,
123 ) -> Result<(), CompileError>;
124 fn emit_movsx(
125 &mut self,
126 sz_src: Size,
127 src: Location,
128 sz_dst: Size,
129 dst: Location,
130 ) -> Result<(), CompileError>;
131 fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
132 fn emit_lock_xadd(
133 &mut self,
134 sz: Size,
135 src: Location,
136 dst: Location,
137 ) -> Result<(), CompileError>;
138 fn emit_lock_cmpxchg(
139 &mut self,
140 sz: Size,
141 src: Location,
142 dst: Location,
143 ) -> Result<(), CompileError>;
144 fn emit_rep_stosq(&mut self) -> Result<(), CompileError>;
145
146 fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR) -> Result<(), CompileError>;
147 fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR) -> Result<(), CompileError>;
148
149 fn emit_cmovae_gpr_32(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError>;
150 fn emit_cmovae_gpr_64(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError>;
151
152 fn emit_vmovaps(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError>;
153 fn emit_vmovapd(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError>;
154 fn emit_vxorps(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
155 fn emit_vxorpd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
156
157 fn emit_vaddss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
158 fn emit_vaddsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
159 fn emit_vsubss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
160 fn emit_vsubsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
161 fn emit_vmulss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
162 fn emit_vmulsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
163 fn emit_vdivss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
164 fn emit_vdivsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
165 fn emit_vmaxss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
166 fn emit_vmaxsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
167 fn emit_vminss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
168 fn emit_vminsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
169
170 fn emit_vcmpeqss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
171 -> Result<(), CompileError>;
172 fn emit_vcmpeqsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
173 -> Result<(), CompileError>;
174
175 fn emit_vcmpneqss(
176 &mut self,
177 src1: XMM,
178 src2: XMMOrMemory,
179 dst: XMM,
180 ) -> Result<(), CompileError>;
181 fn emit_vcmpneqsd(
182 &mut self,
183 src1: XMM,
184 src2: XMMOrMemory,
185 dst: XMM,
186 ) -> Result<(), CompileError>;
187
188 fn emit_vcmpltss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
189 -> Result<(), CompileError>;
190 fn emit_vcmpltsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
191 -> Result<(), CompileError>;
192
193 fn emit_vcmpless(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
194 -> Result<(), CompileError>;
195 fn emit_vcmplesd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
196 -> Result<(), CompileError>;
197
198 fn emit_vcmpgtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
199 -> Result<(), CompileError>;
200 fn emit_vcmpgtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
201 -> Result<(), CompileError>;
202
203 fn emit_vcmpgess(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
204 -> Result<(), CompileError>;
205 fn emit_vcmpgesd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM)
206 -> Result<(), CompileError>;
207
208 fn emit_vcmpunordss(
209 &mut self,
210 src1: XMM,
211 src2: XMMOrMemory,
212 dst: XMM,
213 ) -> Result<(), CompileError>;
214 fn emit_vcmpunordsd(
215 &mut self,
216 src1: XMM,
217 src2: XMMOrMemory,
218 dst: XMM,
219 ) -> Result<(), CompileError>;
220
221 fn emit_vcmpordss(
222 &mut self,
223 src1: XMM,
224 src2: XMMOrMemory,
225 dst: XMM,
226 ) -> Result<(), CompileError>;
227 fn emit_vcmpordsd(
228 &mut self,
229 src1: XMM,
230 src2: XMMOrMemory,
231 dst: XMM,
232 ) -> Result<(), CompileError>;
233
234 fn emit_vsqrtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
235 fn emit_vsqrtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
236
237 fn emit_vroundss_nearest(
238 &mut self,
239 src1: XMM,
240 src2: XMMOrMemory,
241 dst: XMM,
242 ) -> Result<(), CompileError>;
243 fn emit_vroundss_floor(
244 &mut self,
245 src1: XMM,
246 src2: XMMOrMemory,
247 dst: XMM,
248 ) -> Result<(), CompileError>;
249 fn emit_vroundss_ceil(
250 &mut self,
251 src1: XMM,
252 src2: XMMOrMemory,
253 dst: XMM,
254 ) -> Result<(), CompileError>;
255 fn emit_vroundss_trunc(
256 &mut self,
257 src1: XMM,
258 src2: XMMOrMemory,
259 dst: XMM,
260 ) -> Result<(), CompileError>;
261 fn emit_vroundsd_nearest(
262 &mut self,
263 src1: XMM,
264 src2: XMMOrMemory,
265 dst: XMM,
266 ) -> Result<(), CompileError>;
267 fn emit_vroundsd_floor(
268 &mut self,
269 src1: XMM,
270 src2: XMMOrMemory,
271 dst: XMM,
272 ) -> Result<(), CompileError>;
273 fn emit_vroundsd_ceil(
274 &mut self,
275 src1: XMM,
276 src2: XMMOrMemory,
277 dst: XMM,
278 ) -> Result<(), CompileError>;
279 fn emit_vroundsd_trunc(
280 &mut self,
281 src1: XMM,
282 src2: XMMOrMemory,
283 dst: XMM,
284 ) -> Result<(), CompileError>;
285
286 fn emit_vcvtss2sd(
287 &mut self,
288 src1: XMM,
289 src2: XMMOrMemory,
290 dst: XMM,
291 ) -> Result<(), CompileError>;
292 fn emit_vcvtsd2ss(
293 &mut self,
294 src1: XMM,
295 src2: XMMOrMemory,
296 dst: XMM,
297 ) -> Result<(), CompileError>;
298
299 fn emit_ucomiss(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
300 fn emit_ucomisd(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError>;
301
302 fn emit_cvttss2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
303 fn emit_cvttss2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
304 fn emit_cvttsd2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
305 fn emit_cvttsd2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError>;
306
307 fn emit_vcvtsi2ss_32(
308 &mut self,
309 src1: XMM,
310 src2: GPROrMemory,
311 dst: XMM,
312 ) -> Result<(), CompileError>;
313 fn emit_vcvtsi2ss_64(
314 &mut self,
315 src1: XMM,
316 src2: GPROrMemory,
317 dst: XMM,
318 ) -> Result<(), CompileError>;
319 fn emit_vcvtsi2sd_32(
320 &mut self,
321 src1: XMM,
322 src2: GPROrMemory,
323 dst: XMM,
324 ) -> Result<(), CompileError>;
325 fn emit_vcvtsi2sd_64(
326 &mut self,
327 src1: XMM,
328 src2: GPROrMemory,
329 dst: XMM,
330 ) -> Result<(), CompileError>;
331
332 fn emit_vblendvps(
333 &mut self,
334 src1: XMM,
335 src2: XMMOrMemory,
336 mask: XMM,
337 dst: XMM,
338 ) -> Result<(), CompileError>;
339 fn emit_vblendvpd(
340 &mut self,
341 src1: XMM,
342 src2: XMMOrMemory,
343 mask: XMM,
344 dst: XMM,
345 ) -> Result<(), CompileError>;
346
347 fn emit_test_gpr_64(&mut self, reg: GPR) -> Result<(), CompileError>;
348
349 fn emit_ud2(&mut self) -> Result<(), CompileError>;
350 fn emit_ud1_payload(&mut self, payload: u8) -> Result<(), CompileError>;
351 fn emit_ret(&mut self) -> Result<(), CompileError>;
352 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
353 fn emit_call_location(&mut self, loc: Location) -> Result<(), CompileError>;
354
355 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
356
357 fn emit_bkpt(&mut self) -> Result<(), CompileError>;
358
359 fn emit_host_redirection(&mut self, target: GPR) -> Result<(), CompileError>;
360
361 fn arch_has_itruncf(&self) -> bool {
362 false
363 }
364 fn arch_emit_i32_trunc_sf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
365 codegen_error!("singplepass arch_emit_i32_trunc_sf32 unimplemented")
366 }
367 fn arch_emit_i32_trunc_sf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
368 codegen_error!("singplepass arch_emit_i32_trunc_sf64 unimplemented")
369 }
370 fn arch_emit_i32_trunc_uf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
371 codegen_error!("singplepass arch_emit_i32_trunc_uf32 unimplemented")
372 }
373 fn arch_emit_i32_trunc_uf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
374 codegen_error!("singplepass arch_emit_i32_trunc_uf64 unimplemented")
375 }
376 fn arch_emit_i64_trunc_sf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
377 codegen_error!("singplepass arch_emit_i64_trunc_sf32 unimplemented")
378 }
379 fn arch_emit_i64_trunc_sf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
380 codegen_error!("singplepass arch_emit_i64_trunc_sf64 unimplemented")
381 }
382 fn arch_emit_i64_trunc_uf32(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
383 codegen_error!("singplepass arch_emit_i64_trunc_uf32 unimplemented")
384 }
385 fn arch_emit_i64_trunc_uf64(&mut self, _src: XMM, _dst: GPR) -> Result<(), CompileError> {
386 codegen_error!("singplepass arch_emit_i64_trunc_uf64 unimplemented")
387 }
388
389 fn arch_has_fconverti(&self) -> bool {
390 false
391 }
392 fn arch_emit_f32_convert_si32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
393 codegen_error!("singlepass arch_emit_f32_convert_si32 unimplemented")
394 }
395 fn arch_emit_f32_convert_si64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
396 codegen_error!("singlepass arch_emit_f32_convert_si64 unimplemented")
397 }
398 fn arch_emit_f32_convert_ui32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
399 codegen_error!("singlepass arch_emit_f32_convert_ui32 unimplemented")
400 }
401 fn arch_emit_f32_convert_ui64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
402 codegen_error!("singlepass arch_emit_f32_convert_ui64 unimplemented")
403 }
404 fn arch_emit_f64_convert_si32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
405 codegen_error!("singlepass arch_emit_f64_convert_si32 unimplemented")
406 }
407 fn arch_emit_f64_convert_si64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
408 codegen_error!("singlepass arch_emit_f64_convert_si64 unimplemented")
409 }
410 fn arch_emit_f64_convert_ui32(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
411 codegen_error!("singlepass arch_emit_f64_convert_ui32 unimplemented")
412 }
413 fn arch_emit_f64_convert_ui64(&mut self, _src: GPR, _dst: XMM) -> Result<(), CompileError> {
414 codegen_error!("singlepass arch_emit_f64_convert_ui64 unimplemented")
415 }
416
417 fn arch_has_fneg(&self) -> bool {
418 false
419 }
420 fn arch_emit_f32_neg(&mut self, _src: XMM, _dst: XMM) -> Result<(), CompileError> {
421 codegen_error!("singlepass arch_emit_f32_neg unimplemented")
422 }
423 fn arch_emit_f64_neg(&mut self, _src: XMM, _dst: XMM) -> Result<(), CompileError> {
424 codegen_error!("singlepass arch_emit_f64_neg unimplemented")
425 }
426
427 fn arch_has_xzcnt(&self) -> bool {
428 false
429 }
430 fn arch_emit_lzcnt(
431 &mut self,
432 _sz: Size,
433 _src: Location,
434 _dst: Location,
435 ) -> Result<(), CompileError> {
436 codegen_error!("singlepass arch_emit_lzcnt unimplemented")
437 }
438 fn arch_emit_tzcnt(
439 &mut self,
440 _sz: Size,
441 _src: Location,
442 _dst: Location,
443 ) -> Result<(), CompileError> {
444 codegen_error!("singlepass arch_emit_tzcnt unimplemented")
445 }
446
447 fn arch_emit_indirect_call_with_trampoline(
448 &mut self,
449 _loc: Location,
450 ) -> Result<(), CompileError> {
451 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
452 }
453
454 fn arch_emit_entry_trampoline(&mut self) -> Result<(), CompileError> {
456 Ok(())
457 }
458
459 fn arch_mov64_imm_offset(&self) -> Result<usize, CompileError> {
462 codegen_error!("singlepass arch_mov64_imm_offset unimplemented")
463 }
464}
465
466macro_rules! unop_gpr {
467 ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => {
468 match ($sz, $loc) {
469 (Size::S32, Location::GPR(loc)) => {
470 dynasm!($assembler ; $ins Rd(loc));
471 },
472 (Size::S64, Location::GPR(loc)) => {
473 dynasm!($assembler ; $ins Rq(loc));
474 },
475 _ => $otherwise
476 }
477 };
478}
479
480macro_rules! unop_mem {
481 ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => {
482 match ($sz, $loc) {
483 (Size::S32, Location::Memory(loc, disp)) => {
484 dynasm!($assembler ; $ins DWORD [Rq(loc) + disp] );
485 },
486 (Size::S64, Location::Memory(loc, disp)) => {
487 dynasm!($assembler ; $ins QWORD [Rq(loc) + disp] );
488 },
489 _ => $otherwise
490 }
491 };
492}
493
494macro_rules! unop_gpr_or_mem {
495 ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => {
496 unop_gpr!($ins, $assembler, $sz, $loc, {
497 unop_mem!($ins, $assembler, $sz, $loc, $otherwise)
498 })
499 };
500}
501
502macro_rules! binop_imm32_gpr {
503 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
504 match ($sz, $src, $dst) {
505 (Size::S32, Location::Imm32(src), Location::GPR(dst)) => {
506 dynasm!($assembler ; $ins Rd(dst), src as i32); },
508 (Size::S64, Location::Imm32(src), Location::GPR(dst)) => {
509 dynasm!($assembler ; $ins Rq(dst), src as i32); },
511 _ => $otherwise
512 }
513 };
514}
515
516macro_rules! binop_imm32_mem {
517 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
518 match ($sz, $src, $dst) {
519 (Size::S32, Location::Imm32(src), Location::Memory(dst, disp)) => {
520 dynasm!($assembler ; $ins DWORD [Rq(dst) + disp], src as i32);
521 },
522 (Size::S64, Location::Imm32(src), Location::Memory(dst, disp)) => {
523 dynasm!($assembler ; $ins QWORD [Rq(dst) + disp], src as i32);
524 },
525 _ => $otherwise
526 }
527 };
528}
529
530macro_rules! binop_imm64_gpr {
531 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
532 match ($sz, $src, $dst) {
533 (Size::S64, Location::Imm64(src), Location::GPR(dst)) => {
534 dynasm!($assembler ; $ins Rq(dst), QWORD src as i64); },
536 _ => $otherwise
537 }
538 };
539}
540
541macro_rules! binop_gpr_gpr {
542 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
543 match ($sz, $src, $dst) {
544 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
545 dynasm!($assembler ; $ins Rd(dst), Rd(src)); },
547 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
548 dynasm!($assembler ; $ins Rq(dst), Rq(src)); },
550 _ => $otherwise
551 }
552 };
553}
554
555macro_rules! binop_gpr_mem {
556 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
557 match ($sz, $src, $dst) {
558 (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
559 dynasm!($assembler ; $ins [Rq(dst) + disp], Rd(src)); },
561 (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
562 dynasm!($assembler ; $ins [Rq(dst) + disp], Rq(src)); },
564 _ => $otherwise
565 }
566 };
567}
568
569macro_rules! binop_mem_gpr {
570 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
571 match ($sz, $src, $dst) {
572 (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => {
573 dynasm!($assembler ; $ins Rd(dst), [Rq(src) + disp]); },
575 (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => {
576 dynasm!($assembler ; $ins Rq(dst), [Rq(src) + disp]); },
578 _ => $otherwise
579 }
580 };
581}
582
583macro_rules! binop_all_nofp {
584 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
585 binop_imm32_gpr!($ins, $assembler, $sz, $src, $dst, {
586 binop_imm32_mem!($ins, $assembler, $sz, $src, $dst, {
587 binop_gpr_gpr!($ins, $assembler, $sz, $src, $dst, {
588 binop_gpr_mem!($ins, $assembler, $sz, $src, $dst, {
589 binop_mem_gpr!($ins, $assembler, $sz, $src, $dst, $otherwise)
590 })
591 })
592 })
593 })
594 };
595}
596
597macro_rules! binop_shift {
598 ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => {
599 match ($sz, $src, $dst) {
600 (Size::S32, Location::GPR(GPR::RCX), Location::GPR(dst)) => {
601 dynasm!($assembler ; $ins Rd(dst), cl);
602 },
603 (Size::S32, Location::GPR(GPR::RCX), Location::Memory(dst, disp)) => {
604 dynasm!($assembler ; $ins DWORD [Rq(dst) + disp], cl);
605 },
606 (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
607 dynasm!($assembler ; $ins Rd(dst), imm as i8);
608 },
609 (Size::S32, Location::Imm8(imm), Location::Memory(dst, disp)) => {
610 dynasm!($assembler ; $ins DWORD [Rq(dst) + disp], imm as i8);
611 },
612 (Size::S64, Location::GPR(GPR::RCX), Location::GPR(dst)) => {
613 dynasm!($assembler ; $ins Rq(dst), cl);
614 },
615 (Size::S64, Location::GPR(GPR::RCX), Location::Memory(dst, disp)) => {
616 dynasm!($assembler ; $ins QWORD [Rq(dst) + disp], cl);
617 },
618 (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
619 dynasm!($assembler ; $ins Rq(dst), imm as i8);
620 },
621 (Size::S64, Location::Imm8(imm), Location::Memory(dst, disp)) => {
622 dynasm!($assembler ; $ins QWORD [Rq(dst) + disp], imm as i8);
623 },
624 _ => $otherwise
625 }
626 }
627}
628
629macro_rules! jmp_op {
630 ($ins:ident, $assembler:tt, $label:ident) => {
631 dynasm!($assembler ; $ins =>$label)
632 }
633}
634
635fn move_src_to_dst(emitter: &mut AssemblerX64, precision: Precision, src: XMM, dst: XMM) {
642 if src == dst {
643 return;
644 }
645 match precision {
646 Precision::Single => match src {
647 XMM::XMM0 => dynasm!(emitter ; movss Rx(dst), xmm0),
648 XMM::XMM1 => dynasm!(emitter ; movss Rx(dst), xmm1),
649 XMM::XMM2 => dynasm!(emitter ; movss Rx(dst), xmm2),
650 XMM::XMM3 => dynasm!(emitter ; movss Rx(dst), xmm3),
651 XMM::XMM4 => dynasm!(emitter ; movss Rx(dst), xmm4),
652 XMM::XMM5 => dynasm!(emitter ; movss Rx(dst), xmm5),
653 XMM::XMM6 => dynasm!(emitter ; movss Rx(dst), xmm6),
654 XMM::XMM7 => dynasm!(emitter ; movss Rx(dst), xmm7),
655 XMM::XMM8 => dynasm!(emitter ; movss Rx(dst), xmm8),
656 XMM::XMM9 => dynasm!(emitter ; movss Rx(dst), xmm9),
657 XMM::XMM10 => dynasm!(emitter ; movss Rx(dst), xmm10),
658 XMM::XMM11 => dynasm!(emitter ; movss Rx(dst), xmm11),
659 XMM::XMM12 => dynasm!(emitter ; movss Rx(dst), xmm12),
660 XMM::XMM13 => dynasm!(emitter ; movss Rx(dst), xmm13),
661 XMM::XMM14 => dynasm!(emitter ; movss Rx(dst), xmm14),
662 XMM::XMM15 => dynasm!(emitter ; movss Rx(dst), xmm15),
663 },
664 Precision::Double => match src {
665 XMM::XMM0 => dynasm!(emitter ; movsd Rx(dst), xmm0),
666 XMM::XMM1 => dynasm!(emitter ; movsd Rx(dst), xmm1),
667 XMM::XMM2 => dynasm!(emitter ; movsd Rx(dst), xmm2),
668 XMM::XMM3 => dynasm!(emitter ; movsd Rx(dst), xmm3),
669 XMM::XMM4 => dynasm!(emitter ; movsd Rx(dst), xmm4),
670 XMM::XMM5 => dynasm!(emitter ; movsd Rx(dst), xmm5),
671 XMM::XMM6 => dynasm!(emitter ; movsd Rx(dst), xmm6),
672 XMM::XMM7 => dynasm!(emitter ; movsd Rx(dst), xmm7),
673 XMM::XMM8 => dynasm!(emitter ; movsd Rx(dst), xmm8),
674 XMM::XMM9 => dynasm!(emitter ; movsd Rx(dst), xmm9),
675 XMM::XMM10 => dynasm!(emitter ; movsd Rx(dst), xmm10),
676 XMM::XMM11 => dynasm!(emitter ; movsd Rx(dst), xmm11),
677 XMM::XMM12 => dynasm!(emitter ; movsd Rx(dst), xmm12),
678 XMM::XMM13 => dynasm!(emitter ; movsd Rx(dst), xmm13),
679 XMM::XMM14 => dynasm!(emitter ; movsd Rx(dst), xmm14),
680 XMM::XMM15 => dynasm!(emitter ; movsd Rx(dst), xmm15),
681 },
682 }
683}
684
685macro_rules! avx_fn {
686 ($ins:ident, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
687 match $src2 {
689 XMMOrMemory::XMM(x) => match $src1 {
690 XMM::XMM0 => dynasm!($emitter ; $ins Rx($dst), xmm0, Rx(x)),
691 XMM::XMM1 => dynasm!($emitter ; $ins Rx($dst), xmm1, Rx(x)),
692 XMM::XMM2 => dynasm!($emitter ; $ins Rx($dst), xmm2, Rx(x)),
693 XMM::XMM4 => dynasm!($emitter ; $ins Rx($dst), xmm4, Rx(x)),
694 XMM::XMM3 => dynasm!($emitter ; $ins Rx($dst), xmm3, Rx(x)),
695 XMM::XMM5 => dynasm!($emitter ; $ins Rx($dst), xmm5, Rx(x)),
696 XMM::XMM6 => dynasm!($emitter ; $ins Rx($dst), xmm6, Rx(x)),
697 XMM::XMM7 => dynasm!($emitter ; $ins Rx($dst), xmm7, Rx(x)),
698 XMM::XMM8 => dynasm!($emitter ; $ins Rx($dst), xmm8, Rx(x)),
699 XMM::XMM9 => dynasm!($emitter ; $ins Rx($dst), xmm9, Rx(x)),
700 XMM::XMM10 => dynasm!($emitter ; $ins Rx($dst), xmm10, Rx(x)),
701 XMM::XMM11 => dynasm!($emitter ; $ins Rx($dst), xmm11, Rx(x)),
702 XMM::XMM12 => dynasm!($emitter ; $ins Rx($dst), xmm12, Rx(x)),
703 XMM::XMM13 => dynasm!($emitter ; $ins Rx($dst), xmm13, Rx(x)),
704 XMM::XMM14 => dynasm!($emitter ; $ins Rx($dst), xmm14, Rx(x)),
705 XMM::XMM15 => dynasm!($emitter ; $ins Rx($dst), xmm15, Rx(x)),
706 },
707 XMMOrMemory::Memory(base, disp) => match $src1 {
708 XMM::XMM0 => dynasm!($emitter ; $ins Rx($dst), xmm0, [Rq(base) + disp]),
709 XMM::XMM1 => dynasm!($emitter ; $ins Rx($dst), xmm1, [Rq(base) + disp]),
710 XMM::XMM2 => dynasm!($emitter ; $ins Rx($dst), xmm2, [Rq(base) + disp]),
711 XMM::XMM3 => dynasm!($emitter ; $ins Rx($dst), xmm3, [Rq(base) + disp]),
712 XMM::XMM4 => dynasm!($emitter ; $ins Rx($dst), xmm4, [Rq(base) + disp]),
713 XMM::XMM5 => dynasm!($emitter ; $ins Rx($dst), xmm5, [Rq(base) + disp]),
714 XMM::XMM6 => dynasm!($emitter ; $ins Rx($dst), xmm6, [Rq(base) + disp]),
715 XMM::XMM7 => dynasm!($emitter ; $ins Rx($dst), xmm7, [Rq(base) + disp]),
716 XMM::XMM8 => dynasm!($emitter ; $ins Rx($dst), xmm8, [Rq(base) + disp]),
717 XMM::XMM9 => dynasm!($emitter ; $ins Rx($dst), xmm9, [Rq(base) + disp]),
718 XMM::XMM10 => dynasm!($emitter ; $ins Rx($dst), xmm10, [Rq(base) + disp]),
719 XMM::XMM11 => dynasm!($emitter ; $ins Rx($dst), xmm11, [Rq(base) + disp]),
720 XMM::XMM12 => dynasm!($emitter ; $ins Rx($dst), xmm12, [Rq(base) + disp]),
721 XMM::XMM13 => dynasm!($emitter ; $ins Rx($dst), xmm13, [Rq(base) + disp]),
722 XMM::XMM14 => dynasm!($emitter ; $ins Rx($dst), xmm14, [Rq(base) + disp]),
723 XMM::XMM15 => dynasm!($emitter ; $ins Rx($dst), xmm15, [Rq(base) + disp]),
724 },
725 }
726 }
727}
728
729macro_rules! sse_fn {
730 ($ins:ident, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
731 match $src2 {
732 XMMOrMemory::XMM(x) => {
733 if x == $dst {
734 dynasm!($emitter ; $ins Rx($dst), Rx($src1))
735 } else {
736 move_src_to_dst($emitter, $precision, $src1, $dst);
737 dynasm!($emitter ; $ins Rx($dst), Rx(x))
738 }
739 }
740 XMMOrMemory::Memory(base, disp) => {
741 move_src_to_dst($emitter, $precision, $src1, $dst);
742 dynasm!($emitter ; $ins Rx($dst), [Rq(base) + disp])
743 }
744 }
745 };
746 ($ins:ident, $mode:expr, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
747 match $src2 {
748 XMMOrMemory::XMM(x) => {
749 move_src_to_dst($emitter, $precision, $src1, $dst);
750 dynasm!($emitter ; $ins Rx($dst), Rx(x), $mode)
751 }
752 XMMOrMemory::Memory(base, disp) => {
753 move_src_to_dst($emitter, $precision, $src1, $dst);
754 dynasm!($emitter ; $ins Rx($dst), [Rq(base) + disp], $mode)
755 }
756 }
757 };
758}
759
760macro_rules! avx_i2f_64_fn {
761 ($ins:ident, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
762 match $src2 {
763 GPROrMemory::GPR(x) => match $src1 {
764 XMM::XMM0 => dynasm!($emitter ; $ins Rx($dst), xmm0, Rq(x)),
765 XMM::XMM1 => dynasm!($emitter ; $ins Rx($dst), xmm1, Rq(x)),
766 XMM::XMM2 => dynasm!($emitter ; $ins Rx($dst), xmm2, Rq(x)),
767 XMM::XMM3 => dynasm!($emitter ; $ins Rx($dst), xmm3, Rq(x)),
768 XMM::XMM4 => dynasm!($emitter ; $ins Rx($dst), xmm4, Rq(x)),
769 XMM::XMM5 => dynasm!($emitter ; $ins Rx($dst), xmm5, Rq(x)),
770 XMM::XMM6 => dynasm!($emitter ; $ins Rx($dst), xmm6, Rq(x)),
771 XMM::XMM7 => dynasm!($emitter ; $ins Rx($dst), xmm7, Rq(x)),
772 XMM::XMM8 => dynasm!($emitter ; $ins Rx($dst), xmm8, Rq(x)),
773 XMM::XMM9 => dynasm!($emitter ; $ins Rx($dst), xmm9, Rq(x)),
774 XMM::XMM10 => dynasm!($emitter ; $ins Rx($dst), xmm10, Rq(x)),
775 XMM::XMM11 => dynasm!($emitter ; $ins Rx($dst), xmm11, Rq(x)),
776 XMM::XMM12 => dynasm!($emitter ; $ins Rx($dst), xmm12, Rq(x)),
777 XMM::XMM13 => dynasm!($emitter ; $ins Rx($dst), xmm13, Rq(x)),
778 XMM::XMM14 => dynasm!($emitter ; $ins Rx($dst), xmm14, Rq(x)),
779 XMM::XMM15 => dynasm!($emitter ; $ins Rx($dst), xmm15, Rq(x)),
780 },
781 GPROrMemory::Memory(base, disp) => match $src1 {
782 XMM::XMM0 => dynasm!($emitter ; $ins Rx($dst), xmm0, QWORD [Rq(base) + disp]),
783 XMM::XMM1 => dynasm!($emitter ; $ins Rx($dst), xmm1, QWORD [Rq(base) + disp]),
784 XMM::XMM2 => dynasm!($emitter ; $ins Rx($dst), xmm2, QWORD [Rq(base) + disp]),
785 XMM::XMM3 => dynasm!($emitter ; $ins Rx($dst), xmm3, QWORD [Rq(base) + disp]),
786 XMM::XMM4 => dynasm!($emitter ; $ins Rx($dst), xmm4, QWORD [Rq(base) + disp]),
787 XMM::XMM5 => dynasm!($emitter ; $ins Rx($dst), xmm5, QWORD [Rq(base) + disp]),
788 XMM::XMM6 => dynasm!($emitter ; $ins Rx($dst), xmm6, QWORD [Rq(base) + disp]),
789 XMM::XMM7 => dynasm!($emitter ; $ins Rx($dst), xmm7, QWORD [Rq(base) + disp]),
790 XMM::XMM8 => dynasm!($emitter ; $ins Rx($dst), xmm8, QWORD [Rq(base) + disp]),
791 XMM::XMM9 => dynasm!($emitter ; $ins Rx($dst), xmm9, QWORD [Rq(base) + disp]),
792 XMM::XMM10 => dynasm!($emitter ; $ins Rx($dst), xmm10, QWORD [Rq(base) + disp]),
793 XMM::XMM11 => dynasm!($emitter ; $ins Rx($dst), xmm11, QWORD [Rq(base) + disp]),
794 XMM::XMM12 => dynasm!($emitter ; $ins Rx($dst), xmm12, QWORD [Rq(base) + disp]),
795 XMM::XMM13 => dynasm!($emitter ; $ins Rx($dst), xmm13, QWORD [Rq(base) + disp]),
796 XMM::XMM14 => dynasm!($emitter ; $ins Rx($dst), xmm14, QWORD [Rq(base) + disp]),
797 XMM::XMM15 => dynasm!($emitter ; $ins Rx($dst), xmm15, QWORD [Rq(base) + disp]),
798 },
799 }
800 }
801}
802
803macro_rules! sse_i2f_64_fn {
804 ($ins:ident, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
805 match $src2 {
806 GPROrMemory::GPR(x) => {
807 move_src_to_dst($emitter, $precision, $src1, $dst);
808 dynasm!($emitter ; $ins Rx($dst), Rq(x))
809 },
810 GPROrMemory::Memory(base, disp) => {
811 move_src_to_dst($emitter, $precision, $src1, $dst);
812 dynasm!($emitter ; $ins Rx($dst), QWORD [Rq(base) + disp])
813 }
814 }
815 }
816}
817
818macro_rules! avx_i2f_32_fn {
819 ($ins:ident, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
820 match $src2 {
821 GPROrMemory::GPR(x) => match $src1 {
822 XMM::XMM0 => dynasm!($emitter ; $ins Rx($dst), xmm0, Rd(x)),
823 XMM::XMM1 => dynasm!($emitter ; $ins Rx($dst), xmm1, Rd(x)),
824 XMM::XMM2 => dynasm!($emitter ; $ins Rx($dst), xmm2, Rd(x)),
825 XMM::XMM3 => dynasm!($emitter ; $ins Rx($dst), xmm3, Rd(x)),
826 XMM::XMM4 => dynasm!($emitter ; $ins Rx($dst), xmm4, Rd(x)),
827 XMM::XMM5 => dynasm!($emitter ; $ins Rx($dst), xmm5, Rd(x)),
828 XMM::XMM6 => dynasm!($emitter ; $ins Rx($dst), xmm6, Rd(x)),
829 XMM::XMM7 => dynasm!($emitter ; $ins Rx($dst), xmm7, Rd(x)),
830 XMM::XMM8 => dynasm!($emitter ; $ins Rx($dst), xmm8, Rd(x)),
831 XMM::XMM9 => dynasm!($emitter ; $ins Rx($dst), xmm9, Rd(x)),
832 XMM::XMM10 => dynasm!($emitter ; $ins Rx($dst), xmm10, Rd(x)),
833 XMM::XMM11 => dynasm!($emitter ; $ins Rx($dst), xmm11, Rd(x)),
834 XMM::XMM12 => dynasm!($emitter ; $ins Rx($dst), xmm12, Rd(x)),
835 XMM::XMM13 => dynasm!($emitter ; $ins Rx($dst), xmm13, Rd(x)),
836 XMM::XMM14 => dynasm!($emitter ; $ins Rx($dst), xmm14, Rd(x)),
837 XMM::XMM15 => dynasm!($emitter ; $ins Rx($dst), xmm15, Rd(x)),
838 },
839 GPROrMemory::Memory(base, disp) => match $src1 {
840 XMM::XMM0 => dynasm!($emitter ; $ins Rx($dst), xmm0, DWORD [Rq(base) + disp]),
841 XMM::XMM1 => dynasm!($emitter ; $ins Rx($dst), xmm1, DWORD [Rq(base) + disp]),
842 XMM::XMM2 => dynasm!($emitter ; $ins Rx($dst), xmm2, DWORD [Rq(base) + disp]),
843 XMM::XMM3 => dynasm!($emitter ; $ins Rx($dst), xmm3, DWORD [Rq(base) + disp]),
844 XMM::XMM4 => dynasm!($emitter ; $ins Rx($dst), xmm4, DWORD [Rq(base) + disp]),
845 XMM::XMM5 => dynasm!($emitter ; $ins Rx($dst), xmm5, DWORD [Rq(base) + disp]),
846 XMM::XMM6 => dynasm!($emitter ; $ins Rx($dst), xmm6, DWORD [Rq(base) + disp]),
847 XMM::XMM7 => dynasm!($emitter ; $ins Rx($dst), xmm7, DWORD [Rq(base) + disp]),
848 XMM::XMM8 => dynasm!($emitter ; $ins Rx($dst), xmm8, DWORD [Rq(base) + disp]),
849 XMM::XMM9 => dynasm!($emitter ; $ins Rx($dst), xmm9, DWORD [Rq(base) + disp]),
850 XMM::XMM10 => dynasm!($emitter ; $ins Rx($dst), xmm10, DWORD [Rq(base) + disp]),
851 XMM::XMM11 => dynasm!($emitter ; $ins Rx($dst), xmm11, DWORD [Rq(base) + disp]),
852 XMM::XMM12 => dynasm!($emitter ; $ins Rx($dst), xmm12, DWORD [Rq(base) + disp]),
853 XMM::XMM13 => dynasm!($emitter ; $ins Rx($dst), xmm13, DWORD [Rq(base) + disp]),
854 XMM::XMM14 => dynasm!($emitter ; $ins Rx($dst), xmm14, DWORD [Rq(base) + disp]),
855 XMM::XMM15 => dynasm!($emitter ; $ins Rx($dst), xmm15, DWORD [Rq(base) + disp]),
856 },
857 }
858 }
859}
860
861macro_rules! sse_i2f_32_fn {
862 ($ins:ident, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
863 match $src2 {
864 GPROrMemory::GPR(x) => {
865 move_src_to_dst($emitter, $precision, $src1, $dst);
866 dynasm!($emitter; $ins Rx($src1), Rd(x))
867 },
868 GPROrMemory::Memory(base, disp) => {
869 move_src_to_dst($emitter, $precision, $src1, $dst);
870 dynasm!($emitter; $ins Rx($dst), DWORD [Rq(base) + disp])
871 }
872 }
873 }
874}
875
876macro_rules! avx_round_fn {
877 ($ins:ident, $mode:expr, $emitter:ident, $src1:ident, $src2:ident, $dst:ident) => {
878 match $src2 {
879 XMMOrMemory::XMM(x) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), Rx(x), $mode),
880 XMMOrMemory::Memory(base, disp) => dynasm!($emitter ; $ins Rx($dst), Rx($src1), [Rq(base) + disp], $mode),
881 }
882 }
883}
884
885macro_rules! sse_round_fn {
886 ($ins:ident, $mode:expr, $emitter:ident, $precision:expr, $src1:ident, $src2:ident, $dst:ident) => {
887 match $src2 {
888 XMMOrMemory::XMM(x) => {
889 if x != $dst {
890 move_src_to_dst($emitter, $precision, $src1, $dst);
891 }
892 dynasm!($emitter ; $ins Rx(x), Rx($dst), $mode)
893 }
894 XMMOrMemory::Memory(base, disp) => {
895 dynasm!($emitter ; $ins Rx($dst), [Rq(base) + disp], $mode)
896 },
897 }
898 }
899}
900
901impl EmitterX64 for AssemblerX64 {
902 fn get_simd_arch(&self) -> Option<&CpuFeature> {
903 Some(&self.simd_arch)
904 }
905
906 fn get_label(&mut self) -> DynamicLabel {
907 self.new_dynamic_label()
908 }
909
910 fn get_offset(&self) -> AssemblyOffset {
911 self.offset()
912 }
913
914 fn get_jmp_instr_size(&self) -> u8 {
915 5
916 }
917
918 fn finalize_function(&mut self) -> Result<(), CompileError> {
919 Ok(())
920 }
921
922 fn arch_has_xzcnt(&self) -> bool {
923 match &self.target {
924 Some(target) => {
925 target.cpu_features().contains(CpuFeature::LZCNT)
926 && target.cpu_features().contains(CpuFeature::BMI1)
927 }
928 None => false,
929 }
930 }
931
932 fn arch_emit_lzcnt(
933 &mut self,
934 sz: Size,
935 src: Location,
936 dst: Location,
937 ) -> Result<(), CompileError> {
938 binop_gpr_gpr!(lzcnt, self, sz, src, dst, {
939 binop_mem_gpr!(lzcnt, self, sz, src, dst, {
940 codegen_error!("singlepass cannot emit lzcnt")
941 })
942 });
943 Ok(())
944 }
945
946 fn arch_emit_tzcnt(
947 &mut self,
948 sz: Size,
949 src: Location,
950 dst: Location,
951 ) -> Result<(), CompileError> {
952 binop_gpr_gpr!(tzcnt, self, sz, src, dst, {
953 binop_mem_gpr!(tzcnt, self, sz, src, dst, {
954 codegen_error!("singlepass cannot emit tzcnt")
955 })
956 });
957 Ok(())
958 }
959
960 fn emit_u64(&mut self, x: u64) -> Result<(), CompileError> {
961 self.push_u64(x);
962 Ok(())
963 }
964
965 fn emit_bytes(&mut self, bytes: &[u8]) -> Result<(), CompileError> {
966 for &b in bytes {
967 self.push(b);
968 }
969 Ok(())
970 }
971
972 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
973 dynasm!(self ; => label);
974 Ok(())
975 }
976
977 fn emit_nop(&mut self) -> Result<(), CompileError> {
978 dynasm!(self ; nop);
979 Ok(())
980 }
981
982 fn emit_nop_n(&mut self, mut n: usize) -> Result<(), CompileError> {
983 while n >= 9 {
995 n -= 9;
996 self.emit_bytes(&[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00])?;
997 }
999 let seq: &[u8] = match n {
1000 0 => &[],
1001 1 => &[0x90],
1002 2 => &[0x66, 0x90],
1003 3 => &[0x0f, 0x1f, 0x00],
1004 4 => &[0x0f, 0x1f, 0x40, 0x00],
1005 5 => &[0x0f, 0x1f, 0x44, 0x00, 0x00],
1006 6 => &[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00],
1007 7 => &[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00],
1008 8 => &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
1009 _ => codegen_error!("singlepass emit_nop_n unreachable"),
1010 };
1011 self.emit_bytes(seq)
1012 }
1013
1014 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1015 if let (Location::Imm32(0), Location::GPR(x)) = (src, dst) {
1017 dynasm!(self ; xor Rd(x), Rd(x));
1018 return Ok(());
1019 }
1020
1021 binop_all_nofp!(mov, self, sz, src, dst, {
1022 binop_imm64_gpr!(mov, self, sz, src, dst, {
1023 match (sz, src, dst) {
1024 (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
1025 dynasm!(self ; mov [Rq(dst) + disp], Rb(src));
1026 }
1027 (Size::S8, Location::Memory(src, disp), Location::GPR(dst)) => {
1028 dynasm!(self ; mov Rb(dst), [Rq(src) + disp]);
1029 }
1030 (Size::S8, Location::Imm32(src), Location::GPR(dst)) => {
1031 dynasm!(self ; mov Rb(dst), src as i8);
1032 }
1033 (Size::S8, Location::Imm64(src), Location::GPR(dst)) => {
1034 dynasm!(self ; mov Rb(dst), src as i8);
1035 }
1036 (Size::S8, Location::Imm32(src), Location::Memory(dst, disp)) => {
1037 dynasm!(self ; mov BYTE [Rq(dst) + disp], src as i8);
1038 }
1039 (Size::S8, Location::Imm64(src), Location::Memory(dst, disp)) => {
1040 dynasm!(self ; mov BYTE [Rq(dst) + disp], src as i8);
1041 }
1042 (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
1043 dynasm!(self ; mov [Rq(dst) + disp], Rw(src));
1044 }
1045 (Size::S16, Location::Memory(src, disp), Location::GPR(dst)) => {
1046 dynasm!(self ; mov Rw(dst), [Rq(src) + disp]);
1047 }
1048 (Size::S16, Location::Imm32(src), Location::GPR(dst)) => {
1049 dynasm!(self ; mov Rw(dst), src as i16);
1050 }
1051 (Size::S16, Location::Imm64(src), Location::GPR(dst)) => {
1052 dynasm!(self ; mov Rw(dst), src as i16);
1053 }
1054 (Size::S16, Location::Imm32(src), Location::Memory(dst, disp)) => {
1055 dynasm!(self ; mov WORD [Rq(dst) + disp], src as i16);
1056 }
1057 (Size::S16, Location::Imm64(src), Location::Memory(dst, disp)) => {
1058 dynasm!(self ; mov WORD [Rq(dst) + disp], src as i16);
1059 }
1060 (Size::S32, Location::Imm64(src), Location::GPR(dst)) => {
1061 dynasm!(self ; mov Rd(dst), src as i32);
1062 }
1063 (Size::S32, Location::Imm64(src), Location::Memory(dst, disp)) => {
1064 dynasm!(self ; mov DWORD [Rq(dst) + disp], src as i32);
1065 }
1066 (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1067 dynasm!(self ; movd Rx(dst), Rd(src));
1068 }
1069 (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1070 dynasm!(self ; movd Rd(dst), Rx(src));
1071 }
1072 (Size::S32, Location::Memory(src, disp), Location::SIMD(dst)) => {
1073 dynasm!(self ; movd Rx(dst), [Rq(src) + disp]);
1074 }
1075 (Size::S32, Location::SIMD(src), Location::Memory(dst, disp)) => {
1076 dynasm!(self ; movd [Rq(dst) + disp], Rx(src));
1077 }
1078 (Size::S64, Location::Imm64(src), Location::GPR(dst)) => {
1079 dynasm!(self ; mov Rd(dst), src as i32);
1080 }
1081 (Size::S64, Location::Imm32(src), Location::GPR(dst)) => {
1082 dynasm!(self ; mov Rd(dst), src as i32);
1083 }
1084 (Size::S64, Location::Imm8(src), Location::GPR(dst)) => {
1085 dynasm!(self ; mov Rd(dst), src as i32);
1086 }
1087
1088 (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1089 dynasm!(self ; movq Rx(dst), Rq(src));
1090 }
1091 (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1092 dynasm!(self ; movq Rq(dst), Rx(src));
1093 }
1094 (Size::S64, Location::Memory(src, disp), Location::SIMD(dst)) => {
1095 dynasm!(self ; movq Rx(dst), [Rq(src) + disp]);
1096 }
1097 (Size::S64, Location::SIMD(src), Location::Memory(dst, disp)) => {
1098 dynasm!(self ; movq [Rq(dst) + disp], Rx(src));
1099 }
1100 (_, Location::SIMD(src), Location::SIMD(dst)) => {
1101 dynasm!(self ; movq Rx(dst), Rx(src));
1102 }
1103
1104 _ => codegen_error!("singlepass can't emit MOV {:?} {:?} {:?}", sz, src, dst),
1105 }
1106 })
1107 });
1108 Ok(())
1109 }
1110 fn emit_lea(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1111 match (sz, src, dst) {
1112 (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => {
1113 dynasm!(self ; lea Rd(dst), [Rq(src) + disp]);
1114 }
1115 (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => {
1116 dynasm!(self ; lea Rq(dst), [Rq(src) + disp]);
1117 }
1118 (Size::S32, Location::Memory2(src1, src2, mult, disp), Location::GPR(dst)) => {
1119 match mult {
1120 Multiplier::Zero => dynasm!(self ; lea Rd(dst), [Rq(src1) + disp]),
1121 Multiplier::One => {
1122 dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) + disp])
1123 }
1124 Multiplier::Two => {
1125 dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) * 2 + disp])
1126 }
1127 Multiplier::Four => {
1128 dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) * 4 + disp])
1129 }
1130 Multiplier::Height => {
1131 dynasm!(self ; lea Rd(dst), [Rq(src1) + Rq(src2) * 8 + disp])
1132 }
1133 };
1134 }
1135 (Size::S64, Location::Memory2(src1, src2, mult, disp), Location::GPR(dst)) => {
1136 match mult {
1137 Multiplier::Zero => dynasm!(self ; lea Rq(dst), [Rq(src1) + disp]),
1138 Multiplier::One => {
1139 dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) + disp])
1140 }
1141 Multiplier::Two => {
1142 dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) * 2 + disp])
1143 }
1144 Multiplier::Four => {
1145 dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) * 4 + disp])
1146 }
1147 Multiplier::Height => {
1148 dynasm!(self ; lea Rq(dst), [Rq(src1) + Rq(src2) * 8 + disp])
1149 }
1150 };
1151 }
1152 _ => codegen_error!("singlepass can't emit LEA {:?} {:?} {:?}", sz, src, dst),
1153 }
1154 Ok(())
1155 }
1156 fn emit_lea_label(&mut self, label: Label, dst: Location) -> Result<(), CompileError> {
1157 match dst {
1158 Location::GPR(x) => {
1159 dynasm!(self ; lea Rq(x), [=>label]);
1160 }
1161 _ => codegen_error!("singlepass can't emit LEA label={:?} {:?}", label, dst),
1162 }
1163 Ok(())
1164 }
1165 fn emit_cdq(&mut self) -> Result<(), CompileError> {
1166 dynasm!(self ; cdq);
1167 Ok(())
1168 }
1169 fn emit_cqo(&mut self) -> Result<(), CompileError> {
1170 dynasm!(self ; cqo);
1171 Ok(())
1172 }
1173 fn emit_xor(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1174 binop_all_nofp!(xor, self, sz, src, dst, {
1175 codegen_error!("singlepass can't emit XOR {:?} {:?} {:?}", sz, src, dst)
1176 });
1177 Ok(())
1178 }
1179 fn emit_jmp(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
1180 match condition {
1181 Condition::None => jmp_op!(jmp, self, label),
1182 Condition::Above => jmp_op!(ja, self, label),
1183 Condition::AboveEqual => jmp_op!(jae, self, label),
1184 Condition::Below => jmp_op!(jb, self, label),
1185 Condition::BelowEqual => jmp_op!(jbe, self, label),
1186 Condition::Greater => jmp_op!(jg, self, label),
1187 Condition::GreaterEqual => jmp_op!(jge, self, label),
1188 Condition::Less => jmp_op!(jl, self, label),
1189 Condition::LessEqual => jmp_op!(jle, self, label),
1190 Condition::Equal => jmp_op!(je, self, label),
1191 Condition::NotEqual => jmp_op!(jne, self, label),
1192 Condition::Signed => jmp_op!(js, self, label),
1193 Condition::Carry => jmp_op!(jc, self, label),
1194 }
1195 Ok(())
1196 }
1197 fn emit_jmp_location(&mut self, loc: Location) -> Result<(), CompileError> {
1198 match loc {
1199 Location::GPR(x) => dynasm!(self ; jmp Rq(x)),
1200 Location::Memory(base, disp) => dynasm!(self ; jmp QWORD [Rq(base) + disp]),
1201 _ => codegen_error!("singlepass can't emit JMP {:?}", loc),
1202 }
1203 Ok(())
1204 }
1205 fn emit_set(&mut self, condition: Condition, dst: GPR) -> Result<(), CompileError> {
1206 match condition {
1207 Condition::Above => dynasm!(self ; seta Rb(dst)),
1208 Condition::AboveEqual => dynasm!(self ; setae Rb(dst)),
1209 Condition::Below => dynasm!(self ; setb Rb(dst)),
1210 Condition::BelowEqual => dynasm!(self ; setbe Rb(dst)),
1211 Condition::Greater => dynasm!(self ; setg Rb(dst)),
1212 Condition::GreaterEqual => dynasm!(self ; setge Rb(dst)),
1213 Condition::Less => dynasm!(self ; setl Rb(dst)),
1214 Condition::LessEqual => dynasm!(self ; setle Rb(dst)),
1215 Condition::Equal => dynasm!(self ; sete Rb(dst)),
1216 Condition::NotEqual => dynasm!(self ; setne Rb(dst)),
1217 Condition::Signed => dynasm!(self ; sets Rb(dst)),
1218 Condition::Carry => dynasm!(self ; setc Rb(dst)),
1219 _ => codegen_error!("singlepass can't emit SET {:?} {:?}", condition, dst),
1220 }
1221 Ok(())
1222 }
1223 fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1224 match (sz, src) {
1225 (Size::S64, Location::Imm32(src)) => dynasm!(self ; push src as i32),
1226 (Size::S64, Location::GPR(src)) => dynasm!(self ; push Rq(src)),
1227 (Size::S64, Location::Memory(src, disp)) => {
1228 dynasm!(self ; push QWORD [Rq(src) + disp])
1229 }
1230 _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1231 }
1232 Ok(())
1233 }
1234 fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1235 match (sz, dst) {
1236 (Size::S64, Location::GPR(dst)) => dynasm!(self ; pop Rq(dst)),
1237 (Size::S64, Location::Memory(dst, disp)) => {
1238 dynasm!(self ; pop QWORD [Rq(dst) + disp])
1239 }
1240 _ => codegen_error!("singlepass can't emit POP {:?} {:?}", sz, dst),
1241 }
1242 Ok(())
1243 }
1244
1245 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1246 binop_all_nofp!(cmp, self, sz, left, right, {
1247 codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right);
1248 });
1249 Ok(())
1250 }
1251
1252 fn emit_add(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1253 if let Location::Imm32(0) = src {
1255 return Ok(());
1256 }
1257 binop_all_nofp!(add, self, sz, src, dst, {
1258 codegen_error!("singlepass can't emit ADD {:?} {:?} {:?}", sz, src, dst)
1259 });
1260 Ok(())
1261 }
1262 fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1263 if let Location::Imm32(0) = src {
1265 return Ok(());
1266 }
1267 binop_all_nofp!(sub, self, sz, src, dst, {
1268 codegen_error!("singlepass can't emit SUB {:?} {:?} {:?}", sz, src, dst)
1269 });
1270 Ok(())
1271 }
1272 fn emit_neg(&mut self, sz: Size, value: Location) -> Result<(), CompileError> {
1273 match (sz, value) {
1274 (Size::S8, Location::GPR(value)) => dynasm!(self ; neg Rb(value)),
1275 (Size::S8, Location::Memory(value, disp)) => {
1276 dynasm!(self ; neg [Rq(value) + disp])
1277 }
1278 (Size::S16, Location::GPR(value)) => dynasm!(self ; neg Rw(value)),
1279 (Size::S16, Location::Memory(value, disp)) => {
1280 dynasm!(self ; neg [Rq(value) + disp])
1281 }
1282 (Size::S32, Location::GPR(value)) => dynasm!(self ; neg Rd(value)),
1283 (Size::S32, Location::Memory(value, disp)) => {
1284 dynasm!(self ; neg [Rq(value) + disp])
1285 }
1286 (Size::S64, Location::GPR(value)) => dynasm!(self ; neg Rq(value)),
1287 (Size::S64, Location::Memory(value, disp)) => {
1288 dynasm!(self ; neg [Rq(value) + disp])
1289 }
1290 _ => codegen_error!("singlepass can't emit NEG {:?} {:?}", sz, value),
1291 }
1292 Ok(())
1293 }
1294 fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1295 binop_gpr_gpr!(imul, self, sz, src, dst, {
1296 binop_mem_gpr!(imul, self, sz, src, dst, {
1297 codegen_error!("singlepass can't emit IMUL {:?} {:?} {:?}", sz, src, dst)
1298 })
1299 });
1300 Ok(())
1301 }
1302 fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR) -> Result<(), CompileError> {
1303 dynasm!(self ; imul Rq(dst), Rq(dst), src as i32);
1304 Ok(())
1305 }
1306 fn emit_div(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError> {
1307 unop_gpr_or_mem!(div, self, sz, divisor, {
1308 codegen_error!("singlepass can't emit DIV {:?} {:?}", sz, divisor)
1309 });
1310 Ok(())
1311 }
1312 fn emit_idiv(&mut self, sz: Size, divisor: Location) -> Result<(), CompileError> {
1313 unop_gpr_or_mem!(idiv, self, sz, divisor, {
1314 codegen_error!("singlepass can't emit IDIV {:?} {:?}", sz, divisor)
1315 });
1316 Ok(())
1317 }
1318 fn emit_shl(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1319 binop_shift!(shl, self, sz, src, dst, {
1320 codegen_error!("singlepass can't emit SHL {:?} {:?} {:?}", sz, src, dst)
1321 });
1322 Ok(())
1323 }
1324 fn emit_shr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1325 binop_shift!(shr, self, sz, src, dst, {
1326 codegen_error!("singlepass can't emit SHR {:?} {:?} {:?}", sz, src, dst)
1327 });
1328 Ok(())
1329 }
1330 fn emit_sar(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1331 binop_shift!(sar, self, sz, src, dst, {
1332 codegen_error!("singlepass can't emit SAR {:?} {:?} {:?}", sz, src, dst)
1333 });
1334 Ok(())
1335 }
1336 fn emit_rol(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1337 binop_shift!(rol, self, sz, src, dst, {
1338 codegen_error!("singlepass can't emit ROL {:?} {:?} {:?}", sz, src, dst)
1339 });
1340 Ok(())
1341 }
1342 fn emit_ror(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1343 binop_shift!(ror, self, sz, src, dst, {
1344 codegen_error!("singlepass can't emit ROR {:?} {:?} {:?}", sz, src, dst)
1345 });
1346 Ok(())
1347 }
1348 fn emit_and(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1349 binop_all_nofp!(and, self, sz, src, dst, {
1350 codegen_error!("singlepass can't emit AND {:?} {:?} {:?}", sz, src, dst)
1351 });
1352 Ok(())
1353 }
1354 fn emit_test(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1355 binop_all_nofp!(test, self, sz, src, dst, {
1356 codegen_error!("singlepass can't emit TEST {:?} {:?} {:?}", sz, src, dst)
1357 });
1358 Ok(())
1359 }
1360 fn emit_or(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1361 binop_all_nofp!(or, self, sz, src, dst, {
1362 codegen_error!("singlepass can't emit OR {:?} {:?} {:?}", sz, src, dst)
1363 });
1364 Ok(())
1365 }
1366 fn emit_bsr(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1367 binop_gpr_gpr!(bsr, self, sz, src, dst, {
1368 binop_mem_gpr!(bsr, self, sz, src, dst, {
1369 codegen_error!("singlepass can't emit BSR {:?} {:?} {:?}", sz, src, dst)
1370 })
1371 });
1372 Ok(())
1373 }
1374 fn emit_bsf(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1375 binop_gpr_gpr!(bsf, self, sz, src, dst, {
1376 binop_mem_gpr!(bsf, self, sz, src, dst, {
1377 codegen_error!("singlepass can't emit BSF {:?} {:?} {:?}", sz, src, dst)
1378 })
1379 });
1380 Ok(())
1381 }
1382 fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1383 binop_gpr_gpr!(popcnt, self, sz, src, dst, {
1384 binop_mem_gpr!(popcnt, self, sz, src, dst, {
1385 codegen_error!("singlepass can't emit POPCNT {:?} {:?} {:?}", sz, src, dst)
1386 })
1387 });
1388 Ok(())
1389 }
1390 fn emit_movzx(
1391 &mut self,
1392 sz_src: Size,
1393 src: Location,
1394 sz_dst: Size,
1395 dst: Location,
1396 ) -> Result<(), CompileError> {
1397 match (sz_src, src, sz_dst, dst) {
1398 (Size::S8, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1399 dynasm!(self ; movzx Rd(dst), Rb(src));
1400 }
1401 (Size::S16, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1402 dynasm!(self ; movzx Rd(dst), Rw(src));
1403 }
1404 (Size::S8, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1405 dynasm!(self ; movzx Rd(dst), BYTE [Rq(src) + disp]);
1406 }
1407 (Size::S16, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1408 dynasm!(self ; movzx Rd(dst), WORD [Rq(src) + disp]);
1409 }
1410 (Size::S16, Location::Imm32(imm), Size::S32, Location::GPR(dst)) => {
1411 dynasm!(self ; mov Rd(dst), imm as i32);
1412 }
1413 (Size::S8, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1414 dynasm!(self ; movzx Rq(dst), Rb(src));
1415 }
1416 (Size::S16, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1417 dynasm!(self ; movzx Rq(dst), Rw(src));
1418 }
1419 (Size::S8, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1420 dynasm!(self ; movzx Rq(dst), BYTE [Rq(src) + disp]);
1421 }
1422 (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1423 dynasm!(self ; movzx Rq(dst), WORD [Rq(src) + disp]);
1424 }
1425 (Size::S32, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1426 if src != dst {
1427 dynasm!(self ; mov Rd(dst), Rd(src));
1428 }
1429 }
1430 (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1431 dynasm!(self ; mov Rd(dst), DWORD [Rq(src) + disp]);
1432 }
1433 (Size::S8, Location::Imm32(imm), Size::S32, Location::GPR(dst)) => {
1434 dynasm!(self ; mov Rq(dst), imm as i32);
1435 }
1436 (Size::S16, Location::Imm32(imm), Size::S64, Location::GPR(dst)) => {
1437 dynasm!(self ; mov Rq(dst), imm as i32);
1438 }
1439 (Size::S32, Location::Imm32(imm), Size::S64, Location::GPR(dst)) => {
1440 dynasm!(self ; mov Rq(dst), imm as i32);
1441 }
1442 (Size::S8, Location::Imm64(imm), Size::S32, Location::GPR(dst)) => {
1443 dynasm!(self ; mov Rq(dst), imm as i32);
1444 }
1445 (Size::S16, Location::Imm64(imm), Size::S64, Location::GPR(dst)) => {
1446 dynasm!(self ; mov Rq(dst), imm as i32);
1447 }
1448 (Size::S32, Location::Imm64(imm), Size::S64, Location::GPR(dst)) => {
1449 dynasm!(self ; mov Rq(dst), imm as i32);
1450 }
1451 _ => {
1452 codegen_error!(
1453 "singlepass can't emit MOVZX {:?} {:?} {:?} {:?}",
1454 sz_src,
1455 src,
1456 sz_dst,
1457 dst
1458 )
1459 }
1460 }
1461 Ok(())
1462 }
1463 fn emit_movsx(
1464 &mut self,
1465 sz_src: Size,
1466 src: Location,
1467 sz_dst: Size,
1468 dst: Location,
1469 ) -> Result<(), CompileError> {
1470 match (sz_src, src, sz_dst, dst) {
1471 (Size::S8, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1472 dynasm!(self ; movsx Rd(dst), Rb(src));
1473 }
1474 (Size::S16, Location::GPR(src), Size::S32, Location::GPR(dst)) => {
1475 dynasm!(self ; movsx Rd(dst), Rw(src));
1476 }
1477 (Size::S8, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1478 dynasm!(self ; movsx Rd(dst), BYTE [Rq(src) + disp]);
1479 }
1480 (Size::S16, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => {
1481 dynasm!(self ; movsx Rd(dst), WORD [Rq(src) + disp]);
1482 }
1483 (Size::S8, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1484 dynasm!(self ; movsx Rq(dst), Rb(src));
1485 }
1486 (Size::S16, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1487 dynasm!(self ; movsx Rq(dst), Rw(src));
1488 }
1489 (Size::S32, Location::GPR(src), Size::S64, Location::GPR(dst)) => {
1490 dynasm!(self ; movsx Rq(dst), Rd(src));
1491 }
1492 (Size::S8, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1493 dynasm!(self ; movsx Rq(dst), BYTE [Rq(src) + disp]);
1494 }
1495 (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1496 dynasm!(self ; movsx Rq(dst), WORD [Rq(src) + disp]);
1497 }
1498 (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => {
1499 dynasm!(self ; movsx Rq(dst), DWORD [Rq(src) + disp]);
1500 }
1501 _ => {
1502 codegen_error!(
1503 "singlepass can't emit MOVSX {:?} {:?} {:?} {:?}",
1504 sz_src,
1505 src,
1506 sz_dst,
1507 dst
1508 )
1509 }
1510 }
1511 Ok(())
1512 }
1513
1514 fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1515 match (sz, src, dst) {
1516 (Size::S8, Location::GPR(src), Location::GPR(dst)) => {
1517 dynasm!(self ; xchg Rb(dst), Rb(src));
1518 }
1519 (Size::S16, Location::GPR(src), Location::GPR(dst)) => {
1520 dynasm!(self ; xchg Rw(dst), Rw(src));
1521 }
1522 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1523 dynasm!(self ; xchg Rd(dst), Rd(src));
1524 }
1525 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1526 dynasm!(self ; xchg Rq(dst), Rq(src));
1527 }
1528 (Size::S8, Location::Memory(src, disp), Location::GPR(dst)) => {
1529 dynasm!(self ; xchg Rb(dst), [Rq(src) + disp]);
1530 }
1531 (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
1532 dynasm!(self ; xchg [Rq(dst) + disp], Rb(src));
1533 }
1534 (Size::S16, Location::Memory(src, disp), Location::GPR(dst)) => {
1535 dynasm!(self ; xchg Rw(dst), [Rq(src) + disp]);
1536 }
1537 (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
1538 dynasm!(self ; xchg [Rq(dst) + disp], Rw(src));
1539 }
1540 (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => {
1541 dynasm!(self ; xchg Rd(dst), [Rq(src) + disp]);
1542 }
1543 (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
1544 dynasm!(self ; xchg [Rq(dst) + disp], Rd(src));
1545 }
1546 (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => {
1547 dynasm!(self ; xchg Rq(dst), [Rq(src) + disp]);
1548 }
1549 (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
1550 dynasm!(self ; xchg [Rq(dst) + disp], Rq(src));
1551 }
1552 _ => codegen_error!("singlepass can't emit XCHG {:?} {:?} {:?}", sz, src, dst),
1553 }
1554 Ok(())
1555 }
1556
1557 fn emit_lock_xadd(
1558 &mut self,
1559 sz: Size,
1560 src: Location,
1561 dst: Location,
1562 ) -> Result<(), CompileError> {
1563 match (sz, src, dst) {
1564 (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
1565 dynasm!(self ; lock xadd [Rq(dst) + disp], Rb(src));
1566 }
1567 (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
1568 dynasm!(self ; lock xadd [Rq(dst) + disp], Rw(src));
1569 }
1570 (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
1571 dynasm!(self ; lock xadd [Rq(dst) + disp], Rd(src));
1572 }
1573 (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
1574 dynasm!(self ; lock xadd [Rq(dst) + disp], Rq(src));
1575 }
1576 _ => codegen_error!(
1577 "singlepass can't emit LOCK XADD {:?} {:?} {:?}",
1578 sz,
1579 src,
1580 dst
1581 ),
1582 }
1583 Ok(())
1584 }
1585
1586 fn emit_lock_cmpxchg(
1587 &mut self,
1588 sz: Size,
1589 src: Location,
1590 dst: Location,
1591 ) -> Result<(), CompileError> {
1592 match (sz, src, dst) {
1593 (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => {
1594 dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rb(src));
1595 }
1596 (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => {
1597 dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rw(src));
1598 }
1599 (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => {
1600 dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rd(src));
1601 }
1602 (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => {
1603 dynasm!(self ; lock cmpxchg [Rq(dst) + disp], Rq(src));
1604 }
1605 _ => codegen_error!(
1606 "singlepass can't emit LOCK CMPXCHG {:?} {:?} {:?}",
1607 sz,
1608 src,
1609 dst
1610 ),
1611 }
1612 Ok(())
1613 }
1614
1615 fn emit_rep_stosq(&mut self) -> Result<(), CompileError> {
1616 dynasm!(self ; rep stosq);
1617 Ok(())
1618 }
1619 fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR) -> Result<(), CompileError> {
1620 dynasm!(self ; btc Rd(dst), BYTE src as i8);
1621 Ok(())
1622 }
1623
1624 fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR) -> Result<(), CompileError> {
1625 dynasm!(self ; btc Rq(dst), BYTE src as i8);
1626 Ok(())
1627 }
1628
1629 fn emit_cmovae_gpr_32(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError> {
1630 dynasm!(self ; cmovae Rd(dst), Rd(src));
1631 Ok(())
1632 }
1633
1634 fn emit_cmovae_gpr_64(&mut self, src: GPR, dst: GPR) -> Result<(), CompileError> {
1635 dynasm!(self ; cmovae Rq(dst), Rq(src));
1636 Ok(())
1637 }
1638
1639 fn emit_vmovaps(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError> {
1640 match (src, dst) {
1641 (XMMOrMemory::XMM(src), XMMOrMemory::XMM(dst)) => {
1642 dynasm!(self ; movaps Rx(dst), Rx(src))
1643 }
1644 (XMMOrMemory::Memory(base, disp), XMMOrMemory::XMM(dst)) => {
1645 dynasm!(self ; movaps Rx(dst), [Rq(base) + disp])
1646 }
1647 (XMMOrMemory::XMM(src), XMMOrMemory::Memory(base, disp)) => {
1648 dynasm!(self ; movaps [Rq(base) + disp], Rx(src))
1649 }
1650 _ => codegen_error!("singlepass can't emit VMOVAPS {:?} {:?}", src, dst),
1651 };
1652 Ok(())
1653 }
1654
1655 fn emit_vmovapd(&mut self, src: XMMOrMemory, dst: XMMOrMemory) -> Result<(), CompileError> {
1656 match (src, dst) {
1657 (XMMOrMemory::XMM(src), XMMOrMemory::XMM(dst)) => {
1658 dynasm!(self ; movapd Rx(dst), Rx(src))
1659 }
1660 (XMMOrMemory::Memory(base, disp), XMMOrMemory::XMM(dst)) => {
1661 dynasm!(self ; movapd Rx(dst), [Rq(base) + disp])
1662 }
1663 (XMMOrMemory::XMM(src), XMMOrMemory::Memory(base, disp)) => {
1664 dynasm!(self ; movapd [Rq(base) + disp], Rx(src))
1665 }
1666 _ => codegen_error!("singlepass can't emit VMOVAPD {:?} {:?}", src, dst),
1667 };
1668 Ok(())
1669 }
1670 fn emit_vxorps(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1671 match self.get_simd_arch() {
1672 Some(CpuFeature::AVX) => avx_fn!(vxorps, self, src1, src2, dst),
1673 Some(CpuFeature::SSE42) => sse_fn!(xorps, self, Precision::Single, src1, src2, dst),
1674 _ => {}
1675 }
1676 Ok(())
1677 }
1678 fn emit_vxorpd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1679 match self.get_simd_arch() {
1680 Some(CpuFeature::AVX) => avx_fn!(vxorpd, self, src1, src2, dst),
1681 Some(CpuFeature::SSE42) => sse_fn!(xorpd, self, Precision::Double, src1, src2, dst),
1682 _ => {}
1683 }
1684 Ok(())
1685 }
1686 fn emit_vaddss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1687 match self.get_simd_arch() {
1688 Some(CpuFeature::AVX) => avx_fn!(vaddss, self, src1, src2, dst),
1689 Some(CpuFeature::SSE42) => sse_fn!(addss, self, Precision::Single, src1, src2, dst),
1690 _ => {}
1691 }
1692 Ok(())
1693 }
1694 fn emit_vaddsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1695 match self.get_simd_arch() {
1696 Some(CpuFeature::AVX) => avx_fn!(vaddsd, self, src1, src2, dst),
1697 Some(CpuFeature::SSE42) => sse_fn!(addsd, self, Precision::Double, src1, src2, dst),
1698 _ => {}
1699 }
1700 Ok(())
1701 }
1702 fn emit_vsubss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1703 match self.get_simd_arch() {
1704 Some(CpuFeature::AVX) => avx_fn!(vsubss, self, src1, src2, dst),
1705 Some(CpuFeature::SSE42) => sse_fn!(subss, self, Precision::Single, src1, src2, dst),
1706 _ => {}
1707 }
1708 Ok(())
1709 }
1710 fn emit_vsubsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1711 match self.get_simd_arch() {
1712 Some(CpuFeature::AVX) => avx_fn!(vsubsd, self, src1, src2, dst),
1713 Some(CpuFeature::SSE42) => sse_fn!(subsd, self, Precision::Double, src1, src2, dst),
1714 _ => {}
1715 }
1716 Ok(())
1717 }
1718 fn emit_vmulss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1719 match self.get_simd_arch() {
1720 Some(CpuFeature::AVX) => avx_fn!(vmulss, self, src1, src2, dst),
1721 Some(CpuFeature::SSE42) => sse_fn!(mulss, self, Precision::Single, src1, src2, dst),
1722 _ => {}
1723 }
1724 Ok(())
1725 }
1726 fn emit_vmulsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1727 match self.get_simd_arch() {
1728 Some(CpuFeature::AVX) => avx_fn!(vmulsd, self, src1, src2, dst),
1729 Some(CpuFeature::SSE42) => sse_fn!(mulsd, self, Precision::Double, src1, src2, dst),
1730 _ => {}
1731 }
1732 Ok(())
1733 }
1734 fn emit_vdivss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1735 match self.get_simd_arch() {
1736 Some(CpuFeature::AVX) => avx_fn!(vdivss, self, src1, src2, dst),
1737 Some(CpuFeature::SSE42) => sse_fn!(divss, self, Precision::Single, src1, src2, dst),
1738 _ => {}
1739 }
1740 Ok(())
1741 }
1742 fn emit_vdivsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1743 match self.get_simd_arch() {
1744 Some(CpuFeature::AVX) => avx_fn!(vdivsd, self, src1, src2, dst),
1745 Some(CpuFeature::SSE42) => sse_fn!(divsd, self, Precision::Double, src1, src2, dst),
1746 _ => {}
1747 }
1748 Ok(())
1749 }
1750 fn emit_vmaxss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1751 match self.get_simd_arch() {
1752 Some(CpuFeature::AVX) => avx_fn!(vmaxss, self, src1, src2, dst),
1753 Some(CpuFeature::SSE42) => sse_fn!(maxss, self, Precision::Single, src1, src2, dst),
1754 _ => {}
1755 }
1756 Ok(())
1757 }
1758 fn emit_vmaxsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1759 match self.get_simd_arch() {
1760 Some(CpuFeature::AVX) => avx_fn!(vmaxsd, self, src1, src2, dst),
1761 Some(CpuFeature::SSE42) => sse_fn!(maxsd, self, Precision::Double, src1, src2, dst),
1762 _ => {}
1763 }
1764 Ok(())
1765 }
1766 fn emit_vminss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1767 match self.get_simd_arch() {
1768 Some(CpuFeature::AVX) => avx_fn!(vminss, self, src1, src2, dst),
1769 Some(CpuFeature::SSE42) => sse_fn!(minss, self, Precision::Single, src1, src2, dst),
1770 _ => {}
1771 }
1772 Ok(())
1773 }
1774 fn emit_vminsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1775 match self.get_simd_arch() {
1776 Some(CpuFeature::AVX) => avx_fn!(vminsd, self, src1, src2, dst),
1777 Some(CpuFeature::SSE42) => sse_fn!(minsd, self, Precision::Double, src1, src2, dst),
1778 _ => {}
1779 }
1780 Ok(())
1781 }
1782 fn emit_vcmpeqss(
1783 &mut self,
1784 src1: XMM,
1785 src2: XMMOrMemory,
1786 dst: XMM,
1787 ) -> Result<(), CompileError> {
1788 match self.get_simd_arch() {
1789 Some(CpuFeature::AVX) => avx_fn!(vcmpeqss, self, src1, src2, dst),
1790 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 0, self, Precision::Single, src1, src2, dst),
1791 _ => {}
1792 }
1793 Ok(())
1794 }
1795 fn emit_vcmpeqsd(
1796 &mut self,
1797 src1: XMM,
1798 src2: XMMOrMemory,
1799 dst: XMM,
1800 ) -> Result<(), CompileError> {
1801 match self.get_simd_arch() {
1802 Some(CpuFeature::AVX) => avx_fn!(vcmpeqsd, self, src1, src2, dst),
1803 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 0, self, Precision::Double, src1, src2, dst),
1804 _ => {}
1805 }
1806 Ok(())
1807 }
1808 fn emit_vcmpneqss(
1809 &mut self,
1810 src1: XMM,
1811 src2: XMMOrMemory,
1812 dst: XMM,
1813 ) -> Result<(), CompileError> {
1814 match self.get_simd_arch() {
1815 Some(CpuFeature::AVX) => avx_fn!(vcmpneqss, self, src1, src2, dst),
1816 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 4, self, Precision::Single, src1, src2, dst),
1817 _ => {}
1818 }
1819 Ok(())
1820 }
1821 fn emit_vcmpneqsd(
1822 &mut self,
1823 src1: XMM,
1824 src2: XMMOrMemory,
1825 dst: XMM,
1826 ) -> Result<(), CompileError> {
1827 match self.get_simd_arch() {
1828 Some(CpuFeature::AVX) => avx_fn!(vcmpneqsd, self, src1, src2, dst),
1829 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 4, self, Precision::Double, src1, src2, dst),
1830 _ => {}
1831 }
1832 Ok(())
1833 }
1834 fn emit_vcmpltss(
1835 &mut self,
1836 src1: XMM,
1837 src2: XMMOrMemory,
1838 dst: XMM,
1839 ) -> Result<(), CompileError> {
1840 match self.get_simd_arch() {
1841 Some(CpuFeature::AVX) => avx_fn!(vcmpltss, self, src1, src2, dst),
1842 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 1, self, Precision::Single, src1, src2, dst),
1843 _ => {}
1844 }
1845 Ok(())
1846 }
1847 fn emit_vcmpltsd(
1848 &mut self,
1849 src1: XMM,
1850 src2: XMMOrMemory,
1851 dst: XMM,
1852 ) -> Result<(), CompileError> {
1853 match self.get_simd_arch() {
1854 Some(CpuFeature::AVX) => avx_fn!(vcmpltsd, self, src1, src2, dst),
1855 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 1, self, Precision::Double, src1, src2, dst),
1856 _ => {}
1857 }
1858 Ok(())
1859 }
1860 fn emit_vcmpless(
1861 &mut self,
1862 src1: XMM,
1863 src2: XMMOrMemory,
1864 dst: XMM,
1865 ) -> Result<(), CompileError> {
1866 match self.get_simd_arch() {
1867 Some(CpuFeature::AVX) => avx_fn!(vcmpless, self, src1, src2, dst),
1868 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 2, self, Precision::Single, src1, src2, dst),
1869 _ => {}
1870 }
1871 Ok(())
1872 }
1873 fn emit_vcmplesd(
1874 &mut self,
1875 src1: XMM,
1876 src2: XMMOrMemory,
1877 dst: XMM,
1878 ) -> Result<(), CompileError> {
1879 match self.get_simd_arch() {
1880 Some(CpuFeature::AVX) => avx_fn!(vcmplesd, self, src1, src2, dst),
1881 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 2, self, Precision::Double, src1, src2, dst),
1882 _ => {}
1883 }
1884 Ok(())
1885 }
1886 fn emit_vcmpgtss(
1887 &mut self,
1888 src1: XMM,
1889 src2: XMMOrMemory,
1890 dst: XMM,
1891 ) -> Result<(), CompileError> {
1892 match self.get_simd_arch() {
1893 Some(CpuFeature::AVX) => avx_fn!(vcmpgtss, self, src1, src2, dst),
1894 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 6, self, Precision::Single, src1, src2, dst),
1895 _ => {}
1896 }
1897 Ok(())
1898 }
1899 fn emit_vcmpgtsd(
1900 &mut self,
1901 src1: XMM,
1902 src2: XMMOrMemory,
1903 dst: XMM,
1904 ) -> Result<(), CompileError> {
1905 match self.get_simd_arch() {
1906 Some(CpuFeature::AVX) => avx_fn!(vcmpgtsd, self, src1, src2, dst),
1907 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 6, self, Precision::Double, src1, src2, dst),
1908 _ => {}
1909 }
1910 Ok(())
1911 }
1912 fn emit_vcmpgess(
1913 &mut self,
1914 src1: XMM,
1915 src2: XMMOrMemory,
1916 dst: XMM,
1917 ) -> Result<(), CompileError> {
1918 match self.get_simd_arch() {
1919 Some(CpuFeature::AVX) => avx_fn!(vcmpgess, self, src1, src2, dst),
1920 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 5, self, Precision::Single, src1, src2, dst),
1921 _ => {}
1922 }
1923 Ok(())
1924 }
1925 fn emit_vcmpgesd(
1926 &mut self,
1927 src1: XMM,
1928 src2: XMMOrMemory,
1929 dst: XMM,
1930 ) -> Result<(), CompileError> {
1931 match self.get_simd_arch() {
1932 Some(CpuFeature::AVX) => avx_fn!(vcmpgesd, self, src1, src2, dst),
1933 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 5, self, Precision::Double, src1, src2, dst),
1934 _ => {}
1935 }
1936 Ok(())
1937 }
1938 fn emit_vcmpunordss(
1939 &mut self,
1940 src1: XMM,
1941 src2: XMMOrMemory,
1942 dst: XMM,
1943 ) -> Result<(), CompileError> {
1944 match self.get_simd_arch() {
1945 Some(CpuFeature::AVX) => avx_fn!(vcmpunordss, self, src1, src2, dst),
1946 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 3, self, Precision::Single, src1, src2, dst),
1947 _ => {}
1948 }
1949 Ok(())
1950 }
1951 fn emit_vcmpunordsd(
1952 &mut self,
1953 src1: XMM,
1954 src2: XMMOrMemory,
1955 dst: XMM,
1956 ) -> Result<(), CompileError> {
1957 match self.get_simd_arch() {
1958 Some(CpuFeature::AVX) => avx_fn!(vcmpunordsd, self, src1, src2, dst),
1959 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 3, self, Precision::Double, src1, src2, dst),
1960 _ => {}
1961 }
1962 Ok(())
1963 }
1964 fn emit_vcmpordss(
1965 &mut self,
1966 src1: XMM,
1967 src2: XMMOrMemory,
1968 dst: XMM,
1969 ) -> Result<(), CompileError> {
1970 match self.get_simd_arch() {
1971 Some(CpuFeature::AVX) => avx_fn!(vcmpordss, self, src1, src2, dst),
1972 Some(CpuFeature::SSE42) => sse_fn!(cmpss, 7, self, Precision::Single, src1, src2, dst),
1973 _ => {}
1974 }
1975 Ok(())
1976 }
1977 fn emit_vcmpordsd(
1978 &mut self,
1979 src1: XMM,
1980 src2: XMMOrMemory,
1981 dst: XMM,
1982 ) -> Result<(), CompileError> {
1983 match self.get_simd_arch() {
1984 Some(CpuFeature::AVX) => avx_fn!(vcmpordsd, self, src1, src2, dst),
1985 Some(CpuFeature::SSE42) => sse_fn!(cmpsd, 7, self, Precision::Double, src1, src2, dst),
1986 _ => {}
1987 }
1988 Ok(())
1989 }
1990 fn emit_vsqrtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1991 match self.get_simd_arch() {
1992 Some(CpuFeature::AVX) => avx_fn!(vsqrtss, self, src1, src2, dst),
1993 Some(CpuFeature::SSE42) => sse_fn!(sqrtss, self, Precision::Single, src1, src2, dst),
1994 _ => {}
1995 }
1996 Ok(())
1997 }
1998 fn emit_vsqrtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
1999 match self.get_simd_arch() {
2000 Some(CpuFeature::AVX) => avx_fn!(vsqrtsd, self, src1, src2, dst),
2001 Some(CpuFeature::SSE42) => sse_fn!(sqrtsd, self, Precision::Double, src1, src2, dst),
2002 _ => {}
2003 }
2004 Ok(())
2005 }
2006 fn emit_vcvtss2sd(
2007 &mut self,
2008 src1: XMM,
2009 src2: XMMOrMemory,
2010 dst: XMM,
2011 ) -> Result<(), CompileError> {
2012 match self.get_simd_arch() {
2013 Some(CpuFeature::AVX) => avx_fn!(vcvtss2sd, self, src1, src2, dst),
2014 Some(CpuFeature::SSE42) => sse_fn!(cvtss2sd, self, Precision::Single, src1, src2, dst),
2015 _ => {}
2016 }
2017 Ok(())
2018 }
2019 fn emit_vcvtsd2ss(
2020 &mut self,
2021 src1: XMM,
2022 src2: XMMOrMemory,
2023 dst: XMM,
2024 ) -> Result<(), CompileError> {
2025 match self.get_simd_arch() {
2026 Some(CpuFeature::AVX) => avx_fn!(vcvtsd2ss, self, src1, src2, dst),
2027 Some(CpuFeature::SSE42) => sse_fn!(cvtsd2ss, self, Precision::Double, src1, src2, dst),
2028 _ => {}
2029 }
2030 Ok(())
2031 }
2032 fn emit_vroundss_nearest(
2033 &mut self,
2034 src1: XMM,
2035 src2: XMMOrMemory,
2036 dst: XMM,
2037 ) -> Result<(), CompileError> {
2038 match self.get_simd_arch() {
2039 Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 0, self, src1, src2, dst),
2040 Some(CpuFeature::SSE42) => {
2041 sse_round_fn!(roundss, 0, self, Precision::Single, src1, src2, dst)
2042 }
2043 _ => {}
2044 }
2045 Ok(())
2046 }
2047 fn emit_vroundsd_nearest(
2048 &mut self,
2049 src1: XMM,
2050 src2: XMMOrMemory,
2051 dst: XMM,
2052 ) -> Result<(), CompileError> {
2053 match self.get_simd_arch() {
2054 Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 0, self, src1, src2, dst),
2055 Some(CpuFeature::SSE42) => {
2056 sse_round_fn!(roundsd, 0, self, Precision::Double, src1, src2, dst)
2057 }
2058 _ => {}
2059 }
2060 Ok(())
2061 }
2062 fn emit_vroundss_floor(
2063 &mut self,
2064 src1: XMM,
2065 src2: XMMOrMemory,
2066 dst: XMM,
2067 ) -> Result<(), CompileError> {
2068 match self.get_simd_arch() {
2069 Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 1, self, src1, src2, dst),
2070 Some(CpuFeature::SSE42) => {
2071 sse_round_fn!(roundss, 1, self, Precision::Single, src1, src2, dst)
2072 }
2073 _ => {}
2074 }
2075 Ok(())
2076 }
2077 fn emit_vroundsd_floor(
2078 &mut self,
2079 src1: XMM,
2080 src2: XMMOrMemory,
2081 dst: XMM,
2082 ) -> Result<(), CompileError> {
2083 match self.get_simd_arch() {
2084 Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 1, self, src1, src2, dst),
2085 Some(CpuFeature::SSE42) => {
2086 sse_round_fn!(roundsd, 1, self, Precision::Double, src1, src2, dst)
2087 }
2088 _ => {}
2089 }
2090 Ok(())
2091 }
2092 fn emit_vroundss_ceil(
2093 &mut self,
2094 src1: XMM,
2095 src2: XMMOrMemory,
2096 dst: XMM,
2097 ) -> Result<(), CompileError> {
2098 match self.get_simd_arch() {
2099 Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 2, self, src1, src2, dst),
2100 Some(CpuFeature::SSE42) => {
2101 sse_round_fn!(roundss, 2, self, Precision::Single, src1, src2, dst)
2102 }
2103 _ => {}
2104 }
2105 Ok(())
2106 }
2107 fn emit_vroundsd_ceil(
2108 &mut self,
2109 src1: XMM,
2110 src2: XMMOrMemory,
2111 dst: XMM,
2112 ) -> Result<(), CompileError> {
2113 match self.get_simd_arch() {
2114 Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 2, self, src1, src2, dst),
2115 Some(CpuFeature::SSE42) => {
2116 sse_round_fn!(roundsd, 2, self, Precision::Double, src1, src2, dst)
2117 }
2118 _ => {}
2119 }
2120 Ok(())
2121 }
2122 fn emit_vroundss_trunc(
2123 &mut self,
2124 src1: XMM,
2125 src2: XMMOrMemory,
2126 dst: XMM,
2127 ) -> Result<(), CompileError> {
2128 match self.get_simd_arch() {
2129 Some(CpuFeature::AVX) => avx_round_fn!(vroundss, 3, self, src1, src2, dst),
2130 Some(CpuFeature::SSE42) => {
2131 sse_round_fn!(roundss, 3, self, Precision::Single, src1, src2, dst)
2132 }
2133 _ => {}
2134 }
2135 Ok(())
2136 }
2137 fn emit_vroundsd_trunc(
2138 &mut self,
2139 src1: XMM,
2140 src2: XMMOrMemory,
2141 dst: XMM,
2142 ) -> Result<(), CompileError> {
2143 match self.get_simd_arch() {
2144 Some(CpuFeature::AVX) => avx_round_fn!(vroundsd, 3, self, src1, src2, dst),
2145 Some(CpuFeature::SSE42) => {
2146 sse_round_fn!(roundsd, 3, self, Precision::Double, src1, src2, dst)
2147 }
2148 _ => {}
2149 }
2150 Ok(())
2151 }
2152 fn emit_vcvtsi2ss_32(
2153 &mut self,
2154 src1: XMM,
2155 src2: GPROrMemory,
2156 dst: XMM,
2157 ) -> Result<(), CompileError> {
2158 match self.get_simd_arch() {
2159 Some(CpuFeature::AVX) => avx_i2f_32_fn!(vcvtsi2ss, self, src1, src2, dst),
2160 Some(CpuFeature::SSE42) => {
2161 sse_i2f_32_fn!(cvtsi2ss, self, Precision::Single, src1, src2, dst)
2162 }
2163 _ => {}
2164 }
2165 Ok(())
2166 }
2167 fn emit_vcvtsi2sd_32(
2168 &mut self,
2169 src1: XMM,
2170 src2: GPROrMemory,
2171 dst: XMM,
2172 ) -> Result<(), CompileError> {
2173 match self.get_simd_arch() {
2174 Some(CpuFeature::AVX) => avx_i2f_32_fn!(vcvtsi2sd, self, src1, src2, dst),
2175 Some(CpuFeature::SSE42) => {
2176 sse_i2f_32_fn!(cvtsi2sd, self, Precision::Double, src1, src2, dst)
2177 }
2178 _ => {}
2179 }
2180 Ok(())
2181 }
2182 fn emit_vcvtsi2ss_64(
2183 &mut self,
2184 src1: XMM,
2185 src2: GPROrMemory,
2186 dst: XMM,
2187 ) -> Result<(), CompileError> {
2188 match self.get_simd_arch() {
2189 Some(CpuFeature::AVX) => avx_i2f_64_fn!(vcvtsi2ss, self, src1, src2, dst),
2190 Some(CpuFeature::SSE42) => {
2191 sse_i2f_64_fn!(cvtsi2ss, self, Precision::Single, src1, src2, dst)
2192 }
2193 _ => {}
2194 }
2195 Ok(())
2196 }
2197 fn emit_vcvtsi2sd_64(
2198 &mut self,
2199 src1: XMM,
2200 src2: GPROrMemory,
2201 dst: XMM,
2202 ) -> Result<(), CompileError> {
2203 match self.get_simd_arch() {
2204 Some(CpuFeature::AVX) => avx_i2f_64_fn!(vcvtsi2sd, self, src1, src2, dst),
2205 Some(CpuFeature::SSE42) => {
2206 sse_i2f_64_fn!(cvtsi2sd, self, Precision::Double, src1, src2, dst)
2207 }
2208 _ => {}
2209 }
2210 Ok(())
2211 }
2212
2213 fn emit_vblendvps(
2214 &mut self,
2215 src1: XMM,
2216 src2: XMMOrMemory,
2217 mask: XMM,
2218 dst: XMM,
2219 ) -> Result<(), CompileError> {
2220 match self.get_simd_arch() {
2222 Some(CpuFeature::AVX) => match src2 {
2223 XMMOrMemory::XMM(src2) => {
2224 dynasm!( self; vblendvps Rx(dst), Rx(mask), Rx(src2), Rx(src1))
2226 }
2227 XMMOrMemory::Memory(base, disp) => {
2228 dynasm!( self; vblendvps Rx(dst), Rx(mask), [Rq(base) + disp], Rx(src1))
2229 }
2230 },
2231 Some(CpuFeature::SSE42) => match src2 {
2232 XMMOrMemory::XMM(src2) => {
2233 move_src_to_dst(self, Precision::Single, src1, dst);
2234 dynasm!( self; blendvps Rx(dst), Rx(src2))
2235 }
2236 XMMOrMemory::Memory(base, disp) => {
2237 move_src_to_dst(self, Precision::Single, src1, dst);
2238 dynasm!( self; blendvps Rx(dst), [Rq(base) + disp])
2239 }
2240 },
2241 _ => {}
2242 }
2243 Ok(())
2244 }
2245
2246 fn emit_vblendvpd(
2247 &mut self,
2248 src1: XMM,
2249 src2: XMMOrMemory,
2250 mask: XMM,
2251 dst: XMM,
2252 ) -> Result<(), CompileError> {
2253 match self.get_simd_arch() {
2255 Some(CpuFeature::AVX) => match src2 {
2256 XMMOrMemory::XMM(src2) => {
2257 dynasm!( self; vblendvpd Rx(dst), Rx(mask), Rx(src2), Rx(src1))
2259 }
2260 XMMOrMemory::Memory(base, disp) => {
2261 dynasm!( self; vblendvpd Rx(dst), Rx(mask), [Rq(base) + disp], Rx(src1))
2262 }
2263 },
2264 Some(CpuFeature::SSE42) => match src2 {
2265 XMMOrMemory::XMM(src2) => {
2266 move_src_to_dst(self, Precision::Double, src1, dst);
2267 dynasm!( self; blendvpd Rx(dst), Rx(src2))
2268 }
2269 XMMOrMemory::Memory(base, disp) => {
2270 move_src_to_dst(self, Precision::Double, src1, dst);
2271 dynasm!( self; blendvpd Rx(dst), [Rq(base) + disp])
2272 }
2273 },
2274 _ => {}
2275 }
2276 Ok(())
2277 }
2278
2279 fn emit_ucomiss(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
2280 match src {
2281 XMMOrMemory::XMM(x) => dynasm!(self ; ucomiss Rx(dst), Rx(x)),
2282 XMMOrMemory::Memory(base, disp) => {
2283 dynasm!(self ; ucomiss Rx(dst), [Rq(base) + disp])
2284 }
2285 }
2286 Ok(())
2287 }
2288
2289 fn emit_ucomisd(&mut self, src: XMMOrMemory, dst: XMM) -> Result<(), CompileError> {
2290 match src {
2291 XMMOrMemory::XMM(x) => dynasm!(self ; ucomisd Rx(dst), Rx(x)),
2292 XMMOrMemory::Memory(base, disp) => {
2293 dynasm!(self ; ucomisd Rx(dst), [Rq(base) + disp])
2294 }
2295 }
2296 Ok(())
2297 }
2298
2299 fn emit_cvttss2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2300 match src {
2301 XMMOrMemory::XMM(x) => dynasm!(self ; cvttss2si Rd(dst), Rx(x)),
2302 XMMOrMemory::Memory(base, disp) => {
2303 dynasm!(self ; cvttss2si Rd(dst), [Rq(base) + disp])
2304 }
2305 }
2306 Ok(())
2307 }
2308
2309 fn emit_cvttss2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2310 match src {
2311 XMMOrMemory::XMM(x) => dynasm!(self ; cvttss2si Rq(dst), Rx(x)),
2312 XMMOrMemory::Memory(base, disp) => {
2313 dynasm!(self ; cvttss2si Rq(dst), [Rq(base) + disp])
2314 }
2315 }
2316 Ok(())
2317 }
2318
2319 fn emit_cvttsd2si_32(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2320 match src {
2321 XMMOrMemory::XMM(x) => dynasm!(self ; cvttsd2si Rd(dst), Rx(x)),
2322 XMMOrMemory::Memory(base, disp) => {
2323 dynasm!(self ; cvttsd2si Rd(dst), [Rq(base) + disp])
2324 }
2325 }
2326 Ok(())
2327 }
2328
2329 fn emit_cvttsd2si_64(&mut self, src: XMMOrMemory, dst: GPR) -> Result<(), CompileError> {
2330 match src {
2331 XMMOrMemory::XMM(x) => dynasm!(self ; cvttsd2si Rq(dst), Rx(x)),
2332 XMMOrMemory::Memory(base, disp) => {
2333 dynasm!(self ; cvttsd2si Rq(dst), [Rq(base) + disp])
2334 }
2335 }
2336 Ok(())
2337 }
2338
2339 fn emit_test_gpr_64(&mut self, reg: GPR) -> Result<(), CompileError> {
2340 dynasm!(self ; test Rq(reg), Rq(reg));
2341 Ok(())
2342 }
2343
2344 fn emit_ud2(&mut self) -> Result<(), CompileError> {
2345 dynasm!(self ; ud2);
2346 Ok(())
2347 }
2348 #[allow(clippy::useless_conversion)]
2349 fn emit_ud1_payload(&mut self, payload: u8) -> Result<(), CompileError> {
2350 assert!(payload & 0xf0 == 0);
2351 let reg1 = (payload >> 3) & 1;
2352 let reg2 = payload & 7;
2353
2354 dynasm!(self ; ud1 Rd(reg1), Rd(reg2));
2355 Ok(())
2356 }
2357 fn emit_ret(&mut self) -> Result<(), CompileError> {
2358 dynasm!(self ; ret);
2359 Ok(())
2360 }
2361
2362 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2363 dynasm!(self ; call =>label);
2364 Ok(())
2365 }
2366 fn emit_call_location(&mut self, loc: Location) -> Result<(), CompileError> {
2367 match loc {
2368 Location::GPR(x) => dynasm!(self ; call Rq(x)),
2369 Location::Memory(base, disp) => dynasm!(self ; call QWORD [Rq(base) + disp]),
2370 _ => codegen_error!("singlepass can't emit CALL {:?}", loc),
2371 }
2372 Ok(())
2373 }
2374
2375 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2376 dynasm!(self ; call Rq(reg));
2377 Ok(())
2378 }
2379
2380 fn emit_bkpt(&mut self) -> Result<(), CompileError> {
2381 dynasm!(self ; int3);
2382 Ok(())
2383 }
2384
2385 fn emit_host_redirection(&mut self, target: GPR) -> Result<(), CompileError> {
2386 self.emit_jmp_location(Location::GPR(target))
2387 }
2388
2389 fn arch_mov64_imm_offset(&self) -> Result<usize, CompileError> {
2390 Ok(2)
2391 }
2392}