1use crate::abi::LLVMAbi;
8use crate::error::err;
9use inkwell::values::BasicMetadataValueEnum;
10use inkwell::{
11 AddressSpace,
12 attributes::{Attribute, AttributeLoc},
13 builder::Builder,
14 context::Context,
15 module::{Linkage, Module},
16 targets::TargetData,
17 types::{
18 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
19 PointerType, StructType, VectorType, VoidType,
20 },
21 values::{
22 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
23 PointerValue, VectorValue,
24 },
25};
26use std::collections::{HashMap, hash_map::Entry};
27use std::num::NonZero;
28use target_lexicon::{Architecture, Triple};
29use wasmer_types::entity::{EntityRef, PrimaryMap};
30use wasmer_types::{
31 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
32 LocalTableIndex, MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, TableIndex, Type,
33};
34use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
35
36pub fn type_to_llvm<'ctx>(
37 intrinsics: &Intrinsics<'ctx>,
38 ty: Type,
39) -> Result<BasicTypeEnum<'ctx>, CompileError> {
40 match ty {
41 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
42 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
43 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
44 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
45 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
46 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
47 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
48 }
49}
50
51#[allow(dead_code)]
53pub struct X86_64Intrinsics<'ctx> {
54 pub pshufb128: FunctionValue<'ctx>,
55 pub pmaddubsw128: FunctionValue<'ctx>,
56 pub pmaddwd128: FunctionValue<'ctx>,
57 pub pmulhrsw128: FunctionValue<'ctx>,
58 pub pblendvb: FunctionValue<'ctx>,
59 pub min_ps: FunctionValue<'ctx>,
60 pub min_pd: FunctionValue<'ctx>,
61 pub max_ps: FunctionValue<'ctx>,
62 pub max_pd: FunctionValue<'ctx>,
63 pub cvttps2dq: FunctionValue<'ctx>,
64 pub cvtps2udq128: FunctionValue<'ctx>,
65 pub cvtpd2dq: FunctionValue<'ctx>,
66 pub cvtpd2udq128: FunctionValue<'ctx>,
67}
68
69#[allow(dead_code)]
71pub struct Intrinsics<'ctx> {
72 pub ctlz_i32: FunctionValue<'ctx>,
73 pub ctlz_i64: FunctionValue<'ctx>,
74
75 pub cttz_i32: FunctionValue<'ctx>,
76 pub cttz_i64: FunctionValue<'ctx>,
77
78 pub ctpop_i32: FunctionValue<'ctx>,
79 pub ctpop_i64: FunctionValue<'ctx>,
80 pub ctpop_i8x16: FunctionValue<'ctx>,
81
82 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
83 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
84 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
85 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
86 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
87
88 pub add_f32: FunctionValue<'ctx>,
89 pub add_f64: FunctionValue<'ctx>,
90 pub add_f32x4: FunctionValue<'ctx>,
91 pub add_f64x2: FunctionValue<'ctx>,
92
93 pub sub_f32: FunctionValue<'ctx>,
94 pub sub_f64: FunctionValue<'ctx>,
95 pub sub_f32x4: FunctionValue<'ctx>,
96 pub sub_f64x2: FunctionValue<'ctx>,
97
98 pub mul_f32: FunctionValue<'ctx>,
99 pub mul_f64: FunctionValue<'ctx>,
100 pub mul_f32x4: FunctionValue<'ctx>,
101 pub mul_f64x2: FunctionValue<'ctx>,
102 pub muladd_f32x4: FunctionValue<'ctx>,
103 pub muladd_f64x2: FunctionValue<'ctx>,
104
105 pub div_f32: FunctionValue<'ctx>,
106 pub div_f64: FunctionValue<'ctx>,
107 pub div_f32x4: FunctionValue<'ctx>,
108 pub div_f64x2: FunctionValue<'ctx>,
109
110 pub sqrt_f32: FunctionValue<'ctx>,
111 pub sqrt_f64: FunctionValue<'ctx>,
112 pub sqrt_f32x4: FunctionValue<'ctx>,
113 pub sqrt_f64x2: FunctionValue<'ctx>,
114
115 pub cmp_f32: FunctionValue<'ctx>,
116 pub cmp_f64: FunctionValue<'ctx>,
117 pub cmp_f32x4: FunctionValue<'ctx>,
118 pub cmp_f64x2: FunctionValue<'ctx>,
119
120 pub minimum_f32: FunctionValue<'ctx>,
121 pub minimum_f64: FunctionValue<'ctx>,
122 pub minimum_f32x4: FunctionValue<'ctx>,
123 pub minimum_f64x2: FunctionValue<'ctx>,
124
125 pub maximum_f32: FunctionValue<'ctx>,
126 pub maximum_f64: FunctionValue<'ctx>,
127 pub maximum_f32x4: FunctionValue<'ctx>,
128 pub maximum_f64x2: FunctionValue<'ctx>,
129
130 pub ceil_f32: FunctionValue<'ctx>,
131 pub ceil_f64: FunctionValue<'ctx>,
132 pub ceil_f32x4: FunctionValue<'ctx>,
133 pub ceil_f64x2: FunctionValue<'ctx>,
134
135 pub floor_f32: FunctionValue<'ctx>,
136 pub floor_f64: FunctionValue<'ctx>,
137 pub floor_f32x4: FunctionValue<'ctx>,
138 pub floor_f64x2: FunctionValue<'ctx>,
139
140 pub trunc_f32: FunctionValue<'ctx>,
141 pub trunc_f64: FunctionValue<'ctx>,
142 pub trunc_f32x4: FunctionValue<'ctx>,
143 pub trunc_f64x2: FunctionValue<'ctx>,
144
145 pub fpext_f32: FunctionValue<'ctx>,
146 pub fptrunc_f64: FunctionValue<'ctx>,
147
148 pub nearbyint_f32: FunctionValue<'ctx>,
149 pub nearbyint_f64: FunctionValue<'ctx>,
150 pub nearbyint_f32x4: FunctionValue<'ctx>,
151 pub nearbyint_f64x2: FunctionValue<'ctx>,
152
153 pub fabs_f32: FunctionValue<'ctx>,
154 pub fabs_f64: FunctionValue<'ctx>,
155 pub fabs_f32x4: FunctionValue<'ctx>,
156 pub fabs_f64x2: FunctionValue<'ctx>,
157
158 pub copysign_f32: FunctionValue<'ctx>,
159 pub copysign_f64: FunctionValue<'ctx>,
160 pub copysign_f32x4: FunctionValue<'ctx>,
161 pub copysign_f64x2: FunctionValue<'ctx>,
162
163 pub sadd_sat_i8x16: FunctionValue<'ctx>,
164 pub sadd_sat_i16x8: FunctionValue<'ctx>,
165 pub uadd_sat_i8x16: FunctionValue<'ctx>,
166 pub uadd_sat_i16x8: FunctionValue<'ctx>,
167
168 pub ssub_sat_i8x16: FunctionValue<'ctx>,
169 pub ssub_sat_i16x8: FunctionValue<'ctx>,
170 pub usub_sat_i8x16: FunctionValue<'ctx>,
171 pub usub_sat_i16x8: FunctionValue<'ctx>,
172
173 pub expect_i1: FunctionValue<'ctx>,
174 pub trap: FunctionValue<'ctx>,
175 pub debug_trap: FunctionValue<'ctx>,
176
177 pub personality: FunctionValue<'ctx>,
178 pub personality2: FunctionValue<'ctx>,
179 pub readonly: Attribute,
180 pub stack_probe: Attribute,
181 pub uwtable: Attribute,
182 pub nounwind: Attribute,
183 pub frame_pointer: Attribute,
184 pub chkstk: FunctionValue<'ctx>,
186
187 pub void_ty: VoidType<'ctx>,
188 pub i1_ty: IntType<'ctx>,
189 pub i2_ty: IntType<'ctx>,
190 pub i4_ty: IntType<'ctx>,
191 pub i8_ty: IntType<'ctx>,
192 pub i16_ty: IntType<'ctx>,
193 pub i32_ty: IntType<'ctx>,
194 pub i64_ty: IntType<'ctx>,
195 pub i128_ty: IntType<'ctx>,
196 pub isize_ty: IntType<'ctx>,
197 pub f32_ty: FloatType<'ctx>,
198 pub f64_ty: FloatType<'ctx>,
199
200 pub i1x128_ty: VectorType<'ctx>,
201 pub i8x16_ty: VectorType<'ctx>,
202 pub i16x8_ty: VectorType<'ctx>,
203 pub i32x4_ty: VectorType<'ctx>,
204 pub i64x2_ty: VectorType<'ctx>,
205 pub f32x4_ty: VectorType<'ctx>,
206 pub f64x2_ty: VectorType<'ctx>,
207 pub i32x8_ty: VectorType<'ctx>,
208
209 pub ptr_ty: PointerType<'ctx>,
210
211 pub x86_64: X86_64Intrinsics<'ctx>,
212
213 pub anyfunc_ty: StructType<'ctx>,
214
215 pub i1_zero: IntValue<'ctx>,
216 pub i8_zero: IntValue<'ctx>,
217 pub i32_zero: IntValue<'ctx>,
218 pub i64_zero: IntValue<'ctx>,
219 pub i128_zero: IntValue<'ctx>,
220 pub isize_zero: IntValue<'ctx>,
221 pub f32_zero: FloatValue<'ctx>,
222 pub f64_zero: FloatValue<'ctx>,
223 pub f32x4_zero: VectorValue<'ctx>,
224 pub f64x2_zero: VectorValue<'ctx>,
225 pub i32_consts: [IntValue<'ctx>; 16],
226
227 pub trap_unreachable: BasicValueEnum<'ctx>,
228 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
229 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
230 pub trap_memory_oob: BasicValueEnum<'ctx>,
231 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
232 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
233 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
234 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
235 pub trap_table_access_oob: BasicValueEnum<'ctx>,
236
237 pub experimental_stackmap: FunctionValue<'ctx>,
238
239 pub table_copy: FunctionValue<'ctx>,
241 pub table_init: FunctionValue<'ctx>,
242 pub table_fill: FunctionValue<'ctx>,
243 pub table_size: FunctionValue<'ctx>,
244 pub imported_table_size: FunctionValue<'ctx>,
245 pub table_get: FunctionValue<'ctx>,
246 pub imported_table_get: FunctionValue<'ctx>,
247 pub table_set: FunctionValue<'ctx>,
248 pub imported_table_set: FunctionValue<'ctx>,
249 pub table_grow: FunctionValue<'ctx>,
250 pub imported_table_grow: FunctionValue<'ctx>,
251 pub memory_init: FunctionValue<'ctx>,
252 pub data_drop: FunctionValue<'ctx>,
253 pub func_ref: FunctionValue<'ctx>,
254 pub elem_drop: FunctionValue<'ctx>,
255 pub memory_copy: FunctionValue<'ctx>,
256 pub memory_fill: FunctionValue<'ctx>,
257 pub imported_memory_fill: FunctionValue<'ctx>,
258 pub memory_size_ty: FunctionType<'ctx>,
259 pub memory_grow_ty: FunctionType<'ctx>,
260 pub memory_wait32: FunctionValue<'ctx>,
261 pub memory_wait32_ty: FunctionType<'ctx>,
262 pub imported_memory_wait32: FunctionValue<'ctx>,
263 pub memory_wait64: FunctionValue<'ctx>,
264 pub memory_wait64_ty: FunctionType<'ctx>,
265 pub imported_memory_wait64: FunctionValue<'ctx>,
266 pub memory_notify: FunctionValue<'ctx>,
267 pub memory_notify_ty: FunctionType<'ctx>,
268 pub imported_memory_notify: FunctionValue<'ctx>,
269
270 pub throw_trap: FunctionValue<'ctx>,
271
272 pub throw: FunctionValue<'ctx>,
274 pub alloc_exception: FunctionValue<'ctx>,
275 pub read_exnref: FunctionValue<'ctx>,
276 pub exception_into_exnref: FunctionValue<'ctx>,
277 pub lpad_exception_ty: StructType<'ctx>,
278
279 pub debug_ptr: FunctionValue<'ctx>,
281 pub debug_str: FunctionValue<'ctx>,
282
283 pub vmfunction_import_ty: StructType<'ctx>,
285 pub vmfunction_import_body_element: u32,
286 pub vmfunction_import_vmctx_element: u32,
287 pub vmfunction_import_include_m0_param_element: u32,
288
289 pub vmmemory_definition_ty: StructType<'ctx>,
290 pub vmmemory_definition_base_element: u32,
291 pub vmmemory_definition_current_length_element: u32,
292}
293
294#[derive(Debug, Hash, PartialEq, Eq)]
295enum MemoryOp {
296 Size,
297 Grow,
298 Wait32,
299 Wait64,
300 Notify,
301}
302
303impl<'ctx> Intrinsics<'ctx> {
304 pub fn declare(
306 module: &Module<'ctx>,
307 context: &'ctx Context,
308 target_data: &TargetData,
309 target_triple: &Triple,
310 binary_fmt: &target_lexicon::BinaryFormat,
311 ) -> Self {
312 let is_riscv64 = matches!(target_triple.architecture, Architecture::Riscv64(..));
313 let void_ty = context.void_type();
314 let i1_ty = context.bool_type();
315 let i2_ty = context
316 .custom_width_int_type(NonZero::new(2).unwrap())
317 .unwrap();
318 let i4_ty = context
319 .custom_width_int_type(NonZero::new(4).unwrap())
320 .unwrap();
321 let i8_ty = context.i8_type();
322 let i16_ty = context.i16_type();
323 let i32_ty = context.i32_type();
324 let i64_ty = context.i64_type();
325 let i128_ty = context.i128_type();
326 let isize_ty = context.ptr_sized_int_type(target_data, None);
327 let f32_ty = context.f32_type();
328 let f64_ty = context.f64_type();
329
330 let i1x4_ty = i1_ty.vec_type(4);
331 let i1x2_ty = i1_ty.vec_type(2);
332 let i1x128_ty = i1_ty.vec_type(128);
333 let i8x16_ty = i8_ty.vec_type(16);
334 let i16x8_ty = i16_ty.vec_type(8);
335 let i32x4_ty = i32_ty.vec_type(4);
336 let i64x2_ty = i64_ty.vec_type(2);
337 let f32x4_ty = f32_ty.vec_type(4);
338 let f64x2_ty = f64_ty.vec_type(2);
339 let i32x8_ty = i32_ty.vec_type(8);
340
341 let ptr_ty = context.ptr_type(AddressSpace::default());
342
343 let i1_zero = i1_ty.const_int(0, false);
344 let i8_zero = i8_ty.const_int(0, false);
345 let i32_zero = i32_ty.const_int(0, false);
346 let i64_zero = i64_ty.const_int(0, false);
347 let i128_zero = i128_ty.const_int(0, false);
348 let isize_zero = isize_ty.const_int(0, false);
349 let f32_zero = f32_ty.const_float(0.0);
350 let f64_zero = f64_ty.const_float(0.0);
351 let f32x4_zero = f32x4_ty.const_zero();
352 let f64x2_zero = f64x2_ty.const_zero();
353 let i32_consts = [
354 i32_ty.const_int(0, false),
355 i32_ty.const_int(1, false),
356 i32_ty.const_int(2, false),
357 i32_ty.const_int(3, false),
358 i32_ty.const_int(4, false),
359 i32_ty.const_int(5, false),
360 i32_ty.const_int(6, false),
361 i32_ty.const_int(7, false),
362 i32_ty.const_int(8, false),
363 i32_ty.const_int(9, false),
364 i32_ty.const_int(10, false),
365 i32_ty.const_int(11, false),
366 i32_ty.const_int(12, false),
367 i32_ty.const_int(13, false),
368 i32_ty.const_int(14, false),
369 i32_ty.const_int(15, false),
370 ];
371
372 let md_ty = context.metadata_type();
373
374 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
375 let i1_ty_basic = i1_ty.as_basic_type_enum();
376
377 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
378 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
379 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
380 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
381 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
382 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
383 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
384 let i32x4_ty_basic_md: BasicMetadataTypeEnum = i32x4_ty.into();
385 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
386 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
387 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
388
389 let ctx_ptr_ty = ptr_ty;
390 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
391 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
392
393 let anyfunc_ty = context.struct_type(
397 &[
398 i8_ptr_ty_basic,
399 i32_ty.into(),
400 ctx_ptr_ty_basic,
401 ptr_ty.into(),
402 ],
403 false,
404 );
405 let funcref_ty = ptr_ty;
406 let anyref_ty = ptr_ty;
407 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
408
409 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
410 let ret_i8x16_take_i8x16_i8x16 =
411 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
412 let ret_i8x16_take_i8x16_i8x16_i8x16 = i8x16_ty.fn_type(
413 &[i8x16_ty_basic_md, i8x16_ty_basic_md, i8x16_ty_basic_md],
414 false,
415 );
416 let ret_i16x8_take_i16x8_i16x8 =
417 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
418
419 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
420 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
421
422 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
423 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
424
425 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
426 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
427 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
428 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
429
430 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
431 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
432 let ret_f32x4_take_f32x4_f32x4 =
433 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
434 let ret_f64x2_take_f64x2_f64x2 =
435 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
436 let ret_f32x4_take_f32x4_f32x4_f32x4_md_md = f32x4_ty.fn_type(
437 &[
438 f32x4_ty_basic_md,
439 f32x4_ty_basic_md,
440 f32x4_ty_basic_md,
441 md_ty_basic_md,
442 md_ty_basic_md,
443 ],
444 false,
445 );
446 let ret_f64x2_take_f64x2_f64x2_f64x2_md_md = f64x2_ty.fn_type(
447 &[
448 f64x2_ty_basic_md,
449 f64x2_ty_basic_md,
450 f64x2_ty_basic_md,
451 md_ty_basic_md,
452 md_ty_basic_md,
453 ],
454 false,
455 );
456
457 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
458 let ret_f32_take_f64_md_md =
459 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
460
461 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
462
463 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
464 &[
465 f32_ty_basic_md,
466 f32_ty_basic_md,
467 md_ty_basic_md,
468 md_ty_basic_md,
469 ],
470 false,
471 );
472 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
473 &[
474 f64_ty_basic_md,
475 f64_ty_basic_md,
476 md_ty_basic_md,
477 md_ty_basic_md,
478 ],
479 false,
480 );
481 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
482 &[
483 f32x4_ty_basic_md,
484 f32x4_ty_basic_md,
485 md_ty_basic_md,
486 md_ty_basic_md,
487 ],
488 false,
489 );
490 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
491 &[
492 f64x2_ty_basic_md,
493 f64x2_ty_basic_md,
494 md_ty_basic_md,
495 md_ty_basic_md,
496 ],
497 false,
498 );
499
500 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
501 &[
502 f32_ty_basic_md,
503 f32_ty_basic_md,
504 md_ty_basic_md,
505 md_ty_basic_md,
506 ],
507 false,
508 );
509 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
510 &[
511 f64_ty_basic_md,
512 f64_ty_basic_md,
513 md_ty_basic_md,
514 md_ty_basic_md,
515 ],
516 false,
517 );
518 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
519 &[
520 f32x4_ty_basic_md,
521 f32x4_ty_basic_md,
522 md_ty_basic_md,
523 md_ty_basic_md,
524 ],
525 false,
526 );
527 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
528 &[
529 f64x2_ty_basic_md,
530 f64x2_ty_basic_md,
531 md_ty_basic_md,
532 md_ty_basic_md,
533 ],
534 false,
535 );
536 let ret_i32x4_take_f32x4 = i32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
537 let ret_i32x4_take_f32x4_i32x4_i8 =
538 i32x4_ty.fn_type(&[f32x4_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
539 let ret_i32x4_take_f64x2 = i32x4_ty.fn_type(&[f64x2_ty_basic_md], false);
540 let ret_i32x4_take_f64x2_i32x4_i8 =
541 i32x4_ty.fn_type(&[f64x2_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
542
543 let add_function_with_attrs =
544 |name: &str, ty: FunctionType<'ctx>, linkage: Option<Linkage>| -> FunctionValue<'ctx> {
545 let function = module.add_function(name, ty, linkage);
546 if is_riscv64 {
551 for (i, param) in ty.get_param_types().iter().enumerate() {
552 if param == &i32_ty_basic_md {
553 function.add_attribute(
554 AttributeLoc::Param(i as u32),
555 context.create_enum_attribute(
556 Attribute::get_named_enum_kind_id("signext"),
557 0,
558 ),
559 );
560 function.add_attribute(
561 AttributeLoc::Param(i as u32),
562 context.create_enum_attribute(
563 Attribute::get_named_enum_kind_id("noundef"),
564 0,
565 ),
566 );
567 }
568 }
569 }
570 function
571 };
572
573 let intrinsics = Self {
574 ctlz_i32: add_function_with_attrs("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
575 ctlz_i64: add_function_with_attrs("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
576
577 cttz_i32: add_function_with_attrs("llvm.cttz.i32", ret_i32_take_i32_i1, None),
578 cttz_i64: add_function_with_attrs("llvm.cttz.i64", ret_i64_take_i64_i1, None),
579
580 ctpop_i32: add_function_with_attrs("llvm.ctpop.i32", ret_i32_take_i32, None),
581 ctpop_i64: add_function_with_attrs("llvm.ctpop.i64", ret_i64_take_i64, None),
582 ctpop_i8x16: add_function_with_attrs("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
583
584 fp_rounding_md: context.metadata_string("round.tonearest").into(),
585 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
586
587 fp_ogt_md: context.metadata_string("ogt").into(),
588 fp_olt_md: context.metadata_string("olt").into(),
589 fp_uno_md: context.metadata_string("uno").into(),
590
591 sqrt_f32: add_function_with_attrs("llvm.sqrt.f32", ret_f32_take_f32, None),
592 sqrt_f64: add_function_with_attrs("llvm.sqrt.f64", ret_f64_take_f64, None),
593 sqrt_f32x4: add_function_with_attrs("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
594 sqrt_f64x2: add_function_with_attrs("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
595
596 ceil_f32: add_function_with_attrs("llvm.ceil.f32", ret_f32_take_f32, None),
597 ceil_f64: add_function_with_attrs("llvm.ceil.f64", ret_f64_take_f64, None),
598 ceil_f32x4: add_function_with_attrs("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
599 ceil_f64x2: add_function_with_attrs("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
600
601 floor_f32: add_function_with_attrs("llvm.floor.f32", ret_f32_take_f32, None),
602 floor_f64: add_function_with_attrs("llvm.floor.f64", ret_f64_take_f64, None),
603 floor_f32x4: add_function_with_attrs("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
604 floor_f64x2: add_function_with_attrs("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
605
606 trunc_f32: add_function_with_attrs("llvm.trunc.f32", ret_f32_take_f32, None),
607 trunc_f64: add_function_with_attrs("llvm.trunc.f64", ret_f64_take_f64, None),
608 trunc_f32x4: add_function_with_attrs("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
609 trunc_f64x2: add_function_with_attrs("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
610
611 nearbyint_f32: add_function_with_attrs("llvm.nearbyint.f32", ret_f32_take_f32, None),
612 nearbyint_f64: add_function_with_attrs("llvm.nearbyint.f64", ret_f64_take_f64, None),
613 nearbyint_f32x4: add_function_with_attrs(
614 "llvm.nearbyint.v4f32",
615 ret_f32x4_take_f32x4,
616 None,
617 ),
618 nearbyint_f64x2: add_function_with_attrs(
619 "llvm.nearbyint.v2f64",
620 ret_f64x2_take_f64x2,
621 None,
622 ),
623
624 add_f32: add_function_with_attrs(
625 "llvm.experimental.constrained.fadd.f32",
626 ret_f32_take_f32_f32_md_md,
627 None,
628 ),
629 add_f64: add_function_with_attrs(
630 "llvm.experimental.constrained.fadd.f64",
631 ret_f64_take_f64_f64_md_md,
632 None,
633 ),
634 add_f32x4: add_function_with_attrs(
635 "llvm.experimental.constrained.fadd.v4f32",
636 ret_f32x4_take_f32x4_f32x4_md_md,
637 None,
638 ),
639 add_f64x2: add_function_with_attrs(
640 "llvm.experimental.constrained.fadd.v2f64",
641 ret_f64x2_take_f64x2_f64x2_md_md,
642 None,
643 ),
644
645 sub_f32: add_function_with_attrs(
646 "llvm.experimental.constrained.fsub.f32",
647 ret_f32_take_f32_f32_md_md,
648 None,
649 ),
650 sub_f64: add_function_with_attrs(
651 "llvm.experimental.constrained.fsub.f64",
652 ret_f64_take_f64_f64_md_md,
653 None,
654 ),
655 sub_f32x4: add_function_with_attrs(
656 "llvm.experimental.constrained.fsub.v4f32",
657 ret_f32x4_take_f32x4_f32x4_md_md,
658 None,
659 ),
660 sub_f64x2: add_function_with_attrs(
661 "llvm.experimental.constrained.fsub.v2f64",
662 ret_f64x2_take_f64x2_f64x2_md_md,
663 None,
664 ),
665
666 mul_f32: add_function_with_attrs(
667 "llvm.experimental.constrained.fmul.f32",
668 ret_f32_take_f32_f32_md_md,
669 None,
670 ),
671 mul_f64: add_function_with_attrs(
672 "llvm.experimental.constrained.fmul.f64",
673 ret_f64_take_f64_f64_md_md,
674 None,
675 ),
676 mul_f32x4: add_function_with_attrs(
677 "llvm.experimental.constrained.fmul.v4f32",
678 ret_f32x4_take_f32x4_f32x4_md_md,
679 None,
680 ),
681 mul_f64x2: add_function_with_attrs(
682 "llvm.experimental.constrained.fmul.v2f64",
683 ret_f64x2_take_f64x2_f64x2_md_md,
684 None,
685 ),
686 muladd_f32x4: add_function_with_attrs(
687 "llvm.experimental.constrained.fmuladd.v4f32",
688 ret_f32x4_take_f32x4_f32x4_f32x4_md_md,
689 None,
690 ),
691 muladd_f64x2: add_function_with_attrs(
692 "llvm.experimental.constrained.fmuladd.v2f64",
693 ret_f64x2_take_f64x2_f64x2_f64x2_md_md,
694 None,
695 ),
696
697 div_f32: add_function_with_attrs(
698 "llvm.experimental.constrained.fdiv.f32",
699 ret_f32_take_f32_f32_md_md,
700 None,
701 ),
702 div_f64: add_function_with_attrs(
703 "llvm.experimental.constrained.fdiv.f64",
704 ret_f64_take_f64_f64_md_md,
705 None,
706 ),
707 div_f32x4: add_function_with_attrs(
708 "llvm.experimental.constrained.fdiv.v4f32",
709 ret_f32x4_take_f32x4_f32x4_md_md,
710 None,
711 ),
712 div_f64x2: add_function_with_attrs(
713 "llvm.experimental.constrained.fdiv.v2f64",
714 ret_f64x2_take_f64x2_f64x2_md_md,
715 None,
716 ),
717
718 cmp_f32: add_function_with_attrs(
719 "llvm.experimental.constrained.fcmp.f32",
720 ret_i1_take_f32_f32_md_md,
721 None,
722 ),
723 cmp_f64: add_function_with_attrs(
724 "llvm.experimental.constrained.fcmp.f64",
725 ret_i1_take_f64_f64_md_md,
726 None,
727 ),
728 cmp_f32x4: add_function_with_attrs(
729 "llvm.experimental.constrained.fcmp.v4f32",
730 ret_i1x4_take_f32x4_f32x4_md_md,
731 None,
732 ),
733 cmp_f64x2: add_function_with_attrs(
734 "llvm.experimental.constrained.fcmp.v2f64",
735 ret_i1x2_take_f64x2_f64x2_md_md,
736 None,
737 ),
738
739 minimum_f32: add_function_with_attrs("llvm.minimum.f32", ret_f32_take_f32_f32, None),
740 minimum_f64: add_function_with_attrs("llvm.minimum.f64", ret_f64_take_f64_f64, None),
741 minimum_f32x4: add_function_with_attrs(
742 "llvm.minimum.v4f32",
743 ret_f32x4_take_f32x4_f32x4,
744 None,
745 ),
746 minimum_f64x2: add_function_with_attrs(
747 "llvm.minimum.v2f64",
748 ret_f64x2_take_f64x2_f64x2,
749 None,
750 ),
751
752 maximum_f32: add_function_with_attrs("llvm.maximum.f32", ret_f32_take_f32_f32, None),
753 maximum_f64: add_function_with_attrs("llvm.maximum.f64", ret_f64_take_f64_f64, None),
754 maximum_f32x4: add_function_with_attrs(
755 "llvm.maximum.v4f32",
756 ret_f32x4_take_f32x4_f32x4,
757 None,
758 ),
759 maximum_f64x2: add_function_with_attrs(
760 "llvm.maximum.v2f64",
761 ret_f64x2_take_f64x2_f64x2,
762 None,
763 ),
764
765 fpext_f32: add_function_with_attrs(
766 "llvm.experimental.constrained.fpext.f64.f32",
767 ret_f64_take_f32_md,
768 None,
769 ),
770 fptrunc_f64: add_function_with_attrs(
771 "llvm.experimental.constrained.fptrunc.f32.f64",
772 ret_f32_take_f64_md_md,
773 None,
774 ),
775
776 fabs_f32: add_function_with_attrs("llvm.fabs.f32", ret_f32_take_f32, None),
777 fabs_f64: add_function_with_attrs("llvm.fabs.f64", ret_f64_take_f64, None),
778 fabs_f32x4: add_function_with_attrs("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
779 fabs_f64x2: add_function_with_attrs("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
780
781 copysign_f32: add_function_with_attrs("llvm.copysign.f32", ret_f32_take_f32_f32, None),
782 copysign_f64: add_function_with_attrs("llvm.copysign.f64", ret_f64_take_f64_f64, None),
783 copysign_f32x4: add_function_with_attrs(
784 "llvm.copysign.v4f32",
785 ret_f32x4_take_f32x4_f32x4,
786 None,
787 ),
788 copysign_f64x2: add_function_with_attrs(
789 "llvm.copysign.v2f64",
790 ret_f64x2_take_f64x2_f64x2,
791 None,
792 ),
793
794 sadd_sat_i8x16: add_function_with_attrs(
795 "llvm.sadd.sat.v16i8",
796 ret_i8x16_take_i8x16_i8x16,
797 None,
798 ),
799 sadd_sat_i16x8: add_function_with_attrs(
800 "llvm.sadd.sat.v8i16",
801 ret_i16x8_take_i16x8_i16x8,
802 None,
803 ),
804 uadd_sat_i8x16: add_function_with_attrs(
805 "llvm.uadd.sat.v16i8",
806 ret_i8x16_take_i8x16_i8x16,
807 None,
808 ),
809 uadd_sat_i16x8: add_function_with_attrs(
810 "llvm.uadd.sat.v8i16",
811 ret_i16x8_take_i16x8_i16x8,
812 None,
813 ),
814
815 ssub_sat_i8x16: add_function_with_attrs(
816 "llvm.ssub.sat.v16i8",
817 ret_i8x16_take_i8x16_i8x16,
818 None,
819 ),
820 ssub_sat_i16x8: add_function_with_attrs(
821 "llvm.ssub.sat.v8i16",
822 ret_i16x8_take_i16x8_i16x8,
823 None,
824 ),
825 usub_sat_i8x16: add_function_with_attrs(
826 "llvm.usub.sat.v16i8",
827 ret_i8x16_take_i8x16_i8x16,
828 None,
829 ),
830 usub_sat_i16x8: add_function_with_attrs(
831 "llvm.usub.sat.v8i16",
832 ret_i16x8_take_i16x8_i16x8,
833 None,
834 ),
835
836 expect_i1: add_function_with_attrs("llvm.expect.i1", ret_i1_take_i1_i1, None),
837 trap: add_function_with_attrs("llvm.trap", void_ty.fn_type(&[], false), None),
838 debug_trap: add_function_with_attrs(
839 "llvm.debugtrap",
840 void_ty.fn_type(&[], false),
841 None,
842 ),
843 personality: add_function_with_attrs(
844 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
845 "__gxx_personality_v0"
848 } else {
849 "wasmer_eh_personality"
850 },
851 i32_ty.fn_type(
852 &[
853 i32_ty.into(),
854 i32_ty.into(),
855 i64_ty.into(),
856 ptr_ty.into(),
857 ptr_ty.into(),
858 ],
859 false,
860 ),
861 None,
862 ),
863 personality2: add_function_with_attrs(
864 "wasmer_eh_personality2",
865 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
866 None,
867 ),
868 readonly: context
869 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
870 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
871 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
872 nounwind: context
873 .create_enum_attribute(Attribute::get_named_enum_kind_id("nounwind"), 1),
874 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
875 chkstk: add_function_with_attrs("__chkstk", void_ty.fn_type(&[], false), None),
876 void_ty,
877 i1_ty,
878 i2_ty,
879 i4_ty,
880 i8_ty,
881 i16_ty,
882 i32_ty,
883 i64_ty,
884 i128_ty,
885 isize_ty,
886 f32_ty,
887 f64_ty,
888
889 i1x128_ty,
890 i8x16_ty,
891 i16x8_ty,
892 i32x4_ty,
893 i64x2_ty,
894 f32x4_ty,
895 f64x2_ty,
896 i32x8_ty,
897
898 anyfunc_ty,
899 i1_zero,
900 i8_zero,
901 i32_zero,
902 i64_zero,
903 i128_zero,
904 isize_zero,
905 f32_zero,
906 f64_zero,
907 f32x4_zero,
908 f64x2_zero,
909 i32_consts,
910
911 trap_unreachable: i32_ty
912 .const_int(TrapCode::UnreachableCodeReached as _, false)
913 .as_basic_value_enum(),
914 trap_call_indirect_null: i32_ty
915 .const_int(TrapCode::IndirectCallToNull as _, false)
916 .as_basic_value_enum(),
917 trap_call_indirect_sig: i32_ty
918 .const_int(TrapCode::BadSignature as _, false)
919 .as_basic_value_enum(),
920 trap_memory_oob: i32_ty
921 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
922 .as_basic_value_enum(),
923 trap_illegal_arithmetic: i32_ty
924 .const_int(TrapCode::IntegerOverflow as _, false)
925 .as_basic_value_enum(),
926 trap_integer_division_by_zero: i32_ty
927 .const_int(TrapCode::IntegerDivisionByZero as _, false)
928 .as_basic_value_enum(),
929 trap_bad_conversion_to_integer: i32_ty
930 .const_int(TrapCode::BadConversionToInteger as _, false)
931 .as_basic_value_enum(),
932 trap_unaligned_atomic: i32_ty
933 .const_int(TrapCode::UnalignedAtomic as _, false)
934 .as_basic_value_enum(),
935 trap_table_access_oob: i32_ty
936 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
937 .as_basic_value_enum(),
938
939 experimental_stackmap: module.add_function(
943 "llvm.experimental.stackmap",
944 void_ty.fn_type(
945 &[
946 i64_ty_basic_md, i32_ty_basic_md, ],
949 true,
950 ),
951 None,
952 ),
953
954 table_copy: add_function_with_attrs(
956 "wasmer_vm_table_copy",
957 void_ty.fn_type(
958 &[
959 ctx_ptr_ty_basic_md,
960 i32_ty_basic_md,
961 i32_ty_basic_md,
962 i32_ty_basic_md,
963 i32_ty_basic_md,
964 i32_ty_basic_md,
965 ],
966 false,
967 ),
968 None,
969 ),
970 table_init: add_function_with_attrs(
971 "wasmer_vm_table_init",
972 void_ty.fn_type(
973 &[
974 ctx_ptr_ty_basic_md,
975 i32_ty_basic_md,
976 i32_ty_basic_md,
977 i32_ty_basic_md,
978 i32_ty_basic_md,
979 i32_ty_basic_md,
980 ],
981 false,
982 ),
983 None,
984 ),
985 table_fill: add_function_with_attrs(
986 "wasmer_vm_table_fill",
987 void_ty.fn_type(
988 &[
989 ctx_ptr_ty_basic_md,
990 i32_ty_basic_md,
991 i32_ty_basic_md,
992 anyref_ty_basic_md,
993 i32_ty_basic_md,
994 ],
995 false,
996 ),
997 None,
998 ),
999 table_size: add_function_with_attrs(
1000 "wasmer_vm_table_size",
1001 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1002 None,
1003 ),
1004 imported_table_size: add_function_with_attrs(
1005 "wasmer_vm_imported_table_size",
1006 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1007 None,
1008 ),
1009 table_get: add_function_with_attrs(
1010 "wasmer_vm_table_get",
1011 anyref_ty.fn_type(
1012 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1013 false,
1014 ),
1015 None,
1016 ),
1017 imported_table_get: add_function_with_attrs(
1018 "wasmer_vm_imported_table_get",
1019 anyref_ty.fn_type(
1020 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1021 false,
1022 ),
1023 None,
1024 ),
1025 table_set: add_function_with_attrs(
1026 "wasmer_vm_table_set",
1027 void_ty.fn_type(
1028 &[
1029 ctx_ptr_ty_basic_md,
1030 i32_ty_basic_md,
1031 i32_ty_basic_md,
1032 anyref_ty_basic_md,
1033 ],
1034 false,
1035 ),
1036 None,
1037 ),
1038 imported_table_set: add_function_with_attrs(
1039 "wasmer_vm_imported_table_set",
1040 void_ty.fn_type(
1041 &[
1042 ctx_ptr_ty_basic_md,
1043 i32_ty_basic_md,
1044 i32_ty_basic_md,
1045 anyref_ty_basic_md,
1046 ],
1047 false,
1048 ),
1049 None,
1050 ),
1051 table_grow: add_function_with_attrs(
1052 "wasmer_vm_table_grow",
1053 i32_ty.fn_type(
1054 &[
1055 ctx_ptr_ty_basic_md,
1056 anyref_ty_basic_md,
1057 i32_ty_basic_md,
1058 i32_ty_basic_md,
1059 ],
1060 false,
1061 ),
1062 None,
1063 ),
1064 imported_table_grow: add_function_with_attrs(
1065 "wasmer_vm_imported_table_grow",
1066 i32_ty.fn_type(
1067 &[
1068 ctx_ptr_ty_basic_md,
1069 anyref_ty_basic_md,
1070 i32_ty_basic_md,
1071 i32_ty_basic_md,
1072 ],
1073 false,
1074 ),
1075 None,
1076 ),
1077 memory_init: add_function_with_attrs(
1078 "wasmer_vm_memory32_init",
1079 void_ty.fn_type(
1080 &[
1081 ctx_ptr_ty_basic_md,
1082 i32_ty_basic_md,
1083 i32_ty_basic_md,
1084 i32_ty_basic_md,
1085 i32_ty_basic_md,
1086 i32_ty_basic_md,
1087 ],
1088 false,
1089 ),
1090 None,
1091 ),
1092 memory_copy: add_function_with_attrs(
1093 "wasmer_vm_memory32_copy",
1094 void_ty.fn_type(
1095 &[
1096 ctx_ptr_ty_basic_md,
1097 i32_ty_basic_md,
1098 i32_ty_basic_md,
1099 i32_ty_basic_md,
1100 i32_ty_basic_md,
1101 i32_ty_basic_md,
1102 ],
1103 false,
1104 ),
1105 None,
1106 ),
1107 memory_fill: add_function_with_attrs(
1108 "wasmer_vm_memory32_fill",
1109 void_ty.fn_type(
1110 &[
1111 ctx_ptr_ty_basic_md,
1112 i32_ty_basic_md,
1113 i32_ty_basic_md,
1114 i32_ty_basic_md,
1115 i32_ty_basic_md,
1116 ],
1117 false,
1118 ),
1119 None,
1120 ),
1121 imported_memory_fill: add_function_with_attrs(
1122 "wasmer_vm_imported_memory32_fill",
1123 void_ty.fn_type(
1124 &[
1125 ctx_ptr_ty_basic_md,
1126 i32_ty_basic_md,
1127 i32_ty_basic_md,
1128 i32_ty_basic_md,
1129 i32_ty_basic_md,
1130 ],
1131 false,
1132 ),
1133 None,
1134 ),
1135 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1136 memory_grow_ty: i32_ty.fn_type(
1137 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1138 false,
1139 ),
1140 data_drop: add_function_with_attrs(
1141 "wasmer_vm_data_drop",
1142 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1143 None,
1144 ),
1145 func_ref: add_function_with_attrs(
1146 "wasmer_vm_func_ref",
1147 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1148 None,
1149 ),
1150 elem_drop: add_function_with_attrs(
1151 "wasmer_vm_elem_drop",
1152 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1153 None,
1154 ),
1155 throw_trap: add_function_with_attrs(
1156 "wasmer_vm_raise_trap",
1157 void_ty.fn_type(&[i32_ty_basic_md], false),
1158 None,
1159 ),
1160
1161 throw: add_function_with_attrs(
1162 "wasmer_vm_throw",
1163 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1164 None,
1165 ),
1166 alloc_exception: add_function_with_attrs(
1167 "wasmer_vm_alloc_exception",
1168 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1169 None,
1170 ),
1171 read_exnref: add_function_with_attrs(
1172 "wasmer_vm_read_exnref",
1173 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1174 None,
1175 ),
1176 exception_into_exnref: add_function_with_attrs(
1177 "wasmer_vm_exception_into_exnref",
1178 i32_ty.fn_type(&[ptr_ty.into()], false),
1179 None,
1180 ),
1181 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1182
1183 debug_ptr: add_function_with_attrs(
1184 "wasmer_vm_dbg_usize",
1185 void_ty.fn_type(&[ptr_ty.into()], false),
1186 None,
1187 ),
1188 debug_str: add_function_with_attrs(
1189 "wasmer_vm_dbg_str",
1190 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1191 None,
1192 ),
1193 memory_wait32: add_function_with_attrs(
1194 "wasmer_vm_memory32_atomic_wait32",
1195 i32_ty.fn_type(
1196 &[
1197 ctx_ptr_ty_basic_md,
1198 i32_ty_basic_md,
1199 i32_ty_basic_md,
1200 i32_ty_basic_md,
1201 i64_ty_basic_md,
1202 ],
1203 false,
1204 ),
1205 None,
1206 ),
1207 memory_wait32_ty: i32_ty.fn_type(
1208 &[
1209 ctx_ptr_ty_basic_md,
1210 i32_ty_basic_md,
1211 i32_ty_basic_md,
1212 i32_ty_basic_md,
1213 i64_ty_basic_md,
1214 ],
1215 false,
1216 ),
1217 imported_memory_wait32: add_function_with_attrs(
1218 "wasmer_vm_imported_memory32_atomic_wait32",
1219 i32_ty.fn_type(
1220 &[
1221 ctx_ptr_ty_basic_md,
1222 i32_ty_basic_md,
1223 i32_ty_basic_md,
1224 i32_ty_basic_md,
1225 i64_ty_basic_md,
1226 ],
1227 false,
1228 ),
1229 None,
1230 ),
1231 memory_wait64: add_function_with_attrs(
1232 "wasmer_vm_memory32_atomic_wait64",
1233 i32_ty.fn_type(
1234 &[
1235 ctx_ptr_ty_basic_md,
1236 i32_ty_basic_md,
1237 i32_ty_basic_md,
1238 i64_ty_basic_md,
1239 i64_ty_basic_md,
1240 ],
1241 false,
1242 ),
1243 None,
1244 ),
1245 memory_wait64_ty: i32_ty.fn_type(
1246 &[
1247 ctx_ptr_ty_basic_md,
1248 i32_ty_basic_md,
1249 i32_ty_basic_md,
1250 i64_ty_basic_md,
1251 i64_ty_basic_md,
1252 ],
1253 false,
1254 ),
1255 imported_memory_wait64: add_function_with_attrs(
1256 "wasmer_vm_imported_memory32_atomic_wait64",
1257 i32_ty.fn_type(
1258 &[
1259 ctx_ptr_ty_basic_md,
1260 i32_ty_basic_md,
1261 i32_ty_basic_md,
1262 i64_ty_basic_md,
1263 i64_ty_basic_md,
1264 ],
1265 false,
1266 ),
1267 None,
1268 ),
1269 memory_notify: add_function_with_attrs(
1270 "wasmer_vm_memory32_atomic_notify",
1271 i32_ty.fn_type(
1272 &[
1273 ctx_ptr_ty_basic_md,
1274 i32_ty_basic_md,
1275 i32_ty_basic_md,
1276 i32_ty_basic_md,
1277 ],
1278 false,
1279 ),
1280 None,
1281 ),
1282 memory_notify_ty: i32_ty.fn_type(
1283 &[
1284 ctx_ptr_ty_basic_md,
1285 i32_ty_basic_md,
1286 i32_ty_basic_md,
1287 i32_ty_basic_md,
1288 ],
1289 false,
1290 ),
1291 imported_memory_notify: add_function_with_attrs(
1292 "wasmer_vm_imported_memory32_atomic_notify",
1293 i32_ty.fn_type(
1294 &[
1295 ctx_ptr_ty_basic_md,
1296 i32_ty_basic_md,
1297 i32_ty_basic_md,
1298 i32_ty_basic_md,
1299 ],
1300 false,
1301 ),
1302 None,
1303 ),
1304
1305 vmfunction_import_ty: context.struct_type(
1306 &[
1307 i8_ptr_ty_basic,
1308 i8_ptr_ty_basic,
1309 i8_ptr_ty_basic,
1310 i1_ty_basic,
1311 ],
1312 false,
1313 ),
1314 vmfunction_import_body_element: 0,
1315 vmfunction_import_vmctx_element: 1,
1316 vmfunction_import_include_m0_param_element: 3,
1317 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1318 vmmemory_definition_base_element: 0,
1319 vmmemory_definition_current_length_element: 1,
1320
1321 ptr_ty,
1322
1323 x86_64: X86_64Intrinsics {
1324 pshufb128: add_function_with_attrs(
1325 "llvm.x86.ssse3.pshuf.b.128",
1326 ret_i8x16_take_i8x16_i8x16,
1327 None,
1328 ),
1329 pmaddubsw128: add_function_with_attrs(
1330 "llvm.x86.ssse3.pmadd.ub.sw.128",
1331 i16x8_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false),
1332 None,
1333 ),
1334 pmaddwd128: add_function_with_attrs(
1335 "llvm.x86.sse2.pmadd.wd",
1336 i32x4_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false),
1337 None,
1338 ),
1339 pmulhrsw128: add_function_with_attrs(
1340 "llvm.x86.ssse3.pmul.hr.sw.128",
1341 ret_i16x8_take_i16x8_i16x8,
1342 None,
1343 ),
1344 pblendvb: add_function_with_attrs(
1345 "llvm.x86.sse41.pblendvb",
1346 ret_i8x16_take_i8x16_i8x16_i8x16,
1347 None,
1348 ),
1349 min_ps: add_function_with_attrs(
1350 "llvm.x86.sse.min.ps",
1351 ret_f32x4_take_f32x4_f32x4,
1352 None,
1353 ),
1354 min_pd: add_function_with_attrs(
1355 "llvm.x86.sse2.min.pd",
1356 ret_f64x2_take_f64x2_f64x2,
1357 None,
1358 ),
1359 max_ps: add_function_with_attrs(
1360 "llvm.x86.sse.max.ps",
1361 ret_f32x4_take_f32x4_f32x4,
1362 None,
1363 ),
1364 max_pd: add_function_with_attrs(
1365 "llvm.x86.sse2.max.pd",
1366 ret_f64x2_take_f64x2_f64x2,
1367 None,
1368 ),
1369 cvttps2dq: add_function_with_attrs(
1370 "llvm.x86.sse2.cvttps2dq",
1371 ret_i32x4_take_f32x4,
1372 None,
1373 ),
1374 cvtps2udq128: add_function_with_attrs(
1375 "llvm.x86.avx512.mask.cvtps2udq.128",
1376 ret_i32x4_take_f32x4_i32x4_i8,
1377 None,
1378 ),
1379 cvtpd2dq: add_function_with_attrs(
1380 "llvm.x86.sse2.cvtpd2dq",
1381 ret_i32x4_take_f64x2,
1382 None,
1383 ),
1384 cvtpd2udq128: add_function_with_attrs(
1385 "llvm.x86.avx512.mask.cvtpd2udq.128",
1386 ret_i32x4_take_f64x2_i32x4_i8,
1387 None,
1388 ),
1389 },
1390 };
1391
1392 let noreturn =
1393 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1394 intrinsics
1395 .throw_trap
1396 .add_attribute(AttributeLoc::Function, noreturn);
1397 intrinsics
1398 }
1399}
1400
1401#[derive(Clone, Copy)]
1402pub enum MemoryCache<'ctx> {
1403 Dynamic {
1405 ptr_to_base_ptr: PointerValue<'ctx>,
1406 ptr_to_current_length: PointerValue<'ctx>,
1407 },
1408 Static { base_ptr: PointerValue<'ctx> },
1410}
1411
1412#[derive(Clone)]
1413struct TableCache<'ctx> {
1414 ptr_to_base_ptr: PointerValue<'ctx>,
1415 ptr_to_bounds: PointerValue<'ctx>,
1416}
1417
1418#[derive(Clone, Copy)]
1419pub enum GlobalCache<'ctx> {
1420 Mut {
1421 ptr_to_value: PointerValue<'ctx>,
1422 value_type: BasicTypeEnum<'ctx>,
1423 },
1424 Const {
1425 value: BasicValueEnum<'ctx>,
1426 },
1427}
1428
1429#[derive(Clone)]
1430pub struct FunctionCache<'ctx> {
1431 pub func: PointerValue<'ctx>,
1432 pub llvm_func_type: FunctionType<'ctx>,
1433 pub vmctx: BasicValueEnum<'ctx>,
1434 pub imported_include_m0_param: Option<IntValue<'ctx>>,
1435 pub attrs: Vec<(Attribute, AttributeLoc)>,
1436}
1437
1438pub struct CtxType<'ctx, 'a> {
1439 ctx_ptr_value: PointerValue<'ctx>,
1440 m0: Option<PointerValue<'ctx>>,
1441
1442 wasm_module: &'a WasmerCompilerModule,
1443 cache_builder: &'a Builder<'ctx>,
1444 abi: &'a LLVMAbi,
1445
1446 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1447 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1448 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1449 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1450 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1451
1452 offsets: VMOffsets,
1453}
1454
1455impl<'ctx, 'a> CtxType<'ctx, 'a> {
1456 pub fn new(
1457 wasm_module: &'a WasmerCompilerModule,
1458 func_value: &FunctionValue<'ctx>,
1459 cache_builder: &'a Builder<'ctx>,
1460 abi: &'a LLVMAbi,
1461 pointer_width: u8,
1462 m0: Option<PointerValue<'ctx>>,
1463 ) -> CtxType<'ctx, 'a> {
1464 CtxType {
1465 m0,
1466 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1467
1468 wasm_module,
1469 cache_builder,
1470 abi,
1471
1472 cached_memories: HashMap::new(),
1473 cached_tables: HashMap::new(),
1474 cached_globals: HashMap::new(),
1475 cached_functions: HashMap::new(),
1476 cached_memory_op: HashMap::new(),
1477
1478 offsets: VMOffsets::new(pointer_width, wasm_module),
1479 }
1480 }
1481
1482 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1483 self.ctx_ptr_value.as_basic_value_enum()
1484 }
1485
1486 pub fn memory(
1487 &mut self,
1488 index: MemoryIndex,
1489 intrinsics: &Intrinsics<'ctx>,
1490 module: &Module<'ctx>,
1491 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1492 ) -> Result<MemoryCache<'ctx>, CompileError> {
1493 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1494 &mut self.cached_memories,
1495 self.wasm_module,
1496 self.ctx_ptr_value,
1497 &self.cache_builder,
1498 &self.offsets,
1499 );
1500 let memory_style = &memory_styles[index];
1501 match cached_memories.get(&index) {
1502 Some(r) => Ok(*r),
1503 None => {
1504 let memory_definition_ptr =
1505 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1506 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1507 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1508 unsafe {
1509 err!(cache_builder.build_gep(
1510 intrinsics.i8_ty,
1511 ctx_ptr_value,
1512 &[offset],
1513 ""
1514 ))
1515 }
1516 } else {
1517 let offset = offsets.vmctx_vmmemory_import(index);
1518 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1519 let memory_definition_ptr_ptr = unsafe {
1520 err!(cache_builder.build_gep(
1521 intrinsics.i8_ty,
1522 ctx_ptr_value,
1523 &[offset],
1524 ""
1525 ))
1526 };
1527 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1528 memory_definition_ptr_ptr,
1529 intrinsics.ptr_ty,
1530 "",
1531 ))
1532 .into_pointer_value();
1533 let memory_definition_ptr = err!(cache_builder.build_load(
1534 intrinsics.ptr_ty,
1535 memory_definition_ptr_ptr,
1536 ""
1537 ))
1538 .into_pointer_value();
1539 tbaa_label(
1540 module,
1541 intrinsics,
1542 format!("memory {} definition", index.as_u32()),
1543 memory_definition_ptr.as_instruction_value().unwrap(),
1544 );
1545 memory_definition_ptr
1546 };
1547 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1548 memory_definition_ptr,
1549 intrinsics.ptr_ty,
1550 "",
1551 ))
1552 .into_pointer_value();
1553 let base_ptr = err!(cache_builder.build_struct_gep(
1554 intrinsics.vmmemory_definition_ty,
1555 memory_definition_ptr,
1556 intrinsics.vmmemory_definition_base_element,
1557 "",
1558 ));
1559 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1560 let current_length_ptr = err!(cache_builder.build_struct_gep(
1561 intrinsics.vmmemory_definition_ty,
1562 memory_definition_ptr,
1563 intrinsics.vmmemory_definition_current_length_element,
1564 "",
1565 ));
1566 MemoryCache::Dynamic {
1567 ptr_to_base_ptr: base_ptr,
1568 ptr_to_current_length: current_length_ptr,
1569 }
1570 } else {
1571 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1572 .into_pointer_value();
1573 tbaa_label(
1574 module,
1575 intrinsics,
1576 format!("memory base_ptr {}", index.as_u32()),
1577 base_ptr.as_instruction_value().unwrap(),
1578 );
1579 MemoryCache::Static { base_ptr }
1580 };
1581
1582 self.cached_memories.insert(index, value);
1583 Ok(*self.cached_memories.get(&index).unwrap())
1584 }
1585 }
1586 }
1587
1588 fn build_table_prepare(
1589 table_index: TableIndex,
1590 intrinsics: &Intrinsics<'ctx>,
1591 module: &Module<'ctx>,
1592 wasm_module: &WasmerCompilerModule,
1593 ctx_ptr_value: PointerValue<'ctx>,
1594 offsets: &VMOffsets,
1595 builder: &Builder<'ctx>,
1596 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1597 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1598 let offset = intrinsics.i64_ty.const_int(
1599 offsets
1600 .vmctx_vmtable_definition_base(local_table_index)
1601 .into(),
1602 false,
1603 );
1604 let ptr_to_base_ptr =
1605 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1606 let ptr_to_base_ptr =
1607 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1608 .into_pointer_value();
1609 let offset = intrinsics.i64_ty.const_int(
1610 offsets
1611 .vmctx_vmtable_definition_current_elements(local_table_index)
1612 .into(),
1613 false,
1614 );
1615 let ptr_to_bounds =
1616 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1617 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1618 .into_pointer_value();
1619 Ok((ptr_to_base_ptr, ptr_to_bounds))
1620 } else {
1621 let offset = intrinsics.i64_ty.const_int(
1622 offsets.vmctx_vmtable_import_definition(table_index).into(),
1623 false,
1624 );
1625 let definition_ptr_ptr =
1626 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1627 let definition_ptr_ptr =
1628 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1629 .into_pointer_value();
1630 let definition_ptr =
1631 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1632 .into_pointer_value();
1633 tbaa_label(
1634 module,
1635 intrinsics,
1636 format!("table {} definition", table_index.as_u32()),
1637 definition_ptr.as_instruction_value().unwrap(),
1638 );
1639
1640 let offset = intrinsics
1641 .i64_ty
1642 .const_int(offsets.vmtable_definition_base().into(), false);
1643 let ptr_to_base_ptr =
1644 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1645 let ptr_to_base_ptr =
1646 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1647 .into_pointer_value();
1648 let offset = intrinsics
1649 .i64_ty
1650 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1651 let ptr_to_bounds =
1652 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1653 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1654 .into_pointer_value();
1655 Ok((ptr_to_base_ptr, ptr_to_bounds))
1656 }
1657 }
1658
1659 fn table_prepare(
1660 &mut self,
1661 table_index: TableIndex,
1662 intrinsics: &Intrinsics<'ctx>,
1663 module: &Module<'ctx>,
1664 body_builder: &Builder<'ctx>,
1665 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1666 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1667 &mut self.cached_tables,
1668 self.wasm_module,
1669 self.ctx_ptr_value,
1670 &self.cache_builder,
1671 &self.offsets,
1672 );
1673
1674 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1675 CompileError::Codegen(format!(
1676 "Table index out of bounds: {}",
1677 table_index.as_u32()
1678 ))
1679 })?;
1680
1681 if is_growable {
1684 Self::build_table_prepare(
1685 table_index,
1686 intrinsics,
1687 module,
1688 wasm_module,
1689 ctx_ptr_value,
1690 offsets,
1691 body_builder,
1692 )
1693 } else {
1694 let TableCache {
1695 ptr_to_base_ptr,
1696 ptr_to_bounds,
1697 } = match cached_tables.entry(table_index) {
1698 Entry::Occupied(entry) => entry.get().clone(),
1699 Entry::Vacant(entry) => {
1700 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1701 table_index,
1702 intrinsics,
1703 module,
1704 wasm_module,
1705 ctx_ptr_value,
1706 offsets,
1707 cache_builder,
1708 )?;
1709
1710 let v = TableCache {
1711 ptr_to_base_ptr,
1712 ptr_to_bounds,
1713 };
1714
1715 entry.insert(v.clone());
1716
1717 v
1718 }
1719 };
1720
1721 Ok((ptr_to_base_ptr, ptr_to_bounds))
1722 }
1723 }
1724
1725 pub fn table(
1726 &mut self,
1727 index: TableIndex,
1728 intrinsics: &Intrinsics<'ctx>,
1729 module: &Module<'ctx>,
1730 body_builder: &Builder<'ctx>,
1731 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1732 let (ptr_to_base_ptr, ptr_to_bounds) =
1733 self.table_prepare(index, intrinsics, module, body_builder)?;
1734
1735 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1737 &body_builder
1738 } else {
1739 &self.cache_builder
1740 };
1741
1742 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1743 .into_pointer_value();
1744 let bounds =
1745 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1746 tbaa_label(
1747 module,
1748 intrinsics,
1749 format!("table_base_ptr {}", index.index()),
1750 base_ptr.as_instruction_value().unwrap(),
1751 );
1752 tbaa_label(
1753 module,
1754 intrinsics,
1755 format!("table_bounds {}", index.index()),
1756 bounds.as_instruction_value().unwrap(),
1757 );
1758 Ok((base_ptr, bounds))
1759 }
1760
1761 pub fn fixed_funcref_table_anyfuncs(
1763 &self,
1764 index: LocalTableIndex,
1765 intrinsics: &Intrinsics<'ctx>,
1766 builder: &Builder<'ctx>,
1767 ) -> Result<PointerValue<'ctx>, CompileError> {
1768 let offset = intrinsics.i64_ty.const_int(
1769 self.offsets
1770 .vmctx_fixed_funcref_table_anyfuncs(index)
1771 .expect("fixed funcref table must have inline VMContext storage")
1772 .into(),
1773 false,
1774 );
1775 let ptr =
1776 unsafe { err!(builder.build_gep(intrinsics.i8_ty, self.ctx_ptr_value, &[offset], "")) };
1777 Ok(err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value())
1778 }
1779
1780 pub fn global(
1781 &mut self,
1782 index: GlobalIndex,
1783 intrinsics: &Intrinsics<'ctx>,
1784 module: &Module<'ctx>,
1785 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1786 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1787 &mut self.cached_globals,
1788 self.wasm_module,
1789 self.ctx_ptr_value,
1790 &self.cache_builder,
1791 &self.offsets,
1792 );
1793 match cached_globals.entry(index) {
1794 Entry::Occupied(entry) => Ok(entry.into_mut()),
1795 Entry::Vacant(entry) => {
1796 let global_type = wasm_module.globals[index];
1797 let global_value_type = global_type.ty;
1798
1799 let global_mutability = global_type.mutability;
1800 let global_ptr = if let Some(local_global_index) =
1801 wasm_module.local_global_index(index)
1802 {
1803 let offset = offsets.vmctx_vmglobal_definition(local_global_index);
1804 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1805 unsafe {
1806 err!(cache_builder.build_gep(
1807 intrinsics.i8_ty,
1808 ctx_ptr_value,
1809 &[offset],
1810 ""
1811 ))
1812 }
1813 } else {
1814 let offset = offsets.vmctx_vmglobal_import(index);
1815 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1816 let global_ptr_ptr = unsafe {
1817 err!(cache_builder.build_gep(
1818 intrinsics.i8_ty,
1819 ctx_ptr_value,
1820 &[offset],
1821 ""
1822 ))
1823 };
1824 let global_ptr_ptr =
1825 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1826 .into_pointer_value();
1827 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1828 .into_pointer_value()
1829 };
1830 let global_ptr =
1831 err!(cache_builder.build_bit_cast(global_ptr, intrinsics.ptr_ty, "",))
1832 .into_pointer_value();
1833
1834 let ret = entry.insert(match global_mutability {
1835 Mutability::Const => {
1836 let value = err!(cache_builder.build_load(
1837 type_to_llvm(intrinsics, global_value_type)?,
1838 global_ptr,
1839 "",
1840 ));
1841 tbaa_label(
1842 module,
1843 intrinsics,
1844 format!("global {}", index.as_u32()),
1845 value.as_instruction_value().unwrap(),
1846 );
1847 GlobalCache::Const { value }
1848 }
1849 Mutability::Var => GlobalCache::Mut {
1850 ptr_to_value: global_ptr,
1851 value_type: type_to_llvm(intrinsics, global_value_type)?,
1852 },
1853 });
1854
1855 Ok(ret)
1856 }
1857 }
1858 }
1859
1860 pub fn add_func(
1861 &mut self,
1862 function_index: FunctionIndex,
1863 func: PointerValue<'ctx>,
1864 llvm_func_type: FunctionType<'ctx>,
1865 vmctx: BasicValueEnum<'ctx>,
1866 attrs: &[(Attribute, AttributeLoc)],
1867 ) {
1868 match self.cached_functions.entry(function_index) {
1869 Entry::Occupied(_) => unreachable!("duplicate function"),
1870 Entry::Vacant(entry) => {
1871 entry.insert(FunctionCache {
1872 func,
1873 llvm_func_type,
1874 vmctx,
1875 imported_include_m0_param: None,
1876 attrs: attrs.to_vec(),
1877 });
1878 }
1879 }
1880 }
1881
1882 #[allow(clippy::too_many_arguments)]
1883 pub fn local_func(
1884 &mut self,
1885 _local_function_index: LocalFunctionIndex,
1886 function_index: FunctionIndex,
1887 intrinsics: &Intrinsics<'ctx>,
1888 module: &Module<'ctx>,
1889 context: &'ctx Context,
1890 func_type: &FuncType,
1891 function_name: &str,
1892 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1893 let (cached_functions, ctx_ptr_value, offsets) = (
1894 &mut self.cached_functions,
1895 &self.ctx_ptr_value,
1896 &self.offsets,
1897 );
1898 Ok(match cached_functions.entry(function_index) {
1899 Entry::Occupied(entry) => entry.into_mut(),
1900 Entry::Vacant(entry) => {
1901 debug_assert!(module.get_function(function_name).is_none());
1902 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1903 context,
1904 intrinsics,
1905 Some(offsets),
1906 func_type,
1907 self.m0.is_some(),
1908 )?;
1909 let func =
1910 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1911 for (attr, attr_loc) in &llvm_func_attrs {
1912 func.add_attribute(*attr_loc, *attr);
1913 }
1914 entry.insert(FunctionCache {
1915 func: func.as_global_value().as_pointer_value(),
1916 llvm_func_type,
1917 vmctx: ctx_ptr_value.as_basic_value_enum(),
1918 imported_include_m0_param: None,
1919 attrs: llvm_func_attrs,
1920 })
1921 }
1922 })
1923 }
1924
1925 pub fn imported_func(
1926 &mut self,
1927 function_index: FunctionIndex,
1928 intrinsics: &Intrinsics<'ctx>,
1929 context: &'ctx Context,
1930 func_type: &FuncType,
1931 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1932 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1933 &mut self.cached_functions,
1934 self.wasm_module,
1935 &self.ctx_ptr_value,
1936 &self.cache_builder,
1937 &self.offsets,
1938 );
1939 match cached_functions.entry(function_index) {
1940 Entry::Occupied(entry) => Ok(entry.into_mut()),
1941 Entry::Vacant(entry) => {
1942 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1943 context,
1944 intrinsics,
1945 Some(offsets),
1946 func_type,
1947 self.m0.is_some(),
1948 )?;
1949 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1950 let offset = offsets.vmctx_vmfunction_import(function_index);
1951 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1952 let vmfunction_import_ptr = unsafe {
1953 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1954 };
1955 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1956 vmfunction_import_ptr,
1957 intrinsics.ptr_ty,
1958 "",
1959 ))
1960 .into_pointer_value();
1961 vmfunction_import_ptr.set_name("vmfunction_import_ptr");
1962
1963 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1964 intrinsics.vmfunction_import_ty,
1965 vmfunction_import_ptr,
1966 intrinsics.vmfunction_import_body_element,
1967 "",
1968 ));
1969 body_ptr_ptr.set_name("body_ptr_ptr");
1970 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
1971 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
1972 .into_pointer_value();
1973 body_ptr.set_name("body_ptr");
1974 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
1975 intrinsics.vmfunction_import_ty,
1976 vmfunction_import_ptr,
1977 intrinsics.vmfunction_import_vmctx_element,
1978 "",
1979 ));
1980 vmctx_ptr_ptr.set_name("vmctx_ptr_ptr");
1981 let vmctx_ptr =
1982 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
1983 vmctx_ptr.set_name("vmctx_ptr");
1984 let include_m0_param_ptr = err!(cache_builder.build_struct_gep(
1985 intrinsics.vmfunction_import_ty,
1986 vmfunction_import_ptr,
1987 intrinsics.vmfunction_import_include_m0_param_element,
1988 "",
1989 ));
1990 include_m0_param_ptr.set_name("include_m0_param_ptr");
1991 let imported_include_m0_param =
1992 err!(cache_builder.build_load(intrinsics.i1_ty, include_m0_param_ptr, "",))
1993 .into_int_value();
1994 imported_include_m0_param.set_name("imported_include_m0_param");
1995
1996 Ok(entry.insert(FunctionCache {
1997 func: body_ptr,
1998 llvm_func_type,
1999 vmctx: vmctx_ptr,
2000 imported_include_m0_param: Some(imported_include_m0_param),
2001 attrs: llvm_func_attrs,
2002 }))
2003 }
2004 }
2005 }
2006
2007 pub fn memory_grow(
2008 &mut self,
2009 memory_index: MemoryIndex,
2010 intrinsics: &Intrinsics<'ctx>,
2011 ) -> Result<PointerValue<'ctx>, CompileError> {
2012 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2013 &mut self.cached_memory_op,
2014 &self.wasm_module,
2015 &self.offsets,
2016 &self.cache_builder,
2017 &self.ctx_ptr_value,
2018 );
2019 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
2020 Entry::Occupied(entry) => Ok(*entry.get()),
2021 Entry::Vacant(entry) => {
2022 let (grow_fn, grow_fn_ty) =
2023 if wasm_module.local_memory_index(memory_index).is_some() {
2024 (
2025 VMBuiltinFunctionIndex::get_memory32_grow_index(),
2026 intrinsics.ptr_ty,
2027 )
2028 } else {
2029 (
2030 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
2031 intrinsics.ptr_ty,
2032 )
2033 };
2034 let offset = offsets.vmctx_builtin_function(grow_fn);
2035 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2036 let grow_fn_ptr_ptr = unsafe {
2037 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2038 };
2039
2040 let grow_fn_ptr_ptr =
2041 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2042 .into_pointer_value();
2043 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
2044 .into_pointer_value();
2045
2046 entry.insert(val);
2047 Ok(val)
2048 }
2049 }
2050 }
2051
2052 pub fn memory_size(
2053 &mut self,
2054 memory_index: MemoryIndex,
2055 intrinsics: &Intrinsics<'ctx>,
2056 ) -> Result<PointerValue<'ctx>, CompileError> {
2057 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2058 &mut self.cached_memory_op,
2059 &self.wasm_module,
2060 &self.offsets,
2061 &self.cache_builder,
2062 &self.ctx_ptr_value,
2063 );
2064
2065 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
2066 Entry::Occupied(entry) => Ok(*entry.get()),
2067 Entry::Vacant(entry) => {
2068 let (size_fn, size_fn_ty) =
2069 if wasm_module.local_memory_index(memory_index).is_some() {
2070 (
2071 VMBuiltinFunctionIndex::get_memory32_size_index(),
2072 intrinsics.ptr_ty,
2073 )
2074 } else {
2075 (
2076 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
2077 intrinsics.ptr_ty,
2078 )
2079 };
2080 let offset = offsets.vmctx_builtin_function(size_fn);
2081 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2082 let size_fn_ptr_ptr = unsafe {
2083 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2084 };
2085
2086 let size_fn_ptr_ptr =
2087 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2088 .into_pointer_value();
2089
2090 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2091 .into_pointer_value();
2092 entry.insert(val);
2093 Ok(val)
2094 }
2095 }
2096 }
2097
2098 pub fn memory_wait32(
2099 &mut self,
2100 memory_index: MemoryIndex,
2101 intrinsics: &Intrinsics<'ctx>,
2102 ) -> Result<PointerValue<'ctx>, CompileError> {
2103 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2104 &mut self.cached_memory_op,
2105 &self.wasm_module,
2106 &self.offsets,
2107 &self.cache_builder,
2108 &self.ctx_ptr_value,
2109 );
2110 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
2111 Entry::Occupied(entry) => Ok(*entry.get()),
2112 Entry::Vacant(entry) => {
2113 let (size_fn, size_fn_ty) =
2114 if wasm_module.local_memory_index(memory_index).is_some() {
2115 (
2116 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
2117 intrinsics.ptr_ty,
2118 )
2119 } else {
2120 (
2121 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
2122 intrinsics.ptr_ty,
2123 )
2124 };
2125 let offset = offsets.vmctx_builtin_function(size_fn);
2126 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2127 let size_fn_ptr_ptr = unsafe {
2128 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2129 };
2130
2131 let size_fn_ptr_ptr =
2132 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2133 .into_pointer_value();
2134
2135 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2136 .into_pointer_value();
2137
2138 entry.insert(val);
2139 Ok(val)
2140 }
2141 }
2142 }
2143
2144 pub fn memory_wait64(
2145 &mut self,
2146 memory_index: MemoryIndex,
2147 intrinsics: &Intrinsics<'ctx>,
2148 ) -> Result<PointerValue<'ctx>, CompileError> {
2149 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2150 &mut self.cached_memory_op,
2151 &self.wasm_module,
2152 &self.offsets,
2153 &self.cache_builder,
2154 &self.ctx_ptr_value,
2155 );
2156
2157 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
2158 Entry::Occupied(entry) => Ok(*entry.get()),
2159 Entry::Vacant(entry) => {
2160 let (size_fn, size_fn_ty) =
2161 if wasm_module.local_memory_index(memory_index).is_some() {
2162 (
2163 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
2164 intrinsics.ptr_ty,
2165 )
2166 } else {
2167 (
2168 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
2169 intrinsics.ptr_ty,
2170 )
2171 };
2172 let offset = offsets.vmctx_builtin_function(size_fn);
2173 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2174 let size_fn_ptr_ptr = unsafe {
2175 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2176 };
2177
2178 let size_fn_ptr_ptr =
2179 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2180 .into_pointer_value();
2181
2182 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2183 .into_pointer_value();
2184 entry.insert(val);
2185 Ok(val)
2186 }
2187 }
2188 }
2189
2190 pub fn memory_notify(
2191 &mut self,
2192 memory_index: MemoryIndex,
2193 intrinsics: &Intrinsics<'ctx>,
2194 ) -> Result<PointerValue<'ctx>, CompileError> {
2195 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2196 &mut self.cached_memory_op,
2197 &self.wasm_module,
2198 &self.offsets,
2199 &self.cache_builder,
2200 &self.ctx_ptr_value,
2201 );
2202 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2203 Entry::Occupied(entry) => Ok(*entry.get()),
2204 Entry::Vacant(entry) => {
2205 let (size_fn, size_fn_ty) =
2206 if wasm_module.local_memory_index(memory_index).is_some() {
2207 (
2208 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2209 intrinsics.ptr_ty,
2210 )
2211 } else {
2212 (
2213 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2214 intrinsics.ptr_ty,
2215 )
2216 };
2217 let offset = offsets.vmctx_builtin_function(size_fn);
2218 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2219 let size_fn_ptr_ptr = unsafe {
2220 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2221 };
2222
2223 let size_fn_ptr_ptr =
2224 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2225 .into_pointer_value();
2226
2227 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2228 .into_pointer_value();
2229
2230 entry.insert(val);
2231 Ok(val)
2232 }
2233 }
2234 }
2235
2236 pub fn get_offsets(&self) -> &VMOffsets {
2237 &self.offsets
2238 }
2239}
2240
2241pub fn tbaa_label<'ctx>(
2244 module: &Module<'ctx>,
2245 intrinsics: &Intrinsics<'ctx>,
2246 label: String,
2247 instruction: InstructionValue<'ctx>,
2248) {
2249 let context = module.get_context();
2263
2264 let tbaa_root = module
2266 .get_global_metadata("wasmer_tbaa_root")
2267 .pop()
2268 .unwrap_or_else(|| {
2269 module
2270 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2271 .unwrap();
2272 module.get_global_metadata("wasmer_tbaa_root")[0]
2273 });
2274
2275 let type_label = context.metadata_string(label.as_str());
2278 let type_tbaa = module
2279 .get_global_metadata(label.as_str())
2280 .pop()
2281 .unwrap_or_else(|| {
2282 module
2283 .add_global_metadata(
2284 label.as_str(),
2285 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2286 )
2287 .unwrap();
2288 module.get_global_metadata(label.as_str())[0]
2289 });
2290
2291 let label = label + "_memop";
2298 let type_tbaa = module
2299 .get_global_metadata(label.as_str())
2300 .pop()
2301 .unwrap_or_else(|| {
2302 module
2303 .add_global_metadata(
2304 label.as_str(),
2305 &context.metadata_node(&[
2306 type_tbaa.into(),
2307 type_tbaa.into(),
2308 intrinsics.i64_zero.into(),
2309 ]),
2310 )
2311 .unwrap();
2312 module.get_global_metadata(label.as_str())[0]
2313 });
2314
2315 let tbaa_kind = context.get_kind_id("tbaa");
2317 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2318}
2319
2320fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2321 let table = module.tables.get(index)?;
2322 match table.maximum {
2323 None => Some(true),
2324 Some(max) => Some(max > table.minimum),
2325 }
2326}