1use crate::abi::Abi;
8use crate::error::err;
9use inkwell::values::BasicMetadataValueEnum;
10use inkwell::{
11 AddressSpace,
12 attributes::{Attribute, AttributeLoc},
13 builder::Builder,
14 context::Context,
15 module::{Linkage, Module},
16 targets::TargetData,
17 types::{
18 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
19 PointerType, StructType, VectorType, VoidType,
20 },
21 values::{
22 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
23 PointerValue, VectorValue,
24 },
25};
26use std::collections::{HashMap, hash_map::Entry};
27use target_lexicon::{Architecture, Triple};
28use wasmer_types::entity::{EntityRef, PrimaryMap};
29use wasmer_types::{
30 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
31 LocalTableIndex, MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, TableIndex, Type,
32};
33use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
34
35pub fn type_to_llvm<'ctx>(
36 intrinsics: &Intrinsics<'ctx>,
37 ty: Type,
38) -> Result<BasicTypeEnum<'ctx>, CompileError> {
39 match ty {
40 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
41 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
42 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
43 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
44 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
45 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
46 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
47 }
48}
49
50#[allow(dead_code)]
52pub struct X86_64Intrinsics<'ctx> {
53 pub pshufb128: FunctionValue<'ctx>,
54 pub pmaddubsw128: FunctionValue<'ctx>,
55 pub pmaddwd128: FunctionValue<'ctx>,
56 pub pmulhrsw128: FunctionValue<'ctx>,
57 pub pblendvb: FunctionValue<'ctx>,
58 pub min_ps: FunctionValue<'ctx>,
59 pub min_pd: FunctionValue<'ctx>,
60 pub max_ps: FunctionValue<'ctx>,
61 pub max_pd: FunctionValue<'ctx>,
62 pub cvttps2dq: FunctionValue<'ctx>,
63 pub cvtps2udq128: FunctionValue<'ctx>,
64 pub cvtpd2dq: FunctionValue<'ctx>,
65 pub cvtpd2udq128: FunctionValue<'ctx>,
66}
67
68#[allow(dead_code)]
70pub struct Intrinsics<'ctx> {
71 pub ctlz_i32: FunctionValue<'ctx>,
72 pub ctlz_i64: FunctionValue<'ctx>,
73
74 pub cttz_i32: FunctionValue<'ctx>,
75 pub cttz_i64: FunctionValue<'ctx>,
76
77 pub ctpop_i32: FunctionValue<'ctx>,
78 pub ctpop_i64: FunctionValue<'ctx>,
79 pub ctpop_i8x16: FunctionValue<'ctx>,
80
81 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
82 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
83 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
84 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
85 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
86
87 pub add_f32: FunctionValue<'ctx>,
88 pub add_f64: FunctionValue<'ctx>,
89 pub add_f32x4: FunctionValue<'ctx>,
90 pub add_f64x2: FunctionValue<'ctx>,
91
92 pub sub_f32: FunctionValue<'ctx>,
93 pub sub_f64: FunctionValue<'ctx>,
94 pub sub_f32x4: FunctionValue<'ctx>,
95 pub sub_f64x2: FunctionValue<'ctx>,
96
97 pub mul_f32: FunctionValue<'ctx>,
98 pub mul_f64: FunctionValue<'ctx>,
99 pub mul_f32x4: FunctionValue<'ctx>,
100 pub mul_f64x2: FunctionValue<'ctx>,
101 pub muladd_f32x4: FunctionValue<'ctx>,
102 pub muladd_f64x2: FunctionValue<'ctx>,
103
104 pub div_f32: FunctionValue<'ctx>,
105 pub div_f64: FunctionValue<'ctx>,
106 pub div_f32x4: FunctionValue<'ctx>,
107 pub div_f64x2: FunctionValue<'ctx>,
108
109 pub sqrt_f32: FunctionValue<'ctx>,
110 pub sqrt_f64: FunctionValue<'ctx>,
111 pub sqrt_f32x4: FunctionValue<'ctx>,
112 pub sqrt_f64x2: FunctionValue<'ctx>,
113
114 pub cmp_f32: FunctionValue<'ctx>,
115 pub cmp_f64: FunctionValue<'ctx>,
116 pub cmp_f32x4: FunctionValue<'ctx>,
117 pub cmp_f64x2: FunctionValue<'ctx>,
118
119 pub minimum_f32: FunctionValue<'ctx>,
120 pub minimum_f64: FunctionValue<'ctx>,
121 pub minimum_f32x4: FunctionValue<'ctx>,
122 pub minimum_f64x2: FunctionValue<'ctx>,
123
124 pub maximum_f32: FunctionValue<'ctx>,
125 pub maximum_f64: FunctionValue<'ctx>,
126 pub maximum_f32x4: FunctionValue<'ctx>,
127 pub maximum_f64x2: FunctionValue<'ctx>,
128
129 pub ceil_f32: FunctionValue<'ctx>,
130 pub ceil_f64: FunctionValue<'ctx>,
131 pub ceil_f32x4: FunctionValue<'ctx>,
132 pub ceil_f64x2: FunctionValue<'ctx>,
133
134 pub floor_f32: FunctionValue<'ctx>,
135 pub floor_f64: FunctionValue<'ctx>,
136 pub floor_f32x4: FunctionValue<'ctx>,
137 pub floor_f64x2: FunctionValue<'ctx>,
138
139 pub trunc_f32: FunctionValue<'ctx>,
140 pub trunc_f64: FunctionValue<'ctx>,
141 pub trunc_f32x4: FunctionValue<'ctx>,
142 pub trunc_f64x2: FunctionValue<'ctx>,
143
144 pub fpext_f32: FunctionValue<'ctx>,
145 pub fptrunc_f64: FunctionValue<'ctx>,
146
147 pub nearbyint_f32: FunctionValue<'ctx>,
148 pub nearbyint_f64: FunctionValue<'ctx>,
149 pub nearbyint_f32x4: FunctionValue<'ctx>,
150 pub nearbyint_f64x2: FunctionValue<'ctx>,
151
152 pub fabs_f32: FunctionValue<'ctx>,
153 pub fabs_f64: FunctionValue<'ctx>,
154 pub fabs_f32x4: FunctionValue<'ctx>,
155 pub fabs_f64x2: FunctionValue<'ctx>,
156
157 pub copysign_f32: FunctionValue<'ctx>,
158 pub copysign_f64: FunctionValue<'ctx>,
159 pub copysign_f32x4: FunctionValue<'ctx>,
160 pub copysign_f64x2: FunctionValue<'ctx>,
161
162 pub sadd_sat_i8x16: FunctionValue<'ctx>,
163 pub sadd_sat_i16x8: FunctionValue<'ctx>,
164 pub uadd_sat_i8x16: FunctionValue<'ctx>,
165 pub uadd_sat_i16x8: FunctionValue<'ctx>,
166
167 pub ssub_sat_i8x16: FunctionValue<'ctx>,
168 pub ssub_sat_i16x8: FunctionValue<'ctx>,
169 pub usub_sat_i8x16: FunctionValue<'ctx>,
170 pub usub_sat_i16x8: FunctionValue<'ctx>,
171
172 pub expect_i1: FunctionValue<'ctx>,
173 pub trap: FunctionValue<'ctx>,
174 pub debug_trap: FunctionValue<'ctx>,
175
176 pub personality: FunctionValue<'ctx>,
177 pub personality2: FunctionValue<'ctx>,
178 pub readonly: Attribute,
179 pub stack_probe: Attribute,
180 pub uwtable: Attribute,
181 pub frame_pointer: Attribute,
182 pub chkstk: FunctionValue<'ctx>,
184
185 pub void_ty: VoidType<'ctx>,
186 pub i1_ty: IntType<'ctx>,
187 pub i2_ty: IntType<'ctx>,
188 pub i4_ty: IntType<'ctx>,
189 pub i8_ty: IntType<'ctx>,
190 pub i16_ty: IntType<'ctx>,
191 pub i32_ty: IntType<'ctx>,
192 pub i64_ty: IntType<'ctx>,
193 pub i128_ty: IntType<'ctx>,
194 pub isize_ty: IntType<'ctx>,
195 pub f32_ty: FloatType<'ctx>,
196 pub f64_ty: FloatType<'ctx>,
197
198 pub i1x128_ty: VectorType<'ctx>,
199 pub i8x16_ty: VectorType<'ctx>,
200 pub i16x8_ty: VectorType<'ctx>,
201 pub i32x4_ty: VectorType<'ctx>,
202 pub i64x2_ty: VectorType<'ctx>,
203 pub f32x4_ty: VectorType<'ctx>,
204 pub f64x2_ty: VectorType<'ctx>,
205 pub i32x8_ty: VectorType<'ctx>,
206
207 pub ptr_ty: PointerType<'ctx>,
208
209 pub x86_64: X86_64Intrinsics<'ctx>,
210
211 pub anyfunc_ty: StructType<'ctx>,
212
213 pub i1_zero: IntValue<'ctx>,
214 pub i8_zero: IntValue<'ctx>,
215 pub i32_zero: IntValue<'ctx>,
216 pub i64_zero: IntValue<'ctx>,
217 pub i128_zero: IntValue<'ctx>,
218 pub isize_zero: IntValue<'ctx>,
219 pub f32_zero: FloatValue<'ctx>,
220 pub f64_zero: FloatValue<'ctx>,
221 pub f32x4_zero: VectorValue<'ctx>,
222 pub f64x2_zero: VectorValue<'ctx>,
223 pub i32_consts: [IntValue<'ctx>; 16],
224
225 pub trap_unreachable: BasicValueEnum<'ctx>,
226 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
227 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
228 pub trap_memory_oob: BasicValueEnum<'ctx>,
229 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
230 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
231 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
232 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
233 pub trap_table_access_oob: BasicValueEnum<'ctx>,
234
235 pub experimental_stackmap: FunctionValue<'ctx>,
236
237 pub table_copy: FunctionValue<'ctx>,
239 pub table_init: FunctionValue<'ctx>,
240 pub table_fill: FunctionValue<'ctx>,
241 pub table_size: FunctionValue<'ctx>,
242 pub imported_table_size: FunctionValue<'ctx>,
243 pub table_get: FunctionValue<'ctx>,
244 pub imported_table_get: FunctionValue<'ctx>,
245 pub table_set: FunctionValue<'ctx>,
246 pub imported_table_set: FunctionValue<'ctx>,
247 pub table_grow: FunctionValue<'ctx>,
248 pub imported_table_grow: FunctionValue<'ctx>,
249 pub memory_init: FunctionValue<'ctx>,
250 pub data_drop: FunctionValue<'ctx>,
251 pub func_ref: FunctionValue<'ctx>,
252 pub elem_drop: FunctionValue<'ctx>,
253 pub memory_copy: FunctionValue<'ctx>,
254 pub imported_memory_copy: FunctionValue<'ctx>,
255 pub memory_fill: FunctionValue<'ctx>,
256 pub imported_memory_fill: FunctionValue<'ctx>,
257 pub memory_size_ty: FunctionType<'ctx>,
258 pub memory_grow_ty: FunctionType<'ctx>,
259 pub memory_wait32: FunctionValue<'ctx>,
260 pub memory_wait32_ty: FunctionType<'ctx>,
261 pub imported_memory_wait32: FunctionValue<'ctx>,
262 pub memory_wait64: FunctionValue<'ctx>,
263 pub memory_wait64_ty: FunctionType<'ctx>,
264 pub imported_memory_wait64: FunctionValue<'ctx>,
265 pub memory_notify: FunctionValue<'ctx>,
266 pub memory_notify_ty: FunctionType<'ctx>,
267 pub imported_memory_notify: FunctionValue<'ctx>,
268
269 pub throw_trap: FunctionValue<'ctx>,
270
271 pub throw: FunctionValue<'ctx>,
273 pub alloc_exception: FunctionValue<'ctx>,
274 pub read_exnref: FunctionValue<'ctx>,
275 pub exception_into_exnref: FunctionValue<'ctx>,
276 pub lpad_exception_ty: StructType<'ctx>,
277
278 pub debug_ptr: FunctionValue<'ctx>,
280 pub debug_str: FunctionValue<'ctx>,
281
282 pub vmfunction_import_ty: StructType<'ctx>,
284 pub vmfunction_import_body_element: u32,
285 pub vmfunction_import_vmctx_element: u32,
286 pub vmfunction_import_include_m0_param_element: u32,
287
288 pub vmmemory_definition_ty: StructType<'ctx>,
289 pub vmmemory_definition_base_element: u32,
290 pub vmmemory_definition_current_length_element: u32,
291}
292
293#[derive(Debug, Hash, PartialEq, Eq)]
294enum MemoryOp {
295 Size,
296 Grow,
297 Wait32,
298 Wait64,
299 Notify,
300}
301
302impl<'ctx> Intrinsics<'ctx> {
303 pub fn declare(
305 module: &Module<'ctx>,
306 context: &'ctx Context,
307 target_data: &TargetData,
308 target_triple: &Triple,
309 binary_fmt: &target_lexicon::BinaryFormat,
310 ) -> Self {
311 let is_riscv64 = matches!(target_triple.architecture, Architecture::Riscv64(..));
312 let void_ty = context.void_type();
313 let i1_ty = context.bool_type();
314 let i2_ty = context.custom_width_int_type(2);
315 let i4_ty = context.custom_width_int_type(4);
316 let i8_ty = context.i8_type();
317 let i16_ty = context.i16_type();
318 let i32_ty = context.i32_type();
319 let i64_ty = context.i64_type();
320 let i128_ty = context.i128_type();
321 let isize_ty = context.ptr_sized_int_type(target_data, None);
322 let f32_ty = context.f32_type();
323 let f64_ty = context.f64_type();
324
325 let i1x4_ty = i1_ty.vec_type(4);
326 let i1x2_ty = i1_ty.vec_type(2);
327 let i1x128_ty = i1_ty.vec_type(128);
328 let i8x16_ty = i8_ty.vec_type(16);
329 let i16x8_ty = i16_ty.vec_type(8);
330 let i32x4_ty = i32_ty.vec_type(4);
331 let i64x2_ty = i64_ty.vec_type(2);
332 let f32x4_ty = f32_ty.vec_type(4);
333 let f64x2_ty = f64_ty.vec_type(2);
334 let i32x8_ty = i32_ty.vec_type(8);
335
336 let ptr_ty = context.ptr_type(AddressSpace::default());
337
338 let i1_zero = i1_ty.const_int(0, false);
339 let i8_zero = i8_ty.const_int(0, false);
340 let i32_zero = i32_ty.const_int(0, false);
341 let i64_zero = i64_ty.const_int(0, false);
342 let i128_zero = i128_ty.const_int(0, false);
343 let isize_zero = isize_ty.const_int(0, false);
344 let f32_zero = f32_ty.const_float(0.0);
345 let f64_zero = f64_ty.const_float(0.0);
346 let f32x4_zero = f32x4_ty.const_zero();
347 let f64x2_zero = f64x2_ty.const_zero();
348 let i32_consts = [
349 i32_ty.const_int(0, false),
350 i32_ty.const_int(1, false),
351 i32_ty.const_int(2, false),
352 i32_ty.const_int(3, false),
353 i32_ty.const_int(4, false),
354 i32_ty.const_int(5, false),
355 i32_ty.const_int(6, false),
356 i32_ty.const_int(7, false),
357 i32_ty.const_int(8, false),
358 i32_ty.const_int(9, false),
359 i32_ty.const_int(10, false),
360 i32_ty.const_int(11, false),
361 i32_ty.const_int(12, false),
362 i32_ty.const_int(13, false),
363 i32_ty.const_int(14, false),
364 i32_ty.const_int(15, false),
365 ];
366
367 let md_ty = context.metadata_type();
368
369 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
370 let i1_ty_basic = i1_ty.as_basic_type_enum();
371
372 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
373 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
374 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
375 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
376 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
377 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
378 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
379 let i32x4_ty_basic_md: BasicMetadataTypeEnum = i32x4_ty.into();
380 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
381 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
382 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
383
384 let ctx_ptr_ty = ptr_ty;
385 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
386 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
387
388 let anyfunc_ty = context.struct_type(
392 &[
393 i8_ptr_ty_basic,
394 i32_ty.into(),
395 ctx_ptr_ty_basic,
396 ptr_ty.into(),
397 ],
398 false,
399 );
400 let funcref_ty = ptr_ty;
401 let anyref_ty = ptr_ty;
402 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
403
404 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
405 let ret_i8x16_take_i8x16_i8x16 =
406 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
407 let ret_i8x16_take_i8x16_i8x16_i8x16 = i8x16_ty.fn_type(
408 &[i8x16_ty_basic_md, i8x16_ty_basic_md, i8x16_ty_basic_md],
409 false,
410 );
411 let ret_i16x8_take_i16x8_i16x8 =
412 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
413
414 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
415 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
416
417 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
418 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
419
420 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
421 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
422 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
423 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
424
425 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
426 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
427 let ret_f32x4_take_f32x4_f32x4 =
428 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
429 let ret_f64x2_take_f64x2_f64x2 =
430 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
431 let ret_f32x4_take_f32x4_f32x4_f32x4_md_md = f32x4_ty.fn_type(
432 &[
433 f32x4_ty_basic_md,
434 f32x4_ty_basic_md,
435 f32x4_ty_basic_md,
436 md_ty_basic_md,
437 md_ty_basic_md,
438 ],
439 false,
440 );
441 let ret_f64x2_take_f64x2_f64x2_f64x2_md_md = f64x2_ty.fn_type(
442 &[
443 f64x2_ty_basic_md,
444 f64x2_ty_basic_md,
445 f64x2_ty_basic_md,
446 md_ty_basic_md,
447 md_ty_basic_md,
448 ],
449 false,
450 );
451
452 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
453 let ret_f32_take_f64_md_md =
454 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
455
456 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
457
458 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
459 &[
460 f32_ty_basic_md,
461 f32_ty_basic_md,
462 md_ty_basic_md,
463 md_ty_basic_md,
464 ],
465 false,
466 );
467 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
468 &[
469 f64_ty_basic_md,
470 f64_ty_basic_md,
471 md_ty_basic_md,
472 md_ty_basic_md,
473 ],
474 false,
475 );
476 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
477 &[
478 f32x4_ty_basic_md,
479 f32x4_ty_basic_md,
480 md_ty_basic_md,
481 md_ty_basic_md,
482 ],
483 false,
484 );
485 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
486 &[
487 f64x2_ty_basic_md,
488 f64x2_ty_basic_md,
489 md_ty_basic_md,
490 md_ty_basic_md,
491 ],
492 false,
493 );
494
495 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
496 &[
497 f32_ty_basic_md,
498 f32_ty_basic_md,
499 md_ty_basic_md,
500 md_ty_basic_md,
501 ],
502 false,
503 );
504 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
505 &[
506 f64_ty_basic_md,
507 f64_ty_basic_md,
508 md_ty_basic_md,
509 md_ty_basic_md,
510 ],
511 false,
512 );
513 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
514 &[
515 f32x4_ty_basic_md,
516 f32x4_ty_basic_md,
517 md_ty_basic_md,
518 md_ty_basic_md,
519 ],
520 false,
521 );
522 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
523 &[
524 f64x2_ty_basic_md,
525 f64x2_ty_basic_md,
526 md_ty_basic_md,
527 md_ty_basic_md,
528 ],
529 false,
530 );
531 let ret_i32x4_take_f32x4 = i32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
532 let ret_i32x4_take_f32x4_i32x4_i8 =
533 i32x4_ty.fn_type(&[f32x4_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
534 let ret_i32x4_take_f64x2 = i32x4_ty.fn_type(&[f64x2_ty_basic_md], false);
535 let ret_i32x4_take_f64x2_i32x4_i8 =
536 i32x4_ty.fn_type(&[f64x2_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
537
538 let add_function_with_attrs =
539 |name: &str, ty: FunctionType<'ctx>, linkage: Option<Linkage>| -> FunctionValue<'ctx> {
540 let function = module.add_function(name, ty, linkage);
541 if is_riscv64 {
546 for (i, param) in ty.get_param_types().iter().enumerate() {
547 if param == &i32_ty_basic_md {
548 function.add_attribute(
549 AttributeLoc::Param(i as u32),
550 context.create_enum_attribute(
551 Attribute::get_named_enum_kind_id("signext"),
552 0,
553 ),
554 );
555 function.add_attribute(
556 AttributeLoc::Param(i as u32),
557 context.create_enum_attribute(
558 Attribute::get_named_enum_kind_id("noundef"),
559 0,
560 ),
561 );
562 }
563 }
564 }
565 function
566 };
567
568 let intrinsics = Self {
569 ctlz_i32: add_function_with_attrs("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
570 ctlz_i64: add_function_with_attrs("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
571
572 cttz_i32: add_function_with_attrs("llvm.cttz.i32", ret_i32_take_i32_i1, None),
573 cttz_i64: add_function_with_attrs("llvm.cttz.i64", ret_i64_take_i64_i1, None),
574
575 ctpop_i32: add_function_with_attrs("llvm.ctpop.i32", ret_i32_take_i32, None),
576 ctpop_i64: add_function_with_attrs("llvm.ctpop.i64", ret_i64_take_i64, None),
577 ctpop_i8x16: add_function_with_attrs("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
578
579 fp_rounding_md: context.metadata_string("round.tonearest").into(),
580 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
581
582 fp_ogt_md: context.metadata_string("ogt").into(),
583 fp_olt_md: context.metadata_string("olt").into(),
584 fp_uno_md: context.metadata_string("uno").into(),
585
586 sqrt_f32: add_function_with_attrs("llvm.sqrt.f32", ret_f32_take_f32, None),
587 sqrt_f64: add_function_with_attrs("llvm.sqrt.f64", ret_f64_take_f64, None),
588 sqrt_f32x4: add_function_with_attrs("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
589 sqrt_f64x2: add_function_with_attrs("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
590
591 ceil_f32: add_function_with_attrs("llvm.ceil.f32", ret_f32_take_f32, None),
592 ceil_f64: add_function_with_attrs("llvm.ceil.f64", ret_f64_take_f64, None),
593 ceil_f32x4: add_function_with_attrs("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
594 ceil_f64x2: add_function_with_attrs("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
595
596 floor_f32: add_function_with_attrs("llvm.floor.f32", ret_f32_take_f32, None),
597 floor_f64: add_function_with_attrs("llvm.floor.f64", ret_f64_take_f64, None),
598 floor_f32x4: add_function_with_attrs("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
599 floor_f64x2: add_function_with_attrs("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
600
601 trunc_f32: add_function_with_attrs("llvm.trunc.f32", ret_f32_take_f32, None),
602 trunc_f64: add_function_with_attrs("llvm.trunc.f64", ret_f64_take_f64, None),
603 trunc_f32x4: add_function_with_attrs("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
604 trunc_f64x2: add_function_with_attrs("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
605
606 nearbyint_f32: add_function_with_attrs("llvm.nearbyint.f32", ret_f32_take_f32, None),
607 nearbyint_f64: add_function_with_attrs("llvm.nearbyint.f64", ret_f64_take_f64, None),
608 nearbyint_f32x4: add_function_with_attrs(
609 "llvm.nearbyint.v4f32",
610 ret_f32x4_take_f32x4,
611 None,
612 ),
613 nearbyint_f64x2: add_function_with_attrs(
614 "llvm.nearbyint.v2f64",
615 ret_f64x2_take_f64x2,
616 None,
617 ),
618
619 add_f32: add_function_with_attrs(
620 "llvm.experimental.constrained.fadd.f32",
621 ret_f32_take_f32_f32_md_md,
622 None,
623 ),
624 add_f64: add_function_with_attrs(
625 "llvm.experimental.constrained.fadd.f64",
626 ret_f64_take_f64_f64_md_md,
627 None,
628 ),
629 add_f32x4: add_function_with_attrs(
630 "llvm.experimental.constrained.fadd.v4f32",
631 ret_f32x4_take_f32x4_f32x4_md_md,
632 None,
633 ),
634 add_f64x2: add_function_with_attrs(
635 "llvm.experimental.constrained.fadd.v2f64",
636 ret_f64x2_take_f64x2_f64x2_md_md,
637 None,
638 ),
639
640 sub_f32: add_function_with_attrs(
641 "llvm.experimental.constrained.fsub.f32",
642 ret_f32_take_f32_f32_md_md,
643 None,
644 ),
645 sub_f64: add_function_with_attrs(
646 "llvm.experimental.constrained.fsub.f64",
647 ret_f64_take_f64_f64_md_md,
648 None,
649 ),
650 sub_f32x4: add_function_with_attrs(
651 "llvm.experimental.constrained.fsub.v4f32",
652 ret_f32x4_take_f32x4_f32x4_md_md,
653 None,
654 ),
655 sub_f64x2: add_function_with_attrs(
656 "llvm.experimental.constrained.fsub.v2f64",
657 ret_f64x2_take_f64x2_f64x2_md_md,
658 None,
659 ),
660
661 mul_f32: add_function_with_attrs(
662 "llvm.experimental.constrained.fmul.f32",
663 ret_f32_take_f32_f32_md_md,
664 None,
665 ),
666 mul_f64: add_function_with_attrs(
667 "llvm.experimental.constrained.fmul.f64",
668 ret_f64_take_f64_f64_md_md,
669 None,
670 ),
671 mul_f32x4: add_function_with_attrs(
672 "llvm.experimental.constrained.fmul.v4f32",
673 ret_f32x4_take_f32x4_f32x4_md_md,
674 None,
675 ),
676 mul_f64x2: add_function_with_attrs(
677 "llvm.experimental.constrained.fmul.v2f64",
678 ret_f64x2_take_f64x2_f64x2_md_md,
679 None,
680 ),
681 muladd_f32x4: add_function_with_attrs(
682 "llvm.experimental.constrained.fmuladd.v4f32",
683 ret_f32x4_take_f32x4_f32x4_f32x4_md_md,
684 None,
685 ),
686 muladd_f64x2: add_function_with_attrs(
687 "llvm.experimental.constrained.fmuladd.v2f64",
688 ret_f64x2_take_f64x2_f64x2_f64x2_md_md,
689 None,
690 ),
691
692 div_f32: add_function_with_attrs(
693 "llvm.experimental.constrained.fdiv.f32",
694 ret_f32_take_f32_f32_md_md,
695 None,
696 ),
697 div_f64: add_function_with_attrs(
698 "llvm.experimental.constrained.fdiv.f64",
699 ret_f64_take_f64_f64_md_md,
700 None,
701 ),
702 div_f32x4: add_function_with_attrs(
703 "llvm.experimental.constrained.fdiv.v4f32",
704 ret_f32x4_take_f32x4_f32x4_md_md,
705 None,
706 ),
707 div_f64x2: add_function_with_attrs(
708 "llvm.experimental.constrained.fdiv.v2f64",
709 ret_f64x2_take_f64x2_f64x2_md_md,
710 None,
711 ),
712
713 cmp_f32: add_function_with_attrs(
714 "llvm.experimental.constrained.fcmp.f32",
715 ret_i1_take_f32_f32_md_md,
716 None,
717 ),
718 cmp_f64: add_function_with_attrs(
719 "llvm.experimental.constrained.fcmp.f64",
720 ret_i1_take_f64_f64_md_md,
721 None,
722 ),
723 cmp_f32x4: add_function_with_attrs(
724 "llvm.experimental.constrained.fcmp.v4f32",
725 ret_i1x4_take_f32x4_f32x4_md_md,
726 None,
727 ),
728 cmp_f64x2: add_function_with_attrs(
729 "llvm.experimental.constrained.fcmp.v2f64",
730 ret_i1x2_take_f64x2_f64x2_md_md,
731 None,
732 ),
733
734 minimum_f32: add_function_with_attrs("llvm.minimum.f32", ret_f32_take_f32_f32, None),
735 minimum_f64: add_function_with_attrs("llvm.minimum.f64", ret_f64_take_f64_f64, None),
736 minimum_f32x4: add_function_with_attrs(
737 "llvm.minimum.v4f32",
738 ret_f32x4_take_f32x4_f32x4,
739 None,
740 ),
741 minimum_f64x2: add_function_with_attrs(
742 "llvm.minimum.v2f64",
743 ret_f64x2_take_f64x2_f64x2,
744 None,
745 ),
746
747 maximum_f32: add_function_with_attrs("llvm.maximum.f32", ret_f32_take_f32_f32, None),
748 maximum_f64: add_function_with_attrs("llvm.maximum.f64", ret_f64_take_f64_f64, None),
749 maximum_f32x4: add_function_with_attrs(
750 "llvm.maximum.v4f32",
751 ret_f32x4_take_f32x4_f32x4,
752 None,
753 ),
754 maximum_f64x2: add_function_with_attrs(
755 "llvm.maximum.v2f64",
756 ret_f64x2_take_f64x2_f64x2,
757 None,
758 ),
759
760 fpext_f32: add_function_with_attrs(
761 "llvm.experimental.constrained.fpext.f64.f32",
762 ret_f64_take_f32_md,
763 None,
764 ),
765 fptrunc_f64: add_function_with_attrs(
766 "llvm.experimental.constrained.fptrunc.f32.f64",
767 ret_f32_take_f64_md_md,
768 None,
769 ),
770
771 fabs_f32: add_function_with_attrs("llvm.fabs.f32", ret_f32_take_f32, None),
772 fabs_f64: add_function_with_attrs("llvm.fabs.f64", ret_f64_take_f64, None),
773 fabs_f32x4: add_function_with_attrs("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
774 fabs_f64x2: add_function_with_attrs("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
775
776 copysign_f32: add_function_with_attrs("llvm.copysign.f32", ret_f32_take_f32_f32, None),
777 copysign_f64: add_function_with_attrs("llvm.copysign.f64", ret_f64_take_f64_f64, None),
778 copysign_f32x4: add_function_with_attrs(
779 "llvm.copysign.v4f32",
780 ret_f32x4_take_f32x4_f32x4,
781 None,
782 ),
783 copysign_f64x2: add_function_with_attrs(
784 "llvm.copysign.v2f64",
785 ret_f64x2_take_f64x2_f64x2,
786 None,
787 ),
788
789 sadd_sat_i8x16: add_function_with_attrs(
790 "llvm.sadd.sat.v16i8",
791 ret_i8x16_take_i8x16_i8x16,
792 None,
793 ),
794 sadd_sat_i16x8: add_function_with_attrs(
795 "llvm.sadd.sat.v8i16",
796 ret_i16x8_take_i16x8_i16x8,
797 None,
798 ),
799 uadd_sat_i8x16: add_function_with_attrs(
800 "llvm.uadd.sat.v16i8",
801 ret_i8x16_take_i8x16_i8x16,
802 None,
803 ),
804 uadd_sat_i16x8: add_function_with_attrs(
805 "llvm.uadd.sat.v8i16",
806 ret_i16x8_take_i16x8_i16x8,
807 None,
808 ),
809
810 ssub_sat_i8x16: add_function_with_attrs(
811 "llvm.ssub.sat.v16i8",
812 ret_i8x16_take_i8x16_i8x16,
813 None,
814 ),
815 ssub_sat_i16x8: add_function_with_attrs(
816 "llvm.ssub.sat.v8i16",
817 ret_i16x8_take_i16x8_i16x8,
818 None,
819 ),
820 usub_sat_i8x16: add_function_with_attrs(
821 "llvm.usub.sat.v16i8",
822 ret_i8x16_take_i8x16_i8x16,
823 None,
824 ),
825 usub_sat_i16x8: add_function_with_attrs(
826 "llvm.usub.sat.v8i16",
827 ret_i16x8_take_i16x8_i16x8,
828 None,
829 ),
830
831 expect_i1: add_function_with_attrs("llvm.expect.i1", ret_i1_take_i1_i1, None),
832 trap: add_function_with_attrs("llvm.trap", void_ty.fn_type(&[], false), None),
833 debug_trap: add_function_with_attrs(
834 "llvm.debugtrap",
835 void_ty.fn_type(&[], false),
836 None,
837 ),
838 personality: add_function_with_attrs(
839 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
840 "__gxx_personality_v0"
843 } else {
844 "wasmer_eh_personality"
845 },
846 i32_ty.fn_type(
847 &[
848 i32_ty.into(),
849 i32_ty.into(),
850 i64_ty.into(),
851 ptr_ty.into(),
852 ptr_ty.into(),
853 ],
854 false,
855 ),
856 None,
857 ),
858 personality2: add_function_with_attrs(
859 "wasmer_eh_personality2",
860 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
861 None,
862 ),
863 readonly: context
864 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
865 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
866 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
867 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
868 chkstk: add_function_with_attrs("__chkstk", void_ty.fn_type(&[], false), None),
869 void_ty,
870 i1_ty,
871 i2_ty,
872 i4_ty,
873 i8_ty,
874 i16_ty,
875 i32_ty,
876 i64_ty,
877 i128_ty,
878 isize_ty,
879 f32_ty,
880 f64_ty,
881
882 i1x128_ty,
883 i8x16_ty,
884 i16x8_ty,
885 i32x4_ty,
886 i64x2_ty,
887 f32x4_ty,
888 f64x2_ty,
889 i32x8_ty,
890
891 anyfunc_ty,
892 i1_zero,
893 i8_zero,
894 i32_zero,
895 i64_zero,
896 i128_zero,
897 isize_zero,
898 f32_zero,
899 f64_zero,
900 f32x4_zero,
901 f64x2_zero,
902 i32_consts,
903
904 trap_unreachable: i32_ty
905 .const_int(TrapCode::UnreachableCodeReached as _, false)
906 .as_basic_value_enum(),
907 trap_call_indirect_null: i32_ty
908 .const_int(TrapCode::IndirectCallToNull as _, false)
909 .as_basic_value_enum(),
910 trap_call_indirect_sig: i32_ty
911 .const_int(TrapCode::BadSignature as _, false)
912 .as_basic_value_enum(),
913 trap_memory_oob: i32_ty
914 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
915 .as_basic_value_enum(),
916 trap_illegal_arithmetic: i32_ty
917 .const_int(TrapCode::IntegerOverflow as _, false)
918 .as_basic_value_enum(),
919 trap_integer_division_by_zero: i32_ty
920 .const_int(TrapCode::IntegerDivisionByZero as _, false)
921 .as_basic_value_enum(),
922 trap_bad_conversion_to_integer: i32_ty
923 .const_int(TrapCode::BadConversionToInteger as _, false)
924 .as_basic_value_enum(),
925 trap_unaligned_atomic: i32_ty
926 .const_int(TrapCode::UnalignedAtomic as _, false)
927 .as_basic_value_enum(),
928 trap_table_access_oob: i32_ty
929 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
930 .as_basic_value_enum(),
931
932 experimental_stackmap: module.add_function(
936 "llvm.experimental.stackmap",
937 void_ty.fn_type(
938 &[
939 i64_ty_basic_md, i32_ty_basic_md, ],
942 true,
943 ),
944 None,
945 ),
946
947 table_copy: add_function_with_attrs(
949 "wasmer_vm_table_copy",
950 void_ty.fn_type(
951 &[
952 ctx_ptr_ty_basic_md,
953 i32_ty_basic_md,
954 i32_ty_basic_md,
955 i32_ty_basic_md,
956 i32_ty_basic_md,
957 i32_ty_basic_md,
958 ],
959 false,
960 ),
961 None,
962 ),
963 table_init: add_function_with_attrs(
964 "wasmer_vm_table_init",
965 void_ty.fn_type(
966 &[
967 ctx_ptr_ty_basic_md,
968 i32_ty_basic_md,
969 i32_ty_basic_md,
970 i32_ty_basic_md,
971 i32_ty_basic_md,
972 i32_ty_basic_md,
973 ],
974 false,
975 ),
976 None,
977 ),
978 table_fill: add_function_with_attrs(
979 "wasmer_vm_table_fill",
980 void_ty.fn_type(
981 &[
982 ctx_ptr_ty_basic_md,
983 i32_ty_basic_md,
984 i32_ty_basic_md,
985 anyref_ty_basic_md,
986 i32_ty_basic_md,
987 ],
988 false,
989 ),
990 None,
991 ),
992 table_size: add_function_with_attrs(
993 "wasmer_vm_table_size",
994 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
995 None,
996 ),
997 imported_table_size: add_function_with_attrs(
998 "wasmer_vm_imported_table_size",
999 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1000 None,
1001 ),
1002 table_get: add_function_with_attrs(
1003 "wasmer_vm_table_get",
1004 anyref_ty.fn_type(
1005 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1006 false,
1007 ),
1008 None,
1009 ),
1010 imported_table_get: add_function_with_attrs(
1011 "wasmer_vm_imported_table_get",
1012 anyref_ty.fn_type(
1013 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1014 false,
1015 ),
1016 None,
1017 ),
1018 table_set: add_function_with_attrs(
1019 "wasmer_vm_table_set",
1020 void_ty.fn_type(
1021 &[
1022 ctx_ptr_ty_basic_md,
1023 i32_ty_basic_md,
1024 i32_ty_basic_md,
1025 anyref_ty_basic_md,
1026 ],
1027 false,
1028 ),
1029 None,
1030 ),
1031 imported_table_set: add_function_with_attrs(
1032 "wasmer_vm_imported_table_set",
1033 void_ty.fn_type(
1034 &[
1035 ctx_ptr_ty_basic_md,
1036 i32_ty_basic_md,
1037 i32_ty_basic_md,
1038 anyref_ty_basic_md,
1039 ],
1040 false,
1041 ),
1042 None,
1043 ),
1044 table_grow: add_function_with_attrs(
1045 "wasmer_vm_table_grow",
1046 i32_ty.fn_type(
1047 &[
1048 ctx_ptr_ty_basic_md,
1049 anyref_ty_basic_md,
1050 i32_ty_basic_md,
1051 i32_ty_basic_md,
1052 ],
1053 false,
1054 ),
1055 None,
1056 ),
1057 imported_table_grow: add_function_with_attrs(
1058 "wasmer_vm_imported_table_grow",
1059 i32_ty.fn_type(
1060 &[
1061 ctx_ptr_ty_basic_md,
1062 anyref_ty_basic_md,
1063 i32_ty_basic_md,
1064 i32_ty_basic_md,
1065 ],
1066 false,
1067 ),
1068 None,
1069 ),
1070 memory_init: add_function_with_attrs(
1071 "wasmer_vm_memory32_init",
1072 void_ty.fn_type(
1073 &[
1074 ctx_ptr_ty_basic_md,
1075 i32_ty_basic_md,
1076 i32_ty_basic_md,
1077 i32_ty_basic_md,
1078 i32_ty_basic_md,
1079 i32_ty_basic_md,
1080 ],
1081 false,
1082 ),
1083 None,
1084 ),
1085 memory_copy: add_function_with_attrs(
1086 "wasmer_vm_memory32_copy",
1087 void_ty.fn_type(
1088 &[
1089 ctx_ptr_ty_basic_md,
1090 i32_ty_basic_md,
1091 i32_ty_basic_md,
1092 i32_ty_basic_md,
1093 i32_ty_basic_md,
1094 ],
1095 false,
1096 ),
1097 None,
1098 ),
1099 imported_memory_copy: add_function_with_attrs(
1100 "wasmer_vm_imported_memory32_copy",
1101 void_ty.fn_type(
1102 &[
1103 ctx_ptr_ty_basic_md,
1104 i32_ty_basic_md,
1105 i32_ty_basic_md,
1106 i32_ty_basic_md,
1107 i32_ty_basic_md,
1108 ],
1109 false,
1110 ),
1111 None,
1112 ),
1113 memory_fill: add_function_with_attrs(
1114 "wasmer_vm_memory32_fill",
1115 void_ty.fn_type(
1116 &[
1117 ctx_ptr_ty_basic_md,
1118 i32_ty_basic_md,
1119 i32_ty_basic_md,
1120 i32_ty_basic_md,
1121 i32_ty_basic_md,
1122 ],
1123 false,
1124 ),
1125 None,
1126 ),
1127 imported_memory_fill: add_function_with_attrs(
1128 "wasmer_vm_imported_memory32_fill",
1129 void_ty.fn_type(
1130 &[
1131 ctx_ptr_ty_basic_md,
1132 i32_ty_basic_md,
1133 i32_ty_basic_md,
1134 i32_ty_basic_md,
1135 i32_ty_basic_md,
1136 ],
1137 false,
1138 ),
1139 None,
1140 ),
1141 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1142 memory_grow_ty: i32_ty.fn_type(
1143 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1144 false,
1145 ),
1146 data_drop: add_function_with_attrs(
1147 "wasmer_vm_data_drop",
1148 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1149 None,
1150 ),
1151 func_ref: add_function_with_attrs(
1152 "wasmer_vm_func_ref",
1153 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1154 None,
1155 ),
1156 elem_drop: add_function_with_attrs(
1157 "wasmer_vm_elem_drop",
1158 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1159 None,
1160 ),
1161 throw_trap: add_function_with_attrs(
1162 "wasmer_vm_raise_trap",
1163 void_ty.fn_type(&[i32_ty_basic_md], false),
1164 None,
1165 ),
1166
1167 throw: add_function_with_attrs(
1168 "wasmer_vm_throw",
1169 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1170 None,
1171 ),
1172 alloc_exception: add_function_with_attrs(
1173 "wasmer_vm_alloc_exception",
1174 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1175 None,
1176 ),
1177 read_exnref: add_function_with_attrs(
1178 "wasmer_vm_read_exnref",
1179 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1180 None,
1181 ),
1182 exception_into_exnref: add_function_with_attrs(
1183 "wasmer_vm_exception_into_exnref",
1184 i32_ty.fn_type(&[ptr_ty.into()], false),
1185 None,
1186 ),
1187 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1188
1189 debug_ptr: add_function_with_attrs(
1190 "wasmer_vm_dbg_usize",
1191 void_ty.fn_type(&[ptr_ty.into()], false),
1192 None,
1193 ),
1194 debug_str: add_function_with_attrs(
1195 "wasmer_vm_dbg_str",
1196 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1197 None,
1198 ),
1199 memory_wait32: add_function_with_attrs(
1200 "wasmer_vm_memory32_atomic_wait32",
1201 i32_ty.fn_type(
1202 &[
1203 ctx_ptr_ty_basic_md,
1204 i32_ty_basic_md,
1205 i32_ty_basic_md,
1206 i32_ty_basic_md,
1207 i64_ty_basic_md,
1208 ],
1209 false,
1210 ),
1211 None,
1212 ),
1213 memory_wait32_ty: i32_ty.fn_type(
1214 &[
1215 ctx_ptr_ty_basic_md,
1216 i32_ty_basic_md,
1217 i32_ty_basic_md,
1218 i32_ty_basic_md,
1219 i64_ty_basic_md,
1220 ],
1221 false,
1222 ),
1223 imported_memory_wait32: add_function_with_attrs(
1224 "wasmer_vm_imported_memory32_atomic_wait32",
1225 i32_ty.fn_type(
1226 &[
1227 ctx_ptr_ty_basic_md,
1228 i32_ty_basic_md,
1229 i32_ty_basic_md,
1230 i32_ty_basic_md,
1231 i64_ty_basic_md,
1232 ],
1233 false,
1234 ),
1235 None,
1236 ),
1237 memory_wait64: add_function_with_attrs(
1238 "wasmer_vm_memory32_atomic_wait64",
1239 i32_ty.fn_type(
1240 &[
1241 ctx_ptr_ty_basic_md,
1242 i32_ty_basic_md,
1243 i32_ty_basic_md,
1244 i64_ty_basic_md,
1245 i64_ty_basic_md,
1246 ],
1247 false,
1248 ),
1249 None,
1250 ),
1251 memory_wait64_ty: i32_ty.fn_type(
1252 &[
1253 ctx_ptr_ty_basic_md,
1254 i32_ty_basic_md,
1255 i32_ty_basic_md,
1256 i64_ty_basic_md,
1257 i64_ty_basic_md,
1258 ],
1259 false,
1260 ),
1261 imported_memory_wait64: add_function_with_attrs(
1262 "wasmer_vm_imported_memory32_atomic_wait64",
1263 i32_ty.fn_type(
1264 &[
1265 ctx_ptr_ty_basic_md,
1266 i32_ty_basic_md,
1267 i32_ty_basic_md,
1268 i64_ty_basic_md,
1269 i64_ty_basic_md,
1270 ],
1271 false,
1272 ),
1273 None,
1274 ),
1275 memory_notify: add_function_with_attrs(
1276 "wasmer_vm_memory32_atomic_notify",
1277 i32_ty.fn_type(
1278 &[
1279 ctx_ptr_ty_basic_md,
1280 i32_ty_basic_md,
1281 i32_ty_basic_md,
1282 i32_ty_basic_md,
1283 ],
1284 false,
1285 ),
1286 None,
1287 ),
1288 memory_notify_ty: i32_ty.fn_type(
1289 &[
1290 ctx_ptr_ty_basic_md,
1291 i32_ty_basic_md,
1292 i32_ty_basic_md,
1293 i32_ty_basic_md,
1294 ],
1295 false,
1296 ),
1297 imported_memory_notify: add_function_with_attrs(
1298 "wasmer_vm_imported_memory32_atomic_notify",
1299 i32_ty.fn_type(
1300 &[
1301 ctx_ptr_ty_basic_md,
1302 i32_ty_basic_md,
1303 i32_ty_basic_md,
1304 i32_ty_basic_md,
1305 ],
1306 false,
1307 ),
1308 None,
1309 ),
1310
1311 vmfunction_import_ty: context.struct_type(
1312 &[
1313 i8_ptr_ty_basic,
1314 i8_ptr_ty_basic,
1315 i8_ptr_ty_basic,
1316 i1_ty_basic,
1317 ],
1318 false,
1319 ),
1320 vmfunction_import_body_element: 0,
1321 vmfunction_import_vmctx_element: 1,
1322 vmfunction_import_include_m0_param_element: 3,
1323 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1324 vmmemory_definition_base_element: 0,
1325 vmmemory_definition_current_length_element: 1,
1326
1327 ptr_ty,
1328
1329 x86_64: X86_64Intrinsics {
1330 pshufb128: add_function_with_attrs(
1331 "llvm.x86.ssse3.pshuf.b.128",
1332 ret_i8x16_take_i8x16_i8x16,
1333 None,
1334 ),
1335 pmaddubsw128: add_function_with_attrs(
1336 "llvm.x86.ssse3.pmadd.ub.sw.128",
1337 i16x8_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false),
1338 None,
1339 ),
1340 pmaddwd128: add_function_with_attrs(
1341 "llvm.x86.sse2.pmadd.wd",
1342 i32x4_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false),
1343 None,
1344 ),
1345 pmulhrsw128: add_function_with_attrs(
1346 "llvm.x86.ssse3.pmul.hr.sw.128",
1347 ret_i16x8_take_i16x8_i16x8,
1348 None,
1349 ),
1350 pblendvb: add_function_with_attrs(
1351 "llvm.x86.sse41.pblendvb",
1352 ret_i8x16_take_i8x16_i8x16_i8x16,
1353 None,
1354 ),
1355 min_ps: add_function_with_attrs(
1356 "llvm.x86.sse.min.ps",
1357 ret_f32x4_take_f32x4_f32x4,
1358 None,
1359 ),
1360 min_pd: add_function_with_attrs(
1361 "llvm.x86.sse2.min.pd",
1362 ret_f64x2_take_f64x2_f64x2,
1363 None,
1364 ),
1365 max_ps: add_function_with_attrs(
1366 "llvm.x86.sse.max.ps",
1367 ret_f32x4_take_f32x4_f32x4,
1368 None,
1369 ),
1370 max_pd: add_function_with_attrs(
1371 "llvm.x86.sse2.max.pd",
1372 ret_f64x2_take_f64x2_f64x2,
1373 None,
1374 ),
1375 cvttps2dq: add_function_with_attrs(
1376 "llvm.x86.sse2.cvttps2dq",
1377 ret_i32x4_take_f32x4,
1378 None,
1379 ),
1380 cvtps2udq128: add_function_with_attrs(
1381 "llvm.x86.avx512.mask.cvtps2udq.128",
1382 ret_i32x4_take_f32x4_i32x4_i8,
1383 None,
1384 ),
1385 cvtpd2dq: add_function_with_attrs(
1386 "llvm.x86.sse2.cvtpd2dq",
1387 ret_i32x4_take_f64x2,
1388 None,
1389 ),
1390 cvtpd2udq128: add_function_with_attrs(
1391 "llvm.x86.avx512.mask.cvtpd2udq.128",
1392 ret_i32x4_take_f64x2_i32x4_i8,
1393 None,
1394 ),
1395 },
1396 };
1397
1398 let noreturn =
1399 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1400 intrinsics
1401 .throw_trap
1402 .add_attribute(AttributeLoc::Function, noreturn);
1403 intrinsics
1404 }
1405}
1406
1407#[derive(Clone, Copy)]
1408pub enum MemoryCache<'ctx> {
1409 Dynamic {
1411 ptr_to_base_ptr: PointerValue<'ctx>,
1412 ptr_to_current_length: PointerValue<'ctx>,
1413 },
1414 Static { base_ptr: PointerValue<'ctx> },
1416}
1417
1418#[derive(Clone)]
1419struct TableCache<'ctx> {
1420 ptr_to_base_ptr: PointerValue<'ctx>,
1421 ptr_to_bounds: PointerValue<'ctx>,
1422}
1423
1424#[derive(Clone, Copy)]
1425pub enum GlobalCache<'ctx> {
1426 Mut {
1427 ptr_to_value: PointerValue<'ctx>,
1428 value_type: BasicTypeEnum<'ctx>,
1429 },
1430 Const {
1431 value: BasicValueEnum<'ctx>,
1432 },
1433}
1434
1435#[derive(Clone)]
1436pub struct FunctionCache<'ctx> {
1437 pub func: PointerValue<'ctx>,
1438 pub llvm_func_type: FunctionType<'ctx>,
1439 pub vmctx: BasicValueEnum<'ctx>,
1440 pub imported_include_m0_param: Option<IntValue<'ctx>>,
1441 pub attrs: Vec<(Attribute, AttributeLoc)>,
1442}
1443
1444pub struct CtxType<'ctx, 'a> {
1445 ctx_ptr_value: PointerValue<'ctx>,
1446 m0: Option<PointerValue<'ctx>>,
1447
1448 wasm_module: &'a WasmerCompilerModule,
1449 cache_builder: &'a Builder<'ctx>,
1450 abi: &'a dyn Abi,
1451
1452 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1453 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1454 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1455 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1456 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1457
1458 offsets: VMOffsets,
1459}
1460
1461impl<'ctx, 'a> CtxType<'ctx, 'a> {
1462 pub fn new(
1463 wasm_module: &'a WasmerCompilerModule,
1464 func_value: &FunctionValue<'ctx>,
1465 cache_builder: &'a Builder<'ctx>,
1466 abi: &'a dyn Abi,
1467 pointer_width: u8,
1468 m0: Option<PointerValue<'ctx>>,
1469 ) -> CtxType<'ctx, 'a> {
1470 CtxType {
1471 m0,
1472 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1473
1474 wasm_module,
1475 cache_builder,
1476 abi,
1477
1478 cached_memories: HashMap::new(),
1479 cached_tables: HashMap::new(),
1480 cached_globals: HashMap::new(),
1481 cached_functions: HashMap::new(),
1482 cached_memory_op: HashMap::new(),
1483
1484 offsets: VMOffsets::new(pointer_width, wasm_module),
1485 }
1486 }
1487
1488 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1489 self.ctx_ptr_value.as_basic_value_enum()
1490 }
1491
1492 pub fn memory(
1493 &mut self,
1494 index: MemoryIndex,
1495 intrinsics: &Intrinsics<'ctx>,
1496 module: &Module<'ctx>,
1497 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1498 ) -> Result<MemoryCache<'ctx>, CompileError> {
1499 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1500 &mut self.cached_memories,
1501 self.wasm_module,
1502 self.ctx_ptr_value,
1503 &self.cache_builder,
1504 &self.offsets,
1505 );
1506 let memory_style = &memory_styles[index];
1507 match cached_memories.get(&index) {
1508 Some(r) => Ok(*r),
1509 None => {
1510 let memory_definition_ptr =
1511 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1512 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1513 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1514 unsafe {
1515 err!(cache_builder.build_gep(
1516 intrinsics.i8_ty,
1517 ctx_ptr_value,
1518 &[offset],
1519 ""
1520 ))
1521 }
1522 } else {
1523 let offset = offsets.vmctx_vmmemory_import(index);
1524 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1525 let memory_definition_ptr_ptr = unsafe {
1526 err!(cache_builder.build_gep(
1527 intrinsics.i8_ty,
1528 ctx_ptr_value,
1529 &[offset],
1530 ""
1531 ))
1532 };
1533 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1534 memory_definition_ptr_ptr,
1535 intrinsics.ptr_ty,
1536 "",
1537 ))
1538 .into_pointer_value();
1539 let memory_definition_ptr = err!(cache_builder.build_load(
1540 intrinsics.ptr_ty,
1541 memory_definition_ptr_ptr,
1542 ""
1543 ))
1544 .into_pointer_value();
1545 tbaa_label(
1546 module,
1547 intrinsics,
1548 format!("memory {} definition", index.as_u32()),
1549 memory_definition_ptr.as_instruction_value().unwrap(),
1550 );
1551 memory_definition_ptr
1552 };
1553 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1554 memory_definition_ptr,
1555 intrinsics.ptr_ty,
1556 "",
1557 ))
1558 .into_pointer_value();
1559 let base_ptr = err!(cache_builder.build_struct_gep(
1560 intrinsics.vmmemory_definition_ty,
1561 memory_definition_ptr,
1562 intrinsics.vmmemory_definition_base_element,
1563 "",
1564 ));
1565 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1566 let current_length_ptr = err!(cache_builder.build_struct_gep(
1567 intrinsics.vmmemory_definition_ty,
1568 memory_definition_ptr,
1569 intrinsics.vmmemory_definition_current_length_element,
1570 "",
1571 ));
1572 MemoryCache::Dynamic {
1573 ptr_to_base_ptr: base_ptr,
1574 ptr_to_current_length: current_length_ptr,
1575 }
1576 } else {
1577 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1578 .into_pointer_value();
1579 tbaa_label(
1580 module,
1581 intrinsics,
1582 format!("memory base_ptr {}", index.as_u32()),
1583 base_ptr.as_instruction_value().unwrap(),
1584 );
1585 MemoryCache::Static { base_ptr }
1586 };
1587
1588 self.cached_memories.insert(index, value);
1589 Ok(*self.cached_memories.get(&index).unwrap())
1590 }
1591 }
1592 }
1593
1594 fn build_table_prepare(
1595 table_index: TableIndex,
1596 intrinsics: &Intrinsics<'ctx>,
1597 module: &Module<'ctx>,
1598 wasm_module: &WasmerCompilerModule,
1599 ctx_ptr_value: PointerValue<'ctx>,
1600 offsets: &VMOffsets,
1601 builder: &Builder<'ctx>,
1602 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1603 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1604 let offset = intrinsics.i64_ty.const_int(
1605 offsets
1606 .vmctx_vmtable_definition_base(local_table_index)
1607 .into(),
1608 false,
1609 );
1610 let ptr_to_base_ptr =
1611 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1612 let ptr_to_base_ptr =
1613 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1614 .into_pointer_value();
1615 let offset = intrinsics.i64_ty.const_int(
1616 offsets
1617 .vmctx_vmtable_definition_current_elements(local_table_index)
1618 .into(),
1619 false,
1620 );
1621 let ptr_to_bounds =
1622 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1623 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1624 .into_pointer_value();
1625 Ok((ptr_to_base_ptr, ptr_to_bounds))
1626 } else {
1627 let offset = intrinsics.i64_ty.const_int(
1628 offsets.vmctx_vmtable_import_definition(table_index).into(),
1629 false,
1630 );
1631 let definition_ptr_ptr =
1632 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1633 let definition_ptr_ptr =
1634 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1635 .into_pointer_value();
1636 let definition_ptr =
1637 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1638 .into_pointer_value();
1639 tbaa_label(
1640 module,
1641 intrinsics,
1642 format!("table {} definition", table_index.as_u32()),
1643 definition_ptr.as_instruction_value().unwrap(),
1644 );
1645
1646 let offset = intrinsics
1647 .i64_ty
1648 .const_int(offsets.vmtable_definition_base().into(), false);
1649 let ptr_to_base_ptr =
1650 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1651 let ptr_to_base_ptr =
1652 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1653 .into_pointer_value();
1654 let offset = intrinsics
1655 .i64_ty
1656 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1657 let ptr_to_bounds =
1658 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1659 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1660 .into_pointer_value();
1661 Ok((ptr_to_base_ptr, ptr_to_bounds))
1662 }
1663 }
1664
1665 fn table_prepare(
1666 &mut self,
1667 table_index: TableIndex,
1668 intrinsics: &Intrinsics<'ctx>,
1669 module: &Module<'ctx>,
1670 body_builder: &Builder<'ctx>,
1671 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1672 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1673 &mut self.cached_tables,
1674 self.wasm_module,
1675 self.ctx_ptr_value,
1676 &self.cache_builder,
1677 &self.offsets,
1678 );
1679
1680 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1681 CompileError::Codegen(format!(
1682 "Table index out of bounds: {}",
1683 table_index.as_u32()
1684 ))
1685 })?;
1686
1687 if is_growable {
1690 Self::build_table_prepare(
1691 table_index,
1692 intrinsics,
1693 module,
1694 wasm_module,
1695 ctx_ptr_value,
1696 offsets,
1697 body_builder,
1698 )
1699 } else {
1700 let TableCache {
1701 ptr_to_base_ptr,
1702 ptr_to_bounds,
1703 } = match cached_tables.entry(table_index) {
1704 Entry::Occupied(entry) => entry.get().clone(),
1705 Entry::Vacant(entry) => {
1706 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1707 table_index,
1708 intrinsics,
1709 module,
1710 wasm_module,
1711 ctx_ptr_value,
1712 offsets,
1713 cache_builder,
1714 )?;
1715
1716 let v = TableCache {
1717 ptr_to_base_ptr,
1718 ptr_to_bounds,
1719 };
1720
1721 entry.insert(v.clone());
1722
1723 v
1724 }
1725 };
1726
1727 Ok((ptr_to_base_ptr, ptr_to_bounds))
1728 }
1729 }
1730
1731 pub fn table(
1732 &mut self,
1733 index: TableIndex,
1734 intrinsics: &Intrinsics<'ctx>,
1735 module: &Module<'ctx>,
1736 body_builder: &Builder<'ctx>,
1737 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1738 let (ptr_to_base_ptr, ptr_to_bounds) =
1739 self.table_prepare(index, intrinsics, module, body_builder)?;
1740
1741 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1743 &body_builder
1744 } else {
1745 &self.cache_builder
1746 };
1747
1748 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1749 .into_pointer_value();
1750 let bounds =
1751 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1752 tbaa_label(
1753 module,
1754 intrinsics,
1755 format!("table_base_ptr {}", index.index()),
1756 base_ptr.as_instruction_value().unwrap(),
1757 );
1758 tbaa_label(
1759 module,
1760 intrinsics,
1761 format!("table_bounds {}", index.index()),
1762 bounds.as_instruction_value().unwrap(),
1763 );
1764 Ok((base_ptr, bounds))
1765 }
1766
1767 pub fn fixed_funcref_table_anyfuncs(
1769 &self,
1770 index: LocalTableIndex,
1771 intrinsics: &Intrinsics<'ctx>,
1772 builder: &Builder<'ctx>,
1773 ) -> Result<PointerValue<'ctx>, CompileError> {
1774 let offset = intrinsics.i64_ty.const_int(
1775 self.offsets
1776 .vmctx_fixed_funcref_table_anyfuncs(index)
1777 .expect("fixed funcref table must have inline VMContext storage")
1778 .into(),
1779 false,
1780 );
1781 let ptr =
1782 unsafe { err!(builder.build_gep(intrinsics.i8_ty, self.ctx_ptr_value, &[offset], "")) };
1783 Ok(err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value())
1784 }
1785
1786 pub fn global(
1787 &mut self,
1788 index: GlobalIndex,
1789 intrinsics: &Intrinsics<'ctx>,
1790 module: &Module<'ctx>,
1791 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1792 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1793 &mut self.cached_globals,
1794 self.wasm_module,
1795 self.ctx_ptr_value,
1796 &self.cache_builder,
1797 &self.offsets,
1798 );
1799 match cached_globals.entry(index) {
1800 Entry::Occupied(entry) => Ok(entry.into_mut()),
1801 Entry::Vacant(entry) => {
1802 let global_type = wasm_module.globals[index];
1803 let global_value_type = global_type.ty;
1804
1805 let global_mutability = global_type.mutability;
1806 let global_ptr = if let Some(local_global_index) =
1807 wasm_module.local_global_index(index)
1808 {
1809 let offset = offsets.vmctx_vmglobal_definition(local_global_index);
1810 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1811 unsafe {
1812 err!(cache_builder.build_gep(
1813 intrinsics.i8_ty,
1814 ctx_ptr_value,
1815 &[offset],
1816 ""
1817 ))
1818 }
1819 } else {
1820 let offset = offsets.vmctx_vmglobal_import(index);
1821 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1822 let global_ptr_ptr = unsafe {
1823 err!(cache_builder.build_gep(
1824 intrinsics.i8_ty,
1825 ctx_ptr_value,
1826 &[offset],
1827 ""
1828 ))
1829 };
1830 let global_ptr_ptr =
1831 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1832 .into_pointer_value();
1833 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1834 .into_pointer_value()
1835 };
1836 let global_ptr =
1837 err!(cache_builder.build_bit_cast(global_ptr, intrinsics.ptr_ty, "",))
1838 .into_pointer_value();
1839
1840 let ret = entry.insert(match global_mutability {
1841 Mutability::Const => {
1842 let value = err!(cache_builder.build_load(
1843 type_to_llvm(intrinsics, global_value_type)?,
1844 global_ptr,
1845 "",
1846 ));
1847 tbaa_label(
1848 module,
1849 intrinsics,
1850 format!("global {}", index.as_u32()),
1851 value.as_instruction_value().unwrap(),
1852 );
1853 GlobalCache::Const { value }
1854 }
1855 Mutability::Var => GlobalCache::Mut {
1856 ptr_to_value: global_ptr,
1857 value_type: type_to_llvm(intrinsics, global_value_type)?,
1858 },
1859 });
1860
1861 Ok(ret)
1862 }
1863 }
1864 }
1865
1866 pub fn add_func(
1867 &mut self,
1868 function_index: FunctionIndex,
1869 func: PointerValue<'ctx>,
1870 llvm_func_type: FunctionType<'ctx>,
1871 vmctx: BasicValueEnum<'ctx>,
1872 attrs: &[(Attribute, AttributeLoc)],
1873 ) {
1874 match self.cached_functions.entry(function_index) {
1875 Entry::Occupied(_) => unreachable!("duplicate function"),
1876 Entry::Vacant(entry) => {
1877 entry.insert(FunctionCache {
1878 func,
1879 llvm_func_type,
1880 vmctx,
1881 imported_include_m0_param: None,
1882 attrs: attrs.to_vec(),
1883 });
1884 }
1885 }
1886 }
1887
1888 #[allow(clippy::too_many_arguments)]
1889 pub fn local_func(
1890 &mut self,
1891 _local_function_index: LocalFunctionIndex,
1892 function_index: FunctionIndex,
1893 intrinsics: &Intrinsics<'ctx>,
1894 module: &Module<'ctx>,
1895 context: &'ctx Context,
1896 func_type: &FuncType,
1897 function_name: &str,
1898 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1899 let (cached_functions, ctx_ptr_value, offsets) = (
1900 &mut self.cached_functions,
1901 &self.ctx_ptr_value,
1902 &self.offsets,
1903 );
1904 Ok(match cached_functions.entry(function_index) {
1905 Entry::Occupied(entry) => entry.into_mut(),
1906 Entry::Vacant(entry) => {
1907 debug_assert!(module.get_function(function_name).is_none());
1908 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1909 context,
1910 intrinsics,
1911 Some(offsets),
1912 func_type,
1913 self.m0.is_some(),
1914 )?;
1915 let func =
1916 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1917 for (attr, attr_loc) in &llvm_func_attrs {
1918 func.add_attribute(*attr_loc, *attr);
1919 }
1920 entry.insert(FunctionCache {
1921 func: func.as_global_value().as_pointer_value(),
1922 llvm_func_type,
1923 vmctx: ctx_ptr_value.as_basic_value_enum(),
1924 imported_include_m0_param: None,
1925 attrs: llvm_func_attrs,
1926 })
1927 }
1928 })
1929 }
1930
1931 pub fn imported_func(
1932 &mut self,
1933 function_index: FunctionIndex,
1934 intrinsics: &Intrinsics<'ctx>,
1935 context: &'ctx Context,
1936 func_type: &FuncType,
1937 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1938 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1939 &mut self.cached_functions,
1940 self.wasm_module,
1941 &self.ctx_ptr_value,
1942 &self.cache_builder,
1943 &self.offsets,
1944 );
1945 match cached_functions.entry(function_index) {
1946 Entry::Occupied(entry) => Ok(entry.into_mut()),
1947 Entry::Vacant(entry) => {
1948 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1949 context,
1950 intrinsics,
1951 Some(offsets),
1952 func_type,
1953 self.m0.is_some(),
1954 )?;
1955 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1956 let offset = offsets.vmctx_vmfunction_import(function_index);
1957 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1958 let vmfunction_import_ptr = unsafe {
1959 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1960 };
1961 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1962 vmfunction_import_ptr,
1963 intrinsics.ptr_ty,
1964 "",
1965 ))
1966 .into_pointer_value();
1967 vmfunction_import_ptr.set_name("vmfunction_import_ptr");
1968
1969 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1970 intrinsics.vmfunction_import_ty,
1971 vmfunction_import_ptr,
1972 intrinsics.vmfunction_import_body_element,
1973 "",
1974 ));
1975 body_ptr_ptr.set_name("body_ptr_ptr");
1976 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
1977 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
1978 .into_pointer_value();
1979 body_ptr.set_name("body_ptr");
1980 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
1981 intrinsics.vmfunction_import_ty,
1982 vmfunction_import_ptr,
1983 intrinsics.vmfunction_import_vmctx_element,
1984 "",
1985 ));
1986 vmctx_ptr_ptr.set_name("vmctx_ptr_ptr");
1987 let vmctx_ptr =
1988 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
1989 vmctx_ptr.set_name("vmctx_ptr");
1990 let include_m0_param_ptr = err!(cache_builder.build_struct_gep(
1991 intrinsics.vmfunction_import_ty,
1992 vmfunction_import_ptr,
1993 intrinsics.vmfunction_import_include_m0_param_element,
1994 "",
1995 ));
1996 include_m0_param_ptr.set_name("include_m0_param_ptr");
1997 let imported_include_m0_param =
1998 err!(cache_builder.build_load(intrinsics.i1_ty, include_m0_param_ptr, "",))
1999 .into_int_value();
2000 imported_include_m0_param.set_name("imported_include_m0_param");
2001
2002 Ok(entry.insert(FunctionCache {
2003 func: body_ptr,
2004 llvm_func_type,
2005 vmctx: vmctx_ptr,
2006 imported_include_m0_param: Some(imported_include_m0_param),
2007 attrs: llvm_func_attrs,
2008 }))
2009 }
2010 }
2011 }
2012
2013 pub fn memory_grow(
2014 &mut self,
2015 memory_index: MemoryIndex,
2016 intrinsics: &Intrinsics<'ctx>,
2017 ) -> Result<PointerValue<'ctx>, CompileError> {
2018 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2019 &mut self.cached_memory_op,
2020 &self.wasm_module,
2021 &self.offsets,
2022 &self.cache_builder,
2023 &self.ctx_ptr_value,
2024 );
2025 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
2026 Entry::Occupied(entry) => Ok(*entry.get()),
2027 Entry::Vacant(entry) => {
2028 let (grow_fn, grow_fn_ty) =
2029 if wasm_module.local_memory_index(memory_index).is_some() {
2030 (
2031 VMBuiltinFunctionIndex::get_memory32_grow_index(),
2032 intrinsics.ptr_ty,
2033 )
2034 } else {
2035 (
2036 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
2037 intrinsics.ptr_ty,
2038 )
2039 };
2040 let offset = offsets.vmctx_builtin_function(grow_fn);
2041 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2042 let grow_fn_ptr_ptr = unsafe {
2043 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2044 };
2045
2046 let grow_fn_ptr_ptr =
2047 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2048 .into_pointer_value();
2049 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
2050 .into_pointer_value();
2051
2052 entry.insert(val);
2053 Ok(val)
2054 }
2055 }
2056 }
2057
2058 pub fn memory_size(
2059 &mut self,
2060 memory_index: MemoryIndex,
2061 intrinsics: &Intrinsics<'ctx>,
2062 ) -> Result<PointerValue<'ctx>, CompileError> {
2063 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2064 &mut self.cached_memory_op,
2065 &self.wasm_module,
2066 &self.offsets,
2067 &self.cache_builder,
2068 &self.ctx_ptr_value,
2069 );
2070
2071 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
2072 Entry::Occupied(entry) => Ok(*entry.get()),
2073 Entry::Vacant(entry) => {
2074 let (size_fn, size_fn_ty) =
2075 if wasm_module.local_memory_index(memory_index).is_some() {
2076 (
2077 VMBuiltinFunctionIndex::get_memory32_size_index(),
2078 intrinsics.ptr_ty,
2079 )
2080 } else {
2081 (
2082 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
2083 intrinsics.ptr_ty,
2084 )
2085 };
2086 let offset = offsets.vmctx_builtin_function(size_fn);
2087 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2088 let size_fn_ptr_ptr = unsafe {
2089 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2090 };
2091
2092 let size_fn_ptr_ptr =
2093 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2094 .into_pointer_value();
2095
2096 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2097 .into_pointer_value();
2098 entry.insert(val);
2099 Ok(val)
2100 }
2101 }
2102 }
2103
2104 pub fn memory_wait32(
2105 &mut self,
2106 memory_index: MemoryIndex,
2107 intrinsics: &Intrinsics<'ctx>,
2108 ) -> Result<PointerValue<'ctx>, CompileError> {
2109 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2110 &mut self.cached_memory_op,
2111 &self.wasm_module,
2112 &self.offsets,
2113 &self.cache_builder,
2114 &self.ctx_ptr_value,
2115 );
2116 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
2117 Entry::Occupied(entry) => Ok(*entry.get()),
2118 Entry::Vacant(entry) => {
2119 let (size_fn, size_fn_ty) =
2120 if wasm_module.local_memory_index(memory_index).is_some() {
2121 (
2122 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
2123 intrinsics.ptr_ty,
2124 )
2125 } else {
2126 (
2127 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
2128 intrinsics.ptr_ty,
2129 )
2130 };
2131 let offset = offsets.vmctx_builtin_function(size_fn);
2132 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2133 let size_fn_ptr_ptr = unsafe {
2134 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2135 };
2136
2137 let size_fn_ptr_ptr =
2138 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2139 .into_pointer_value();
2140
2141 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2142 .into_pointer_value();
2143
2144 entry.insert(val);
2145 Ok(val)
2146 }
2147 }
2148 }
2149
2150 pub fn memory_wait64(
2151 &mut self,
2152 memory_index: MemoryIndex,
2153 intrinsics: &Intrinsics<'ctx>,
2154 ) -> Result<PointerValue<'ctx>, CompileError> {
2155 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2156 &mut self.cached_memory_op,
2157 &self.wasm_module,
2158 &self.offsets,
2159 &self.cache_builder,
2160 &self.ctx_ptr_value,
2161 );
2162
2163 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
2164 Entry::Occupied(entry) => Ok(*entry.get()),
2165 Entry::Vacant(entry) => {
2166 let (size_fn, size_fn_ty) =
2167 if wasm_module.local_memory_index(memory_index).is_some() {
2168 (
2169 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
2170 intrinsics.ptr_ty,
2171 )
2172 } else {
2173 (
2174 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
2175 intrinsics.ptr_ty,
2176 )
2177 };
2178 let offset = offsets.vmctx_builtin_function(size_fn);
2179 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2180 let size_fn_ptr_ptr = unsafe {
2181 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2182 };
2183
2184 let size_fn_ptr_ptr =
2185 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2186 .into_pointer_value();
2187
2188 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2189 .into_pointer_value();
2190 entry.insert(val);
2191 Ok(val)
2192 }
2193 }
2194 }
2195
2196 pub fn memory_notify(
2197 &mut self,
2198 memory_index: MemoryIndex,
2199 intrinsics: &Intrinsics<'ctx>,
2200 ) -> Result<PointerValue<'ctx>, CompileError> {
2201 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2202 &mut self.cached_memory_op,
2203 &self.wasm_module,
2204 &self.offsets,
2205 &self.cache_builder,
2206 &self.ctx_ptr_value,
2207 );
2208 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2209 Entry::Occupied(entry) => Ok(*entry.get()),
2210 Entry::Vacant(entry) => {
2211 let (size_fn, size_fn_ty) =
2212 if wasm_module.local_memory_index(memory_index).is_some() {
2213 (
2214 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2215 intrinsics.ptr_ty,
2216 )
2217 } else {
2218 (
2219 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2220 intrinsics.ptr_ty,
2221 )
2222 };
2223 let offset = offsets.vmctx_builtin_function(size_fn);
2224 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2225 let size_fn_ptr_ptr = unsafe {
2226 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2227 };
2228
2229 let size_fn_ptr_ptr =
2230 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2231 .into_pointer_value();
2232
2233 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2234 .into_pointer_value();
2235
2236 entry.insert(val);
2237 Ok(val)
2238 }
2239 }
2240 }
2241
2242 pub fn get_offsets(&self) -> &VMOffsets {
2243 &self.offsets
2244 }
2245}
2246
2247pub fn tbaa_label<'ctx>(
2250 module: &Module<'ctx>,
2251 intrinsics: &Intrinsics<'ctx>,
2252 label: String,
2253 instruction: InstructionValue<'ctx>,
2254) {
2255 let context = module.get_context();
2269
2270 let tbaa_root = module
2272 .get_global_metadata("wasmer_tbaa_root")
2273 .pop()
2274 .unwrap_or_else(|| {
2275 module
2276 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2277 .unwrap();
2278 module.get_global_metadata("wasmer_tbaa_root")[0]
2279 });
2280
2281 let type_label = context.metadata_string(label.as_str());
2284 let type_tbaa = module
2285 .get_global_metadata(label.as_str())
2286 .pop()
2287 .unwrap_or_else(|| {
2288 module
2289 .add_global_metadata(
2290 label.as_str(),
2291 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2292 )
2293 .unwrap();
2294 module.get_global_metadata(label.as_str())[0]
2295 });
2296
2297 let label = label + "_memop";
2304 let type_tbaa = module
2305 .get_global_metadata(label.as_str())
2306 .pop()
2307 .unwrap_or_else(|| {
2308 module
2309 .add_global_metadata(
2310 label.as_str(),
2311 &context.metadata_node(&[
2312 type_tbaa.into(),
2313 type_tbaa.into(),
2314 intrinsics.i64_zero.into(),
2315 ]),
2316 )
2317 .unwrap();
2318 module.get_global_metadata(label.as_str())[0]
2319 });
2320
2321 let tbaa_kind = context.get_kind_id("tbaa");
2323 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2324}
2325
2326fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2327 let table = module.tables.get(index)?;
2328 match table.maximum {
2329 None => Some(true),
2330 Some(max) => Some(max > table.minimum),
2331 }
2332}