1use crate::LLVM;
8use crate::abi::Abi;
9use crate::error::err;
10use inkwell::values::BasicMetadataValueEnum;
11use inkwell::{
12 AddressSpace,
13 attributes::{Attribute, AttributeLoc},
14 builder::Builder,
15 context::Context,
16 module::{Linkage, Module},
17 targets::TargetData,
18 types::{
19 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
20 PointerType, StructType, VectorType, VoidType,
21 },
22 values::{
23 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
24 PointerValue, VectorValue,
25 },
26};
27use std::collections::{HashMap, hash_map::Entry};
28use wasmer_types::entity::{EntityRef, PrimaryMap};
29use wasmer_types::{
30 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
31 MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, SignatureIndex, TableIndex, Type,
32};
33use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
34
35pub fn type_to_llvm_ptr<'ctx>(
36 intrinsics: &Intrinsics<'ctx>,
37 _ty: Type,
38) -> Result<PointerType<'ctx>, CompileError> {
39 Ok(intrinsics.ptr_ty)
41}
42
43pub fn type_to_llvm<'ctx>(
44 intrinsics: &Intrinsics<'ctx>,
45 ty: Type,
46) -> Result<BasicTypeEnum<'ctx>, CompileError> {
47 match ty {
48 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
49 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
50 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
51 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
52 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
53 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
54 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
55 }
56}
57
58#[allow(dead_code)]
60pub struct Intrinsics<'ctx> {
61 pub ctlz_i32: FunctionValue<'ctx>,
62 pub ctlz_i64: FunctionValue<'ctx>,
63
64 pub cttz_i32: FunctionValue<'ctx>,
65 pub cttz_i64: FunctionValue<'ctx>,
66
67 pub ctpop_i32: FunctionValue<'ctx>,
68 pub ctpop_i64: FunctionValue<'ctx>,
69 pub ctpop_i8x16: FunctionValue<'ctx>,
70
71 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
72 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
73 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
74 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
75 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
76
77 pub add_f32: FunctionValue<'ctx>,
78 pub add_f64: FunctionValue<'ctx>,
79 pub add_f32x4: FunctionValue<'ctx>,
80 pub add_f64x2: FunctionValue<'ctx>,
81
82 pub sub_f32: FunctionValue<'ctx>,
83 pub sub_f64: FunctionValue<'ctx>,
84 pub sub_f32x4: FunctionValue<'ctx>,
85 pub sub_f64x2: FunctionValue<'ctx>,
86
87 pub mul_f32: FunctionValue<'ctx>,
88 pub mul_f64: FunctionValue<'ctx>,
89 pub mul_f32x4: FunctionValue<'ctx>,
90 pub mul_f64x2: FunctionValue<'ctx>,
91
92 pub div_f32: FunctionValue<'ctx>,
93 pub div_f64: FunctionValue<'ctx>,
94 pub div_f32x4: FunctionValue<'ctx>,
95 pub div_f64x2: FunctionValue<'ctx>,
96
97 pub sqrt_f32: FunctionValue<'ctx>,
98 pub sqrt_f64: FunctionValue<'ctx>,
99 pub sqrt_f32x4: FunctionValue<'ctx>,
100 pub sqrt_f64x2: FunctionValue<'ctx>,
101
102 pub cmp_f32: FunctionValue<'ctx>,
103 pub cmp_f64: FunctionValue<'ctx>,
104 pub cmp_f32x4: FunctionValue<'ctx>,
105 pub cmp_f64x2: FunctionValue<'ctx>,
106
107 pub minimum_f32: FunctionValue<'ctx>,
108 pub minimum_f64: FunctionValue<'ctx>,
109 pub minimum_f32x4: FunctionValue<'ctx>,
110 pub minimum_f64x2: FunctionValue<'ctx>,
111
112 pub maximum_f32: FunctionValue<'ctx>,
113 pub maximum_f64: FunctionValue<'ctx>,
114 pub maximum_f32x4: FunctionValue<'ctx>,
115 pub maximum_f64x2: FunctionValue<'ctx>,
116
117 pub ceil_f32: FunctionValue<'ctx>,
118 pub ceil_f64: FunctionValue<'ctx>,
119 pub ceil_f32x4: FunctionValue<'ctx>,
120 pub ceil_f64x2: FunctionValue<'ctx>,
121
122 pub floor_f32: FunctionValue<'ctx>,
123 pub floor_f64: FunctionValue<'ctx>,
124 pub floor_f32x4: FunctionValue<'ctx>,
125 pub floor_f64x2: FunctionValue<'ctx>,
126
127 pub trunc_f32: FunctionValue<'ctx>,
128 pub trunc_f64: FunctionValue<'ctx>,
129 pub trunc_f32x4: FunctionValue<'ctx>,
130 pub trunc_f64x2: FunctionValue<'ctx>,
131
132 pub fpext_f32: FunctionValue<'ctx>,
133 pub fptrunc_f64: FunctionValue<'ctx>,
134
135 pub nearbyint_f32: FunctionValue<'ctx>,
136 pub nearbyint_f64: FunctionValue<'ctx>,
137 pub nearbyint_f32x4: FunctionValue<'ctx>,
138 pub nearbyint_f64x2: FunctionValue<'ctx>,
139
140 pub fabs_f32: FunctionValue<'ctx>,
141 pub fabs_f64: FunctionValue<'ctx>,
142 pub fabs_f32x4: FunctionValue<'ctx>,
143 pub fabs_f64x2: FunctionValue<'ctx>,
144
145 pub copysign_f32: FunctionValue<'ctx>,
146 pub copysign_f64: FunctionValue<'ctx>,
147 pub copysign_f32x4: FunctionValue<'ctx>,
148 pub copysign_f64x2: FunctionValue<'ctx>,
149
150 pub sadd_sat_i8x16: FunctionValue<'ctx>,
151 pub sadd_sat_i16x8: FunctionValue<'ctx>,
152 pub uadd_sat_i8x16: FunctionValue<'ctx>,
153 pub uadd_sat_i16x8: FunctionValue<'ctx>,
154
155 pub ssub_sat_i8x16: FunctionValue<'ctx>,
156 pub ssub_sat_i16x8: FunctionValue<'ctx>,
157 pub usub_sat_i8x16: FunctionValue<'ctx>,
158 pub usub_sat_i16x8: FunctionValue<'ctx>,
159
160 pub expect_i1: FunctionValue<'ctx>,
161 pub trap: FunctionValue<'ctx>,
162 pub debug_trap: FunctionValue<'ctx>,
163
164 pub personality: FunctionValue<'ctx>,
165 pub personality2: FunctionValue<'ctx>,
166 pub readonly: Attribute,
167 pub stack_probe: Attribute,
168 pub uwtable: Attribute,
169 pub frame_pointer: Attribute,
170 pub chkstk: FunctionValue<'ctx>,
172
173 pub void_ty: VoidType<'ctx>,
174 pub i1_ty: IntType<'ctx>,
175 pub i2_ty: IntType<'ctx>,
176 pub i4_ty: IntType<'ctx>,
177 pub i8_ty: IntType<'ctx>,
178 pub i16_ty: IntType<'ctx>,
179 pub i32_ty: IntType<'ctx>,
180 pub i64_ty: IntType<'ctx>,
181 pub i128_ty: IntType<'ctx>,
182 pub isize_ty: IntType<'ctx>,
183 pub f32_ty: FloatType<'ctx>,
184 pub f64_ty: FloatType<'ctx>,
185
186 pub i1x128_ty: VectorType<'ctx>,
187 pub i8x16_ty: VectorType<'ctx>,
188 pub i16x8_ty: VectorType<'ctx>,
189 pub i32x4_ty: VectorType<'ctx>,
190 pub i64x2_ty: VectorType<'ctx>,
191 pub f32x4_ty: VectorType<'ctx>,
192 pub f64x2_ty: VectorType<'ctx>,
193 pub i32x8_ty: VectorType<'ctx>,
194
195 pub ptr_ty: PointerType<'ctx>,
196
197 pub anyfunc_ty: StructType<'ctx>,
198
199 pub i1_zero: IntValue<'ctx>,
200 pub i8_zero: IntValue<'ctx>,
201 pub i32_zero: IntValue<'ctx>,
202 pub i64_zero: IntValue<'ctx>,
203 pub i128_zero: IntValue<'ctx>,
204 pub isize_zero: IntValue<'ctx>,
205 pub f32_zero: FloatValue<'ctx>,
206 pub f64_zero: FloatValue<'ctx>,
207 pub f32x4_zero: VectorValue<'ctx>,
208 pub f64x2_zero: VectorValue<'ctx>,
209 pub i32_consts: [IntValue<'ctx>; 16],
210
211 pub trap_unreachable: BasicValueEnum<'ctx>,
212 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
213 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
214 pub trap_memory_oob: BasicValueEnum<'ctx>,
215 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
216 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
217 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
218 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
219 pub trap_table_access_oob: BasicValueEnum<'ctx>,
220
221 pub experimental_stackmap: FunctionValue<'ctx>,
222
223 pub table_copy: FunctionValue<'ctx>,
225 pub table_init: FunctionValue<'ctx>,
226 pub table_fill: FunctionValue<'ctx>,
227 pub table_size: FunctionValue<'ctx>,
228 pub imported_table_size: FunctionValue<'ctx>,
229 pub table_get: FunctionValue<'ctx>,
230 pub imported_table_get: FunctionValue<'ctx>,
231 pub table_set: FunctionValue<'ctx>,
232 pub imported_table_set: FunctionValue<'ctx>,
233 pub table_grow: FunctionValue<'ctx>,
234 pub imported_table_grow: FunctionValue<'ctx>,
235 pub memory_init: FunctionValue<'ctx>,
236 pub data_drop: FunctionValue<'ctx>,
237 pub func_ref: FunctionValue<'ctx>,
238 pub elem_drop: FunctionValue<'ctx>,
239 pub memory_copy: FunctionValue<'ctx>,
240 pub imported_memory_copy: FunctionValue<'ctx>,
241 pub memory_fill: FunctionValue<'ctx>,
242 pub imported_memory_fill: FunctionValue<'ctx>,
243 pub memory_size_ty: FunctionType<'ctx>,
244 pub memory_grow_ty: FunctionType<'ctx>,
245 pub memory_wait32: FunctionValue<'ctx>,
246 pub memory_wait32_ty: FunctionType<'ctx>,
247 pub imported_memory_wait32: FunctionValue<'ctx>,
248 pub memory_wait64: FunctionValue<'ctx>,
249 pub memory_wait64_ty: FunctionType<'ctx>,
250 pub imported_memory_wait64: FunctionValue<'ctx>,
251 pub memory_notify: FunctionValue<'ctx>,
252 pub memory_notify_ty: FunctionType<'ctx>,
253 pub imported_memory_notify: FunctionValue<'ctx>,
254
255 pub throw_trap: FunctionValue<'ctx>,
256
257 pub throw: FunctionValue<'ctx>,
259 pub alloc_exception: FunctionValue<'ctx>,
260 pub read_exnref: FunctionValue<'ctx>,
261 pub exception_into_exnref: FunctionValue<'ctx>,
262 pub lpad_exception_ty: StructType<'ctx>,
263
264 pub debug_ptr: FunctionValue<'ctx>,
266 pub debug_str: FunctionValue<'ctx>,
267
268 pub vmfunction_import_ty: StructType<'ctx>,
270 pub vmfunction_import_body_element: u32,
271 pub vmfunction_import_vmctx_element: u32,
272
273 pub vmmemory_definition_ty: StructType<'ctx>,
274 pub vmmemory_definition_base_element: u32,
275 pub vmmemory_definition_current_length_element: u32,
276}
277
278#[derive(Debug, Hash, PartialEq, Eq)]
279enum MemoryOp {
280 Size,
281 Grow,
282 Wait32,
283 Wait64,
284 Notify,
285}
286
287impl<'ctx> Intrinsics<'ctx> {
288 pub fn declare(
290 module: &Module<'ctx>,
291 context: &'ctx Context,
292 target_data: &TargetData,
293 binary_fmt: &target_lexicon::BinaryFormat,
294 ) -> Self {
295 let void_ty = context.void_type();
296 let i1_ty = context.bool_type();
297 let i2_ty = context.custom_width_int_type(2);
298 let i4_ty = context.custom_width_int_type(4);
299 let i8_ty = context.i8_type();
300 let i16_ty = context.i16_type();
301 let i32_ty = context.i32_type();
302 let i64_ty = context.i64_type();
303 let i128_ty = context.i128_type();
304 let isize_ty = context.ptr_sized_int_type(target_data, None);
305 let f32_ty = context.f32_type();
306 let f64_ty = context.f64_type();
307
308 let i1x4_ty = i1_ty.vec_type(4);
309 let i1x2_ty = i1_ty.vec_type(2);
310 let i1x128_ty = i1_ty.vec_type(128);
311 let i8x16_ty = i8_ty.vec_type(16);
312 let i16x8_ty = i16_ty.vec_type(8);
313 let i32x4_ty = i32_ty.vec_type(4);
314 let i64x2_ty = i64_ty.vec_type(2);
315 let f32x4_ty = f32_ty.vec_type(4);
316 let f64x2_ty = f64_ty.vec_type(2);
317 let i32x8_ty = i32_ty.vec_type(8);
318
319 let ptr_ty = context.ptr_type(AddressSpace::default());
320
321 let i1_zero = i1_ty.const_int(0, false);
322 let i8_zero = i8_ty.const_int(0, false);
323 let i32_zero = i32_ty.const_int(0, false);
324 let i64_zero = i64_ty.const_int(0, false);
325 let i128_zero = i128_ty.const_int(0, false);
326 let isize_zero = isize_ty.const_int(0, false);
327 let f32_zero = f32_ty.const_float(0.0);
328 let f64_zero = f64_ty.const_float(0.0);
329 let f32x4_zero = f32x4_ty.const_zero();
330 let f64x2_zero = f64x2_ty.const_zero();
331 let i32_consts = [
332 i32_ty.const_int(0, false),
333 i32_ty.const_int(1, false),
334 i32_ty.const_int(2, false),
335 i32_ty.const_int(3, false),
336 i32_ty.const_int(4, false),
337 i32_ty.const_int(5, false),
338 i32_ty.const_int(6, false),
339 i32_ty.const_int(7, false),
340 i32_ty.const_int(8, false),
341 i32_ty.const_int(9, false),
342 i32_ty.const_int(10, false),
343 i32_ty.const_int(11, false),
344 i32_ty.const_int(12, false),
345 i32_ty.const_int(13, false),
346 i32_ty.const_int(14, false),
347 i32_ty.const_int(15, false),
348 ];
349
350 let md_ty = context.metadata_type();
351
352 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
353
354 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
355 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
356 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
357 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
358 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
359 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
360 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
361 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
362 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
363 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
364
365 let ctx_ptr_ty = ptr_ty;
366 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
367 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
368
369 let sigindex_ty = i32_ty;
370
371 let anyfunc_ty = context.struct_type(
372 &[i8_ptr_ty_basic, sigindex_ty.into(), ctx_ptr_ty_basic],
373 false,
374 );
375 let funcref_ty = ptr_ty;
376 let anyref_ty = ptr_ty;
377 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
378
379 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
380 let ret_i8x16_take_i8x16_i8x16 =
381 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
382 let ret_i16x8_take_i16x8_i16x8 =
383 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
384
385 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
386 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
387
388 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
389 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
390
391 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
392 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
393 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
394 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
395
396 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
397 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
398 let ret_f32x4_take_f32x4_f32x4 =
399 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
400 let ret_f64x2_take_f64x2_f64x2 =
401 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
402
403 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
404 let ret_f32_take_f64_md_md =
405 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
406
407 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
408
409 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
410 &[
411 f32_ty_basic_md,
412 f32_ty_basic_md,
413 md_ty_basic_md,
414 md_ty_basic_md,
415 ],
416 false,
417 );
418 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
419 &[
420 f64_ty_basic_md,
421 f64_ty_basic_md,
422 md_ty_basic_md,
423 md_ty_basic_md,
424 ],
425 false,
426 );
427 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
428 &[
429 f32x4_ty_basic_md,
430 f32x4_ty_basic_md,
431 md_ty_basic_md,
432 md_ty_basic_md,
433 ],
434 false,
435 );
436 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
437 &[
438 f64x2_ty_basic_md,
439 f64x2_ty_basic_md,
440 md_ty_basic_md,
441 md_ty_basic_md,
442 ],
443 false,
444 );
445
446 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
447 &[
448 f32_ty_basic_md,
449 f32_ty_basic_md,
450 md_ty_basic_md,
451 md_ty_basic_md,
452 ],
453 false,
454 );
455 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
456 &[
457 f64_ty_basic_md,
458 f64_ty_basic_md,
459 md_ty_basic_md,
460 md_ty_basic_md,
461 ],
462 false,
463 );
464 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
465 &[
466 f32x4_ty_basic_md,
467 f32x4_ty_basic_md,
468 md_ty_basic_md,
469 md_ty_basic_md,
470 ],
471 false,
472 );
473 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
474 &[
475 f64x2_ty_basic_md,
476 f64x2_ty_basic_md,
477 md_ty_basic_md,
478 md_ty_basic_md,
479 ],
480 false,
481 );
482
483 let intrinsics = Self {
484 ctlz_i32: module.add_function("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
485 ctlz_i64: module.add_function("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
486
487 cttz_i32: module.add_function("llvm.cttz.i32", ret_i32_take_i32_i1, None),
488 cttz_i64: module.add_function("llvm.cttz.i64", ret_i64_take_i64_i1, None),
489
490 ctpop_i32: module.add_function("llvm.ctpop.i32", ret_i32_take_i32, None),
491 ctpop_i64: module.add_function("llvm.ctpop.i64", ret_i64_take_i64, None),
492 ctpop_i8x16: module.add_function("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
493
494 fp_rounding_md: context.metadata_string("round.tonearest").into(),
495 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
496
497 fp_ogt_md: context.metadata_string("ogt").into(),
498 fp_olt_md: context.metadata_string("olt").into(),
499 fp_uno_md: context.metadata_string("uno").into(),
500
501 sqrt_f32: module.add_function("llvm.sqrt.f32", ret_f32_take_f32, None),
502 sqrt_f64: module.add_function("llvm.sqrt.f64", ret_f64_take_f64, None),
503 sqrt_f32x4: module.add_function("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
504 sqrt_f64x2: module.add_function("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
505
506 ceil_f32: module.add_function("llvm.ceil.f32", ret_f32_take_f32, None),
507 ceil_f64: module.add_function("llvm.ceil.f64", ret_f64_take_f64, None),
508 ceil_f32x4: module.add_function("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
509 ceil_f64x2: module.add_function("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
510
511 floor_f32: module.add_function("llvm.floor.f32", ret_f32_take_f32, None),
512 floor_f64: module.add_function("llvm.floor.f64", ret_f64_take_f64, None),
513 floor_f32x4: module.add_function("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
514 floor_f64x2: module.add_function("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
515
516 trunc_f32: module.add_function("llvm.trunc.f32", ret_f32_take_f32, None),
517 trunc_f64: module.add_function("llvm.trunc.f64", ret_f64_take_f64, None),
518 trunc_f32x4: module.add_function("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
519 trunc_f64x2: module.add_function("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
520
521 nearbyint_f32: module.add_function("llvm.nearbyint.f32", ret_f32_take_f32, None),
522 nearbyint_f64: module.add_function("llvm.nearbyint.f64", ret_f64_take_f64, None),
523 nearbyint_f32x4: module.add_function(
524 "llvm.nearbyint.v4f32",
525 ret_f32x4_take_f32x4,
526 None,
527 ),
528 nearbyint_f64x2: module.add_function(
529 "llvm.nearbyint.v2f64",
530 ret_f64x2_take_f64x2,
531 None,
532 ),
533
534 add_f32: module.add_function(
535 "llvm.experimental.constrained.fadd.f32",
536 ret_f32_take_f32_f32_md_md,
537 None,
538 ),
539 add_f64: module.add_function(
540 "llvm.experimental.constrained.fadd.f64",
541 ret_f64_take_f64_f64_md_md,
542 None,
543 ),
544 add_f32x4: module.add_function(
545 "llvm.experimental.constrained.fadd.v4f32",
546 ret_f32x4_take_f32x4_f32x4_md_md,
547 None,
548 ),
549 add_f64x2: module.add_function(
550 "llvm.experimental.constrained.fadd.v2f64",
551 ret_f64x2_take_f64x2_f64x2_md_md,
552 None,
553 ),
554
555 sub_f32: module.add_function(
556 "llvm.experimental.constrained.fsub.f32",
557 ret_f32_take_f32_f32_md_md,
558 None,
559 ),
560 sub_f64: module.add_function(
561 "llvm.experimental.constrained.fsub.f64",
562 ret_f64_take_f64_f64_md_md,
563 None,
564 ),
565 sub_f32x4: module.add_function(
566 "llvm.experimental.constrained.fsub.v4f32",
567 ret_f32x4_take_f32x4_f32x4_md_md,
568 None,
569 ),
570 sub_f64x2: module.add_function(
571 "llvm.experimental.constrained.fsub.v2f64",
572 ret_f64x2_take_f64x2_f64x2_md_md,
573 None,
574 ),
575
576 mul_f32: module.add_function(
577 "llvm.experimental.constrained.fmul.f32",
578 ret_f32_take_f32_f32_md_md,
579 None,
580 ),
581 mul_f64: module.add_function(
582 "llvm.experimental.constrained.fmul.f64",
583 ret_f64_take_f64_f64_md_md,
584 None,
585 ),
586 mul_f32x4: module.add_function(
587 "llvm.experimental.constrained.fmul.v4f32",
588 ret_f32x4_take_f32x4_f32x4_md_md,
589 None,
590 ),
591 mul_f64x2: module.add_function(
592 "llvm.experimental.constrained.fmul.v2f64",
593 ret_f64x2_take_f64x2_f64x2_md_md,
594 None,
595 ),
596
597 div_f32: module.add_function(
598 "llvm.experimental.constrained.fdiv.f32",
599 ret_f32_take_f32_f32_md_md,
600 None,
601 ),
602 div_f64: module.add_function(
603 "llvm.experimental.constrained.fdiv.f64",
604 ret_f64_take_f64_f64_md_md,
605 None,
606 ),
607 div_f32x4: module.add_function(
608 "llvm.experimental.constrained.fdiv.v4f32",
609 ret_f32x4_take_f32x4_f32x4_md_md,
610 None,
611 ),
612 div_f64x2: module.add_function(
613 "llvm.experimental.constrained.fdiv.v2f64",
614 ret_f64x2_take_f64x2_f64x2_md_md,
615 None,
616 ),
617
618 cmp_f32: module.add_function(
619 "llvm.experimental.constrained.fcmp.f32",
620 ret_i1_take_f32_f32_md_md,
621 None,
622 ),
623 cmp_f64: module.add_function(
624 "llvm.experimental.constrained.fcmp.f64",
625 ret_i1_take_f64_f64_md_md,
626 None,
627 ),
628 cmp_f32x4: module.add_function(
629 "llvm.experimental.constrained.fcmp.v4f32",
630 ret_i1x4_take_f32x4_f32x4_md_md,
631 None,
632 ),
633 cmp_f64x2: module.add_function(
634 "llvm.experimental.constrained.fcmp.v2f64",
635 ret_i1x2_take_f64x2_f64x2_md_md,
636 None,
637 ),
638
639 minimum_f32: module.add_function("llvm.minimum.f32", ret_f32_take_f32_f32, None),
640 minimum_f64: module.add_function("llvm.minimum.f64", ret_f64_take_f64_f64, None),
641 minimum_f32x4: module.add_function(
642 "llvm.minimum.v4f32",
643 ret_f32x4_take_f32x4_f32x4,
644 None,
645 ),
646 minimum_f64x2: module.add_function(
647 "llvm.minimum.v2f64",
648 ret_f64x2_take_f64x2_f64x2,
649 None,
650 ),
651
652 maximum_f32: module.add_function("llvm.maximum.f32", ret_f32_take_f32_f32, None),
653 maximum_f64: module.add_function("llvm.maximum.f64", ret_f64_take_f64_f64, None),
654 maximum_f32x4: module.add_function(
655 "llvm.maximum.v4f32",
656 ret_f32x4_take_f32x4_f32x4,
657 None,
658 ),
659 maximum_f64x2: module.add_function(
660 "llvm.maximum.v2f64",
661 ret_f64x2_take_f64x2_f64x2,
662 None,
663 ),
664
665 fpext_f32: module.add_function(
666 "llvm.experimental.constrained.fpext.f64.f32",
667 ret_f64_take_f32_md,
668 None,
669 ),
670 fptrunc_f64: module.add_function(
671 "llvm.experimental.constrained.fptrunc.f32.f64",
672 ret_f32_take_f64_md_md,
673 None,
674 ),
675
676 fabs_f32: module.add_function("llvm.fabs.f32", ret_f32_take_f32, None),
677 fabs_f64: module.add_function("llvm.fabs.f64", ret_f64_take_f64, None),
678 fabs_f32x4: module.add_function("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
679 fabs_f64x2: module.add_function("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
680
681 copysign_f32: module.add_function("llvm.copysign.f32", ret_f32_take_f32_f32, None),
682 copysign_f64: module.add_function("llvm.copysign.f64", ret_f64_take_f64_f64, None),
683 copysign_f32x4: module.add_function(
684 "llvm.copysign.v4f32",
685 ret_f32x4_take_f32x4_f32x4,
686 None,
687 ),
688 copysign_f64x2: module.add_function(
689 "llvm.copysign.v2f64",
690 ret_f64x2_take_f64x2_f64x2,
691 None,
692 ),
693
694 sadd_sat_i8x16: module.add_function(
695 "llvm.sadd.sat.v16i8",
696 ret_i8x16_take_i8x16_i8x16,
697 None,
698 ),
699 sadd_sat_i16x8: module.add_function(
700 "llvm.sadd.sat.v8i16",
701 ret_i16x8_take_i16x8_i16x8,
702 None,
703 ),
704 uadd_sat_i8x16: module.add_function(
705 "llvm.uadd.sat.v16i8",
706 ret_i8x16_take_i8x16_i8x16,
707 None,
708 ),
709 uadd_sat_i16x8: module.add_function(
710 "llvm.uadd.sat.v8i16",
711 ret_i16x8_take_i16x8_i16x8,
712 None,
713 ),
714
715 ssub_sat_i8x16: module.add_function(
716 "llvm.ssub.sat.v16i8",
717 ret_i8x16_take_i8x16_i8x16,
718 None,
719 ),
720 ssub_sat_i16x8: module.add_function(
721 "llvm.ssub.sat.v8i16",
722 ret_i16x8_take_i16x8_i16x8,
723 None,
724 ),
725 usub_sat_i8x16: module.add_function(
726 "llvm.usub.sat.v16i8",
727 ret_i8x16_take_i8x16_i8x16,
728 None,
729 ),
730 usub_sat_i16x8: module.add_function(
731 "llvm.usub.sat.v8i16",
732 ret_i16x8_take_i16x8_i16x8,
733 None,
734 ),
735
736 expect_i1: module.add_function("llvm.expect.i1", ret_i1_take_i1_i1, None),
737 trap: module.add_function("llvm.trap", void_ty.fn_type(&[], false), None),
738 debug_trap: module.add_function("llvm.debugtrap", void_ty.fn_type(&[], false), None),
739 personality: module.add_function(
740 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
741 "__gxx_personality_v0"
744 } else {
745 "wasmer_eh_personality"
746 },
747 i32_ty.fn_type(
748 &[
749 i32_ty.into(),
750 i32_ty.into(),
751 i64_ty.into(),
752 ptr_ty.into(),
753 ptr_ty.into(),
754 ],
755 false,
756 ),
757 None,
758 ),
759 personality2: module.add_function(
760 "wasmer_eh_personality2",
761 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
762 None,
763 ),
764 readonly: context
765 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
766 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
767 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
768 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
769 chkstk: module.add_function("__chkstk", void_ty.fn_type(&[], false), None),
770 void_ty,
771 i1_ty,
772 i2_ty,
773 i4_ty,
774 i8_ty,
775 i16_ty,
776 i32_ty,
777 i64_ty,
778 i128_ty,
779 isize_ty,
780 f32_ty,
781 f64_ty,
782
783 i1x128_ty,
784 i8x16_ty,
785 i16x8_ty,
786 i32x4_ty,
787 i64x2_ty,
788 f32x4_ty,
789 f64x2_ty,
790 i32x8_ty,
791
792 anyfunc_ty,
793 i1_zero,
794 i8_zero,
795 i32_zero,
796 i64_zero,
797 i128_zero,
798 isize_zero,
799 f32_zero,
800 f64_zero,
801 f32x4_zero,
802 f64x2_zero,
803 i32_consts,
804
805 trap_unreachable: i32_ty
806 .const_int(TrapCode::UnreachableCodeReached as _, false)
807 .as_basic_value_enum(),
808 trap_call_indirect_null: i32_ty
809 .const_int(TrapCode::IndirectCallToNull as _, false)
810 .as_basic_value_enum(),
811 trap_call_indirect_sig: i32_ty
812 .const_int(TrapCode::BadSignature as _, false)
813 .as_basic_value_enum(),
814 trap_memory_oob: i32_ty
815 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
816 .as_basic_value_enum(),
817 trap_illegal_arithmetic: i32_ty
818 .const_int(TrapCode::IntegerOverflow as _, false)
819 .as_basic_value_enum(),
820 trap_integer_division_by_zero: i32_ty
821 .const_int(TrapCode::IntegerDivisionByZero as _, false)
822 .as_basic_value_enum(),
823 trap_bad_conversion_to_integer: i32_ty
824 .const_int(TrapCode::BadConversionToInteger as _, false)
825 .as_basic_value_enum(),
826 trap_unaligned_atomic: i32_ty
827 .const_int(TrapCode::UnalignedAtomic as _, false)
828 .as_basic_value_enum(),
829 trap_table_access_oob: i32_ty
830 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
831 .as_basic_value_enum(),
832
833 experimental_stackmap: module.add_function(
834 "llvm.experimental.stackmap",
835 void_ty.fn_type(
836 &[
837 i64_ty_basic_md, i32_ty_basic_md, ],
840 true,
841 ),
842 None,
843 ),
844
845 table_copy: module.add_function(
847 "wasmer_vm_table_copy",
848 void_ty.fn_type(
849 &[
850 ctx_ptr_ty_basic_md,
851 i32_ty_basic_md,
852 i32_ty_basic_md,
853 i32_ty_basic_md,
854 i32_ty_basic_md,
855 i32_ty_basic_md,
856 ],
857 false,
858 ),
859 None,
860 ),
861 table_init: module.add_function(
862 "wasmer_vm_table_init",
863 void_ty.fn_type(
864 &[
865 ctx_ptr_ty_basic_md,
866 i32_ty_basic_md,
867 i32_ty_basic_md,
868 i32_ty_basic_md,
869 i32_ty_basic_md,
870 i32_ty_basic_md,
871 ],
872 false,
873 ),
874 None,
875 ),
876 table_fill: module.add_function(
877 "wasmer_vm_table_fill",
878 void_ty.fn_type(
879 &[
880 ctx_ptr_ty_basic_md,
881 i32_ty_basic_md,
882 i32_ty_basic_md,
883 anyref_ty_basic_md,
884 i32_ty_basic_md,
885 ],
886 false,
887 ),
888 None,
889 ),
890 table_size: module.add_function(
891 "wasmer_vm_table_size",
892 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
893 None,
894 ),
895 imported_table_size: module.add_function(
896 "wasmer_vm_imported_table_size",
897 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
898 None,
899 ),
900 table_get: module.add_function(
901 "wasmer_vm_table_get",
902 anyref_ty.fn_type(
903 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
904 false,
905 ),
906 None,
907 ),
908 imported_table_get: module.add_function(
909 "wasmer_vm_imported_table_get",
910 anyref_ty.fn_type(
911 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
912 false,
913 ),
914 None,
915 ),
916 table_set: module.add_function(
917 "wasmer_vm_table_set",
918 void_ty.fn_type(
919 &[
920 ctx_ptr_ty_basic_md,
921 i32_ty_basic_md,
922 i32_ty_basic_md,
923 anyref_ty_basic_md,
924 ],
925 false,
926 ),
927 None,
928 ),
929 imported_table_set: module.add_function(
930 "wasmer_vm_imported_table_set",
931 void_ty.fn_type(
932 &[
933 ctx_ptr_ty_basic_md,
934 i32_ty_basic_md,
935 i32_ty_basic_md,
936 anyref_ty_basic_md,
937 ],
938 false,
939 ),
940 None,
941 ),
942 table_grow: module.add_function(
943 "wasmer_vm_table_grow",
944 i32_ty.fn_type(
945 &[
946 ctx_ptr_ty_basic_md,
947 anyref_ty_basic_md,
948 i32_ty_basic_md,
949 i32_ty_basic_md,
950 ],
951 false,
952 ),
953 None,
954 ),
955 imported_table_grow: module.add_function(
956 "wasmer_vm_imported_table_grow",
957 i32_ty.fn_type(
958 &[
959 ctx_ptr_ty_basic_md,
960 anyref_ty_basic_md,
961 i32_ty_basic_md,
962 i32_ty_basic_md,
963 ],
964 false,
965 ),
966 None,
967 ),
968 memory_init: module.add_function(
969 "wasmer_vm_memory32_init",
970 void_ty.fn_type(
971 &[
972 ctx_ptr_ty_basic_md,
973 i32_ty_basic_md,
974 i32_ty_basic_md,
975 i32_ty_basic_md,
976 i32_ty_basic_md,
977 i32_ty_basic_md,
978 ],
979 false,
980 ),
981 None,
982 ),
983 memory_copy: module.add_function(
984 "wasmer_vm_memory32_copy",
985 void_ty.fn_type(
986 &[
987 ctx_ptr_ty_basic_md,
988 i32_ty_basic_md,
989 i32_ty_basic_md,
990 i32_ty_basic_md,
991 i32_ty_basic_md,
992 ],
993 false,
994 ),
995 None,
996 ),
997 imported_memory_copy: module.add_function(
998 "wasmer_vm_imported_memory32_copy",
999 void_ty.fn_type(
1000 &[
1001 ctx_ptr_ty_basic_md,
1002 i32_ty_basic_md,
1003 i32_ty_basic_md,
1004 i32_ty_basic_md,
1005 i32_ty_basic_md,
1006 ],
1007 false,
1008 ),
1009 None,
1010 ),
1011 memory_fill: module.add_function(
1012 "wasmer_vm_memory32_fill",
1013 void_ty.fn_type(
1014 &[
1015 ctx_ptr_ty_basic_md,
1016 i32_ty_basic_md,
1017 i32_ty_basic_md,
1018 i32_ty_basic_md,
1019 i32_ty_basic_md,
1020 ],
1021 false,
1022 ),
1023 None,
1024 ),
1025 imported_memory_fill: module.add_function(
1026 "wasmer_vm_imported_memory32_fill",
1027 void_ty.fn_type(
1028 &[
1029 ctx_ptr_ty_basic_md,
1030 i32_ty_basic_md,
1031 i32_ty_basic_md,
1032 i32_ty_basic_md,
1033 i32_ty_basic_md,
1034 ],
1035 false,
1036 ),
1037 None,
1038 ),
1039 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1040 memory_grow_ty: i32_ty.fn_type(
1041 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1042 false,
1043 ),
1044 data_drop: module.add_function(
1045 "wasmer_vm_data_drop",
1046 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1047 None,
1048 ),
1049 func_ref: module.add_function(
1050 "wasmer_vm_func_ref",
1051 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1052 None,
1053 ),
1054 elem_drop: module.add_function(
1055 "wasmer_vm_elem_drop",
1056 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1057 None,
1058 ),
1059 throw_trap: module.add_function(
1060 "wasmer_vm_raise_trap",
1061 void_ty.fn_type(&[i32_ty_basic_md], false),
1062 None,
1063 ),
1064
1065 throw: module.add_function(
1066 "wasmer_vm_throw",
1067 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1068 None,
1069 ),
1070 alloc_exception: module.add_function(
1071 "wasmer_vm_alloc_exception",
1072 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1073 None,
1074 ),
1075 read_exnref: module.add_function(
1076 "wasmer_vm_read_exnref",
1077 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1078 None,
1079 ),
1080 exception_into_exnref: module.add_function(
1081 "wasmer_vm_exception_into_exnref",
1082 i32_ty.fn_type(&[ptr_ty.into()], false),
1083 None,
1084 ),
1085 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1086
1087 debug_ptr: module.add_function(
1088 "wasmer_vm_dbg_usize",
1089 void_ty.fn_type(&[ptr_ty.into()], false),
1090 None,
1091 ),
1092 debug_str: module.add_function(
1093 "wasmer_vm_dbg_str",
1094 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1095 None,
1096 ),
1097 memory_wait32: module.add_function(
1098 "wasmer_vm_memory32_atomic_wait32",
1099 i32_ty.fn_type(
1100 &[
1101 ctx_ptr_ty_basic_md,
1102 i32_ty_basic_md,
1103 i32_ty_basic_md,
1104 i32_ty_basic_md,
1105 i64_ty_basic_md,
1106 ],
1107 false,
1108 ),
1109 None,
1110 ),
1111 memory_wait32_ty: i32_ty.fn_type(
1112 &[
1113 ctx_ptr_ty_basic_md,
1114 i32_ty_basic_md,
1115 i32_ty_basic_md,
1116 i32_ty_basic_md,
1117 i64_ty_basic_md,
1118 ],
1119 false,
1120 ),
1121 imported_memory_wait32: module.add_function(
1122 "wasmer_vm_imported_memory32_atomic_wait32",
1123 i32_ty.fn_type(
1124 &[
1125 ctx_ptr_ty_basic_md,
1126 i32_ty_basic_md,
1127 i32_ty_basic_md,
1128 i32_ty_basic_md,
1129 i64_ty_basic_md,
1130 ],
1131 false,
1132 ),
1133 None,
1134 ),
1135 memory_wait64: module.add_function(
1136 "wasmer_vm_memory32_atomic_wait64",
1137 i32_ty.fn_type(
1138 &[
1139 ctx_ptr_ty_basic_md,
1140 i32_ty_basic_md,
1141 i32_ty_basic_md,
1142 i64_ty_basic_md,
1143 i64_ty_basic_md,
1144 ],
1145 false,
1146 ),
1147 None,
1148 ),
1149 memory_wait64_ty: i32_ty.fn_type(
1150 &[
1151 ctx_ptr_ty_basic_md,
1152 i32_ty_basic_md,
1153 i32_ty_basic_md,
1154 i64_ty_basic_md,
1155 i64_ty_basic_md,
1156 ],
1157 false,
1158 ),
1159 imported_memory_wait64: module.add_function(
1160 "wasmer_vm_imported_memory32_atomic_wait64",
1161 i32_ty.fn_type(
1162 &[
1163 ctx_ptr_ty_basic_md,
1164 i32_ty_basic_md,
1165 i32_ty_basic_md,
1166 i64_ty_basic_md,
1167 i64_ty_basic_md,
1168 ],
1169 false,
1170 ),
1171 None,
1172 ),
1173 memory_notify: module.add_function(
1174 "wasmer_vm_memory32_atomic_notify",
1175 i32_ty.fn_type(
1176 &[
1177 ctx_ptr_ty_basic_md,
1178 i32_ty_basic_md,
1179 i32_ty_basic_md,
1180 i32_ty_basic_md,
1181 ],
1182 false,
1183 ),
1184 None,
1185 ),
1186 memory_notify_ty: i32_ty.fn_type(
1187 &[
1188 ctx_ptr_ty_basic_md,
1189 i32_ty_basic_md,
1190 i32_ty_basic_md,
1191 i32_ty_basic_md,
1192 ],
1193 false,
1194 ),
1195 imported_memory_notify: module.add_function(
1196 "wasmer_vm_imported_memory32_atomic_notify",
1197 i32_ty.fn_type(
1198 &[
1199 ctx_ptr_ty_basic_md,
1200 i32_ty_basic_md,
1201 i32_ty_basic_md,
1202 i32_ty_basic_md,
1203 ],
1204 false,
1205 ),
1206 None,
1207 ),
1208
1209 vmfunction_import_ty: context.struct_type(&[i8_ptr_ty_basic, i8_ptr_ty_basic], false),
1210 vmfunction_import_body_element: 0,
1211 vmfunction_import_vmctx_element: 1,
1212 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1213 vmmemory_definition_base_element: 0,
1214 vmmemory_definition_current_length_element: 1,
1215
1216 ptr_ty,
1218 };
1219
1220 let noreturn =
1221 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1222 intrinsics
1223 .throw_trap
1224 .add_attribute(AttributeLoc::Function, noreturn);
1225 intrinsics
1226 }
1227}
1228
1229#[derive(Clone, Copy)]
1230pub enum MemoryCache<'ctx> {
1231 Dynamic {
1233 ptr_to_base_ptr: PointerValue<'ctx>,
1234 ptr_to_current_length: PointerValue<'ctx>,
1235 },
1236 Static { base_ptr: PointerValue<'ctx> },
1238}
1239
1240#[derive(Clone)]
1241struct TableCache<'ctx> {
1242 ptr_to_base_ptr: PointerValue<'ctx>,
1243 ptr_to_bounds: PointerValue<'ctx>,
1244}
1245
1246#[derive(Clone, Copy)]
1247pub enum GlobalCache<'ctx> {
1248 Mut {
1249 ptr_to_value: PointerValue<'ctx>,
1250 value_type: BasicTypeEnum<'ctx>,
1251 },
1252 Const {
1253 value: BasicValueEnum<'ctx>,
1254 },
1255}
1256
1257#[derive(Clone)]
1258pub struct FunctionCache<'ctx> {
1259 pub func: PointerValue<'ctx>,
1260 pub llvm_func_type: FunctionType<'ctx>,
1261 pub vmctx: BasicValueEnum<'ctx>,
1262 pub attrs: Vec<(Attribute, AttributeLoc)>,
1263}
1264
1265pub struct CtxType<'ctx, 'a> {
1266 ctx_ptr_value: PointerValue<'ctx>,
1267
1268 config: &'a LLVM,
1269 wasm_module: &'a WasmerCompilerModule,
1270 cache_builder: &'a Builder<'ctx>,
1271 abi: &'a dyn Abi,
1272
1273 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1274 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1275 cached_sigindices: HashMap<SignatureIndex, IntValue<'ctx>>,
1276 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1277 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1278 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1279
1280 offsets: VMOffsets,
1281}
1282
1283impl<'ctx, 'a> CtxType<'ctx, 'a> {
1284 pub fn new(
1285 wasm_module: &'a WasmerCompilerModule,
1286 func_value: &FunctionValue<'ctx>,
1287 cache_builder: &'a Builder<'ctx>,
1288 abi: &'a dyn Abi,
1289 config: &'a LLVM,
1290 ) -> CtxType<'ctx, 'a> {
1291 CtxType {
1292 config,
1293 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1294
1295 wasm_module,
1296 cache_builder,
1297 abi,
1298
1299 cached_memories: HashMap::new(),
1300 cached_tables: HashMap::new(),
1301 cached_sigindices: HashMap::new(),
1302 cached_globals: HashMap::new(),
1303 cached_functions: HashMap::new(),
1304 cached_memory_op: HashMap::new(),
1305
1306 offsets: VMOffsets::new(8, wasm_module),
1308 }
1309 }
1310
1311 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1312 self.ctx_ptr_value.as_basic_value_enum()
1313 }
1314
1315 pub fn memory(
1316 &mut self,
1317 index: MemoryIndex,
1318 intrinsics: &Intrinsics<'ctx>,
1319 module: &Module<'ctx>,
1320 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1321 ) -> Result<MemoryCache<'ctx>, CompileError> {
1322 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1323 &mut self.cached_memories,
1324 self.wasm_module,
1325 self.ctx_ptr_value,
1326 &self.cache_builder,
1327 &self.offsets,
1328 );
1329 let memory_style = &memory_styles[index];
1330 match cached_memories.get(&index) {
1331 Some(r) => Ok(*r),
1332 None => {
1333 let memory_definition_ptr =
1334 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1335 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1336 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1337 unsafe {
1338 err!(cache_builder.build_gep(
1339 intrinsics.i8_ty,
1340 ctx_ptr_value,
1341 &[offset],
1342 ""
1343 ))
1344 }
1345 } else {
1346 let offset = offsets.vmctx_vmmemory_import(index);
1347 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1348 let memory_definition_ptr_ptr = unsafe {
1349 err!(cache_builder.build_gep(
1350 intrinsics.i8_ty,
1351 ctx_ptr_value,
1352 &[offset],
1353 ""
1354 ))
1355 };
1356 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1357 memory_definition_ptr_ptr,
1358 intrinsics.ptr_ty,
1359 "",
1360 ))
1361 .into_pointer_value();
1362 let memory_definition_ptr = err!(cache_builder.build_load(
1363 intrinsics.ptr_ty,
1364 memory_definition_ptr_ptr,
1365 ""
1366 ))
1367 .into_pointer_value();
1368 tbaa_label(
1369 module,
1370 intrinsics,
1371 format!("memory {} definition", index.as_u32()),
1372 memory_definition_ptr.as_instruction_value().unwrap(),
1373 );
1374 memory_definition_ptr
1375 };
1376 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1377 memory_definition_ptr,
1378 intrinsics.ptr_ty,
1379 "",
1380 ))
1381 .into_pointer_value();
1382 let base_ptr = err!(cache_builder.build_struct_gep(
1383 intrinsics.vmmemory_definition_ty,
1384 memory_definition_ptr,
1385 intrinsics.vmmemory_definition_base_element,
1386 "",
1387 ));
1388 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1389 let current_length_ptr = err!(cache_builder.build_struct_gep(
1390 intrinsics.vmmemory_definition_ty,
1391 memory_definition_ptr,
1392 intrinsics.vmmemory_definition_current_length_element,
1393 "",
1394 ));
1395 MemoryCache::Dynamic {
1396 ptr_to_base_ptr: base_ptr,
1397 ptr_to_current_length: current_length_ptr,
1398 }
1399 } else {
1400 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1401 .into_pointer_value();
1402 tbaa_label(
1403 module,
1404 intrinsics,
1405 format!("memory base_ptr {}", index.as_u32()),
1406 base_ptr.as_instruction_value().unwrap(),
1407 );
1408 MemoryCache::Static { base_ptr }
1409 };
1410
1411 self.cached_memories.insert(index, value);
1412 Ok(*self.cached_memories.get(&index).unwrap())
1413 }
1414 }
1415 }
1416
1417 fn build_table_prepare(
1418 table_index: TableIndex,
1419 intrinsics: &Intrinsics<'ctx>,
1420 module: &Module<'ctx>,
1421 wasm_module: &WasmerCompilerModule,
1422 ctx_ptr_value: PointerValue<'ctx>,
1423 offsets: &VMOffsets,
1424 builder: &Builder<'ctx>,
1425 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1426 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1427 let offset = intrinsics.i64_ty.const_int(
1428 offsets
1429 .vmctx_vmtable_definition_base(local_table_index)
1430 .into(),
1431 false,
1432 );
1433 let ptr_to_base_ptr =
1434 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1435 let ptr_to_base_ptr =
1436 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1437 .into_pointer_value();
1438 let offset = intrinsics.i64_ty.const_int(
1439 offsets
1440 .vmctx_vmtable_definition_current_elements(local_table_index)
1441 .into(),
1442 false,
1443 );
1444 let ptr_to_bounds =
1445 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1446 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1447 .into_pointer_value();
1448 Ok((ptr_to_base_ptr, ptr_to_bounds))
1449 } else {
1450 let offset = intrinsics.i64_ty.const_int(
1451 offsets.vmctx_vmtable_import_definition(table_index).into(),
1452 false,
1453 );
1454 let definition_ptr_ptr =
1455 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1456 let definition_ptr_ptr =
1457 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1458 .into_pointer_value();
1459 let definition_ptr =
1460 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1461 .into_pointer_value();
1462 tbaa_label(
1463 module,
1464 intrinsics,
1465 format!("table {} definition", table_index.as_u32()),
1466 definition_ptr.as_instruction_value().unwrap(),
1467 );
1468
1469 let offset = intrinsics
1470 .i64_ty
1471 .const_int(offsets.vmtable_definition_base().into(), false);
1472 let ptr_to_base_ptr =
1473 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1474 let ptr_to_base_ptr =
1475 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1476 .into_pointer_value();
1477 let offset = intrinsics
1478 .i64_ty
1479 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1480 let ptr_to_bounds =
1481 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1482 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1483 .into_pointer_value();
1484 Ok((ptr_to_base_ptr, ptr_to_bounds))
1485 }
1486 }
1487
1488 fn table_prepare(
1489 &mut self,
1490 table_index: TableIndex,
1491 intrinsics: &Intrinsics<'ctx>,
1492 module: &Module<'ctx>,
1493 body_builder: &Builder<'ctx>,
1494 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1495 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1496 &mut self.cached_tables,
1497 self.wasm_module,
1498 self.ctx_ptr_value,
1499 &self.cache_builder,
1500 &self.offsets,
1501 );
1502
1503 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1504 CompileError::Codegen(format!(
1505 "Table index out of bounds: {}",
1506 table_index.as_u32()
1507 ))
1508 })?;
1509
1510 if is_growable {
1513 Self::build_table_prepare(
1514 table_index,
1515 intrinsics,
1516 module,
1517 wasm_module,
1518 ctx_ptr_value,
1519 offsets,
1520 body_builder,
1521 )
1522 } else {
1523 let TableCache {
1524 ptr_to_base_ptr,
1525 ptr_to_bounds,
1526 } = match cached_tables.entry(table_index) {
1527 Entry::Occupied(entry) => entry.get().clone(),
1528 Entry::Vacant(entry) => {
1529 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1530 table_index,
1531 intrinsics,
1532 module,
1533 wasm_module,
1534 ctx_ptr_value,
1535 offsets,
1536 cache_builder,
1537 )?;
1538
1539 let v = TableCache {
1540 ptr_to_base_ptr,
1541 ptr_to_bounds,
1542 };
1543
1544 entry.insert(v.clone());
1545
1546 v
1547 }
1548 };
1549
1550 Ok((ptr_to_base_ptr, ptr_to_bounds))
1551 }
1552 }
1553
1554 pub fn table(
1555 &mut self,
1556 index: TableIndex,
1557 intrinsics: &Intrinsics<'ctx>,
1558 module: &Module<'ctx>,
1559 body_builder: &Builder<'ctx>,
1560 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1561 let (ptr_to_base_ptr, ptr_to_bounds) =
1562 self.table_prepare(index, intrinsics, module, body_builder)?;
1563
1564 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1566 &body_builder
1567 } else {
1568 &self.cache_builder
1569 };
1570
1571 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1572 .into_pointer_value();
1573 let bounds =
1574 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1575 tbaa_label(
1576 module,
1577 intrinsics,
1578 format!("table_base_ptr {}", index.index()),
1579 base_ptr.as_instruction_value().unwrap(),
1580 );
1581 tbaa_label(
1582 module,
1583 intrinsics,
1584 format!("table_bounds {}", index.index()),
1585 bounds.as_instruction_value().unwrap(),
1586 );
1587 Ok((base_ptr, bounds))
1588 }
1589
1590 pub fn dynamic_sigindex(
1591 &mut self,
1592 index: SignatureIndex,
1593 intrinsics: &Intrinsics<'ctx>,
1594 module: &Module<'ctx>,
1595 ) -> Result<IntValue<'ctx>, CompileError> {
1596 let (cached_sigindices, ctx_ptr_value, cache_builder, offsets) = (
1597 &mut self.cached_sigindices,
1598 self.ctx_ptr_value,
1599 &self.cache_builder,
1600 &self.offsets,
1601 );
1602
1603 match cached_sigindices.entry(index) {
1604 Entry::Occupied(entry) => Ok(*entry.get()),
1605 Entry::Vacant(entry) => {
1606 let byte_offset = intrinsics
1607 .i64_ty
1608 .const_int(offsets.vmctx_vmshared_signature_id(index).into(), false);
1609
1610 let sigindex_ptr = unsafe {
1611 err!(cache_builder.build_gep(
1612 intrinsics.i8_ty,
1613 ctx_ptr_value,
1614 &[byte_offset],
1615 "dynamic_sigindex",
1616 ))
1617 };
1618
1619 let sigindex_ptr =
1620 err!(cache_builder.build_bit_cast(sigindex_ptr, intrinsics.ptr_ty, ""))
1621 .into_pointer_value();
1622
1623 let sigindex =
1624 err!(cache_builder.build_load(intrinsics.i32_ty, sigindex_ptr, "sigindex"))
1625 .into_int_value();
1626 tbaa_label(
1627 module,
1628 intrinsics,
1629 format!("sigindex {}", index.as_u32()),
1630 sigindex.as_instruction_value().unwrap(),
1631 );
1632
1633 entry.insert(sigindex);
1634 Ok(sigindex)
1635 }
1636 }
1637 }
1638
1639 pub fn global(
1640 &mut self,
1641 index: GlobalIndex,
1642 intrinsics: &Intrinsics<'ctx>,
1643 module: &Module<'ctx>,
1644 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1645 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1646 &mut self.cached_globals,
1647 self.wasm_module,
1648 self.ctx_ptr_value,
1649 &self.cache_builder,
1650 &self.offsets,
1651 );
1652 match cached_globals.entry(index) {
1653 Entry::Occupied(entry) => Ok(entry.into_mut()),
1654 Entry::Vacant(entry) => {
1655 let global_type = wasm_module.globals[index];
1656 let global_value_type = global_type.ty;
1657
1658 let global_mutability = global_type.mutability;
1659 let offset = if let Some(local_global_index) = wasm_module.local_global_index(index)
1660 {
1661 offsets.vmctx_vmglobal_definition(local_global_index)
1662 } else {
1663 offsets.vmctx_vmglobal_import(index)
1664 };
1665 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1666 let global_ptr = {
1667 let global_ptr_ptr = unsafe {
1668 err!(cache_builder.build_gep(
1669 intrinsics.i8_ty,
1670 ctx_ptr_value,
1671 &[offset],
1672 ""
1673 ))
1674 };
1675 let global_ptr_ptr =
1676 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1677 .into_pointer_value();
1678 let global_ptr =
1679 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1680 .into_pointer_value();
1681 tbaa_label(
1682 module,
1683 intrinsics,
1684 format!("global_ptr {}", index.as_u32()),
1685 global_ptr.as_instruction_value().unwrap(),
1686 );
1687 global_ptr
1688 };
1689 let global_ptr = err!(cache_builder.build_bit_cast(
1690 global_ptr,
1691 type_to_llvm_ptr(intrinsics, global_value_type)?,
1692 "",
1693 ))
1694 .into_pointer_value();
1695
1696 let ret = entry.insert(match global_mutability {
1697 Mutability::Const => {
1698 let value = err!(cache_builder.build_load(
1699 type_to_llvm(intrinsics, global_value_type)?,
1700 global_ptr,
1701 "",
1702 ));
1703 tbaa_label(
1704 module,
1705 intrinsics,
1706 format!("global {}", index.as_u32()),
1707 value.as_instruction_value().unwrap(),
1708 );
1709 GlobalCache::Const { value }
1710 }
1711 Mutability::Var => GlobalCache::Mut {
1712 ptr_to_value: global_ptr,
1713 value_type: type_to_llvm(intrinsics, global_value_type)?,
1714 },
1715 });
1716
1717 Ok(ret)
1718 }
1719 }
1720 }
1721
1722 pub fn add_func(
1723 &mut self,
1724 function_index: FunctionIndex,
1725 func: PointerValue<'ctx>,
1726 llvm_func_type: FunctionType<'ctx>,
1727 vmctx: BasicValueEnum<'ctx>,
1728 attrs: &[(Attribute, AttributeLoc)],
1729 ) {
1730 match self.cached_functions.entry(function_index) {
1731 Entry::Occupied(_) => unreachable!("duplicate function"),
1732 Entry::Vacant(entry) => {
1733 entry.insert(FunctionCache {
1734 func,
1735 llvm_func_type,
1736 vmctx,
1737 attrs: attrs.to_vec(),
1738 });
1739 }
1740 }
1741 }
1742
1743 #[allow(clippy::too_many_arguments)]
1744 pub fn local_func(
1745 &mut self,
1746 _local_function_index: LocalFunctionIndex,
1747 function_index: FunctionIndex,
1748 intrinsics: &Intrinsics<'ctx>,
1749 module: &Module<'ctx>,
1750 context: &'ctx Context,
1751 func_type: &FuncType,
1752 function_name: &str,
1753 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1754 let (cached_functions, ctx_ptr_value, offsets) = (
1755 &mut self.cached_functions,
1756 &self.ctx_ptr_value,
1757 &self.offsets,
1758 );
1759 Ok(match cached_functions.entry(function_index) {
1760 Entry::Occupied(entry) => entry.into_mut(),
1761 Entry::Vacant(entry) => {
1762 debug_assert!(module.get_function(function_name).is_none());
1763 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1764 context,
1765 intrinsics,
1766 Some(offsets),
1767 func_type,
1768 if self.config.enable_g0m0_opt {
1769 Some(crate::abi::G0M0FunctionKind::Local)
1770 } else {
1771 None
1772 },
1773 )?;
1774 let func =
1775 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1776 for (attr, attr_loc) in &llvm_func_attrs {
1777 func.add_attribute(*attr_loc, *attr);
1778 }
1779 entry.insert(FunctionCache {
1780 func: func.as_global_value().as_pointer_value(),
1781 llvm_func_type,
1782 vmctx: ctx_ptr_value.as_basic_value_enum(),
1783 attrs: llvm_func_attrs,
1784 })
1785 }
1786 })
1787 }
1788
1789 pub fn func(
1790 &mut self,
1791 function_index: FunctionIndex,
1792 intrinsics: &Intrinsics<'ctx>,
1793 context: &'ctx Context,
1794 func_type: &FuncType,
1795 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1796 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1797 &mut self.cached_functions,
1798 self.wasm_module,
1799 &self.ctx_ptr_value,
1800 &self.cache_builder,
1801 &self.offsets,
1802 );
1803 match cached_functions.entry(function_index) {
1804 Entry::Occupied(entry) => Ok(entry.into_mut()),
1805 Entry::Vacant(entry) => {
1806 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1807 context,
1808 intrinsics,
1809 Some(offsets),
1810 func_type,
1811 None,
1812 )?;
1813 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1814 let offset = offsets.vmctx_vmfunction_import(function_index);
1815 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1816 let vmfunction_import_ptr = unsafe {
1817 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1818 };
1819 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1820 vmfunction_import_ptr,
1821 intrinsics.ptr_ty,
1822 "",
1823 ))
1824 .into_pointer_value();
1825
1826 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1827 intrinsics.vmfunction_import_ty,
1828 vmfunction_import_ptr,
1829 intrinsics.vmfunction_import_body_element,
1830 "",
1831 ));
1832 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
1833 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
1834 .into_pointer_value();
1835 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
1836 intrinsics.vmfunction_import_ty,
1837 vmfunction_import_ptr,
1838 intrinsics.vmfunction_import_vmctx_element,
1839 "",
1840 ));
1841 let vmctx_ptr =
1842 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
1843
1844 Ok(entry.insert(FunctionCache {
1845 func: body_ptr,
1846 llvm_func_type,
1847 vmctx: vmctx_ptr,
1848 attrs: llvm_func_attrs,
1849 }))
1850 }
1851 }
1852 }
1853
1854 pub fn memory_grow(
1855 &mut self,
1856 memory_index: MemoryIndex,
1857 intrinsics: &Intrinsics<'ctx>,
1858 ) -> Result<PointerValue<'ctx>, CompileError> {
1859 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1860 &mut self.cached_memory_op,
1861 &self.wasm_module,
1862 &self.offsets,
1863 &self.cache_builder,
1864 &self.ctx_ptr_value,
1865 );
1866 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
1867 Entry::Occupied(entry) => Ok(*entry.get()),
1868 Entry::Vacant(entry) => {
1869 let (grow_fn, grow_fn_ty) =
1870 if wasm_module.local_memory_index(memory_index).is_some() {
1871 (
1872 VMBuiltinFunctionIndex::get_memory32_grow_index(),
1873 intrinsics.ptr_ty,
1874 )
1875 } else {
1876 (
1877 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
1878 intrinsics.ptr_ty,
1879 )
1880 };
1881 let offset = offsets.vmctx_builtin_function(grow_fn);
1882 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1883 let grow_fn_ptr_ptr = unsafe {
1884 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1885 };
1886
1887 let grow_fn_ptr_ptr =
1888 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
1889 .into_pointer_value();
1890 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
1891 .into_pointer_value();
1892
1893 entry.insert(val);
1894 Ok(val)
1895 }
1896 }
1897 }
1898
1899 pub fn memory_size(
1900 &mut self,
1901 memory_index: MemoryIndex,
1902 intrinsics: &Intrinsics<'ctx>,
1903 ) -> Result<PointerValue<'ctx>, CompileError> {
1904 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1905 &mut self.cached_memory_op,
1906 &self.wasm_module,
1907 &self.offsets,
1908 &self.cache_builder,
1909 &self.ctx_ptr_value,
1910 );
1911
1912 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
1913 Entry::Occupied(entry) => Ok(*entry.get()),
1914 Entry::Vacant(entry) => {
1915 let (size_fn, size_fn_ty) =
1916 if wasm_module.local_memory_index(memory_index).is_some() {
1917 (
1918 VMBuiltinFunctionIndex::get_memory32_size_index(),
1919 intrinsics.ptr_ty,
1920 )
1921 } else {
1922 (
1923 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
1924 intrinsics.ptr_ty,
1925 )
1926 };
1927 let offset = offsets.vmctx_builtin_function(size_fn);
1928 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1929 let size_fn_ptr_ptr = unsafe {
1930 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1931 };
1932
1933 let size_fn_ptr_ptr =
1934 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
1935 .into_pointer_value();
1936
1937 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
1938 .into_pointer_value();
1939 entry.insert(val);
1940 Ok(val)
1941 }
1942 }
1943 }
1944
1945 pub fn memory_wait32(
1946 &mut self,
1947 memory_index: MemoryIndex,
1948 intrinsics: &Intrinsics<'ctx>,
1949 ) -> Result<PointerValue<'ctx>, CompileError> {
1950 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1951 &mut self.cached_memory_op,
1952 &self.wasm_module,
1953 &self.offsets,
1954 &self.cache_builder,
1955 &self.ctx_ptr_value,
1956 );
1957 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
1958 Entry::Occupied(entry) => Ok(*entry.get()),
1959 Entry::Vacant(entry) => {
1960 let (size_fn, size_fn_ty) =
1961 if wasm_module.local_memory_index(memory_index).is_some() {
1962 (
1963 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
1964 intrinsics.ptr_ty,
1965 )
1966 } else {
1967 (
1968 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
1969 intrinsics.ptr_ty,
1970 )
1971 };
1972 let offset = offsets.vmctx_builtin_function(size_fn);
1973 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1974 let size_fn_ptr_ptr = unsafe {
1975 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1976 };
1977
1978 let size_fn_ptr_ptr =
1979 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
1980 .into_pointer_value();
1981
1982 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
1983 .into_pointer_value();
1984
1985 entry.insert(val);
1986 Ok(val)
1987 }
1988 }
1989 }
1990
1991 pub fn memory_wait64(
1992 &mut self,
1993 memory_index: MemoryIndex,
1994 intrinsics: &Intrinsics<'ctx>,
1995 ) -> Result<PointerValue<'ctx>, CompileError> {
1996 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1997 &mut self.cached_memory_op,
1998 &self.wasm_module,
1999 &self.offsets,
2000 &self.cache_builder,
2001 &self.ctx_ptr_value,
2002 );
2003
2004 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
2005 Entry::Occupied(entry) => Ok(*entry.get()),
2006 Entry::Vacant(entry) => {
2007 let (size_fn, size_fn_ty) =
2008 if wasm_module.local_memory_index(memory_index).is_some() {
2009 (
2010 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
2011 intrinsics.ptr_ty,
2012 )
2013 } else {
2014 (
2015 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
2016 intrinsics.ptr_ty,
2017 )
2018 };
2019 let offset = offsets.vmctx_builtin_function(size_fn);
2020 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2021 let size_fn_ptr_ptr = unsafe {
2022 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2023 };
2024
2025 let size_fn_ptr_ptr =
2026 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2027 .into_pointer_value();
2028
2029 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2030 .into_pointer_value();
2031 entry.insert(val);
2032 Ok(val)
2033 }
2034 }
2035 }
2036
2037 pub fn memory_notify(
2038 &mut self,
2039 memory_index: MemoryIndex,
2040 intrinsics: &Intrinsics<'ctx>,
2041 ) -> Result<PointerValue<'ctx>, CompileError> {
2042 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2043 &mut self.cached_memory_op,
2044 &self.wasm_module,
2045 &self.offsets,
2046 &self.cache_builder,
2047 &self.ctx_ptr_value,
2048 );
2049 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2050 Entry::Occupied(entry) => Ok(*entry.get()),
2051 Entry::Vacant(entry) => {
2052 let (size_fn, size_fn_ty) =
2053 if wasm_module.local_memory_index(memory_index).is_some() {
2054 (
2055 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2056 intrinsics.ptr_ty,
2057 )
2058 } else {
2059 (
2060 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2061 intrinsics.ptr_ty,
2062 )
2063 };
2064 let offset = offsets.vmctx_builtin_function(size_fn);
2065 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2066 let size_fn_ptr_ptr = unsafe {
2067 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2068 };
2069
2070 let size_fn_ptr_ptr =
2071 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2072 .into_pointer_value();
2073
2074 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2075 .into_pointer_value();
2076
2077 entry.insert(val);
2078 Ok(val)
2079 }
2080 }
2081 }
2082
2083 pub fn get_offsets(&self) -> &VMOffsets {
2084 &self.offsets
2085 }
2086}
2087
2088pub fn tbaa_label<'ctx>(
2091 module: &Module<'ctx>,
2092 intrinsics: &Intrinsics<'ctx>,
2093 label: String,
2094 instruction: InstructionValue<'ctx>,
2095) {
2096 let context = module.get_context();
2110
2111 let tbaa_root = module
2113 .get_global_metadata("wasmer_tbaa_root")
2114 .pop()
2115 .unwrap_or_else(|| {
2116 module
2117 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2118 .unwrap();
2119 module.get_global_metadata("wasmer_tbaa_root")[0]
2120 });
2121
2122 let type_label = context.metadata_string(label.as_str());
2125 let type_tbaa = module
2126 .get_global_metadata(label.as_str())
2127 .pop()
2128 .unwrap_or_else(|| {
2129 module
2130 .add_global_metadata(
2131 label.as_str(),
2132 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2133 )
2134 .unwrap();
2135 module.get_global_metadata(label.as_str())[0]
2136 });
2137
2138 let label = label + "_memop";
2145 let type_tbaa = module
2146 .get_global_metadata(label.as_str())
2147 .pop()
2148 .unwrap_or_else(|| {
2149 module
2150 .add_global_metadata(
2151 label.as_str(),
2152 &context.metadata_node(&[
2153 type_tbaa.into(),
2154 type_tbaa.into(),
2155 intrinsics.i64_zero.into(),
2156 ]),
2157 )
2158 .unwrap();
2159 module.get_global_metadata(label.as_str())[0]
2160 });
2161
2162 let tbaa_kind = context.get_kind_id("tbaa");
2164 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2165}
2166
2167fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2168 let table = module.tables.get(index)?;
2169 match table.maximum {
2170 None => Some(true),
2171 Some(max) => Some(max > table.minimum),
2172 }
2173}