1use crate::LLVM;
8use crate::abi::Abi;
9use crate::error::err;
10use inkwell::values::BasicMetadataValueEnum;
11use inkwell::{
12 AddressSpace,
13 attributes::{Attribute, AttributeLoc},
14 builder::Builder,
15 context::Context,
16 module::{Linkage, Module},
17 targets::TargetData,
18 types::{
19 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
20 PointerType, StructType, VectorType, VoidType,
21 },
22 values::{
23 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
24 PointerValue, VectorValue,
25 },
26};
27use std::collections::{HashMap, hash_map::Entry};
28use wasmer_types::entity::{EntityRef, PrimaryMap};
29use wasmer_types::{
30 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
31 MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, SignatureIndex, TableIndex, Type,
32};
33use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
34
35pub fn type_to_llvm_ptr<'ctx>(
36 intrinsics: &Intrinsics<'ctx>,
37 _ty: Type,
38) -> Result<PointerType<'ctx>, CompileError> {
39 Ok(intrinsics.ptr_ty)
41}
42
43pub fn type_to_llvm<'ctx>(
44 intrinsics: &Intrinsics<'ctx>,
45 ty: Type,
46) -> Result<BasicTypeEnum<'ctx>, CompileError> {
47 match ty {
48 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
49 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
50 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
51 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
52 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
53 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
54 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
55 }
56}
57
58#[allow(dead_code)]
60pub struct Intrinsics<'ctx> {
61 pub ctlz_i32: FunctionValue<'ctx>,
62 pub ctlz_i64: FunctionValue<'ctx>,
63
64 pub cttz_i32: FunctionValue<'ctx>,
65 pub cttz_i64: FunctionValue<'ctx>,
66
67 pub ctpop_i32: FunctionValue<'ctx>,
68 pub ctpop_i64: FunctionValue<'ctx>,
69 pub ctpop_i8x16: FunctionValue<'ctx>,
70
71 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
72 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
73 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
74 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
75 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
76
77 pub add_f32: FunctionValue<'ctx>,
78 pub add_f64: FunctionValue<'ctx>,
79 pub add_f32x4: FunctionValue<'ctx>,
80 pub add_f64x2: FunctionValue<'ctx>,
81
82 pub sub_f32: FunctionValue<'ctx>,
83 pub sub_f64: FunctionValue<'ctx>,
84 pub sub_f32x4: FunctionValue<'ctx>,
85 pub sub_f64x2: FunctionValue<'ctx>,
86
87 pub mul_f32: FunctionValue<'ctx>,
88 pub mul_f64: FunctionValue<'ctx>,
89 pub mul_f32x4: FunctionValue<'ctx>,
90 pub mul_f64x2: FunctionValue<'ctx>,
91
92 pub div_f32: FunctionValue<'ctx>,
93 pub div_f64: FunctionValue<'ctx>,
94 pub div_f32x4: FunctionValue<'ctx>,
95 pub div_f64x2: FunctionValue<'ctx>,
96
97 pub sqrt_f32: FunctionValue<'ctx>,
98 pub sqrt_f64: FunctionValue<'ctx>,
99 pub sqrt_f32x4: FunctionValue<'ctx>,
100 pub sqrt_f64x2: FunctionValue<'ctx>,
101
102 pub cmp_f32: FunctionValue<'ctx>,
103 pub cmp_f64: FunctionValue<'ctx>,
104 pub cmp_f32x4: FunctionValue<'ctx>,
105 pub cmp_f64x2: FunctionValue<'ctx>,
106
107 pub ceil_f32: FunctionValue<'ctx>,
108 pub ceil_f64: FunctionValue<'ctx>,
109 pub ceil_f32x4: FunctionValue<'ctx>,
110 pub ceil_f64x2: FunctionValue<'ctx>,
111
112 pub floor_f32: FunctionValue<'ctx>,
113 pub floor_f64: FunctionValue<'ctx>,
114 pub floor_f32x4: FunctionValue<'ctx>,
115 pub floor_f64x2: FunctionValue<'ctx>,
116
117 pub trunc_f32: FunctionValue<'ctx>,
118 pub trunc_f64: FunctionValue<'ctx>,
119 pub trunc_f32x4: FunctionValue<'ctx>,
120 pub trunc_f64x2: FunctionValue<'ctx>,
121
122 pub fpext_f32: FunctionValue<'ctx>,
123 pub fptrunc_f64: FunctionValue<'ctx>,
124
125 pub nearbyint_f32: FunctionValue<'ctx>,
126 pub nearbyint_f64: FunctionValue<'ctx>,
127 pub nearbyint_f32x4: FunctionValue<'ctx>,
128 pub nearbyint_f64x2: FunctionValue<'ctx>,
129
130 pub fabs_f32: FunctionValue<'ctx>,
131 pub fabs_f64: FunctionValue<'ctx>,
132 pub fabs_f32x4: FunctionValue<'ctx>,
133 pub fabs_f64x2: FunctionValue<'ctx>,
134
135 pub copysign_f32: FunctionValue<'ctx>,
136 pub copysign_f64: FunctionValue<'ctx>,
137 pub copysign_f32x4: FunctionValue<'ctx>,
138 pub copysign_f64x2: FunctionValue<'ctx>,
139
140 pub sadd_sat_i8x16: FunctionValue<'ctx>,
141 pub sadd_sat_i16x8: FunctionValue<'ctx>,
142 pub uadd_sat_i8x16: FunctionValue<'ctx>,
143 pub uadd_sat_i16x8: FunctionValue<'ctx>,
144
145 pub ssub_sat_i8x16: FunctionValue<'ctx>,
146 pub ssub_sat_i16x8: FunctionValue<'ctx>,
147 pub usub_sat_i8x16: FunctionValue<'ctx>,
148 pub usub_sat_i16x8: FunctionValue<'ctx>,
149
150 pub expect_i1: FunctionValue<'ctx>,
151 pub trap: FunctionValue<'ctx>,
152 pub debug_trap: FunctionValue<'ctx>,
153
154 pub personality: FunctionValue<'ctx>,
155 pub personality2: FunctionValue<'ctx>,
156 pub readonly: Attribute,
157 pub stack_probe: Attribute,
158 pub uwtable: Attribute,
159 pub frame_pointer: Attribute,
160
161 pub void_ty: VoidType<'ctx>,
162 pub i1_ty: IntType<'ctx>,
163 pub i2_ty: IntType<'ctx>,
164 pub i4_ty: IntType<'ctx>,
165 pub i8_ty: IntType<'ctx>,
166 pub i16_ty: IntType<'ctx>,
167 pub i32_ty: IntType<'ctx>,
168 pub i64_ty: IntType<'ctx>,
169 pub i128_ty: IntType<'ctx>,
170 pub isize_ty: IntType<'ctx>,
171 pub f32_ty: FloatType<'ctx>,
172 pub f64_ty: FloatType<'ctx>,
173
174 pub i1x128_ty: VectorType<'ctx>,
175 pub i8x16_ty: VectorType<'ctx>,
176 pub i16x8_ty: VectorType<'ctx>,
177 pub i32x4_ty: VectorType<'ctx>,
178 pub i64x2_ty: VectorType<'ctx>,
179 pub f32x4_ty: VectorType<'ctx>,
180 pub f64x2_ty: VectorType<'ctx>,
181 pub i32x8_ty: VectorType<'ctx>,
182
183 pub ptr_ty: PointerType<'ctx>,
184
185 pub anyfunc_ty: StructType<'ctx>,
186
187 pub i1_zero: IntValue<'ctx>,
188 pub i8_zero: IntValue<'ctx>,
189 pub i32_zero: IntValue<'ctx>,
190 pub i64_zero: IntValue<'ctx>,
191 pub i128_zero: IntValue<'ctx>,
192 pub isize_zero: IntValue<'ctx>,
193 pub f32_zero: FloatValue<'ctx>,
194 pub f64_zero: FloatValue<'ctx>,
195 pub f32x4_zero: VectorValue<'ctx>,
196 pub f64x2_zero: VectorValue<'ctx>,
197 pub i32_consts: [IntValue<'ctx>; 16],
198
199 pub trap_unreachable: BasicValueEnum<'ctx>,
200 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
201 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
202 pub trap_memory_oob: BasicValueEnum<'ctx>,
203 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
204 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
205 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
206 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
207 pub trap_table_access_oob: BasicValueEnum<'ctx>,
208
209 pub experimental_stackmap: FunctionValue<'ctx>,
210
211 pub table_copy: FunctionValue<'ctx>,
213 pub table_init: FunctionValue<'ctx>,
214 pub table_fill: FunctionValue<'ctx>,
215 pub table_size: FunctionValue<'ctx>,
216 pub imported_table_size: FunctionValue<'ctx>,
217 pub table_get: FunctionValue<'ctx>,
218 pub imported_table_get: FunctionValue<'ctx>,
219 pub table_set: FunctionValue<'ctx>,
220 pub imported_table_set: FunctionValue<'ctx>,
221 pub table_grow: FunctionValue<'ctx>,
222 pub imported_table_grow: FunctionValue<'ctx>,
223 pub memory_init: FunctionValue<'ctx>,
224 pub data_drop: FunctionValue<'ctx>,
225 pub func_ref: FunctionValue<'ctx>,
226 pub elem_drop: FunctionValue<'ctx>,
227 pub memory_copy: FunctionValue<'ctx>,
228 pub imported_memory_copy: FunctionValue<'ctx>,
229 pub memory_fill: FunctionValue<'ctx>,
230 pub imported_memory_fill: FunctionValue<'ctx>,
231 pub memory_size_ty: FunctionType<'ctx>,
232 pub memory_grow_ty: FunctionType<'ctx>,
233 pub memory_wait32: FunctionValue<'ctx>,
234 pub memory_wait32_ty: FunctionType<'ctx>,
235 pub imported_memory_wait32: FunctionValue<'ctx>,
236 pub memory_wait64: FunctionValue<'ctx>,
237 pub memory_wait64_ty: FunctionType<'ctx>,
238 pub imported_memory_wait64: FunctionValue<'ctx>,
239 pub memory_notify: FunctionValue<'ctx>,
240 pub memory_notify_ty: FunctionType<'ctx>,
241 pub imported_memory_notify: FunctionValue<'ctx>,
242
243 pub throw_trap: FunctionValue<'ctx>,
244
245 pub throw: FunctionValue<'ctx>,
247 pub alloc_exception: FunctionValue<'ctx>,
248 pub read_exnref: FunctionValue<'ctx>,
249 pub exception_into_exnref: FunctionValue<'ctx>,
250 pub lpad_exception_ty: StructType<'ctx>,
251
252 pub debug_ptr: FunctionValue<'ctx>,
254 pub debug_str: FunctionValue<'ctx>,
255
256 pub vmfunction_import_ty: StructType<'ctx>,
258 pub vmfunction_import_body_element: u32,
259 pub vmfunction_import_vmctx_element: u32,
260
261 pub vmmemory_definition_ty: StructType<'ctx>,
262 pub vmmemory_definition_base_element: u32,
263 pub vmmemory_definition_current_length_element: u32,
264}
265
266#[derive(Debug, Hash, PartialEq, Eq)]
267enum MemoryOp {
268 Size,
269 Grow,
270 Wait32,
271 Wait64,
272 Notify,
273}
274
275impl<'ctx> Intrinsics<'ctx> {
276 pub fn declare(
278 module: &Module<'ctx>,
279 context: &'ctx Context,
280 target_data: &TargetData,
281 binary_fmt: &target_lexicon::BinaryFormat,
282 ) -> Self {
283 let void_ty = context.void_type();
284 let i1_ty = context.bool_type();
285 let i2_ty = context.custom_width_int_type(2);
286 let i4_ty = context.custom_width_int_type(4);
287 let i8_ty = context.i8_type();
288 let i16_ty = context.i16_type();
289 let i32_ty = context.i32_type();
290 let i64_ty = context.i64_type();
291 let i128_ty = context.i128_type();
292 let isize_ty = context.ptr_sized_int_type(target_data, None);
293 let f32_ty = context.f32_type();
294 let f64_ty = context.f64_type();
295
296 let i1x4_ty = i1_ty.vec_type(4);
297 let i1x2_ty = i1_ty.vec_type(2);
298 let i1x128_ty = i1_ty.vec_type(128);
299 let i8x16_ty = i8_ty.vec_type(16);
300 let i16x8_ty = i16_ty.vec_type(8);
301 let i32x4_ty = i32_ty.vec_type(4);
302 let i64x2_ty = i64_ty.vec_type(2);
303 let f32x4_ty = f32_ty.vec_type(4);
304 let f64x2_ty = f64_ty.vec_type(2);
305 let i32x8_ty = i32_ty.vec_type(8);
306
307 let ptr_ty = context.ptr_type(AddressSpace::default());
308
309 let i1_zero = i1_ty.const_int(0, false);
310 let i8_zero = i8_ty.const_int(0, false);
311 let i32_zero = i32_ty.const_int(0, false);
312 let i64_zero = i64_ty.const_int(0, false);
313 let i128_zero = i128_ty.const_int(0, false);
314 let isize_zero = isize_ty.const_int(0, false);
315 let f32_zero = f32_ty.const_float(0.0);
316 let f64_zero = f64_ty.const_float(0.0);
317 let f32x4_zero = f32x4_ty.const_zero();
318 let f64x2_zero = f64x2_ty.const_zero();
319 let i32_consts = [
320 i32_ty.const_int(0, false),
321 i32_ty.const_int(1, false),
322 i32_ty.const_int(2, false),
323 i32_ty.const_int(3, false),
324 i32_ty.const_int(4, false),
325 i32_ty.const_int(5, false),
326 i32_ty.const_int(6, false),
327 i32_ty.const_int(7, false),
328 i32_ty.const_int(8, false),
329 i32_ty.const_int(9, false),
330 i32_ty.const_int(10, false),
331 i32_ty.const_int(11, false),
332 i32_ty.const_int(12, false),
333 i32_ty.const_int(13, false),
334 i32_ty.const_int(14, false),
335 i32_ty.const_int(15, false),
336 ];
337
338 let md_ty = context.metadata_type();
339
340 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
341
342 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
343 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
344 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
345 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
346 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
347 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
348 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
349 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
350 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
351 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
352
353 let ctx_ptr_ty = ptr_ty;
354 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
355 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
356
357 let sigindex_ty = i32_ty;
358
359 let anyfunc_ty = context.struct_type(
360 &[i8_ptr_ty_basic, sigindex_ty.into(), ctx_ptr_ty_basic],
361 false,
362 );
363 let funcref_ty = ptr_ty;
364 let anyref_ty = ptr_ty;
365 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
366
367 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
368 let ret_i8x16_take_i8x16_i8x16 =
369 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
370 let ret_i16x8_take_i16x8_i16x8 =
371 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
372
373 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
374 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
375
376 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
377 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
378
379 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
380 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
381 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
382 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
383
384 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
385 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
386 let ret_f32x4_take_f32x4_f32x4 =
387 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
388 let ret_f64x2_take_f64x2_f64x2 =
389 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
390
391 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
392 let ret_f32_take_f64_md_md =
393 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
394
395 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
396
397 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
398 &[
399 f32_ty_basic_md,
400 f32_ty_basic_md,
401 md_ty_basic_md,
402 md_ty_basic_md,
403 ],
404 false,
405 );
406 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
407 &[
408 f64_ty_basic_md,
409 f64_ty_basic_md,
410 md_ty_basic_md,
411 md_ty_basic_md,
412 ],
413 false,
414 );
415 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
416 &[
417 f32x4_ty_basic_md,
418 f32x4_ty_basic_md,
419 md_ty_basic_md,
420 md_ty_basic_md,
421 ],
422 false,
423 );
424 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
425 &[
426 f64x2_ty_basic_md,
427 f64x2_ty_basic_md,
428 md_ty_basic_md,
429 md_ty_basic_md,
430 ],
431 false,
432 );
433
434 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
435 &[
436 f32_ty_basic_md,
437 f32_ty_basic_md,
438 md_ty_basic_md,
439 md_ty_basic_md,
440 ],
441 false,
442 );
443 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
444 &[
445 f64_ty_basic_md,
446 f64_ty_basic_md,
447 md_ty_basic_md,
448 md_ty_basic_md,
449 ],
450 false,
451 );
452 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
453 &[
454 f32x4_ty_basic_md,
455 f32x4_ty_basic_md,
456 md_ty_basic_md,
457 md_ty_basic_md,
458 ],
459 false,
460 );
461 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
462 &[
463 f64x2_ty_basic_md,
464 f64x2_ty_basic_md,
465 md_ty_basic_md,
466 md_ty_basic_md,
467 ],
468 false,
469 );
470
471 let intrinsics = Self {
472 ctlz_i32: module.add_function("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
473 ctlz_i64: module.add_function("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
474
475 cttz_i32: module.add_function("llvm.cttz.i32", ret_i32_take_i32_i1, None),
476 cttz_i64: module.add_function("llvm.cttz.i64", ret_i64_take_i64_i1, None),
477
478 ctpop_i32: module.add_function("llvm.ctpop.i32", ret_i32_take_i32, None),
479 ctpop_i64: module.add_function("llvm.ctpop.i64", ret_i64_take_i64, None),
480 ctpop_i8x16: module.add_function("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
481
482 fp_rounding_md: context.metadata_string("round.tonearest").into(),
483 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
484
485 fp_ogt_md: context.metadata_string("ogt").into(),
486 fp_olt_md: context.metadata_string("olt").into(),
487 fp_uno_md: context.metadata_string("uno").into(),
488
489 sqrt_f32: module.add_function("llvm.sqrt.f32", ret_f32_take_f32, None),
490 sqrt_f64: module.add_function("llvm.sqrt.f64", ret_f64_take_f64, None),
491 sqrt_f32x4: module.add_function("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
492 sqrt_f64x2: module.add_function("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
493
494 ceil_f32: module.add_function("llvm.ceil.f32", ret_f32_take_f32, None),
495 ceil_f64: module.add_function("llvm.ceil.f64", ret_f64_take_f64, None),
496 ceil_f32x4: module.add_function("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
497 ceil_f64x2: module.add_function("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
498
499 floor_f32: module.add_function("llvm.floor.f32", ret_f32_take_f32, None),
500 floor_f64: module.add_function("llvm.floor.f64", ret_f64_take_f64, None),
501 floor_f32x4: module.add_function("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
502 floor_f64x2: module.add_function("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
503
504 trunc_f32: module.add_function("llvm.trunc.f32", ret_f32_take_f32, None),
505 trunc_f64: module.add_function("llvm.trunc.f64", ret_f64_take_f64, None),
506 trunc_f32x4: module.add_function("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
507 trunc_f64x2: module.add_function("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
508
509 nearbyint_f32: module.add_function("llvm.nearbyint.f32", ret_f32_take_f32, None),
510 nearbyint_f64: module.add_function("llvm.nearbyint.f64", ret_f64_take_f64, None),
511 nearbyint_f32x4: module.add_function(
512 "llvm.nearbyint.v4f32",
513 ret_f32x4_take_f32x4,
514 None,
515 ),
516 nearbyint_f64x2: module.add_function(
517 "llvm.nearbyint.v2f64",
518 ret_f64x2_take_f64x2,
519 None,
520 ),
521
522 add_f32: module.add_function(
523 "llvm.experimental.constrained.fadd.f32",
524 ret_f32_take_f32_f32_md_md,
525 None,
526 ),
527 add_f64: module.add_function(
528 "llvm.experimental.constrained.fadd.f64",
529 ret_f64_take_f64_f64_md_md,
530 None,
531 ),
532 add_f32x4: module.add_function(
533 "llvm.experimental.constrained.fadd.v4f32",
534 ret_f32x4_take_f32x4_f32x4_md_md,
535 None,
536 ),
537 add_f64x2: module.add_function(
538 "llvm.experimental.constrained.fadd.v2f64",
539 ret_f64x2_take_f64x2_f64x2_md_md,
540 None,
541 ),
542
543 sub_f32: module.add_function(
544 "llvm.experimental.constrained.fsub.f32",
545 ret_f32_take_f32_f32_md_md,
546 None,
547 ),
548 sub_f64: module.add_function(
549 "llvm.experimental.constrained.fsub.f64",
550 ret_f64_take_f64_f64_md_md,
551 None,
552 ),
553 sub_f32x4: module.add_function(
554 "llvm.experimental.constrained.fsub.v4f32",
555 ret_f32x4_take_f32x4_f32x4_md_md,
556 None,
557 ),
558 sub_f64x2: module.add_function(
559 "llvm.experimental.constrained.fsub.v2f64",
560 ret_f64x2_take_f64x2_f64x2_md_md,
561 None,
562 ),
563
564 mul_f32: module.add_function(
565 "llvm.experimental.constrained.fmul.f32",
566 ret_f32_take_f32_f32_md_md,
567 None,
568 ),
569 mul_f64: module.add_function(
570 "llvm.experimental.constrained.fmul.f64",
571 ret_f64_take_f64_f64_md_md,
572 None,
573 ),
574 mul_f32x4: module.add_function(
575 "llvm.experimental.constrained.fmul.v4f32",
576 ret_f32x4_take_f32x4_f32x4_md_md,
577 None,
578 ),
579 mul_f64x2: module.add_function(
580 "llvm.experimental.constrained.fmul.v2f64",
581 ret_f64x2_take_f64x2_f64x2_md_md,
582 None,
583 ),
584
585 div_f32: module.add_function(
586 "llvm.experimental.constrained.fdiv.f32",
587 ret_f32_take_f32_f32_md_md,
588 None,
589 ),
590 div_f64: module.add_function(
591 "llvm.experimental.constrained.fdiv.f64",
592 ret_f64_take_f64_f64_md_md,
593 None,
594 ),
595 div_f32x4: module.add_function(
596 "llvm.experimental.constrained.fdiv.v4f32",
597 ret_f32x4_take_f32x4_f32x4_md_md,
598 None,
599 ),
600 div_f64x2: module.add_function(
601 "llvm.experimental.constrained.fdiv.v2f64",
602 ret_f64x2_take_f64x2_f64x2_md_md,
603 None,
604 ),
605
606 cmp_f32: module.add_function(
607 "llvm.experimental.constrained.fcmp.f32",
608 ret_i1_take_f32_f32_md_md,
609 None,
610 ),
611 cmp_f64: module.add_function(
612 "llvm.experimental.constrained.fcmp.f64",
613 ret_i1_take_f64_f64_md_md,
614 None,
615 ),
616 cmp_f32x4: module.add_function(
617 "llvm.experimental.constrained.fcmp.v4f32",
618 ret_i1x4_take_f32x4_f32x4_md_md,
619 None,
620 ),
621 cmp_f64x2: module.add_function(
622 "llvm.experimental.constrained.fcmp.v2f64",
623 ret_i1x2_take_f64x2_f64x2_md_md,
624 None,
625 ),
626
627 fpext_f32: module.add_function(
628 "llvm.experimental.constrained.fpext.f64.f32",
629 ret_f64_take_f32_md,
630 None,
631 ),
632 fptrunc_f64: module.add_function(
633 "llvm.experimental.constrained.fptrunc.f32.f64",
634 ret_f32_take_f64_md_md,
635 None,
636 ),
637
638 fabs_f32: module.add_function("llvm.fabs.f32", ret_f32_take_f32, None),
639 fabs_f64: module.add_function("llvm.fabs.f64", ret_f64_take_f64, None),
640 fabs_f32x4: module.add_function("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
641 fabs_f64x2: module.add_function("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
642
643 copysign_f32: module.add_function("llvm.copysign.f32", ret_f32_take_f32_f32, None),
644 copysign_f64: module.add_function("llvm.copysign.f64", ret_f64_take_f64_f64, None),
645 copysign_f32x4: module.add_function(
646 "llvm.copysign.v4f32",
647 ret_f32x4_take_f32x4_f32x4,
648 None,
649 ),
650 copysign_f64x2: module.add_function(
651 "llvm.copysign.v2f64",
652 ret_f64x2_take_f64x2_f64x2,
653 None,
654 ),
655
656 sadd_sat_i8x16: module.add_function(
657 "llvm.sadd.sat.v16i8",
658 ret_i8x16_take_i8x16_i8x16,
659 None,
660 ),
661 sadd_sat_i16x8: module.add_function(
662 "llvm.sadd.sat.v8i16",
663 ret_i16x8_take_i16x8_i16x8,
664 None,
665 ),
666 uadd_sat_i8x16: module.add_function(
667 "llvm.uadd.sat.v16i8",
668 ret_i8x16_take_i8x16_i8x16,
669 None,
670 ),
671 uadd_sat_i16x8: module.add_function(
672 "llvm.uadd.sat.v8i16",
673 ret_i16x8_take_i16x8_i16x8,
674 None,
675 ),
676
677 ssub_sat_i8x16: module.add_function(
678 "llvm.ssub.sat.v16i8",
679 ret_i8x16_take_i8x16_i8x16,
680 None,
681 ),
682 ssub_sat_i16x8: module.add_function(
683 "llvm.ssub.sat.v8i16",
684 ret_i16x8_take_i16x8_i16x8,
685 None,
686 ),
687 usub_sat_i8x16: module.add_function(
688 "llvm.usub.sat.v16i8",
689 ret_i8x16_take_i8x16_i8x16,
690 None,
691 ),
692 usub_sat_i16x8: module.add_function(
693 "llvm.usub.sat.v8i16",
694 ret_i16x8_take_i16x8_i16x8,
695 None,
696 ),
697
698 expect_i1: module.add_function("llvm.expect.i1", ret_i1_take_i1_i1, None),
699 trap: module.add_function("llvm.trap", void_ty.fn_type(&[], false), None),
700 debug_trap: module.add_function("llvm.debugtrap", void_ty.fn_type(&[], false), None),
701 personality: module.add_function(
702 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
703 "__gxx_personality_v0"
706 } else {
707 "wasmer_eh_personality"
708 },
709 i32_ty.fn_type(
710 &[
711 i32_ty.into(),
712 i32_ty.into(),
713 i64_ty.into(),
714 ptr_ty.into(),
715 ptr_ty.into(),
716 ],
717 false,
718 ),
719 None,
720 ),
721 personality2: module.add_function(
722 "wasmer_eh_personality2",
723 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
724 None,
725 ),
726 readonly: context
727 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
728 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
729 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
730 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
731 void_ty,
732 i1_ty,
733 i2_ty,
734 i4_ty,
735 i8_ty,
736 i16_ty,
737 i32_ty,
738 i64_ty,
739 i128_ty,
740 isize_ty,
741 f32_ty,
742 f64_ty,
743
744 i1x128_ty,
745 i8x16_ty,
746 i16x8_ty,
747 i32x4_ty,
748 i64x2_ty,
749 f32x4_ty,
750 f64x2_ty,
751 i32x8_ty,
752
753 anyfunc_ty,
754 i1_zero,
755 i8_zero,
756 i32_zero,
757 i64_zero,
758 i128_zero,
759 isize_zero,
760 f32_zero,
761 f64_zero,
762 f32x4_zero,
763 f64x2_zero,
764 i32_consts,
765
766 trap_unreachable: i32_ty
767 .const_int(TrapCode::UnreachableCodeReached as _, false)
768 .as_basic_value_enum(),
769 trap_call_indirect_null: i32_ty
770 .const_int(TrapCode::IndirectCallToNull as _, false)
771 .as_basic_value_enum(),
772 trap_call_indirect_sig: i32_ty
773 .const_int(TrapCode::BadSignature as _, false)
774 .as_basic_value_enum(),
775 trap_memory_oob: i32_ty
776 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
777 .as_basic_value_enum(),
778 trap_illegal_arithmetic: i32_ty
779 .const_int(TrapCode::IntegerOverflow as _, false)
780 .as_basic_value_enum(),
781 trap_integer_division_by_zero: i32_ty
782 .const_int(TrapCode::IntegerDivisionByZero as _, false)
783 .as_basic_value_enum(),
784 trap_bad_conversion_to_integer: i32_ty
785 .const_int(TrapCode::BadConversionToInteger as _, false)
786 .as_basic_value_enum(),
787 trap_unaligned_atomic: i32_ty
788 .const_int(TrapCode::UnalignedAtomic as _, false)
789 .as_basic_value_enum(),
790 trap_table_access_oob: i32_ty
791 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
792 .as_basic_value_enum(),
793
794 experimental_stackmap: module.add_function(
795 "llvm.experimental.stackmap",
796 void_ty.fn_type(
797 &[
798 i64_ty_basic_md, i32_ty_basic_md, ],
801 true,
802 ),
803 None,
804 ),
805
806 table_copy: module.add_function(
808 "wasmer_vm_table_copy",
809 void_ty.fn_type(
810 &[
811 ctx_ptr_ty_basic_md,
812 i32_ty_basic_md,
813 i32_ty_basic_md,
814 i32_ty_basic_md,
815 i32_ty_basic_md,
816 i32_ty_basic_md,
817 ],
818 false,
819 ),
820 None,
821 ),
822 table_init: module.add_function(
823 "wasmer_vm_table_init",
824 void_ty.fn_type(
825 &[
826 ctx_ptr_ty_basic_md,
827 i32_ty_basic_md,
828 i32_ty_basic_md,
829 i32_ty_basic_md,
830 i32_ty_basic_md,
831 i32_ty_basic_md,
832 ],
833 false,
834 ),
835 None,
836 ),
837 table_fill: module.add_function(
838 "wasmer_vm_table_fill",
839 void_ty.fn_type(
840 &[
841 ctx_ptr_ty_basic_md,
842 i32_ty_basic_md,
843 i32_ty_basic_md,
844 anyref_ty_basic_md,
845 i32_ty_basic_md,
846 ],
847 false,
848 ),
849 None,
850 ),
851 table_size: module.add_function(
852 "wasmer_vm_table_size",
853 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
854 None,
855 ),
856 imported_table_size: module.add_function(
857 "wasmer_vm_imported_table_size",
858 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
859 None,
860 ),
861 table_get: module.add_function(
862 "wasmer_vm_table_get",
863 anyref_ty.fn_type(
864 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
865 false,
866 ),
867 None,
868 ),
869 imported_table_get: module.add_function(
870 "wasmer_vm_imported_table_get",
871 anyref_ty.fn_type(
872 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
873 false,
874 ),
875 None,
876 ),
877 table_set: module.add_function(
878 "wasmer_vm_table_set",
879 void_ty.fn_type(
880 &[
881 ctx_ptr_ty_basic_md,
882 i32_ty_basic_md,
883 i32_ty_basic_md,
884 anyref_ty_basic_md,
885 ],
886 false,
887 ),
888 None,
889 ),
890 imported_table_set: module.add_function(
891 "wasmer_vm_imported_table_set",
892 void_ty.fn_type(
893 &[
894 ctx_ptr_ty_basic_md,
895 i32_ty_basic_md,
896 i32_ty_basic_md,
897 anyref_ty_basic_md,
898 ],
899 false,
900 ),
901 None,
902 ),
903 table_grow: module.add_function(
904 "wasmer_vm_table_grow",
905 i32_ty.fn_type(
906 &[
907 ctx_ptr_ty_basic_md,
908 anyref_ty_basic_md,
909 i32_ty_basic_md,
910 i32_ty_basic_md,
911 ],
912 false,
913 ),
914 None,
915 ),
916 imported_table_grow: module.add_function(
917 "wasmer_vm_imported_table_grow",
918 i32_ty.fn_type(
919 &[
920 ctx_ptr_ty_basic_md,
921 anyref_ty_basic_md,
922 i32_ty_basic_md,
923 i32_ty_basic_md,
924 ],
925 false,
926 ),
927 None,
928 ),
929 memory_init: module.add_function(
930 "wasmer_vm_memory32_init",
931 void_ty.fn_type(
932 &[
933 ctx_ptr_ty_basic_md,
934 i32_ty_basic_md,
935 i32_ty_basic_md,
936 i32_ty_basic_md,
937 i32_ty_basic_md,
938 i32_ty_basic_md,
939 ],
940 false,
941 ),
942 None,
943 ),
944 memory_copy: module.add_function(
945 "wasmer_vm_memory32_copy",
946 void_ty.fn_type(
947 &[
948 ctx_ptr_ty_basic_md,
949 i32_ty_basic_md,
950 i32_ty_basic_md,
951 i32_ty_basic_md,
952 i32_ty_basic_md,
953 ],
954 false,
955 ),
956 None,
957 ),
958 imported_memory_copy: module.add_function(
959 "wasmer_vm_imported_memory32_copy",
960 void_ty.fn_type(
961 &[
962 ctx_ptr_ty_basic_md,
963 i32_ty_basic_md,
964 i32_ty_basic_md,
965 i32_ty_basic_md,
966 i32_ty_basic_md,
967 ],
968 false,
969 ),
970 None,
971 ),
972 memory_fill: module.add_function(
973 "wasmer_vm_memory32_fill",
974 void_ty.fn_type(
975 &[
976 ctx_ptr_ty_basic_md,
977 i32_ty_basic_md,
978 i32_ty_basic_md,
979 i32_ty_basic_md,
980 i32_ty_basic_md,
981 ],
982 false,
983 ),
984 None,
985 ),
986 imported_memory_fill: module.add_function(
987 "wasmer_vm_imported_memory32_fill",
988 void_ty.fn_type(
989 &[
990 ctx_ptr_ty_basic_md,
991 i32_ty_basic_md,
992 i32_ty_basic_md,
993 i32_ty_basic_md,
994 i32_ty_basic_md,
995 ],
996 false,
997 ),
998 None,
999 ),
1000 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1001 memory_grow_ty: i32_ty.fn_type(
1002 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1003 false,
1004 ),
1005 data_drop: module.add_function(
1006 "wasmer_vm_data_drop",
1007 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1008 None,
1009 ),
1010 func_ref: module.add_function(
1011 "wasmer_vm_func_ref",
1012 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1013 None,
1014 ),
1015 elem_drop: module.add_function(
1016 "wasmer_vm_elem_drop",
1017 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1018 None,
1019 ),
1020 throw_trap: module.add_function(
1021 "wasmer_vm_raise_trap",
1022 void_ty.fn_type(&[i32_ty_basic_md], false),
1023 None,
1024 ),
1025
1026 throw: module.add_function(
1027 "wasmer_vm_throw",
1028 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1029 None,
1030 ),
1031 alloc_exception: module.add_function(
1032 "wasmer_vm_alloc_exception",
1033 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1034 None,
1035 ),
1036 read_exnref: module.add_function(
1037 "wasmer_vm_read_exnref",
1038 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1039 None,
1040 ),
1041 exception_into_exnref: module.add_function(
1042 "wasmer_vm_exception_into_exnref",
1043 i32_ty.fn_type(&[ptr_ty.into()], false),
1044 None,
1045 ),
1046 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1047
1048 debug_ptr: module.add_function(
1049 "wasmer_vm_dbg_usize",
1050 void_ty.fn_type(&[ptr_ty.into()], false),
1051 None,
1052 ),
1053 debug_str: module.add_function(
1054 "wasmer_vm_dbg_str",
1055 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1056 None,
1057 ),
1058 memory_wait32: module.add_function(
1059 "wasmer_vm_memory32_atomic_wait32",
1060 i32_ty.fn_type(
1061 &[
1062 ctx_ptr_ty_basic_md,
1063 i32_ty_basic_md,
1064 i32_ty_basic_md,
1065 i32_ty_basic_md,
1066 i64_ty_basic_md,
1067 ],
1068 false,
1069 ),
1070 None,
1071 ),
1072 memory_wait32_ty: i32_ty.fn_type(
1073 &[
1074 ctx_ptr_ty_basic_md,
1075 i32_ty_basic_md,
1076 i32_ty_basic_md,
1077 i32_ty_basic_md,
1078 i64_ty_basic_md,
1079 ],
1080 false,
1081 ),
1082 imported_memory_wait32: module.add_function(
1083 "wasmer_vm_imported_memory32_atomic_wait32",
1084 i32_ty.fn_type(
1085 &[
1086 ctx_ptr_ty_basic_md,
1087 i32_ty_basic_md,
1088 i32_ty_basic_md,
1089 i32_ty_basic_md,
1090 i64_ty_basic_md,
1091 ],
1092 false,
1093 ),
1094 None,
1095 ),
1096 memory_wait64: module.add_function(
1097 "wasmer_vm_memory32_atomic_wait64",
1098 i32_ty.fn_type(
1099 &[
1100 ctx_ptr_ty_basic_md,
1101 i32_ty_basic_md,
1102 i32_ty_basic_md,
1103 i64_ty_basic_md,
1104 i64_ty_basic_md,
1105 ],
1106 false,
1107 ),
1108 None,
1109 ),
1110 memory_wait64_ty: i32_ty.fn_type(
1111 &[
1112 ctx_ptr_ty_basic_md,
1113 i32_ty_basic_md,
1114 i32_ty_basic_md,
1115 i64_ty_basic_md,
1116 i64_ty_basic_md,
1117 ],
1118 false,
1119 ),
1120 imported_memory_wait64: module.add_function(
1121 "wasmer_vm_imported_memory32_atomic_wait64",
1122 i32_ty.fn_type(
1123 &[
1124 ctx_ptr_ty_basic_md,
1125 i32_ty_basic_md,
1126 i32_ty_basic_md,
1127 i64_ty_basic_md,
1128 i64_ty_basic_md,
1129 ],
1130 false,
1131 ),
1132 None,
1133 ),
1134 memory_notify: module.add_function(
1135 "wasmer_vm_memory32_atomic_notify",
1136 i32_ty.fn_type(
1137 &[
1138 ctx_ptr_ty_basic_md,
1139 i32_ty_basic_md,
1140 i32_ty_basic_md,
1141 i32_ty_basic_md,
1142 ],
1143 false,
1144 ),
1145 None,
1146 ),
1147 memory_notify_ty: i32_ty.fn_type(
1148 &[
1149 ctx_ptr_ty_basic_md,
1150 i32_ty_basic_md,
1151 i32_ty_basic_md,
1152 i32_ty_basic_md,
1153 ],
1154 false,
1155 ),
1156 imported_memory_notify: module.add_function(
1157 "wasmer_vm_imported_memory32_atomic_notify",
1158 i32_ty.fn_type(
1159 &[
1160 ctx_ptr_ty_basic_md,
1161 i32_ty_basic_md,
1162 i32_ty_basic_md,
1163 i32_ty_basic_md,
1164 ],
1165 false,
1166 ),
1167 None,
1168 ),
1169
1170 vmfunction_import_ty: context.struct_type(&[i8_ptr_ty_basic, i8_ptr_ty_basic], false),
1171 vmfunction_import_body_element: 0,
1172 vmfunction_import_vmctx_element: 1,
1173 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1174 vmmemory_definition_base_element: 0,
1175 vmmemory_definition_current_length_element: 1,
1176
1177 ptr_ty,
1179 };
1180
1181 let noreturn =
1182 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1183 intrinsics
1184 .throw_trap
1185 .add_attribute(AttributeLoc::Function, noreturn);
1186 intrinsics
1187 }
1188}
1189
1190#[derive(Clone, Copy)]
1191pub enum MemoryCache<'ctx> {
1192 Dynamic {
1194 ptr_to_base_ptr: PointerValue<'ctx>,
1195 ptr_to_current_length: PointerValue<'ctx>,
1196 },
1197 Static { base_ptr: PointerValue<'ctx> },
1199}
1200
1201#[derive(Clone)]
1202struct TableCache<'ctx> {
1203 ptr_to_base_ptr: PointerValue<'ctx>,
1204 ptr_to_bounds: PointerValue<'ctx>,
1205}
1206
1207#[derive(Clone, Copy)]
1208pub enum GlobalCache<'ctx> {
1209 Mut {
1210 ptr_to_value: PointerValue<'ctx>,
1211 value_type: BasicTypeEnum<'ctx>,
1212 },
1213 Const {
1214 value: BasicValueEnum<'ctx>,
1215 },
1216}
1217
1218#[derive(Clone)]
1219pub struct FunctionCache<'ctx> {
1220 pub func: PointerValue<'ctx>,
1221 pub llvm_func_type: FunctionType<'ctx>,
1222 pub vmctx: BasicValueEnum<'ctx>,
1223 pub attrs: Vec<(Attribute, AttributeLoc)>,
1224}
1225
1226pub struct CtxType<'ctx, 'a> {
1227 ctx_ptr_value: PointerValue<'ctx>,
1228
1229 config: &'a LLVM,
1230 wasm_module: &'a WasmerCompilerModule,
1231 cache_builder: &'a Builder<'ctx>,
1232 abi: &'a dyn Abi,
1233
1234 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1235 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1236 cached_sigindices: HashMap<SignatureIndex, IntValue<'ctx>>,
1237 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1238 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1239 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1240
1241 offsets: VMOffsets,
1242}
1243
1244impl<'ctx, 'a> CtxType<'ctx, 'a> {
1245 pub fn new(
1246 wasm_module: &'a WasmerCompilerModule,
1247 func_value: &FunctionValue<'ctx>,
1248 cache_builder: &'a Builder<'ctx>,
1249 abi: &'a dyn Abi,
1250 config: &'a LLVM,
1251 ) -> CtxType<'ctx, 'a> {
1252 CtxType {
1253 config,
1254 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1255
1256 wasm_module,
1257 cache_builder,
1258 abi,
1259
1260 cached_memories: HashMap::new(),
1261 cached_tables: HashMap::new(),
1262 cached_sigindices: HashMap::new(),
1263 cached_globals: HashMap::new(),
1264 cached_functions: HashMap::new(),
1265 cached_memory_op: HashMap::new(),
1266
1267 offsets: VMOffsets::new(8, wasm_module),
1269 }
1270 }
1271
1272 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1273 self.ctx_ptr_value.as_basic_value_enum()
1274 }
1275
1276 pub fn memory(
1277 &mut self,
1278 index: MemoryIndex,
1279 intrinsics: &Intrinsics<'ctx>,
1280 module: &Module<'ctx>,
1281 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1282 ) -> Result<MemoryCache<'ctx>, CompileError> {
1283 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1284 &mut self.cached_memories,
1285 self.wasm_module,
1286 self.ctx_ptr_value,
1287 &self.cache_builder,
1288 &self.offsets,
1289 );
1290 let memory_style = &memory_styles[index];
1291 match cached_memories.get(&index) {
1292 Some(r) => Ok(*r),
1293 None => {
1294 let memory_definition_ptr =
1295 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1296 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1297 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1298 unsafe {
1299 err!(cache_builder.build_gep(
1300 intrinsics.i8_ty,
1301 ctx_ptr_value,
1302 &[offset],
1303 ""
1304 ))
1305 }
1306 } else {
1307 let offset = offsets.vmctx_vmmemory_import(index);
1308 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1309 let memory_definition_ptr_ptr = unsafe {
1310 err!(cache_builder.build_gep(
1311 intrinsics.i8_ty,
1312 ctx_ptr_value,
1313 &[offset],
1314 ""
1315 ))
1316 };
1317 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1318 memory_definition_ptr_ptr,
1319 intrinsics.ptr_ty,
1320 "",
1321 ))
1322 .into_pointer_value();
1323 let memory_definition_ptr = err!(cache_builder.build_load(
1324 intrinsics.ptr_ty,
1325 memory_definition_ptr_ptr,
1326 ""
1327 ))
1328 .into_pointer_value();
1329 tbaa_label(
1330 module,
1331 intrinsics,
1332 format!("memory {} definition", index.as_u32()),
1333 memory_definition_ptr.as_instruction_value().unwrap(),
1334 );
1335 memory_definition_ptr
1336 };
1337 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1338 memory_definition_ptr,
1339 intrinsics.ptr_ty,
1340 "",
1341 ))
1342 .into_pointer_value();
1343 let base_ptr = err!(cache_builder.build_struct_gep(
1344 intrinsics.vmmemory_definition_ty,
1345 memory_definition_ptr,
1346 intrinsics.vmmemory_definition_base_element,
1347 "",
1348 ));
1349 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1350 let current_length_ptr = err!(cache_builder.build_struct_gep(
1351 intrinsics.vmmemory_definition_ty,
1352 memory_definition_ptr,
1353 intrinsics.vmmemory_definition_current_length_element,
1354 "",
1355 ));
1356 MemoryCache::Dynamic {
1357 ptr_to_base_ptr: base_ptr,
1358 ptr_to_current_length: current_length_ptr,
1359 }
1360 } else {
1361 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1362 .into_pointer_value();
1363 tbaa_label(
1364 module,
1365 intrinsics,
1366 format!("memory base_ptr {}", index.as_u32()),
1367 base_ptr.as_instruction_value().unwrap(),
1368 );
1369 MemoryCache::Static { base_ptr }
1370 };
1371
1372 self.cached_memories.insert(index, value);
1373 Ok(*self.cached_memories.get(&index).unwrap())
1374 }
1375 }
1376 }
1377
1378 fn build_table_prepare(
1379 table_index: TableIndex,
1380 intrinsics: &Intrinsics<'ctx>,
1381 module: &Module<'ctx>,
1382 wasm_module: &WasmerCompilerModule,
1383 ctx_ptr_value: PointerValue<'ctx>,
1384 offsets: &VMOffsets,
1385 builder: &Builder<'ctx>,
1386 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1387 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1388 let offset = intrinsics.i64_ty.const_int(
1389 offsets
1390 .vmctx_vmtable_definition_base(local_table_index)
1391 .into(),
1392 false,
1393 );
1394 let ptr_to_base_ptr =
1395 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1396 let ptr_to_base_ptr =
1397 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1398 .into_pointer_value();
1399 let offset = intrinsics.i64_ty.const_int(
1400 offsets
1401 .vmctx_vmtable_definition_current_elements(local_table_index)
1402 .into(),
1403 false,
1404 );
1405 let ptr_to_bounds =
1406 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1407 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1408 .into_pointer_value();
1409 Ok((ptr_to_base_ptr, ptr_to_bounds))
1410 } else {
1411 let offset = intrinsics.i64_ty.const_int(
1412 offsets.vmctx_vmtable_import_definition(table_index).into(),
1413 false,
1414 );
1415 let definition_ptr_ptr =
1416 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1417 let definition_ptr_ptr =
1418 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1419 .into_pointer_value();
1420 let definition_ptr =
1421 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1422 .into_pointer_value();
1423 tbaa_label(
1424 module,
1425 intrinsics,
1426 format!("table {} definition", table_index.as_u32()),
1427 definition_ptr.as_instruction_value().unwrap(),
1428 );
1429
1430 let offset = intrinsics
1431 .i64_ty
1432 .const_int(offsets.vmtable_definition_base().into(), false);
1433 let ptr_to_base_ptr =
1434 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1435 let ptr_to_base_ptr =
1436 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1437 .into_pointer_value();
1438 let offset = intrinsics
1439 .i64_ty
1440 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1441 let ptr_to_bounds =
1442 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1443 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1444 .into_pointer_value();
1445 Ok((ptr_to_base_ptr, ptr_to_bounds))
1446 }
1447 }
1448
1449 fn table_prepare(
1450 &mut self,
1451 table_index: TableIndex,
1452 intrinsics: &Intrinsics<'ctx>,
1453 module: &Module<'ctx>,
1454 body_builder: &Builder<'ctx>,
1455 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1456 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1457 &mut self.cached_tables,
1458 self.wasm_module,
1459 self.ctx_ptr_value,
1460 &self.cache_builder,
1461 &self.offsets,
1462 );
1463
1464 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1465 CompileError::Codegen(format!(
1466 "Table index out of bounds: {}",
1467 table_index.as_u32()
1468 ))
1469 })?;
1470
1471 if is_growable {
1474 Self::build_table_prepare(
1475 table_index,
1476 intrinsics,
1477 module,
1478 wasm_module,
1479 ctx_ptr_value,
1480 offsets,
1481 body_builder,
1482 )
1483 } else {
1484 let TableCache {
1485 ptr_to_base_ptr,
1486 ptr_to_bounds,
1487 } = match cached_tables.entry(table_index) {
1488 Entry::Occupied(entry) => entry.get().clone(),
1489 Entry::Vacant(entry) => {
1490 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1491 table_index,
1492 intrinsics,
1493 module,
1494 wasm_module,
1495 ctx_ptr_value,
1496 offsets,
1497 cache_builder,
1498 )?;
1499
1500 let v = TableCache {
1501 ptr_to_base_ptr,
1502 ptr_to_bounds,
1503 };
1504
1505 entry.insert(v.clone());
1506
1507 v
1508 }
1509 };
1510
1511 Ok((ptr_to_base_ptr, ptr_to_bounds))
1512 }
1513 }
1514
1515 pub fn table(
1516 &mut self,
1517 index: TableIndex,
1518 intrinsics: &Intrinsics<'ctx>,
1519 module: &Module<'ctx>,
1520 body_builder: &Builder<'ctx>,
1521 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1522 let (ptr_to_base_ptr, ptr_to_bounds) =
1523 self.table_prepare(index, intrinsics, module, body_builder)?;
1524
1525 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1527 &body_builder
1528 } else {
1529 &self.cache_builder
1530 };
1531
1532 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1533 .into_pointer_value();
1534 let bounds =
1535 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1536 tbaa_label(
1537 module,
1538 intrinsics,
1539 format!("table_base_ptr {}", index.index()),
1540 base_ptr.as_instruction_value().unwrap(),
1541 );
1542 tbaa_label(
1543 module,
1544 intrinsics,
1545 format!("table_bounds {}", index.index()),
1546 bounds.as_instruction_value().unwrap(),
1547 );
1548 Ok((base_ptr, bounds))
1549 }
1550
1551 pub fn dynamic_sigindex(
1552 &mut self,
1553 index: SignatureIndex,
1554 intrinsics: &Intrinsics<'ctx>,
1555 module: &Module<'ctx>,
1556 ) -> Result<IntValue<'ctx>, CompileError> {
1557 let (cached_sigindices, ctx_ptr_value, cache_builder, offsets) = (
1558 &mut self.cached_sigindices,
1559 self.ctx_ptr_value,
1560 &self.cache_builder,
1561 &self.offsets,
1562 );
1563
1564 match cached_sigindices.entry(index) {
1565 Entry::Occupied(entry) => Ok(*entry.get()),
1566 Entry::Vacant(entry) => {
1567 let byte_offset = intrinsics
1568 .i64_ty
1569 .const_int(offsets.vmctx_vmshared_signature_id(index).into(), false);
1570
1571 let sigindex_ptr = unsafe {
1572 err!(cache_builder.build_gep(
1573 intrinsics.i8_ty,
1574 ctx_ptr_value,
1575 &[byte_offset],
1576 "dynamic_sigindex",
1577 ))
1578 };
1579
1580 let sigindex_ptr =
1581 err!(cache_builder.build_bit_cast(sigindex_ptr, intrinsics.ptr_ty, ""))
1582 .into_pointer_value();
1583
1584 let sigindex =
1585 err!(cache_builder.build_load(intrinsics.i32_ty, sigindex_ptr, "sigindex"))
1586 .into_int_value();
1587 tbaa_label(
1588 module,
1589 intrinsics,
1590 format!("sigindex {}", index.as_u32()),
1591 sigindex.as_instruction_value().unwrap(),
1592 );
1593
1594 entry.insert(sigindex);
1595 Ok(sigindex)
1596 }
1597 }
1598 }
1599
1600 pub fn global(
1601 &mut self,
1602 index: GlobalIndex,
1603 intrinsics: &Intrinsics<'ctx>,
1604 module: &Module<'ctx>,
1605 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1606 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1607 &mut self.cached_globals,
1608 self.wasm_module,
1609 self.ctx_ptr_value,
1610 &self.cache_builder,
1611 &self.offsets,
1612 );
1613 match cached_globals.entry(index) {
1614 Entry::Occupied(entry) => Ok(entry.into_mut()),
1615 Entry::Vacant(entry) => {
1616 let global_type = wasm_module.globals[index];
1617 let global_value_type = global_type.ty;
1618
1619 let global_mutability = global_type.mutability;
1620 let offset = if let Some(local_global_index) = wasm_module.local_global_index(index)
1621 {
1622 offsets.vmctx_vmglobal_definition(local_global_index)
1623 } else {
1624 offsets.vmctx_vmglobal_import(index)
1625 };
1626 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1627 let global_ptr = {
1628 let global_ptr_ptr = unsafe {
1629 err!(cache_builder.build_gep(
1630 intrinsics.i8_ty,
1631 ctx_ptr_value,
1632 &[offset],
1633 ""
1634 ))
1635 };
1636 let global_ptr_ptr =
1637 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1638 .into_pointer_value();
1639 let global_ptr =
1640 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1641 .into_pointer_value();
1642 tbaa_label(
1643 module,
1644 intrinsics,
1645 format!("global_ptr {}", index.as_u32()),
1646 global_ptr.as_instruction_value().unwrap(),
1647 );
1648 global_ptr
1649 };
1650 let global_ptr = err!(cache_builder.build_bit_cast(
1651 global_ptr,
1652 type_to_llvm_ptr(intrinsics, global_value_type)?,
1653 "",
1654 ))
1655 .into_pointer_value();
1656
1657 let ret = entry.insert(match global_mutability {
1658 Mutability::Const => {
1659 let value = err!(cache_builder.build_load(
1660 type_to_llvm(intrinsics, global_value_type)?,
1661 global_ptr,
1662 "",
1663 ));
1664 tbaa_label(
1665 module,
1666 intrinsics,
1667 format!("global {}", index.as_u32()),
1668 value.as_instruction_value().unwrap(),
1669 );
1670 GlobalCache::Const { value }
1671 }
1672 Mutability::Var => GlobalCache::Mut {
1673 ptr_to_value: global_ptr,
1674 value_type: type_to_llvm(intrinsics, global_value_type)?,
1675 },
1676 });
1677
1678 Ok(ret)
1679 }
1680 }
1681 }
1682
1683 pub fn add_func(
1684 &mut self,
1685 function_index: FunctionIndex,
1686 func: PointerValue<'ctx>,
1687 llvm_func_type: FunctionType<'ctx>,
1688 vmctx: BasicValueEnum<'ctx>,
1689 attrs: &[(Attribute, AttributeLoc)],
1690 ) {
1691 match self.cached_functions.entry(function_index) {
1692 Entry::Occupied(_) => unreachable!("duplicate function"),
1693 Entry::Vacant(entry) => {
1694 entry.insert(FunctionCache {
1695 func,
1696 llvm_func_type,
1697 vmctx,
1698 attrs: attrs.to_vec(),
1699 });
1700 }
1701 }
1702 }
1703
1704 #[allow(clippy::too_many_arguments)]
1705 pub fn local_func(
1706 &mut self,
1707 _local_function_index: LocalFunctionIndex,
1708 function_index: FunctionIndex,
1709 intrinsics: &Intrinsics<'ctx>,
1710 module: &Module<'ctx>,
1711 context: &'ctx Context,
1712 func_type: &FuncType,
1713 function_name: &str,
1714 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1715 let (cached_functions, ctx_ptr_value, offsets) = (
1716 &mut self.cached_functions,
1717 &self.ctx_ptr_value,
1718 &self.offsets,
1719 );
1720 Ok(match cached_functions.entry(function_index) {
1721 Entry::Occupied(entry) => entry.into_mut(),
1722 Entry::Vacant(entry) => {
1723 debug_assert!(module.get_function(function_name).is_none());
1724 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1725 context,
1726 intrinsics,
1727 Some(offsets),
1728 func_type,
1729 if self.config.enable_g0m0_opt {
1730 Some(crate::abi::G0M0FunctionKind::Local)
1731 } else {
1732 None
1733 },
1734 )?;
1735 let func =
1736 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1737 for (attr, attr_loc) in &llvm_func_attrs {
1738 func.add_attribute(*attr_loc, *attr);
1739 }
1740 entry.insert(FunctionCache {
1741 func: func.as_global_value().as_pointer_value(),
1742 llvm_func_type,
1743 vmctx: ctx_ptr_value.as_basic_value_enum(),
1744 attrs: llvm_func_attrs,
1745 })
1746 }
1747 })
1748 }
1749
1750 pub fn func(
1751 &mut self,
1752 function_index: FunctionIndex,
1753 intrinsics: &Intrinsics<'ctx>,
1754 context: &'ctx Context,
1755 func_type: &FuncType,
1756 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1757 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1758 &mut self.cached_functions,
1759 self.wasm_module,
1760 &self.ctx_ptr_value,
1761 &self.cache_builder,
1762 &self.offsets,
1763 );
1764 match cached_functions.entry(function_index) {
1765 Entry::Occupied(entry) => Ok(entry.into_mut()),
1766 Entry::Vacant(entry) => {
1767 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1768 context,
1769 intrinsics,
1770 Some(offsets),
1771 func_type,
1772 None,
1773 )?;
1774 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1775 let offset = offsets.vmctx_vmfunction_import(function_index);
1776 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1777 let vmfunction_import_ptr = unsafe {
1778 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1779 };
1780 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1781 vmfunction_import_ptr,
1782 intrinsics.ptr_ty,
1783 "",
1784 ))
1785 .into_pointer_value();
1786
1787 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1788 intrinsics.vmfunction_import_ty,
1789 vmfunction_import_ptr,
1790 intrinsics.vmfunction_import_body_element,
1791 "",
1792 ));
1793 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
1794 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
1795 .into_pointer_value();
1796 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
1797 intrinsics.vmfunction_import_ty,
1798 vmfunction_import_ptr,
1799 intrinsics.vmfunction_import_vmctx_element,
1800 "",
1801 ));
1802 let vmctx_ptr =
1803 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
1804
1805 Ok(entry.insert(FunctionCache {
1806 func: body_ptr,
1807 llvm_func_type,
1808 vmctx: vmctx_ptr,
1809 attrs: llvm_func_attrs,
1810 }))
1811 }
1812 }
1813 }
1814
1815 pub fn memory_grow(
1816 &mut self,
1817 memory_index: MemoryIndex,
1818 intrinsics: &Intrinsics<'ctx>,
1819 ) -> Result<PointerValue<'ctx>, CompileError> {
1820 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1821 &mut self.cached_memory_op,
1822 &self.wasm_module,
1823 &self.offsets,
1824 &self.cache_builder,
1825 &self.ctx_ptr_value,
1826 );
1827 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
1828 Entry::Occupied(entry) => Ok(*entry.get()),
1829 Entry::Vacant(entry) => {
1830 let (grow_fn, grow_fn_ty) =
1831 if wasm_module.local_memory_index(memory_index).is_some() {
1832 (
1833 VMBuiltinFunctionIndex::get_memory32_grow_index(),
1834 intrinsics.ptr_ty,
1835 )
1836 } else {
1837 (
1838 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
1839 intrinsics.ptr_ty,
1840 )
1841 };
1842 let offset = offsets.vmctx_builtin_function(grow_fn);
1843 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1844 let grow_fn_ptr_ptr = unsafe {
1845 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1846 };
1847
1848 let grow_fn_ptr_ptr =
1849 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
1850 .into_pointer_value();
1851 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
1852 .into_pointer_value();
1853
1854 entry.insert(val);
1855 Ok(val)
1856 }
1857 }
1858 }
1859
1860 pub fn memory_size(
1861 &mut self,
1862 memory_index: MemoryIndex,
1863 intrinsics: &Intrinsics<'ctx>,
1864 ) -> Result<PointerValue<'ctx>, CompileError> {
1865 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1866 &mut self.cached_memory_op,
1867 &self.wasm_module,
1868 &self.offsets,
1869 &self.cache_builder,
1870 &self.ctx_ptr_value,
1871 );
1872
1873 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
1874 Entry::Occupied(entry) => Ok(*entry.get()),
1875 Entry::Vacant(entry) => {
1876 let (size_fn, size_fn_ty) =
1877 if wasm_module.local_memory_index(memory_index).is_some() {
1878 (
1879 VMBuiltinFunctionIndex::get_memory32_size_index(),
1880 intrinsics.ptr_ty,
1881 )
1882 } else {
1883 (
1884 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
1885 intrinsics.ptr_ty,
1886 )
1887 };
1888 let offset = offsets.vmctx_builtin_function(size_fn);
1889 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1890 let size_fn_ptr_ptr = unsafe {
1891 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1892 };
1893
1894 let size_fn_ptr_ptr =
1895 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
1896 .into_pointer_value();
1897
1898 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
1899 .into_pointer_value();
1900 entry.insert(val);
1901 Ok(val)
1902 }
1903 }
1904 }
1905
1906 pub fn memory_wait32(
1907 &mut self,
1908 memory_index: MemoryIndex,
1909 intrinsics: &Intrinsics<'ctx>,
1910 ) -> Result<PointerValue<'ctx>, CompileError> {
1911 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1912 &mut self.cached_memory_op,
1913 &self.wasm_module,
1914 &self.offsets,
1915 &self.cache_builder,
1916 &self.ctx_ptr_value,
1917 );
1918 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
1919 Entry::Occupied(entry) => Ok(*entry.get()),
1920 Entry::Vacant(entry) => {
1921 let (size_fn, size_fn_ty) =
1922 if wasm_module.local_memory_index(memory_index).is_some() {
1923 (
1924 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
1925 intrinsics.ptr_ty,
1926 )
1927 } else {
1928 (
1929 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
1930 intrinsics.ptr_ty,
1931 )
1932 };
1933 let offset = offsets.vmctx_builtin_function(size_fn);
1934 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1935 let size_fn_ptr_ptr = unsafe {
1936 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1937 };
1938
1939 let size_fn_ptr_ptr =
1940 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
1941 .into_pointer_value();
1942
1943 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
1944 .into_pointer_value();
1945
1946 entry.insert(val);
1947 Ok(val)
1948 }
1949 }
1950 }
1951
1952 pub fn memory_wait64(
1953 &mut self,
1954 memory_index: MemoryIndex,
1955 intrinsics: &Intrinsics<'ctx>,
1956 ) -> Result<PointerValue<'ctx>, CompileError> {
1957 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
1958 &mut self.cached_memory_op,
1959 &self.wasm_module,
1960 &self.offsets,
1961 &self.cache_builder,
1962 &self.ctx_ptr_value,
1963 );
1964
1965 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
1966 Entry::Occupied(entry) => Ok(*entry.get()),
1967 Entry::Vacant(entry) => {
1968 let (size_fn, size_fn_ty) =
1969 if wasm_module.local_memory_index(memory_index).is_some() {
1970 (
1971 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
1972 intrinsics.ptr_ty,
1973 )
1974 } else {
1975 (
1976 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
1977 intrinsics.ptr_ty,
1978 )
1979 };
1980 let offset = offsets.vmctx_builtin_function(size_fn);
1981 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1982 let size_fn_ptr_ptr = unsafe {
1983 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1984 };
1985
1986 let size_fn_ptr_ptr =
1987 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
1988 .into_pointer_value();
1989
1990 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
1991 .into_pointer_value();
1992 entry.insert(val);
1993 Ok(val)
1994 }
1995 }
1996 }
1997
1998 pub fn memory_notify(
1999 &mut self,
2000 memory_index: MemoryIndex,
2001 intrinsics: &Intrinsics<'ctx>,
2002 ) -> Result<PointerValue<'ctx>, CompileError> {
2003 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2004 &mut self.cached_memory_op,
2005 &self.wasm_module,
2006 &self.offsets,
2007 &self.cache_builder,
2008 &self.ctx_ptr_value,
2009 );
2010 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2011 Entry::Occupied(entry) => Ok(*entry.get()),
2012 Entry::Vacant(entry) => {
2013 let (size_fn, size_fn_ty) =
2014 if wasm_module.local_memory_index(memory_index).is_some() {
2015 (
2016 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2017 intrinsics.ptr_ty,
2018 )
2019 } else {
2020 (
2021 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2022 intrinsics.ptr_ty,
2023 )
2024 };
2025 let offset = offsets.vmctx_builtin_function(size_fn);
2026 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2027 let size_fn_ptr_ptr = unsafe {
2028 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2029 };
2030
2031 let size_fn_ptr_ptr =
2032 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2033 .into_pointer_value();
2034
2035 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2036 .into_pointer_value();
2037
2038 entry.insert(val);
2039 Ok(val)
2040 }
2041 }
2042 }
2043
2044 pub fn get_offsets(&self) -> &VMOffsets {
2045 &self.offsets
2046 }
2047}
2048
2049pub fn tbaa_label<'ctx>(
2052 module: &Module<'ctx>,
2053 intrinsics: &Intrinsics<'ctx>,
2054 label: String,
2055 instruction: InstructionValue<'ctx>,
2056) {
2057 let context = module.get_context();
2071
2072 let tbaa_root = module
2074 .get_global_metadata("wasmer_tbaa_root")
2075 .pop()
2076 .unwrap_or_else(|| {
2077 module
2078 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2079 .unwrap();
2080 module.get_global_metadata("wasmer_tbaa_root")[0]
2081 });
2082
2083 let type_label = context.metadata_string(label.as_str());
2086 let type_tbaa = module
2087 .get_global_metadata(label.as_str())
2088 .pop()
2089 .unwrap_or_else(|| {
2090 module
2091 .add_global_metadata(
2092 label.as_str(),
2093 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2094 )
2095 .unwrap();
2096 module.get_global_metadata(label.as_str())[0]
2097 });
2098
2099 let label = label + "_memop";
2106 let type_tbaa = module
2107 .get_global_metadata(label.as_str())
2108 .pop()
2109 .unwrap_or_else(|| {
2110 module
2111 .add_global_metadata(
2112 label.as_str(),
2113 &context.metadata_node(&[
2114 type_tbaa.into(),
2115 type_tbaa.into(),
2116 intrinsics.i64_zero.into(),
2117 ]),
2118 )
2119 .unwrap();
2120 module.get_global_metadata(label.as_str())[0]
2121 });
2122
2123 let tbaa_kind = context.get_kind_id("tbaa");
2125 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2126}
2127
2128fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2129 let table = module.tables.get(index)?;
2130 match table.maximum {
2131 None => Some(true),
2132 Some(max) => Some(max > table.minimum),
2133 }
2134}