1use crate::abi::Abi;
8use crate::error::err;
9use inkwell::values::BasicMetadataValueEnum;
10use inkwell::{
11 AddressSpace,
12 attributes::{Attribute, AttributeLoc},
13 builder::Builder,
14 context::Context,
15 module::{Linkage, Module},
16 targets::TargetData,
17 types::{
18 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
19 PointerType, StructType, VectorType, VoidType,
20 },
21 values::{
22 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
23 PointerValue, VectorValue,
24 },
25};
26use std::collections::{HashMap, hash_map::Entry};
27use target_lexicon::{Architecture, Triple};
28use wasmer_types::entity::{EntityRef, PrimaryMap};
29use wasmer_types::{
30 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
31 MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, SignatureIndex, TableIndex, Type,
32};
33use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
34
35pub fn type_to_llvm<'ctx>(
36 intrinsics: &Intrinsics<'ctx>,
37 ty: Type,
38) -> Result<BasicTypeEnum<'ctx>, CompileError> {
39 match ty {
40 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
41 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
42 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
43 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
44 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
45 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
46 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
47 }
48}
49
50#[allow(dead_code)]
52pub struct X86_64Intrinsics<'ctx> {
53 pub pshufb128: FunctionValue<'ctx>,
54 pub pmaddubsw128: FunctionValue<'ctx>,
55 pub pmaddwd128: FunctionValue<'ctx>,
56 pub pmulhrsw128: FunctionValue<'ctx>,
57 pub pblendvb: FunctionValue<'ctx>,
58 pub min_ps: FunctionValue<'ctx>,
59 pub min_pd: FunctionValue<'ctx>,
60 pub max_ps: FunctionValue<'ctx>,
61 pub max_pd: FunctionValue<'ctx>,
62 pub cvttps2dq: FunctionValue<'ctx>,
63 pub cvtps2udq128: FunctionValue<'ctx>,
64 pub cvtpd2dq: FunctionValue<'ctx>,
65 pub cvtpd2udq128: FunctionValue<'ctx>,
66}
67
68#[allow(dead_code)]
70pub struct Intrinsics<'ctx> {
71 pub ctlz_i32: FunctionValue<'ctx>,
72 pub ctlz_i64: FunctionValue<'ctx>,
73
74 pub cttz_i32: FunctionValue<'ctx>,
75 pub cttz_i64: FunctionValue<'ctx>,
76
77 pub ctpop_i32: FunctionValue<'ctx>,
78 pub ctpop_i64: FunctionValue<'ctx>,
79 pub ctpop_i8x16: FunctionValue<'ctx>,
80
81 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
82 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
83 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
84 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
85 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
86
87 pub add_f32: FunctionValue<'ctx>,
88 pub add_f64: FunctionValue<'ctx>,
89 pub add_f32x4: FunctionValue<'ctx>,
90 pub add_f64x2: FunctionValue<'ctx>,
91
92 pub sub_f32: FunctionValue<'ctx>,
93 pub sub_f64: FunctionValue<'ctx>,
94 pub sub_f32x4: FunctionValue<'ctx>,
95 pub sub_f64x2: FunctionValue<'ctx>,
96
97 pub mul_f32: FunctionValue<'ctx>,
98 pub mul_f64: FunctionValue<'ctx>,
99 pub mul_f32x4: FunctionValue<'ctx>,
100 pub mul_f64x2: FunctionValue<'ctx>,
101 pub muladd_f32x4: FunctionValue<'ctx>,
102 pub muladd_f64x2: FunctionValue<'ctx>,
103
104 pub div_f32: FunctionValue<'ctx>,
105 pub div_f64: FunctionValue<'ctx>,
106 pub div_f32x4: FunctionValue<'ctx>,
107 pub div_f64x2: FunctionValue<'ctx>,
108
109 pub sqrt_f32: FunctionValue<'ctx>,
110 pub sqrt_f64: FunctionValue<'ctx>,
111 pub sqrt_f32x4: FunctionValue<'ctx>,
112 pub sqrt_f64x2: FunctionValue<'ctx>,
113
114 pub cmp_f32: FunctionValue<'ctx>,
115 pub cmp_f64: FunctionValue<'ctx>,
116 pub cmp_f32x4: FunctionValue<'ctx>,
117 pub cmp_f64x2: FunctionValue<'ctx>,
118
119 pub minimum_f32: FunctionValue<'ctx>,
120 pub minimum_f64: FunctionValue<'ctx>,
121 pub minimum_f32x4: FunctionValue<'ctx>,
122 pub minimum_f64x2: FunctionValue<'ctx>,
123
124 pub maximum_f32: FunctionValue<'ctx>,
125 pub maximum_f64: FunctionValue<'ctx>,
126 pub maximum_f32x4: FunctionValue<'ctx>,
127 pub maximum_f64x2: FunctionValue<'ctx>,
128
129 pub ceil_f32: FunctionValue<'ctx>,
130 pub ceil_f64: FunctionValue<'ctx>,
131 pub ceil_f32x4: FunctionValue<'ctx>,
132 pub ceil_f64x2: FunctionValue<'ctx>,
133
134 pub floor_f32: FunctionValue<'ctx>,
135 pub floor_f64: FunctionValue<'ctx>,
136 pub floor_f32x4: FunctionValue<'ctx>,
137 pub floor_f64x2: FunctionValue<'ctx>,
138
139 pub trunc_f32: FunctionValue<'ctx>,
140 pub trunc_f64: FunctionValue<'ctx>,
141 pub trunc_f32x4: FunctionValue<'ctx>,
142 pub trunc_f64x2: FunctionValue<'ctx>,
143
144 pub fpext_f32: FunctionValue<'ctx>,
145 pub fptrunc_f64: FunctionValue<'ctx>,
146
147 pub nearbyint_f32: FunctionValue<'ctx>,
148 pub nearbyint_f64: FunctionValue<'ctx>,
149 pub nearbyint_f32x4: FunctionValue<'ctx>,
150 pub nearbyint_f64x2: FunctionValue<'ctx>,
151
152 pub fabs_f32: FunctionValue<'ctx>,
153 pub fabs_f64: FunctionValue<'ctx>,
154 pub fabs_f32x4: FunctionValue<'ctx>,
155 pub fabs_f64x2: FunctionValue<'ctx>,
156
157 pub copysign_f32: FunctionValue<'ctx>,
158 pub copysign_f64: FunctionValue<'ctx>,
159 pub copysign_f32x4: FunctionValue<'ctx>,
160 pub copysign_f64x2: FunctionValue<'ctx>,
161
162 pub sadd_sat_i8x16: FunctionValue<'ctx>,
163 pub sadd_sat_i16x8: FunctionValue<'ctx>,
164 pub uadd_sat_i8x16: FunctionValue<'ctx>,
165 pub uadd_sat_i16x8: FunctionValue<'ctx>,
166
167 pub ssub_sat_i8x16: FunctionValue<'ctx>,
168 pub ssub_sat_i16x8: FunctionValue<'ctx>,
169 pub usub_sat_i8x16: FunctionValue<'ctx>,
170 pub usub_sat_i16x8: FunctionValue<'ctx>,
171
172 pub expect_i1: FunctionValue<'ctx>,
173 pub trap: FunctionValue<'ctx>,
174 pub debug_trap: FunctionValue<'ctx>,
175
176 pub personality: FunctionValue<'ctx>,
177 pub personality2: FunctionValue<'ctx>,
178 pub readonly: Attribute,
179 pub stack_probe: Attribute,
180 pub uwtable: Attribute,
181 pub frame_pointer: Attribute,
182 pub chkstk: FunctionValue<'ctx>,
184
185 pub void_ty: VoidType<'ctx>,
186 pub i1_ty: IntType<'ctx>,
187 pub i2_ty: IntType<'ctx>,
188 pub i4_ty: IntType<'ctx>,
189 pub i8_ty: IntType<'ctx>,
190 pub i16_ty: IntType<'ctx>,
191 pub i32_ty: IntType<'ctx>,
192 pub i64_ty: IntType<'ctx>,
193 pub i128_ty: IntType<'ctx>,
194 pub isize_ty: IntType<'ctx>,
195 pub f32_ty: FloatType<'ctx>,
196 pub f64_ty: FloatType<'ctx>,
197
198 pub i1x128_ty: VectorType<'ctx>,
199 pub i8x16_ty: VectorType<'ctx>,
200 pub i16x8_ty: VectorType<'ctx>,
201 pub i32x4_ty: VectorType<'ctx>,
202 pub i64x2_ty: VectorType<'ctx>,
203 pub f32x4_ty: VectorType<'ctx>,
204 pub f64x2_ty: VectorType<'ctx>,
205 pub i32x8_ty: VectorType<'ctx>,
206
207 pub ptr_ty: PointerType<'ctx>,
208
209 pub x86_64: X86_64Intrinsics<'ctx>,
210
211 pub anyfunc_ty: StructType<'ctx>,
212
213 pub i1_zero: IntValue<'ctx>,
214 pub i8_zero: IntValue<'ctx>,
215 pub i32_zero: IntValue<'ctx>,
216 pub i64_zero: IntValue<'ctx>,
217 pub i128_zero: IntValue<'ctx>,
218 pub isize_zero: IntValue<'ctx>,
219 pub f32_zero: FloatValue<'ctx>,
220 pub f64_zero: FloatValue<'ctx>,
221 pub f32x4_zero: VectorValue<'ctx>,
222 pub f64x2_zero: VectorValue<'ctx>,
223 pub i32_consts: [IntValue<'ctx>; 16],
224
225 pub trap_unreachable: BasicValueEnum<'ctx>,
226 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
227 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
228 pub trap_memory_oob: BasicValueEnum<'ctx>,
229 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
230 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
231 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
232 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
233 pub trap_table_access_oob: BasicValueEnum<'ctx>,
234
235 pub experimental_stackmap: FunctionValue<'ctx>,
236
237 pub table_copy: FunctionValue<'ctx>,
239 pub table_init: FunctionValue<'ctx>,
240 pub table_fill: FunctionValue<'ctx>,
241 pub table_size: FunctionValue<'ctx>,
242 pub imported_table_size: FunctionValue<'ctx>,
243 pub table_get: FunctionValue<'ctx>,
244 pub imported_table_get: FunctionValue<'ctx>,
245 pub table_set: FunctionValue<'ctx>,
246 pub imported_table_set: FunctionValue<'ctx>,
247 pub table_grow: FunctionValue<'ctx>,
248 pub imported_table_grow: FunctionValue<'ctx>,
249 pub memory_init: FunctionValue<'ctx>,
250 pub data_drop: FunctionValue<'ctx>,
251 pub func_ref: FunctionValue<'ctx>,
252 pub elem_drop: FunctionValue<'ctx>,
253 pub memory_copy: FunctionValue<'ctx>,
254 pub imported_memory_copy: FunctionValue<'ctx>,
255 pub memory_fill: FunctionValue<'ctx>,
256 pub imported_memory_fill: FunctionValue<'ctx>,
257 pub memory_size_ty: FunctionType<'ctx>,
258 pub memory_grow_ty: FunctionType<'ctx>,
259 pub memory_wait32: FunctionValue<'ctx>,
260 pub memory_wait32_ty: FunctionType<'ctx>,
261 pub imported_memory_wait32: FunctionValue<'ctx>,
262 pub memory_wait64: FunctionValue<'ctx>,
263 pub memory_wait64_ty: FunctionType<'ctx>,
264 pub imported_memory_wait64: FunctionValue<'ctx>,
265 pub memory_notify: FunctionValue<'ctx>,
266 pub memory_notify_ty: FunctionType<'ctx>,
267 pub imported_memory_notify: FunctionValue<'ctx>,
268
269 pub throw_trap: FunctionValue<'ctx>,
270
271 pub throw: FunctionValue<'ctx>,
273 pub alloc_exception: FunctionValue<'ctx>,
274 pub read_exnref: FunctionValue<'ctx>,
275 pub exception_into_exnref: FunctionValue<'ctx>,
276 pub lpad_exception_ty: StructType<'ctx>,
277
278 pub debug_ptr: FunctionValue<'ctx>,
280 pub debug_str: FunctionValue<'ctx>,
281
282 pub vmfunction_import_ty: StructType<'ctx>,
284 pub vmfunction_import_body_element: u32,
285 pub vmfunction_import_vmctx_element: u32,
286 pub vmfunction_import_include_m0_param_element: u32,
287
288 pub vmmemory_definition_ty: StructType<'ctx>,
289 pub vmmemory_definition_base_element: u32,
290 pub vmmemory_definition_current_length_element: u32,
291}
292
293#[derive(Debug, Hash, PartialEq, Eq)]
294enum MemoryOp {
295 Size,
296 Grow,
297 Wait32,
298 Wait64,
299 Notify,
300}
301
302impl<'ctx> Intrinsics<'ctx> {
303 pub fn declare(
305 module: &Module<'ctx>,
306 context: &'ctx Context,
307 target_data: &TargetData,
308 target_triple: &Triple,
309 binary_fmt: &target_lexicon::BinaryFormat,
310 ) -> Self {
311 let is_riscv64 = matches!(target_triple.architecture, Architecture::Riscv64(..));
312 let void_ty = context.void_type();
313 let i1_ty = context.bool_type();
314 let i2_ty = context.custom_width_int_type(2);
315 let i4_ty = context.custom_width_int_type(4);
316 let i8_ty = context.i8_type();
317 let i16_ty = context.i16_type();
318 let i32_ty = context.i32_type();
319 let i64_ty = context.i64_type();
320 let i128_ty = context.i128_type();
321 let isize_ty = context.ptr_sized_int_type(target_data, None);
322 let f32_ty = context.f32_type();
323 let f64_ty = context.f64_type();
324
325 let i1x4_ty = i1_ty.vec_type(4);
326 let i1x2_ty = i1_ty.vec_type(2);
327 let i1x128_ty = i1_ty.vec_type(128);
328 let i8x16_ty = i8_ty.vec_type(16);
329 let i16x8_ty = i16_ty.vec_type(8);
330 let i32x4_ty = i32_ty.vec_type(4);
331 let i64x2_ty = i64_ty.vec_type(2);
332 let f32x4_ty = f32_ty.vec_type(4);
333 let f64x2_ty = f64_ty.vec_type(2);
334 let i32x8_ty = i32_ty.vec_type(8);
335
336 let ptr_ty = context.ptr_type(AddressSpace::default());
337
338 let i1_zero = i1_ty.const_int(0, false);
339 let i8_zero = i8_ty.const_int(0, false);
340 let i32_zero = i32_ty.const_int(0, false);
341 let i64_zero = i64_ty.const_int(0, false);
342 let i128_zero = i128_ty.const_int(0, false);
343 let isize_zero = isize_ty.const_int(0, false);
344 let f32_zero = f32_ty.const_float(0.0);
345 let f64_zero = f64_ty.const_float(0.0);
346 let f32x4_zero = f32x4_ty.const_zero();
347 let f64x2_zero = f64x2_ty.const_zero();
348 let i32_consts = [
349 i32_ty.const_int(0, false),
350 i32_ty.const_int(1, false),
351 i32_ty.const_int(2, false),
352 i32_ty.const_int(3, false),
353 i32_ty.const_int(4, false),
354 i32_ty.const_int(5, false),
355 i32_ty.const_int(6, false),
356 i32_ty.const_int(7, false),
357 i32_ty.const_int(8, false),
358 i32_ty.const_int(9, false),
359 i32_ty.const_int(10, false),
360 i32_ty.const_int(11, false),
361 i32_ty.const_int(12, false),
362 i32_ty.const_int(13, false),
363 i32_ty.const_int(14, false),
364 i32_ty.const_int(15, false),
365 ];
366
367 let md_ty = context.metadata_type();
368
369 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
370 let i1_ty_basic = i1_ty.as_basic_type_enum();
371
372 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
373 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
374 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
375 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
376 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
377 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
378 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
379 let i32x4_ty_basic_md: BasicMetadataTypeEnum = i32x4_ty.into();
380 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
381 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
382 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
383
384 let ctx_ptr_ty = ptr_ty;
385 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
386 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
387
388 let sigindex_ty = i32_ty;
389
390 let anyfunc_ty = context.struct_type(
391 &[i8_ptr_ty_basic, sigindex_ty.into(), ctx_ptr_ty_basic],
392 false,
393 );
394 let funcref_ty = ptr_ty;
395 let anyref_ty = ptr_ty;
396 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
397
398 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
399 let ret_i8x16_take_i8x16_i8x16 =
400 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
401 let ret_i8x16_take_i8x16_i8x16_i8x16 = i8x16_ty.fn_type(
402 &[i8x16_ty_basic_md, i8x16_ty_basic_md, i8x16_ty_basic_md],
403 false,
404 );
405 let ret_i16x8_take_i16x8_i16x8 =
406 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
407
408 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
409 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
410
411 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
412 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
413
414 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
415 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
416 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
417 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
418
419 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
420 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
421 let ret_f32x4_take_f32x4_f32x4 =
422 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
423 let ret_f64x2_take_f64x2_f64x2 =
424 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
425 let ret_f32x4_take_f32x4_f32x4_f32x4_md_md = f32x4_ty.fn_type(
426 &[
427 f32x4_ty_basic_md,
428 f32x4_ty_basic_md,
429 f32x4_ty_basic_md,
430 md_ty_basic_md,
431 md_ty_basic_md,
432 ],
433 false,
434 );
435 let ret_f64x2_take_f64x2_f64x2_f64x2_md_md = f64x2_ty.fn_type(
436 &[
437 f64x2_ty_basic_md,
438 f64x2_ty_basic_md,
439 f64x2_ty_basic_md,
440 md_ty_basic_md,
441 md_ty_basic_md,
442 ],
443 false,
444 );
445
446 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
447 let ret_f32_take_f64_md_md =
448 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
449
450 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
451
452 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
453 &[
454 f32_ty_basic_md,
455 f32_ty_basic_md,
456 md_ty_basic_md,
457 md_ty_basic_md,
458 ],
459 false,
460 );
461 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
462 &[
463 f64_ty_basic_md,
464 f64_ty_basic_md,
465 md_ty_basic_md,
466 md_ty_basic_md,
467 ],
468 false,
469 );
470 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
471 &[
472 f32x4_ty_basic_md,
473 f32x4_ty_basic_md,
474 md_ty_basic_md,
475 md_ty_basic_md,
476 ],
477 false,
478 );
479 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
480 &[
481 f64x2_ty_basic_md,
482 f64x2_ty_basic_md,
483 md_ty_basic_md,
484 md_ty_basic_md,
485 ],
486 false,
487 );
488
489 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
490 &[
491 f32_ty_basic_md,
492 f32_ty_basic_md,
493 md_ty_basic_md,
494 md_ty_basic_md,
495 ],
496 false,
497 );
498 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
499 &[
500 f64_ty_basic_md,
501 f64_ty_basic_md,
502 md_ty_basic_md,
503 md_ty_basic_md,
504 ],
505 false,
506 );
507 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
508 &[
509 f32x4_ty_basic_md,
510 f32x4_ty_basic_md,
511 md_ty_basic_md,
512 md_ty_basic_md,
513 ],
514 false,
515 );
516 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
517 &[
518 f64x2_ty_basic_md,
519 f64x2_ty_basic_md,
520 md_ty_basic_md,
521 md_ty_basic_md,
522 ],
523 false,
524 );
525 let ret_i32x4_take_f32x4 = i32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
526 let ret_i32x4_take_f32x4_i32x4_i8 =
527 i32x4_ty.fn_type(&[f32x4_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
528 let ret_i32x4_take_f64x2 = i32x4_ty.fn_type(&[f64x2_ty_basic_md], false);
529 let ret_i32x4_take_f64x2_i32x4_i8 =
530 i32x4_ty.fn_type(&[f64x2_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
531
532 let add_function_with_attrs =
533 |name: &str, ty: FunctionType<'ctx>, linkage: Option<Linkage>| -> FunctionValue<'ctx> {
534 let function = module.add_function(name, ty, linkage);
535 if is_riscv64 {
540 for (i, param) in ty.get_param_types().iter().enumerate() {
541 if param == &i32_ty_basic_md {
542 function.add_attribute(
543 AttributeLoc::Param(i as u32),
544 context.create_enum_attribute(
545 Attribute::get_named_enum_kind_id("signext"),
546 0,
547 ),
548 );
549 function.add_attribute(
550 AttributeLoc::Param(i as u32),
551 context.create_enum_attribute(
552 Attribute::get_named_enum_kind_id("noundef"),
553 0,
554 ),
555 );
556 }
557 }
558 }
559 function
560 };
561
562 let intrinsics = Self {
563 ctlz_i32: add_function_with_attrs("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
564 ctlz_i64: add_function_with_attrs("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
565
566 cttz_i32: add_function_with_attrs("llvm.cttz.i32", ret_i32_take_i32_i1, None),
567 cttz_i64: add_function_with_attrs("llvm.cttz.i64", ret_i64_take_i64_i1, None),
568
569 ctpop_i32: add_function_with_attrs("llvm.ctpop.i32", ret_i32_take_i32, None),
570 ctpop_i64: add_function_with_attrs("llvm.ctpop.i64", ret_i64_take_i64, None),
571 ctpop_i8x16: add_function_with_attrs("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
572
573 fp_rounding_md: context.metadata_string("round.tonearest").into(),
574 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
575
576 fp_ogt_md: context.metadata_string("ogt").into(),
577 fp_olt_md: context.metadata_string("olt").into(),
578 fp_uno_md: context.metadata_string("uno").into(),
579
580 sqrt_f32: add_function_with_attrs("llvm.sqrt.f32", ret_f32_take_f32, None),
581 sqrt_f64: add_function_with_attrs("llvm.sqrt.f64", ret_f64_take_f64, None),
582 sqrt_f32x4: add_function_with_attrs("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
583 sqrt_f64x2: add_function_with_attrs("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
584
585 ceil_f32: add_function_with_attrs("llvm.ceil.f32", ret_f32_take_f32, None),
586 ceil_f64: add_function_with_attrs("llvm.ceil.f64", ret_f64_take_f64, None),
587 ceil_f32x4: add_function_with_attrs("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
588 ceil_f64x2: add_function_with_attrs("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
589
590 floor_f32: add_function_with_attrs("llvm.floor.f32", ret_f32_take_f32, None),
591 floor_f64: add_function_with_attrs("llvm.floor.f64", ret_f64_take_f64, None),
592 floor_f32x4: add_function_with_attrs("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
593 floor_f64x2: add_function_with_attrs("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
594
595 trunc_f32: add_function_with_attrs("llvm.trunc.f32", ret_f32_take_f32, None),
596 trunc_f64: add_function_with_attrs("llvm.trunc.f64", ret_f64_take_f64, None),
597 trunc_f32x4: add_function_with_attrs("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
598 trunc_f64x2: add_function_with_attrs("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
599
600 nearbyint_f32: add_function_with_attrs("llvm.nearbyint.f32", ret_f32_take_f32, None),
601 nearbyint_f64: add_function_with_attrs("llvm.nearbyint.f64", ret_f64_take_f64, None),
602 nearbyint_f32x4: add_function_with_attrs(
603 "llvm.nearbyint.v4f32",
604 ret_f32x4_take_f32x4,
605 None,
606 ),
607 nearbyint_f64x2: add_function_with_attrs(
608 "llvm.nearbyint.v2f64",
609 ret_f64x2_take_f64x2,
610 None,
611 ),
612
613 add_f32: add_function_with_attrs(
614 "llvm.experimental.constrained.fadd.f32",
615 ret_f32_take_f32_f32_md_md,
616 None,
617 ),
618 add_f64: add_function_with_attrs(
619 "llvm.experimental.constrained.fadd.f64",
620 ret_f64_take_f64_f64_md_md,
621 None,
622 ),
623 add_f32x4: add_function_with_attrs(
624 "llvm.experimental.constrained.fadd.v4f32",
625 ret_f32x4_take_f32x4_f32x4_md_md,
626 None,
627 ),
628 add_f64x2: add_function_with_attrs(
629 "llvm.experimental.constrained.fadd.v2f64",
630 ret_f64x2_take_f64x2_f64x2_md_md,
631 None,
632 ),
633
634 sub_f32: add_function_with_attrs(
635 "llvm.experimental.constrained.fsub.f32",
636 ret_f32_take_f32_f32_md_md,
637 None,
638 ),
639 sub_f64: add_function_with_attrs(
640 "llvm.experimental.constrained.fsub.f64",
641 ret_f64_take_f64_f64_md_md,
642 None,
643 ),
644 sub_f32x4: add_function_with_attrs(
645 "llvm.experimental.constrained.fsub.v4f32",
646 ret_f32x4_take_f32x4_f32x4_md_md,
647 None,
648 ),
649 sub_f64x2: add_function_with_attrs(
650 "llvm.experimental.constrained.fsub.v2f64",
651 ret_f64x2_take_f64x2_f64x2_md_md,
652 None,
653 ),
654
655 mul_f32: add_function_with_attrs(
656 "llvm.experimental.constrained.fmul.f32",
657 ret_f32_take_f32_f32_md_md,
658 None,
659 ),
660 mul_f64: add_function_with_attrs(
661 "llvm.experimental.constrained.fmul.f64",
662 ret_f64_take_f64_f64_md_md,
663 None,
664 ),
665 mul_f32x4: add_function_with_attrs(
666 "llvm.experimental.constrained.fmul.v4f32",
667 ret_f32x4_take_f32x4_f32x4_md_md,
668 None,
669 ),
670 mul_f64x2: add_function_with_attrs(
671 "llvm.experimental.constrained.fmul.v2f64",
672 ret_f64x2_take_f64x2_f64x2_md_md,
673 None,
674 ),
675 muladd_f32x4: add_function_with_attrs(
676 "llvm.experimental.constrained.fmuladd.v4f32",
677 ret_f32x4_take_f32x4_f32x4_f32x4_md_md,
678 None,
679 ),
680 muladd_f64x2: add_function_with_attrs(
681 "llvm.experimental.constrained.fmuladd.v2f64",
682 ret_f64x2_take_f64x2_f64x2_f64x2_md_md,
683 None,
684 ),
685
686 div_f32: add_function_with_attrs(
687 "llvm.experimental.constrained.fdiv.f32",
688 ret_f32_take_f32_f32_md_md,
689 None,
690 ),
691 div_f64: add_function_with_attrs(
692 "llvm.experimental.constrained.fdiv.f64",
693 ret_f64_take_f64_f64_md_md,
694 None,
695 ),
696 div_f32x4: add_function_with_attrs(
697 "llvm.experimental.constrained.fdiv.v4f32",
698 ret_f32x4_take_f32x4_f32x4_md_md,
699 None,
700 ),
701 div_f64x2: add_function_with_attrs(
702 "llvm.experimental.constrained.fdiv.v2f64",
703 ret_f64x2_take_f64x2_f64x2_md_md,
704 None,
705 ),
706
707 cmp_f32: add_function_with_attrs(
708 "llvm.experimental.constrained.fcmp.f32",
709 ret_i1_take_f32_f32_md_md,
710 None,
711 ),
712 cmp_f64: add_function_with_attrs(
713 "llvm.experimental.constrained.fcmp.f64",
714 ret_i1_take_f64_f64_md_md,
715 None,
716 ),
717 cmp_f32x4: add_function_with_attrs(
718 "llvm.experimental.constrained.fcmp.v4f32",
719 ret_i1x4_take_f32x4_f32x4_md_md,
720 None,
721 ),
722 cmp_f64x2: add_function_with_attrs(
723 "llvm.experimental.constrained.fcmp.v2f64",
724 ret_i1x2_take_f64x2_f64x2_md_md,
725 None,
726 ),
727
728 minimum_f32: add_function_with_attrs("llvm.minimum.f32", ret_f32_take_f32_f32, None),
729 minimum_f64: add_function_with_attrs("llvm.minimum.f64", ret_f64_take_f64_f64, None),
730 minimum_f32x4: add_function_with_attrs(
731 "llvm.minimum.v4f32",
732 ret_f32x4_take_f32x4_f32x4,
733 None,
734 ),
735 minimum_f64x2: add_function_with_attrs(
736 "llvm.minimum.v2f64",
737 ret_f64x2_take_f64x2_f64x2,
738 None,
739 ),
740
741 maximum_f32: add_function_with_attrs("llvm.maximum.f32", ret_f32_take_f32_f32, None),
742 maximum_f64: add_function_with_attrs("llvm.maximum.f64", ret_f64_take_f64_f64, None),
743 maximum_f32x4: add_function_with_attrs(
744 "llvm.maximum.v4f32",
745 ret_f32x4_take_f32x4_f32x4,
746 None,
747 ),
748 maximum_f64x2: add_function_with_attrs(
749 "llvm.maximum.v2f64",
750 ret_f64x2_take_f64x2_f64x2,
751 None,
752 ),
753
754 fpext_f32: add_function_with_attrs(
755 "llvm.experimental.constrained.fpext.f64.f32",
756 ret_f64_take_f32_md,
757 None,
758 ),
759 fptrunc_f64: add_function_with_attrs(
760 "llvm.experimental.constrained.fptrunc.f32.f64",
761 ret_f32_take_f64_md_md,
762 None,
763 ),
764
765 fabs_f32: add_function_with_attrs("llvm.fabs.f32", ret_f32_take_f32, None),
766 fabs_f64: add_function_with_attrs("llvm.fabs.f64", ret_f64_take_f64, None),
767 fabs_f32x4: add_function_with_attrs("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
768 fabs_f64x2: add_function_with_attrs("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
769
770 copysign_f32: add_function_with_attrs("llvm.copysign.f32", ret_f32_take_f32_f32, None),
771 copysign_f64: add_function_with_attrs("llvm.copysign.f64", ret_f64_take_f64_f64, None),
772 copysign_f32x4: add_function_with_attrs(
773 "llvm.copysign.v4f32",
774 ret_f32x4_take_f32x4_f32x4,
775 None,
776 ),
777 copysign_f64x2: add_function_with_attrs(
778 "llvm.copysign.v2f64",
779 ret_f64x2_take_f64x2_f64x2,
780 None,
781 ),
782
783 sadd_sat_i8x16: add_function_with_attrs(
784 "llvm.sadd.sat.v16i8",
785 ret_i8x16_take_i8x16_i8x16,
786 None,
787 ),
788 sadd_sat_i16x8: add_function_with_attrs(
789 "llvm.sadd.sat.v8i16",
790 ret_i16x8_take_i16x8_i16x8,
791 None,
792 ),
793 uadd_sat_i8x16: add_function_with_attrs(
794 "llvm.uadd.sat.v16i8",
795 ret_i8x16_take_i8x16_i8x16,
796 None,
797 ),
798 uadd_sat_i16x8: add_function_with_attrs(
799 "llvm.uadd.sat.v8i16",
800 ret_i16x8_take_i16x8_i16x8,
801 None,
802 ),
803
804 ssub_sat_i8x16: add_function_with_attrs(
805 "llvm.ssub.sat.v16i8",
806 ret_i8x16_take_i8x16_i8x16,
807 None,
808 ),
809 ssub_sat_i16x8: add_function_with_attrs(
810 "llvm.ssub.sat.v8i16",
811 ret_i16x8_take_i16x8_i16x8,
812 None,
813 ),
814 usub_sat_i8x16: add_function_with_attrs(
815 "llvm.usub.sat.v16i8",
816 ret_i8x16_take_i8x16_i8x16,
817 None,
818 ),
819 usub_sat_i16x8: add_function_with_attrs(
820 "llvm.usub.sat.v8i16",
821 ret_i16x8_take_i16x8_i16x8,
822 None,
823 ),
824
825 expect_i1: add_function_with_attrs("llvm.expect.i1", ret_i1_take_i1_i1, None),
826 trap: add_function_with_attrs("llvm.trap", void_ty.fn_type(&[], false), None),
827 debug_trap: add_function_with_attrs(
828 "llvm.debugtrap",
829 void_ty.fn_type(&[], false),
830 None,
831 ),
832 personality: add_function_with_attrs(
833 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
834 "__gxx_personality_v0"
837 } else {
838 "wasmer_eh_personality"
839 },
840 i32_ty.fn_type(
841 &[
842 i32_ty.into(),
843 i32_ty.into(),
844 i64_ty.into(),
845 ptr_ty.into(),
846 ptr_ty.into(),
847 ],
848 false,
849 ),
850 None,
851 ),
852 personality2: add_function_with_attrs(
853 "wasmer_eh_personality2",
854 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
855 None,
856 ),
857 readonly: context
858 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
859 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
860 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
861 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
862 chkstk: add_function_with_attrs("__chkstk", void_ty.fn_type(&[], false), None),
863 void_ty,
864 i1_ty,
865 i2_ty,
866 i4_ty,
867 i8_ty,
868 i16_ty,
869 i32_ty,
870 i64_ty,
871 i128_ty,
872 isize_ty,
873 f32_ty,
874 f64_ty,
875
876 i1x128_ty,
877 i8x16_ty,
878 i16x8_ty,
879 i32x4_ty,
880 i64x2_ty,
881 f32x4_ty,
882 f64x2_ty,
883 i32x8_ty,
884
885 anyfunc_ty,
886 i1_zero,
887 i8_zero,
888 i32_zero,
889 i64_zero,
890 i128_zero,
891 isize_zero,
892 f32_zero,
893 f64_zero,
894 f32x4_zero,
895 f64x2_zero,
896 i32_consts,
897
898 trap_unreachable: i32_ty
899 .const_int(TrapCode::UnreachableCodeReached as _, false)
900 .as_basic_value_enum(),
901 trap_call_indirect_null: i32_ty
902 .const_int(TrapCode::IndirectCallToNull as _, false)
903 .as_basic_value_enum(),
904 trap_call_indirect_sig: i32_ty
905 .const_int(TrapCode::BadSignature as _, false)
906 .as_basic_value_enum(),
907 trap_memory_oob: i32_ty
908 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
909 .as_basic_value_enum(),
910 trap_illegal_arithmetic: i32_ty
911 .const_int(TrapCode::IntegerOverflow as _, false)
912 .as_basic_value_enum(),
913 trap_integer_division_by_zero: i32_ty
914 .const_int(TrapCode::IntegerDivisionByZero as _, false)
915 .as_basic_value_enum(),
916 trap_bad_conversion_to_integer: i32_ty
917 .const_int(TrapCode::BadConversionToInteger as _, false)
918 .as_basic_value_enum(),
919 trap_unaligned_atomic: i32_ty
920 .const_int(TrapCode::UnalignedAtomic as _, false)
921 .as_basic_value_enum(),
922 trap_table_access_oob: i32_ty
923 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
924 .as_basic_value_enum(),
925
926 experimental_stackmap: module.add_function(
930 "llvm.experimental.stackmap",
931 void_ty.fn_type(
932 &[
933 i64_ty_basic_md, i32_ty_basic_md, ],
936 true,
937 ),
938 None,
939 ),
940
941 table_copy: add_function_with_attrs(
943 "wasmer_vm_table_copy",
944 void_ty.fn_type(
945 &[
946 ctx_ptr_ty_basic_md,
947 i32_ty_basic_md,
948 i32_ty_basic_md,
949 i32_ty_basic_md,
950 i32_ty_basic_md,
951 i32_ty_basic_md,
952 ],
953 false,
954 ),
955 None,
956 ),
957 table_init: add_function_with_attrs(
958 "wasmer_vm_table_init",
959 void_ty.fn_type(
960 &[
961 ctx_ptr_ty_basic_md,
962 i32_ty_basic_md,
963 i32_ty_basic_md,
964 i32_ty_basic_md,
965 i32_ty_basic_md,
966 i32_ty_basic_md,
967 ],
968 false,
969 ),
970 None,
971 ),
972 table_fill: add_function_with_attrs(
973 "wasmer_vm_table_fill",
974 void_ty.fn_type(
975 &[
976 ctx_ptr_ty_basic_md,
977 i32_ty_basic_md,
978 i32_ty_basic_md,
979 anyref_ty_basic_md,
980 i32_ty_basic_md,
981 ],
982 false,
983 ),
984 None,
985 ),
986 table_size: add_function_with_attrs(
987 "wasmer_vm_table_size",
988 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
989 None,
990 ),
991 imported_table_size: add_function_with_attrs(
992 "wasmer_vm_imported_table_size",
993 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
994 None,
995 ),
996 table_get: add_function_with_attrs(
997 "wasmer_vm_table_get",
998 anyref_ty.fn_type(
999 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1000 false,
1001 ),
1002 None,
1003 ),
1004 imported_table_get: add_function_with_attrs(
1005 "wasmer_vm_imported_table_get",
1006 anyref_ty.fn_type(
1007 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1008 false,
1009 ),
1010 None,
1011 ),
1012 table_set: add_function_with_attrs(
1013 "wasmer_vm_table_set",
1014 void_ty.fn_type(
1015 &[
1016 ctx_ptr_ty_basic_md,
1017 i32_ty_basic_md,
1018 i32_ty_basic_md,
1019 anyref_ty_basic_md,
1020 ],
1021 false,
1022 ),
1023 None,
1024 ),
1025 imported_table_set: add_function_with_attrs(
1026 "wasmer_vm_imported_table_set",
1027 void_ty.fn_type(
1028 &[
1029 ctx_ptr_ty_basic_md,
1030 i32_ty_basic_md,
1031 i32_ty_basic_md,
1032 anyref_ty_basic_md,
1033 ],
1034 false,
1035 ),
1036 None,
1037 ),
1038 table_grow: add_function_with_attrs(
1039 "wasmer_vm_table_grow",
1040 i32_ty.fn_type(
1041 &[
1042 ctx_ptr_ty_basic_md,
1043 anyref_ty_basic_md,
1044 i32_ty_basic_md,
1045 i32_ty_basic_md,
1046 ],
1047 false,
1048 ),
1049 None,
1050 ),
1051 imported_table_grow: add_function_with_attrs(
1052 "wasmer_vm_imported_table_grow",
1053 i32_ty.fn_type(
1054 &[
1055 ctx_ptr_ty_basic_md,
1056 anyref_ty_basic_md,
1057 i32_ty_basic_md,
1058 i32_ty_basic_md,
1059 ],
1060 false,
1061 ),
1062 None,
1063 ),
1064 memory_init: add_function_with_attrs(
1065 "wasmer_vm_memory32_init",
1066 void_ty.fn_type(
1067 &[
1068 ctx_ptr_ty_basic_md,
1069 i32_ty_basic_md,
1070 i32_ty_basic_md,
1071 i32_ty_basic_md,
1072 i32_ty_basic_md,
1073 i32_ty_basic_md,
1074 ],
1075 false,
1076 ),
1077 None,
1078 ),
1079 memory_copy: add_function_with_attrs(
1080 "wasmer_vm_memory32_copy",
1081 void_ty.fn_type(
1082 &[
1083 ctx_ptr_ty_basic_md,
1084 i32_ty_basic_md,
1085 i32_ty_basic_md,
1086 i32_ty_basic_md,
1087 i32_ty_basic_md,
1088 ],
1089 false,
1090 ),
1091 None,
1092 ),
1093 imported_memory_copy: add_function_with_attrs(
1094 "wasmer_vm_imported_memory32_copy",
1095 void_ty.fn_type(
1096 &[
1097 ctx_ptr_ty_basic_md,
1098 i32_ty_basic_md,
1099 i32_ty_basic_md,
1100 i32_ty_basic_md,
1101 i32_ty_basic_md,
1102 ],
1103 false,
1104 ),
1105 None,
1106 ),
1107 memory_fill: add_function_with_attrs(
1108 "wasmer_vm_memory32_fill",
1109 void_ty.fn_type(
1110 &[
1111 ctx_ptr_ty_basic_md,
1112 i32_ty_basic_md,
1113 i32_ty_basic_md,
1114 i32_ty_basic_md,
1115 i32_ty_basic_md,
1116 ],
1117 false,
1118 ),
1119 None,
1120 ),
1121 imported_memory_fill: add_function_with_attrs(
1122 "wasmer_vm_imported_memory32_fill",
1123 void_ty.fn_type(
1124 &[
1125 ctx_ptr_ty_basic_md,
1126 i32_ty_basic_md,
1127 i32_ty_basic_md,
1128 i32_ty_basic_md,
1129 i32_ty_basic_md,
1130 ],
1131 false,
1132 ),
1133 None,
1134 ),
1135 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1136 memory_grow_ty: i32_ty.fn_type(
1137 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1138 false,
1139 ),
1140 data_drop: add_function_with_attrs(
1141 "wasmer_vm_data_drop",
1142 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1143 None,
1144 ),
1145 func_ref: add_function_with_attrs(
1146 "wasmer_vm_func_ref",
1147 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1148 None,
1149 ),
1150 elem_drop: add_function_with_attrs(
1151 "wasmer_vm_elem_drop",
1152 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1153 None,
1154 ),
1155 throw_trap: add_function_with_attrs(
1156 "wasmer_vm_raise_trap",
1157 void_ty.fn_type(&[i32_ty_basic_md], false),
1158 None,
1159 ),
1160
1161 throw: add_function_with_attrs(
1162 "wasmer_vm_throw",
1163 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1164 None,
1165 ),
1166 alloc_exception: add_function_with_attrs(
1167 "wasmer_vm_alloc_exception",
1168 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1169 None,
1170 ),
1171 read_exnref: add_function_with_attrs(
1172 "wasmer_vm_read_exnref",
1173 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1174 None,
1175 ),
1176 exception_into_exnref: add_function_with_attrs(
1177 "wasmer_vm_exception_into_exnref",
1178 i32_ty.fn_type(&[ptr_ty.into()], false),
1179 None,
1180 ),
1181 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1182
1183 debug_ptr: add_function_with_attrs(
1184 "wasmer_vm_dbg_usize",
1185 void_ty.fn_type(&[ptr_ty.into()], false),
1186 None,
1187 ),
1188 debug_str: add_function_with_attrs(
1189 "wasmer_vm_dbg_str",
1190 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1191 None,
1192 ),
1193 memory_wait32: add_function_with_attrs(
1194 "wasmer_vm_memory32_atomic_wait32",
1195 i32_ty.fn_type(
1196 &[
1197 ctx_ptr_ty_basic_md,
1198 i32_ty_basic_md,
1199 i32_ty_basic_md,
1200 i32_ty_basic_md,
1201 i64_ty_basic_md,
1202 ],
1203 false,
1204 ),
1205 None,
1206 ),
1207 memory_wait32_ty: i32_ty.fn_type(
1208 &[
1209 ctx_ptr_ty_basic_md,
1210 i32_ty_basic_md,
1211 i32_ty_basic_md,
1212 i32_ty_basic_md,
1213 i64_ty_basic_md,
1214 ],
1215 false,
1216 ),
1217 imported_memory_wait32: add_function_with_attrs(
1218 "wasmer_vm_imported_memory32_atomic_wait32",
1219 i32_ty.fn_type(
1220 &[
1221 ctx_ptr_ty_basic_md,
1222 i32_ty_basic_md,
1223 i32_ty_basic_md,
1224 i32_ty_basic_md,
1225 i64_ty_basic_md,
1226 ],
1227 false,
1228 ),
1229 None,
1230 ),
1231 memory_wait64: add_function_with_attrs(
1232 "wasmer_vm_memory32_atomic_wait64",
1233 i32_ty.fn_type(
1234 &[
1235 ctx_ptr_ty_basic_md,
1236 i32_ty_basic_md,
1237 i32_ty_basic_md,
1238 i64_ty_basic_md,
1239 i64_ty_basic_md,
1240 ],
1241 false,
1242 ),
1243 None,
1244 ),
1245 memory_wait64_ty: i32_ty.fn_type(
1246 &[
1247 ctx_ptr_ty_basic_md,
1248 i32_ty_basic_md,
1249 i32_ty_basic_md,
1250 i64_ty_basic_md,
1251 i64_ty_basic_md,
1252 ],
1253 false,
1254 ),
1255 imported_memory_wait64: add_function_with_attrs(
1256 "wasmer_vm_imported_memory32_atomic_wait64",
1257 i32_ty.fn_type(
1258 &[
1259 ctx_ptr_ty_basic_md,
1260 i32_ty_basic_md,
1261 i32_ty_basic_md,
1262 i64_ty_basic_md,
1263 i64_ty_basic_md,
1264 ],
1265 false,
1266 ),
1267 None,
1268 ),
1269 memory_notify: add_function_with_attrs(
1270 "wasmer_vm_memory32_atomic_notify",
1271 i32_ty.fn_type(
1272 &[
1273 ctx_ptr_ty_basic_md,
1274 i32_ty_basic_md,
1275 i32_ty_basic_md,
1276 i32_ty_basic_md,
1277 ],
1278 false,
1279 ),
1280 None,
1281 ),
1282 memory_notify_ty: i32_ty.fn_type(
1283 &[
1284 ctx_ptr_ty_basic_md,
1285 i32_ty_basic_md,
1286 i32_ty_basic_md,
1287 i32_ty_basic_md,
1288 ],
1289 false,
1290 ),
1291 imported_memory_notify: add_function_with_attrs(
1292 "wasmer_vm_imported_memory32_atomic_notify",
1293 i32_ty.fn_type(
1294 &[
1295 ctx_ptr_ty_basic_md,
1296 i32_ty_basic_md,
1297 i32_ty_basic_md,
1298 i32_ty_basic_md,
1299 ],
1300 false,
1301 ),
1302 None,
1303 ),
1304
1305 vmfunction_import_ty: context.struct_type(
1306 &[
1307 i8_ptr_ty_basic,
1308 i8_ptr_ty_basic,
1309 i8_ptr_ty_basic,
1310 i1_ty_basic,
1311 ],
1312 false,
1313 ),
1314 vmfunction_import_body_element: 0,
1315 vmfunction_import_vmctx_element: 1,
1316 vmfunction_import_include_m0_param_element: 3,
1317 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1318 vmmemory_definition_base_element: 0,
1319 vmmemory_definition_current_length_element: 1,
1320
1321 ptr_ty,
1322
1323 x86_64: X86_64Intrinsics {
1324 pshufb128: add_function_with_attrs(
1325 "llvm.x86.ssse3.pshuf.b.128",
1326 ret_i8x16_take_i8x16_i8x16,
1327 None,
1328 ),
1329 pmaddubsw128: add_function_with_attrs(
1330 "llvm.x86.ssse3.pmadd.ub.sw.128",
1331 i16x8_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false),
1332 None,
1333 ),
1334 pmaddwd128: add_function_with_attrs(
1335 "llvm.x86.sse2.pmadd.wd",
1336 i32x4_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false),
1337 None,
1338 ),
1339 pmulhrsw128: add_function_with_attrs(
1340 "llvm.x86.ssse3.pmul.hr.sw.128",
1341 ret_i16x8_take_i16x8_i16x8,
1342 None,
1343 ),
1344 pblendvb: add_function_with_attrs(
1345 "llvm.x86.sse41.pblendvb",
1346 ret_i8x16_take_i8x16_i8x16_i8x16,
1347 None,
1348 ),
1349 min_ps: add_function_with_attrs(
1350 "llvm.x86.sse.min.ps",
1351 ret_f32x4_take_f32x4_f32x4,
1352 None,
1353 ),
1354 min_pd: add_function_with_attrs(
1355 "llvm.x86.sse2.min.pd",
1356 ret_f64x2_take_f64x2_f64x2,
1357 None,
1358 ),
1359 max_ps: add_function_with_attrs(
1360 "llvm.x86.sse.max.ps",
1361 ret_f32x4_take_f32x4_f32x4,
1362 None,
1363 ),
1364 max_pd: add_function_with_attrs(
1365 "llvm.x86.sse2.max.pd",
1366 ret_f64x2_take_f64x2_f64x2,
1367 None,
1368 ),
1369 cvttps2dq: add_function_with_attrs(
1370 "llvm.x86.sse2.cvttps2dq",
1371 ret_i32x4_take_f32x4,
1372 None,
1373 ),
1374 cvtps2udq128: add_function_with_attrs(
1375 "llvm.x86.avx512.mask.cvtps2udq.128",
1376 ret_i32x4_take_f32x4_i32x4_i8,
1377 None,
1378 ),
1379 cvtpd2dq: add_function_with_attrs(
1380 "llvm.x86.sse2.cvtpd2dq",
1381 ret_i32x4_take_f64x2,
1382 None,
1383 ),
1384 cvtpd2udq128: add_function_with_attrs(
1385 "llvm.x86.avx512.mask.cvtpd2udq.128",
1386 ret_i32x4_take_f64x2_i32x4_i8,
1387 None,
1388 ),
1389 },
1390 };
1391
1392 let noreturn =
1393 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1394 intrinsics
1395 .throw_trap
1396 .add_attribute(AttributeLoc::Function, noreturn);
1397 intrinsics
1398 }
1399}
1400
1401#[derive(Clone, Copy)]
1402pub enum MemoryCache<'ctx> {
1403 Dynamic {
1405 ptr_to_base_ptr: PointerValue<'ctx>,
1406 ptr_to_current_length: PointerValue<'ctx>,
1407 },
1408 Static { base_ptr: PointerValue<'ctx> },
1410}
1411
1412#[derive(Clone)]
1413struct TableCache<'ctx> {
1414 ptr_to_base_ptr: PointerValue<'ctx>,
1415 ptr_to_bounds: PointerValue<'ctx>,
1416}
1417
1418#[derive(Clone, Copy)]
1419pub enum GlobalCache<'ctx> {
1420 Mut {
1421 ptr_to_value: PointerValue<'ctx>,
1422 value_type: BasicTypeEnum<'ctx>,
1423 },
1424 Const {
1425 value: BasicValueEnum<'ctx>,
1426 },
1427}
1428
1429#[derive(Clone)]
1430pub struct FunctionCache<'ctx> {
1431 pub func: PointerValue<'ctx>,
1432 pub llvm_func_type: FunctionType<'ctx>,
1433 pub vmctx: BasicValueEnum<'ctx>,
1434 pub imported_include_m0_param: Option<IntValue<'ctx>>,
1435 pub attrs: Vec<(Attribute, AttributeLoc)>,
1436}
1437
1438pub struct CtxType<'ctx, 'a> {
1439 ctx_ptr_value: PointerValue<'ctx>,
1440 m0: Option<PointerValue<'ctx>>,
1441
1442 wasm_module: &'a WasmerCompilerModule,
1443 cache_builder: &'a Builder<'ctx>,
1444 abi: &'a dyn Abi,
1445
1446 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1447 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1448 cached_sigindices: HashMap<SignatureIndex, IntValue<'ctx>>,
1449 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1450 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1451 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1452
1453 offsets: VMOffsets,
1454}
1455
1456impl<'ctx, 'a> CtxType<'ctx, 'a> {
1457 pub fn new(
1458 wasm_module: &'a WasmerCompilerModule,
1459 func_value: &FunctionValue<'ctx>,
1460 cache_builder: &'a Builder<'ctx>,
1461 abi: &'a dyn Abi,
1462 pointer_width: u8,
1463 m0: Option<PointerValue<'ctx>>,
1464 ) -> CtxType<'ctx, 'a> {
1465 CtxType {
1466 m0,
1467 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1468
1469 wasm_module,
1470 cache_builder,
1471 abi,
1472
1473 cached_memories: HashMap::new(),
1474 cached_tables: HashMap::new(),
1475 cached_sigindices: HashMap::new(),
1476 cached_globals: HashMap::new(),
1477 cached_functions: HashMap::new(),
1478 cached_memory_op: HashMap::new(),
1479
1480 offsets: VMOffsets::new(pointer_width, wasm_module),
1481 }
1482 }
1483
1484 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1485 self.ctx_ptr_value.as_basic_value_enum()
1486 }
1487
1488 pub fn memory(
1489 &mut self,
1490 index: MemoryIndex,
1491 intrinsics: &Intrinsics<'ctx>,
1492 module: &Module<'ctx>,
1493 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1494 ) -> Result<MemoryCache<'ctx>, CompileError> {
1495 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1496 &mut self.cached_memories,
1497 self.wasm_module,
1498 self.ctx_ptr_value,
1499 &self.cache_builder,
1500 &self.offsets,
1501 );
1502 let memory_style = &memory_styles[index];
1503 match cached_memories.get(&index) {
1504 Some(r) => Ok(*r),
1505 None => {
1506 let memory_definition_ptr =
1507 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1508 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1509 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1510 unsafe {
1511 err!(cache_builder.build_gep(
1512 intrinsics.i8_ty,
1513 ctx_ptr_value,
1514 &[offset],
1515 ""
1516 ))
1517 }
1518 } else {
1519 let offset = offsets.vmctx_vmmemory_import(index);
1520 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1521 let memory_definition_ptr_ptr = unsafe {
1522 err!(cache_builder.build_gep(
1523 intrinsics.i8_ty,
1524 ctx_ptr_value,
1525 &[offset],
1526 ""
1527 ))
1528 };
1529 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1530 memory_definition_ptr_ptr,
1531 intrinsics.ptr_ty,
1532 "",
1533 ))
1534 .into_pointer_value();
1535 let memory_definition_ptr = err!(cache_builder.build_load(
1536 intrinsics.ptr_ty,
1537 memory_definition_ptr_ptr,
1538 ""
1539 ))
1540 .into_pointer_value();
1541 tbaa_label(
1542 module,
1543 intrinsics,
1544 format!("memory {} definition", index.as_u32()),
1545 memory_definition_ptr.as_instruction_value().unwrap(),
1546 );
1547 memory_definition_ptr
1548 };
1549 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1550 memory_definition_ptr,
1551 intrinsics.ptr_ty,
1552 "",
1553 ))
1554 .into_pointer_value();
1555 let base_ptr = err!(cache_builder.build_struct_gep(
1556 intrinsics.vmmemory_definition_ty,
1557 memory_definition_ptr,
1558 intrinsics.vmmemory_definition_base_element,
1559 "",
1560 ));
1561 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1562 let current_length_ptr = err!(cache_builder.build_struct_gep(
1563 intrinsics.vmmemory_definition_ty,
1564 memory_definition_ptr,
1565 intrinsics.vmmemory_definition_current_length_element,
1566 "",
1567 ));
1568 MemoryCache::Dynamic {
1569 ptr_to_base_ptr: base_ptr,
1570 ptr_to_current_length: current_length_ptr,
1571 }
1572 } else {
1573 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1574 .into_pointer_value();
1575 tbaa_label(
1576 module,
1577 intrinsics,
1578 format!("memory base_ptr {}", index.as_u32()),
1579 base_ptr.as_instruction_value().unwrap(),
1580 );
1581 MemoryCache::Static { base_ptr }
1582 };
1583
1584 self.cached_memories.insert(index, value);
1585 Ok(*self.cached_memories.get(&index).unwrap())
1586 }
1587 }
1588 }
1589
1590 fn build_table_prepare(
1591 table_index: TableIndex,
1592 intrinsics: &Intrinsics<'ctx>,
1593 module: &Module<'ctx>,
1594 wasm_module: &WasmerCompilerModule,
1595 ctx_ptr_value: PointerValue<'ctx>,
1596 offsets: &VMOffsets,
1597 builder: &Builder<'ctx>,
1598 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1599 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1600 let offset = intrinsics.i64_ty.const_int(
1601 offsets
1602 .vmctx_vmtable_definition_base(local_table_index)
1603 .into(),
1604 false,
1605 );
1606 let ptr_to_base_ptr =
1607 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1608 let ptr_to_base_ptr =
1609 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1610 .into_pointer_value();
1611 let offset = intrinsics.i64_ty.const_int(
1612 offsets
1613 .vmctx_vmtable_definition_current_elements(local_table_index)
1614 .into(),
1615 false,
1616 );
1617 let ptr_to_bounds =
1618 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1619 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1620 .into_pointer_value();
1621 Ok((ptr_to_base_ptr, ptr_to_bounds))
1622 } else {
1623 let offset = intrinsics.i64_ty.const_int(
1624 offsets.vmctx_vmtable_import_definition(table_index).into(),
1625 false,
1626 );
1627 let definition_ptr_ptr =
1628 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1629 let definition_ptr_ptr =
1630 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1631 .into_pointer_value();
1632 let definition_ptr =
1633 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1634 .into_pointer_value();
1635 tbaa_label(
1636 module,
1637 intrinsics,
1638 format!("table {} definition", table_index.as_u32()),
1639 definition_ptr.as_instruction_value().unwrap(),
1640 );
1641
1642 let offset = intrinsics
1643 .i64_ty
1644 .const_int(offsets.vmtable_definition_base().into(), false);
1645 let ptr_to_base_ptr =
1646 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1647 let ptr_to_base_ptr =
1648 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1649 .into_pointer_value();
1650 let offset = intrinsics
1651 .i64_ty
1652 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1653 let ptr_to_bounds =
1654 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1655 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1656 .into_pointer_value();
1657 Ok((ptr_to_base_ptr, ptr_to_bounds))
1658 }
1659 }
1660
1661 fn table_prepare(
1662 &mut self,
1663 table_index: TableIndex,
1664 intrinsics: &Intrinsics<'ctx>,
1665 module: &Module<'ctx>,
1666 body_builder: &Builder<'ctx>,
1667 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1668 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1669 &mut self.cached_tables,
1670 self.wasm_module,
1671 self.ctx_ptr_value,
1672 &self.cache_builder,
1673 &self.offsets,
1674 );
1675
1676 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1677 CompileError::Codegen(format!(
1678 "Table index out of bounds: {}",
1679 table_index.as_u32()
1680 ))
1681 })?;
1682
1683 if is_growable {
1686 Self::build_table_prepare(
1687 table_index,
1688 intrinsics,
1689 module,
1690 wasm_module,
1691 ctx_ptr_value,
1692 offsets,
1693 body_builder,
1694 )
1695 } else {
1696 let TableCache {
1697 ptr_to_base_ptr,
1698 ptr_to_bounds,
1699 } = match cached_tables.entry(table_index) {
1700 Entry::Occupied(entry) => entry.get().clone(),
1701 Entry::Vacant(entry) => {
1702 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1703 table_index,
1704 intrinsics,
1705 module,
1706 wasm_module,
1707 ctx_ptr_value,
1708 offsets,
1709 cache_builder,
1710 )?;
1711
1712 let v = TableCache {
1713 ptr_to_base_ptr,
1714 ptr_to_bounds,
1715 };
1716
1717 entry.insert(v.clone());
1718
1719 v
1720 }
1721 };
1722
1723 Ok((ptr_to_base_ptr, ptr_to_bounds))
1724 }
1725 }
1726
1727 pub fn table(
1728 &mut self,
1729 index: TableIndex,
1730 intrinsics: &Intrinsics<'ctx>,
1731 module: &Module<'ctx>,
1732 body_builder: &Builder<'ctx>,
1733 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1734 let (ptr_to_base_ptr, ptr_to_bounds) =
1735 self.table_prepare(index, intrinsics, module, body_builder)?;
1736
1737 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1739 &body_builder
1740 } else {
1741 &self.cache_builder
1742 };
1743
1744 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1745 .into_pointer_value();
1746 let bounds =
1747 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1748 tbaa_label(
1749 module,
1750 intrinsics,
1751 format!("table_base_ptr {}", index.index()),
1752 base_ptr.as_instruction_value().unwrap(),
1753 );
1754 tbaa_label(
1755 module,
1756 intrinsics,
1757 format!("table_bounds {}", index.index()),
1758 bounds.as_instruction_value().unwrap(),
1759 );
1760 Ok((base_ptr, bounds))
1761 }
1762
1763 pub fn dynamic_sigindex(
1764 &mut self,
1765 index: SignatureIndex,
1766 intrinsics: &Intrinsics<'ctx>,
1767 module: &Module<'ctx>,
1768 ) -> Result<IntValue<'ctx>, CompileError> {
1769 let (cached_sigindices, ctx_ptr_value, cache_builder, offsets) = (
1770 &mut self.cached_sigindices,
1771 self.ctx_ptr_value,
1772 &self.cache_builder,
1773 &self.offsets,
1774 );
1775
1776 match cached_sigindices.entry(index) {
1777 Entry::Occupied(entry) => Ok(*entry.get()),
1778 Entry::Vacant(entry) => {
1779 let byte_offset = intrinsics
1780 .i64_ty
1781 .const_int(offsets.vmctx_vmshared_signature_id(index).into(), false);
1782
1783 let sigindex_ptr = unsafe {
1784 err!(cache_builder.build_gep(
1785 intrinsics.i8_ty,
1786 ctx_ptr_value,
1787 &[byte_offset],
1788 "dynamic_sigindex",
1789 ))
1790 };
1791
1792 let sigindex_ptr =
1793 err!(cache_builder.build_bit_cast(sigindex_ptr, intrinsics.ptr_ty, ""))
1794 .into_pointer_value();
1795
1796 let sigindex =
1797 err!(cache_builder.build_load(intrinsics.i32_ty, sigindex_ptr, "sigindex"))
1798 .into_int_value();
1799 tbaa_label(
1800 module,
1801 intrinsics,
1802 format!("sigindex {}", index.as_u32()),
1803 sigindex.as_instruction_value().unwrap(),
1804 );
1805
1806 entry.insert(sigindex);
1807 Ok(sigindex)
1808 }
1809 }
1810 }
1811
1812 pub fn global(
1813 &mut self,
1814 index: GlobalIndex,
1815 intrinsics: &Intrinsics<'ctx>,
1816 module: &Module<'ctx>,
1817 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1818 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1819 &mut self.cached_globals,
1820 self.wasm_module,
1821 self.ctx_ptr_value,
1822 &self.cache_builder,
1823 &self.offsets,
1824 );
1825 match cached_globals.entry(index) {
1826 Entry::Occupied(entry) => Ok(entry.into_mut()),
1827 Entry::Vacant(entry) => {
1828 let global_type = wasm_module.globals[index];
1829 let global_value_type = global_type.ty;
1830
1831 let global_mutability = global_type.mutability;
1832 let offset = if let Some(local_global_index) = wasm_module.local_global_index(index)
1833 {
1834 offsets.vmctx_vmglobal_definition(local_global_index)
1835 } else {
1836 offsets.vmctx_vmglobal_import(index)
1837 };
1838 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1839 let global_ptr = {
1840 let global_ptr_ptr = unsafe {
1841 err!(cache_builder.build_gep(
1842 intrinsics.i8_ty,
1843 ctx_ptr_value,
1844 &[offset],
1845 ""
1846 ))
1847 };
1848 let global_ptr_ptr =
1849 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1850 .into_pointer_value();
1851 let global_ptr =
1852 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1853 .into_pointer_value();
1854 tbaa_label(
1855 module,
1856 intrinsics,
1857 format!("global_ptr {}", index.as_u32()),
1858 global_ptr.as_instruction_value().unwrap(),
1859 );
1860 global_ptr
1861 };
1862 let global_ptr =
1863 err!(cache_builder.build_bit_cast(global_ptr, intrinsics.ptr_ty, "",))
1864 .into_pointer_value();
1865
1866 let ret = entry.insert(match global_mutability {
1867 Mutability::Const => {
1868 let value = err!(cache_builder.build_load(
1869 type_to_llvm(intrinsics, global_value_type)?,
1870 global_ptr,
1871 "",
1872 ));
1873 tbaa_label(
1874 module,
1875 intrinsics,
1876 format!("global {}", index.as_u32()),
1877 value.as_instruction_value().unwrap(),
1878 );
1879 GlobalCache::Const { value }
1880 }
1881 Mutability::Var => GlobalCache::Mut {
1882 ptr_to_value: global_ptr,
1883 value_type: type_to_llvm(intrinsics, global_value_type)?,
1884 },
1885 });
1886
1887 Ok(ret)
1888 }
1889 }
1890 }
1891
1892 pub fn add_func(
1893 &mut self,
1894 function_index: FunctionIndex,
1895 func: PointerValue<'ctx>,
1896 llvm_func_type: FunctionType<'ctx>,
1897 vmctx: BasicValueEnum<'ctx>,
1898 attrs: &[(Attribute, AttributeLoc)],
1899 ) {
1900 match self.cached_functions.entry(function_index) {
1901 Entry::Occupied(_) => unreachable!("duplicate function"),
1902 Entry::Vacant(entry) => {
1903 entry.insert(FunctionCache {
1904 func,
1905 llvm_func_type,
1906 vmctx,
1907 imported_include_m0_param: None,
1908 attrs: attrs.to_vec(),
1909 });
1910 }
1911 }
1912 }
1913
1914 #[allow(clippy::too_many_arguments)]
1915 pub fn local_func(
1916 &mut self,
1917 _local_function_index: LocalFunctionIndex,
1918 function_index: FunctionIndex,
1919 intrinsics: &Intrinsics<'ctx>,
1920 module: &Module<'ctx>,
1921 context: &'ctx Context,
1922 func_type: &FuncType,
1923 function_name: &str,
1924 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1925 let (cached_functions, ctx_ptr_value, offsets) = (
1926 &mut self.cached_functions,
1927 &self.ctx_ptr_value,
1928 &self.offsets,
1929 );
1930 Ok(match cached_functions.entry(function_index) {
1931 Entry::Occupied(entry) => entry.into_mut(),
1932 Entry::Vacant(entry) => {
1933 debug_assert!(module.get_function(function_name).is_none());
1934 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1935 context,
1936 intrinsics,
1937 Some(offsets),
1938 func_type,
1939 self.m0.is_some(),
1940 )?;
1941 let func =
1942 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1943 for (attr, attr_loc) in &llvm_func_attrs {
1944 func.add_attribute(*attr_loc, *attr);
1945 }
1946 entry.insert(FunctionCache {
1947 func: func.as_global_value().as_pointer_value(),
1948 llvm_func_type,
1949 vmctx: ctx_ptr_value.as_basic_value_enum(),
1950 imported_include_m0_param: None,
1951 attrs: llvm_func_attrs,
1952 })
1953 }
1954 })
1955 }
1956
1957 pub fn imported_func(
1958 &mut self,
1959 function_index: FunctionIndex,
1960 intrinsics: &Intrinsics<'ctx>,
1961 context: &'ctx Context,
1962 func_type: &FuncType,
1963 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1964 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1965 &mut self.cached_functions,
1966 self.wasm_module,
1967 &self.ctx_ptr_value,
1968 &self.cache_builder,
1969 &self.offsets,
1970 );
1971 match cached_functions.entry(function_index) {
1972 Entry::Occupied(entry) => Ok(entry.into_mut()),
1973 Entry::Vacant(entry) => {
1974 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1975 context,
1976 intrinsics,
1977 Some(offsets),
1978 func_type,
1979 self.m0.is_some(),
1980 )?;
1981 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1982 let offset = offsets.vmctx_vmfunction_import(function_index);
1983 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1984 let vmfunction_import_ptr = unsafe {
1985 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1986 };
1987 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1988 vmfunction_import_ptr,
1989 intrinsics.ptr_ty,
1990 "",
1991 ))
1992 .into_pointer_value();
1993 vmfunction_import_ptr.set_name("vmfunction_import_ptr");
1994
1995 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1996 intrinsics.vmfunction_import_ty,
1997 vmfunction_import_ptr,
1998 intrinsics.vmfunction_import_body_element,
1999 "",
2000 ));
2001 body_ptr_ptr.set_name("body_ptr_ptr");
2002 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
2003 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
2004 .into_pointer_value();
2005 body_ptr.set_name("body_ptr");
2006 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
2007 intrinsics.vmfunction_import_ty,
2008 vmfunction_import_ptr,
2009 intrinsics.vmfunction_import_vmctx_element,
2010 "",
2011 ));
2012 vmctx_ptr_ptr.set_name("vmctx_ptr_ptr");
2013 let vmctx_ptr =
2014 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
2015 vmctx_ptr.set_name("vmctx_ptr");
2016 let include_m0_param_ptr = err!(cache_builder.build_struct_gep(
2017 intrinsics.vmfunction_import_ty,
2018 vmfunction_import_ptr,
2019 intrinsics.vmfunction_import_include_m0_param_element,
2020 "",
2021 ));
2022 include_m0_param_ptr.set_name("include_m0_param_ptr");
2023 let imported_include_m0_param =
2024 err!(cache_builder.build_load(intrinsics.i1_ty, include_m0_param_ptr, "",))
2025 .into_int_value();
2026 imported_include_m0_param.set_name("imported_include_m0_param");
2027
2028 Ok(entry.insert(FunctionCache {
2029 func: body_ptr,
2030 llvm_func_type,
2031 vmctx: vmctx_ptr,
2032 imported_include_m0_param: Some(imported_include_m0_param),
2033 attrs: llvm_func_attrs,
2034 }))
2035 }
2036 }
2037 }
2038
2039 pub fn memory_grow(
2040 &mut self,
2041 memory_index: MemoryIndex,
2042 intrinsics: &Intrinsics<'ctx>,
2043 ) -> Result<PointerValue<'ctx>, CompileError> {
2044 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2045 &mut self.cached_memory_op,
2046 &self.wasm_module,
2047 &self.offsets,
2048 &self.cache_builder,
2049 &self.ctx_ptr_value,
2050 );
2051 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
2052 Entry::Occupied(entry) => Ok(*entry.get()),
2053 Entry::Vacant(entry) => {
2054 let (grow_fn, grow_fn_ty) =
2055 if wasm_module.local_memory_index(memory_index).is_some() {
2056 (
2057 VMBuiltinFunctionIndex::get_memory32_grow_index(),
2058 intrinsics.ptr_ty,
2059 )
2060 } else {
2061 (
2062 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
2063 intrinsics.ptr_ty,
2064 )
2065 };
2066 let offset = offsets.vmctx_builtin_function(grow_fn);
2067 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2068 let grow_fn_ptr_ptr = unsafe {
2069 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2070 };
2071
2072 let grow_fn_ptr_ptr =
2073 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2074 .into_pointer_value();
2075 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
2076 .into_pointer_value();
2077
2078 entry.insert(val);
2079 Ok(val)
2080 }
2081 }
2082 }
2083
2084 pub fn memory_size(
2085 &mut self,
2086 memory_index: MemoryIndex,
2087 intrinsics: &Intrinsics<'ctx>,
2088 ) -> Result<PointerValue<'ctx>, CompileError> {
2089 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2090 &mut self.cached_memory_op,
2091 &self.wasm_module,
2092 &self.offsets,
2093 &self.cache_builder,
2094 &self.ctx_ptr_value,
2095 );
2096
2097 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
2098 Entry::Occupied(entry) => Ok(*entry.get()),
2099 Entry::Vacant(entry) => {
2100 let (size_fn, size_fn_ty) =
2101 if wasm_module.local_memory_index(memory_index).is_some() {
2102 (
2103 VMBuiltinFunctionIndex::get_memory32_size_index(),
2104 intrinsics.ptr_ty,
2105 )
2106 } else {
2107 (
2108 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
2109 intrinsics.ptr_ty,
2110 )
2111 };
2112 let offset = offsets.vmctx_builtin_function(size_fn);
2113 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2114 let size_fn_ptr_ptr = unsafe {
2115 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2116 };
2117
2118 let size_fn_ptr_ptr =
2119 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2120 .into_pointer_value();
2121
2122 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2123 .into_pointer_value();
2124 entry.insert(val);
2125 Ok(val)
2126 }
2127 }
2128 }
2129
2130 pub fn memory_wait32(
2131 &mut self,
2132 memory_index: MemoryIndex,
2133 intrinsics: &Intrinsics<'ctx>,
2134 ) -> Result<PointerValue<'ctx>, CompileError> {
2135 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2136 &mut self.cached_memory_op,
2137 &self.wasm_module,
2138 &self.offsets,
2139 &self.cache_builder,
2140 &self.ctx_ptr_value,
2141 );
2142 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
2143 Entry::Occupied(entry) => Ok(*entry.get()),
2144 Entry::Vacant(entry) => {
2145 let (size_fn, size_fn_ty) =
2146 if wasm_module.local_memory_index(memory_index).is_some() {
2147 (
2148 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
2149 intrinsics.ptr_ty,
2150 )
2151 } else {
2152 (
2153 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
2154 intrinsics.ptr_ty,
2155 )
2156 };
2157 let offset = offsets.vmctx_builtin_function(size_fn);
2158 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2159 let size_fn_ptr_ptr = unsafe {
2160 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2161 };
2162
2163 let size_fn_ptr_ptr =
2164 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2165 .into_pointer_value();
2166
2167 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2168 .into_pointer_value();
2169
2170 entry.insert(val);
2171 Ok(val)
2172 }
2173 }
2174 }
2175
2176 pub fn memory_wait64(
2177 &mut self,
2178 memory_index: MemoryIndex,
2179 intrinsics: &Intrinsics<'ctx>,
2180 ) -> Result<PointerValue<'ctx>, CompileError> {
2181 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2182 &mut self.cached_memory_op,
2183 &self.wasm_module,
2184 &self.offsets,
2185 &self.cache_builder,
2186 &self.ctx_ptr_value,
2187 );
2188
2189 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
2190 Entry::Occupied(entry) => Ok(*entry.get()),
2191 Entry::Vacant(entry) => {
2192 let (size_fn, size_fn_ty) =
2193 if wasm_module.local_memory_index(memory_index).is_some() {
2194 (
2195 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
2196 intrinsics.ptr_ty,
2197 )
2198 } else {
2199 (
2200 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
2201 intrinsics.ptr_ty,
2202 )
2203 };
2204 let offset = offsets.vmctx_builtin_function(size_fn);
2205 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2206 let size_fn_ptr_ptr = unsafe {
2207 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2208 };
2209
2210 let size_fn_ptr_ptr =
2211 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2212 .into_pointer_value();
2213
2214 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2215 .into_pointer_value();
2216 entry.insert(val);
2217 Ok(val)
2218 }
2219 }
2220 }
2221
2222 pub fn memory_notify(
2223 &mut self,
2224 memory_index: MemoryIndex,
2225 intrinsics: &Intrinsics<'ctx>,
2226 ) -> Result<PointerValue<'ctx>, CompileError> {
2227 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2228 &mut self.cached_memory_op,
2229 &self.wasm_module,
2230 &self.offsets,
2231 &self.cache_builder,
2232 &self.ctx_ptr_value,
2233 );
2234 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2235 Entry::Occupied(entry) => Ok(*entry.get()),
2236 Entry::Vacant(entry) => {
2237 let (size_fn, size_fn_ty) =
2238 if wasm_module.local_memory_index(memory_index).is_some() {
2239 (
2240 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2241 intrinsics.ptr_ty,
2242 )
2243 } else {
2244 (
2245 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2246 intrinsics.ptr_ty,
2247 )
2248 };
2249 let offset = offsets.vmctx_builtin_function(size_fn);
2250 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2251 let size_fn_ptr_ptr = unsafe {
2252 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2253 };
2254
2255 let size_fn_ptr_ptr =
2256 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2257 .into_pointer_value();
2258
2259 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2260 .into_pointer_value();
2261
2262 entry.insert(val);
2263 Ok(val)
2264 }
2265 }
2266 }
2267
2268 pub fn get_offsets(&self) -> &VMOffsets {
2269 &self.offsets
2270 }
2271}
2272
2273pub fn tbaa_label<'ctx>(
2276 module: &Module<'ctx>,
2277 intrinsics: &Intrinsics<'ctx>,
2278 label: String,
2279 instruction: InstructionValue<'ctx>,
2280) {
2281 let context = module.get_context();
2295
2296 let tbaa_root = module
2298 .get_global_metadata("wasmer_tbaa_root")
2299 .pop()
2300 .unwrap_or_else(|| {
2301 module
2302 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2303 .unwrap();
2304 module.get_global_metadata("wasmer_tbaa_root")[0]
2305 });
2306
2307 let type_label = context.metadata_string(label.as_str());
2310 let type_tbaa = module
2311 .get_global_metadata(label.as_str())
2312 .pop()
2313 .unwrap_or_else(|| {
2314 module
2315 .add_global_metadata(
2316 label.as_str(),
2317 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2318 )
2319 .unwrap();
2320 module.get_global_metadata(label.as_str())[0]
2321 });
2322
2323 let label = label + "_memop";
2330 let type_tbaa = module
2331 .get_global_metadata(label.as_str())
2332 .pop()
2333 .unwrap_or_else(|| {
2334 module
2335 .add_global_metadata(
2336 label.as_str(),
2337 &context.metadata_node(&[
2338 type_tbaa.into(),
2339 type_tbaa.into(),
2340 intrinsics.i64_zero.into(),
2341 ]),
2342 )
2343 .unwrap();
2344 module.get_global_metadata(label.as_str())[0]
2345 });
2346
2347 let tbaa_kind = context.get_kind_id("tbaa");
2349 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2350}
2351
2352fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2353 let table = module.tables.get(index)?;
2354 match table.maximum {
2355 None => Some(true),
2356 Some(max) => Some(max > table.minimum),
2357 }
2358}