1use crate::abi::Abi;
8use crate::error::err;
9use inkwell::values::BasicMetadataValueEnum;
10use inkwell::{
11 AddressSpace,
12 attributes::{Attribute, AttributeLoc},
13 builder::Builder,
14 context::Context,
15 module::{Linkage, Module},
16 targets::TargetData,
17 types::{
18 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
19 PointerType, StructType, VectorType, VoidType,
20 },
21 values::{
22 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
23 PointerValue, VectorValue,
24 },
25};
26use std::collections::{HashMap, hash_map::Entry};
27use std::num::NonZero;
28use target_lexicon::{Architecture, Triple};
29use wasmer_types::entity::{EntityRef, PrimaryMap};
30use wasmer_types::{
31 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
32 LocalTableIndex, MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, TableIndex, Type,
33};
34use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
35
36pub fn type_to_llvm<'ctx>(
37 intrinsics: &Intrinsics<'ctx>,
38 ty: Type,
39) -> Result<BasicTypeEnum<'ctx>, CompileError> {
40 match ty {
41 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
42 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
43 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
44 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
45 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
46 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
47 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
48 }
49}
50
51#[allow(dead_code)]
53pub struct X86_64Intrinsics<'ctx> {
54 pub pshufb128: FunctionValue<'ctx>,
55 pub pmaddubsw128: FunctionValue<'ctx>,
56 pub pmaddwd128: FunctionValue<'ctx>,
57 pub pmulhrsw128: FunctionValue<'ctx>,
58 pub pblendvb: FunctionValue<'ctx>,
59 pub min_ps: FunctionValue<'ctx>,
60 pub min_pd: FunctionValue<'ctx>,
61 pub max_ps: FunctionValue<'ctx>,
62 pub max_pd: FunctionValue<'ctx>,
63 pub cvttps2dq: FunctionValue<'ctx>,
64 pub cvtps2udq128: FunctionValue<'ctx>,
65 pub cvtpd2dq: FunctionValue<'ctx>,
66 pub cvtpd2udq128: FunctionValue<'ctx>,
67}
68
69#[allow(dead_code)]
71pub struct Intrinsics<'ctx> {
72 pub ctlz_i32: FunctionValue<'ctx>,
73 pub ctlz_i64: FunctionValue<'ctx>,
74
75 pub cttz_i32: FunctionValue<'ctx>,
76 pub cttz_i64: FunctionValue<'ctx>,
77
78 pub ctpop_i32: FunctionValue<'ctx>,
79 pub ctpop_i64: FunctionValue<'ctx>,
80 pub ctpop_i8x16: FunctionValue<'ctx>,
81
82 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
83 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
84 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
85 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
86 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
87
88 pub add_f32: FunctionValue<'ctx>,
89 pub add_f64: FunctionValue<'ctx>,
90 pub add_f32x4: FunctionValue<'ctx>,
91 pub add_f64x2: FunctionValue<'ctx>,
92
93 pub sub_f32: FunctionValue<'ctx>,
94 pub sub_f64: FunctionValue<'ctx>,
95 pub sub_f32x4: FunctionValue<'ctx>,
96 pub sub_f64x2: FunctionValue<'ctx>,
97
98 pub mul_f32: FunctionValue<'ctx>,
99 pub mul_f64: FunctionValue<'ctx>,
100 pub mul_f32x4: FunctionValue<'ctx>,
101 pub mul_f64x2: FunctionValue<'ctx>,
102 pub muladd_f32x4: FunctionValue<'ctx>,
103 pub muladd_f64x2: FunctionValue<'ctx>,
104
105 pub div_f32: FunctionValue<'ctx>,
106 pub div_f64: FunctionValue<'ctx>,
107 pub div_f32x4: FunctionValue<'ctx>,
108 pub div_f64x2: FunctionValue<'ctx>,
109
110 pub sqrt_f32: FunctionValue<'ctx>,
111 pub sqrt_f64: FunctionValue<'ctx>,
112 pub sqrt_f32x4: FunctionValue<'ctx>,
113 pub sqrt_f64x2: FunctionValue<'ctx>,
114
115 pub cmp_f32: FunctionValue<'ctx>,
116 pub cmp_f64: FunctionValue<'ctx>,
117 pub cmp_f32x4: FunctionValue<'ctx>,
118 pub cmp_f64x2: FunctionValue<'ctx>,
119
120 pub minimum_f32: FunctionValue<'ctx>,
121 pub minimum_f64: FunctionValue<'ctx>,
122 pub minimum_f32x4: FunctionValue<'ctx>,
123 pub minimum_f64x2: FunctionValue<'ctx>,
124
125 pub maximum_f32: FunctionValue<'ctx>,
126 pub maximum_f64: FunctionValue<'ctx>,
127 pub maximum_f32x4: FunctionValue<'ctx>,
128 pub maximum_f64x2: FunctionValue<'ctx>,
129
130 pub ceil_f32: FunctionValue<'ctx>,
131 pub ceil_f64: FunctionValue<'ctx>,
132 pub ceil_f32x4: FunctionValue<'ctx>,
133 pub ceil_f64x2: FunctionValue<'ctx>,
134
135 pub floor_f32: FunctionValue<'ctx>,
136 pub floor_f64: FunctionValue<'ctx>,
137 pub floor_f32x4: FunctionValue<'ctx>,
138 pub floor_f64x2: FunctionValue<'ctx>,
139
140 pub trunc_f32: FunctionValue<'ctx>,
141 pub trunc_f64: FunctionValue<'ctx>,
142 pub trunc_f32x4: FunctionValue<'ctx>,
143 pub trunc_f64x2: FunctionValue<'ctx>,
144
145 pub fpext_f32: FunctionValue<'ctx>,
146 pub fptrunc_f64: FunctionValue<'ctx>,
147
148 pub nearbyint_f32: FunctionValue<'ctx>,
149 pub nearbyint_f64: FunctionValue<'ctx>,
150 pub nearbyint_f32x4: FunctionValue<'ctx>,
151 pub nearbyint_f64x2: FunctionValue<'ctx>,
152
153 pub fabs_f32: FunctionValue<'ctx>,
154 pub fabs_f64: FunctionValue<'ctx>,
155 pub fabs_f32x4: FunctionValue<'ctx>,
156 pub fabs_f64x2: FunctionValue<'ctx>,
157
158 pub copysign_f32: FunctionValue<'ctx>,
159 pub copysign_f64: FunctionValue<'ctx>,
160 pub copysign_f32x4: FunctionValue<'ctx>,
161 pub copysign_f64x2: FunctionValue<'ctx>,
162
163 pub sadd_sat_i8x16: FunctionValue<'ctx>,
164 pub sadd_sat_i16x8: FunctionValue<'ctx>,
165 pub uadd_sat_i8x16: FunctionValue<'ctx>,
166 pub uadd_sat_i16x8: FunctionValue<'ctx>,
167
168 pub ssub_sat_i8x16: FunctionValue<'ctx>,
169 pub ssub_sat_i16x8: FunctionValue<'ctx>,
170 pub usub_sat_i8x16: FunctionValue<'ctx>,
171 pub usub_sat_i16x8: FunctionValue<'ctx>,
172
173 pub expect_i1: FunctionValue<'ctx>,
174 pub trap: FunctionValue<'ctx>,
175 pub debug_trap: FunctionValue<'ctx>,
176
177 pub personality: FunctionValue<'ctx>,
178 pub personality2: FunctionValue<'ctx>,
179 pub readonly: Attribute,
180 pub stack_probe: Attribute,
181 pub uwtable: Attribute,
182 pub frame_pointer: Attribute,
183 pub chkstk: FunctionValue<'ctx>,
185
186 pub void_ty: VoidType<'ctx>,
187 pub i1_ty: IntType<'ctx>,
188 pub i2_ty: IntType<'ctx>,
189 pub i4_ty: IntType<'ctx>,
190 pub i8_ty: IntType<'ctx>,
191 pub i16_ty: IntType<'ctx>,
192 pub i32_ty: IntType<'ctx>,
193 pub i64_ty: IntType<'ctx>,
194 pub i128_ty: IntType<'ctx>,
195 pub isize_ty: IntType<'ctx>,
196 pub f32_ty: FloatType<'ctx>,
197 pub f64_ty: FloatType<'ctx>,
198
199 pub i1x128_ty: VectorType<'ctx>,
200 pub i8x16_ty: VectorType<'ctx>,
201 pub i16x8_ty: VectorType<'ctx>,
202 pub i32x4_ty: VectorType<'ctx>,
203 pub i64x2_ty: VectorType<'ctx>,
204 pub f32x4_ty: VectorType<'ctx>,
205 pub f64x2_ty: VectorType<'ctx>,
206 pub i32x8_ty: VectorType<'ctx>,
207
208 pub ptr_ty: PointerType<'ctx>,
209
210 pub x86_64: X86_64Intrinsics<'ctx>,
211
212 pub anyfunc_ty: StructType<'ctx>,
213
214 pub i1_zero: IntValue<'ctx>,
215 pub i8_zero: IntValue<'ctx>,
216 pub i32_zero: IntValue<'ctx>,
217 pub i64_zero: IntValue<'ctx>,
218 pub i128_zero: IntValue<'ctx>,
219 pub isize_zero: IntValue<'ctx>,
220 pub f32_zero: FloatValue<'ctx>,
221 pub f64_zero: FloatValue<'ctx>,
222 pub f32x4_zero: VectorValue<'ctx>,
223 pub f64x2_zero: VectorValue<'ctx>,
224 pub i32_consts: [IntValue<'ctx>; 16],
225
226 pub trap_unreachable: BasicValueEnum<'ctx>,
227 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
228 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
229 pub trap_memory_oob: BasicValueEnum<'ctx>,
230 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
231 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
232 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
233 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
234 pub trap_table_access_oob: BasicValueEnum<'ctx>,
235
236 pub experimental_stackmap: FunctionValue<'ctx>,
237
238 pub table_copy: FunctionValue<'ctx>,
240 pub table_init: FunctionValue<'ctx>,
241 pub table_fill: FunctionValue<'ctx>,
242 pub table_size: FunctionValue<'ctx>,
243 pub imported_table_size: FunctionValue<'ctx>,
244 pub table_get: FunctionValue<'ctx>,
245 pub imported_table_get: FunctionValue<'ctx>,
246 pub table_set: FunctionValue<'ctx>,
247 pub imported_table_set: FunctionValue<'ctx>,
248 pub table_grow: FunctionValue<'ctx>,
249 pub imported_table_grow: FunctionValue<'ctx>,
250 pub memory_init: FunctionValue<'ctx>,
251 pub data_drop: FunctionValue<'ctx>,
252 pub func_ref: FunctionValue<'ctx>,
253 pub elem_drop: FunctionValue<'ctx>,
254 pub memory_copy: FunctionValue<'ctx>,
255 pub imported_memory_copy: FunctionValue<'ctx>,
256 pub memory_fill: FunctionValue<'ctx>,
257 pub imported_memory_fill: FunctionValue<'ctx>,
258 pub memory_size_ty: FunctionType<'ctx>,
259 pub memory_grow_ty: FunctionType<'ctx>,
260 pub memory_wait32: FunctionValue<'ctx>,
261 pub memory_wait32_ty: FunctionType<'ctx>,
262 pub imported_memory_wait32: FunctionValue<'ctx>,
263 pub memory_wait64: FunctionValue<'ctx>,
264 pub memory_wait64_ty: FunctionType<'ctx>,
265 pub imported_memory_wait64: FunctionValue<'ctx>,
266 pub memory_notify: FunctionValue<'ctx>,
267 pub memory_notify_ty: FunctionType<'ctx>,
268 pub imported_memory_notify: FunctionValue<'ctx>,
269
270 pub throw_trap: FunctionValue<'ctx>,
271
272 pub throw: FunctionValue<'ctx>,
274 pub alloc_exception: FunctionValue<'ctx>,
275 pub read_exnref: FunctionValue<'ctx>,
276 pub exception_into_exnref: FunctionValue<'ctx>,
277 pub lpad_exception_ty: StructType<'ctx>,
278
279 pub debug_ptr: FunctionValue<'ctx>,
281 pub debug_str: FunctionValue<'ctx>,
282
283 pub vmfunction_import_ty: StructType<'ctx>,
285 pub vmfunction_import_body_element: u32,
286 pub vmfunction_import_vmctx_element: u32,
287 pub vmfunction_import_include_m0_param_element: u32,
288
289 pub vmmemory_definition_ty: StructType<'ctx>,
290 pub vmmemory_definition_base_element: u32,
291 pub vmmemory_definition_current_length_element: u32,
292}
293
294#[derive(Debug, Hash, PartialEq, Eq)]
295enum MemoryOp {
296 Size,
297 Grow,
298 Wait32,
299 Wait64,
300 Notify,
301}
302
303impl<'ctx> Intrinsics<'ctx> {
304 pub fn declare(
306 module: &Module<'ctx>,
307 context: &'ctx Context,
308 target_data: &TargetData,
309 target_triple: &Triple,
310 binary_fmt: &target_lexicon::BinaryFormat,
311 ) -> Self {
312 let is_riscv64 = matches!(target_triple.architecture, Architecture::Riscv64(..));
313 let void_ty = context.void_type();
314 let i1_ty = context.bool_type();
315 let i2_ty = context
316 .custom_width_int_type(NonZero::new(2).unwrap())
317 .unwrap();
318 let i4_ty = context
319 .custom_width_int_type(NonZero::new(4).unwrap())
320 .unwrap();
321 let i8_ty = context.i8_type();
322 let i16_ty = context.i16_type();
323 let i32_ty = context.i32_type();
324 let i64_ty = context.i64_type();
325 let i128_ty = context.i128_type();
326 let isize_ty = context.ptr_sized_int_type(target_data, None);
327 let f32_ty = context.f32_type();
328 let f64_ty = context.f64_type();
329
330 let i1x4_ty = i1_ty.vec_type(4);
331 let i1x2_ty = i1_ty.vec_type(2);
332 let i1x128_ty = i1_ty.vec_type(128);
333 let i8x16_ty = i8_ty.vec_type(16);
334 let i16x8_ty = i16_ty.vec_type(8);
335 let i32x4_ty = i32_ty.vec_type(4);
336 let i64x2_ty = i64_ty.vec_type(2);
337 let f32x4_ty = f32_ty.vec_type(4);
338 let f64x2_ty = f64_ty.vec_type(2);
339 let i32x8_ty = i32_ty.vec_type(8);
340
341 let ptr_ty = context.ptr_type(AddressSpace::default());
342
343 let i1_zero = i1_ty.const_int(0, false);
344 let i8_zero = i8_ty.const_int(0, false);
345 let i32_zero = i32_ty.const_int(0, false);
346 let i64_zero = i64_ty.const_int(0, false);
347 let i128_zero = i128_ty.const_int(0, false);
348 let isize_zero = isize_ty.const_int(0, false);
349 let f32_zero = f32_ty.const_float(0.0);
350 let f64_zero = f64_ty.const_float(0.0);
351 let f32x4_zero = f32x4_ty.const_zero();
352 let f64x2_zero = f64x2_ty.const_zero();
353 let i32_consts = [
354 i32_ty.const_int(0, false),
355 i32_ty.const_int(1, false),
356 i32_ty.const_int(2, false),
357 i32_ty.const_int(3, false),
358 i32_ty.const_int(4, false),
359 i32_ty.const_int(5, false),
360 i32_ty.const_int(6, false),
361 i32_ty.const_int(7, false),
362 i32_ty.const_int(8, false),
363 i32_ty.const_int(9, false),
364 i32_ty.const_int(10, false),
365 i32_ty.const_int(11, false),
366 i32_ty.const_int(12, false),
367 i32_ty.const_int(13, false),
368 i32_ty.const_int(14, false),
369 i32_ty.const_int(15, false),
370 ];
371
372 let md_ty = context.metadata_type();
373
374 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
375 let i1_ty_basic = i1_ty.as_basic_type_enum();
376
377 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
378 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
379 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
380 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
381 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
382 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
383 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
384 let i32x4_ty_basic_md: BasicMetadataTypeEnum = i32x4_ty.into();
385 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
386 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
387 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
388
389 let ctx_ptr_ty = ptr_ty;
390 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
391 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
392
393 let anyfunc_ty = context.struct_type(
397 &[
398 i8_ptr_ty_basic,
399 i32_ty.into(),
400 ctx_ptr_ty_basic,
401 ptr_ty.into(),
402 ],
403 false,
404 );
405 let funcref_ty = ptr_ty;
406 let anyref_ty = ptr_ty;
407 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
408
409 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
410 let ret_i8x16_take_i8x16_i8x16 =
411 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
412 let ret_i8x16_take_i8x16_i8x16_i8x16 = i8x16_ty.fn_type(
413 &[i8x16_ty_basic_md, i8x16_ty_basic_md, i8x16_ty_basic_md],
414 false,
415 );
416 let ret_i16x8_take_i16x8_i16x8 =
417 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
418
419 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
420 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
421
422 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
423 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
424
425 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
426 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
427 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
428 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
429
430 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
431 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
432 let ret_f32x4_take_f32x4_f32x4 =
433 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
434 let ret_f64x2_take_f64x2_f64x2 =
435 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
436 let ret_f32x4_take_f32x4_f32x4_f32x4_md_md = f32x4_ty.fn_type(
437 &[
438 f32x4_ty_basic_md,
439 f32x4_ty_basic_md,
440 f32x4_ty_basic_md,
441 md_ty_basic_md,
442 md_ty_basic_md,
443 ],
444 false,
445 );
446 let ret_f64x2_take_f64x2_f64x2_f64x2_md_md = f64x2_ty.fn_type(
447 &[
448 f64x2_ty_basic_md,
449 f64x2_ty_basic_md,
450 f64x2_ty_basic_md,
451 md_ty_basic_md,
452 md_ty_basic_md,
453 ],
454 false,
455 );
456
457 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
458 let ret_f32_take_f64_md_md =
459 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
460
461 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
462
463 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
464 &[
465 f32_ty_basic_md,
466 f32_ty_basic_md,
467 md_ty_basic_md,
468 md_ty_basic_md,
469 ],
470 false,
471 );
472 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
473 &[
474 f64_ty_basic_md,
475 f64_ty_basic_md,
476 md_ty_basic_md,
477 md_ty_basic_md,
478 ],
479 false,
480 );
481 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
482 &[
483 f32x4_ty_basic_md,
484 f32x4_ty_basic_md,
485 md_ty_basic_md,
486 md_ty_basic_md,
487 ],
488 false,
489 );
490 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
491 &[
492 f64x2_ty_basic_md,
493 f64x2_ty_basic_md,
494 md_ty_basic_md,
495 md_ty_basic_md,
496 ],
497 false,
498 );
499
500 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
501 &[
502 f32_ty_basic_md,
503 f32_ty_basic_md,
504 md_ty_basic_md,
505 md_ty_basic_md,
506 ],
507 false,
508 );
509 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
510 &[
511 f64_ty_basic_md,
512 f64_ty_basic_md,
513 md_ty_basic_md,
514 md_ty_basic_md,
515 ],
516 false,
517 );
518 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
519 &[
520 f32x4_ty_basic_md,
521 f32x4_ty_basic_md,
522 md_ty_basic_md,
523 md_ty_basic_md,
524 ],
525 false,
526 );
527 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
528 &[
529 f64x2_ty_basic_md,
530 f64x2_ty_basic_md,
531 md_ty_basic_md,
532 md_ty_basic_md,
533 ],
534 false,
535 );
536 let ret_i32x4_take_f32x4 = i32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
537 let ret_i32x4_take_f32x4_i32x4_i8 =
538 i32x4_ty.fn_type(&[f32x4_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
539 let ret_i32x4_take_f64x2 = i32x4_ty.fn_type(&[f64x2_ty_basic_md], false);
540 let ret_i32x4_take_f64x2_i32x4_i8 =
541 i32x4_ty.fn_type(&[f64x2_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
542
543 let add_function_with_attrs =
544 |name: &str, ty: FunctionType<'ctx>, linkage: Option<Linkage>| -> FunctionValue<'ctx> {
545 let function = module.add_function(name, ty, linkage);
546 if is_riscv64 {
551 for (i, param) in ty.get_param_types().iter().enumerate() {
552 if param == &i32_ty_basic_md {
553 function.add_attribute(
554 AttributeLoc::Param(i as u32),
555 context.create_enum_attribute(
556 Attribute::get_named_enum_kind_id("signext"),
557 0,
558 ),
559 );
560 function.add_attribute(
561 AttributeLoc::Param(i as u32),
562 context.create_enum_attribute(
563 Attribute::get_named_enum_kind_id("noundef"),
564 0,
565 ),
566 );
567 }
568 }
569 }
570 function
571 };
572
573 let intrinsics = Self {
574 ctlz_i32: add_function_with_attrs("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
575 ctlz_i64: add_function_with_attrs("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
576
577 cttz_i32: add_function_with_attrs("llvm.cttz.i32", ret_i32_take_i32_i1, None),
578 cttz_i64: add_function_with_attrs("llvm.cttz.i64", ret_i64_take_i64_i1, None),
579
580 ctpop_i32: add_function_with_attrs("llvm.ctpop.i32", ret_i32_take_i32, None),
581 ctpop_i64: add_function_with_attrs("llvm.ctpop.i64", ret_i64_take_i64, None),
582 ctpop_i8x16: add_function_with_attrs("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
583
584 fp_rounding_md: context.metadata_string("round.tonearest").into(),
585 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
586
587 fp_ogt_md: context.metadata_string("ogt").into(),
588 fp_olt_md: context.metadata_string("olt").into(),
589 fp_uno_md: context.metadata_string("uno").into(),
590
591 sqrt_f32: add_function_with_attrs("llvm.sqrt.f32", ret_f32_take_f32, None),
592 sqrt_f64: add_function_with_attrs("llvm.sqrt.f64", ret_f64_take_f64, None),
593 sqrt_f32x4: add_function_with_attrs("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
594 sqrt_f64x2: add_function_with_attrs("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
595
596 ceil_f32: add_function_with_attrs("llvm.ceil.f32", ret_f32_take_f32, None),
597 ceil_f64: add_function_with_attrs("llvm.ceil.f64", ret_f64_take_f64, None),
598 ceil_f32x4: add_function_with_attrs("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
599 ceil_f64x2: add_function_with_attrs("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
600
601 floor_f32: add_function_with_attrs("llvm.floor.f32", ret_f32_take_f32, None),
602 floor_f64: add_function_with_attrs("llvm.floor.f64", ret_f64_take_f64, None),
603 floor_f32x4: add_function_with_attrs("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
604 floor_f64x2: add_function_with_attrs("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
605
606 trunc_f32: add_function_with_attrs("llvm.trunc.f32", ret_f32_take_f32, None),
607 trunc_f64: add_function_with_attrs("llvm.trunc.f64", ret_f64_take_f64, None),
608 trunc_f32x4: add_function_with_attrs("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
609 trunc_f64x2: add_function_with_attrs("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
610
611 nearbyint_f32: add_function_with_attrs("llvm.nearbyint.f32", ret_f32_take_f32, None),
612 nearbyint_f64: add_function_with_attrs("llvm.nearbyint.f64", ret_f64_take_f64, None),
613 nearbyint_f32x4: add_function_with_attrs(
614 "llvm.nearbyint.v4f32",
615 ret_f32x4_take_f32x4,
616 None,
617 ),
618 nearbyint_f64x2: add_function_with_attrs(
619 "llvm.nearbyint.v2f64",
620 ret_f64x2_take_f64x2,
621 None,
622 ),
623
624 add_f32: add_function_with_attrs(
625 "llvm.experimental.constrained.fadd.f32",
626 ret_f32_take_f32_f32_md_md,
627 None,
628 ),
629 add_f64: add_function_with_attrs(
630 "llvm.experimental.constrained.fadd.f64",
631 ret_f64_take_f64_f64_md_md,
632 None,
633 ),
634 add_f32x4: add_function_with_attrs(
635 "llvm.experimental.constrained.fadd.v4f32",
636 ret_f32x4_take_f32x4_f32x4_md_md,
637 None,
638 ),
639 add_f64x2: add_function_with_attrs(
640 "llvm.experimental.constrained.fadd.v2f64",
641 ret_f64x2_take_f64x2_f64x2_md_md,
642 None,
643 ),
644
645 sub_f32: add_function_with_attrs(
646 "llvm.experimental.constrained.fsub.f32",
647 ret_f32_take_f32_f32_md_md,
648 None,
649 ),
650 sub_f64: add_function_with_attrs(
651 "llvm.experimental.constrained.fsub.f64",
652 ret_f64_take_f64_f64_md_md,
653 None,
654 ),
655 sub_f32x4: add_function_with_attrs(
656 "llvm.experimental.constrained.fsub.v4f32",
657 ret_f32x4_take_f32x4_f32x4_md_md,
658 None,
659 ),
660 sub_f64x2: add_function_with_attrs(
661 "llvm.experimental.constrained.fsub.v2f64",
662 ret_f64x2_take_f64x2_f64x2_md_md,
663 None,
664 ),
665
666 mul_f32: add_function_with_attrs(
667 "llvm.experimental.constrained.fmul.f32",
668 ret_f32_take_f32_f32_md_md,
669 None,
670 ),
671 mul_f64: add_function_with_attrs(
672 "llvm.experimental.constrained.fmul.f64",
673 ret_f64_take_f64_f64_md_md,
674 None,
675 ),
676 mul_f32x4: add_function_with_attrs(
677 "llvm.experimental.constrained.fmul.v4f32",
678 ret_f32x4_take_f32x4_f32x4_md_md,
679 None,
680 ),
681 mul_f64x2: add_function_with_attrs(
682 "llvm.experimental.constrained.fmul.v2f64",
683 ret_f64x2_take_f64x2_f64x2_md_md,
684 None,
685 ),
686 muladd_f32x4: add_function_with_attrs(
687 "llvm.experimental.constrained.fmuladd.v4f32",
688 ret_f32x4_take_f32x4_f32x4_f32x4_md_md,
689 None,
690 ),
691 muladd_f64x2: add_function_with_attrs(
692 "llvm.experimental.constrained.fmuladd.v2f64",
693 ret_f64x2_take_f64x2_f64x2_f64x2_md_md,
694 None,
695 ),
696
697 div_f32: add_function_with_attrs(
698 "llvm.experimental.constrained.fdiv.f32",
699 ret_f32_take_f32_f32_md_md,
700 None,
701 ),
702 div_f64: add_function_with_attrs(
703 "llvm.experimental.constrained.fdiv.f64",
704 ret_f64_take_f64_f64_md_md,
705 None,
706 ),
707 div_f32x4: add_function_with_attrs(
708 "llvm.experimental.constrained.fdiv.v4f32",
709 ret_f32x4_take_f32x4_f32x4_md_md,
710 None,
711 ),
712 div_f64x2: add_function_with_attrs(
713 "llvm.experimental.constrained.fdiv.v2f64",
714 ret_f64x2_take_f64x2_f64x2_md_md,
715 None,
716 ),
717
718 cmp_f32: add_function_with_attrs(
719 "llvm.experimental.constrained.fcmp.f32",
720 ret_i1_take_f32_f32_md_md,
721 None,
722 ),
723 cmp_f64: add_function_with_attrs(
724 "llvm.experimental.constrained.fcmp.f64",
725 ret_i1_take_f64_f64_md_md,
726 None,
727 ),
728 cmp_f32x4: add_function_with_attrs(
729 "llvm.experimental.constrained.fcmp.v4f32",
730 ret_i1x4_take_f32x4_f32x4_md_md,
731 None,
732 ),
733 cmp_f64x2: add_function_with_attrs(
734 "llvm.experimental.constrained.fcmp.v2f64",
735 ret_i1x2_take_f64x2_f64x2_md_md,
736 None,
737 ),
738
739 minimum_f32: add_function_with_attrs("llvm.minimum.f32", ret_f32_take_f32_f32, None),
740 minimum_f64: add_function_with_attrs("llvm.minimum.f64", ret_f64_take_f64_f64, None),
741 minimum_f32x4: add_function_with_attrs(
742 "llvm.minimum.v4f32",
743 ret_f32x4_take_f32x4_f32x4,
744 None,
745 ),
746 minimum_f64x2: add_function_with_attrs(
747 "llvm.minimum.v2f64",
748 ret_f64x2_take_f64x2_f64x2,
749 None,
750 ),
751
752 maximum_f32: add_function_with_attrs("llvm.maximum.f32", ret_f32_take_f32_f32, None),
753 maximum_f64: add_function_with_attrs("llvm.maximum.f64", ret_f64_take_f64_f64, None),
754 maximum_f32x4: add_function_with_attrs(
755 "llvm.maximum.v4f32",
756 ret_f32x4_take_f32x4_f32x4,
757 None,
758 ),
759 maximum_f64x2: add_function_with_attrs(
760 "llvm.maximum.v2f64",
761 ret_f64x2_take_f64x2_f64x2,
762 None,
763 ),
764
765 fpext_f32: add_function_with_attrs(
766 "llvm.experimental.constrained.fpext.f64.f32",
767 ret_f64_take_f32_md,
768 None,
769 ),
770 fptrunc_f64: add_function_with_attrs(
771 "llvm.experimental.constrained.fptrunc.f32.f64",
772 ret_f32_take_f64_md_md,
773 None,
774 ),
775
776 fabs_f32: add_function_with_attrs("llvm.fabs.f32", ret_f32_take_f32, None),
777 fabs_f64: add_function_with_attrs("llvm.fabs.f64", ret_f64_take_f64, None),
778 fabs_f32x4: add_function_with_attrs("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
779 fabs_f64x2: add_function_with_attrs("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
780
781 copysign_f32: add_function_with_attrs("llvm.copysign.f32", ret_f32_take_f32_f32, None),
782 copysign_f64: add_function_with_attrs("llvm.copysign.f64", ret_f64_take_f64_f64, None),
783 copysign_f32x4: add_function_with_attrs(
784 "llvm.copysign.v4f32",
785 ret_f32x4_take_f32x4_f32x4,
786 None,
787 ),
788 copysign_f64x2: add_function_with_attrs(
789 "llvm.copysign.v2f64",
790 ret_f64x2_take_f64x2_f64x2,
791 None,
792 ),
793
794 sadd_sat_i8x16: add_function_with_attrs(
795 "llvm.sadd.sat.v16i8",
796 ret_i8x16_take_i8x16_i8x16,
797 None,
798 ),
799 sadd_sat_i16x8: add_function_with_attrs(
800 "llvm.sadd.sat.v8i16",
801 ret_i16x8_take_i16x8_i16x8,
802 None,
803 ),
804 uadd_sat_i8x16: add_function_with_attrs(
805 "llvm.uadd.sat.v16i8",
806 ret_i8x16_take_i8x16_i8x16,
807 None,
808 ),
809 uadd_sat_i16x8: add_function_with_attrs(
810 "llvm.uadd.sat.v8i16",
811 ret_i16x8_take_i16x8_i16x8,
812 None,
813 ),
814
815 ssub_sat_i8x16: add_function_with_attrs(
816 "llvm.ssub.sat.v16i8",
817 ret_i8x16_take_i8x16_i8x16,
818 None,
819 ),
820 ssub_sat_i16x8: add_function_with_attrs(
821 "llvm.ssub.sat.v8i16",
822 ret_i16x8_take_i16x8_i16x8,
823 None,
824 ),
825 usub_sat_i8x16: add_function_with_attrs(
826 "llvm.usub.sat.v16i8",
827 ret_i8x16_take_i8x16_i8x16,
828 None,
829 ),
830 usub_sat_i16x8: add_function_with_attrs(
831 "llvm.usub.sat.v8i16",
832 ret_i16x8_take_i16x8_i16x8,
833 None,
834 ),
835
836 expect_i1: add_function_with_attrs("llvm.expect.i1", ret_i1_take_i1_i1, None),
837 trap: add_function_with_attrs("llvm.trap", void_ty.fn_type(&[], false), None),
838 debug_trap: add_function_with_attrs(
839 "llvm.debugtrap",
840 void_ty.fn_type(&[], false),
841 None,
842 ),
843 personality: add_function_with_attrs(
844 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
845 "__gxx_personality_v0"
848 } else {
849 "wasmer_eh_personality"
850 },
851 i32_ty.fn_type(
852 &[
853 i32_ty.into(),
854 i32_ty.into(),
855 i64_ty.into(),
856 ptr_ty.into(),
857 ptr_ty.into(),
858 ],
859 false,
860 ),
861 None,
862 ),
863 personality2: add_function_with_attrs(
864 "wasmer_eh_personality2",
865 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
866 None,
867 ),
868 readonly: context
869 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
870 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
871 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
872 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
873 chkstk: add_function_with_attrs("__chkstk", void_ty.fn_type(&[], false), None),
874 void_ty,
875 i1_ty,
876 i2_ty,
877 i4_ty,
878 i8_ty,
879 i16_ty,
880 i32_ty,
881 i64_ty,
882 i128_ty,
883 isize_ty,
884 f32_ty,
885 f64_ty,
886
887 i1x128_ty,
888 i8x16_ty,
889 i16x8_ty,
890 i32x4_ty,
891 i64x2_ty,
892 f32x4_ty,
893 f64x2_ty,
894 i32x8_ty,
895
896 anyfunc_ty,
897 i1_zero,
898 i8_zero,
899 i32_zero,
900 i64_zero,
901 i128_zero,
902 isize_zero,
903 f32_zero,
904 f64_zero,
905 f32x4_zero,
906 f64x2_zero,
907 i32_consts,
908
909 trap_unreachable: i32_ty
910 .const_int(TrapCode::UnreachableCodeReached as _, false)
911 .as_basic_value_enum(),
912 trap_call_indirect_null: i32_ty
913 .const_int(TrapCode::IndirectCallToNull as _, false)
914 .as_basic_value_enum(),
915 trap_call_indirect_sig: i32_ty
916 .const_int(TrapCode::BadSignature as _, false)
917 .as_basic_value_enum(),
918 trap_memory_oob: i32_ty
919 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
920 .as_basic_value_enum(),
921 trap_illegal_arithmetic: i32_ty
922 .const_int(TrapCode::IntegerOverflow as _, false)
923 .as_basic_value_enum(),
924 trap_integer_division_by_zero: i32_ty
925 .const_int(TrapCode::IntegerDivisionByZero as _, false)
926 .as_basic_value_enum(),
927 trap_bad_conversion_to_integer: i32_ty
928 .const_int(TrapCode::BadConversionToInteger as _, false)
929 .as_basic_value_enum(),
930 trap_unaligned_atomic: i32_ty
931 .const_int(TrapCode::UnalignedAtomic as _, false)
932 .as_basic_value_enum(),
933 trap_table_access_oob: i32_ty
934 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
935 .as_basic_value_enum(),
936
937 experimental_stackmap: module.add_function(
941 "llvm.experimental.stackmap",
942 void_ty.fn_type(
943 &[
944 i64_ty_basic_md, i32_ty_basic_md, ],
947 true,
948 ),
949 None,
950 ),
951
952 table_copy: add_function_with_attrs(
954 "wasmer_vm_table_copy",
955 void_ty.fn_type(
956 &[
957 ctx_ptr_ty_basic_md,
958 i32_ty_basic_md,
959 i32_ty_basic_md,
960 i32_ty_basic_md,
961 i32_ty_basic_md,
962 i32_ty_basic_md,
963 ],
964 false,
965 ),
966 None,
967 ),
968 table_init: add_function_with_attrs(
969 "wasmer_vm_table_init",
970 void_ty.fn_type(
971 &[
972 ctx_ptr_ty_basic_md,
973 i32_ty_basic_md,
974 i32_ty_basic_md,
975 i32_ty_basic_md,
976 i32_ty_basic_md,
977 i32_ty_basic_md,
978 ],
979 false,
980 ),
981 None,
982 ),
983 table_fill: add_function_with_attrs(
984 "wasmer_vm_table_fill",
985 void_ty.fn_type(
986 &[
987 ctx_ptr_ty_basic_md,
988 i32_ty_basic_md,
989 i32_ty_basic_md,
990 anyref_ty_basic_md,
991 i32_ty_basic_md,
992 ],
993 false,
994 ),
995 None,
996 ),
997 table_size: add_function_with_attrs(
998 "wasmer_vm_table_size",
999 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1000 None,
1001 ),
1002 imported_table_size: add_function_with_attrs(
1003 "wasmer_vm_imported_table_size",
1004 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1005 None,
1006 ),
1007 table_get: add_function_with_attrs(
1008 "wasmer_vm_table_get",
1009 anyref_ty.fn_type(
1010 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1011 false,
1012 ),
1013 None,
1014 ),
1015 imported_table_get: add_function_with_attrs(
1016 "wasmer_vm_imported_table_get",
1017 anyref_ty.fn_type(
1018 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1019 false,
1020 ),
1021 None,
1022 ),
1023 table_set: add_function_with_attrs(
1024 "wasmer_vm_table_set",
1025 void_ty.fn_type(
1026 &[
1027 ctx_ptr_ty_basic_md,
1028 i32_ty_basic_md,
1029 i32_ty_basic_md,
1030 anyref_ty_basic_md,
1031 ],
1032 false,
1033 ),
1034 None,
1035 ),
1036 imported_table_set: add_function_with_attrs(
1037 "wasmer_vm_imported_table_set",
1038 void_ty.fn_type(
1039 &[
1040 ctx_ptr_ty_basic_md,
1041 i32_ty_basic_md,
1042 i32_ty_basic_md,
1043 anyref_ty_basic_md,
1044 ],
1045 false,
1046 ),
1047 None,
1048 ),
1049 table_grow: add_function_with_attrs(
1050 "wasmer_vm_table_grow",
1051 i32_ty.fn_type(
1052 &[
1053 ctx_ptr_ty_basic_md,
1054 anyref_ty_basic_md,
1055 i32_ty_basic_md,
1056 i32_ty_basic_md,
1057 ],
1058 false,
1059 ),
1060 None,
1061 ),
1062 imported_table_grow: add_function_with_attrs(
1063 "wasmer_vm_imported_table_grow",
1064 i32_ty.fn_type(
1065 &[
1066 ctx_ptr_ty_basic_md,
1067 anyref_ty_basic_md,
1068 i32_ty_basic_md,
1069 i32_ty_basic_md,
1070 ],
1071 false,
1072 ),
1073 None,
1074 ),
1075 memory_init: add_function_with_attrs(
1076 "wasmer_vm_memory32_init",
1077 void_ty.fn_type(
1078 &[
1079 ctx_ptr_ty_basic_md,
1080 i32_ty_basic_md,
1081 i32_ty_basic_md,
1082 i32_ty_basic_md,
1083 i32_ty_basic_md,
1084 i32_ty_basic_md,
1085 ],
1086 false,
1087 ),
1088 None,
1089 ),
1090 memory_copy: add_function_with_attrs(
1091 "wasmer_vm_memory32_copy",
1092 void_ty.fn_type(
1093 &[
1094 ctx_ptr_ty_basic_md,
1095 i32_ty_basic_md,
1096 i32_ty_basic_md,
1097 i32_ty_basic_md,
1098 i32_ty_basic_md,
1099 ],
1100 false,
1101 ),
1102 None,
1103 ),
1104 imported_memory_copy: add_function_with_attrs(
1105 "wasmer_vm_imported_memory32_copy",
1106 void_ty.fn_type(
1107 &[
1108 ctx_ptr_ty_basic_md,
1109 i32_ty_basic_md,
1110 i32_ty_basic_md,
1111 i32_ty_basic_md,
1112 i32_ty_basic_md,
1113 ],
1114 false,
1115 ),
1116 None,
1117 ),
1118 memory_fill: add_function_with_attrs(
1119 "wasmer_vm_memory32_fill",
1120 void_ty.fn_type(
1121 &[
1122 ctx_ptr_ty_basic_md,
1123 i32_ty_basic_md,
1124 i32_ty_basic_md,
1125 i32_ty_basic_md,
1126 i32_ty_basic_md,
1127 ],
1128 false,
1129 ),
1130 None,
1131 ),
1132 imported_memory_fill: add_function_with_attrs(
1133 "wasmer_vm_imported_memory32_fill",
1134 void_ty.fn_type(
1135 &[
1136 ctx_ptr_ty_basic_md,
1137 i32_ty_basic_md,
1138 i32_ty_basic_md,
1139 i32_ty_basic_md,
1140 i32_ty_basic_md,
1141 ],
1142 false,
1143 ),
1144 None,
1145 ),
1146 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1147 memory_grow_ty: i32_ty.fn_type(
1148 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1149 false,
1150 ),
1151 data_drop: add_function_with_attrs(
1152 "wasmer_vm_data_drop",
1153 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1154 None,
1155 ),
1156 func_ref: add_function_with_attrs(
1157 "wasmer_vm_func_ref",
1158 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1159 None,
1160 ),
1161 elem_drop: add_function_with_attrs(
1162 "wasmer_vm_elem_drop",
1163 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1164 None,
1165 ),
1166 throw_trap: add_function_with_attrs(
1167 "wasmer_vm_raise_trap",
1168 void_ty.fn_type(&[i32_ty_basic_md], false),
1169 None,
1170 ),
1171
1172 throw: add_function_with_attrs(
1173 "wasmer_vm_throw",
1174 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1175 None,
1176 ),
1177 alloc_exception: add_function_with_attrs(
1178 "wasmer_vm_alloc_exception",
1179 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1180 None,
1181 ),
1182 read_exnref: add_function_with_attrs(
1183 "wasmer_vm_read_exnref",
1184 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1185 None,
1186 ),
1187 exception_into_exnref: add_function_with_attrs(
1188 "wasmer_vm_exception_into_exnref",
1189 i32_ty.fn_type(&[ptr_ty.into()], false),
1190 None,
1191 ),
1192 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1193
1194 debug_ptr: add_function_with_attrs(
1195 "wasmer_vm_dbg_usize",
1196 void_ty.fn_type(&[ptr_ty.into()], false),
1197 None,
1198 ),
1199 debug_str: add_function_with_attrs(
1200 "wasmer_vm_dbg_str",
1201 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1202 None,
1203 ),
1204 memory_wait32: add_function_with_attrs(
1205 "wasmer_vm_memory32_atomic_wait32",
1206 i32_ty.fn_type(
1207 &[
1208 ctx_ptr_ty_basic_md,
1209 i32_ty_basic_md,
1210 i32_ty_basic_md,
1211 i32_ty_basic_md,
1212 i64_ty_basic_md,
1213 ],
1214 false,
1215 ),
1216 None,
1217 ),
1218 memory_wait32_ty: i32_ty.fn_type(
1219 &[
1220 ctx_ptr_ty_basic_md,
1221 i32_ty_basic_md,
1222 i32_ty_basic_md,
1223 i32_ty_basic_md,
1224 i64_ty_basic_md,
1225 ],
1226 false,
1227 ),
1228 imported_memory_wait32: add_function_with_attrs(
1229 "wasmer_vm_imported_memory32_atomic_wait32",
1230 i32_ty.fn_type(
1231 &[
1232 ctx_ptr_ty_basic_md,
1233 i32_ty_basic_md,
1234 i32_ty_basic_md,
1235 i32_ty_basic_md,
1236 i64_ty_basic_md,
1237 ],
1238 false,
1239 ),
1240 None,
1241 ),
1242 memory_wait64: add_function_with_attrs(
1243 "wasmer_vm_memory32_atomic_wait64",
1244 i32_ty.fn_type(
1245 &[
1246 ctx_ptr_ty_basic_md,
1247 i32_ty_basic_md,
1248 i32_ty_basic_md,
1249 i64_ty_basic_md,
1250 i64_ty_basic_md,
1251 ],
1252 false,
1253 ),
1254 None,
1255 ),
1256 memory_wait64_ty: i32_ty.fn_type(
1257 &[
1258 ctx_ptr_ty_basic_md,
1259 i32_ty_basic_md,
1260 i32_ty_basic_md,
1261 i64_ty_basic_md,
1262 i64_ty_basic_md,
1263 ],
1264 false,
1265 ),
1266 imported_memory_wait64: add_function_with_attrs(
1267 "wasmer_vm_imported_memory32_atomic_wait64",
1268 i32_ty.fn_type(
1269 &[
1270 ctx_ptr_ty_basic_md,
1271 i32_ty_basic_md,
1272 i32_ty_basic_md,
1273 i64_ty_basic_md,
1274 i64_ty_basic_md,
1275 ],
1276 false,
1277 ),
1278 None,
1279 ),
1280 memory_notify: add_function_with_attrs(
1281 "wasmer_vm_memory32_atomic_notify",
1282 i32_ty.fn_type(
1283 &[
1284 ctx_ptr_ty_basic_md,
1285 i32_ty_basic_md,
1286 i32_ty_basic_md,
1287 i32_ty_basic_md,
1288 ],
1289 false,
1290 ),
1291 None,
1292 ),
1293 memory_notify_ty: i32_ty.fn_type(
1294 &[
1295 ctx_ptr_ty_basic_md,
1296 i32_ty_basic_md,
1297 i32_ty_basic_md,
1298 i32_ty_basic_md,
1299 ],
1300 false,
1301 ),
1302 imported_memory_notify: add_function_with_attrs(
1303 "wasmer_vm_imported_memory32_atomic_notify",
1304 i32_ty.fn_type(
1305 &[
1306 ctx_ptr_ty_basic_md,
1307 i32_ty_basic_md,
1308 i32_ty_basic_md,
1309 i32_ty_basic_md,
1310 ],
1311 false,
1312 ),
1313 None,
1314 ),
1315
1316 vmfunction_import_ty: context.struct_type(
1317 &[
1318 i8_ptr_ty_basic,
1319 i8_ptr_ty_basic,
1320 i8_ptr_ty_basic,
1321 i1_ty_basic,
1322 ],
1323 false,
1324 ),
1325 vmfunction_import_body_element: 0,
1326 vmfunction_import_vmctx_element: 1,
1327 vmfunction_import_include_m0_param_element: 3,
1328 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1329 vmmemory_definition_base_element: 0,
1330 vmmemory_definition_current_length_element: 1,
1331
1332 ptr_ty,
1333
1334 x86_64: X86_64Intrinsics {
1335 pshufb128: add_function_with_attrs(
1336 "llvm.x86.ssse3.pshuf.b.128",
1337 ret_i8x16_take_i8x16_i8x16,
1338 None,
1339 ),
1340 pmaddubsw128: add_function_with_attrs(
1341 "llvm.x86.ssse3.pmadd.ub.sw.128",
1342 i16x8_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false),
1343 None,
1344 ),
1345 pmaddwd128: add_function_with_attrs(
1346 "llvm.x86.sse2.pmadd.wd",
1347 i32x4_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false),
1348 None,
1349 ),
1350 pmulhrsw128: add_function_with_attrs(
1351 "llvm.x86.ssse3.pmul.hr.sw.128",
1352 ret_i16x8_take_i16x8_i16x8,
1353 None,
1354 ),
1355 pblendvb: add_function_with_attrs(
1356 "llvm.x86.sse41.pblendvb",
1357 ret_i8x16_take_i8x16_i8x16_i8x16,
1358 None,
1359 ),
1360 min_ps: add_function_with_attrs(
1361 "llvm.x86.sse.min.ps",
1362 ret_f32x4_take_f32x4_f32x4,
1363 None,
1364 ),
1365 min_pd: add_function_with_attrs(
1366 "llvm.x86.sse2.min.pd",
1367 ret_f64x2_take_f64x2_f64x2,
1368 None,
1369 ),
1370 max_ps: add_function_with_attrs(
1371 "llvm.x86.sse.max.ps",
1372 ret_f32x4_take_f32x4_f32x4,
1373 None,
1374 ),
1375 max_pd: add_function_with_attrs(
1376 "llvm.x86.sse2.max.pd",
1377 ret_f64x2_take_f64x2_f64x2,
1378 None,
1379 ),
1380 cvttps2dq: add_function_with_attrs(
1381 "llvm.x86.sse2.cvttps2dq",
1382 ret_i32x4_take_f32x4,
1383 None,
1384 ),
1385 cvtps2udq128: add_function_with_attrs(
1386 "llvm.x86.avx512.mask.cvtps2udq.128",
1387 ret_i32x4_take_f32x4_i32x4_i8,
1388 None,
1389 ),
1390 cvtpd2dq: add_function_with_attrs(
1391 "llvm.x86.sse2.cvtpd2dq",
1392 ret_i32x4_take_f64x2,
1393 None,
1394 ),
1395 cvtpd2udq128: add_function_with_attrs(
1396 "llvm.x86.avx512.mask.cvtpd2udq.128",
1397 ret_i32x4_take_f64x2_i32x4_i8,
1398 None,
1399 ),
1400 },
1401 };
1402
1403 let noreturn =
1404 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1405 intrinsics
1406 .throw_trap
1407 .add_attribute(AttributeLoc::Function, noreturn);
1408 intrinsics
1409 }
1410}
1411
1412#[derive(Clone, Copy)]
1413pub enum MemoryCache<'ctx> {
1414 Dynamic {
1416 ptr_to_base_ptr: PointerValue<'ctx>,
1417 ptr_to_current_length: PointerValue<'ctx>,
1418 },
1419 Static { base_ptr: PointerValue<'ctx> },
1421}
1422
1423#[derive(Clone)]
1424struct TableCache<'ctx> {
1425 ptr_to_base_ptr: PointerValue<'ctx>,
1426 ptr_to_bounds: PointerValue<'ctx>,
1427}
1428
1429#[derive(Clone, Copy)]
1430pub enum GlobalCache<'ctx> {
1431 Mut {
1432 ptr_to_value: PointerValue<'ctx>,
1433 value_type: BasicTypeEnum<'ctx>,
1434 },
1435 Const {
1436 value: BasicValueEnum<'ctx>,
1437 },
1438}
1439
1440#[derive(Clone)]
1441pub struct FunctionCache<'ctx> {
1442 pub func: PointerValue<'ctx>,
1443 pub llvm_func_type: FunctionType<'ctx>,
1444 pub vmctx: BasicValueEnum<'ctx>,
1445 pub imported_include_m0_param: Option<IntValue<'ctx>>,
1446 pub attrs: Vec<(Attribute, AttributeLoc)>,
1447}
1448
1449pub struct CtxType<'ctx, 'a> {
1450 ctx_ptr_value: PointerValue<'ctx>,
1451 m0: Option<PointerValue<'ctx>>,
1452
1453 wasm_module: &'a WasmerCompilerModule,
1454 cache_builder: &'a Builder<'ctx>,
1455 abi: &'a dyn Abi,
1456
1457 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1458 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1459 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1460 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1461 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1462
1463 offsets: VMOffsets,
1464}
1465
1466impl<'ctx, 'a> CtxType<'ctx, 'a> {
1467 pub fn new(
1468 wasm_module: &'a WasmerCompilerModule,
1469 func_value: &FunctionValue<'ctx>,
1470 cache_builder: &'a Builder<'ctx>,
1471 abi: &'a dyn Abi,
1472 pointer_width: u8,
1473 m0: Option<PointerValue<'ctx>>,
1474 ) -> CtxType<'ctx, 'a> {
1475 CtxType {
1476 m0,
1477 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1478
1479 wasm_module,
1480 cache_builder,
1481 abi,
1482
1483 cached_memories: HashMap::new(),
1484 cached_tables: HashMap::new(),
1485 cached_globals: HashMap::new(),
1486 cached_functions: HashMap::new(),
1487 cached_memory_op: HashMap::new(),
1488
1489 offsets: VMOffsets::new(pointer_width, wasm_module),
1490 }
1491 }
1492
1493 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1494 self.ctx_ptr_value.as_basic_value_enum()
1495 }
1496
1497 pub fn memory(
1498 &mut self,
1499 index: MemoryIndex,
1500 intrinsics: &Intrinsics<'ctx>,
1501 module: &Module<'ctx>,
1502 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1503 ) -> Result<MemoryCache<'ctx>, CompileError> {
1504 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1505 &mut self.cached_memories,
1506 self.wasm_module,
1507 self.ctx_ptr_value,
1508 &self.cache_builder,
1509 &self.offsets,
1510 );
1511 let memory_style = &memory_styles[index];
1512 match cached_memories.get(&index) {
1513 Some(r) => Ok(*r),
1514 None => {
1515 let memory_definition_ptr =
1516 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1517 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1518 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1519 unsafe {
1520 err!(cache_builder.build_gep(
1521 intrinsics.i8_ty,
1522 ctx_ptr_value,
1523 &[offset],
1524 ""
1525 ))
1526 }
1527 } else {
1528 let offset = offsets.vmctx_vmmemory_import(index);
1529 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1530 let memory_definition_ptr_ptr = unsafe {
1531 err!(cache_builder.build_gep(
1532 intrinsics.i8_ty,
1533 ctx_ptr_value,
1534 &[offset],
1535 ""
1536 ))
1537 };
1538 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1539 memory_definition_ptr_ptr,
1540 intrinsics.ptr_ty,
1541 "",
1542 ))
1543 .into_pointer_value();
1544 let memory_definition_ptr = err!(cache_builder.build_load(
1545 intrinsics.ptr_ty,
1546 memory_definition_ptr_ptr,
1547 ""
1548 ))
1549 .into_pointer_value();
1550 tbaa_label(
1551 module,
1552 intrinsics,
1553 format!("memory {} definition", index.as_u32()),
1554 memory_definition_ptr.as_instruction_value().unwrap(),
1555 );
1556 memory_definition_ptr
1557 };
1558 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1559 memory_definition_ptr,
1560 intrinsics.ptr_ty,
1561 "",
1562 ))
1563 .into_pointer_value();
1564 let base_ptr = err!(cache_builder.build_struct_gep(
1565 intrinsics.vmmemory_definition_ty,
1566 memory_definition_ptr,
1567 intrinsics.vmmemory_definition_base_element,
1568 "",
1569 ));
1570 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1571 let current_length_ptr = err!(cache_builder.build_struct_gep(
1572 intrinsics.vmmemory_definition_ty,
1573 memory_definition_ptr,
1574 intrinsics.vmmemory_definition_current_length_element,
1575 "",
1576 ));
1577 MemoryCache::Dynamic {
1578 ptr_to_base_ptr: base_ptr,
1579 ptr_to_current_length: current_length_ptr,
1580 }
1581 } else {
1582 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1583 .into_pointer_value();
1584 tbaa_label(
1585 module,
1586 intrinsics,
1587 format!("memory base_ptr {}", index.as_u32()),
1588 base_ptr.as_instruction_value().unwrap(),
1589 );
1590 MemoryCache::Static { base_ptr }
1591 };
1592
1593 self.cached_memories.insert(index, value);
1594 Ok(*self.cached_memories.get(&index).unwrap())
1595 }
1596 }
1597 }
1598
1599 fn build_table_prepare(
1600 table_index: TableIndex,
1601 intrinsics: &Intrinsics<'ctx>,
1602 module: &Module<'ctx>,
1603 wasm_module: &WasmerCompilerModule,
1604 ctx_ptr_value: PointerValue<'ctx>,
1605 offsets: &VMOffsets,
1606 builder: &Builder<'ctx>,
1607 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1608 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1609 let offset = intrinsics.i64_ty.const_int(
1610 offsets
1611 .vmctx_vmtable_definition_base(local_table_index)
1612 .into(),
1613 false,
1614 );
1615 let ptr_to_base_ptr =
1616 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1617 let ptr_to_base_ptr =
1618 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1619 .into_pointer_value();
1620 let offset = intrinsics.i64_ty.const_int(
1621 offsets
1622 .vmctx_vmtable_definition_current_elements(local_table_index)
1623 .into(),
1624 false,
1625 );
1626 let ptr_to_bounds =
1627 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1628 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1629 .into_pointer_value();
1630 Ok((ptr_to_base_ptr, ptr_to_bounds))
1631 } else {
1632 let offset = intrinsics.i64_ty.const_int(
1633 offsets.vmctx_vmtable_import_definition(table_index).into(),
1634 false,
1635 );
1636 let definition_ptr_ptr =
1637 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1638 let definition_ptr_ptr =
1639 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1640 .into_pointer_value();
1641 let definition_ptr =
1642 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1643 .into_pointer_value();
1644 tbaa_label(
1645 module,
1646 intrinsics,
1647 format!("table {} definition", table_index.as_u32()),
1648 definition_ptr.as_instruction_value().unwrap(),
1649 );
1650
1651 let offset = intrinsics
1652 .i64_ty
1653 .const_int(offsets.vmtable_definition_base().into(), false);
1654 let ptr_to_base_ptr =
1655 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1656 let ptr_to_base_ptr =
1657 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1658 .into_pointer_value();
1659 let offset = intrinsics
1660 .i64_ty
1661 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1662 let ptr_to_bounds =
1663 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1664 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1665 .into_pointer_value();
1666 Ok((ptr_to_base_ptr, ptr_to_bounds))
1667 }
1668 }
1669
1670 fn table_prepare(
1671 &mut self,
1672 table_index: TableIndex,
1673 intrinsics: &Intrinsics<'ctx>,
1674 module: &Module<'ctx>,
1675 body_builder: &Builder<'ctx>,
1676 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1677 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1678 &mut self.cached_tables,
1679 self.wasm_module,
1680 self.ctx_ptr_value,
1681 &self.cache_builder,
1682 &self.offsets,
1683 );
1684
1685 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1686 CompileError::Codegen(format!(
1687 "Table index out of bounds: {}",
1688 table_index.as_u32()
1689 ))
1690 })?;
1691
1692 if is_growable {
1695 Self::build_table_prepare(
1696 table_index,
1697 intrinsics,
1698 module,
1699 wasm_module,
1700 ctx_ptr_value,
1701 offsets,
1702 body_builder,
1703 )
1704 } else {
1705 let TableCache {
1706 ptr_to_base_ptr,
1707 ptr_to_bounds,
1708 } = match cached_tables.entry(table_index) {
1709 Entry::Occupied(entry) => entry.get().clone(),
1710 Entry::Vacant(entry) => {
1711 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1712 table_index,
1713 intrinsics,
1714 module,
1715 wasm_module,
1716 ctx_ptr_value,
1717 offsets,
1718 cache_builder,
1719 )?;
1720
1721 let v = TableCache {
1722 ptr_to_base_ptr,
1723 ptr_to_bounds,
1724 };
1725
1726 entry.insert(v.clone());
1727
1728 v
1729 }
1730 };
1731
1732 Ok((ptr_to_base_ptr, ptr_to_bounds))
1733 }
1734 }
1735
1736 pub fn table(
1737 &mut self,
1738 index: TableIndex,
1739 intrinsics: &Intrinsics<'ctx>,
1740 module: &Module<'ctx>,
1741 body_builder: &Builder<'ctx>,
1742 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1743 let (ptr_to_base_ptr, ptr_to_bounds) =
1744 self.table_prepare(index, intrinsics, module, body_builder)?;
1745
1746 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1748 &body_builder
1749 } else {
1750 &self.cache_builder
1751 };
1752
1753 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1754 .into_pointer_value();
1755 let bounds =
1756 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1757 tbaa_label(
1758 module,
1759 intrinsics,
1760 format!("table_base_ptr {}", index.index()),
1761 base_ptr.as_instruction_value().unwrap(),
1762 );
1763 tbaa_label(
1764 module,
1765 intrinsics,
1766 format!("table_bounds {}", index.index()),
1767 bounds.as_instruction_value().unwrap(),
1768 );
1769 Ok((base_ptr, bounds))
1770 }
1771
1772 pub fn fixed_funcref_table_anyfuncs(
1774 &self,
1775 index: LocalTableIndex,
1776 intrinsics: &Intrinsics<'ctx>,
1777 builder: &Builder<'ctx>,
1778 ) -> Result<PointerValue<'ctx>, CompileError> {
1779 let offset = intrinsics.i64_ty.const_int(
1780 self.offsets
1781 .vmctx_fixed_funcref_table_anyfuncs(index)
1782 .expect("fixed funcref table must have inline VMContext storage")
1783 .into(),
1784 false,
1785 );
1786 let ptr =
1787 unsafe { err!(builder.build_gep(intrinsics.i8_ty, self.ctx_ptr_value, &[offset], "")) };
1788 Ok(err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value())
1789 }
1790
1791 pub fn global(
1792 &mut self,
1793 index: GlobalIndex,
1794 intrinsics: &Intrinsics<'ctx>,
1795 module: &Module<'ctx>,
1796 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1797 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1798 &mut self.cached_globals,
1799 self.wasm_module,
1800 self.ctx_ptr_value,
1801 &self.cache_builder,
1802 &self.offsets,
1803 );
1804 match cached_globals.entry(index) {
1805 Entry::Occupied(entry) => Ok(entry.into_mut()),
1806 Entry::Vacant(entry) => {
1807 let global_type = wasm_module.globals[index];
1808 let global_value_type = global_type.ty;
1809
1810 let global_mutability = global_type.mutability;
1811 let global_ptr = if let Some(local_global_index) =
1812 wasm_module.local_global_index(index)
1813 {
1814 let offset = offsets.vmctx_vmglobal_definition(local_global_index);
1815 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1816 unsafe {
1817 err!(cache_builder.build_gep(
1818 intrinsics.i8_ty,
1819 ctx_ptr_value,
1820 &[offset],
1821 ""
1822 ))
1823 }
1824 } else {
1825 let offset = offsets.vmctx_vmglobal_import(index);
1826 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1827 let global_ptr_ptr = unsafe {
1828 err!(cache_builder.build_gep(
1829 intrinsics.i8_ty,
1830 ctx_ptr_value,
1831 &[offset],
1832 ""
1833 ))
1834 };
1835 let global_ptr_ptr =
1836 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1837 .into_pointer_value();
1838 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1839 .into_pointer_value()
1840 };
1841 let global_ptr =
1842 err!(cache_builder.build_bit_cast(global_ptr, intrinsics.ptr_ty, "",))
1843 .into_pointer_value();
1844
1845 let ret = entry.insert(match global_mutability {
1846 Mutability::Const => {
1847 let value = err!(cache_builder.build_load(
1848 type_to_llvm(intrinsics, global_value_type)?,
1849 global_ptr,
1850 "",
1851 ));
1852 tbaa_label(
1853 module,
1854 intrinsics,
1855 format!("global {}", index.as_u32()),
1856 value.as_instruction_value().unwrap(),
1857 );
1858 GlobalCache::Const { value }
1859 }
1860 Mutability::Var => GlobalCache::Mut {
1861 ptr_to_value: global_ptr,
1862 value_type: type_to_llvm(intrinsics, global_value_type)?,
1863 },
1864 });
1865
1866 Ok(ret)
1867 }
1868 }
1869 }
1870
1871 pub fn add_func(
1872 &mut self,
1873 function_index: FunctionIndex,
1874 func: PointerValue<'ctx>,
1875 llvm_func_type: FunctionType<'ctx>,
1876 vmctx: BasicValueEnum<'ctx>,
1877 attrs: &[(Attribute, AttributeLoc)],
1878 ) {
1879 match self.cached_functions.entry(function_index) {
1880 Entry::Occupied(_) => unreachable!("duplicate function"),
1881 Entry::Vacant(entry) => {
1882 entry.insert(FunctionCache {
1883 func,
1884 llvm_func_type,
1885 vmctx,
1886 imported_include_m0_param: None,
1887 attrs: attrs.to_vec(),
1888 });
1889 }
1890 }
1891 }
1892
1893 #[allow(clippy::too_many_arguments)]
1894 pub fn local_func(
1895 &mut self,
1896 _local_function_index: LocalFunctionIndex,
1897 function_index: FunctionIndex,
1898 intrinsics: &Intrinsics<'ctx>,
1899 module: &Module<'ctx>,
1900 context: &'ctx Context,
1901 func_type: &FuncType,
1902 function_name: &str,
1903 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1904 let (cached_functions, ctx_ptr_value, offsets) = (
1905 &mut self.cached_functions,
1906 &self.ctx_ptr_value,
1907 &self.offsets,
1908 );
1909 Ok(match cached_functions.entry(function_index) {
1910 Entry::Occupied(entry) => entry.into_mut(),
1911 Entry::Vacant(entry) => {
1912 debug_assert!(module.get_function(function_name).is_none());
1913 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1914 context,
1915 intrinsics,
1916 Some(offsets),
1917 func_type,
1918 self.m0.is_some(),
1919 )?;
1920 let func =
1921 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1922 for (attr, attr_loc) in &llvm_func_attrs {
1923 func.add_attribute(*attr_loc, *attr);
1924 }
1925 entry.insert(FunctionCache {
1926 func: func.as_global_value().as_pointer_value(),
1927 llvm_func_type,
1928 vmctx: ctx_ptr_value.as_basic_value_enum(),
1929 imported_include_m0_param: None,
1930 attrs: llvm_func_attrs,
1931 })
1932 }
1933 })
1934 }
1935
1936 pub fn imported_func(
1937 &mut self,
1938 function_index: FunctionIndex,
1939 intrinsics: &Intrinsics<'ctx>,
1940 context: &'ctx Context,
1941 func_type: &FuncType,
1942 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1943 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1944 &mut self.cached_functions,
1945 self.wasm_module,
1946 &self.ctx_ptr_value,
1947 &self.cache_builder,
1948 &self.offsets,
1949 );
1950 match cached_functions.entry(function_index) {
1951 Entry::Occupied(entry) => Ok(entry.into_mut()),
1952 Entry::Vacant(entry) => {
1953 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1954 context,
1955 intrinsics,
1956 Some(offsets),
1957 func_type,
1958 self.m0.is_some(),
1959 )?;
1960 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1961 let offset = offsets.vmctx_vmfunction_import(function_index);
1962 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1963 let vmfunction_import_ptr = unsafe {
1964 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1965 };
1966 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1967 vmfunction_import_ptr,
1968 intrinsics.ptr_ty,
1969 "",
1970 ))
1971 .into_pointer_value();
1972 vmfunction_import_ptr.set_name("vmfunction_import_ptr");
1973
1974 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1975 intrinsics.vmfunction_import_ty,
1976 vmfunction_import_ptr,
1977 intrinsics.vmfunction_import_body_element,
1978 "",
1979 ));
1980 body_ptr_ptr.set_name("body_ptr_ptr");
1981 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
1982 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
1983 .into_pointer_value();
1984 body_ptr.set_name("body_ptr");
1985 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
1986 intrinsics.vmfunction_import_ty,
1987 vmfunction_import_ptr,
1988 intrinsics.vmfunction_import_vmctx_element,
1989 "",
1990 ));
1991 vmctx_ptr_ptr.set_name("vmctx_ptr_ptr");
1992 let vmctx_ptr =
1993 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
1994 vmctx_ptr.set_name("vmctx_ptr");
1995 let include_m0_param_ptr = err!(cache_builder.build_struct_gep(
1996 intrinsics.vmfunction_import_ty,
1997 vmfunction_import_ptr,
1998 intrinsics.vmfunction_import_include_m0_param_element,
1999 "",
2000 ));
2001 include_m0_param_ptr.set_name("include_m0_param_ptr");
2002 let imported_include_m0_param =
2003 err!(cache_builder.build_load(intrinsics.i1_ty, include_m0_param_ptr, "",))
2004 .into_int_value();
2005 imported_include_m0_param.set_name("imported_include_m0_param");
2006
2007 Ok(entry.insert(FunctionCache {
2008 func: body_ptr,
2009 llvm_func_type,
2010 vmctx: vmctx_ptr,
2011 imported_include_m0_param: Some(imported_include_m0_param),
2012 attrs: llvm_func_attrs,
2013 }))
2014 }
2015 }
2016 }
2017
2018 pub fn memory_grow(
2019 &mut self,
2020 memory_index: MemoryIndex,
2021 intrinsics: &Intrinsics<'ctx>,
2022 ) -> Result<PointerValue<'ctx>, CompileError> {
2023 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2024 &mut self.cached_memory_op,
2025 &self.wasm_module,
2026 &self.offsets,
2027 &self.cache_builder,
2028 &self.ctx_ptr_value,
2029 );
2030 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
2031 Entry::Occupied(entry) => Ok(*entry.get()),
2032 Entry::Vacant(entry) => {
2033 let (grow_fn, grow_fn_ty) =
2034 if wasm_module.local_memory_index(memory_index).is_some() {
2035 (
2036 VMBuiltinFunctionIndex::get_memory32_grow_index(),
2037 intrinsics.ptr_ty,
2038 )
2039 } else {
2040 (
2041 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
2042 intrinsics.ptr_ty,
2043 )
2044 };
2045 let offset = offsets.vmctx_builtin_function(grow_fn);
2046 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2047 let grow_fn_ptr_ptr = unsafe {
2048 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2049 };
2050
2051 let grow_fn_ptr_ptr =
2052 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2053 .into_pointer_value();
2054 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
2055 .into_pointer_value();
2056
2057 entry.insert(val);
2058 Ok(val)
2059 }
2060 }
2061 }
2062
2063 pub fn memory_size(
2064 &mut self,
2065 memory_index: MemoryIndex,
2066 intrinsics: &Intrinsics<'ctx>,
2067 ) -> Result<PointerValue<'ctx>, CompileError> {
2068 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2069 &mut self.cached_memory_op,
2070 &self.wasm_module,
2071 &self.offsets,
2072 &self.cache_builder,
2073 &self.ctx_ptr_value,
2074 );
2075
2076 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
2077 Entry::Occupied(entry) => Ok(*entry.get()),
2078 Entry::Vacant(entry) => {
2079 let (size_fn, size_fn_ty) =
2080 if wasm_module.local_memory_index(memory_index).is_some() {
2081 (
2082 VMBuiltinFunctionIndex::get_memory32_size_index(),
2083 intrinsics.ptr_ty,
2084 )
2085 } else {
2086 (
2087 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
2088 intrinsics.ptr_ty,
2089 )
2090 };
2091 let offset = offsets.vmctx_builtin_function(size_fn);
2092 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2093 let size_fn_ptr_ptr = unsafe {
2094 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2095 };
2096
2097 let size_fn_ptr_ptr =
2098 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2099 .into_pointer_value();
2100
2101 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2102 .into_pointer_value();
2103 entry.insert(val);
2104 Ok(val)
2105 }
2106 }
2107 }
2108
2109 pub fn memory_wait32(
2110 &mut self,
2111 memory_index: MemoryIndex,
2112 intrinsics: &Intrinsics<'ctx>,
2113 ) -> Result<PointerValue<'ctx>, CompileError> {
2114 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2115 &mut self.cached_memory_op,
2116 &self.wasm_module,
2117 &self.offsets,
2118 &self.cache_builder,
2119 &self.ctx_ptr_value,
2120 );
2121 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
2122 Entry::Occupied(entry) => Ok(*entry.get()),
2123 Entry::Vacant(entry) => {
2124 let (size_fn, size_fn_ty) =
2125 if wasm_module.local_memory_index(memory_index).is_some() {
2126 (
2127 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
2128 intrinsics.ptr_ty,
2129 )
2130 } else {
2131 (
2132 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
2133 intrinsics.ptr_ty,
2134 )
2135 };
2136 let offset = offsets.vmctx_builtin_function(size_fn);
2137 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2138 let size_fn_ptr_ptr = unsafe {
2139 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2140 };
2141
2142 let size_fn_ptr_ptr =
2143 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2144 .into_pointer_value();
2145
2146 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2147 .into_pointer_value();
2148
2149 entry.insert(val);
2150 Ok(val)
2151 }
2152 }
2153 }
2154
2155 pub fn memory_wait64(
2156 &mut self,
2157 memory_index: MemoryIndex,
2158 intrinsics: &Intrinsics<'ctx>,
2159 ) -> Result<PointerValue<'ctx>, CompileError> {
2160 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2161 &mut self.cached_memory_op,
2162 &self.wasm_module,
2163 &self.offsets,
2164 &self.cache_builder,
2165 &self.ctx_ptr_value,
2166 );
2167
2168 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
2169 Entry::Occupied(entry) => Ok(*entry.get()),
2170 Entry::Vacant(entry) => {
2171 let (size_fn, size_fn_ty) =
2172 if wasm_module.local_memory_index(memory_index).is_some() {
2173 (
2174 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
2175 intrinsics.ptr_ty,
2176 )
2177 } else {
2178 (
2179 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
2180 intrinsics.ptr_ty,
2181 )
2182 };
2183 let offset = offsets.vmctx_builtin_function(size_fn);
2184 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2185 let size_fn_ptr_ptr = unsafe {
2186 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2187 };
2188
2189 let size_fn_ptr_ptr =
2190 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2191 .into_pointer_value();
2192
2193 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2194 .into_pointer_value();
2195 entry.insert(val);
2196 Ok(val)
2197 }
2198 }
2199 }
2200
2201 pub fn memory_notify(
2202 &mut self,
2203 memory_index: MemoryIndex,
2204 intrinsics: &Intrinsics<'ctx>,
2205 ) -> Result<PointerValue<'ctx>, CompileError> {
2206 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2207 &mut self.cached_memory_op,
2208 &self.wasm_module,
2209 &self.offsets,
2210 &self.cache_builder,
2211 &self.ctx_ptr_value,
2212 );
2213 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2214 Entry::Occupied(entry) => Ok(*entry.get()),
2215 Entry::Vacant(entry) => {
2216 let (size_fn, size_fn_ty) =
2217 if wasm_module.local_memory_index(memory_index).is_some() {
2218 (
2219 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2220 intrinsics.ptr_ty,
2221 )
2222 } else {
2223 (
2224 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2225 intrinsics.ptr_ty,
2226 )
2227 };
2228 let offset = offsets.vmctx_builtin_function(size_fn);
2229 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2230 let size_fn_ptr_ptr = unsafe {
2231 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2232 };
2233
2234 let size_fn_ptr_ptr =
2235 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2236 .into_pointer_value();
2237
2238 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2239 .into_pointer_value();
2240
2241 entry.insert(val);
2242 Ok(val)
2243 }
2244 }
2245 }
2246
2247 pub fn get_offsets(&self) -> &VMOffsets {
2248 &self.offsets
2249 }
2250}
2251
2252pub fn tbaa_label<'ctx>(
2255 module: &Module<'ctx>,
2256 intrinsics: &Intrinsics<'ctx>,
2257 label: String,
2258 instruction: InstructionValue<'ctx>,
2259) {
2260 let context = module.get_context();
2274
2275 let tbaa_root = module
2277 .get_global_metadata("wasmer_tbaa_root")
2278 .pop()
2279 .unwrap_or_else(|| {
2280 module
2281 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2282 .unwrap();
2283 module.get_global_metadata("wasmer_tbaa_root")[0]
2284 });
2285
2286 let type_label = context.metadata_string(label.as_str());
2289 let type_tbaa = module
2290 .get_global_metadata(label.as_str())
2291 .pop()
2292 .unwrap_or_else(|| {
2293 module
2294 .add_global_metadata(
2295 label.as_str(),
2296 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2297 )
2298 .unwrap();
2299 module.get_global_metadata(label.as_str())[0]
2300 });
2301
2302 let label = label + "_memop";
2309 let type_tbaa = module
2310 .get_global_metadata(label.as_str())
2311 .pop()
2312 .unwrap_or_else(|| {
2313 module
2314 .add_global_metadata(
2315 label.as_str(),
2316 &context.metadata_node(&[
2317 type_tbaa.into(),
2318 type_tbaa.into(),
2319 intrinsics.i64_zero.into(),
2320 ]),
2321 )
2322 .unwrap();
2323 module.get_global_metadata(label.as_str())[0]
2324 });
2325
2326 let tbaa_kind = context.get_kind_id("tbaa");
2328 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2329}
2330
2331fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2332 let table = module.tables.get(index)?;
2333 match table.maximum {
2334 None => Some(true),
2335 Some(max) => Some(max > table.minimum),
2336 }
2337}