1use crate::abi::LLVMAbi;
8use crate::error::err;
9use inkwell::values::BasicMetadataValueEnum;
10use inkwell::{
11 AddressSpace,
12 attributes::{Attribute, AttributeLoc},
13 builder::Builder,
14 context::Context,
15 module::{Linkage, Module},
16 targets::TargetData,
17 types::{
18 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
19 PointerType, StructType, VectorType, VoidType,
20 },
21 values::{
22 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
23 PointerValue, VectorValue,
24 },
25};
26use std::collections::{HashMap, hash_map::Entry};
27use std::num::NonZero;
28use target_lexicon::{Architecture, Triple};
29use wasmer_types::entity::{EntityRef, PrimaryMap};
30use wasmer_types::{
31 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
32 LocalTableIndex, MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, TableIndex, Type,
33};
34use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
35
36pub fn type_to_llvm<'ctx>(
37 intrinsics: &Intrinsics<'ctx>,
38 ty: Type,
39) -> Result<BasicTypeEnum<'ctx>, CompileError> {
40 match ty {
41 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
42 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
43 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
44 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
45 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
46 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
47 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
48 }
49}
50
51#[allow(dead_code)]
53pub struct X86_64Intrinsics<'ctx> {
54 pub pshufb128: FunctionValue<'ctx>,
55 pub pmaddubsw128: FunctionValue<'ctx>,
56 pub pmaddwd128: FunctionValue<'ctx>,
57 pub pmulhrsw128: FunctionValue<'ctx>,
58 pub pblendvb: FunctionValue<'ctx>,
59 pub min_ps: FunctionValue<'ctx>,
60 pub min_pd: FunctionValue<'ctx>,
61 pub max_ps: FunctionValue<'ctx>,
62 pub max_pd: FunctionValue<'ctx>,
63 pub cvttps2dq: FunctionValue<'ctx>,
64 pub cvtps2udq128: FunctionValue<'ctx>,
65 pub cvtpd2dq: FunctionValue<'ctx>,
66 pub cvtpd2udq128: FunctionValue<'ctx>,
67}
68
69#[allow(dead_code)]
71pub struct Intrinsics<'ctx> {
72 pub ctlz_i32: FunctionValue<'ctx>,
73 pub ctlz_i64: FunctionValue<'ctx>,
74
75 pub cttz_i32: FunctionValue<'ctx>,
76 pub cttz_i64: FunctionValue<'ctx>,
77
78 pub ctpop_i32: FunctionValue<'ctx>,
79 pub ctpop_i64: FunctionValue<'ctx>,
80 pub ctpop_i8x16: FunctionValue<'ctx>,
81
82 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
83 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
84 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
85 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
86 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
87
88 pub add_f32: FunctionValue<'ctx>,
89 pub add_f64: FunctionValue<'ctx>,
90 pub add_f32x4: FunctionValue<'ctx>,
91 pub add_f64x2: FunctionValue<'ctx>,
92
93 pub sub_f32: FunctionValue<'ctx>,
94 pub sub_f64: FunctionValue<'ctx>,
95 pub sub_f32x4: FunctionValue<'ctx>,
96 pub sub_f64x2: FunctionValue<'ctx>,
97
98 pub mul_f32: FunctionValue<'ctx>,
99 pub mul_f64: FunctionValue<'ctx>,
100 pub mul_f32x4: FunctionValue<'ctx>,
101 pub mul_f64x2: FunctionValue<'ctx>,
102 pub muladd_f32x4: FunctionValue<'ctx>,
103 pub muladd_f64x2: FunctionValue<'ctx>,
104
105 pub div_f32: FunctionValue<'ctx>,
106 pub div_f64: FunctionValue<'ctx>,
107 pub div_f32x4: FunctionValue<'ctx>,
108 pub div_f64x2: FunctionValue<'ctx>,
109
110 pub sqrt_f32: FunctionValue<'ctx>,
111 pub sqrt_f64: FunctionValue<'ctx>,
112 pub sqrt_f32x4: FunctionValue<'ctx>,
113 pub sqrt_f64x2: FunctionValue<'ctx>,
114
115 pub cmp_f32: FunctionValue<'ctx>,
116 pub cmp_f64: FunctionValue<'ctx>,
117 pub cmp_f32x4: FunctionValue<'ctx>,
118 pub cmp_f64x2: FunctionValue<'ctx>,
119
120 pub minimum_f32: FunctionValue<'ctx>,
121 pub minimum_f64: FunctionValue<'ctx>,
122 pub minimum_f32x4: FunctionValue<'ctx>,
123 pub minimum_f64x2: FunctionValue<'ctx>,
124
125 pub maximum_f32: FunctionValue<'ctx>,
126 pub maximum_f64: FunctionValue<'ctx>,
127 pub maximum_f32x4: FunctionValue<'ctx>,
128 pub maximum_f64x2: FunctionValue<'ctx>,
129
130 pub ceil_f32: FunctionValue<'ctx>,
131 pub ceil_f64: FunctionValue<'ctx>,
132 pub ceil_f32x4: FunctionValue<'ctx>,
133 pub ceil_f64x2: FunctionValue<'ctx>,
134
135 pub floor_f32: FunctionValue<'ctx>,
136 pub floor_f64: FunctionValue<'ctx>,
137 pub floor_f32x4: FunctionValue<'ctx>,
138 pub floor_f64x2: FunctionValue<'ctx>,
139
140 pub trunc_f32: FunctionValue<'ctx>,
141 pub trunc_f64: FunctionValue<'ctx>,
142 pub trunc_f32x4: FunctionValue<'ctx>,
143 pub trunc_f64x2: FunctionValue<'ctx>,
144
145 pub fpext_f32: FunctionValue<'ctx>,
146 pub fptrunc_f64: FunctionValue<'ctx>,
147
148 pub nearbyint_f32: FunctionValue<'ctx>,
149 pub nearbyint_f64: FunctionValue<'ctx>,
150 pub nearbyint_f32x4: FunctionValue<'ctx>,
151 pub nearbyint_f64x2: FunctionValue<'ctx>,
152
153 pub fabs_f32: FunctionValue<'ctx>,
154 pub fabs_f64: FunctionValue<'ctx>,
155 pub fabs_f32x4: FunctionValue<'ctx>,
156 pub fabs_f64x2: FunctionValue<'ctx>,
157
158 pub copysign_f32: FunctionValue<'ctx>,
159 pub copysign_f64: FunctionValue<'ctx>,
160 pub copysign_f32x4: FunctionValue<'ctx>,
161 pub copysign_f64x2: FunctionValue<'ctx>,
162
163 pub sadd_sat_i8x16: FunctionValue<'ctx>,
164 pub sadd_sat_i16x8: FunctionValue<'ctx>,
165 pub uadd_sat_i8x16: FunctionValue<'ctx>,
166 pub uadd_sat_i16x8: FunctionValue<'ctx>,
167
168 pub ssub_sat_i8x16: FunctionValue<'ctx>,
169 pub ssub_sat_i16x8: FunctionValue<'ctx>,
170 pub usub_sat_i8x16: FunctionValue<'ctx>,
171 pub usub_sat_i16x8: FunctionValue<'ctx>,
172
173 pub expect_i1: FunctionValue<'ctx>,
174 pub trap: FunctionValue<'ctx>,
175 pub debug_trap: FunctionValue<'ctx>,
176
177 pub personality: FunctionValue<'ctx>,
178 pub personality2: FunctionValue<'ctx>,
179 pub readonly: Attribute,
180 pub stack_probe: Attribute,
181 pub uwtable: Attribute,
182 pub nounwind: Attribute,
183 pub frame_pointer: Attribute,
184 pub chkstk: FunctionValue<'ctx>,
186
187 pub void_ty: VoidType<'ctx>,
188 pub i1_ty: IntType<'ctx>,
189 pub i2_ty: IntType<'ctx>,
190 pub i4_ty: IntType<'ctx>,
191 pub i8_ty: IntType<'ctx>,
192 pub i16_ty: IntType<'ctx>,
193 pub i32_ty: IntType<'ctx>,
194 pub i64_ty: IntType<'ctx>,
195 pub i128_ty: IntType<'ctx>,
196 pub isize_ty: IntType<'ctx>,
197 pub f32_ty: FloatType<'ctx>,
198 pub f64_ty: FloatType<'ctx>,
199
200 pub i1x128_ty: VectorType<'ctx>,
201 pub i8x16_ty: VectorType<'ctx>,
202 pub i16x8_ty: VectorType<'ctx>,
203 pub i32x4_ty: VectorType<'ctx>,
204 pub i64x2_ty: VectorType<'ctx>,
205 pub f32x4_ty: VectorType<'ctx>,
206 pub f64x2_ty: VectorType<'ctx>,
207 pub i32x8_ty: VectorType<'ctx>,
208
209 pub ptr_ty: PointerType<'ctx>,
210
211 pub x86_64: X86_64Intrinsics<'ctx>,
212
213 pub anyfunc_ty: StructType<'ctx>,
214
215 pub i1_zero: IntValue<'ctx>,
216 pub i8_zero: IntValue<'ctx>,
217 pub i32_zero: IntValue<'ctx>,
218 pub i64_zero: IntValue<'ctx>,
219 pub i128_zero: IntValue<'ctx>,
220 pub isize_zero: IntValue<'ctx>,
221 pub f32_zero: FloatValue<'ctx>,
222 pub f64_zero: FloatValue<'ctx>,
223 pub f32x4_zero: VectorValue<'ctx>,
224 pub f64x2_zero: VectorValue<'ctx>,
225 pub i32_consts: [IntValue<'ctx>; 16],
226
227 pub trap_unreachable: BasicValueEnum<'ctx>,
228 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
229 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
230 pub trap_memory_oob: BasicValueEnum<'ctx>,
231 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
232 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
233 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
234 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
235 pub trap_table_access_oob: BasicValueEnum<'ctx>,
236
237 pub experimental_stackmap: FunctionValue<'ctx>,
238
239 pub table_copy: FunctionValue<'ctx>,
241 pub table_init: FunctionValue<'ctx>,
242 pub table_fill: FunctionValue<'ctx>,
243 pub table_size: FunctionValue<'ctx>,
244 pub imported_table_size: FunctionValue<'ctx>,
245 pub table_get: FunctionValue<'ctx>,
246 pub imported_table_get: FunctionValue<'ctx>,
247 pub table_set: FunctionValue<'ctx>,
248 pub imported_table_set: FunctionValue<'ctx>,
249 pub table_grow: FunctionValue<'ctx>,
250 pub imported_table_grow: FunctionValue<'ctx>,
251 pub memory_init: FunctionValue<'ctx>,
252 pub data_drop: FunctionValue<'ctx>,
253 pub func_ref: FunctionValue<'ctx>,
254 pub elem_drop: FunctionValue<'ctx>,
255 pub memory_copy: FunctionValue<'ctx>,
256 pub memory_fill: FunctionValue<'ctx>,
257 pub imported_memory_fill: FunctionValue<'ctx>,
258 pub memory_size_ty: FunctionType<'ctx>,
259 pub memory_grow_ty: FunctionType<'ctx>,
260 pub memory_wait32: FunctionValue<'ctx>,
261 pub memory_wait32_ty: FunctionType<'ctx>,
262 pub imported_memory_wait32: FunctionValue<'ctx>,
263 pub memory_wait64: FunctionValue<'ctx>,
264 pub memory_wait64_ty: FunctionType<'ctx>,
265 pub imported_memory_wait64: FunctionValue<'ctx>,
266 pub memory_notify: FunctionValue<'ctx>,
267 pub memory_notify_ty: FunctionType<'ctx>,
268 pub imported_memory_notify: FunctionValue<'ctx>,
269
270 pub throw_trap: FunctionValue<'ctx>,
271
272 pub throw: FunctionValue<'ctx>,
274 pub alloc_exception: FunctionValue<'ctx>,
275 pub read_exnref: FunctionValue<'ctx>,
276 pub exception_into_exnref: FunctionValue<'ctx>,
277 pub lpad_exception_ty: StructType<'ctx>,
278
279 pub debug_ptr: FunctionValue<'ctx>,
281 pub debug_str: FunctionValue<'ctx>,
282
283 pub vmfunction_import_ty: StructType<'ctx>,
285 pub vmfunction_import_body_element: u32,
286 pub vmfunction_import_vmctx_element: u32,
287 pub vmfunction_import_include_m0_param_element: u32,
288
289 pub vmmemory_definition_ty: StructType<'ctx>,
290 pub vmmemory_definition_base_element: u32,
291 pub vmmemory_definition_current_length_element: u32,
292}
293
294#[derive(Debug, Hash, PartialEq, Eq)]
295enum MemoryOp {
296 Size,
297 Grow,
298 Wait32,
299 Wait64,
300 Notify,
301}
302
303impl<'ctx> Intrinsics<'ctx> {
304 pub fn declare(
306 module: &Module<'ctx>,
307 context: &'ctx Context,
308 target_data: &TargetData,
309 target_triple: &Triple,
310 binary_fmt: &target_lexicon::BinaryFormat,
311 ) -> Self {
312 let is_riscv64 = matches!(target_triple.architecture, Architecture::Riscv64(..));
313 let void_ty = context.void_type();
314 let i1_ty = context.bool_type();
315 let i2_ty = context
316 .custom_width_int_type(NonZero::new(2).unwrap())
317 .unwrap();
318 let i4_ty = context
319 .custom_width_int_type(NonZero::new(4).unwrap())
320 .unwrap();
321 let i8_ty = context.i8_type();
322 let i16_ty = context.i16_type();
323 let i32_ty = context.i32_type();
324 let i64_ty = context.i64_type();
325 let i128_ty = context.i128_type();
326 let isize_ty = context.ptr_sized_int_type(target_data, None);
327 let f32_ty = context.f32_type();
328 let f64_ty = context.f64_type();
329
330 let i1x4_ty = i1_ty.vec_type(4);
331 let i1x2_ty = i1_ty.vec_type(2);
332 let i1x128_ty = i1_ty.vec_type(128);
333 let i8x16_ty = i8_ty.vec_type(16);
334 let i16x8_ty = i16_ty.vec_type(8);
335 let i32x4_ty = i32_ty.vec_type(4);
336 let i64x2_ty = i64_ty.vec_type(2);
337 let f32x4_ty = f32_ty.vec_type(4);
338 let f64x2_ty = f64_ty.vec_type(2);
339 let i32x8_ty = i32_ty.vec_type(8);
340
341 let ptr_ty = context.ptr_type(AddressSpace::default());
342
343 let i1_zero = i1_ty.const_int(0, false);
344 let i8_zero = i8_ty.const_int(0, false);
345 let i32_zero = i32_ty.const_int(0, false);
346 let i64_zero = i64_ty.const_int(0, false);
347 let i128_zero = i128_ty.const_int(0, false);
348 let isize_zero = isize_ty.const_int(0, false);
349 let f32_zero = f32_ty.const_float(0.0);
350 let f64_zero = f64_ty.const_float(0.0);
351 let f32x4_zero = f32x4_ty.const_zero();
352 let f64x2_zero = f64x2_ty.const_zero();
353 let i32_consts = [
354 i32_ty.const_int(0, false),
355 i32_ty.const_int(1, false),
356 i32_ty.const_int(2, false),
357 i32_ty.const_int(3, false),
358 i32_ty.const_int(4, false),
359 i32_ty.const_int(5, false),
360 i32_ty.const_int(6, false),
361 i32_ty.const_int(7, false),
362 i32_ty.const_int(8, false),
363 i32_ty.const_int(9, false),
364 i32_ty.const_int(10, false),
365 i32_ty.const_int(11, false),
366 i32_ty.const_int(12, false),
367 i32_ty.const_int(13, false),
368 i32_ty.const_int(14, false),
369 i32_ty.const_int(15, false),
370 ];
371
372 let md_ty = context.metadata_type();
373
374 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
375 let i1_ty_basic = i1_ty.as_basic_type_enum();
376
377 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
378 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
379 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
380 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
381 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
382 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
383 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
384 let i32x4_ty_basic_md: BasicMetadataTypeEnum = i32x4_ty.into();
385 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
386 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
387 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
388
389 let ctx_ptr_ty = ptr_ty;
390 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
391 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
392
393 let anyfunc_ty = context.struct_type(
397 &[
398 i8_ptr_ty_basic,
399 i32_ty.into(),
400 ctx_ptr_ty_basic,
401 ptr_ty.into(),
402 ],
403 false,
404 );
405 let funcref_ty = ptr_ty;
406 let anyref_ty = ptr_ty;
407 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
408
409 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
410 let ret_i8x16_take_i8x16_i8x16 =
411 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
412 let ret_i8x16_take_i8x16_i8x16_i8x16 = i8x16_ty.fn_type(
413 &[i8x16_ty_basic_md, i8x16_ty_basic_md, i8x16_ty_basic_md],
414 false,
415 );
416 let ret_i16x8_take_i16x8_i16x8 =
417 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
418
419 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
420 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
421
422 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
423 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
424
425 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
426 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
427 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
428 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
429
430 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
431 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
432 let ret_f32x4_take_f32x4_f32x4 =
433 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
434 let ret_f64x2_take_f64x2_f64x2 =
435 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
436 let ret_f32x4_take_f32x4_f32x4_f32x4_md_md = f32x4_ty.fn_type(
437 &[
438 f32x4_ty_basic_md,
439 f32x4_ty_basic_md,
440 f32x4_ty_basic_md,
441 md_ty_basic_md,
442 md_ty_basic_md,
443 ],
444 false,
445 );
446 let ret_f64x2_take_f64x2_f64x2_f64x2_md_md = f64x2_ty.fn_type(
447 &[
448 f64x2_ty_basic_md,
449 f64x2_ty_basic_md,
450 f64x2_ty_basic_md,
451 md_ty_basic_md,
452 md_ty_basic_md,
453 ],
454 false,
455 );
456
457 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
458 let ret_f32_take_f64_md_md =
459 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
460
461 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
462
463 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
464 &[
465 f32_ty_basic_md,
466 f32_ty_basic_md,
467 md_ty_basic_md,
468 md_ty_basic_md,
469 ],
470 false,
471 );
472 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
473 &[
474 f64_ty_basic_md,
475 f64_ty_basic_md,
476 md_ty_basic_md,
477 md_ty_basic_md,
478 ],
479 false,
480 );
481 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
482 &[
483 f32x4_ty_basic_md,
484 f32x4_ty_basic_md,
485 md_ty_basic_md,
486 md_ty_basic_md,
487 ],
488 false,
489 );
490 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
491 &[
492 f64x2_ty_basic_md,
493 f64x2_ty_basic_md,
494 md_ty_basic_md,
495 md_ty_basic_md,
496 ],
497 false,
498 );
499
500 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
501 &[
502 f32_ty_basic_md,
503 f32_ty_basic_md,
504 md_ty_basic_md,
505 md_ty_basic_md,
506 ],
507 false,
508 );
509 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
510 &[
511 f64_ty_basic_md,
512 f64_ty_basic_md,
513 md_ty_basic_md,
514 md_ty_basic_md,
515 ],
516 false,
517 );
518 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
519 &[
520 f32x4_ty_basic_md,
521 f32x4_ty_basic_md,
522 md_ty_basic_md,
523 md_ty_basic_md,
524 ],
525 false,
526 );
527 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
528 &[
529 f64x2_ty_basic_md,
530 f64x2_ty_basic_md,
531 md_ty_basic_md,
532 md_ty_basic_md,
533 ],
534 false,
535 );
536 let ret_i32x4_take_f32x4 = i32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
537 let ret_i32x4_take_f32x4_i32x4_i8 =
538 i32x4_ty.fn_type(&[f32x4_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
539 let ret_i32x4_take_f64x2 = i32x4_ty.fn_type(&[f64x2_ty_basic_md], false);
540 let ret_i32x4_take_f64x2_i32x4_i8 =
541 i32x4_ty.fn_type(&[f64x2_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
542
543 let add_function_with_attrs =
544 |name: &str, ty: FunctionType<'ctx>, linkage: Option<Linkage>| -> FunctionValue<'ctx> {
545 let function = module.add_function(name, ty, linkage);
546 if is_riscv64 {
551 for (i, param) in ty.get_param_types().iter().enumerate() {
552 if param == &i32_ty_basic_md {
553 function.add_attribute(
554 AttributeLoc::Param(i as u32),
555 context.create_enum_attribute(
556 Attribute::get_named_enum_kind_id("signext"),
557 0,
558 ),
559 );
560 function.add_attribute(
561 AttributeLoc::Param(i as u32),
562 context.create_enum_attribute(
563 Attribute::get_named_enum_kind_id("noundef"),
564 0,
565 ),
566 );
567 }
568 }
569 }
570 function
571 };
572
573 let intrinsics = Self {
574 ctlz_i32: add_function_with_attrs("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
575 ctlz_i64: add_function_with_attrs("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
576
577 cttz_i32: add_function_with_attrs("llvm.cttz.i32", ret_i32_take_i32_i1, None),
578 cttz_i64: add_function_with_attrs("llvm.cttz.i64", ret_i64_take_i64_i1, None),
579
580 ctpop_i32: add_function_with_attrs("llvm.ctpop.i32", ret_i32_take_i32, None),
581 ctpop_i64: add_function_with_attrs("llvm.ctpop.i64", ret_i64_take_i64, None),
582 ctpop_i8x16: add_function_with_attrs("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
583
584 fp_rounding_md: context.metadata_string("round.tonearest").into(),
585 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
586
587 fp_ogt_md: context.metadata_string("ogt").into(),
588 fp_olt_md: context.metadata_string("olt").into(),
589 fp_uno_md: context.metadata_string("uno").into(),
590
591 sqrt_f32: add_function_with_attrs("llvm.sqrt.f32", ret_f32_take_f32, None),
592 sqrt_f64: add_function_with_attrs("llvm.sqrt.f64", ret_f64_take_f64, None),
593 sqrt_f32x4: add_function_with_attrs("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
594 sqrt_f64x2: add_function_with_attrs("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
595
596 ceil_f32: add_function_with_attrs("llvm.ceil.f32", ret_f32_take_f32, None),
597 ceil_f64: add_function_with_attrs("llvm.ceil.f64", ret_f64_take_f64, None),
598 ceil_f32x4: add_function_with_attrs("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
599 ceil_f64x2: add_function_with_attrs("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
600
601 floor_f32: add_function_with_attrs("llvm.floor.f32", ret_f32_take_f32, None),
602 floor_f64: add_function_with_attrs("llvm.floor.f64", ret_f64_take_f64, None),
603 floor_f32x4: add_function_with_attrs("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
604 floor_f64x2: add_function_with_attrs("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
605
606 trunc_f32: add_function_with_attrs("llvm.trunc.f32", ret_f32_take_f32, None),
607 trunc_f64: add_function_with_attrs("llvm.trunc.f64", ret_f64_take_f64, None),
608 trunc_f32x4: add_function_with_attrs("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
609 trunc_f64x2: add_function_with_attrs("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
610
611 nearbyint_f32: add_function_with_attrs("llvm.nearbyint.f32", ret_f32_take_f32, None),
612 nearbyint_f64: add_function_with_attrs("llvm.nearbyint.f64", ret_f64_take_f64, None),
613 nearbyint_f32x4: add_function_with_attrs(
614 "llvm.nearbyint.v4f32",
615 ret_f32x4_take_f32x4,
616 None,
617 ),
618 nearbyint_f64x2: add_function_with_attrs(
619 "llvm.nearbyint.v2f64",
620 ret_f64x2_take_f64x2,
621 None,
622 ),
623
624 add_f32: add_function_with_attrs(
625 "llvm.experimental.constrained.fadd.f32",
626 ret_f32_take_f32_f32_md_md,
627 None,
628 ),
629 add_f64: add_function_with_attrs(
630 "llvm.experimental.constrained.fadd.f64",
631 ret_f64_take_f64_f64_md_md,
632 None,
633 ),
634 add_f32x4: add_function_with_attrs(
635 "llvm.experimental.constrained.fadd.v4f32",
636 ret_f32x4_take_f32x4_f32x4_md_md,
637 None,
638 ),
639 add_f64x2: add_function_with_attrs(
640 "llvm.experimental.constrained.fadd.v2f64",
641 ret_f64x2_take_f64x2_f64x2_md_md,
642 None,
643 ),
644
645 sub_f32: add_function_with_attrs(
646 "llvm.experimental.constrained.fsub.f32",
647 ret_f32_take_f32_f32_md_md,
648 None,
649 ),
650 sub_f64: add_function_with_attrs(
651 "llvm.experimental.constrained.fsub.f64",
652 ret_f64_take_f64_f64_md_md,
653 None,
654 ),
655 sub_f32x4: add_function_with_attrs(
656 "llvm.experimental.constrained.fsub.v4f32",
657 ret_f32x4_take_f32x4_f32x4_md_md,
658 None,
659 ),
660 sub_f64x2: add_function_with_attrs(
661 "llvm.experimental.constrained.fsub.v2f64",
662 ret_f64x2_take_f64x2_f64x2_md_md,
663 None,
664 ),
665
666 mul_f32: add_function_with_attrs(
667 "llvm.experimental.constrained.fmul.f32",
668 ret_f32_take_f32_f32_md_md,
669 None,
670 ),
671 mul_f64: add_function_with_attrs(
672 "llvm.experimental.constrained.fmul.f64",
673 ret_f64_take_f64_f64_md_md,
674 None,
675 ),
676 mul_f32x4: add_function_with_attrs(
677 "llvm.experimental.constrained.fmul.v4f32",
678 ret_f32x4_take_f32x4_f32x4_md_md,
679 None,
680 ),
681 mul_f64x2: add_function_with_attrs(
682 "llvm.experimental.constrained.fmul.v2f64",
683 ret_f64x2_take_f64x2_f64x2_md_md,
684 None,
685 ),
686 muladd_f32x4: add_function_with_attrs(
687 "llvm.experimental.constrained.fmuladd.v4f32",
688 ret_f32x4_take_f32x4_f32x4_f32x4_md_md,
689 None,
690 ),
691 muladd_f64x2: add_function_with_attrs(
692 "llvm.experimental.constrained.fmuladd.v2f64",
693 ret_f64x2_take_f64x2_f64x2_f64x2_md_md,
694 None,
695 ),
696
697 div_f32: add_function_with_attrs(
698 "llvm.experimental.constrained.fdiv.f32",
699 ret_f32_take_f32_f32_md_md,
700 None,
701 ),
702 div_f64: add_function_with_attrs(
703 "llvm.experimental.constrained.fdiv.f64",
704 ret_f64_take_f64_f64_md_md,
705 None,
706 ),
707 div_f32x4: add_function_with_attrs(
708 "llvm.experimental.constrained.fdiv.v4f32",
709 ret_f32x4_take_f32x4_f32x4_md_md,
710 None,
711 ),
712 div_f64x2: add_function_with_attrs(
713 "llvm.experimental.constrained.fdiv.v2f64",
714 ret_f64x2_take_f64x2_f64x2_md_md,
715 None,
716 ),
717
718 cmp_f32: add_function_with_attrs(
719 "llvm.experimental.constrained.fcmp.f32",
720 ret_i1_take_f32_f32_md_md,
721 None,
722 ),
723 cmp_f64: add_function_with_attrs(
724 "llvm.experimental.constrained.fcmp.f64",
725 ret_i1_take_f64_f64_md_md,
726 None,
727 ),
728 cmp_f32x4: add_function_with_attrs(
729 "llvm.experimental.constrained.fcmp.v4f32",
730 ret_i1x4_take_f32x4_f32x4_md_md,
731 None,
732 ),
733 cmp_f64x2: add_function_with_attrs(
734 "llvm.experimental.constrained.fcmp.v2f64",
735 ret_i1x2_take_f64x2_f64x2_md_md,
736 None,
737 ),
738
739 minimum_f32: add_function_with_attrs("llvm.minimum.f32", ret_f32_take_f32_f32, None),
740 minimum_f64: add_function_with_attrs("llvm.minimum.f64", ret_f64_take_f64_f64, None),
741 minimum_f32x4: add_function_with_attrs(
742 "llvm.minimum.v4f32",
743 ret_f32x4_take_f32x4_f32x4,
744 None,
745 ),
746 minimum_f64x2: add_function_with_attrs(
747 "llvm.minimum.v2f64",
748 ret_f64x2_take_f64x2_f64x2,
749 None,
750 ),
751
752 maximum_f32: add_function_with_attrs("llvm.maximum.f32", ret_f32_take_f32_f32, None),
753 maximum_f64: add_function_with_attrs("llvm.maximum.f64", ret_f64_take_f64_f64, None),
754 maximum_f32x4: add_function_with_attrs(
755 "llvm.maximum.v4f32",
756 ret_f32x4_take_f32x4_f32x4,
757 None,
758 ),
759 maximum_f64x2: add_function_with_attrs(
760 "llvm.maximum.v2f64",
761 ret_f64x2_take_f64x2_f64x2,
762 None,
763 ),
764
765 fpext_f32: add_function_with_attrs(
766 "llvm.experimental.constrained.fpext.f64.f32",
767 ret_f64_take_f32_md,
768 None,
769 ),
770 fptrunc_f64: add_function_with_attrs(
771 "llvm.experimental.constrained.fptrunc.f32.f64",
772 ret_f32_take_f64_md_md,
773 None,
774 ),
775
776 fabs_f32: add_function_with_attrs("llvm.fabs.f32", ret_f32_take_f32, None),
777 fabs_f64: add_function_with_attrs("llvm.fabs.f64", ret_f64_take_f64, None),
778 fabs_f32x4: add_function_with_attrs("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
779 fabs_f64x2: add_function_with_attrs("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
780
781 copysign_f32: add_function_with_attrs("llvm.copysign.f32", ret_f32_take_f32_f32, None),
782 copysign_f64: add_function_with_attrs("llvm.copysign.f64", ret_f64_take_f64_f64, None),
783 copysign_f32x4: add_function_with_attrs(
784 "llvm.copysign.v4f32",
785 ret_f32x4_take_f32x4_f32x4,
786 None,
787 ),
788 copysign_f64x2: add_function_with_attrs(
789 "llvm.copysign.v2f64",
790 ret_f64x2_take_f64x2_f64x2,
791 None,
792 ),
793
794 sadd_sat_i8x16: add_function_with_attrs(
795 "llvm.sadd.sat.v16i8",
796 ret_i8x16_take_i8x16_i8x16,
797 None,
798 ),
799 sadd_sat_i16x8: add_function_with_attrs(
800 "llvm.sadd.sat.v8i16",
801 ret_i16x8_take_i16x8_i16x8,
802 None,
803 ),
804 uadd_sat_i8x16: add_function_with_attrs(
805 "llvm.uadd.sat.v16i8",
806 ret_i8x16_take_i8x16_i8x16,
807 None,
808 ),
809 uadd_sat_i16x8: add_function_with_attrs(
810 "llvm.uadd.sat.v8i16",
811 ret_i16x8_take_i16x8_i16x8,
812 None,
813 ),
814
815 ssub_sat_i8x16: add_function_with_attrs(
816 "llvm.ssub.sat.v16i8",
817 ret_i8x16_take_i8x16_i8x16,
818 None,
819 ),
820 ssub_sat_i16x8: add_function_with_attrs(
821 "llvm.ssub.sat.v8i16",
822 ret_i16x8_take_i16x8_i16x8,
823 None,
824 ),
825 usub_sat_i8x16: add_function_with_attrs(
826 "llvm.usub.sat.v16i8",
827 ret_i8x16_take_i8x16_i8x16,
828 None,
829 ),
830 usub_sat_i16x8: add_function_with_attrs(
831 "llvm.usub.sat.v8i16",
832 ret_i16x8_take_i16x8_i16x8,
833 None,
834 ),
835
836 expect_i1: add_function_with_attrs("llvm.expect.i1", ret_i1_take_i1_i1, None),
837 trap: add_function_with_attrs("llvm.trap", void_ty.fn_type(&[], false), None),
838 debug_trap: add_function_with_attrs(
839 "llvm.debugtrap",
840 void_ty.fn_type(&[], false),
841 None,
842 ),
843 personality: add_function_with_attrs(
844 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
845 "__gxx_personality_v0"
848 } else {
849 "wasmer_eh_personality"
850 },
851 i32_ty.fn_type(
852 &[
853 i32_ty.into(),
854 i32_ty.into(),
855 i64_ty.into(),
856 ptr_ty.into(),
857 ptr_ty.into(),
858 ],
859 false,
860 ),
861 None,
862 ),
863 personality2: add_function_with_attrs(
864 "wasmer_eh_personality2",
865 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
866 None,
867 ),
868 readonly: context
869 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
870 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
871 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
872 nounwind: context
873 .create_enum_attribute(Attribute::get_named_enum_kind_id("nounwind"), 1),
874 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
875 chkstk: add_function_with_attrs("__chkstk", void_ty.fn_type(&[], false), None),
876 void_ty,
877 i1_ty,
878 i2_ty,
879 i4_ty,
880 i8_ty,
881 i16_ty,
882 i32_ty,
883 i64_ty,
884 i128_ty,
885 isize_ty,
886 f32_ty,
887 f64_ty,
888
889 i1x128_ty,
890 i8x16_ty,
891 i16x8_ty,
892 i32x4_ty,
893 i64x2_ty,
894 f32x4_ty,
895 f64x2_ty,
896 i32x8_ty,
897
898 anyfunc_ty,
899 i1_zero,
900 i8_zero,
901 i32_zero,
902 i64_zero,
903 i128_zero,
904 isize_zero,
905 f32_zero,
906 f64_zero,
907 f32x4_zero,
908 f64x2_zero,
909 i32_consts,
910
911 trap_unreachable: i32_ty
912 .const_int(TrapCode::UnreachableCodeReached as _, false)
913 .as_basic_value_enum(),
914 trap_call_indirect_null: i32_ty
915 .const_int(TrapCode::IndirectCallToNull as _, false)
916 .as_basic_value_enum(),
917 trap_call_indirect_sig: i32_ty
918 .const_int(TrapCode::BadSignature as _, false)
919 .as_basic_value_enum(),
920 trap_memory_oob: i32_ty
921 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
922 .as_basic_value_enum(),
923 trap_illegal_arithmetic: i32_ty
924 .const_int(TrapCode::IntegerOverflow as _, false)
925 .as_basic_value_enum(),
926 trap_integer_division_by_zero: i32_ty
927 .const_int(TrapCode::IntegerDivisionByZero as _, false)
928 .as_basic_value_enum(),
929 trap_bad_conversion_to_integer: i32_ty
930 .const_int(TrapCode::BadConversionToInteger as _, false)
931 .as_basic_value_enum(),
932 trap_unaligned_atomic: i32_ty
933 .const_int(TrapCode::UnalignedAtomic as _, false)
934 .as_basic_value_enum(),
935 trap_table_access_oob: i32_ty
936 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
937 .as_basic_value_enum(),
938
939 experimental_stackmap: module.add_function(
943 "llvm.experimental.stackmap",
944 void_ty.fn_type(
945 &[
946 i64_ty_basic_md, i32_ty_basic_md, ],
949 true,
950 ),
951 None,
952 ),
953
954 table_copy: add_function_with_attrs(
956 "wasmer_vm_table_copy",
957 void_ty.fn_type(
958 &[
959 ctx_ptr_ty_basic_md,
960 i32_ty_basic_md,
961 i32_ty_basic_md,
962 i32_ty_basic_md,
963 i32_ty_basic_md,
964 i32_ty_basic_md,
965 ],
966 false,
967 ),
968 None,
969 ),
970 table_init: add_function_with_attrs(
971 "wasmer_vm_table_init",
972 void_ty.fn_type(
973 &[
974 ctx_ptr_ty_basic_md,
975 i32_ty_basic_md,
976 i32_ty_basic_md,
977 i32_ty_basic_md,
978 i32_ty_basic_md,
979 i32_ty_basic_md,
980 ],
981 false,
982 ),
983 None,
984 ),
985 table_fill: add_function_with_attrs(
986 "wasmer_vm_table_fill",
987 void_ty.fn_type(
988 &[
989 ctx_ptr_ty_basic_md,
990 i32_ty_basic_md,
991 i32_ty_basic_md,
992 anyref_ty_basic_md,
993 i32_ty_basic_md,
994 ],
995 false,
996 ),
997 None,
998 ),
999 table_size: add_function_with_attrs(
1000 "wasmer_vm_table_size",
1001 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1002 None,
1003 ),
1004 imported_table_size: add_function_with_attrs(
1005 "wasmer_vm_imported_table_size",
1006 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1007 None,
1008 ),
1009 table_get: add_function_with_attrs(
1010 "wasmer_vm_table_get",
1011 anyref_ty.fn_type(
1012 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1013 false,
1014 ),
1015 None,
1016 ),
1017 imported_table_get: add_function_with_attrs(
1018 "wasmer_vm_imported_table_get",
1019 anyref_ty.fn_type(
1020 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1021 false,
1022 ),
1023 None,
1024 ),
1025 table_set: add_function_with_attrs(
1026 "wasmer_vm_table_set",
1027 void_ty.fn_type(
1028 &[
1029 ctx_ptr_ty_basic_md,
1030 i32_ty_basic_md,
1031 i32_ty_basic_md,
1032 anyref_ty_basic_md,
1033 ],
1034 false,
1035 ),
1036 None,
1037 ),
1038 imported_table_set: add_function_with_attrs(
1039 "wasmer_vm_imported_table_set",
1040 void_ty.fn_type(
1041 &[
1042 ctx_ptr_ty_basic_md,
1043 i32_ty_basic_md,
1044 i32_ty_basic_md,
1045 anyref_ty_basic_md,
1046 ],
1047 false,
1048 ),
1049 None,
1050 ),
1051 table_grow: add_function_with_attrs(
1052 "wasmer_vm_table_grow",
1053 i32_ty.fn_type(
1054 &[
1055 ctx_ptr_ty_basic_md,
1056 anyref_ty_basic_md,
1057 i32_ty_basic_md,
1058 i32_ty_basic_md,
1059 ],
1060 false,
1061 ),
1062 None,
1063 ),
1064 imported_table_grow: add_function_with_attrs(
1065 "wasmer_vm_imported_table_grow",
1066 i32_ty.fn_type(
1067 &[
1068 ctx_ptr_ty_basic_md,
1069 anyref_ty_basic_md,
1070 i32_ty_basic_md,
1071 i32_ty_basic_md,
1072 ],
1073 false,
1074 ),
1075 None,
1076 ),
1077 memory_init: add_function_with_attrs(
1078 "wasmer_vm_memory32_init",
1079 void_ty.fn_type(
1080 &[
1081 ctx_ptr_ty_basic_md,
1082 i32_ty_basic_md,
1083 i32_ty_basic_md,
1084 i32_ty_basic_md,
1085 i32_ty_basic_md,
1086 i32_ty_basic_md,
1087 ],
1088 false,
1089 ),
1090 None,
1091 ),
1092 memory_copy: add_function_with_attrs(
1093 "wasmer_vm_memory32_copy",
1094 void_ty.fn_type(
1095 &[
1096 ctx_ptr_ty_basic_md,
1097 i32_ty_basic_md,
1098 i32_ty_basic_md,
1099 i32_ty_basic_md,
1100 i32_ty_basic_md,
1101 i32_ty_basic_md,
1102 ],
1103 false,
1104 ),
1105 None,
1106 ),
1107 memory_fill: add_function_with_attrs(
1108 "wasmer_vm_memory32_fill",
1109 void_ty.fn_type(
1110 &[
1111 ctx_ptr_ty_basic_md,
1112 i32_ty_basic_md,
1113 i32_ty_basic_md,
1114 i32_ty_basic_md,
1115 i32_ty_basic_md,
1116 ],
1117 false,
1118 ),
1119 None,
1120 ),
1121 imported_memory_fill: add_function_with_attrs(
1122 "wasmer_vm_imported_memory32_fill",
1123 void_ty.fn_type(
1124 &[
1125 ctx_ptr_ty_basic_md,
1126 i32_ty_basic_md,
1127 i32_ty_basic_md,
1128 i32_ty_basic_md,
1129 i32_ty_basic_md,
1130 ],
1131 false,
1132 ),
1133 None,
1134 ),
1135 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1136 memory_grow_ty: i32_ty.fn_type(
1137 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1138 false,
1139 ),
1140 data_drop: add_function_with_attrs(
1141 "wasmer_vm_data_drop",
1142 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1143 None,
1144 ),
1145 func_ref: add_function_with_attrs(
1146 "wasmer_vm_func_ref",
1147 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1148 None,
1149 ),
1150 elem_drop: add_function_with_attrs(
1151 "wasmer_vm_elem_drop",
1152 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1153 None,
1154 ),
1155 throw_trap: add_function_with_attrs(
1156 "wasmer_vm_raise_trap",
1157 void_ty.fn_type(&[i32_ty_basic_md], false),
1158 None,
1159 ),
1160
1161 throw: add_function_with_attrs(
1162 "wasmer_vm_throw",
1163 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1164 None,
1165 ),
1166 alloc_exception: add_function_with_attrs(
1167 "wasmer_vm_alloc_exception",
1168 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1169 None,
1170 ),
1171 read_exnref: add_function_with_attrs(
1172 "wasmer_vm_read_exnref",
1173 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1174 None,
1175 ),
1176 exception_into_exnref: add_function_with_attrs(
1177 "wasmer_vm_exception_into_exnref",
1178 i32_ty.fn_type(&[ptr_ty.into()], false),
1179 None,
1180 ),
1181 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1182
1183 debug_ptr: add_function_with_attrs(
1184 "wasmer_vm_dbg_usize",
1185 void_ty.fn_type(&[ptr_ty.into()], false),
1186 None,
1187 ),
1188 debug_str: add_function_with_attrs(
1189 "wasmer_vm_dbg_str",
1190 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1191 None,
1192 ),
1193 memory_wait32: add_function_with_attrs(
1194 "wasmer_vm_memory32_atomic_wait32",
1195 i32_ty.fn_type(
1196 &[
1197 ctx_ptr_ty_basic_md,
1198 i32_ty_basic_md,
1199 i32_ty_basic_md,
1200 i32_ty_basic_md,
1201 i64_ty_basic_md,
1202 ],
1203 false,
1204 ),
1205 None,
1206 ),
1207 memory_wait32_ty: i32_ty.fn_type(
1208 &[
1209 ctx_ptr_ty_basic_md,
1210 i32_ty_basic_md,
1211 i32_ty_basic_md,
1212 i32_ty_basic_md,
1213 i64_ty_basic_md,
1214 ],
1215 false,
1216 ),
1217 imported_memory_wait32: add_function_with_attrs(
1218 "wasmer_vm_imported_memory32_atomic_wait32",
1219 i32_ty.fn_type(
1220 &[
1221 ctx_ptr_ty_basic_md,
1222 i32_ty_basic_md,
1223 i32_ty_basic_md,
1224 i32_ty_basic_md,
1225 i64_ty_basic_md,
1226 ],
1227 false,
1228 ),
1229 None,
1230 ),
1231 memory_wait64: add_function_with_attrs(
1232 "wasmer_vm_memory32_atomic_wait64",
1233 i32_ty.fn_type(
1234 &[
1235 ctx_ptr_ty_basic_md,
1236 i32_ty_basic_md,
1237 i32_ty_basic_md,
1238 i64_ty_basic_md,
1239 i64_ty_basic_md,
1240 ],
1241 false,
1242 ),
1243 None,
1244 ),
1245 memory_wait64_ty: i32_ty.fn_type(
1246 &[
1247 ctx_ptr_ty_basic_md,
1248 i32_ty_basic_md,
1249 i32_ty_basic_md,
1250 i64_ty_basic_md,
1251 i64_ty_basic_md,
1252 ],
1253 false,
1254 ),
1255 imported_memory_wait64: add_function_with_attrs(
1256 "wasmer_vm_imported_memory32_atomic_wait64",
1257 i32_ty.fn_type(
1258 &[
1259 ctx_ptr_ty_basic_md,
1260 i32_ty_basic_md,
1261 i32_ty_basic_md,
1262 i64_ty_basic_md,
1263 i64_ty_basic_md,
1264 ],
1265 false,
1266 ),
1267 None,
1268 ),
1269 memory_notify: add_function_with_attrs(
1270 "wasmer_vm_memory32_atomic_notify",
1271 i32_ty.fn_type(
1272 &[
1273 ctx_ptr_ty_basic_md,
1274 i32_ty_basic_md,
1275 i32_ty_basic_md,
1276 i32_ty_basic_md,
1277 ],
1278 false,
1279 ),
1280 None,
1281 ),
1282 memory_notify_ty: i32_ty.fn_type(
1283 &[
1284 ctx_ptr_ty_basic_md,
1285 i32_ty_basic_md,
1286 i32_ty_basic_md,
1287 i32_ty_basic_md,
1288 ],
1289 false,
1290 ),
1291 imported_memory_notify: add_function_with_attrs(
1292 "wasmer_vm_imported_memory32_atomic_notify",
1293 i32_ty.fn_type(
1294 &[
1295 ctx_ptr_ty_basic_md,
1296 i32_ty_basic_md,
1297 i32_ty_basic_md,
1298 i32_ty_basic_md,
1299 ],
1300 false,
1301 ),
1302 None,
1303 ),
1304
1305 vmfunction_import_ty: context.struct_type(
1306 &[
1307 i8_ptr_ty_basic,
1308 i8_ptr_ty_basic,
1309 i8_ptr_ty_basic,
1310 i1_ty_basic,
1311 ],
1312 false,
1313 ),
1314 vmfunction_import_body_element: 0,
1315 vmfunction_import_vmctx_element: 1,
1316 vmfunction_import_include_m0_param_element: 3,
1317 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1318 vmmemory_definition_base_element: 0,
1319 vmmemory_definition_current_length_element: 1,
1320
1321 ptr_ty,
1322
1323 x86_64: X86_64Intrinsics {
1324 pshufb128: add_function_with_attrs(
1325 "llvm.x86.ssse3.pshuf.b.128",
1326 ret_i8x16_take_i8x16_i8x16,
1327 None,
1328 ),
1329 pmaddubsw128: add_function_with_attrs(
1330 "llvm.x86.ssse3.pmadd.ub.sw.128",
1331 i16x8_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false),
1332 None,
1333 ),
1334 pmaddwd128: add_function_with_attrs(
1335 "llvm.x86.sse2.pmadd.wd",
1336 i32x4_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false),
1337 None,
1338 ),
1339 pmulhrsw128: add_function_with_attrs(
1340 "llvm.x86.ssse3.pmul.hr.sw.128",
1341 ret_i16x8_take_i16x8_i16x8,
1342 None,
1343 ),
1344 pblendvb: add_function_with_attrs(
1345 "llvm.x86.sse41.pblendvb",
1346 ret_i8x16_take_i8x16_i8x16_i8x16,
1347 None,
1348 ),
1349 min_ps: add_function_with_attrs(
1350 "llvm.x86.sse.min.ps",
1351 ret_f32x4_take_f32x4_f32x4,
1352 None,
1353 ),
1354 min_pd: add_function_with_attrs(
1355 "llvm.x86.sse2.min.pd",
1356 ret_f64x2_take_f64x2_f64x2,
1357 None,
1358 ),
1359 max_ps: add_function_with_attrs(
1360 "llvm.x86.sse.max.ps",
1361 ret_f32x4_take_f32x4_f32x4,
1362 None,
1363 ),
1364 max_pd: add_function_with_attrs(
1365 "llvm.x86.sse2.max.pd",
1366 ret_f64x2_take_f64x2_f64x2,
1367 None,
1368 ),
1369 cvttps2dq: add_function_with_attrs(
1370 "llvm.x86.sse2.cvttps2dq",
1371 ret_i32x4_take_f32x4,
1372 None,
1373 ),
1374 cvtps2udq128: add_function_with_attrs(
1375 "llvm.x86.avx512.mask.cvtps2udq.128",
1376 ret_i32x4_take_f32x4_i32x4_i8,
1377 None,
1378 ),
1379 cvtpd2dq: add_function_with_attrs(
1380 "llvm.x86.sse2.cvtpd2dq",
1381 ret_i32x4_take_f64x2,
1382 None,
1383 ),
1384 cvtpd2udq128: add_function_with_attrs(
1385 "llvm.x86.avx512.mask.cvtpd2udq.128",
1386 ret_i32x4_take_f64x2_i32x4_i8,
1387 None,
1388 ),
1389 },
1390 };
1391
1392 let noreturn =
1393 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1394 intrinsics
1395 .throw_trap
1396 .add_attribute(AttributeLoc::Function, noreturn);
1397 intrinsics
1398 }
1399}
1400
1401#[derive(Clone, Copy)]
1402pub enum MemoryCache<'ctx> {
1403 Dynamic {
1405 ptr_to_base_ptr: PointerValue<'ctx>,
1406 ptr_to_current_length: PointerValue<'ctx>,
1407 },
1408 Static { base_ptr: PointerValue<'ctx> },
1410}
1411
1412#[derive(Clone)]
1413struct TableCache<'ctx> {
1414 ptr_to_base_ptr: PointerValue<'ctx>,
1415 ptr_to_bounds: PointerValue<'ctx>,
1416}
1417
1418#[derive(Clone, Copy)]
1419pub enum GlobalCache<'ctx> {
1420 Mut {
1421 ptr_to_value: PointerValue<'ctx>,
1422 value_type: BasicTypeEnum<'ctx>,
1423 },
1424 Const {
1425 value: BasicValueEnum<'ctx>,
1426 },
1427}
1428
1429#[derive(Clone)]
1430pub struct FunctionCache<'ctx> {
1431 pub func: PointerValue<'ctx>,
1432 pub llvm_func_type: FunctionType<'ctx>,
1433 pub vmctx: BasicValueEnum<'ctx>,
1434 pub imported_include_m0_param: Option<IntValue<'ctx>>,
1435 pub attrs: Vec<(Attribute, AttributeLoc)>,
1436}
1437
1438pub struct CtxType<'ctx, 'a> {
1439 ctx_ptr_value: PointerValue<'ctx>,
1440 m0: Option<PointerValue<'ctx>>,
1441
1442 wasm_module: &'a WasmerCompilerModule,
1443 cache_builder: &'a Builder<'ctx>,
1444 abi: &'a LLVMAbi,
1445
1446 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1447 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1448 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1449 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1450 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1451
1452 offsets: VMOffsets,
1453}
1454
1455impl<'ctx, 'a> CtxType<'ctx, 'a> {
1456 pub fn new(
1457 wasm_module: &'a WasmerCompilerModule,
1458 func_value: &FunctionValue<'ctx>,
1459 cache_builder: &'a Builder<'ctx>,
1460 abi: &'a LLVMAbi,
1461 pointer_width: u8,
1462 m0: Option<PointerValue<'ctx>>,
1463 ) -> Result<CtxType<'ctx, 'a>, CompileError> {
1464 Ok(CtxType {
1465 m0,
1466 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1467
1468 wasm_module,
1469 cache_builder,
1470 abi,
1471
1472 cached_memories: HashMap::new(),
1473 cached_tables: HashMap::new(),
1474 cached_globals: HashMap::new(),
1475 cached_functions: HashMap::new(),
1476 cached_memory_op: HashMap::new(),
1477
1478 offsets: VMOffsets::try_new(pointer_width, wasm_module)
1479 .map_err(CompileError::Resource)?,
1480 })
1481 }
1482
1483 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1484 self.ctx_ptr_value.as_basic_value_enum()
1485 }
1486
1487 pub fn memory(
1488 &mut self,
1489 index: MemoryIndex,
1490 intrinsics: &Intrinsics<'ctx>,
1491 module: &Module<'ctx>,
1492 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1493 ) -> Result<MemoryCache<'ctx>, CompileError> {
1494 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1495 &mut self.cached_memories,
1496 self.wasm_module,
1497 self.ctx_ptr_value,
1498 &self.cache_builder,
1499 &self.offsets,
1500 );
1501 let memory_style = &memory_styles[index];
1502 match cached_memories.get(&index) {
1503 Some(r) => Ok(*r),
1504 None => {
1505 let memory_definition_ptr =
1506 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1507 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1508 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1509 unsafe {
1510 err!(cache_builder.build_gep(
1511 intrinsics.i8_ty,
1512 ctx_ptr_value,
1513 &[offset],
1514 ""
1515 ))
1516 }
1517 } else {
1518 let offset = offsets.vmctx_vmmemory_import(index);
1519 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1520 let memory_definition_ptr_ptr = unsafe {
1521 err!(cache_builder.build_gep(
1522 intrinsics.i8_ty,
1523 ctx_ptr_value,
1524 &[offset],
1525 ""
1526 ))
1527 };
1528 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1529 memory_definition_ptr_ptr,
1530 intrinsics.ptr_ty,
1531 "",
1532 ))
1533 .into_pointer_value();
1534 let memory_definition_ptr = err!(cache_builder.build_load(
1535 intrinsics.ptr_ty,
1536 memory_definition_ptr_ptr,
1537 ""
1538 ))
1539 .into_pointer_value();
1540 tbaa_label(
1541 module,
1542 intrinsics,
1543 format!("memory {} definition", index.as_u32()),
1544 memory_definition_ptr.as_instruction_value().unwrap(),
1545 );
1546 memory_definition_ptr
1547 };
1548 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1549 memory_definition_ptr,
1550 intrinsics.ptr_ty,
1551 "",
1552 ))
1553 .into_pointer_value();
1554 let base_ptr = err!(cache_builder.build_struct_gep(
1555 intrinsics.vmmemory_definition_ty,
1556 memory_definition_ptr,
1557 intrinsics.vmmemory_definition_base_element,
1558 "",
1559 ));
1560 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1561 let current_length_ptr = err!(cache_builder.build_struct_gep(
1562 intrinsics.vmmemory_definition_ty,
1563 memory_definition_ptr,
1564 intrinsics.vmmemory_definition_current_length_element,
1565 "",
1566 ));
1567 MemoryCache::Dynamic {
1568 ptr_to_base_ptr: base_ptr,
1569 ptr_to_current_length: current_length_ptr,
1570 }
1571 } else {
1572 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1573 .into_pointer_value();
1574 tbaa_label(
1575 module,
1576 intrinsics,
1577 format!("memory base_ptr {}", index.as_u32()),
1578 base_ptr.as_instruction_value().unwrap(),
1579 );
1580 MemoryCache::Static { base_ptr }
1581 };
1582
1583 self.cached_memories.insert(index, value);
1584 Ok(*self.cached_memories.get(&index).unwrap())
1585 }
1586 }
1587 }
1588
1589 fn build_table_prepare(
1590 table_index: TableIndex,
1591 intrinsics: &Intrinsics<'ctx>,
1592 module: &Module<'ctx>,
1593 wasm_module: &WasmerCompilerModule,
1594 ctx_ptr_value: PointerValue<'ctx>,
1595 offsets: &VMOffsets,
1596 builder: &Builder<'ctx>,
1597 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1598 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1599 let offset = intrinsics.i64_ty.const_int(
1600 offsets
1601 .vmctx_vmtable_definition_base(local_table_index)
1602 .into(),
1603 false,
1604 );
1605 let ptr_to_base_ptr =
1606 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1607 let ptr_to_base_ptr =
1608 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1609 .into_pointer_value();
1610 let offset = intrinsics.i64_ty.const_int(
1611 offsets
1612 .vmctx_vmtable_definition_current_elements(local_table_index)
1613 .into(),
1614 false,
1615 );
1616 let ptr_to_bounds =
1617 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1618 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1619 .into_pointer_value();
1620 Ok((ptr_to_base_ptr, ptr_to_bounds))
1621 } else {
1622 let offset = intrinsics.i64_ty.const_int(
1623 offsets.vmctx_vmtable_import_definition(table_index).into(),
1624 false,
1625 );
1626 let definition_ptr_ptr =
1627 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1628 let definition_ptr_ptr =
1629 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1630 .into_pointer_value();
1631 let definition_ptr =
1632 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1633 .into_pointer_value();
1634 tbaa_label(
1635 module,
1636 intrinsics,
1637 format!("table {} definition", table_index.as_u32()),
1638 definition_ptr.as_instruction_value().unwrap(),
1639 );
1640
1641 let offset = intrinsics
1642 .i64_ty
1643 .const_int(offsets.vmtable_definition_base().into(), false);
1644 let ptr_to_base_ptr =
1645 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1646 let ptr_to_base_ptr =
1647 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1648 .into_pointer_value();
1649 let offset = intrinsics
1650 .i64_ty
1651 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1652 let ptr_to_bounds =
1653 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1654 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1655 .into_pointer_value();
1656 Ok((ptr_to_base_ptr, ptr_to_bounds))
1657 }
1658 }
1659
1660 fn table_prepare(
1661 &mut self,
1662 table_index: TableIndex,
1663 intrinsics: &Intrinsics<'ctx>,
1664 module: &Module<'ctx>,
1665 body_builder: &Builder<'ctx>,
1666 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1667 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1668 &mut self.cached_tables,
1669 self.wasm_module,
1670 self.ctx_ptr_value,
1671 &self.cache_builder,
1672 &self.offsets,
1673 );
1674
1675 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1676 CompileError::Codegen(format!(
1677 "Table index out of bounds: {}",
1678 table_index.as_u32()
1679 ))
1680 })?;
1681
1682 if is_growable {
1685 Self::build_table_prepare(
1686 table_index,
1687 intrinsics,
1688 module,
1689 wasm_module,
1690 ctx_ptr_value,
1691 offsets,
1692 body_builder,
1693 )
1694 } else {
1695 let TableCache {
1696 ptr_to_base_ptr,
1697 ptr_to_bounds,
1698 } = match cached_tables.entry(table_index) {
1699 Entry::Occupied(entry) => entry.get().clone(),
1700 Entry::Vacant(entry) => {
1701 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1702 table_index,
1703 intrinsics,
1704 module,
1705 wasm_module,
1706 ctx_ptr_value,
1707 offsets,
1708 cache_builder,
1709 )?;
1710
1711 let v = TableCache {
1712 ptr_to_base_ptr,
1713 ptr_to_bounds,
1714 };
1715
1716 entry.insert(v.clone());
1717
1718 v
1719 }
1720 };
1721
1722 Ok((ptr_to_base_ptr, ptr_to_bounds))
1723 }
1724 }
1725
1726 pub fn table(
1727 &mut self,
1728 index: TableIndex,
1729 intrinsics: &Intrinsics<'ctx>,
1730 module: &Module<'ctx>,
1731 body_builder: &Builder<'ctx>,
1732 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1733 let (ptr_to_base_ptr, ptr_to_bounds) =
1734 self.table_prepare(index, intrinsics, module, body_builder)?;
1735
1736 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1738 &body_builder
1739 } else {
1740 &self.cache_builder
1741 };
1742
1743 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1744 .into_pointer_value();
1745 let bounds =
1746 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1747 tbaa_label(
1748 module,
1749 intrinsics,
1750 format!("table_base_ptr {}", index.index()),
1751 base_ptr.as_instruction_value().unwrap(),
1752 );
1753 tbaa_label(
1754 module,
1755 intrinsics,
1756 format!("table_bounds {}", index.index()),
1757 bounds.as_instruction_value().unwrap(),
1758 );
1759 Ok((base_ptr, bounds))
1760 }
1761
1762 pub fn fixed_funcref_table_anyfuncs(
1764 &self,
1765 index: LocalTableIndex,
1766 intrinsics: &Intrinsics<'ctx>,
1767 builder: &Builder<'ctx>,
1768 ) -> Result<PointerValue<'ctx>, CompileError> {
1769 let offset = intrinsics.i64_ty.const_int(
1770 self.offsets
1771 .vmctx_fixed_funcref_table_anyfuncs(index)
1772 .expect("fixed funcref table must have inline VMContext storage")
1773 .into(),
1774 false,
1775 );
1776 let ptr =
1777 unsafe { err!(builder.build_gep(intrinsics.i8_ty, self.ctx_ptr_value, &[offset], "")) };
1778 Ok(err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value())
1779 }
1780
1781 pub fn global(
1782 &mut self,
1783 index: GlobalIndex,
1784 intrinsics: &Intrinsics<'ctx>,
1785 module: &Module<'ctx>,
1786 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1787 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1788 &mut self.cached_globals,
1789 self.wasm_module,
1790 self.ctx_ptr_value,
1791 &self.cache_builder,
1792 &self.offsets,
1793 );
1794 match cached_globals.entry(index) {
1795 Entry::Occupied(entry) => Ok(entry.into_mut()),
1796 Entry::Vacant(entry) => {
1797 let global_type = wasm_module.globals[index];
1798 let global_value_type = global_type.ty;
1799
1800 let global_mutability = global_type.mutability;
1801 let global_ptr = if let Some(local_global_index) =
1802 wasm_module.local_global_index(index)
1803 {
1804 let offset = offsets.vmctx_vmglobal_definition(local_global_index);
1805 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1806 unsafe {
1807 err!(cache_builder.build_gep(
1808 intrinsics.i8_ty,
1809 ctx_ptr_value,
1810 &[offset],
1811 ""
1812 ))
1813 }
1814 } else {
1815 let offset = offsets.vmctx_vmglobal_import(index);
1816 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1817 let global_ptr_ptr = unsafe {
1818 err!(cache_builder.build_gep(
1819 intrinsics.i8_ty,
1820 ctx_ptr_value,
1821 &[offset],
1822 ""
1823 ))
1824 };
1825 let global_ptr_ptr =
1826 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1827 .into_pointer_value();
1828 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1829 .into_pointer_value()
1830 };
1831 let global_ptr =
1832 err!(cache_builder.build_bit_cast(global_ptr, intrinsics.ptr_ty, "",))
1833 .into_pointer_value();
1834
1835 let ret = entry.insert(match global_mutability {
1836 Mutability::Const => {
1837 let value = err!(cache_builder.build_load(
1838 type_to_llvm(intrinsics, global_value_type)?,
1839 global_ptr,
1840 "",
1841 ));
1842 tbaa_label(
1843 module,
1844 intrinsics,
1845 format!("global {}", index.as_u32()),
1846 value.as_instruction_value().unwrap(),
1847 );
1848 GlobalCache::Const { value }
1849 }
1850 Mutability::Var => GlobalCache::Mut {
1851 ptr_to_value: global_ptr,
1852 value_type: type_to_llvm(intrinsics, global_value_type)?,
1853 },
1854 });
1855
1856 Ok(ret)
1857 }
1858 }
1859 }
1860
1861 pub fn add_func(
1862 &mut self,
1863 function_index: FunctionIndex,
1864 func: PointerValue<'ctx>,
1865 llvm_func_type: FunctionType<'ctx>,
1866 vmctx: BasicValueEnum<'ctx>,
1867 attrs: &[(Attribute, AttributeLoc)],
1868 ) {
1869 match self.cached_functions.entry(function_index) {
1870 Entry::Occupied(_) => unreachable!("duplicate function"),
1871 Entry::Vacant(entry) => {
1872 entry.insert(FunctionCache {
1873 func,
1874 llvm_func_type,
1875 vmctx,
1876 imported_include_m0_param: None,
1877 attrs: attrs.to_vec(),
1878 });
1879 }
1880 }
1881 }
1882
1883 #[allow(clippy::too_many_arguments)]
1884 pub fn local_func(
1885 &mut self,
1886 _local_function_index: LocalFunctionIndex,
1887 function_index: FunctionIndex,
1888 intrinsics: &Intrinsics<'ctx>,
1889 module: &Module<'ctx>,
1890 context: &'ctx Context,
1891 func_type: &FuncType,
1892 function_name: &str,
1893 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1894 let (cached_functions, ctx_ptr_value, offsets) = (
1895 &mut self.cached_functions,
1896 &self.ctx_ptr_value,
1897 &self.offsets,
1898 );
1899 Ok(match cached_functions.entry(function_index) {
1900 Entry::Occupied(entry) => entry.into_mut(),
1901 Entry::Vacant(entry) => {
1902 debug_assert!(module.get_function(function_name).is_none());
1903 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1904 context,
1905 intrinsics,
1906 Some(offsets),
1907 func_type,
1908 self.m0.is_some(),
1909 )?;
1910 let func =
1911 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1912 for (attr, attr_loc) in &llvm_func_attrs {
1913 func.add_attribute(*attr_loc, *attr);
1914 }
1915 entry.insert(FunctionCache {
1916 func: func.as_global_value().as_pointer_value(),
1917 llvm_func_type,
1918 vmctx: ctx_ptr_value.as_basic_value_enum(),
1919 imported_include_m0_param: None,
1920 attrs: llvm_func_attrs,
1921 })
1922 }
1923 })
1924 }
1925
1926 pub fn imported_func(
1927 &mut self,
1928 function_index: FunctionIndex,
1929 intrinsics: &Intrinsics<'ctx>,
1930 context: &'ctx Context,
1931 func_type: &FuncType,
1932 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1933 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1934 &mut self.cached_functions,
1935 self.wasm_module,
1936 &self.ctx_ptr_value,
1937 &self.cache_builder,
1938 &self.offsets,
1939 );
1940 match cached_functions.entry(function_index) {
1941 Entry::Occupied(entry) => Ok(entry.into_mut()),
1942 Entry::Vacant(entry) => {
1943 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1944 context,
1945 intrinsics,
1946 Some(offsets),
1947 func_type,
1948 self.m0.is_some(),
1949 )?;
1950 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1951 let offset = offsets.vmctx_vmfunction_import(function_index);
1952 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1953 let vmfunction_import_ptr = unsafe {
1954 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1955 };
1956 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1957 vmfunction_import_ptr,
1958 intrinsics.ptr_ty,
1959 "",
1960 ))
1961 .into_pointer_value();
1962 vmfunction_import_ptr.set_name("vmfunction_import_ptr");
1963
1964 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1965 intrinsics.vmfunction_import_ty,
1966 vmfunction_import_ptr,
1967 intrinsics.vmfunction_import_body_element,
1968 "",
1969 ));
1970 body_ptr_ptr.set_name("body_ptr_ptr");
1971 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
1972 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
1973 .into_pointer_value();
1974 body_ptr.set_name("body_ptr");
1975 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
1976 intrinsics.vmfunction_import_ty,
1977 vmfunction_import_ptr,
1978 intrinsics.vmfunction_import_vmctx_element,
1979 "",
1980 ));
1981 vmctx_ptr_ptr.set_name("vmctx_ptr_ptr");
1982 let vmctx_ptr =
1983 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
1984 vmctx_ptr.set_name("vmctx_ptr");
1985 let include_m0_param_ptr = err!(cache_builder.build_struct_gep(
1986 intrinsics.vmfunction_import_ty,
1987 vmfunction_import_ptr,
1988 intrinsics.vmfunction_import_include_m0_param_element,
1989 "",
1990 ));
1991 include_m0_param_ptr.set_name("include_m0_param_ptr");
1992 let imported_include_m0_param =
1993 err!(cache_builder.build_load(intrinsics.i1_ty, include_m0_param_ptr, "",))
1994 .into_int_value();
1995 imported_include_m0_param.set_name("imported_include_m0_param");
1996
1997 Ok(entry.insert(FunctionCache {
1998 func: body_ptr,
1999 llvm_func_type,
2000 vmctx: vmctx_ptr,
2001 imported_include_m0_param: Some(imported_include_m0_param),
2002 attrs: llvm_func_attrs,
2003 }))
2004 }
2005 }
2006 }
2007
2008 pub fn memory_grow(
2009 &mut self,
2010 memory_index: MemoryIndex,
2011 intrinsics: &Intrinsics<'ctx>,
2012 ) -> Result<PointerValue<'ctx>, CompileError> {
2013 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2014 &mut self.cached_memory_op,
2015 &self.wasm_module,
2016 &self.offsets,
2017 &self.cache_builder,
2018 &self.ctx_ptr_value,
2019 );
2020 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
2021 Entry::Occupied(entry) => Ok(*entry.get()),
2022 Entry::Vacant(entry) => {
2023 let (grow_fn, grow_fn_ty) =
2024 if wasm_module.local_memory_index(memory_index).is_some() {
2025 (
2026 VMBuiltinFunctionIndex::get_memory32_grow_index(),
2027 intrinsics.ptr_ty,
2028 )
2029 } else {
2030 (
2031 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
2032 intrinsics.ptr_ty,
2033 )
2034 };
2035 let offset = offsets.vmctx_builtin_function(grow_fn);
2036 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2037 let grow_fn_ptr_ptr = unsafe {
2038 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2039 };
2040
2041 let grow_fn_ptr_ptr =
2042 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2043 .into_pointer_value();
2044 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
2045 .into_pointer_value();
2046
2047 entry.insert(val);
2048 Ok(val)
2049 }
2050 }
2051 }
2052
2053 pub fn memory_size(
2054 &mut self,
2055 memory_index: MemoryIndex,
2056 intrinsics: &Intrinsics<'ctx>,
2057 ) -> Result<PointerValue<'ctx>, CompileError> {
2058 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2059 &mut self.cached_memory_op,
2060 &self.wasm_module,
2061 &self.offsets,
2062 &self.cache_builder,
2063 &self.ctx_ptr_value,
2064 );
2065
2066 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
2067 Entry::Occupied(entry) => Ok(*entry.get()),
2068 Entry::Vacant(entry) => {
2069 let (size_fn, size_fn_ty) =
2070 if wasm_module.local_memory_index(memory_index).is_some() {
2071 (
2072 VMBuiltinFunctionIndex::get_memory32_size_index(),
2073 intrinsics.ptr_ty,
2074 )
2075 } else {
2076 (
2077 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
2078 intrinsics.ptr_ty,
2079 )
2080 };
2081 let offset = offsets.vmctx_builtin_function(size_fn);
2082 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2083 let size_fn_ptr_ptr = unsafe {
2084 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2085 };
2086
2087 let size_fn_ptr_ptr =
2088 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2089 .into_pointer_value();
2090
2091 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2092 .into_pointer_value();
2093 entry.insert(val);
2094 Ok(val)
2095 }
2096 }
2097 }
2098
2099 pub fn memory_wait32(
2100 &mut self,
2101 memory_index: MemoryIndex,
2102 intrinsics: &Intrinsics<'ctx>,
2103 ) -> Result<PointerValue<'ctx>, CompileError> {
2104 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2105 &mut self.cached_memory_op,
2106 &self.wasm_module,
2107 &self.offsets,
2108 &self.cache_builder,
2109 &self.ctx_ptr_value,
2110 );
2111 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
2112 Entry::Occupied(entry) => Ok(*entry.get()),
2113 Entry::Vacant(entry) => {
2114 let (size_fn, size_fn_ty) =
2115 if wasm_module.local_memory_index(memory_index).is_some() {
2116 (
2117 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
2118 intrinsics.ptr_ty,
2119 )
2120 } else {
2121 (
2122 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
2123 intrinsics.ptr_ty,
2124 )
2125 };
2126 let offset = offsets.vmctx_builtin_function(size_fn);
2127 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2128 let size_fn_ptr_ptr = unsafe {
2129 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2130 };
2131
2132 let size_fn_ptr_ptr =
2133 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2134 .into_pointer_value();
2135
2136 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2137 .into_pointer_value();
2138
2139 entry.insert(val);
2140 Ok(val)
2141 }
2142 }
2143 }
2144
2145 pub fn memory_wait64(
2146 &mut self,
2147 memory_index: MemoryIndex,
2148 intrinsics: &Intrinsics<'ctx>,
2149 ) -> Result<PointerValue<'ctx>, CompileError> {
2150 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2151 &mut self.cached_memory_op,
2152 &self.wasm_module,
2153 &self.offsets,
2154 &self.cache_builder,
2155 &self.ctx_ptr_value,
2156 );
2157
2158 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
2159 Entry::Occupied(entry) => Ok(*entry.get()),
2160 Entry::Vacant(entry) => {
2161 let (size_fn, size_fn_ty) =
2162 if wasm_module.local_memory_index(memory_index).is_some() {
2163 (
2164 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
2165 intrinsics.ptr_ty,
2166 )
2167 } else {
2168 (
2169 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
2170 intrinsics.ptr_ty,
2171 )
2172 };
2173 let offset = offsets.vmctx_builtin_function(size_fn);
2174 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2175 let size_fn_ptr_ptr = unsafe {
2176 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2177 };
2178
2179 let size_fn_ptr_ptr =
2180 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2181 .into_pointer_value();
2182
2183 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2184 .into_pointer_value();
2185 entry.insert(val);
2186 Ok(val)
2187 }
2188 }
2189 }
2190
2191 pub fn memory_notify(
2192 &mut self,
2193 memory_index: MemoryIndex,
2194 intrinsics: &Intrinsics<'ctx>,
2195 ) -> Result<PointerValue<'ctx>, CompileError> {
2196 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2197 &mut self.cached_memory_op,
2198 &self.wasm_module,
2199 &self.offsets,
2200 &self.cache_builder,
2201 &self.ctx_ptr_value,
2202 );
2203 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2204 Entry::Occupied(entry) => Ok(*entry.get()),
2205 Entry::Vacant(entry) => {
2206 let (size_fn, size_fn_ty) =
2207 if wasm_module.local_memory_index(memory_index).is_some() {
2208 (
2209 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2210 intrinsics.ptr_ty,
2211 )
2212 } else {
2213 (
2214 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2215 intrinsics.ptr_ty,
2216 )
2217 };
2218 let offset = offsets.vmctx_builtin_function(size_fn);
2219 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2220 let size_fn_ptr_ptr = unsafe {
2221 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2222 };
2223
2224 let size_fn_ptr_ptr =
2225 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2226 .into_pointer_value();
2227
2228 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2229 .into_pointer_value();
2230
2231 entry.insert(val);
2232 Ok(val)
2233 }
2234 }
2235 }
2236
2237 pub fn get_offsets(&self) -> &VMOffsets {
2238 &self.offsets
2239 }
2240}
2241
2242pub fn tbaa_label<'ctx>(
2245 module: &Module<'ctx>,
2246 intrinsics: &Intrinsics<'ctx>,
2247 label: String,
2248 instruction: InstructionValue<'ctx>,
2249) {
2250 let context = module.get_context();
2264
2265 let tbaa_root = module
2267 .get_global_metadata("wasmer_tbaa_root")
2268 .pop()
2269 .unwrap_or_else(|| {
2270 module
2271 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2272 .unwrap();
2273 module.get_global_metadata("wasmer_tbaa_root")[0]
2274 });
2275
2276 let type_label = context.metadata_string(label.as_str());
2279 let type_tbaa = module
2280 .get_global_metadata(label.as_str())
2281 .pop()
2282 .unwrap_or_else(|| {
2283 module
2284 .add_global_metadata(
2285 label.as_str(),
2286 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2287 )
2288 .unwrap();
2289 module.get_global_metadata(label.as_str())[0]
2290 });
2291
2292 let label = label + "_memop";
2299 let type_tbaa = module
2300 .get_global_metadata(label.as_str())
2301 .pop()
2302 .unwrap_or_else(|| {
2303 module
2304 .add_global_metadata(
2305 label.as_str(),
2306 &context.metadata_node(&[
2307 type_tbaa.into(),
2308 type_tbaa.into(),
2309 intrinsics.i64_zero.into(),
2310 ]),
2311 )
2312 .unwrap();
2313 module.get_global_metadata(label.as_str())[0]
2314 });
2315
2316 let tbaa_kind = context.get_kind_id("tbaa");
2318 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2319}
2320
2321fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2322 let table = module.tables.get(index)?;
2323 match table.maximum {
2324 None => Some(true),
2325 Some(max) => Some(max > table.minimum),
2326 }
2327}