1use crate::abi::Abi;
8use crate::error::err;
9use inkwell::values::BasicMetadataValueEnum;
10use inkwell::{
11 AddressSpace,
12 attributes::{Attribute, AttributeLoc},
13 builder::Builder,
14 context::Context,
15 module::{Linkage, Module},
16 targets::TargetData,
17 types::{
18 BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FloatType, FunctionType, IntType,
19 PointerType, StructType, VectorType, VoidType,
20 },
21 values::{
22 BasicValue, BasicValueEnum, FloatValue, FunctionValue, InstructionValue, IntValue,
23 PointerValue, VectorValue,
24 },
25};
26use std::collections::{HashMap, hash_map::Entry};
27use std::num::NonZero;
28use target_lexicon::{Architecture, Triple};
29use wasmer_types::entity::{EntityRef, PrimaryMap};
30use wasmer_types::{
31 CompileError, FunctionIndex, FunctionType as FuncType, GlobalIndex, LocalFunctionIndex,
32 LocalTableIndex, MemoryIndex, ModuleInfo as WasmerCompilerModule, Mutability, TableIndex, Type,
33};
34use wasmer_vm::{MemoryStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets};
35
36pub fn type_to_llvm<'ctx>(
37 intrinsics: &Intrinsics<'ctx>,
38 ty: Type,
39) -> Result<BasicTypeEnum<'ctx>, CompileError> {
40 match ty {
41 Type::I32 => Ok(intrinsics.i32_ty.as_basic_type_enum()),
42 Type::I64 => Ok(intrinsics.i64_ty.as_basic_type_enum()),
43 Type::F32 => Ok(intrinsics.f32_ty.as_basic_type_enum()),
44 Type::F64 => Ok(intrinsics.f64_ty.as_basic_type_enum()),
45 Type::V128 => Ok(intrinsics.i128_ty.as_basic_type_enum()),
46 Type::ExceptionRef => Ok(intrinsics.i32_ty.as_basic_type_enum()),
47 Type::FuncRef | Type::ExternRef => Ok(intrinsics.ptr_ty.as_basic_type_enum()),
48 }
49}
50
51#[allow(dead_code)]
53pub struct X86_64Intrinsics<'ctx> {
54 pub pshufb128: FunctionValue<'ctx>,
55 pub pmaddubsw128: FunctionValue<'ctx>,
56 pub pmaddwd128: FunctionValue<'ctx>,
57 pub pmulhrsw128: FunctionValue<'ctx>,
58 pub pblendvb: FunctionValue<'ctx>,
59 pub min_ps: FunctionValue<'ctx>,
60 pub min_pd: FunctionValue<'ctx>,
61 pub max_ps: FunctionValue<'ctx>,
62 pub max_pd: FunctionValue<'ctx>,
63 pub cvttps2dq: FunctionValue<'ctx>,
64 pub cvtps2udq128: FunctionValue<'ctx>,
65 pub cvtpd2dq: FunctionValue<'ctx>,
66 pub cvtpd2udq128: FunctionValue<'ctx>,
67}
68
69#[allow(dead_code)]
71pub struct Intrinsics<'ctx> {
72 pub ctlz_i32: FunctionValue<'ctx>,
73 pub ctlz_i64: FunctionValue<'ctx>,
74
75 pub cttz_i32: FunctionValue<'ctx>,
76 pub cttz_i64: FunctionValue<'ctx>,
77
78 pub ctpop_i32: FunctionValue<'ctx>,
79 pub ctpop_i64: FunctionValue<'ctx>,
80 pub ctpop_i8x16: FunctionValue<'ctx>,
81
82 pub fp_rounding_md: BasicMetadataValueEnum<'ctx>,
83 pub fp_exception_md: BasicMetadataValueEnum<'ctx>,
84 pub fp_ogt_md: BasicMetadataValueEnum<'ctx>,
85 pub fp_olt_md: BasicMetadataValueEnum<'ctx>,
86 pub fp_uno_md: BasicMetadataValueEnum<'ctx>,
87
88 pub add_f32: FunctionValue<'ctx>,
89 pub add_f64: FunctionValue<'ctx>,
90 pub add_f32x4: FunctionValue<'ctx>,
91 pub add_f64x2: FunctionValue<'ctx>,
92
93 pub sub_f32: FunctionValue<'ctx>,
94 pub sub_f64: FunctionValue<'ctx>,
95 pub sub_f32x4: FunctionValue<'ctx>,
96 pub sub_f64x2: FunctionValue<'ctx>,
97
98 pub mul_f32: FunctionValue<'ctx>,
99 pub mul_f64: FunctionValue<'ctx>,
100 pub mul_f32x4: FunctionValue<'ctx>,
101 pub mul_f64x2: FunctionValue<'ctx>,
102 pub muladd_f32x4: FunctionValue<'ctx>,
103 pub muladd_f64x2: FunctionValue<'ctx>,
104
105 pub div_f32: FunctionValue<'ctx>,
106 pub div_f64: FunctionValue<'ctx>,
107 pub div_f32x4: FunctionValue<'ctx>,
108 pub div_f64x2: FunctionValue<'ctx>,
109
110 pub sqrt_f32: FunctionValue<'ctx>,
111 pub sqrt_f64: FunctionValue<'ctx>,
112 pub sqrt_f32x4: FunctionValue<'ctx>,
113 pub sqrt_f64x2: FunctionValue<'ctx>,
114
115 pub cmp_f32: FunctionValue<'ctx>,
116 pub cmp_f64: FunctionValue<'ctx>,
117 pub cmp_f32x4: FunctionValue<'ctx>,
118 pub cmp_f64x2: FunctionValue<'ctx>,
119
120 pub minimum_f32: FunctionValue<'ctx>,
121 pub minimum_f64: FunctionValue<'ctx>,
122 pub minimum_f32x4: FunctionValue<'ctx>,
123 pub minimum_f64x2: FunctionValue<'ctx>,
124
125 pub maximum_f32: FunctionValue<'ctx>,
126 pub maximum_f64: FunctionValue<'ctx>,
127 pub maximum_f32x4: FunctionValue<'ctx>,
128 pub maximum_f64x2: FunctionValue<'ctx>,
129
130 pub ceil_f32: FunctionValue<'ctx>,
131 pub ceil_f64: FunctionValue<'ctx>,
132 pub ceil_f32x4: FunctionValue<'ctx>,
133 pub ceil_f64x2: FunctionValue<'ctx>,
134
135 pub floor_f32: FunctionValue<'ctx>,
136 pub floor_f64: FunctionValue<'ctx>,
137 pub floor_f32x4: FunctionValue<'ctx>,
138 pub floor_f64x2: FunctionValue<'ctx>,
139
140 pub trunc_f32: FunctionValue<'ctx>,
141 pub trunc_f64: FunctionValue<'ctx>,
142 pub trunc_f32x4: FunctionValue<'ctx>,
143 pub trunc_f64x2: FunctionValue<'ctx>,
144
145 pub fpext_f32: FunctionValue<'ctx>,
146 pub fptrunc_f64: FunctionValue<'ctx>,
147
148 pub nearbyint_f32: FunctionValue<'ctx>,
149 pub nearbyint_f64: FunctionValue<'ctx>,
150 pub nearbyint_f32x4: FunctionValue<'ctx>,
151 pub nearbyint_f64x2: FunctionValue<'ctx>,
152
153 pub fabs_f32: FunctionValue<'ctx>,
154 pub fabs_f64: FunctionValue<'ctx>,
155 pub fabs_f32x4: FunctionValue<'ctx>,
156 pub fabs_f64x2: FunctionValue<'ctx>,
157
158 pub copysign_f32: FunctionValue<'ctx>,
159 pub copysign_f64: FunctionValue<'ctx>,
160 pub copysign_f32x4: FunctionValue<'ctx>,
161 pub copysign_f64x2: FunctionValue<'ctx>,
162
163 pub sadd_sat_i8x16: FunctionValue<'ctx>,
164 pub sadd_sat_i16x8: FunctionValue<'ctx>,
165 pub uadd_sat_i8x16: FunctionValue<'ctx>,
166 pub uadd_sat_i16x8: FunctionValue<'ctx>,
167
168 pub ssub_sat_i8x16: FunctionValue<'ctx>,
169 pub ssub_sat_i16x8: FunctionValue<'ctx>,
170 pub usub_sat_i8x16: FunctionValue<'ctx>,
171 pub usub_sat_i16x8: FunctionValue<'ctx>,
172
173 pub expect_i1: FunctionValue<'ctx>,
174 pub trap: FunctionValue<'ctx>,
175 pub debug_trap: FunctionValue<'ctx>,
176
177 pub personality: FunctionValue<'ctx>,
178 pub personality2: FunctionValue<'ctx>,
179 pub readonly: Attribute,
180 pub stack_probe: Attribute,
181 pub uwtable: Attribute,
182 pub nounwind: Attribute,
183 pub frame_pointer: Attribute,
184 pub chkstk: FunctionValue<'ctx>,
186
187 pub void_ty: VoidType<'ctx>,
188 pub i1_ty: IntType<'ctx>,
189 pub i2_ty: IntType<'ctx>,
190 pub i4_ty: IntType<'ctx>,
191 pub i8_ty: IntType<'ctx>,
192 pub i16_ty: IntType<'ctx>,
193 pub i32_ty: IntType<'ctx>,
194 pub i64_ty: IntType<'ctx>,
195 pub i128_ty: IntType<'ctx>,
196 pub isize_ty: IntType<'ctx>,
197 pub f32_ty: FloatType<'ctx>,
198 pub f64_ty: FloatType<'ctx>,
199
200 pub i1x128_ty: VectorType<'ctx>,
201 pub i8x16_ty: VectorType<'ctx>,
202 pub i16x8_ty: VectorType<'ctx>,
203 pub i32x4_ty: VectorType<'ctx>,
204 pub i64x2_ty: VectorType<'ctx>,
205 pub f32x4_ty: VectorType<'ctx>,
206 pub f64x2_ty: VectorType<'ctx>,
207 pub i32x8_ty: VectorType<'ctx>,
208
209 pub ptr_ty: PointerType<'ctx>,
210
211 pub x86_64: X86_64Intrinsics<'ctx>,
212
213 pub anyfunc_ty: StructType<'ctx>,
214
215 pub i1_zero: IntValue<'ctx>,
216 pub i8_zero: IntValue<'ctx>,
217 pub i32_zero: IntValue<'ctx>,
218 pub i64_zero: IntValue<'ctx>,
219 pub i128_zero: IntValue<'ctx>,
220 pub isize_zero: IntValue<'ctx>,
221 pub f32_zero: FloatValue<'ctx>,
222 pub f64_zero: FloatValue<'ctx>,
223 pub f32x4_zero: VectorValue<'ctx>,
224 pub f64x2_zero: VectorValue<'ctx>,
225 pub i32_consts: [IntValue<'ctx>; 16],
226
227 pub trap_unreachable: BasicValueEnum<'ctx>,
228 pub trap_call_indirect_null: BasicValueEnum<'ctx>,
229 pub trap_call_indirect_sig: BasicValueEnum<'ctx>,
230 pub trap_memory_oob: BasicValueEnum<'ctx>,
231 pub trap_illegal_arithmetic: BasicValueEnum<'ctx>,
232 pub trap_integer_division_by_zero: BasicValueEnum<'ctx>,
233 pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>,
234 pub trap_unaligned_atomic: BasicValueEnum<'ctx>,
235 pub trap_table_access_oob: BasicValueEnum<'ctx>,
236
237 pub experimental_stackmap: FunctionValue<'ctx>,
238
239 pub table_copy: FunctionValue<'ctx>,
241 pub table_init: FunctionValue<'ctx>,
242 pub table_fill: FunctionValue<'ctx>,
243 pub table_size: FunctionValue<'ctx>,
244 pub imported_table_size: FunctionValue<'ctx>,
245 pub table_get: FunctionValue<'ctx>,
246 pub imported_table_get: FunctionValue<'ctx>,
247 pub table_set: FunctionValue<'ctx>,
248 pub imported_table_set: FunctionValue<'ctx>,
249 pub table_grow: FunctionValue<'ctx>,
250 pub imported_table_grow: FunctionValue<'ctx>,
251 pub memory_init: FunctionValue<'ctx>,
252 pub data_drop: FunctionValue<'ctx>,
253 pub func_ref: FunctionValue<'ctx>,
254 pub elem_drop: FunctionValue<'ctx>,
255 pub memory_copy: FunctionValue<'ctx>,
256 pub imported_memory_copy: FunctionValue<'ctx>,
257 pub memory_fill: FunctionValue<'ctx>,
258 pub imported_memory_fill: FunctionValue<'ctx>,
259 pub memory_size_ty: FunctionType<'ctx>,
260 pub memory_grow_ty: FunctionType<'ctx>,
261 pub memory_wait32: FunctionValue<'ctx>,
262 pub memory_wait32_ty: FunctionType<'ctx>,
263 pub imported_memory_wait32: FunctionValue<'ctx>,
264 pub memory_wait64: FunctionValue<'ctx>,
265 pub memory_wait64_ty: FunctionType<'ctx>,
266 pub imported_memory_wait64: FunctionValue<'ctx>,
267 pub memory_notify: FunctionValue<'ctx>,
268 pub memory_notify_ty: FunctionType<'ctx>,
269 pub imported_memory_notify: FunctionValue<'ctx>,
270
271 pub throw_trap: FunctionValue<'ctx>,
272
273 pub throw: FunctionValue<'ctx>,
275 pub alloc_exception: FunctionValue<'ctx>,
276 pub read_exnref: FunctionValue<'ctx>,
277 pub exception_into_exnref: FunctionValue<'ctx>,
278 pub lpad_exception_ty: StructType<'ctx>,
279
280 pub debug_ptr: FunctionValue<'ctx>,
282 pub debug_str: FunctionValue<'ctx>,
283
284 pub vmfunction_import_ty: StructType<'ctx>,
286 pub vmfunction_import_body_element: u32,
287 pub vmfunction_import_vmctx_element: u32,
288 pub vmfunction_import_include_m0_param_element: u32,
289
290 pub vmmemory_definition_ty: StructType<'ctx>,
291 pub vmmemory_definition_base_element: u32,
292 pub vmmemory_definition_current_length_element: u32,
293}
294
295#[derive(Debug, Hash, PartialEq, Eq)]
296enum MemoryOp {
297 Size,
298 Grow,
299 Wait32,
300 Wait64,
301 Notify,
302}
303
304impl<'ctx> Intrinsics<'ctx> {
305 pub fn declare(
307 module: &Module<'ctx>,
308 context: &'ctx Context,
309 target_data: &TargetData,
310 target_triple: &Triple,
311 binary_fmt: &target_lexicon::BinaryFormat,
312 ) -> Self {
313 let is_riscv64 = matches!(target_triple.architecture, Architecture::Riscv64(..));
314 let void_ty = context.void_type();
315 let i1_ty = context.bool_type();
316 let i2_ty = context
317 .custom_width_int_type(NonZero::new(2).unwrap())
318 .unwrap();
319 let i4_ty = context
320 .custom_width_int_type(NonZero::new(4).unwrap())
321 .unwrap();
322 let i8_ty = context.i8_type();
323 let i16_ty = context.i16_type();
324 let i32_ty = context.i32_type();
325 let i64_ty = context.i64_type();
326 let i128_ty = context.i128_type();
327 let isize_ty = context.ptr_sized_int_type(target_data, None);
328 let f32_ty = context.f32_type();
329 let f64_ty = context.f64_type();
330
331 let i1x4_ty = i1_ty.vec_type(4);
332 let i1x2_ty = i1_ty.vec_type(2);
333 let i1x128_ty = i1_ty.vec_type(128);
334 let i8x16_ty = i8_ty.vec_type(16);
335 let i16x8_ty = i16_ty.vec_type(8);
336 let i32x4_ty = i32_ty.vec_type(4);
337 let i64x2_ty = i64_ty.vec_type(2);
338 let f32x4_ty = f32_ty.vec_type(4);
339 let f64x2_ty = f64_ty.vec_type(2);
340 let i32x8_ty = i32_ty.vec_type(8);
341
342 let ptr_ty = context.ptr_type(AddressSpace::default());
343
344 let i1_zero = i1_ty.const_int(0, false);
345 let i8_zero = i8_ty.const_int(0, false);
346 let i32_zero = i32_ty.const_int(0, false);
347 let i64_zero = i64_ty.const_int(0, false);
348 let i128_zero = i128_ty.const_int(0, false);
349 let isize_zero = isize_ty.const_int(0, false);
350 let f32_zero = f32_ty.const_float(0.0);
351 let f64_zero = f64_ty.const_float(0.0);
352 let f32x4_zero = f32x4_ty.const_zero();
353 let f64x2_zero = f64x2_ty.const_zero();
354 let i32_consts = [
355 i32_ty.const_int(0, false),
356 i32_ty.const_int(1, false),
357 i32_ty.const_int(2, false),
358 i32_ty.const_int(3, false),
359 i32_ty.const_int(4, false),
360 i32_ty.const_int(5, false),
361 i32_ty.const_int(6, false),
362 i32_ty.const_int(7, false),
363 i32_ty.const_int(8, false),
364 i32_ty.const_int(9, false),
365 i32_ty.const_int(10, false),
366 i32_ty.const_int(11, false),
367 i32_ty.const_int(12, false),
368 i32_ty.const_int(13, false),
369 i32_ty.const_int(14, false),
370 i32_ty.const_int(15, false),
371 ];
372
373 let md_ty = context.metadata_type();
374
375 let i8_ptr_ty_basic = ptr_ty.as_basic_type_enum();
376 let i1_ty_basic = i1_ty.as_basic_type_enum();
377
378 let i1_ty_basic_md: BasicMetadataTypeEnum = i1_ty.into();
379 let i32_ty_basic_md: BasicMetadataTypeEnum = i32_ty.into();
380 let i64_ty_basic_md: BasicMetadataTypeEnum = i64_ty.into();
381 let f32_ty_basic_md: BasicMetadataTypeEnum = f32_ty.into();
382 let f64_ty_basic_md: BasicMetadataTypeEnum = f64_ty.into();
383 let i8x16_ty_basic_md: BasicMetadataTypeEnum = i8x16_ty.into();
384 let i16x8_ty_basic_md: BasicMetadataTypeEnum = i16x8_ty.into();
385 let i32x4_ty_basic_md: BasicMetadataTypeEnum = i32x4_ty.into();
386 let f32x4_ty_basic_md: BasicMetadataTypeEnum = f32x4_ty.into();
387 let f64x2_ty_basic_md: BasicMetadataTypeEnum = f64x2_ty.into();
388 let md_ty_basic_md: BasicMetadataTypeEnum = md_ty.into();
389
390 let ctx_ptr_ty = ptr_ty;
391 let ctx_ptr_ty_basic = ctx_ptr_ty.as_basic_type_enum();
392 let ctx_ptr_ty_basic_md: BasicMetadataTypeEnum = ctx_ptr_ty.into();
393
394 let anyfunc_ty = context.struct_type(
398 &[
399 i8_ptr_ty_basic,
400 i32_ty.into(),
401 ctx_ptr_ty_basic,
402 ptr_ty.into(),
403 ],
404 false,
405 );
406 let funcref_ty = ptr_ty;
407 let anyref_ty = ptr_ty;
408 let anyref_ty_basic_md: BasicMetadataTypeEnum = anyref_ty.into();
409
410 let ret_i8x16_take_i8x16 = i8x16_ty.fn_type(&[i8x16_ty_basic_md], false);
411 let ret_i8x16_take_i8x16_i8x16 =
412 i8x16_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false);
413 let ret_i8x16_take_i8x16_i8x16_i8x16 = i8x16_ty.fn_type(
414 &[i8x16_ty_basic_md, i8x16_ty_basic_md, i8x16_ty_basic_md],
415 false,
416 );
417 let ret_i16x8_take_i16x8_i16x8 =
418 i16x8_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false);
419
420 let ret_i32_take_i32_i1 = i32_ty.fn_type(&[i32_ty_basic_md, i1_ty_basic_md], false);
421 let ret_i64_take_i64_i1 = i64_ty.fn_type(&[i64_ty_basic_md, i1_ty_basic_md], false);
422
423 let ret_i32_take_i32 = i32_ty.fn_type(&[i32_ty_basic_md], false);
424 let ret_i64_take_i64 = i64_ty.fn_type(&[i64_ty_basic_md], false);
425
426 let ret_f32_take_f32 = f32_ty.fn_type(&[f32_ty_basic_md], false);
427 let ret_f64_take_f64 = f64_ty.fn_type(&[f64_ty_basic_md], false);
428 let ret_f32x4_take_f32x4 = f32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
429 let ret_f64x2_take_f64x2 = f64x2_ty.fn_type(&[f64x2_ty_basic_md], false);
430
431 let ret_f32_take_f32_f32 = f32_ty.fn_type(&[f32_ty_basic_md, f32_ty_basic_md], false);
432 let ret_f64_take_f64_f64 = f64_ty.fn_type(&[f64_ty_basic_md, f64_ty_basic_md], false);
433 let ret_f32x4_take_f32x4_f32x4 =
434 f32x4_ty.fn_type(&[f32x4_ty_basic_md, f32x4_ty_basic_md], false);
435 let ret_f64x2_take_f64x2_f64x2 =
436 f64x2_ty.fn_type(&[f64x2_ty_basic_md, f64x2_ty_basic_md], false);
437 let ret_f32x4_take_f32x4_f32x4_f32x4_md_md = f32x4_ty.fn_type(
438 &[
439 f32x4_ty_basic_md,
440 f32x4_ty_basic_md,
441 f32x4_ty_basic_md,
442 md_ty_basic_md,
443 md_ty_basic_md,
444 ],
445 false,
446 );
447 let ret_f64x2_take_f64x2_f64x2_f64x2_md_md = f64x2_ty.fn_type(
448 &[
449 f64x2_ty_basic_md,
450 f64x2_ty_basic_md,
451 f64x2_ty_basic_md,
452 md_ty_basic_md,
453 md_ty_basic_md,
454 ],
455 false,
456 );
457
458 let ret_f64_take_f32_md = f64_ty.fn_type(&[f32_ty_basic_md, md_ty_basic_md], false);
459 let ret_f32_take_f64_md_md =
460 f32_ty.fn_type(&[f64_ty_basic_md, md_ty_basic_md, md_ty_basic_md], false);
461
462 let ret_i1_take_i1_i1 = i1_ty.fn_type(&[i1_ty_basic_md, i1_ty_basic_md], false);
463
464 let ret_i1_take_f32_f32_md_md = i1_ty.fn_type(
465 &[
466 f32_ty_basic_md,
467 f32_ty_basic_md,
468 md_ty_basic_md,
469 md_ty_basic_md,
470 ],
471 false,
472 );
473 let ret_i1_take_f64_f64_md_md = i1_ty.fn_type(
474 &[
475 f64_ty_basic_md,
476 f64_ty_basic_md,
477 md_ty_basic_md,
478 md_ty_basic_md,
479 ],
480 false,
481 );
482 let ret_i1x4_take_f32x4_f32x4_md_md = i1x4_ty.fn_type(
483 &[
484 f32x4_ty_basic_md,
485 f32x4_ty_basic_md,
486 md_ty_basic_md,
487 md_ty_basic_md,
488 ],
489 false,
490 );
491 let ret_i1x2_take_f64x2_f64x2_md_md = i1x2_ty.fn_type(
492 &[
493 f64x2_ty_basic_md,
494 f64x2_ty_basic_md,
495 md_ty_basic_md,
496 md_ty_basic_md,
497 ],
498 false,
499 );
500
501 let ret_f32_take_f32_f32_md_md = f32_ty.fn_type(
502 &[
503 f32_ty_basic_md,
504 f32_ty_basic_md,
505 md_ty_basic_md,
506 md_ty_basic_md,
507 ],
508 false,
509 );
510 let ret_f64_take_f64_f64_md_md = f64_ty.fn_type(
511 &[
512 f64_ty_basic_md,
513 f64_ty_basic_md,
514 md_ty_basic_md,
515 md_ty_basic_md,
516 ],
517 false,
518 );
519 let ret_f32x4_take_f32x4_f32x4_md_md = f32x4_ty.fn_type(
520 &[
521 f32x4_ty_basic_md,
522 f32x4_ty_basic_md,
523 md_ty_basic_md,
524 md_ty_basic_md,
525 ],
526 false,
527 );
528 let ret_f64x2_take_f64x2_f64x2_md_md = f64x2_ty.fn_type(
529 &[
530 f64x2_ty_basic_md,
531 f64x2_ty_basic_md,
532 md_ty_basic_md,
533 md_ty_basic_md,
534 ],
535 false,
536 );
537 let ret_i32x4_take_f32x4 = i32x4_ty.fn_type(&[f32x4_ty_basic_md], false);
538 let ret_i32x4_take_f32x4_i32x4_i8 =
539 i32x4_ty.fn_type(&[f32x4_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
540 let ret_i32x4_take_f64x2 = i32x4_ty.fn_type(&[f64x2_ty_basic_md], false);
541 let ret_i32x4_take_f64x2_i32x4_i8 =
542 i32x4_ty.fn_type(&[f64x2_ty_basic_md, i32x4_ty_basic_md, i8_ty.into()], false);
543
544 let add_function_with_attrs =
545 |name: &str, ty: FunctionType<'ctx>, linkage: Option<Linkage>| -> FunctionValue<'ctx> {
546 let function = module.add_function(name, ty, linkage);
547 if is_riscv64 {
552 for (i, param) in ty.get_param_types().iter().enumerate() {
553 if param == &i32_ty_basic_md {
554 function.add_attribute(
555 AttributeLoc::Param(i as u32),
556 context.create_enum_attribute(
557 Attribute::get_named_enum_kind_id("signext"),
558 0,
559 ),
560 );
561 function.add_attribute(
562 AttributeLoc::Param(i as u32),
563 context.create_enum_attribute(
564 Attribute::get_named_enum_kind_id("noundef"),
565 0,
566 ),
567 );
568 }
569 }
570 }
571 function
572 };
573
574 let intrinsics = Self {
575 ctlz_i32: add_function_with_attrs("llvm.ctlz.i32", ret_i32_take_i32_i1, None),
576 ctlz_i64: add_function_with_attrs("llvm.ctlz.i64", ret_i64_take_i64_i1, None),
577
578 cttz_i32: add_function_with_attrs("llvm.cttz.i32", ret_i32_take_i32_i1, None),
579 cttz_i64: add_function_with_attrs("llvm.cttz.i64", ret_i64_take_i64_i1, None),
580
581 ctpop_i32: add_function_with_attrs("llvm.ctpop.i32", ret_i32_take_i32, None),
582 ctpop_i64: add_function_with_attrs("llvm.ctpop.i64", ret_i64_take_i64, None),
583 ctpop_i8x16: add_function_with_attrs("llvm.ctpop.v16i8", ret_i8x16_take_i8x16, None),
584
585 fp_rounding_md: context.metadata_string("round.tonearest").into(),
586 fp_exception_md: context.metadata_string("fpexcept.strict").into(),
587
588 fp_ogt_md: context.metadata_string("ogt").into(),
589 fp_olt_md: context.metadata_string("olt").into(),
590 fp_uno_md: context.metadata_string("uno").into(),
591
592 sqrt_f32: add_function_with_attrs("llvm.sqrt.f32", ret_f32_take_f32, None),
593 sqrt_f64: add_function_with_attrs("llvm.sqrt.f64", ret_f64_take_f64, None),
594 sqrt_f32x4: add_function_with_attrs("llvm.sqrt.v4f32", ret_f32x4_take_f32x4, None),
595 sqrt_f64x2: add_function_with_attrs("llvm.sqrt.v2f64", ret_f64x2_take_f64x2, None),
596
597 ceil_f32: add_function_with_attrs("llvm.ceil.f32", ret_f32_take_f32, None),
598 ceil_f64: add_function_with_attrs("llvm.ceil.f64", ret_f64_take_f64, None),
599 ceil_f32x4: add_function_with_attrs("llvm.ceil.v4f32", ret_f32x4_take_f32x4, None),
600 ceil_f64x2: add_function_with_attrs("llvm.ceil.v2f64", ret_f64x2_take_f64x2, None),
601
602 floor_f32: add_function_with_attrs("llvm.floor.f32", ret_f32_take_f32, None),
603 floor_f64: add_function_with_attrs("llvm.floor.f64", ret_f64_take_f64, None),
604 floor_f32x4: add_function_with_attrs("llvm.floor.v4f32", ret_f32x4_take_f32x4, None),
605 floor_f64x2: add_function_with_attrs("llvm.floor.v2f64", ret_f64x2_take_f64x2, None),
606
607 trunc_f32: add_function_with_attrs("llvm.trunc.f32", ret_f32_take_f32, None),
608 trunc_f64: add_function_with_attrs("llvm.trunc.f64", ret_f64_take_f64, None),
609 trunc_f32x4: add_function_with_attrs("llvm.trunc.v4f32", ret_f32x4_take_f32x4, None),
610 trunc_f64x2: add_function_with_attrs("llvm.trunc.v2f64", ret_f64x2_take_f64x2, None),
611
612 nearbyint_f32: add_function_with_attrs("llvm.nearbyint.f32", ret_f32_take_f32, None),
613 nearbyint_f64: add_function_with_attrs("llvm.nearbyint.f64", ret_f64_take_f64, None),
614 nearbyint_f32x4: add_function_with_attrs(
615 "llvm.nearbyint.v4f32",
616 ret_f32x4_take_f32x4,
617 None,
618 ),
619 nearbyint_f64x2: add_function_with_attrs(
620 "llvm.nearbyint.v2f64",
621 ret_f64x2_take_f64x2,
622 None,
623 ),
624
625 add_f32: add_function_with_attrs(
626 "llvm.experimental.constrained.fadd.f32",
627 ret_f32_take_f32_f32_md_md,
628 None,
629 ),
630 add_f64: add_function_with_attrs(
631 "llvm.experimental.constrained.fadd.f64",
632 ret_f64_take_f64_f64_md_md,
633 None,
634 ),
635 add_f32x4: add_function_with_attrs(
636 "llvm.experimental.constrained.fadd.v4f32",
637 ret_f32x4_take_f32x4_f32x4_md_md,
638 None,
639 ),
640 add_f64x2: add_function_with_attrs(
641 "llvm.experimental.constrained.fadd.v2f64",
642 ret_f64x2_take_f64x2_f64x2_md_md,
643 None,
644 ),
645
646 sub_f32: add_function_with_attrs(
647 "llvm.experimental.constrained.fsub.f32",
648 ret_f32_take_f32_f32_md_md,
649 None,
650 ),
651 sub_f64: add_function_with_attrs(
652 "llvm.experimental.constrained.fsub.f64",
653 ret_f64_take_f64_f64_md_md,
654 None,
655 ),
656 sub_f32x4: add_function_with_attrs(
657 "llvm.experimental.constrained.fsub.v4f32",
658 ret_f32x4_take_f32x4_f32x4_md_md,
659 None,
660 ),
661 sub_f64x2: add_function_with_attrs(
662 "llvm.experimental.constrained.fsub.v2f64",
663 ret_f64x2_take_f64x2_f64x2_md_md,
664 None,
665 ),
666
667 mul_f32: add_function_with_attrs(
668 "llvm.experimental.constrained.fmul.f32",
669 ret_f32_take_f32_f32_md_md,
670 None,
671 ),
672 mul_f64: add_function_with_attrs(
673 "llvm.experimental.constrained.fmul.f64",
674 ret_f64_take_f64_f64_md_md,
675 None,
676 ),
677 mul_f32x4: add_function_with_attrs(
678 "llvm.experimental.constrained.fmul.v4f32",
679 ret_f32x4_take_f32x4_f32x4_md_md,
680 None,
681 ),
682 mul_f64x2: add_function_with_attrs(
683 "llvm.experimental.constrained.fmul.v2f64",
684 ret_f64x2_take_f64x2_f64x2_md_md,
685 None,
686 ),
687 muladd_f32x4: add_function_with_attrs(
688 "llvm.experimental.constrained.fmuladd.v4f32",
689 ret_f32x4_take_f32x4_f32x4_f32x4_md_md,
690 None,
691 ),
692 muladd_f64x2: add_function_with_attrs(
693 "llvm.experimental.constrained.fmuladd.v2f64",
694 ret_f64x2_take_f64x2_f64x2_f64x2_md_md,
695 None,
696 ),
697
698 div_f32: add_function_with_attrs(
699 "llvm.experimental.constrained.fdiv.f32",
700 ret_f32_take_f32_f32_md_md,
701 None,
702 ),
703 div_f64: add_function_with_attrs(
704 "llvm.experimental.constrained.fdiv.f64",
705 ret_f64_take_f64_f64_md_md,
706 None,
707 ),
708 div_f32x4: add_function_with_attrs(
709 "llvm.experimental.constrained.fdiv.v4f32",
710 ret_f32x4_take_f32x4_f32x4_md_md,
711 None,
712 ),
713 div_f64x2: add_function_with_attrs(
714 "llvm.experimental.constrained.fdiv.v2f64",
715 ret_f64x2_take_f64x2_f64x2_md_md,
716 None,
717 ),
718
719 cmp_f32: add_function_with_attrs(
720 "llvm.experimental.constrained.fcmp.f32",
721 ret_i1_take_f32_f32_md_md,
722 None,
723 ),
724 cmp_f64: add_function_with_attrs(
725 "llvm.experimental.constrained.fcmp.f64",
726 ret_i1_take_f64_f64_md_md,
727 None,
728 ),
729 cmp_f32x4: add_function_with_attrs(
730 "llvm.experimental.constrained.fcmp.v4f32",
731 ret_i1x4_take_f32x4_f32x4_md_md,
732 None,
733 ),
734 cmp_f64x2: add_function_with_attrs(
735 "llvm.experimental.constrained.fcmp.v2f64",
736 ret_i1x2_take_f64x2_f64x2_md_md,
737 None,
738 ),
739
740 minimum_f32: add_function_with_attrs("llvm.minimum.f32", ret_f32_take_f32_f32, None),
741 minimum_f64: add_function_with_attrs("llvm.minimum.f64", ret_f64_take_f64_f64, None),
742 minimum_f32x4: add_function_with_attrs(
743 "llvm.minimum.v4f32",
744 ret_f32x4_take_f32x4_f32x4,
745 None,
746 ),
747 minimum_f64x2: add_function_with_attrs(
748 "llvm.minimum.v2f64",
749 ret_f64x2_take_f64x2_f64x2,
750 None,
751 ),
752
753 maximum_f32: add_function_with_attrs("llvm.maximum.f32", ret_f32_take_f32_f32, None),
754 maximum_f64: add_function_with_attrs("llvm.maximum.f64", ret_f64_take_f64_f64, None),
755 maximum_f32x4: add_function_with_attrs(
756 "llvm.maximum.v4f32",
757 ret_f32x4_take_f32x4_f32x4,
758 None,
759 ),
760 maximum_f64x2: add_function_with_attrs(
761 "llvm.maximum.v2f64",
762 ret_f64x2_take_f64x2_f64x2,
763 None,
764 ),
765
766 fpext_f32: add_function_with_attrs(
767 "llvm.experimental.constrained.fpext.f64.f32",
768 ret_f64_take_f32_md,
769 None,
770 ),
771 fptrunc_f64: add_function_with_attrs(
772 "llvm.experimental.constrained.fptrunc.f32.f64",
773 ret_f32_take_f64_md_md,
774 None,
775 ),
776
777 fabs_f32: add_function_with_attrs("llvm.fabs.f32", ret_f32_take_f32, None),
778 fabs_f64: add_function_with_attrs("llvm.fabs.f64", ret_f64_take_f64, None),
779 fabs_f32x4: add_function_with_attrs("llvm.fabs.v4f32", ret_f32x4_take_f32x4, None),
780 fabs_f64x2: add_function_with_attrs("llvm.fabs.v2f64", ret_f64x2_take_f64x2, None),
781
782 copysign_f32: add_function_with_attrs("llvm.copysign.f32", ret_f32_take_f32_f32, None),
783 copysign_f64: add_function_with_attrs("llvm.copysign.f64", ret_f64_take_f64_f64, None),
784 copysign_f32x4: add_function_with_attrs(
785 "llvm.copysign.v4f32",
786 ret_f32x4_take_f32x4_f32x4,
787 None,
788 ),
789 copysign_f64x2: add_function_with_attrs(
790 "llvm.copysign.v2f64",
791 ret_f64x2_take_f64x2_f64x2,
792 None,
793 ),
794
795 sadd_sat_i8x16: add_function_with_attrs(
796 "llvm.sadd.sat.v16i8",
797 ret_i8x16_take_i8x16_i8x16,
798 None,
799 ),
800 sadd_sat_i16x8: add_function_with_attrs(
801 "llvm.sadd.sat.v8i16",
802 ret_i16x8_take_i16x8_i16x8,
803 None,
804 ),
805 uadd_sat_i8x16: add_function_with_attrs(
806 "llvm.uadd.sat.v16i8",
807 ret_i8x16_take_i8x16_i8x16,
808 None,
809 ),
810 uadd_sat_i16x8: add_function_with_attrs(
811 "llvm.uadd.sat.v8i16",
812 ret_i16x8_take_i16x8_i16x8,
813 None,
814 ),
815
816 ssub_sat_i8x16: add_function_with_attrs(
817 "llvm.ssub.sat.v16i8",
818 ret_i8x16_take_i8x16_i8x16,
819 None,
820 ),
821 ssub_sat_i16x8: add_function_with_attrs(
822 "llvm.ssub.sat.v8i16",
823 ret_i16x8_take_i16x8_i16x8,
824 None,
825 ),
826 usub_sat_i8x16: add_function_with_attrs(
827 "llvm.usub.sat.v16i8",
828 ret_i8x16_take_i8x16_i8x16,
829 None,
830 ),
831 usub_sat_i16x8: add_function_with_attrs(
832 "llvm.usub.sat.v8i16",
833 ret_i16x8_take_i16x8_i16x8,
834 None,
835 ),
836
837 expect_i1: add_function_with_attrs("llvm.expect.i1", ret_i1_take_i1_i1, None),
838 trap: add_function_with_attrs("llvm.trap", void_ty.fn_type(&[], false), None),
839 debug_trap: add_function_with_attrs(
840 "llvm.debugtrap",
841 void_ty.fn_type(&[], false),
842 None,
843 ),
844 personality: add_function_with_attrs(
845 if matches!(binary_fmt, target_lexicon::BinaryFormat::Macho) {
846 "__gxx_personality_v0"
849 } else {
850 "wasmer_eh_personality"
851 },
852 i32_ty.fn_type(
853 &[
854 i32_ty.into(),
855 i32_ty.into(),
856 i64_ty.into(),
857 ptr_ty.into(),
858 ptr_ty.into(),
859 ],
860 false,
861 ),
862 None,
863 ),
864 personality2: add_function_with_attrs(
865 "wasmer_eh_personality2",
866 i32_ty.fn_type(&[ptr_ty.into(), ptr_ty.into()], false),
867 None,
868 ),
869 readonly: context
870 .create_enum_attribute(Attribute::get_named_enum_kind_id("readonly"), 0),
871 stack_probe: context.create_string_attribute("probe-stack", "inline-asm"),
872 uwtable: context.create_enum_attribute(Attribute::get_named_enum_kind_id("uwtable"), 1),
873 nounwind: context
874 .create_enum_attribute(Attribute::get_named_enum_kind_id("nounwind"), 1),
875 frame_pointer: context.create_string_attribute("frame-pointer", "non-leaf"),
876 chkstk: add_function_with_attrs("__chkstk", void_ty.fn_type(&[], false), None),
877 void_ty,
878 i1_ty,
879 i2_ty,
880 i4_ty,
881 i8_ty,
882 i16_ty,
883 i32_ty,
884 i64_ty,
885 i128_ty,
886 isize_ty,
887 f32_ty,
888 f64_ty,
889
890 i1x128_ty,
891 i8x16_ty,
892 i16x8_ty,
893 i32x4_ty,
894 i64x2_ty,
895 f32x4_ty,
896 f64x2_ty,
897 i32x8_ty,
898
899 anyfunc_ty,
900 i1_zero,
901 i8_zero,
902 i32_zero,
903 i64_zero,
904 i128_zero,
905 isize_zero,
906 f32_zero,
907 f64_zero,
908 f32x4_zero,
909 f64x2_zero,
910 i32_consts,
911
912 trap_unreachable: i32_ty
913 .const_int(TrapCode::UnreachableCodeReached as _, false)
914 .as_basic_value_enum(),
915 trap_call_indirect_null: i32_ty
916 .const_int(TrapCode::IndirectCallToNull as _, false)
917 .as_basic_value_enum(),
918 trap_call_indirect_sig: i32_ty
919 .const_int(TrapCode::BadSignature as _, false)
920 .as_basic_value_enum(),
921 trap_memory_oob: i32_ty
922 .const_int(TrapCode::HeapAccessOutOfBounds as _, false)
923 .as_basic_value_enum(),
924 trap_illegal_arithmetic: i32_ty
925 .const_int(TrapCode::IntegerOverflow as _, false)
926 .as_basic_value_enum(),
927 trap_integer_division_by_zero: i32_ty
928 .const_int(TrapCode::IntegerDivisionByZero as _, false)
929 .as_basic_value_enum(),
930 trap_bad_conversion_to_integer: i32_ty
931 .const_int(TrapCode::BadConversionToInteger as _, false)
932 .as_basic_value_enum(),
933 trap_unaligned_atomic: i32_ty
934 .const_int(TrapCode::UnalignedAtomic as _, false)
935 .as_basic_value_enum(),
936 trap_table_access_oob: i32_ty
937 .const_int(TrapCode::TableAccessOutOfBounds as _, false)
938 .as_basic_value_enum(),
939
940 experimental_stackmap: module.add_function(
944 "llvm.experimental.stackmap",
945 void_ty.fn_type(
946 &[
947 i64_ty_basic_md, i32_ty_basic_md, ],
950 true,
951 ),
952 None,
953 ),
954
955 table_copy: add_function_with_attrs(
957 "wasmer_vm_table_copy",
958 void_ty.fn_type(
959 &[
960 ctx_ptr_ty_basic_md,
961 i32_ty_basic_md,
962 i32_ty_basic_md,
963 i32_ty_basic_md,
964 i32_ty_basic_md,
965 i32_ty_basic_md,
966 ],
967 false,
968 ),
969 None,
970 ),
971 table_init: add_function_with_attrs(
972 "wasmer_vm_table_init",
973 void_ty.fn_type(
974 &[
975 ctx_ptr_ty_basic_md,
976 i32_ty_basic_md,
977 i32_ty_basic_md,
978 i32_ty_basic_md,
979 i32_ty_basic_md,
980 i32_ty_basic_md,
981 ],
982 false,
983 ),
984 None,
985 ),
986 table_fill: add_function_with_attrs(
987 "wasmer_vm_table_fill",
988 void_ty.fn_type(
989 &[
990 ctx_ptr_ty_basic_md,
991 i32_ty_basic_md,
992 i32_ty_basic_md,
993 anyref_ty_basic_md,
994 i32_ty_basic_md,
995 ],
996 false,
997 ),
998 None,
999 ),
1000 table_size: add_function_with_attrs(
1001 "wasmer_vm_table_size",
1002 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1003 None,
1004 ),
1005 imported_table_size: add_function_with_attrs(
1006 "wasmer_vm_imported_table_size",
1007 i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1008 None,
1009 ),
1010 table_get: add_function_with_attrs(
1011 "wasmer_vm_table_get",
1012 anyref_ty.fn_type(
1013 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1014 false,
1015 ),
1016 None,
1017 ),
1018 imported_table_get: add_function_with_attrs(
1019 "wasmer_vm_imported_table_get",
1020 anyref_ty.fn_type(
1021 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1022 false,
1023 ),
1024 None,
1025 ),
1026 table_set: add_function_with_attrs(
1027 "wasmer_vm_table_set",
1028 void_ty.fn_type(
1029 &[
1030 ctx_ptr_ty_basic_md,
1031 i32_ty_basic_md,
1032 i32_ty_basic_md,
1033 anyref_ty_basic_md,
1034 ],
1035 false,
1036 ),
1037 None,
1038 ),
1039 imported_table_set: add_function_with_attrs(
1040 "wasmer_vm_imported_table_set",
1041 void_ty.fn_type(
1042 &[
1043 ctx_ptr_ty_basic_md,
1044 i32_ty_basic_md,
1045 i32_ty_basic_md,
1046 anyref_ty_basic_md,
1047 ],
1048 false,
1049 ),
1050 None,
1051 ),
1052 table_grow: add_function_with_attrs(
1053 "wasmer_vm_table_grow",
1054 i32_ty.fn_type(
1055 &[
1056 ctx_ptr_ty_basic_md,
1057 anyref_ty_basic_md,
1058 i32_ty_basic_md,
1059 i32_ty_basic_md,
1060 ],
1061 false,
1062 ),
1063 None,
1064 ),
1065 imported_table_grow: add_function_with_attrs(
1066 "wasmer_vm_imported_table_grow",
1067 i32_ty.fn_type(
1068 &[
1069 ctx_ptr_ty_basic_md,
1070 anyref_ty_basic_md,
1071 i32_ty_basic_md,
1072 i32_ty_basic_md,
1073 ],
1074 false,
1075 ),
1076 None,
1077 ),
1078 memory_init: add_function_with_attrs(
1079 "wasmer_vm_memory32_init",
1080 void_ty.fn_type(
1081 &[
1082 ctx_ptr_ty_basic_md,
1083 i32_ty_basic_md,
1084 i32_ty_basic_md,
1085 i32_ty_basic_md,
1086 i32_ty_basic_md,
1087 i32_ty_basic_md,
1088 ],
1089 false,
1090 ),
1091 None,
1092 ),
1093 memory_copy: add_function_with_attrs(
1094 "wasmer_vm_memory32_copy",
1095 void_ty.fn_type(
1096 &[
1097 ctx_ptr_ty_basic_md,
1098 i32_ty_basic_md,
1099 i32_ty_basic_md,
1100 i32_ty_basic_md,
1101 i32_ty_basic_md,
1102 ],
1103 false,
1104 ),
1105 None,
1106 ),
1107 imported_memory_copy: add_function_with_attrs(
1108 "wasmer_vm_imported_memory32_copy",
1109 void_ty.fn_type(
1110 &[
1111 ctx_ptr_ty_basic_md,
1112 i32_ty_basic_md,
1113 i32_ty_basic_md,
1114 i32_ty_basic_md,
1115 i32_ty_basic_md,
1116 ],
1117 false,
1118 ),
1119 None,
1120 ),
1121 memory_fill: add_function_with_attrs(
1122 "wasmer_vm_memory32_fill",
1123 void_ty.fn_type(
1124 &[
1125 ctx_ptr_ty_basic_md,
1126 i32_ty_basic_md,
1127 i32_ty_basic_md,
1128 i32_ty_basic_md,
1129 i32_ty_basic_md,
1130 ],
1131 false,
1132 ),
1133 None,
1134 ),
1135 imported_memory_fill: add_function_with_attrs(
1136 "wasmer_vm_imported_memory32_fill",
1137 void_ty.fn_type(
1138 &[
1139 ctx_ptr_ty_basic_md,
1140 i32_ty_basic_md,
1141 i32_ty_basic_md,
1142 i32_ty_basic_md,
1143 i32_ty_basic_md,
1144 ],
1145 false,
1146 ),
1147 None,
1148 ),
1149 memory_size_ty: i32_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1150 memory_grow_ty: i32_ty.fn_type(
1151 &[ctx_ptr_ty_basic_md, i32_ty_basic_md, i32_ty_basic_md],
1152 false,
1153 ),
1154 data_drop: add_function_with_attrs(
1155 "wasmer_vm_data_drop",
1156 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1157 None,
1158 ),
1159 func_ref: add_function_with_attrs(
1160 "wasmer_vm_func_ref",
1161 funcref_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1162 None,
1163 ),
1164 elem_drop: add_function_with_attrs(
1165 "wasmer_vm_elem_drop",
1166 void_ty.fn_type(&[ctx_ptr_ty_basic_md, i32_ty_basic_md], false),
1167 None,
1168 ),
1169 throw_trap: add_function_with_attrs(
1170 "wasmer_vm_raise_trap",
1171 void_ty.fn_type(&[i32_ty_basic_md], false),
1172 None,
1173 ),
1174
1175 throw: add_function_with_attrs(
1176 "wasmer_vm_throw",
1177 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1178 None,
1179 ),
1180 alloc_exception: add_function_with_attrs(
1181 "wasmer_vm_alloc_exception",
1182 i32_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1183 None,
1184 ),
1185 read_exnref: add_function_with_attrs(
1186 "wasmer_vm_read_exnref",
1187 ptr_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1188 None,
1189 ),
1190 exception_into_exnref: add_function_with_attrs(
1191 "wasmer_vm_exception_into_exnref",
1192 i32_ty.fn_type(&[ptr_ty.into()], false),
1193 None,
1194 ),
1195 lpad_exception_ty: context.struct_type(&[ptr_ty.into(), i32_ty.into()], false),
1196
1197 debug_ptr: add_function_with_attrs(
1198 "wasmer_vm_dbg_usize",
1199 void_ty.fn_type(&[ptr_ty.into()], false),
1200 None,
1201 ),
1202 debug_str: add_function_with_attrs(
1203 "wasmer_vm_dbg_str",
1204 void_ty.fn_type(&[ptr_ty.into(), i32_ty.into()], false),
1205 None,
1206 ),
1207 memory_wait32: add_function_with_attrs(
1208 "wasmer_vm_memory32_atomic_wait32",
1209 i32_ty.fn_type(
1210 &[
1211 ctx_ptr_ty_basic_md,
1212 i32_ty_basic_md,
1213 i32_ty_basic_md,
1214 i32_ty_basic_md,
1215 i64_ty_basic_md,
1216 ],
1217 false,
1218 ),
1219 None,
1220 ),
1221 memory_wait32_ty: i32_ty.fn_type(
1222 &[
1223 ctx_ptr_ty_basic_md,
1224 i32_ty_basic_md,
1225 i32_ty_basic_md,
1226 i32_ty_basic_md,
1227 i64_ty_basic_md,
1228 ],
1229 false,
1230 ),
1231 imported_memory_wait32: add_function_with_attrs(
1232 "wasmer_vm_imported_memory32_atomic_wait32",
1233 i32_ty.fn_type(
1234 &[
1235 ctx_ptr_ty_basic_md,
1236 i32_ty_basic_md,
1237 i32_ty_basic_md,
1238 i32_ty_basic_md,
1239 i64_ty_basic_md,
1240 ],
1241 false,
1242 ),
1243 None,
1244 ),
1245 memory_wait64: add_function_with_attrs(
1246 "wasmer_vm_memory32_atomic_wait64",
1247 i32_ty.fn_type(
1248 &[
1249 ctx_ptr_ty_basic_md,
1250 i32_ty_basic_md,
1251 i32_ty_basic_md,
1252 i64_ty_basic_md,
1253 i64_ty_basic_md,
1254 ],
1255 false,
1256 ),
1257 None,
1258 ),
1259 memory_wait64_ty: i32_ty.fn_type(
1260 &[
1261 ctx_ptr_ty_basic_md,
1262 i32_ty_basic_md,
1263 i32_ty_basic_md,
1264 i64_ty_basic_md,
1265 i64_ty_basic_md,
1266 ],
1267 false,
1268 ),
1269 imported_memory_wait64: add_function_with_attrs(
1270 "wasmer_vm_imported_memory32_atomic_wait64",
1271 i32_ty.fn_type(
1272 &[
1273 ctx_ptr_ty_basic_md,
1274 i32_ty_basic_md,
1275 i32_ty_basic_md,
1276 i64_ty_basic_md,
1277 i64_ty_basic_md,
1278 ],
1279 false,
1280 ),
1281 None,
1282 ),
1283 memory_notify: add_function_with_attrs(
1284 "wasmer_vm_memory32_atomic_notify",
1285 i32_ty.fn_type(
1286 &[
1287 ctx_ptr_ty_basic_md,
1288 i32_ty_basic_md,
1289 i32_ty_basic_md,
1290 i32_ty_basic_md,
1291 ],
1292 false,
1293 ),
1294 None,
1295 ),
1296 memory_notify_ty: i32_ty.fn_type(
1297 &[
1298 ctx_ptr_ty_basic_md,
1299 i32_ty_basic_md,
1300 i32_ty_basic_md,
1301 i32_ty_basic_md,
1302 ],
1303 false,
1304 ),
1305 imported_memory_notify: add_function_with_attrs(
1306 "wasmer_vm_imported_memory32_atomic_notify",
1307 i32_ty.fn_type(
1308 &[
1309 ctx_ptr_ty_basic_md,
1310 i32_ty_basic_md,
1311 i32_ty_basic_md,
1312 i32_ty_basic_md,
1313 ],
1314 false,
1315 ),
1316 None,
1317 ),
1318
1319 vmfunction_import_ty: context.struct_type(
1320 &[
1321 i8_ptr_ty_basic,
1322 i8_ptr_ty_basic,
1323 i8_ptr_ty_basic,
1324 i1_ty_basic,
1325 ],
1326 false,
1327 ),
1328 vmfunction_import_body_element: 0,
1329 vmfunction_import_vmctx_element: 1,
1330 vmfunction_import_include_m0_param_element: 3,
1331 vmmemory_definition_ty: context.struct_type(&[i8_ptr_ty_basic, isize_ty.into()], false),
1332 vmmemory_definition_base_element: 0,
1333 vmmemory_definition_current_length_element: 1,
1334
1335 ptr_ty,
1336
1337 x86_64: X86_64Intrinsics {
1338 pshufb128: add_function_with_attrs(
1339 "llvm.x86.ssse3.pshuf.b.128",
1340 ret_i8x16_take_i8x16_i8x16,
1341 None,
1342 ),
1343 pmaddubsw128: add_function_with_attrs(
1344 "llvm.x86.ssse3.pmadd.ub.sw.128",
1345 i16x8_ty.fn_type(&[i8x16_ty_basic_md, i8x16_ty_basic_md], false),
1346 None,
1347 ),
1348 pmaddwd128: add_function_with_attrs(
1349 "llvm.x86.sse2.pmadd.wd",
1350 i32x4_ty.fn_type(&[i16x8_ty_basic_md, i16x8_ty_basic_md], false),
1351 None,
1352 ),
1353 pmulhrsw128: add_function_with_attrs(
1354 "llvm.x86.ssse3.pmul.hr.sw.128",
1355 ret_i16x8_take_i16x8_i16x8,
1356 None,
1357 ),
1358 pblendvb: add_function_with_attrs(
1359 "llvm.x86.sse41.pblendvb",
1360 ret_i8x16_take_i8x16_i8x16_i8x16,
1361 None,
1362 ),
1363 min_ps: add_function_with_attrs(
1364 "llvm.x86.sse.min.ps",
1365 ret_f32x4_take_f32x4_f32x4,
1366 None,
1367 ),
1368 min_pd: add_function_with_attrs(
1369 "llvm.x86.sse2.min.pd",
1370 ret_f64x2_take_f64x2_f64x2,
1371 None,
1372 ),
1373 max_ps: add_function_with_attrs(
1374 "llvm.x86.sse.max.ps",
1375 ret_f32x4_take_f32x4_f32x4,
1376 None,
1377 ),
1378 max_pd: add_function_with_attrs(
1379 "llvm.x86.sse2.max.pd",
1380 ret_f64x2_take_f64x2_f64x2,
1381 None,
1382 ),
1383 cvttps2dq: add_function_with_attrs(
1384 "llvm.x86.sse2.cvttps2dq",
1385 ret_i32x4_take_f32x4,
1386 None,
1387 ),
1388 cvtps2udq128: add_function_with_attrs(
1389 "llvm.x86.avx512.mask.cvtps2udq.128",
1390 ret_i32x4_take_f32x4_i32x4_i8,
1391 None,
1392 ),
1393 cvtpd2dq: add_function_with_attrs(
1394 "llvm.x86.sse2.cvtpd2dq",
1395 ret_i32x4_take_f64x2,
1396 None,
1397 ),
1398 cvtpd2udq128: add_function_with_attrs(
1399 "llvm.x86.avx512.mask.cvtpd2udq.128",
1400 ret_i32x4_take_f64x2_i32x4_i8,
1401 None,
1402 ),
1403 },
1404 };
1405
1406 let noreturn =
1407 context.create_enum_attribute(Attribute::get_named_enum_kind_id("noreturn"), 0);
1408 intrinsics
1409 .throw_trap
1410 .add_attribute(AttributeLoc::Function, noreturn);
1411 intrinsics
1412 }
1413}
1414
1415#[derive(Clone, Copy)]
1416pub enum MemoryCache<'ctx> {
1417 Dynamic {
1419 ptr_to_base_ptr: PointerValue<'ctx>,
1420 ptr_to_current_length: PointerValue<'ctx>,
1421 },
1422 Static { base_ptr: PointerValue<'ctx> },
1424}
1425
1426#[derive(Clone)]
1427struct TableCache<'ctx> {
1428 ptr_to_base_ptr: PointerValue<'ctx>,
1429 ptr_to_bounds: PointerValue<'ctx>,
1430}
1431
1432#[derive(Clone, Copy)]
1433pub enum GlobalCache<'ctx> {
1434 Mut {
1435 ptr_to_value: PointerValue<'ctx>,
1436 value_type: BasicTypeEnum<'ctx>,
1437 },
1438 Const {
1439 value: BasicValueEnum<'ctx>,
1440 },
1441}
1442
1443#[derive(Clone)]
1444pub struct FunctionCache<'ctx> {
1445 pub func: PointerValue<'ctx>,
1446 pub llvm_func_type: FunctionType<'ctx>,
1447 pub vmctx: BasicValueEnum<'ctx>,
1448 pub imported_include_m0_param: Option<IntValue<'ctx>>,
1449 pub attrs: Vec<(Attribute, AttributeLoc)>,
1450}
1451
1452pub struct CtxType<'ctx, 'a> {
1453 ctx_ptr_value: PointerValue<'ctx>,
1454 m0: Option<PointerValue<'ctx>>,
1455
1456 wasm_module: &'a WasmerCompilerModule,
1457 cache_builder: &'a Builder<'ctx>,
1458 abi: &'a dyn Abi,
1459
1460 cached_memories: HashMap<MemoryIndex, MemoryCache<'ctx>>,
1461 cached_tables: HashMap<TableIndex, TableCache<'ctx>>,
1462 cached_globals: HashMap<GlobalIndex, GlobalCache<'ctx>>,
1463 cached_functions: HashMap<FunctionIndex, FunctionCache<'ctx>>,
1464 cached_memory_op: HashMap<(MemoryIndex, MemoryOp), PointerValue<'ctx>>,
1465
1466 offsets: VMOffsets,
1467}
1468
1469impl<'ctx, 'a> CtxType<'ctx, 'a> {
1470 pub fn new(
1471 wasm_module: &'a WasmerCompilerModule,
1472 func_value: &FunctionValue<'ctx>,
1473 cache_builder: &'a Builder<'ctx>,
1474 abi: &'a dyn Abi,
1475 pointer_width: u8,
1476 m0: Option<PointerValue<'ctx>>,
1477 ) -> CtxType<'ctx, 'a> {
1478 CtxType {
1479 m0,
1480 ctx_ptr_value: abi.get_vmctx_ptr_param(func_value),
1481
1482 wasm_module,
1483 cache_builder,
1484 abi,
1485
1486 cached_memories: HashMap::new(),
1487 cached_tables: HashMap::new(),
1488 cached_globals: HashMap::new(),
1489 cached_functions: HashMap::new(),
1490 cached_memory_op: HashMap::new(),
1491
1492 offsets: VMOffsets::new(pointer_width, wasm_module),
1493 }
1494 }
1495
1496 pub fn basic(&self) -> BasicValueEnum<'ctx> {
1497 self.ctx_ptr_value.as_basic_value_enum()
1498 }
1499
1500 pub fn memory(
1501 &mut self,
1502 index: MemoryIndex,
1503 intrinsics: &Intrinsics<'ctx>,
1504 module: &Module<'ctx>,
1505 memory_styles: &PrimaryMap<MemoryIndex, MemoryStyle>,
1506 ) -> Result<MemoryCache<'ctx>, CompileError> {
1507 let (cached_memories, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1508 &mut self.cached_memories,
1509 self.wasm_module,
1510 self.ctx_ptr_value,
1511 &self.cache_builder,
1512 &self.offsets,
1513 );
1514 let memory_style = &memory_styles[index];
1515 match cached_memories.get(&index) {
1516 Some(r) => Ok(*r),
1517 None => {
1518 let memory_definition_ptr =
1519 if let Some(local_memory_index) = wasm_module.local_memory_index(index) {
1520 let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
1521 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1522 unsafe {
1523 err!(cache_builder.build_gep(
1524 intrinsics.i8_ty,
1525 ctx_ptr_value,
1526 &[offset],
1527 ""
1528 ))
1529 }
1530 } else {
1531 let offset = offsets.vmctx_vmmemory_import(index);
1532 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1533 let memory_definition_ptr_ptr = unsafe {
1534 err!(cache_builder.build_gep(
1535 intrinsics.i8_ty,
1536 ctx_ptr_value,
1537 &[offset],
1538 ""
1539 ))
1540 };
1541 let memory_definition_ptr_ptr = err!(cache_builder.build_bit_cast(
1542 memory_definition_ptr_ptr,
1543 intrinsics.ptr_ty,
1544 "",
1545 ))
1546 .into_pointer_value();
1547 let memory_definition_ptr = err!(cache_builder.build_load(
1548 intrinsics.ptr_ty,
1549 memory_definition_ptr_ptr,
1550 ""
1551 ))
1552 .into_pointer_value();
1553 tbaa_label(
1554 module,
1555 intrinsics,
1556 format!("memory {} definition", index.as_u32()),
1557 memory_definition_ptr.as_instruction_value().unwrap(),
1558 );
1559 memory_definition_ptr
1560 };
1561 let memory_definition_ptr = err!(cache_builder.build_bit_cast(
1562 memory_definition_ptr,
1563 intrinsics.ptr_ty,
1564 "",
1565 ))
1566 .into_pointer_value();
1567 let base_ptr = err!(cache_builder.build_struct_gep(
1568 intrinsics.vmmemory_definition_ty,
1569 memory_definition_ptr,
1570 intrinsics.vmmemory_definition_base_element,
1571 "",
1572 ));
1573 let value = if let MemoryStyle::Dynamic { .. } = memory_style {
1574 let current_length_ptr = err!(cache_builder.build_struct_gep(
1575 intrinsics.vmmemory_definition_ty,
1576 memory_definition_ptr,
1577 intrinsics.vmmemory_definition_current_length_element,
1578 "",
1579 ));
1580 MemoryCache::Dynamic {
1581 ptr_to_base_ptr: base_ptr,
1582 ptr_to_current_length: current_length_ptr,
1583 }
1584 } else {
1585 let base_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, base_ptr, ""))
1586 .into_pointer_value();
1587 tbaa_label(
1588 module,
1589 intrinsics,
1590 format!("memory base_ptr {}", index.as_u32()),
1591 base_ptr.as_instruction_value().unwrap(),
1592 );
1593 MemoryCache::Static { base_ptr }
1594 };
1595
1596 self.cached_memories.insert(index, value);
1597 Ok(*self.cached_memories.get(&index).unwrap())
1598 }
1599 }
1600 }
1601
1602 fn build_table_prepare(
1603 table_index: TableIndex,
1604 intrinsics: &Intrinsics<'ctx>,
1605 module: &Module<'ctx>,
1606 wasm_module: &WasmerCompilerModule,
1607 ctx_ptr_value: PointerValue<'ctx>,
1608 offsets: &VMOffsets,
1609 builder: &Builder<'ctx>,
1610 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1611 if let Some(local_table_index) = wasm_module.local_table_index(table_index) {
1612 let offset = intrinsics.i64_ty.const_int(
1613 offsets
1614 .vmctx_vmtable_definition_base(local_table_index)
1615 .into(),
1616 false,
1617 );
1618 let ptr_to_base_ptr =
1619 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1620 let ptr_to_base_ptr =
1621 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1622 .into_pointer_value();
1623 let offset = intrinsics.i64_ty.const_int(
1624 offsets
1625 .vmctx_vmtable_definition_current_elements(local_table_index)
1626 .into(),
1627 false,
1628 );
1629 let ptr_to_bounds =
1630 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1631 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1632 .into_pointer_value();
1633 Ok((ptr_to_base_ptr, ptr_to_bounds))
1634 } else {
1635 let offset = intrinsics.i64_ty.const_int(
1636 offsets.vmctx_vmtable_import_definition(table_index).into(),
1637 false,
1638 );
1639 let definition_ptr_ptr =
1640 unsafe { err!(builder.build_gep(intrinsics.i8_ty, ctx_ptr_value, &[offset], "")) };
1641 let definition_ptr_ptr =
1642 err!(builder.build_bit_cast(definition_ptr_ptr, intrinsics.ptr_ty, "",))
1643 .into_pointer_value();
1644 let definition_ptr =
1645 err!(builder.build_load(intrinsics.ptr_ty, definition_ptr_ptr, ""))
1646 .into_pointer_value();
1647 tbaa_label(
1648 module,
1649 intrinsics,
1650 format!("table {} definition", table_index.as_u32()),
1651 definition_ptr.as_instruction_value().unwrap(),
1652 );
1653
1654 let offset = intrinsics
1655 .i64_ty
1656 .const_int(offsets.vmtable_definition_base().into(), false);
1657 let ptr_to_base_ptr =
1658 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1659 let ptr_to_base_ptr =
1660 err!(builder.build_bit_cast(ptr_to_base_ptr, intrinsics.ptr_ty, ""))
1661 .into_pointer_value();
1662 let offset = intrinsics
1663 .i64_ty
1664 .const_int(offsets.vmtable_definition_current_elements().into(), false);
1665 let ptr_to_bounds =
1666 unsafe { err!(builder.build_gep(intrinsics.i8_ty, definition_ptr, &[offset], "")) };
1667 let ptr_to_bounds = err!(builder.build_bit_cast(ptr_to_bounds, intrinsics.ptr_ty, ""))
1668 .into_pointer_value();
1669 Ok((ptr_to_base_ptr, ptr_to_bounds))
1670 }
1671 }
1672
1673 fn table_prepare(
1674 &mut self,
1675 table_index: TableIndex,
1676 intrinsics: &Intrinsics<'ctx>,
1677 module: &Module<'ctx>,
1678 body_builder: &Builder<'ctx>,
1679 ) -> Result<(PointerValue<'ctx>, PointerValue<'ctx>), CompileError> {
1680 let (cached_tables, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1681 &mut self.cached_tables,
1682 self.wasm_module,
1683 self.ctx_ptr_value,
1684 &self.cache_builder,
1685 &self.offsets,
1686 );
1687
1688 let is_growable = is_table_growable(wasm_module, table_index).ok_or_else(|| {
1689 CompileError::Codegen(format!(
1690 "Table index out of bounds: {}",
1691 table_index.as_u32()
1692 ))
1693 })?;
1694
1695 if is_growable {
1698 Self::build_table_prepare(
1699 table_index,
1700 intrinsics,
1701 module,
1702 wasm_module,
1703 ctx_ptr_value,
1704 offsets,
1705 body_builder,
1706 )
1707 } else {
1708 let TableCache {
1709 ptr_to_base_ptr,
1710 ptr_to_bounds,
1711 } = match cached_tables.entry(table_index) {
1712 Entry::Occupied(entry) => entry.get().clone(),
1713 Entry::Vacant(entry) => {
1714 let (ptr_to_base_ptr, ptr_to_bounds) = Self::build_table_prepare(
1715 table_index,
1716 intrinsics,
1717 module,
1718 wasm_module,
1719 ctx_ptr_value,
1720 offsets,
1721 cache_builder,
1722 )?;
1723
1724 let v = TableCache {
1725 ptr_to_base_ptr,
1726 ptr_to_bounds,
1727 };
1728
1729 entry.insert(v.clone());
1730
1731 v
1732 }
1733 };
1734
1735 Ok((ptr_to_base_ptr, ptr_to_bounds))
1736 }
1737 }
1738
1739 pub fn table(
1740 &mut self,
1741 index: TableIndex,
1742 intrinsics: &Intrinsics<'ctx>,
1743 module: &Module<'ctx>,
1744 body_builder: &Builder<'ctx>,
1745 ) -> Result<(PointerValue<'ctx>, IntValue<'ctx>), CompileError> {
1746 let (ptr_to_base_ptr, ptr_to_bounds) =
1747 self.table_prepare(index, intrinsics, module, body_builder)?;
1748
1749 let builder = if is_table_growable(self.wasm_module, index).unwrap() {
1751 &body_builder
1752 } else {
1753 &self.cache_builder
1754 };
1755
1756 let base_ptr = err!(builder.build_load(intrinsics.ptr_ty, ptr_to_base_ptr, "base_ptr"))
1757 .into_pointer_value();
1758 let bounds =
1759 err!(builder.build_load(intrinsics.isize_ty, ptr_to_bounds, "bounds")).into_int_value();
1760 tbaa_label(
1761 module,
1762 intrinsics,
1763 format!("table_base_ptr {}", index.index()),
1764 base_ptr.as_instruction_value().unwrap(),
1765 );
1766 tbaa_label(
1767 module,
1768 intrinsics,
1769 format!("table_bounds {}", index.index()),
1770 bounds.as_instruction_value().unwrap(),
1771 );
1772 Ok((base_ptr, bounds))
1773 }
1774
1775 pub fn fixed_funcref_table_anyfuncs(
1777 &self,
1778 index: LocalTableIndex,
1779 intrinsics: &Intrinsics<'ctx>,
1780 builder: &Builder<'ctx>,
1781 ) -> Result<PointerValue<'ctx>, CompileError> {
1782 let offset = intrinsics.i64_ty.const_int(
1783 self.offsets
1784 .vmctx_fixed_funcref_table_anyfuncs(index)
1785 .expect("fixed funcref table must have inline VMContext storage")
1786 .into(),
1787 false,
1788 );
1789 let ptr =
1790 unsafe { err!(builder.build_gep(intrinsics.i8_ty, self.ctx_ptr_value, &[offset], "")) };
1791 Ok(err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value())
1792 }
1793
1794 pub fn global(
1795 &mut self,
1796 index: GlobalIndex,
1797 intrinsics: &Intrinsics<'ctx>,
1798 module: &Module<'ctx>,
1799 ) -> Result<&GlobalCache<'ctx>, CompileError> {
1800 let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1801 &mut self.cached_globals,
1802 self.wasm_module,
1803 self.ctx_ptr_value,
1804 &self.cache_builder,
1805 &self.offsets,
1806 );
1807 match cached_globals.entry(index) {
1808 Entry::Occupied(entry) => Ok(entry.into_mut()),
1809 Entry::Vacant(entry) => {
1810 let global_type = wasm_module.globals[index];
1811 let global_value_type = global_type.ty;
1812
1813 let global_mutability = global_type.mutability;
1814 let global_ptr = if let Some(local_global_index) =
1815 wasm_module.local_global_index(index)
1816 {
1817 let offset = offsets.vmctx_vmglobal_definition(local_global_index);
1818 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1819 unsafe {
1820 err!(cache_builder.build_gep(
1821 intrinsics.i8_ty,
1822 ctx_ptr_value,
1823 &[offset],
1824 ""
1825 ))
1826 }
1827 } else {
1828 let offset = offsets.vmctx_vmglobal_import(index);
1829 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1830 let global_ptr_ptr = unsafe {
1831 err!(cache_builder.build_gep(
1832 intrinsics.i8_ty,
1833 ctx_ptr_value,
1834 &[offset],
1835 ""
1836 ))
1837 };
1838 let global_ptr_ptr =
1839 err!(cache_builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, ""))
1840 .into_pointer_value();
1841 err!(cache_builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, ""))
1842 .into_pointer_value()
1843 };
1844 let global_ptr =
1845 err!(cache_builder.build_bit_cast(global_ptr, intrinsics.ptr_ty, "",))
1846 .into_pointer_value();
1847
1848 let ret = entry.insert(match global_mutability {
1849 Mutability::Const => {
1850 let value = err!(cache_builder.build_load(
1851 type_to_llvm(intrinsics, global_value_type)?,
1852 global_ptr,
1853 "",
1854 ));
1855 tbaa_label(
1856 module,
1857 intrinsics,
1858 format!("global {}", index.as_u32()),
1859 value.as_instruction_value().unwrap(),
1860 );
1861 GlobalCache::Const { value }
1862 }
1863 Mutability::Var => GlobalCache::Mut {
1864 ptr_to_value: global_ptr,
1865 value_type: type_to_llvm(intrinsics, global_value_type)?,
1866 },
1867 });
1868
1869 Ok(ret)
1870 }
1871 }
1872 }
1873
1874 pub fn add_func(
1875 &mut self,
1876 function_index: FunctionIndex,
1877 func: PointerValue<'ctx>,
1878 llvm_func_type: FunctionType<'ctx>,
1879 vmctx: BasicValueEnum<'ctx>,
1880 attrs: &[(Attribute, AttributeLoc)],
1881 ) {
1882 match self.cached_functions.entry(function_index) {
1883 Entry::Occupied(_) => unreachable!("duplicate function"),
1884 Entry::Vacant(entry) => {
1885 entry.insert(FunctionCache {
1886 func,
1887 llvm_func_type,
1888 vmctx,
1889 imported_include_m0_param: None,
1890 attrs: attrs.to_vec(),
1891 });
1892 }
1893 }
1894 }
1895
1896 #[allow(clippy::too_many_arguments)]
1897 pub fn local_func(
1898 &mut self,
1899 _local_function_index: LocalFunctionIndex,
1900 function_index: FunctionIndex,
1901 intrinsics: &Intrinsics<'ctx>,
1902 module: &Module<'ctx>,
1903 context: &'ctx Context,
1904 func_type: &FuncType,
1905 function_name: &str,
1906 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1907 let (cached_functions, ctx_ptr_value, offsets) = (
1908 &mut self.cached_functions,
1909 &self.ctx_ptr_value,
1910 &self.offsets,
1911 );
1912 Ok(match cached_functions.entry(function_index) {
1913 Entry::Occupied(entry) => entry.into_mut(),
1914 Entry::Vacant(entry) => {
1915 debug_assert!(module.get_function(function_name).is_none());
1916 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1917 context,
1918 intrinsics,
1919 Some(offsets),
1920 func_type,
1921 self.m0.is_some(),
1922 )?;
1923 let func =
1924 module.add_function(function_name, llvm_func_type, Some(Linkage::External));
1925 for (attr, attr_loc) in &llvm_func_attrs {
1926 func.add_attribute(*attr_loc, *attr);
1927 }
1928 entry.insert(FunctionCache {
1929 func: func.as_global_value().as_pointer_value(),
1930 llvm_func_type,
1931 vmctx: ctx_ptr_value.as_basic_value_enum(),
1932 imported_include_m0_param: None,
1933 attrs: llvm_func_attrs,
1934 })
1935 }
1936 })
1937 }
1938
1939 pub fn imported_func(
1940 &mut self,
1941 function_index: FunctionIndex,
1942 intrinsics: &Intrinsics<'ctx>,
1943 context: &'ctx Context,
1944 func_type: &FuncType,
1945 ) -> Result<&FunctionCache<'ctx>, CompileError> {
1946 let (cached_functions, wasm_module, ctx_ptr_value, cache_builder, offsets) = (
1947 &mut self.cached_functions,
1948 self.wasm_module,
1949 &self.ctx_ptr_value,
1950 &self.cache_builder,
1951 &self.offsets,
1952 );
1953 match cached_functions.entry(function_index) {
1954 Entry::Occupied(entry) => Ok(entry.into_mut()),
1955 Entry::Vacant(entry) => {
1956 let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm(
1957 context,
1958 intrinsics,
1959 Some(offsets),
1960 func_type,
1961 self.m0.is_some(),
1962 )?;
1963 debug_assert!(wasm_module.local_func_index(function_index).is_none());
1964 let offset = offsets.vmctx_vmfunction_import(function_index);
1965 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
1966 let vmfunction_import_ptr = unsafe {
1967 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
1968 };
1969 let vmfunction_import_ptr = err!(cache_builder.build_bit_cast(
1970 vmfunction_import_ptr,
1971 intrinsics.ptr_ty,
1972 "",
1973 ))
1974 .into_pointer_value();
1975 vmfunction_import_ptr.set_name("vmfunction_import_ptr");
1976
1977 let body_ptr_ptr = err!(cache_builder.build_struct_gep(
1978 intrinsics.vmfunction_import_ty,
1979 vmfunction_import_ptr,
1980 intrinsics.vmfunction_import_body_element,
1981 "",
1982 ));
1983 body_ptr_ptr.set_name("body_ptr_ptr");
1984 let body_ptr = err!(cache_builder.build_load(intrinsics.ptr_ty, body_ptr_ptr, ""));
1985 let body_ptr = err!(cache_builder.build_bit_cast(body_ptr, intrinsics.ptr_ty, ""))
1986 .into_pointer_value();
1987 body_ptr.set_name("body_ptr");
1988 let vmctx_ptr_ptr = err!(cache_builder.build_struct_gep(
1989 intrinsics.vmfunction_import_ty,
1990 vmfunction_import_ptr,
1991 intrinsics.vmfunction_import_vmctx_element,
1992 "",
1993 ));
1994 vmctx_ptr_ptr.set_name("vmctx_ptr_ptr");
1995 let vmctx_ptr =
1996 err!(cache_builder.build_load(intrinsics.ptr_ty, vmctx_ptr_ptr, ""));
1997 vmctx_ptr.set_name("vmctx_ptr");
1998 let include_m0_param_ptr = err!(cache_builder.build_struct_gep(
1999 intrinsics.vmfunction_import_ty,
2000 vmfunction_import_ptr,
2001 intrinsics.vmfunction_import_include_m0_param_element,
2002 "",
2003 ));
2004 include_m0_param_ptr.set_name("include_m0_param_ptr");
2005 let imported_include_m0_param =
2006 err!(cache_builder.build_load(intrinsics.i1_ty, include_m0_param_ptr, "",))
2007 .into_int_value();
2008 imported_include_m0_param.set_name("imported_include_m0_param");
2009
2010 Ok(entry.insert(FunctionCache {
2011 func: body_ptr,
2012 llvm_func_type,
2013 vmctx: vmctx_ptr,
2014 imported_include_m0_param: Some(imported_include_m0_param),
2015 attrs: llvm_func_attrs,
2016 }))
2017 }
2018 }
2019 }
2020
2021 pub fn memory_grow(
2022 &mut self,
2023 memory_index: MemoryIndex,
2024 intrinsics: &Intrinsics<'ctx>,
2025 ) -> Result<PointerValue<'ctx>, CompileError> {
2026 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2027 &mut self.cached_memory_op,
2028 &self.wasm_module,
2029 &self.offsets,
2030 &self.cache_builder,
2031 &self.ctx_ptr_value,
2032 );
2033 match cached_memory_op.entry((memory_index, MemoryOp::Grow)) {
2034 Entry::Occupied(entry) => Ok(*entry.get()),
2035 Entry::Vacant(entry) => {
2036 let (grow_fn, grow_fn_ty) =
2037 if wasm_module.local_memory_index(memory_index).is_some() {
2038 (
2039 VMBuiltinFunctionIndex::get_memory32_grow_index(),
2040 intrinsics.ptr_ty,
2041 )
2042 } else {
2043 (
2044 VMBuiltinFunctionIndex::get_imported_memory32_grow_index(),
2045 intrinsics.ptr_ty,
2046 )
2047 };
2048 let offset = offsets.vmctx_builtin_function(grow_fn);
2049 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2050 let grow_fn_ptr_ptr = unsafe {
2051 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2052 };
2053
2054 let grow_fn_ptr_ptr =
2055 err!(cache_builder.build_bit_cast(grow_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2056 .into_pointer_value();
2057 let val = err!(cache_builder.build_load(grow_fn_ty, grow_fn_ptr_ptr, ""))
2058 .into_pointer_value();
2059
2060 entry.insert(val);
2061 Ok(val)
2062 }
2063 }
2064 }
2065
2066 pub fn memory_size(
2067 &mut self,
2068 memory_index: MemoryIndex,
2069 intrinsics: &Intrinsics<'ctx>,
2070 ) -> Result<PointerValue<'ctx>, CompileError> {
2071 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2072 &mut self.cached_memory_op,
2073 &self.wasm_module,
2074 &self.offsets,
2075 &self.cache_builder,
2076 &self.ctx_ptr_value,
2077 );
2078
2079 match cached_memory_op.entry((memory_index, MemoryOp::Size)) {
2080 Entry::Occupied(entry) => Ok(*entry.get()),
2081 Entry::Vacant(entry) => {
2082 let (size_fn, size_fn_ty) =
2083 if wasm_module.local_memory_index(memory_index).is_some() {
2084 (
2085 VMBuiltinFunctionIndex::get_memory32_size_index(),
2086 intrinsics.ptr_ty,
2087 )
2088 } else {
2089 (
2090 VMBuiltinFunctionIndex::get_imported_memory32_size_index(),
2091 intrinsics.ptr_ty,
2092 )
2093 };
2094 let offset = offsets.vmctx_builtin_function(size_fn);
2095 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2096 let size_fn_ptr_ptr = unsafe {
2097 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2098 };
2099
2100 let size_fn_ptr_ptr =
2101 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2102 .into_pointer_value();
2103
2104 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2105 .into_pointer_value();
2106 entry.insert(val);
2107 Ok(val)
2108 }
2109 }
2110 }
2111
2112 pub fn memory_wait32(
2113 &mut self,
2114 memory_index: MemoryIndex,
2115 intrinsics: &Intrinsics<'ctx>,
2116 ) -> Result<PointerValue<'ctx>, CompileError> {
2117 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2118 &mut self.cached_memory_op,
2119 &self.wasm_module,
2120 &self.offsets,
2121 &self.cache_builder,
2122 &self.ctx_ptr_value,
2123 );
2124 match cached_memory_op.entry((memory_index, MemoryOp::Wait32)) {
2125 Entry::Occupied(entry) => Ok(*entry.get()),
2126 Entry::Vacant(entry) => {
2127 let (size_fn, size_fn_ty) =
2128 if wasm_module.local_memory_index(memory_index).is_some() {
2129 (
2130 VMBuiltinFunctionIndex::get_memory_atomic_wait32_index(),
2131 intrinsics.ptr_ty,
2132 )
2133 } else {
2134 (
2135 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait32_index(),
2136 intrinsics.ptr_ty,
2137 )
2138 };
2139 let offset = offsets.vmctx_builtin_function(size_fn);
2140 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2141 let size_fn_ptr_ptr = unsafe {
2142 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2143 };
2144
2145 let size_fn_ptr_ptr =
2146 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2147 .into_pointer_value();
2148
2149 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2150 .into_pointer_value();
2151
2152 entry.insert(val);
2153 Ok(val)
2154 }
2155 }
2156 }
2157
2158 pub fn memory_wait64(
2159 &mut self,
2160 memory_index: MemoryIndex,
2161 intrinsics: &Intrinsics<'ctx>,
2162 ) -> Result<PointerValue<'ctx>, CompileError> {
2163 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2164 &mut self.cached_memory_op,
2165 &self.wasm_module,
2166 &self.offsets,
2167 &self.cache_builder,
2168 &self.ctx_ptr_value,
2169 );
2170
2171 match cached_memory_op.entry((memory_index, MemoryOp::Wait64)) {
2172 Entry::Occupied(entry) => Ok(*entry.get()),
2173 Entry::Vacant(entry) => {
2174 let (size_fn, size_fn_ty) =
2175 if wasm_module.local_memory_index(memory_index).is_some() {
2176 (
2177 VMBuiltinFunctionIndex::get_memory_atomic_wait64_index(),
2178 intrinsics.ptr_ty,
2179 )
2180 } else {
2181 (
2182 VMBuiltinFunctionIndex::get_imported_memory_atomic_wait64_index(),
2183 intrinsics.ptr_ty,
2184 )
2185 };
2186 let offset = offsets.vmctx_builtin_function(size_fn);
2187 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2188 let size_fn_ptr_ptr = unsafe {
2189 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2190 };
2191
2192 let size_fn_ptr_ptr =
2193 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2194 .into_pointer_value();
2195
2196 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2197 .into_pointer_value();
2198 entry.insert(val);
2199 Ok(val)
2200 }
2201 }
2202 }
2203
2204 pub fn memory_notify(
2205 &mut self,
2206 memory_index: MemoryIndex,
2207 intrinsics: &Intrinsics<'ctx>,
2208 ) -> Result<PointerValue<'ctx>, CompileError> {
2209 let (cached_memory_op, wasm_module, offsets, cache_builder, ctx_ptr_value) = (
2210 &mut self.cached_memory_op,
2211 &self.wasm_module,
2212 &self.offsets,
2213 &self.cache_builder,
2214 &self.ctx_ptr_value,
2215 );
2216 match cached_memory_op.entry((memory_index, MemoryOp::Notify)) {
2217 Entry::Occupied(entry) => Ok(*entry.get()),
2218 Entry::Vacant(entry) => {
2219 let (size_fn, size_fn_ty) =
2220 if wasm_module.local_memory_index(memory_index).is_some() {
2221 (
2222 VMBuiltinFunctionIndex::get_memory_atomic_notify_index(),
2223 intrinsics.ptr_ty,
2224 )
2225 } else {
2226 (
2227 VMBuiltinFunctionIndex::get_imported_memory_atomic_notify_index(),
2228 intrinsics.ptr_ty,
2229 )
2230 };
2231 let offset = offsets.vmctx_builtin_function(size_fn);
2232 let offset = intrinsics.i32_ty.const_int(offset.into(), false);
2233 let size_fn_ptr_ptr = unsafe {
2234 err!(cache_builder.build_gep(intrinsics.i8_ty, *ctx_ptr_value, &[offset], ""))
2235 };
2236
2237 let size_fn_ptr_ptr =
2238 err!(cache_builder.build_bit_cast(size_fn_ptr_ptr, intrinsics.ptr_ty, ""))
2239 .into_pointer_value();
2240
2241 let val = err!(cache_builder.build_load(size_fn_ty, size_fn_ptr_ptr, ""))
2242 .into_pointer_value();
2243
2244 entry.insert(val);
2245 Ok(val)
2246 }
2247 }
2248 }
2249
2250 pub fn get_offsets(&self) -> &VMOffsets {
2251 &self.offsets
2252 }
2253}
2254
2255pub fn tbaa_label<'ctx>(
2258 module: &Module<'ctx>,
2259 intrinsics: &Intrinsics<'ctx>,
2260 label: String,
2261 instruction: InstructionValue<'ctx>,
2262) {
2263 let context = module.get_context();
2277
2278 let tbaa_root = module
2280 .get_global_metadata("wasmer_tbaa_root")
2281 .pop()
2282 .unwrap_or_else(|| {
2283 module
2284 .add_global_metadata("wasmer_tbaa_root", &context.metadata_node(&[]))
2285 .unwrap();
2286 module.get_global_metadata("wasmer_tbaa_root")[0]
2287 });
2288
2289 let type_label = context.metadata_string(label.as_str());
2292 let type_tbaa = module
2293 .get_global_metadata(label.as_str())
2294 .pop()
2295 .unwrap_or_else(|| {
2296 module
2297 .add_global_metadata(
2298 label.as_str(),
2299 &context.metadata_node(&[type_label.into(), tbaa_root.into()]),
2300 )
2301 .unwrap();
2302 module.get_global_metadata(label.as_str())[0]
2303 });
2304
2305 let label = label + "_memop";
2312 let type_tbaa = module
2313 .get_global_metadata(label.as_str())
2314 .pop()
2315 .unwrap_or_else(|| {
2316 module
2317 .add_global_metadata(
2318 label.as_str(),
2319 &context.metadata_node(&[
2320 type_tbaa.into(),
2321 type_tbaa.into(),
2322 intrinsics.i64_zero.into(),
2323 ]),
2324 )
2325 .unwrap();
2326 module.get_global_metadata(label.as_str())[0]
2327 });
2328
2329 let tbaa_kind = context.get_kind_id("tbaa");
2331 instruction.set_metadata(type_tbaa, tbaa_kind).unwrap();
2332}
2333
2334fn is_table_growable(module: &WasmerCompilerModule, index: TableIndex) -> Option<bool> {
2335 let table = module.tables.get(index)?;
2336 match table.maximum {
2337 None => Some(true),
2338 Some(max) => Some(max > table.minimum),
2339 }
2340}