1use crate::config::LLVM;
2use crate::translator::FuncTrampoline;
3use crate::translator::FuncTranslator;
4use inkwell::DLLStorageClass;
5use inkwell::context::Context;
6use inkwell::memory_buffer::MemoryBuffer;
7use inkwell::module::{Linkage, Module};
8use inkwell::targets::FileType;
9use rayon::ThreadPoolBuilder;
10use rayon::iter::ParallelBridge;
11use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
12use std::{
13 borrow::Cow,
14 collections::{HashMap, HashSet},
15 sync::Arc,
16};
17use wasmer_compiler::misc::CompiledKind;
18use wasmer_compiler::progress::ProgressContext;
19use wasmer_compiler::types::function::{Compilation, UnwindInfo};
20use wasmer_compiler::types::module::CompileModuleInfo;
21use wasmer_compiler::types::relocation::RelocationKind;
22use wasmer_compiler::{
23 Compiler, FunctionBodyData, ModuleMiddleware, ModuleTranslationState,
24 types::{
25 relocation::RelocationTarget,
26 section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
27 symbols::{Symbol, SymbolRegistry},
28 },
29};
30use wasmer_compiler::{
31 WASM_LARGE_FUNCTION_THRESHOLD, WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE, build_function_buckets,
32 translate_function_buckets,
33};
34use wasmer_types::entity::{EntityRef, PrimaryMap};
35use wasmer_types::target::Target;
36use wasmer_types::{
37 CompilationProgressCallback, CompileError, FunctionIndex, LocalFunctionIndex, ModuleInfo,
38 SignatureIndex,
39};
40use wasmer_vm::LibCall;
41
42#[derive(Debug)]
45pub struct LLVMCompiler {
46 config: LLVM,
47}
48
49impl LLVMCompiler {
50 pub fn new(config: LLVM) -> LLVMCompiler {
52 LLVMCompiler { config }
53 }
54
55 fn config(&self) -> &LLVM {
57 &self.config
58 }
59}
60
61struct ShortNames {}
62
63impl SymbolRegistry for ShortNames {
64 fn symbol_to_name(&self, symbol: Symbol) -> String {
65 match symbol {
66 Symbol::Metadata => "M".to_string(),
67 Symbol::LocalFunction(index) => format!("f{}", index.index()),
68 Symbol::Section(index) => format!("s{}", index.index()),
69 Symbol::FunctionCallTrampoline(index) => format!("t{}", index.index()),
70 Symbol::DynamicFunctionTrampoline(index) => format!("d{}", index.index()),
71 }
72 }
73
74 fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
75 if name.len() < 2 {
76 return None;
77 }
78 let (ty, idx) = name.split_at(1);
79 if ty.starts_with('M') {
80 return Some(Symbol::Metadata);
81 }
82
83 let idx = idx.parse::<u32>().ok()?;
84 match ty.chars().next().unwrap() {
85 'f' => Some(Symbol::LocalFunction(LocalFunctionIndex::from_u32(idx))),
86 's' => Some(Symbol::Section(SectionIndex::from_u32(idx))),
87 't' => Some(Symbol::FunctionCallTrampoline(SignatureIndex::from_u32(
88 idx,
89 ))),
90 'd' => Some(Symbol::DynamicFunctionTrampoline(FunctionIndex::from_u32(
91 idx,
92 ))),
93 _ => None,
94 }
95 }
96}
97
98pub(crate) struct ModuleBasedSymbolRegistry {
99 wasm_module: Arc<ModuleInfo>,
100 local_func_names: HashMap<String, LocalFunctionIndex>,
101 short_names: ShortNames,
102}
103
104impl ModuleBasedSymbolRegistry {
105 const PROBLEMATIC_PREFIXES: &[&'static str] = &[
106 ".L", "llvm.", ];
109
110 fn new(wasm_module: Arc<ModuleInfo>) -> Self {
111 let local_func_names = HashMap::from_iter(
112 wasm_module
113 .function_names
114 .iter()
115 .map(|(f, v)| (wasm_module.local_func_index(*f), v))
116 .filter(|(f, _)| f.is_some())
117 .map(|(f, v)| (format!("{}_{}", v.clone(), f.unwrap().as_u32()), f.unwrap())),
118 );
119 Self {
120 wasm_module,
121 local_func_names,
122 short_names: ShortNames {},
123 }
124 }
125
126 fn fixup_problematic_name(name: &str) -> Cow<'_, str> {
128 for prefix in Self::PROBLEMATIC_PREFIXES {
129 if name.starts_with(prefix) {
130 return format!("_{name}").into();
131 }
132 }
133 name.into()
134 }
135
136 fn unfixup_problematic_name(name: &str) -> &str {
140 if let Some(stripped_name) = name.strip_prefix('_') {
141 for prefix in Self::PROBLEMATIC_PREFIXES {
142 if stripped_name.starts_with(prefix) {
143 return stripped_name;
144 }
145 }
146 }
147
148 name
149 }
150}
151
152impl SymbolRegistry for ModuleBasedSymbolRegistry {
153 fn symbol_to_name(&self, symbol: Symbol) -> String {
154 match symbol {
155 Symbol::LocalFunction(index) => self
156 .wasm_module
157 .function_names
158 .get(&self.wasm_module.func_index(index))
159 .map(|name| format!("{}_{}", Self::fixup_problematic_name(name), index.as_u32()))
160 .unwrap_or(self.short_names.symbol_to_name(symbol)),
161 _ => self.short_names.symbol_to_name(symbol),
162 }
163 }
164
165 fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
166 let name = Self::unfixup_problematic_name(name);
167 if let Some(idx) = self.local_func_names.get(name) {
168 Some(Symbol::LocalFunction(*idx))
169 } else {
170 self.short_names.name_to_symbol(name)
171 }
172 }
173}
174
175impl LLVMCompiler {
176 #[allow(clippy::too_many_arguments)]
177 fn compile_native_object(
178 &self,
179 target: &Target,
180 compile_info: &CompileModuleInfo,
181 module_translation: &ModuleTranslationState,
182 function_body_inputs: &PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
183 symbol_registry: &dyn SymbolRegistry,
184 wasmer_metadata: &[u8],
185 binary_format: target_lexicon::BinaryFormat,
186 ) -> Result<Vec<u8>, CompileError> {
187 let target_machine = self.config().target_machine(target);
188 let ctx = Context::create();
189
190 let merged_bitcode = function_body_inputs.into_iter().par_bridge().map_init(
193 || {
194 let target_machine = self.config().target_machine(target);
195 let target_machine_no_opt = self.config().target_machine_with_opt(target, false);
196 let pointer_width = target.triple().pointer_width().unwrap().bytes();
197 FuncTranslator::new(
198 target.triple().clone(),
199 target_machine,
200 Some(target_machine_no_opt),
201 binary_format,
202 pointer_width,
203 *target.cpu_features(),
204 false,
205 )
206 .unwrap()
207 },
208 |func_translator, (i, input)| {
209 let module = func_translator.translate_to_module(
210 &compile_info.module,
211 module_translation,
212 &i,
213 input,
214 self.config(),
215 &compile_info.memory_styles,
216 &compile_info.table_styles,
217 symbol_registry,
218 target.triple(),
219 )?;
220
221 Ok(module.write_bitcode_to_memory().as_slice().to_vec())
222 },
223 );
224
225 let trampolines_bitcode = compile_info.module.signatures.iter().par_bridge().map_init(
226 || {
227 let target_machine = self.config().target_machine(target);
228 FuncTrampoline::new(target_machine, target.triple().clone(), binary_format).unwrap()
229 },
230 |func_trampoline, (i, sig)| {
231 let name = symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(i));
232 let module = func_trampoline.trampoline_to_module(
233 sig,
234 self.config(),
235 &name,
236 compile_info,
237 )?;
238 Ok(module.write_bitcode_to_memory().as_slice().to_vec())
239 },
240 );
241
242 let module_hash = compile_info.module.hash_string();
243 let dynamic_trampolines_bitcode =
244 compile_info.module.functions.iter().par_bridge().map_init(
245 || {
246 let target_machine = self.config().target_machine(target);
247 (
248 FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
249 .unwrap(),
250 &compile_info.module.signatures,
251 )
252 },
253 |(func_trampoline, signatures), (i, sig)| {
254 let sig = &signatures[*sig];
255 let name = symbol_registry.symbol_to_name(Symbol::DynamicFunctionTrampoline(i));
256 let module = func_trampoline.dynamic_trampoline_to_module(
257 sig,
258 self.config(),
259 &name,
260 &module_hash,
261 )?;
262 Ok(module.write_bitcode_to_memory().as_slice().to_vec())
263 },
264 );
265
266 let merged_bitcode = merged_bitcode
267 .chain(trampolines_bitcode)
268 .chain(dynamic_trampolines_bitcode)
269 .collect::<Result<Vec<_>, CompileError>>()?
270 .into_par_iter()
271 .reduce_with(|bc1, bc2| {
272 let ctx = Context::create();
273 let membuf = MemoryBuffer::create_from_memory_range(&bc1, "");
274 let m1 = Module::parse_bitcode_from_buffer(&membuf, &ctx).unwrap();
275 let membuf = MemoryBuffer::create_from_memory_range(&bc2, "");
276 let m2 = Module::parse_bitcode_from_buffer(&membuf, &ctx).unwrap();
277 m1.link_in_module(m2).unwrap();
278 m1.write_bitcode_to_memory().as_slice().to_vec()
279 });
280 let merged_module = if let Some(bc) = merged_bitcode {
281 let membuf = MemoryBuffer::create_from_memory_range(&bc, "");
282 Module::parse_bitcode_from_buffer(&membuf, &ctx).unwrap()
283 } else {
284 ctx.create_module("")
285 };
286
287 let i8_ty = ctx.i8_type();
288 let metadata_init = i8_ty.const_array(
289 wasmer_metadata
290 .iter()
291 .map(|v| i8_ty.const_int(*v as u64, false))
292 .collect::<Vec<_>>()
293 .as_slice(),
294 );
295 let metadata_gv = merged_module.add_global(
296 metadata_init.get_type(),
297 None,
298 &symbol_registry.symbol_to_name(wasmer_compiler::types::symbols::Symbol::Metadata),
299 );
300 metadata_gv.set_initializer(&metadata_init);
301 metadata_gv.set_linkage(Linkage::DLLExport);
302 metadata_gv.set_dll_storage_class(DLLStorageClass::Export);
303 metadata_gv.set_alignment(16);
304
305 if self.config().enable_verifier {
306 merged_module.verify().unwrap();
307 }
308
309 let memory_buffer = target_machine
310 .write_to_memory_buffer(&merged_module, FileType::Object)
311 .unwrap();
312 if let Some(ref callbacks) = self.config.callbacks {
313 callbacks.obj_memory_buffer(
314 &CompiledKind::Module,
315 &compile_info.module.hash_string(),
316 &memory_buffer,
317 );
318 }
319
320 tracing::trace!("Finished compling the module!");
321 Ok(memory_buffer.as_slice().to_vec())
322 }
323}
324
325impl Compiler for LLVMCompiler {
326 fn name(&self) -> &str {
327 "llvm"
328 }
329
330 fn get_perfmap_enabled(&self) -> bool {
331 self.config.enable_perfmap
332 }
333
334 fn deterministic_id(&self) -> String {
335 format!(
336 "llvm-{}",
337 match self.config.opt_level {
338 inkwell::OptimizationLevel::None => "opt0",
339 inkwell::OptimizationLevel::Less => "optl",
340 inkwell::OptimizationLevel::Default => "optd",
341 inkwell::OptimizationLevel::Aggressive => "opta",
342 }
343 )
344 }
345
346 fn get_middlewares(&self) -> &[Arc<dyn ModuleMiddleware>] {
348 &self.config.middlewares
349 }
350
351 fn experimental_native_compile_module(
352 &self,
353 target: &Target,
354 compile_info: &CompileModuleInfo,
355 module_translation: &ModuleTranslationState,
356 function_body_inputs: &PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
358 symbol_registry: &dyn SymbolRegistry,
359 wasmer_metadata: &[u8],
361 ) -> Option<Result<Vec<u8>, CompileError>> {
362 Some(self.compile_native_object(
363 target,
364 compile_info,
365 module_translation,
366 function_body_inputs,
367 symbol_registry,
368 wasmer_metadata,
369 self.config.target_binary_format(target),
370 ))
371 }
372
373 fn compile_module(
376 &self,
377 target: &Target,
378 compile_info: &CompileModuleInfo,
379 module_translation: &ModuleTranslationState,
380 function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
381 progress_callback: Option<&CompilationProgressCallback>,
382 ) -> Result<Compilation, CompileError> {
383 let binary_format = self.config.target_binary_format(target);
384
385 let module = &compile_info.module;
386 let module_hash = module.hash_string();
387
388 let total_function_call_trampolines = module.signatures.len();
389 let total_dynamic_trampolines = module.num_imported_functions;
390 let total_steps = WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE
391 * ((total_dynamic_trampolines + total_function_call_trampolines) as u64)
392 + function_body_inputs
393 .iter()
394 .map(|(_, body)| body.data.len() as u64)
395 .sum::<u64>();
396
397 let progress = progress_callback
398 .cloned()
399 .map(|cb| ProgressContext::new(cb, total_steps, "Compiling functions"));
400
401 let mut module_custom_sections = PrimaryMap::new();
404
405 let mut eh_frame_section_bytes = vec![];
406 let mut eh_frame_section_relocations = vec![];
407
408 let mut compact_unwind_section_bytes = vec![];
409 let mut compact_unwind_section_relocations = vec![];
410
411 let mut got_targets: HashSet<wasmer_compiler::types::relocation::RelocationTarget> = if matches!(
412 target.triple().binary_format,
413 target_lexicon::BinaryFormat::Macho
414 ) {
415 HashSet::from_iter(vec![RelocationTarget::LibCall(LibCall::EHPersonality)])
416 } else {
417 HashSet::default()
418 };
419
420 let symbol_registry = ModuleBasedSymbolRegistry::new(module.clone());
421 let module = &compile_info.module;
422 let memory_styles = &compile_info.memory_styles;
423 let table_styles = &compile_info.table_styles;
424
425 let pool = ThreadPoolBuilder::new()
426 .num_threads(self.config.num_threads.get())
427 .build()
428 .map_err(|e| CompileError::Resource(e.to_string()))?;
429
430 let buckets =
431 build_function_buckets(&function_body_inputs, WASM_LARGE_FUNCTION_THRESHOLD / 3);
432 let largest_bucket = buckets.first().map(|b| b.size).unwrap_or_default();
433 tracing::debug!(buckets = buckets.len(), largest_bucket, "buckets built");
434 let functions = translate_function_buckets(
435 &pool,
436 || {
437 let compiler = &self;
438 let target_machine = compiler.config().target_machine_with_opt(target, true);
439 let target_machine_no_opt =
440 compiler.config().target_machine_with_opt(target, false);
441 let pointer_width = target.triple().pointer_width().unwrap().bytes();
442 FuncTranslator::new(
443 target.triple().clone(),
444 target_machine,
445 Some(target_machine_no_opt),
446 binary_format,
447 pointer_width,
448 *target.cpu_features(),
449 self.config.enable_non_volatile_memops,
450 )
451 .unwrap()
452 },
453 |func_translator, i, input| {
454 func_translator.translate(
455 module,
456 module_translation,
457 i,
458 input,
459 self.config(),
460 memory_styles,
461 table_styles,
462 &symbol_registry,
463 target.triple(),
464 )
465 },
466 progress.clone(),
467 &buckets,
468 )?;
469
470 let functions = functions
471 .into_iter()
472 .map(|mut compiled_function| {
473 let first_section = module_custom_sections.len() as u32;
474 for (section_index, custom_section) in compiled_function.custom_sections.iter() {
475 let mut custom_section = custom_section.clone();
477 for reloc in &mut custom_section.relocations {
478 if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
479 reloc.reloc_target = RelocationTarget::CustomSection(
480 SectionIndex::from_u32(first_section + index.as_u32()),
481 )
482 }
483
484 if reloc.kind.needs_got() {
485 got_targets.insert(reloc.reloc_target);
486 }
487 }
488
489 if compiled_function
490 .eh_frame_section_indices
491 .contains(§ion_index)
492 {
493 let offset = eh_frame_section_bytes.len() as u32;
494 for reloc in &mut custom_section.relocations {
495 reloc.offset += offset;
496 }
497 eh_frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice());
498 eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
500 eh_frame_section_relocations.extend(custom_section.relocations);
501 module_custom_sections.push(CustomSection {
503 protection: CustomSectionProtection::Read,
504 alignment: None,
505 bytes: SectionBody::new_with_vec(vec![]),
506 relocations: vec![],
507 });
508 } else if compiled_function
509 .compact_unwind_section_indices
510 .contains(§ion_index)
511 {
512 let offset = compact_unwind_section_bytes.len() as u32;
513 for reloc in &mut custom_section.relocations {
514 reloc.offset += offset;
515 }
516 compact_unwind_section_bytes
517 .extend_from_slice(custom_section.bytes.as_slice());
518 compact_unwind_section_relocations.extend(custom_section.relocations);
519 module_custom_sections.push(CustomSection {
521 protection: CustomSectionProtection::Read,
522 alignment: None,
523 bytes: SectionBody::new_with_vec(vec![]),
524 relocations: vec![],
525 });
526 } else {
527 module_custom_sections.push(custom_section);
528 }
529 }
530 for reloc in &mut compiled_function.compiled_function.relocations {
531 if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
532 reloc.reloc_target = RelocationTarget::CustomSection(
533 SectionIndex::from_u32(first_section + index.as_u32()),
534 )
535 }
536
537 if reloc.kind.needs_got() {
538 got_targets.insert(reloc.reloc_target);
539 }
540 }
541 compiled_function.compiled_function
542 })
543 .collect::<PrimaryMap<LocalFunctionIndex, _>>();
544
545 let progress = progress.clone();
546 let function_call_trampolines = pool.install(|| {
547 module
548 .signatures
549 .values()
550 .collect::<Vec<_>>()
551 .par_iter()
552 .map_init(
553 || {
554 let target_machine = self.config().target_machine(target);
555 FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
556 .unwrap()
557 },
558 |func_trampoline, sig| {
559 let trampoline =
560 func_trampoline.trampoline(sig, self.config(), "", compile_info);
561 if let Some(progress) = progress.as_ref() {
562 progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
563 }
564 trampoline
565 },
566 )
567 .collect::<Vec<_>>()
568 .into_iter()
569 .collect::<Result<PrimaryMap<_, _>, CompileError>>()
570 })?;
571
572 let dynamic_function_trampolines = {
577 let progress = progress.clone();
578 let target_machine = self.config().target_machine(target);
579 let func_trampoline =
580 FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
581 .unwrap();
582 module
583 .imported_function_types()
584 .collect::<Vec<_>>()
585 .into_iter()
586 .enumerate()
587 .map(|(index, func_type)| {
588 let trampoline = func_trampoline.dynamic_trampoline(
589 &func_type,
590 self.config(),
591 "",
592 index as u32,
593 &mut module_custom_sections,
594 &mut eh_frame_section_bytes,
595 &mut eh_frame_section_relocations,
596 &mut compact_unwind_section_bytes,
597 &mut compact_unwind_section_relocations,
598 &module_hash,
599 )?;
600 if let Some(progress) = progress.as_ref() {
601 progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
602 }
603 Ok(trampoline)
604 })
605 .collect::<Vec<_>>()
606 .into_iter()
607 .collect::<Result<PrimaryMap<_, _>, CompileError>>()?
608 };
609
610 let mut unwind_info = UnwindInfo::default();
611
612 if !eh_frame_section_bytes.is_empty() {
613 let eh_frame_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
614 module_custom_sections.push(CustomSection {
615 protection: CustomSectionProtection::Read,
616 alignment: None,
617 bytes: SectionBody::new_with_vec(eh_frame_section_bytes),
618 relocations: eh_frame_section_relocations,
619 });
620 unwind_info.eh_frame = Some(eh_frame_idx);
621 }
622
623 if !compact_unwind_section_bytes.is_empty() {
624 let cu_index = SectionIndex::from_u32(module_custom_sections.len() as u32);
625 module_custom_sections.push(CustomSection {
626 protection: CustomSectionProtection::Read,
627 alignment: None,
628 bytes: SectionBody::new_with_vec(compact_unwind_section_bytes),
629 relocations: compact_unwind_section_relocations,
630 });
631 unwind_info.compact_unwind = Some(cu_index);
632 }
633
634 let mut got = wasmer_compiler::types::function::GOT::empty();
635
636 if !got_targets.is_empty() {
637 let pointer_width = target
638 .triple()
639 .pointer_width()
640 .map_err(|_| CompileError::Codegen("Could not get pointer width".to_string()))?;
641
642 let got_entry_size = match pointer_width {
643 target_lexicon::PointerWidth::U64 => 8,
644 target_lexicon::PointerWidth::U32 => 4,
645 target_lexicon::PointerWidth::U16 => todo!(),
646 };
647
648 let got_entry_reloc_kind = match pointer_width {
649 target_lexicon::PointerWidth::U64 => RelocationKind::Abs8,
650 target_lexicon::PointerWidth::U32 => RelocationKind::Abs4,
651 target_lexicon::PointerWidth::U16 => todo!(),
652 };
653
654 let got_data: Vec<u8> = vec![0; got_targets.len() * got_entry_size];
655 let mut got_relocs = vec![];
656
657 for (i, target) in got_targets.into_iter().enumerate() {
658 got_relocs.push(wasmer_compiler::types::relocation::Relocation {
659 kind: got_entry_reloc_kind,
660 reloc_target: target,
661 offset: (i * got_entry_size) as u32,
662 addend: 0,
663 });
664 }
665
666 let got_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
667 module_custom_sections.push(CustomSection {
668 protection: CustomSectionProtection::Read,
669 alignment: None,
670 bytes: SectionBody::new_with_vec(got_data),
671 relocations: got_relocs,
672 });
673 got.index = Some(got_idx);
674 };
675
676 Ok(Compilation {
677 functions,
678 custom_sections: module_custom_sections,
679 function_call_trampolines,
680 dynamic_function_trampolines,
681 unwind_info,
682 got,
683 })
684 }
685
686 fn with_opts(
687 &mut self,
688 _suggested_compiler_opts: &wasmer_types::target::UserCompilerOptimizations,
689 ) -> Result<(), CompileError> {
690 Ok(())
691 }
692}