1use crate::config::LLVM;
2use crate::trampoline::FuncTrampoline;
3use crate::translator::FuncTranslator;
4use inkwell::DLLStorageClass;
5use inkwell::context::Context;
6use inkwell::memory_buffer::MemoryBuffer;
7use inkwell::module::{Linkage, Module};
8use inkwell::targets::FileType;
9use rayon::ThreadPoolBuilder;
10use rayon::iter::ParallelBridge;
11use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
12use std::borrow::Cow;
13use std::collections::{HashMap, HashSet};
14use std::sync::Arc;
15use wasmer_compiler::misc::CompiledKind;
16use wasmer_compiler::types::function::{Compilation, UnwindInfo};
17use wasmer_compiler::types::module::CompileModuleInfo;
18use wasmer_compiler::types::relocation::RelocationKind;
19use wasmer_compiler::{
20 Compiler, FunctionBodyData, ModuleMiddleware, ModuleTranslationState,
21 types::{
22 relocation::RelocationTarget,
23 section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
24 symbols::{Symbol, SymbolRegistry},
25 },
26};
27use wasmer_types::entity::{EntityRef, PrimaryMap};
28use wasmer_types::target::Target;
29use wasmer_types::{CompileError, FunctionIndex, LocalFunctionIndex, ModuleInfo, SignatureIndex};
30use wasmer_vm::LibCall;
31
32#[derive(Debug)]
35pub struct LLVMCompiler {
36 config: LLVM,
37}
38
39impl LLVMCompiler {
40 pub fn new(config: LLVM) -> LLVMCompiler {
42 LLVMCompiler { config }
43 }
44
45 fn config(&self) -> &LLVM {
47 &self.config
48 }
49}
50
51struct ShortNames {}
52
53impl SymbolRegistry for ShortNames {
54 fn symbol_to_name(&self, symbol: Symbol) -> String {
55 match symbol {
56 Symbol::Metadata => "M".to_string(),
57 Symbol::LocalFunction(index) => format!("f{}", index.index()),
58 Symbol::Section(index) => format!("s{}", index.index()),
59 Symbol::FunctionCallTrampoline(index) => format!("t{}", index.index()),
60 Symbol::DynamicFunctionTrampoline(index) => format!("d{}", index.index()),
61 }
62 }
63
64 fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
65 if name.len() < 2 {
66 return None;
67 }
68 let (ty, idx) = name.split_at(1);
69 if ty.starts_with('M') {
70 return Some(Symbol::Metadata);
71 }
72
73 let idx = idx.parse::<u32>().ok()?;
74 match ty.chars().next().unwrap() {
75 'f' => Some(Symbol::LocalFunction(LocalFunctionIndex::from_u32(idx))),
76 's' => Some(Symbol::Section(SectionIndex::from_u32(idx))),
77 't' => Some(Symbol::FunctionCallTrampoline(SignatureIndex::from_u32(
78 idx,
79 ))),
80 'd' => Some(Symbol::DynamicFunctionTrampoline(FunctionIndex::from_u32(
81 idx,
82 ))),
83 _ => None,
84 }
85 }
86}
87
88struct ModuleBasedSymbolRegistry {
89 wasm_module: Arc<ModuleInfo>,
90 local_func_names: HashMap<String, LocalFunctionIndex>,
91 short_names: ShortNames,
92}
93
94impl ModuleBasedSymbolRegistry {
95 const PROBLEMATIC_PREFIXES: &[&'static str] = &[
96 ".L", "llvm.", ];
99
100 fn new(wasm_module: Arc<ModuleInfo>) -> Self {
101 let local_func_names = HashMap::from_iter(
102 wasm_module
103 .function_names
104 .iter()
105 .map(|(f, v)| (wasm_module.local_func_index(*f), v))
106 .filter(|(f, _)| f.is_some())
107 .map(|(f, v)| (format!("{}_{}", v.clone(), f.unwrap().as_u32()), f.unwrap())),
108 );
109 Self {
110 wasm_module,
111 local_func_names,
112 short_names: ShortNames {},
113 }
114 }
115
116 fn fixup_problematic_name(name: &str) -> Cow<'_, str> {
118 for prefix in Self::PROBLEMATIC_PREFIXES {
119 if name.starts_with(prefix) {
120 return format!("_{name}").into();
121 }
122 }
123 name.into()
124 }
125
126 fn unfixup_problematic_name(name: &str) -> &str {
130 if let Some(stripped_name) = name.strip_prefix('_') {
131 for prefix in Self::PROBLEMATIC_PREFIXES {
132 if stripped_name.starts_with(prefix) {
133 return stripped_name;
134 }
135 }
136 }
137
138 name
139 }
140}
141
142impl SymbolRegistry for ModuleBasedSymbolRegistry {
143 fn symbol_to_name(&self, symbol: Symbol) -> String {
144 match symbol {
145 Symbol::LocalFunction(index) => self
146 .wasm_module
147 .function_names
148 .get(&self.wasm_module.func_index(index))
149 .map(|name| format!("{}_{}", Self::fixup_problematic_name(name), index.as_u32()))
150 .unwrap_or(self.short_names.symbol_to_name(symbol)),
151 _ => self.short_names.symbol_to_name(symbol),
152 }
153 }
154
155 fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
156 let name = Self::unfixup_problematic_name(name);
157 if let Some(idx) = self.local_func_names.get(name) {
158 Some(Symbol::LocalFunction(*idx))
159 } else {
160 self.short_names.name_to_symbol(name)
161 }
162 }
163}
164
165impl LLVMCompiler {
166 #[allow(clippy::too_many_arguments)]
167 fn compile_native_object(
168 &self,
169 target: &Target,
170 compile_info: &CompileModuleInfo,
171 module_translation: &ModuleTranslationState,
172 function_body_inputs: &PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
173 symbol_registry: &dyn SymbolRegistry,
174 wasmer_metadata: &[u8],
175 binary_format: target_lexicon::BinaryFormat,
176 ) -> Result<Vec<u8>, CompileError> {
177 let target_machine = self.config().target_machine(target);
178 let ctx = Context::create();
179
180 let merged_bitcode = function_body_inputs.into_iter().par_bridge().map_init(
183 || {
184 let target_machine = self.config().target_machine(target);
185 FuncTranslator::new(target_machine, binary_format).unwrap()
186 },
187 |func_translator, (i, input)| {
188 let module = func_translator.translate_to_module(
189 &compile_info.module,
190 module_translation,
191 &i,
192 input,
193 self.config(),
194 &compile_info.memory_styles,
195 &compile_info.table_styles,
196 symbol_registry,
197 target.triple(),
198 )?;
199
200 Ok(module.write_bitcode_to_memory().as_slice().to_vec())
201 },
202 );
203
204 let trampolines_bitcode = compile_info.module.signatures.iter().par_bridge().map_init(
205 || {
206 let target_machine = self.config().target_machine(target);
207 FuncTrampoline::new(target_machine, binary_format).unwrap()
208 },
209 |func_trampoline, (i, sig)| {
210 let name = symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(i));
211 let module = func_trampoline.trampoline_to_module(
212 sig,
213 self.config(),
214 &name,
215 compile_info,
216 )?;
217 Ok(module.write_bitcode_to_memory().as_slice().to_vec())
218 },
219 );
220
221 let dynamic_trampolines_bitcode =
222 compile_info.module.functions.iter().par_bridge().map_init(
223 || {
224 let target_machine = self.config().target_machine(target);
225 (
226 FuncTrampoline::new(target_machine, binary_format).unwrap(),
227 &compile_info.module.signatures,
228 )
229 },
230 |(func_trampoline, signatures), (i, sig)| {
231 let sig = &signatures[*sig];
232 let name = symbol_registry.symbol_to_name(Symbol::DynamicFunctionTrampoline(i));
233 let module =
234 func_trampoline.dynamic_trampoline_to_module(sig, self.config(), &name)?;
235 Ok(module.write_bitcode_to_memory().as_slice().to_vec())
236 },
237 );
238
239 let merged_bitcode = merged_bitcode
240 .chain(trampolines_bitcode)
241 .chain(dynamic_trampolines_bitcode)
242 .collect::<Result<Vec<_>, CompileError>>()?
243 .into_par_iter()
244 .reduce_with(|bc1, bc2| {
245 let ctx = Context::create();
246 let membuf = MemoryBuffer::create_from_memory_range(&bc1, "");
247 let m1 = Module::parse_bitcode_from_buffer(&membuf, &ctx).unwrap();
248 let membuf = MemoryBuffer::create_from_memory_range(&bc2, "");
249 let m2 = Module::parse_bitcode_from_buffer(&membuf, &ctx).unwrap();
250 m1.link_in_module(m2).unwrap();
251 m1.write_bitcode_to_memory().as_slice().to_vec()
252 });
253 let merged_module = if let Some(bc) = merged_bitcode {
254 let membuf = MemoryBuffer::create_from_memory_range(&bc, "");
255 Module::parse_bitcode_from_buffer(&membuf, &ctx).unwrap()
256 } else {
257 ctx.create_module("")
258 };
259
260 let i8_ty = ctx.i8_type();
261 let metadata_init = i8_ty.const_array(
262 wasmer_metadata
263 .iter()
264 .map(|v| i8_ty.const_int(*v as u64, false))
265 .collect::<Vec<_>>()
266 .as_slice(),
267 );
268 let metadata_gv = merged_module.add_global(
269 metadata_init.get_type(),
270 None,
271 &symbol_registry.symbol_to_name(wasmer_compiler::types::symbols::Symbol::Metadata),
272 );
273 metadata_gv.set_initializer(&metadata_init);
274 metadata_gv.set_linkage(Linkage::DLLExport);
275 metadata_gv.set_dll_storage_class(DLLStorageClass::Export);
276 metadata_gv.set_alignment(16);
277
278 if self.config().enable_verifier {
279 merged_module.verify().unwrap();
280 }
281
282 let memory_buffer = target_machine
283 .write_to_memory_buffer(&merged_module, FileType::Object)
284 .unwrap();
285 if let Some(ref callbacks) = self.config.callbacks {
286 callbacks.obj_memory_buffer(&CompiledKind::Module, &memory_buffer);
287 }
288
289 tracing::trace!("Finished compling the module!");
290 Ok(memory_buffer.as_slice().to_vec())
291 }
292}
293
294impl Compiler for LLVMCompiler {
295 fn name(&self) -> &str {
296 "llvm"
297 }
298
299 fn get_perfmap_enabled(&self) -> bool {
300 self.config.enable_perfmap
301 }
302
303 fn deterministic_id(&self) -> String {
304 let mut ret = format!(
305 "llvm-{}",
306 match self.config.opt_level {
307 inkwell::OptimizationLevel::None => "opt0",
308 inkwell::OptimizationLevel::Less => "optl",
309 inkwell::OptimizationLevel::Default => "optd",
310 inkwell::OptimizationLevel::Aggressive => "opta",
311 }
312 );
313
314 if self.config.enable_g0m0_opt {
315 ret.push_str("-g0m0");
316 }
317
318 ret
319 }
320
321 fn get_middlewares(&self) -> &[Arc<dyn ModuleMiddleware>] {
323 &self.config.middlewares
324 }
325
326 fn experimental_native_compile_module(
327 &self,
328 target: &Target,
329 compile_info: &CompileModuleInfo,
330 module_translation: &ModuleTranslationState,
331 function_body_inputs: &PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
333 symbol_registry: &dyn SymbolRegistry,
334 wasmer_metadata: &[u8],
336 ) -> Option<Result<Vec<u8>, CompileError>> {
337 Some(self.compile_native_object(
338 target,
339 compile_info,
340 module_translation,
341 function_body_inputs,
342 symbol_registry,
343 wasmer_metadata,
344 self.config.target_binary_format(target),
345 ))
346 }
347
348 fn compile_module(
351 &self,
352 target: &Target,
353 compile_info: &CompileModuleInfo,
354 module_translation: &ModuleTranslationState,
355 function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
356 ) -> Result<Compilation, CompileError> {
357 let memory_styles = &compile_info.memory_styles;
360 let table_styles = &compile_info.table_styles;
361 let binary_format = self.config.target_binary_format(target);
362
363 let module = &compile_info.module;
364
365 let mut module_custom_sections = PrimaryMap::new();
368
369 let mut eh_frame_section_bytes = vec![];
370 let mut eh_frame_section_relocations = vec![];
371
372 let mut compact_unwind_section_bytes = vec![];
373 let mut compact_unwind_section_relocations = vec![];
374
375 let mut got_targets: HashSet<wasmer_compiler::types::relocation::RelocationTarget> = if matches!(
376 target.triple().binary_format,
377 target_lexicon::BinaryFormat::Macho
378 ) {
379 HashSet::from_iter(vec![RelocationTarget::LibCall(LibCall::EHPersonality)])
380 } else {
381 HashSet::default()
382 };
383
384 let symbol_registry = ModuleBasedSymbolRegistry::new(module.clone());
385
386 let functions = if self.config.num_threads.get() > 1 {
387 let pool = ThreadPoolBuilder::new()
388 .num_threads(self.config.num_threads.get())
389 .build()
390 .map_err(|e| CompileError::Resource(e.to_string()))?;
391 pool.install(|| {
392 function_body_inputs
393 .iter()
394 .collect::<Vec<(LocalFunctionIndex, &FunctionBodyData<'_>)>>()
395 .par_iter()
396 .map_init(
397 || {
398 let target_machine = self.config().target_machine(target);
399 FuncTranslator::new(target_machine, binary_format).unwrap()
400 },
401 |func_translator, (i, input)| {
402 func_translator.translate(
406 module,
407 module_translation,
408 i,
409 input,
410 self.config(),
411 memory_styles,
412 table_styles,
413 &symbol_registry,
414 target.triple(),
415 )
416 },
417 )
418 .collect::<Result<Vec<_>, CompileError>>()
419 })?
420 } else {
421 let target_machine = self.config().target_machine(target);
422 let func_translator = FuncTranslator::new(target_machine, binary_format).unwrap();
423
424 function_body_inputs
425 .iter()
426 .collect::<Vec<(LocalFunctionIndex, &FunctionBodyData<'_>)>>()
427 .into_iter()
428 .map(|(i, input)| {
429 func_translator.translate(
433 module,
434 module_translation,
435 &i,
436 input,
437 self.config(),
438 memory_styles,
439 table_styles,
440 &symbol_registry,
441 target.triple(),
442 )
443 })
444 .collect::<Result<Vec<_>, CompileError>>()?
445 };
446
447 let functions = functions
448 .into_iter()
449 .map(|mut compiled_function| {
450 let first_section = module_custom_sections.len() as u32;
451 for (section_index, custom_section) in compiled_function.custom_sections.iter() {
452 let mut custom_section = custom_section.clone();
454 for reloc in &mut custom_section.relocations {
455 if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
456 reloc.reloc_target = RelocationTarget::CustomSection(
457 SectionIndex::from_u32(first_section + index.as_u32()),
458 )
459 }
460
461 if reloc.kind.needs_got() {
462 got_targets.insert(reloc.reloc_target);
463 }
464 }
465
466 if compiled_function
467 .eh_frame_section_indices
468 .contains(§ion_index)
469 {
470 let offset = eh_frame_section_bytes.len() as u32;
471 for reloc in &mut custom_section.relocations {
472 reloc.offset += offset;
473 }
474 eh_frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice());
475 eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
477 eh_frame_section_relocations.extend(custom_section.relocations);
478 module_custom_sections.push(CustomSection {
480 protection: CustomSectionProtection::Read,
481 alignment: None,
482 bytes: SectionBody::new_with_vec(vec![]),
483 relocations: vec![],
484 });
485 } else if compiled_function
486 .compact_unwind_section_indices
487 .contains(§ion_index)
488 {
489 let offset = compact_unwind_section_bytes.len() as u32;
490 for reloc in &mut custom_section.relocations {
491 reloc.offset += offset;
492 }
493 compact_unwind_section_bytes
494 .extend_from_slice(custom_section.bytes.as_slice());
495 compact_unwind_section_relocations.extend(custom_section.relocations);
496 module_custom_sections.push(CustomSection {
498 protection: CustomSectionProtection::Read,
499 alignment: None,
500 bytes: SectionBody::new_with_vec(vec![]),
501 relocations: vec![],
502 });
503 } else {
504 module_custom_sections.push(custom_section);
505 }
506 }
507 for reloc in &mut compiled_function.compiled_function.relocations {
508 if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
509 reloc.reloc_target = RelocationTarget::CustomSection(
510 SectionIndex::from_u32(first_section + index.as_u32()),
511 )
512 }
513
514 if reloc.kind.needs_got() {
515 got_targets.insert(reloc.reloc_target);
516 }
517 }
518 compiled_function.compiled_function
519 })
520 .collect::<PrimaryMap<LocalFunctionIndex, _>>();
521
522 let function_call_trampolines = if self.config.num_threads.get() > 1 {
523 let pool = ThreadPoolBuilder::new()
524 .num_threads(self.config.num_threads.get())
525 .build()
526 .map_err(|e| CompileError::Resource(e.to_string()))?;
527 pool.install(|| {
528 module
529 .signatures
530 .values()
531 .collect::<Vec<_>>()
532 .par_iter()
533 .map_init(
534 || {
535 let target_machine = self.config().target_machine(target);
536 FuncTrampoline::new(target_machine, binary_format).unwrap()
537 },
538 |func_trampoline, sig| {
539 func_trampoline.trampoline(sig, self.config(), "", compile_info)
540 },
541 )
542 .collect::<Vec<_>>()
543 .into_iter()
544 .collect::<Result<PrimaryMap<_, _>, CompileError>>()
545 })?
546 } else {
547 let target_machine = self.config().target_machine(target);
548 let func_trampoline = FuncTrampoline::new(target_machine, binary_format).unwrap();
549 module
550 .signatures
551 .values()
552 .collect::<Vec<_>>()
553 .into_iter()
554 .map(|sig| func_trampoline.trampoline(sig, self.config(), "", compile_info))
555 .collect::<Vec<_>>()
556 .into_iter()
557 .collect::<Result<PrimaryMap<_, _>, CompileError>>()?
558 };
559
560 let dynamic_function_trampolines = {
565 let target_machine = self.config().target_machine(target);
566 let func_trampoline = FuncTrampoline::new(target_machine, binary_format).unwrap();
567 module
568 .imported_function_types()
569 .collect::<Vec<_>>()
570 .into_iter()
571 .enumerate()
572 .map(|(index, func_type)| {
573 func_trampoline.dynamic_trampoline(
574 &func_type,
575 self.config(),
576 "",
577 index as u32,
578 &mut module_custom_sections,
579 &mut eh_frame_section_bytes,
580 &mut eh_frame_section_relocations,
581 &mut compact_unwind_section_bytes,
582 &mut compact_unwind_section_relocations,
583 )
584 })
585 .collect::<Vec<_>>()
586 .into_iter()
587 .collect::<Result<PrimaryMap<_, _>, CompileError>>()?
588 };
589
590 let mut unwind_info = UnwindInfo::default();
591
592 if !eh_frame_section_bytes.is_empty() {
593 let eh_frame_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
594 module_custom_sections.push(CustomSection {
595 protection: CustomSectionProtection::Read,
596 alignment: None,
597 bytes: SectionBody::new_with_vec(eh_frame_section_bytes),
598 relocations: eh_frame_section_relocations,
599 });
600 unwind_info.eh_frame = Some(eh_frame_idx);
601 }
602
603 if !compact_unwind_section_bytes.is_empty() {
604 let cu_index = SectionIndex::from_u32(module_custom_sections.len() as u32);
605 module_custom_sections.push(CustomSection {
606 protection: CustomSectionProtection::Read,
607 alignment: None,
608 bytes: SectionBody::new_with_vec(compact_unwind_section_bytes),
609 relocations: compact_unwind_section_relocations,
610 });
611 unwind_info.compact_unwind = Some(cu_index);
612 }
613
614 let mut got = wasmer_compiler::types::function::GOT::empty();
615
616 if !got_targets.is_empty() {
617 let pointer_width = target
618 .triple()
619 .pointer_width()
620 .map_err(|_| CompileError::Codegen("Could not get pointer width".to_string()))?;
621
622 let got_entry_size = match pointer_width {
623 target_lexicon::PointerWidth::U64 => 8,
624 target_lexicon::PointerWidth::U32 => 4,
625 target_lexicon::PointerWidth::U16 => todo!(),
626 };
627
628 let got_entry_reloc_kind = match pointer_width {
629 target_lexicon::PointerWidth::U64 => RelocationKind::Abs8,
630 target_lexicon::PointerWidth::U32 => RelocationKind::Abs4,
631 target_lexicon::PointerWidth::U16 => todo!(),
632 };
633
634 let got_data: Vec<u8> = vec![0; got_targets.len() * got_entry_size];
635 let mut got_relocs = vec![];
636
637 for (i, target) in got_targets.into_iter().enumerate() {
638 got_relocs.push(wasmer_compiler::types::relocation::Relocation {
639 kind: got_entry_reloc_kind,
640 reloc_target: target,
641 offset: (i * got_entry_size) as u32,
642 addend: 0,
643 });
644 }
645
646 let got_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
647 module_custom_sections.push(CustomSection {
648 protection: CustomSectionProtection::Read,
649 alignment: None,
650 bytes: SectionBody::new_with_vec(got_data),
651 relocations: got_relocs,
652 });
653 got.index = Some(got_idx);
654 };
655
656 tracing::trace!("Finished compling the module!");
657 Ok(Compilation {
658 functions,
659 custom_sections: module_custom_sections,
660 function_call_trampolines,
661 dynamic_function_trampolines,
662 unwind_info,
663 got,
664 })
665 }
666
667 fn with_opts(
668 &mut self,
669 suggested_compiler_opts: &wasmer_types::target::UserCompilerOptimizations,
670 ) -> Result<(), CompileError> {
671 if suggested_compiler_opts.pass_params.is_some_and(|v| v) {
672 self.config.enable_g0m0_opt = true;
673 }
674 Ok(())
675 }
676}