wasmer_compiler_llvm/
compiler.rs

1use crate::config::LLVM;
2use crate::config::OptimizationStyle;
3use crate::object_file::CompiledFunction;
4use crate::translator::FuncTrampoline;
5use crate::translator::FuncTranslator;
6use itertools::Itertools;
7use rayon::ThreadPoolBuilder;
8use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
9use std::{
10    borrow::Cow,
11    collections::{HashMap, HashSet},
12    sync::Arc,
13};
14use wasmer_compiler::progress::ProgressContext;
15use wasmer_compiler::types::function::Compilation;
16use wasmer_compiler::types::function::CompiledFunctionBody;
17use wasmer_compiler::types::function::{RkyvCompilation, UnwindInfo};
18use wasmer_compiler::types::module::CompileModuleInfo;
19use wasmer_compiler::types::relocation::RelocationKind;
20use wasmer_compiler::{
21    CompiledObjects, Compiler, FunctionBodyData, ModuleMiddleware, ModuleTranslationState,
22    WasmSourceMap, emit_metadata_and_link,
23    types::{
24        relocation::RelocationTarget,
25        section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
26        symbols::{Symbol, SymbolRegistry},
27    },
28};
29use wasmer_compiler::{
30    WASM_LARGE_FUNCTION_THRESHOLD, WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE, build_function_buckets,
31    translate_function_buckets,
32};
33use wasmer_types::ExportIndex;
34use wasmer_types::MetadataHeader;
35use wasmer_types::entity::{EntityRef, PrimaryMap};
36use wasmer_types::target::Target;
37use wasmer_types::{
38    CompilationProgressCallback, CompileError, FunctionIndex, LocalFunctionIndex, ModuleInfo,
39    SignatureIndex,
40};
41use wasmer_vm::LibCall;
42
43/// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR,
44/// optimizing it and then translating to assembly.
45#[derive(Debug)]
46pub struct LLVMCompiler {
47    config: LLVM,
48}
49
50impl LLVMCompiler {
51    /// Creates a new LLVM compiler
52    pub fn new(config: LLVM) -> LLVMCompiler {
53        LLVMCompiler { config }
54    }
55
56    /// Gets the config for this Compiler
57    fn config(&self) -> &LLVM {
58        &self.config
59    }
60}
61
62struct ShortNames {}
63
64impl SymbolRegistry for ShortNames {
65    fn symbol_to_name(&self, symbol: Symbol) -> String {
66        match symbol {
67            Symbol::Metadata => "M".to_string(),
68            Symbol::LocalFunction(index) => format!("f{}", index.index()),
69            Symbol::Section(index) => format!("s{}", index.index()),
70            Symbol::FunctionCallTrampoline(index) => format!("t{}", index.index()),
71            Symbol::DynamicFunctionTrampoline(index) => format!("d{}", index.index()),
72        }
73    }
74
75    fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
76        if name.len() < 2 {
77            return None;
78        }
79        let (ty, idx) = name.split_at(1);
80        if ty.starts_with('M') {
81            return Some(Symbol::Metadata);
82        }
83
84        let idx = idx.parse::<u32>().ok()?;
85        match ty.chars().next().unwrap() {
86            'f' => Some(Symbol::LocalFunction(LocalFunctionIndex::from_u32(idx))),
87            's' => Some(Symbol::Section(SectionIndex::from_u32(idx))),
88            't' => Some(Symbol::FunctionCallTrampoline(SignatureIndex::from_u32(
89                idx,
90            ))),
91            'd' => Some(Symbol::DynamicFunctionTrampoline(FunctionIndex::from_u32(
92                idx,
93            ))),
94            _ => None,
95        }
96    }
97}
98
99pub(crate) struct ModuleBasedSymbolRegistry {
100    wasm_module: Arc<ModuleInfo>,
101    local_func_names: HashMap<String, LocalFunctionIndex>,
102    short_names: ShortNames,
103}
104
105impl ModuleBasedSymbolRegistry {
106    const PROBLEMATIC_PREFIXES: &[&'static str] = &[
107        ".L",    // .L is used for local symbols
108        "llvm.", // llvm. is used for LLVM's own intrinsics
109    ];
110
111    fn new(wasm_module: Arc<ModuleInfo>) -> Self {
112        let local_func_names = HashMap::from_iter(
113            wasm_module
114                .function_names
115                .iter()
116                .map(|(f, v)| (wasm_module.local_func_index(*f), v))
117                .filter(|(f, _)| f.is_some())
118                .map(|(f, v)| (format!("{}_{}", v.clone(), f.unwrap().as_u32()), f.unwrap())),
119        );
120        Self {
121            wasm_module,
122            local_func_names,
123            short_names: ShortNames {},
124        }
125    }
126
127    // If the name starts with a problematic prefix, we prefix it with an underscore.
128    fn fixup_problematic_name(name: &str) -> Cow<'_, str> {
129        for prefix in Self::PROBLEMATIC_PREFIXES {
130            if name.starts_with(prefix) {
131                return format!("_{name}").into();
132            }
133        }
134        name.into()
135    }
136
137    // If the name starts with an underscore and the rest starts with a problematic prefix,
138    // remove the underscore to get back the original name. This is necessary to be able
139    // to match the name back to the original name in the wasm module.
140    fn unfixup_problematic_name(name: &str) -> &str {
141        if let Some(stripped_name) = name.strip_prefix('_') {
142            for prefix in Self::PROBLEMATIC_PREFIXES {
143                if stripped_name.starts_with(prefix) {
144                    return stripped_name;
145                }
146            }
147        }
148
149        name
150    }
151}
152
153impl SymbolRegistry for ModuleBasedSymbolRegistry {
154    fn symbol_to_name(&self, symbol: Symbol) -> String {
155        match symbol {
156            Symbol::LocalFunction(index) => self
157                .wasm_module
158                .function_names
159                .get(&self.wasm_module.func_index(index))
160                .map(|name| format!("{}_{}", Self::fixup_problematic_name(name), index.as_u32()))
161                .unwrap_or(self.short_names.symbol_to_name(symbol)),
162            _ => self.short_names.symbol_to_name(symbol),
163        }
164    }
165
166    fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
167        let name = Self::unfixup_problematic_name(name);
168        if let Some(idx) = self.local_func_names.get(name) {
169            Some(Symbol::LocalFunction(*idx))
170        } else {
171            self.short_names.name_to_symbol(name)
172        }
173    }
174}
175
176impl Compiler for LLVMCompiler {
177    fn name(&self) -> &str {
178        "llvm"
179    }
180
181    fn get_perfmap_enabled(&self) -> bool {
182        self.config.enable_perfmap
183    }
184
185    fn get_debugger(&self) -> Option<wasmer_compiler::Debugger> {
186        self.config.debugger
187    }
188
189    fn deterministic_id(&self) -> String {
190        use wasmer_compiler::DeterministicIdComponent as Component;
191
192        let mut components = vec![Component::Llvm];
193        components.push(match self.config.opt_level {
194            inkwell::OptimizationLevel::None => Component::OptNone,
195            inkwell::OptimizationLevel::Less => Component::OptLess,
196            inkwell::OptimizationLevel::Default => Component::OptDefault,
197            inkwell::OptimizationLevel::Aggressive => Component::OptAggressive,
198        });
199        if self.config.enable_nan_canonicalization {
200            components.push(Component::NanCanonicalization);
201        }
202        if self.config.enable_non_volatile_memops {
203            components.push(Component::NonVolatileMemops);
204        }
205        if self.config.is_pic {
206            components.push(Component::Pic);
207        }
208        if self.config.enable_readonly_funcref_table {
209            components.push(Component::ReadonlyFuncrefTable);
210        }
211        // We intentionally use a negative marker to distinguish it from the already
212        // existing compiled Artifacts built with M0 enabled!
213        // TODO: flip it to EnableM0 in the future
214        const _: () = assert!(
215            MetadataHeader::CURRENT_VERSION == 24,
216            "Rename Component::DisableM0",
217        );
218        if !self.config.enable_m0 {
219            components.push(Component::DisableM0);
220        }
221
222        components
223            .into_iter()
224            .map(|component| component.to_string())
225            .collect_vec()
226            .join("-")
227    }
228
229    fn artifact_format(&self) -> String {
230        if self.config.experimental_artifact {
231            wasmer_compiler::ArtifactFormat::Native
232        } else {
233            wasmer_compiler::ArtifactFormat::Rkyv
234        }
235        .to_string()
236    }
237
238    /// Get the middlewares for this compiler
239    fn get_middlewares(&self) -> &[Arc<dyn ModuleMiddleware>] {
240        &self.config.middlewares
241    }
242
243    fn enable_readonly_funcref_table(&self) -> bool {
244        self.config.enable_readonly_funcref_table
245    }
246
247    /// Compile the module using LLVM, producing a compilation result with
248    /// associated relocations.
249    fn compile_module(
250        &self,
251        target: &Target,
252        compile_info: &CompileModuleInfo,
253        compile_info_blob: &[u8],
254        module_translation: &ModuleTranslationState,
255        function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
256        progress_callback: Option<&CompilationProgressCallback>,
257    ) -> Result<Compilation, CompileError> {
258        wasmer_compiler::validate_module_fixed_table_size(
259            &compile_info.module,
260            self.config.max_table_elements,
261        )?;
262        let function_max_stack_usage = function_body_inputs.iter().map(|_| None).collect();
263        let binary_format = self.config.target_binary_format(target);
264
265        let module = &compile_info.module;
266        let module_hash = module.hash_string();
267
268        let total_function_call_trampolines = module.signatures.len();
269        let total_dynamic_trampolines = module.num_imported_functions;
270        let total_steps = WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE
271            * ((total_dynamic_trampolines + total_function_call_trampolines) as u64)
272            + function_body_inputs
273                .iter()
274                .map(|(_, body)| body.data.len() as u64)
275                .sum::<u64>();
276
277        let progress = progress_callback
278            .cloned()
279            .map(|cb| ProgressContext::new(cb, total_steps, "Compiling functions"));
280
281        // TODO: merge constants in sections.
282
283        let mut module_custom_sections = PrimaryMap::new();
284
285        let mut eh_frame_section_bytes = vec![];
286        let mut eh_frame_section_relocations = vec![];
287
288        let mut compact_unwind_section_bytes = vec![];
289        let mut compact_unwind_section_relocations = vec![];
290
291        let mut got_targets: HashSet<wasmer_compiler::types::relocation::RelocationTarget> = if matches!(
292            target.triple().binary_format,
293            target_lexicon::BinaryFormat::Macho
294        ) {
295            HashSet::from_iter(vec![RelocationTarget::LibCall(LibCall::EHPersonality)])
296        } else {
297            HashSet::default()
298        };
299
300        let symbol_registry = ModuleBasedSymbolRegistry::new(module.clone());
301        let module = &compile_info.module;
302        let memory_styles = &compile_info.memory_styles;
303        let table_styles = &compile_info.table_styles;
304        let signature_hashes = &module.signature_hashes;
305
306        let pool = ThreadPoolBuilder::new()
307            .num_threads(self.config.num_threads.get())
308            .build()
309            .map_err(|e| CompileError::Resource(e.to_string()))?;
310
311        let source_map = Arc::new(if self.config.experimental_artifact {
312            WasmSourceMap::new(module, module_translation, &function_body_inputs)
313                .map_err(CompileError::Codegen)?
314        } else {
315            WasmSourceMap::default()
316        });
317        let buckets =
318            build_function_buckets(&function_body_inputs, WASM_LARGE_FUNCTION_THRESHOLD / 3);
319        let largest_bucket = buckets.first().map(|b| b.size).unwrap_or_default();
320        tracing::debug!(buckets = buckets.len(), largest_bucket, "buckets built");
321
322        let functions = translate_function_buckets(
323            &pool,
324            || {
325                let compiler = &self;
326                let target_machines = enum_iterator::all::<OptimizationStyle>()
327                    .map(|style| {
328                        (
329                            style,
330                            compiler.config().target_machine_with_opt(target, style),
331                        )
332                    })
333                    .collect();
334                let pointer_width = target.triple().pointer_width().unwrap().bytes();
335                FuncTranslator::new(
336                    target.triple().clone(),
337                    target_machines,
338                    binary_format,
339                    pointer_width,
340                    *target.cpu_features(),
341                    self.config.enable_non_volatile_memops,
342                    source_map.clone(),
343                    module
344                        .exports
345                        .get("__wasm_apply_data_relocs")
346                        .and_then(|export| {
347                            if let ExportIndex::Function(index) = export {
348                                Some(*index)
349                            } else {
350                                None
351                            }
352                        }),
353                )
354                .unwrap()
355            },
356            |func_translator, i, input| {
357                func_translator.translate(
358                    module,
359                    module_translation,
360                    signature_hashes,
361                    i,
362                    input,
363                    self.config(),
364                    memory_styles,
365                    table_styles,
366                    &symbol_registry,
367                    target.triple(),
368                )
369            },
370            progress.clone(),
371            &buckets,
372        )?;
373
374        let progress = progress.clone();
375        let function_call_trampolines = pool.install(|| {
376            module
377                .signatures
378                .iter()
379                .collect::<Vec<_>>()
380                .par_iter()
381                .map_init(
382                    || {
383                        let target_machine = self.config().target_machine(target);
384                        FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
385                            .unwrap()
386                    },
387                    |func_trampoline, (sig_index, sig)| {
388                        let kind = wasmer_compiler::misc::CompiledKind::FunctionCallTrampoline(
389                            *sig_index,
390                            (*sig).clone(),
391                        );
392                        let trampoline =
393                            func_trampoline.trampoline(sig, self.config(), &kind, compile_info);
394                        if let Some(progress) = progress.as_ref() {
395                            progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
396                        }
397                        trampoline
398                    },
399                )
400                .collect::<Result<Vec<_>, _>>()
401        })?;
402
403        // TODO: I removed the parallel processing of dynamic trampolines because we're passing
404        // the sections bytes and relocations directly into the trampoline generation function.
405        // We can move that logic out and re-enable parallel processing. Hopefully, there aren't
406        // enough dynamic trampolines to actually cause a noticeable performance degradation.
407        let dynamic_function_trampolines = {
408            let progress = progress.clone();
409            let target_machine = self.config().target_machine(target);
410            let func_trampoline =
411                FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
412                    .unwrap();
413            module
414                .imported_function_types()
415                .collect::<Vec<_>>()
416                .into_iter()
417                .enumerate()
418                .map(|(index, func_type)| {
419                    let kind = wasmer_compiler::misc::CompiledKind::DynamicFunctionTrampoline(
420                        FunctionIndex::from_u32(index as u32),
421                        func_type.clone(),
422                    );
423                    let trampoline = func_trampoline.dynamic_trampoline(
424                        &func_type,
425                        self.config(),
426                        &kind,
427                        index as u32,
428                        &mut module_custom_sections,
429                        &mut eh_frame_section_bytes,
430                        &mut eh_frame_section_relocations,
431                        &mut compact_unwind_section_bytes,
432                        &mut compact_unwind_section_relocations,
433                        &module_hash,
434                    )?;
435                    if let Some(progress) = progress.as_ref() {
436                        progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
437                    }
438                    Ok(trampoline)
439                })
440                .collect::<Result<Vec<_>, CompileError>>()?
441        };
442
443        if self.config.experimental_artifact {
444            let object_files = functions
445                .into_iter()
446                .map(|compiled_function| match compiled_function {
447                    CompiledFunction::Elf(path) => path,
448                    CompiledFunction::Rkyv(_) => {
449                        unreachable!()
450                    }
451                })
452                .collect::<Vec<Vec<u8>>>();
453            let trampolines_objects = function_call_trampolines
454                .into_iter()
455                .map(|f| match f {
456                    CompiledFunctionBody::Elf(path) => path,
457                    CompiledFunctionBody::Rkyv(_) => {
458                        unreachable!()
459                    }
460                })
461                .collect::<Vec<Vec<u8>>>();
462            let dynamic_trampolines_objects = dynamic_function_trampolines
463                .into_iter()
464                .map(|f| match f {
465                    CompiledFunctionBody::Elf(path) => path,
466                    CompiledFunctionBody::Rkyv(_) => unreachable!(),
467                })
468                .collect::<Vec<Vec<u8>>>();
469
470            let elf_content = emit_metadata_and_link(
471                &pool,
472                target,
473                compile_info_blob,
474                CompiledObjects {
475                    object_files,
476                    import_trampoline_object_files: Vec::new(),
477                    trampoline_object_files: trampolines_objects,
478                    dynamic_trampoline_object_files: dynamic_trampolines_objects,
479                },
480                self.config
481                    .callbacks
482                    .as_ref()
483                    .map(|callbacks| callbacks.debug_dir().clone()),
484                module.hash().map(|hash| hash.to_string()),
485            )?;
486            Ok(Compilation::Elf {
487                data: elf_content,
488                function_max_stack_usage,
489            })
490        } else {
491            let functions = functions
492                .into_iter()
493                .map(|compiled_function| {
494                    let CompiledFunction::Rkyv(mut compiled_function) = compiled_function else {
495                        unreachable!()
496                    };
497
498                    let first_section = module_custom_sections.len() as u32;
499                    for (section_index, custom_section) in compiled_function.custom_sections.iter()
500                    {
501                        // TODO: remove this call to clone()
502                        let mut custom_section = custom_section.clone();
503                        for reloc in &mut custom_section.relocations {
504                            if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
505                                reloc.reloc_target = RelocationTarget::CustomSection(
506                                    SectionIndex::from_u32(first_section + index.as_u32()),
507                                )
508                            }
509
510                            if reloc.kind.needs_got() {
511                                got_targets.insert(reloc.reloc_target);
512                            }
513                        }
514
515                        if compiled_function
516                            .eh_frame_section_indices
517                            .contains(&section_index)
518                        {
519                            let offset = eh_frame_section_bytes.len() as u32;
520                            for reloc in &mut custom_section.relocations {
521                                reloc.offset += offset;
522                            }
523                            eh_frame_section_bytes
524                                .extend_from_slice(custom_section.bytes.as_slice());
525                            // Terminate the eh_frame info with a zero-length CIE.
526                            eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
527                            eh_frame_section_relocations.extend(custom_section.relocations);
528                            // TODO: we do this to keep the count right, remove it.
529                            module_custom_sections.push(CustomSection {
530                                protection: CustomSectionProtection::Read,
531                                alignment: None,
532                                bytes: SectionBody::new_with_vec(vec![]),
533                                relocations: vec![],
534                            });
535                        } else if compiled_function
536                            .compact_unwind_section_indices
537                            .contains(&section_index)
538                        {
539                            let offset = compact_unwind_section_bytes.len() as u32;
540                            for reloc in &mut custom_section.relocations {
541                                reloc.offset += offset;
542                            }
543                            compact_unwind_section_bytes
544                                .extend_from_slice(custom_section.bytes.as_slice());
545                            compact_unwind_section_relocations.extend(custom_section.relocations);
546                            // TODO: we do this to keep the count right, remove it.
547                            module_custom_sections.push(CustomSection {
548                                protection: CustomSectionProtection::Read,
549                                alignment: None,
550                                bytes: SectionBody::new_with_vec(vec![]),
551                                relocations: vec![],
552                            });
553                        } else {
554                            module_custom_sections.push(custom_section);
555                        }
556                    }
557                    for reloc in &mut compiled_function.compiled_function.relocations {
558                        if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
559                            reloc.reloc_target = RelocationTarget::CustomSection(
560                                SectionIndex::from_u32(first_section + index.as_u32()),
561                            )
562                        }
563
564                        if reloc.kind.needs_got() {
565                            got_targets.insert(reloc.reloc_target);
566                        }
567                    }
568                    compiled_function.compiled_function
569                })
570                .collect::<PrimaryMap<LocalFunctionIndex, _>>();
571
572            let mut unwind_info = UnwindInfo::default();
573
574            if !eh_frame_section_bytes.is_empty() {
575                let eh_frame_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
576                module_custom_sections.push(CustomSection {
577                    protection: CustomSectionProtection::Read,
578                    alignment: None,
579                    bytes: SectionBody::new_with_vec(eh_frame_section_bytes),
580                    relocations: eh_frame_section_relocations,
581                });
582                unwind_info.eh_frame = Some(eh_frame_idx);
583            }
584
585            if !compact_unwind_section_bytes.is_empty() {
586                let cu_index = SectionIndex::from_u32(module_custom_sections.len() as u32);
587                module_custom_sections.push(CustomSection {
588                    protection: CustomSectionProtection::Read,
589                    alignment: None,
590                    bytes: SectionBody::new_with_vec(compact_unwind_section_bytes),
591                    relocations: compact_unwind_section_relocations,
592                });
593                unwind_info.compact_unwind = Some(cu_index);
594            }
595
596            let mut got = wasmer_compiler::types::function::GOT::empty();
597
598            if !got_targets.is_empty() {
599                let got_data: Vec<u8> = vec![0; got_targets.len() * 8];
600                let mut got_relocs = vec![];
601
602                for (i, target) in got_targets.into_iter().enumerate() {
603                    got_relocs.push(wasmer_compiler::types::relocation::Relocation {
604                        kind: RelocationKind::Abs8,
605                        reloc_target: target,
606                        offset: (i * 8) as u32,
607                        addend: 0,
608                    });
609                }
610
611                let got_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
612                module_custom_sections.push(CustomSection {
613                    protection: CustomSectionProtection::Read,
614                    alignment: None,
615                    bytes: SectionBody::new_with_vec(got_data),
616                    relocations: got_relocs,
617                });
618                got.index = Some(got_idx);
619            };
620
621            let function_call_trampolines = function_call_trampolines
622                .into_iter()
623                .map(|f| {
624                    let CompiledFunctionBody::Rkyv(function) = f else {
625                        unreachable!()
626                    };
627                    function
628                })
629                .collect();
630            let dynamic_function_trampolines = dynamic_function_trampolines
631                .into_iter()
632                .map(|f| {
633                    let CompiledFunctionBody::Rkyv(function) = f else {
634                        unreachable!()
635                    };
636                    function
637                })
638                .collect();
639
640            Ok(Compilation::Rkyv {
641                compilation: RkyvCompilation {
642                    functions,
643                    custom_sections: module_custom_sections,
644                    function_call_trampolines,
645                    dynamic_function_trampolines,
646                    unwind_info,
647                    got,
648                },
649                function_max_stack_usage,
650            })
651        }
652    }
653
654    /// Add suggested optimizations to this compiler.
655    fn with_opts(
656        &mut self,
657        _suggested_compiler_opts: &wasmer_types::target::UserCompilerOptimizations,
658    ) -> Result<(), CompileError> {
659        Ok(())
660    }
661}
wasmer_compiler_llvm/compiler.rs

wasmer_compiler_llvm/
compiler.rs
