wasmer_compiler_llvm/
compiler.rs

1use crate::config::LLVM;
2use crate::config::OptimizationStyle;
3use crate::object_file::CompiledFunction;
4use crate::translator::FuncTrampoline;
5use crate::translator::FuncTranslator;
6use itertools::Itertools;
7use rayon::ThreadPoolBuilder;
8use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
9use std::{
10    borrow::Cow,
11    collections::{HashMap, HashSet},
12    sync::Arc,
13};
14use wasmer_compiler::progress::ProgressContext;
15use wasmer_compiler::types::function::Compilation;
16use wasmer_compiler::types::function::CompiledFunctionBody;
17use wasmer_compiler::types::function::{RkyvCompilation, UnwindInfo};
18use wasmer_compiler::types::module::CompileModuleInfo;
19use wasmer_compiler::types::relocation::RelocationKind;
20use wasmer_compiler::{
21    CompiledObjects, Compiler, FunctionBodyData, ModuleMiddleware, ModuleTranslationState,
22    WasmSourceMap, emit_metadata_and_link,
23    types::{
24        relocation::RelocationTarget,
25        section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
26        symbols::{Symbol, SymbolRegistry},
27    },
28};
29use wasmer_compiler::{
30    WASM_LARGE_FUNCTION_THRESHOLD, WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE, build_function_buckets,
31    translate_function_buckets,
32};
33use wasmer_types::ExportIndex;
34use wasmer_types::entity::{EntityRef, PrimaryMap};
35use wasmer_types::target::Target;
36use wasmer_types::{
37    CompilationProgressCallback, CompileError, FunctionIndex, LocalFunctionIndex, ModuleInfo,
38    SignatureIndex,
39};
40use wasmer_vm::LibCall;
41
42/// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR,
43/// optimizing it and then translating to assembly.
44#[derive(Debug)]
45pub struct LLVMCompiler {
46    config: LLVM,
47}
48
49impl LLVMCompiler {
50    /// Creates a new LLVM compiler
51    pub fn new(config: LLVM) -> LLVMCompiler {
52        LLVMCompiler { config }
53    }
54
55    /// Gets the config for this Compiler
56    fn config(&self) -> &LLVM {
57        &self.config
58    }
59}
60
61struct ShortNames {}
62
63impl SymbolRegistry for ShortNames {
64    fn symbol_to_name(&self, symbol: Symbol) -> String {
65        match symbol {
66            Symbol::Metadata => "M".to_string(),
67            Symbol::LocalFunction(index) => format!("f{}", index.index()),
68            Symbol::Section(index) => format!("s{}", index.index()),
69            Symbol::FunctionCallTrampoline(index) => format!("t{}", index.index()),
70            Symbol::DynamicFunctionTrampoline(index) => format!("d{}", index.index()),
71        }
72    }
73
74    fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
75        if name.len() < 2 {
76            return None;
77        }
78        let (ty, idx) = name.split_at(1);
79        if ty.starts_with('M') {
80            return Some(Symbol::Metadata);
81        }
82
83        let idx = idx.parse::<u32>().ok()?;
84        match ty.chars().next().unwrap() {
85            'f' => Some(Symbol::LocalFunction(LocalFunctionIndex::from_u32(idx))),
86            's' => Some(Symbol::Section(SectionIndex::from_u32(idx))),
87            't' => Some(Symbol::FunctionCallTrampoline(SignatureIndex::from_u32(
88                idx,
89            ))),
90            'd' => Some(Symbol::DynamicFunctionTrampoline(FunctionIndex::from_u32(
91                idx,
92            ))),
93            _ => None,
94        }
95    }
96}
97
98pub(crate) struct ModuleBasedSymbolRegistry {
99    wasm_module: Arc<ModuleInfo>,
100    local_func_names: HashMap<String, LocalFunctionIndex>,
101    short_names: ShortNames,
102}
103
104impl ModuleBasedSymbolRegistry {
105    const PROBLEMATIC_PREFIXES: &[&'static str] = &[
106        ".L",    // .L is used for local symbols
107        "llvm.", // llvm. is used for LLVM's own intrinsics
108    ];
109
110    fn new(wasm_module: Arc<ModuleInfo>) -> Self {
111        let local_func_names = HashMap::from_iter(
112            wasm_module
113                .function_names
114                .iter()
115                .map(|(f, v)| (wasm_module.local_func_index(*f), v))
116                .filter(|(f, _)| f.is_some())
117                .map(|(f, v)| (format!("{}_{}", v.clone(), f.unwrap().as_u32()), f.unwrap())),
118        );
119        Self {
120            wasm_module,
121            local_func_names,
122            short_names: ShortNames {},
123        }
124    }
125
126    // If the name starts with a problematic prefix, we prefix it with an underscore.
127    fn fixup_problematic_name(name: &str) -> Cow<'_, str> {
128        for prefix in Self::PROBLEMATIC_PREFIXES {
129            if name.starts_with(prefix) {
130                return format!("_{name}").into();
131            }
132        }
133        name.into()
134    }
135
136    // If the name starts with an underscore and the rest starts with a problematic prefix,
137    // remove the underscore to get back the original name. This is necessary to be able
138    // to match the name back to the original name in the wasm module.
139    fn unfixup_problematic_name(name: &str) -> &str {
140        if let Some(stripped_name) = name.strip_prefix('_') {
141            for prefix in Self::PROBLEMATIC_PREFIXES {
142                if stripped_name.starts_with(prefix) {
143                    return stripped_name;
144                }
145            }
146        }
147
148        name
149    }
150}
151
152impl SymbolRegistry for ModuleBasedSymbolRegistry {
153    fn symbol_to_name(&self, symbol: Symbol) -> String {
154        match symbol {
155            Symbol::LocalFunction(index) => self
156                .wasm_module
157                .function_names
158                .get(&self.wasm_module.func_index(index))
159                .map(|name| format!("{}_{}", Self::fixup_problematic_name(name), index.as_u32()))
160                .unwrap_or(self.short_names.symbol_to_name(symbol)),
161            _ => self.short_names.symbol_to_name(symbol),
162        }
163    }
164
165    fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
166        let name = Self::unfixup_problematic_name(name);
167        if let Some(idx) = self.local_func_names.get(name) {
168            Some(Symbol::LocalFunction(*idx))
169        } else {
170            self.short_names.name_to_symbol(name)
171        }
172    }
173}
174
175impl Compiler for LLVMCompiler {
176    fn name(&self) -> &str {
177        "llvm"
178    }
179
180    fn get_perfmap_enabled(&self) -> bool {
181        self.config.enable_perfmap
182    }
183
184    fn get_debugger(&self) -> Option<wasmer_compiler::Debugger> {
185        self.config.debugger
186    }
187
188    fn deterministic_id(&self) -> String {
189        use wasmer_compiler::DeterministicIdComponent as Component;
190
191        let mut components = vec![Component::Llvm];
192        components.push(match self.config.opt_level {
193            inkwell::OptimizationLevel::None => Component::OptNone,
194            inkwell::OptimizationLevel::Less => Component::OptLess,
195            inkwell::OptimizationLevel::Default => Component::OptDefault,
196            inkwell::OptimizationLevel::Aggressive => Component::OptAggressive,
197        });
198        if self.config.enable_nan_canonicalization {
199            components.push(Component::NanCanonicalization);
200        }
201        if self.config.enable_non_volatile_memops {
202            components.push(Component::NonVolatileMemops);
203        }
204        if self.config.is_pic {
205            components.push(Component::Pic);
206        }
207        if self.config.enable_readonly_funcref_table {
208            components.push(Component::ReadonlyFuncrefTable);
209        }
210
211        components
212            .into_iter()
213            .map(|component| component.to_string())
214            .collect_vec()
215            .join("-")
216    }
217
218    fn artifact_format(&self) -> String {
219        if self.config.experimental_artifact {
220            wasmer_compiler::ArtifactFormat::Native
221        } else {
222            wasmer_compiler::ArtifactFormat::Rkyv
223        }
224        .to_string()
225    }
226
227    /// Get the middlewares for this compiler
228    fn get_middlewares(&self) -> &[Arc<dyn ModuleMiddleware>] {
229        &self.config.middlewares
230    }
231
232    fn enable_readonly_funcref_table(&self) -> bool {
233        self.config.enable_readonly_funcref_table
234    }
235
236    /// Compile the module using LLVM, producing a compilation result with
237    /// associated relocations.
238    fn compile_module(
239        &self,
240        target: &Target,
241        compile_info: &CompileModuleInfo,
242        compile_info_blob: &[u8],
243        module_translation: &ModuleTranslationState,
244        function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
245        progress_callback: Option<&CompilationProgressCallback>,
246    ) -> Result<Compilation, CompileError> {
247        let function_max_stack_usage = function_body_inputs.iter().map(|_| None).collect();
248        let binary_format = self.config.target_binary_format(target);
249
250        let module = &compile_info.module;
251        let module_hash = module.hash_string();
252
253        let total_function_call_trampolines = module.signatures.len();
254        let total_dynamic_trampolines = module.num_imported_functions;
255        let total_steps = WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE
256            * ((total_dynamic_trampolines + total_function_call_trampolines) as u64)
257            + function_body_inputs
258                .iter()
259                .map(|(_, body)| body.data.len() as u64)
260                .sum::<u64>();
261
262        let progress = progress_callback
263            .cloned()
264            .map(|cb| ProgressContext::new(cb, total_steps, "Compiling functions"));
265
266        // TODO: merge constants in sections.
267
268        let mut module_custom_sections = PrimaryMap::new();
269
270        let mut eh_frame_section_bytes = vec![];
271        let mut eh_frame_section_relocations = vec![];
272
273        let mut compact_unwind_section_bytes = vec![];
274        let mut compact_unwind_section_relocations = vec![];
275
276        let mut got_targets: HashSet<wasmer_compiler::types::relocation::RelocationTarget> = if matches!(
277            target.triple().binary_format,
278            target_lexicon::BinaryFormat::Macho
279        ) {
280            HashSet::from_iter(vec![RelocationTarget::LibCall(LibCall::EHPersonality)])
281        } else {
282            HashSet::default()
283        };
284
285        let symbol_registry = ModuleBasedSymbolRegistry::new(module.clone());
286        let module = &compile_info.module;
287        let memory_styles = &compile_info.memory_styles;
288        let table_styles = &compile_info.table_styles;
289        let signature_hashes = &module.signature_hashes;
290
291        let pool = ThreadPoolBuilder::new()
292            .num_threads(self.config.num_threads.get())
293            .build()
294            .map_err(|e| CompileError::Resource(e.to_string()))?;
295
296        let source_map = Arc::new(if self.config.experimental_artifact {
297            WasmSourceMap::new(module, module_translation, &function_body_inputs)
298                .map_err(CompileError::Codegen)?
299        } else {
300            WasmSourceMap::default()
301        });
302        let buckets =
303            build_function_buckets(&function_body_inputs, WASM_LARGE_FUNCTION_THRESHOLD / 3);
304        let largest_bucket = buckets.first().map(|b| b.size).unwrap_or_default();
305        tracing::debug!(buckets = buckets.len(), largest_bucket, "buckets built");
306
307        let functions = translate_function_buckets(
308            &pool,
309            || {
310                let compiler = &self;
311                let target_machines = enum_iterator::all::<OptimizationStyle>()
312                    .map(|style| {
313                        (
314                            style,
315                            compiler.config().target_machine_with_opt(target, style),
316                        )
317                    })
318                    .collect();
319                let pointer_width = target.triple().pointer_width().unwrap().bytes();
320                FuncTranslator::new(
321                    target.triple().clone(),
322                    target_machines,
323                    binary_format,
324                    pointer_width,
325                    *target.cpu_features(),
326                    self.config.enable_non_volatile_memops,
327                    source_map.clone(),
328                    module
329                        .exports
330                        .get("__wasm_apply_data_relocs")
331                        .and_then(|export| {
332                            if let ExportIndex::Function(index) = export {
333                                Some(*index)
334                            } else {
335                                None
336                            }
337                        }),
338                )
339                .unwrap()
340            },
341            |func_translator, i, input| {
342                func_translator.translate(
343                    module,
344                    module_translation,
345                    signature_hashes,
346                    i,
347                    input,
348                    self.config(),
349                    memory_styles,
350                    table_styles,
351                    &symbol_registry,
352                    target.triple(),
353                )
354            },
355            progress.clone(),
356            &buckets,
357        )?;
358
359        let progress = progress.clone();
360        let function_call_trampolines = pool.install(|| {
361            module
362                .signatures
363                .iter()
364                .collect::<Vec<_>>()
365                .par_iter()
366                .map_init(
367                    || {
368                        let target_machine = self.config().target_machine(target);
369                        FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
370                            .unwrap()
371                    },
372                    |func_trampoline, (sig_index, sig)| {
373                        let kind = wasmer_compiler::misc::CompiledKind::FunctionCallTrampoline(
374                            *sig_index,
375                            (*sig).clone(),
376                        );
377                        let trampoline =
378                            func_trampoline.trampoline(sig, self.config(), &kind, compile_info);
379                        if let Some(progress) = progress.as_ref() {
380                            progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
381                        }
382                        trampoline
383                    },
384                )
385                .collect::<Result<Vec<_>, _>>()
386        })?;
387
388        // TODO: I removed the parallel processing of dynamic trampolines because we're passing
389        // the sections bytes and relocations directly into the trampoline generation function.
390        // We can move that logic out and re-enable parallel processing. Hopefully, there aren't
391        // enough dynamic trampolines to actually cause a noticeable performance degradation.
392        let dynamic_function_trampolines = {
393            let progress = progress.clone();
394            let target_machine = self.config().target_machine(target);
395            let func_trampoline =
396                FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
397                    .unwrap();
398            module
399                .imported_function_types()
400                .collect::<Vec<_>>()
401                .into_iter()
402                .enumerate()
403                .map(|(index, func_type)| {
404                    let kind = wasmer_compiler::misc::CompiledKind::DynamicFunctionTrampoline(
405                        FunctionIndex::from_u32(index as u32),
406                        func_type.clone(),
407                    );
408                    let trampoline = func_trampoline.dynamic_trampoline(
409                        &func_type,
410                        self.config(),
411                        &kind,
412                        index as u32,
413                        &mut module_custom_sections,
414                        &mut eh_frame_section_bytes,
415                        &mut eh_frame_section_relocations,
416                        &mut compact_unwind_section_bytes,
417                        &mut compact_unwind_section_relocations,
418                        &module_hash,
419                    )?;
420                    if let Some(progress) = progress.as_ref() {
421                        progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
422                    }
423                    Ok(trampoline)
424                })
425                .collect::<Result<Vec<_>, CompileError>>()?
426        };
427
428        if self.config.experimental_artifact {
429            let object_files = functions
430                .into_iter()
431                .map(|compiled_function| match compiled_function {
432                    CompiledFunction::Elf(path) => path,
433                    CompiledFunction::Rkyv(_) => {
434                        unreachable!()
435                    }
436                })
437                .collect::<Vec<Vec<u8>>>();
438            let trampolines_objects = function_call_trampolines
439                .into_iter()
440                .map(|f| match f {
441                    CompiledFunctionBody::Elf(path) => path,
442                    CompiledFunctionBody::Rkyv(_) => {
443                        unreachable!()
444                    }
445                })
446                .collect::<Vec<Vec<u8>>>();
447            let dynamic_trampolines_objects = dynamic_function_trampolines
448                .into_iter()
449                .map(|f| match f {
450                    CompiledFunctionBody::Elf(path) => path,
451                    CompiledFunctionBody::Rkyv(_) => unreachable!(),
452                })
453                .collect::<Vec<Vec<u8>>>();
454
455            let elf_content = emit_metadata_and_link(
456                &pool,
457                target,
458                compile_info_blob,
459                CompiledObjects {
460                    object_files,
461                    import_trampoline_object_files: Vec::new(),
462                    trampoline_object_files: trampolines_objects,
463                    dynamic_trampoline_object_files: dynamic_trampolines_objects,
464                },
465                self.config
466                    .callbacks
467                    .as_ref()
468                    .map(|callbacks| callbacks.debug_dir().clone()),
469                module.hash().map(|hash| hash.to_string()),
470            )?;
471            Ok(Compilation::Elf {
472                data: elf_content,
473                function_max_stack_usage,
474            })
475        } else {
476            let functions = functions
477                .into_iter()
478                .map(|compiled_function| {
479                    let CompiledFunction::Rkyv(mut compiled_function) = compiled_function else {
480                        unreachable!()
481                    };
482
483                    let first_section = module_custom_sections.len() as u32;
484                    for (section_index, custom_section) in compiled_function.custom_sections.iter()
485                    {
486                        // TODO: remove this call to clone()
487                        let mut custom_section = custom_section.clone();
488                        for reloc in &mut custom_section.relocations {
489                            if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
490                                reloc.reloc_target = RelocationTarget::CustomSection(
491                                    SectionIndex::from_u32(first_section + index.as_u32()),
492                                )
493                            }
494
495                            if reloc.kind.needs_got() {
496                                got_targets.insert(reloc.reloc_target);
497                            }
498                        }
499
500                        if compiled_function
501                            .eh_frame_section_indices
502                            .contains(&section_index)
503                        {
504                            let offset = eh_frame_section_bytes.len() as u32;
505                            for reloc in &mut custom_section.relocations {
506                                reloc.offset += offset;
507                            }
508                            eh_frame_section_bytes
509                                .extend_from_slice(custom_section.bytes.as_slice());
510                            // Terminate the eh_frame info with a zero-length CIE.
511                            eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
512                            eh_frame_section_relocations.extend(custom_section.relocations);
513                            // TODO: we do this to keep the count right, remove it.
514                            module_custom_sections.push(CustomSection {
515                                protection: CustomSectionProtection::Read,
516                                alignment: None,
517                                bytes: SectionBody::new_with_vec(vec![]),
518                                relocations: vec![],
519                            });
520                        } else if compiled_function
521                            .compact_unwind_section_indices
522                            .contains(&section_index)
523                        {
524                            let offset = compact_unwind_section_bytes.len() as u32;
525                            for reloc in &mut custom_section.relocations {
526                                reloc.offset += offset;
527                            }
528                            compact_unwind_section_bytes
529                                .extend_from_slice(custom_section.bytes.as_slice());
530                            compact_unwind_section_relocations.extend(custom_section.relocations);
531                            // TODO: we do this to keep the count right, remove it.
532                            module_custom_sections.push(CustomSection {
533                                protection: CustomSectionProtection::Read,
534                                alignment: None,
535                                bytes: SectionBody::new_with_vec(vec![]),
536                                relocations: vec![],
537                            });
538                        } else {
539                            module_custom_sections.push(custom_section);
540                        }
541                    }
542                    for reloc in &mut compiled_function.compiled_function.relocations {
543                        if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
544                            reloc.reloc_target = RelocationTarget::CustomSection(
545                                SectionIndex::from_u32(first_section + index.as_u32()),
546                            )
547                        }
548
549                        if reloc.kind.needs_got() {
550                            got_targets.insert(reloc.reloc_target);
551                        }
552                    }
553                    compiled_function.compiled_function
554                })
555                .collect::<PrimaryMap<LocalFunctionIndex, _>>();
556
557            let mut unwind_info = UnwindInfo::default();
558
559            if !eh_frame_section_bytes.is_empty() {
560                let eh_frame_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
561                module_custom_sections.push(CustomSection {
562                    protection: CustomSectionProtection::Read,
563                    alignment: None,
564                    bytes: SectionBody::new_with_vec(eh_frame_section_bytes),
565                    relocations: eh_frame_section_relocations,
566                });
567                unwind_info.eh_frame = Some(eh_frame_idx);
568            }
569
570            if !compact_unwind_section_bytes.is_empty() {
571                let cu_index = SectionIndex::from_u32(module_custom_sections.len() as u32);
572                module_custom_sections.push(CustomSection {
573                    protection: CustomSectionProtection::Read,
574                    alignment: None,
575                    bytes: SectionBody::new_with_vec(compact_unwind_section_bytes),
576                    relocations: compact_unwind_section_relocations,
577                });
578                unwind_info.compact_unwind = Some(cu_index);
579            }
580
581            let mut got = wasmer_compiler::types::function::GOT::empty();
582
583            if !got_targets.is_empty() {
584                let got_data: Vec<u8> = vec![0; got_targets.len() * 8];
585                let mut got_relocs = vec![];
586
587                for (i, target) in got_targets.into_iter().enumerate() {
588                    got_relocs.push(wasmer_compiler::types::relocation::Relocation {
589                        kind: RelocationKind::Abs8,
590                        reloc_target: target,
591                        offset: (i * 8) as u32,
592                        addend: 0,
593                    });
594                }
595
596                let got_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
597                module_custom_sections.push(CustomSection {
598                    protection: CustomSectionProtection::Read,
599                    alignment: None,
600                    bytes: SectionBody::new_with_vec(got_data),
601                    relocations: got_relocs,
602                });
603                got.index = Some(got_idx);
604            };
605
606            let function_call_trampolines = function_call_trampolines
607                .into_iter()
608                .map(|f| {
609                    let CompiledFunctionBody::Rkyv(function) = f else {
610                        unreachable!()
611                    };
612                    function
613                })
614                .collect();
615            let dynamic_function_trampolines = dynamic_function_trampolines
616                .into_iter()
617                .map(|f| {
618                    let CompiledFunctionBody::Rkyv(function) = f else {
619                        unreachable!()
620                    };
621                    function
622                })
623                .collect();
624
625            Ok(Compilation::Rkyv {
626                compilation: RkyvCompilation {
627                    functions,
628                    custom_sections: module_custom_sections,
629                    function_call_trampolines,
630                    dynamic_function_trampolines,
631                    unwind_info,
632                    got,
633                },
634                function_max_stack_usage,
635            })
636        }
637    }
638
639    fn with_opts(
640        &mut self,
641        _suggested_compiler_opts: &wasmer_types::target::UserCompilerOptimizations,
642    ) -> Result<(), CompileError> {
643        Ok(())
644    }
645}
wasmer_compiler_llvm/compiler.rs

wasmer_compiler_llvm/
compiler.rs
