wasmer_compiler_llvm/
compiler.rs

1use crate::config::LLVM;
2use crate::config::OptimizationStyle;
3use crate::object_file::CompiledFunction;
4use crate::translator::FuncTrampoline;
5use crate::translator::FuncTranslator;
6use rayon::ThreadPoolBuilder;
7use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
8use std::io::Read;
9use std::{
10    borrow::Cow,
11    collections::{HashMap, HashSet},
12    path::PathBuf,
13    sync::Arc,
14};
15use tempfile::{NamedTempFile, tempdir};
16use wasmer_compiler::progress::ProgressContext;
17use wasmer_compiler::types::function::Compilation;
18use wasmer_compiler::types::function::CompiledFunctionBody;
19use wasmer_compiler::types::function::{RkyvCompilation, UnwindInfo};
20use wasmer_compiler::types::module::CompileModuleInfo;
21use wasmer_compiler::types::relocation::RelocationKind;
22use wasmer_compiler::{
23    CompiledObjects, Compiler, FunctionBodyData, ModuleMiddleware, ModuleTranslationState,
24    emit_metadata_and_link,
25    types::{
26        relocation::RelocationTarget,
27        section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
28        symbols::{Symbol, SymbolRegistry},
29    },
30};
31use wasmer_compiler::{
32    WASM_LARGE_FUNCTION_THRESHOLD, WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE, build_function_buckets,
33    translate_function_buckets,
34};
35use wasmer_types::ExportIndex;
36use wasmer_types::entity::{EntityRef, PrimaryMap};
37use wasmer_types::target::Target;
38use wasmer_types::{
39    CompilationProgressCallback, CompileError, FunctionIndex, LocalFunctionIndex, ModuleInfo,
40    SignatureIndex,
41};
42use wasmer_vm::LibCall;
43
44/// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR,
45/// optimizing it and then translating to assembly.
46#[derive(Debug)]
47pub struct LLVMCompiler {
48    config: LLVM,
49}
50
51impl LLVMCompiler {
52    /// Creates a new LLVM compiler
53    pub fn new(config: LLVM) -> LLVMCompiler {
54        LLVMCompiler { config }
55    }
56
57    /// Gets the config for this Compiler
58    fn config(&self) -> &LLVM {
59        &self.config
60    }
61}
62
63struct ShortNames {}
64
65impl SymbolRegistry for ShortNames {
66    fn symbol_to_name(&self, symbol: Symbol) -> String {
67        match symbol {
68            Symbol::Metadata => "M".to_string(),
69            Symbol::LocalFunction(index) => format!("f{}", index.index()),
70            Symbol::Section(index) => format!("s{}", index.index()),
71            Symbol::FunctionCallTrampoline(index) => format!("t{}", index.index()),
72            Symbol::DynamicFunctionTrampoline(index) => format!("d{}", index.index()),
73        }
74    }
75
76    fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
77        if name.len() < 2 {
78            return None;
79        }
80        let (ty, idx) = name.split_at(1);
81        if ty.starts_with('M') {
82            return Some(Symbol::Metadata);
83        }
84
85        let idx = idx.parse::<u32>().ok()?;
86        match ty.chars().next().unwrap() {
87            'f' => Some(Symbol::LocalFunction(LocalFunctionIndex::from_u32(idx))),
88            's' => Some(Symbol::Section(SectionIndex::from_u32(idx))),
89            't' => Some(Symbol::FunctionCallTrampoline(SignatureIndex::from_u32(
90                idx,
91            ))),
92            'd' => Some(Symbol::DynamicFunctionTrampoline(FunctionIndex::from_u32(
93                idx,
94            ))),
95            _ => None,
96        }
97    }
98}
99
100pub(crate) struct ModuleBasedSymbolRegistry {
101    wasm_module: Arc<ModuleInfo>,
102    local_func_names: HashMap<String, LocalFunctionIndex>,
103    short_names: ShortNames,
104}
105
106impl ModuleBasedSymbolRegistry {
107    const PROBLEMATIC_PREFIXES: &[&'static str] = &[
108        ".L",    // .L is used for local symbols
109        "llvm.", // llvm. is used for LLVM's own intrinsics
110    ];
111
112    fn new(wasm_module: Arc<ModuleInfo>) -> Self {
113        let local_func_names = HashMap::from_iter(
114            wasm_module
115                .function_names
116                .iter()
117                .map(|(f, v)| (wasm_module.local_func_index(*f), v))
118                .filter(|(f, _)| f.is_some())
119                .map(|(f, v)| (format!("{}_{}", v.clone(), f.unwrap().as_u32()), f.unwrap())),
120        );
121        Self {
122            wasm_module,
123            local_func_names,
124            short_names: ShortNames {},
125        }
126    }
127
128    // If the name starts with a problematic prefix, we prefix it with an underscore.
129    fn fixup_problematic_name(name: &str) -> Cow<'_, str> {
130        for prefix in Self::PROBLEMATIC_PREFIXES {
131            if name.starts_with(prefix) {
132                return format!("_{name}").into();
133            }
134        }
135        name.into()
136    }
137
138    // If the name starts with an underscore and the rest starts with a problematic prefix,
139    // remove the underscore to get back the original name. This is necessary to be able
140    // to match the name back to the original name in the wasm module.
141    fn unfixup_problematic_name(name: &str) -> &str {
142        if let Some(stripped_name) = name.strip_prefix('_') {
143            for prefix in Self::PROBLEMATIC_PREFIXES {
144                if stripped_name.starts_with(prefix) {
145                    return stripped_name;
146                }
147            }
148        }
149
150        name
151    }
152}
153
154impl SymbolRegistry for ModuleBasedSymbolRegistry {
155    fn symbol_to_name(&self, symbol: Symbol) -> String {
156        match symbol {
157            Symbol::LocalFunction(index) => self
158                .wasm_module
159                .function_names
160                .get(&self.wasm_module.func_index(index))
161                .map(|name| format!("{}_{}", Self::fixup_problematic_name(name), index.as_u32()))
162                .unwrap_or(self.short_names.symbol_to_name(symbol)),
163            _ => self.short_names.symbol_to_name(symbol),
164        }
165    }
166
167    fn name_to_symbol(&self, name: &str) -> Option<Symbol> {
168        let name = Self::unfixup_problematic_name(name);
169        if let Some(idx) = self.local_func_names.get(name) {
170            Some(Symbol::LocalFunction(*idx))
171        } else {
172            self.short_names.name_to_symbol(name)
173        }
174    }
175}
176
177impl Compiler for LLVMCompiler {
178    fn name(&self) -> &str {
179        "llvm"
180    }
181
182    fn get_perfmap_enabled(&self) -> bool {
183        self.config.enable_perfmap
184    }
185
186    fn deterministic_id(&self) -> String {
187        format!(
188            "llvm-{}{}",
189            match self.config.opt_level {
190                inkwell::OptimizationLevel::None => "opt0",
191                inkwell::OptimizationLevel::Less => "optl",
192                inkwell::OptimizationLevel::Default => "optd",
193                inkwell::OptimizationLevel::Aggressive => "opta",
194            },
195            if cfg!(feature = "experimental-artifact") {
196                "-elf"
197            } else {
198                ""
199            }
200        )
201    }
202
203    /// Get the middlewares for this compiler
204    fn get_middlewares(&self) -> &[Arc<dyn ModuleMiddleware>] {
205        &self.config.middlewares
206    }
207
208    fn enable_readonly_funcref_table(&self) -> bool {
209        self.config.enable_readonly_funcref_table
210    }
211
212    /// Compile the module using LLVM, producing a compilation result with
213    /// associated relocations.
214    fn compile_module(
215        &self,
216        target: &Target,
217        compile_info: &CompileModuleInfo,
218        compile_info_blob: &[u8],
219        module_translation: &ModuleTranslationState,
220        function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
221        progress_callback: Option<&CompilationProgressCallback>,
222    ) -> Result<Compilation, CompileError> {
223        let binary_format = self.config.target_binary_format(target);
224
225        let module = &compile_info.module;
226        let module_hash = module.hash_string();
227
228        let total_function_call_trampolines = module.signatures.len();
229        let total_dynamic_trampolines = module.num_imported_functions;
230        let total_steps = WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE
231            * ((total_dynamic_trampolines + total_function_call_trampolines) as u64)
232            + function_body_inputs
233                .iter()
234                .map(|(_, body)| body.data.len() as u64)
235                .sum::<u64>();
236
237        let progress = progress_callback
238            .cloned()
239            .map(|cb| ProgressContext::new(cb, total_steps, "Compiling functions"));
240
241        // TODO: merge constants in sections.
242
243        let mut module_custom_sections = PrimaryMap::new();
244
245        let mut eh_frame_section_bytes = vec![];
246        let mut eh_frame_section_relocations = vec![];
247
248        let mut compact_unwind_section_bytes = vec![];
249        let mut compact_unwind_section_relocations = vec![];
250
251        let mut got_targets: HashSet<wasmer_compiler::types::relocation::RelocationTarget> = if matches!(
252            target.triple().binary_format,
253            target_lexicon::BinaryFormat::Macho
254        ) {
255            HashSet::from_iter(vec![RelocationTarget::LibCall(LibCall::EHPersonality)])
256        } else {
257            HashSet::default()
258        };
259
260        let symbol_registry = ModuleBasedSymbolRegistry::new(module.clone());
261        let module = &compile_info.module;
262        let memory_styles = &compile_info.memory_styles;
263        let table_styles = &compile_info.table_styles;
264        let signature_hashes = &module.signature_hashes;
265
266        let pool = ThreadPoolBuilder::new()
267            .num_threads(self.config.num_threads.get())
268            .build()
269            .map_err(|e| CompileError::Resource(e.to_string()))?;
270
271        let buckets =
272            build_function_buckets(&function_body_inputs, WASM_LARGE_FUNCTION_THRESHOLD / 3);
273        let largest_bucket = buckets.first().map(|b| b.size).unwrap_or_default();
274        tracing::debug!(buckets = buckets.len(), largest_bucket, "buckets built");
275
276        let build_directory = tempdir().map_err(|err| {
277            CompileError::Codegen(format!("cannot create temporary build folder: {err}"))
278        })?;
279        let functions = translate_function_buckets(
280            &pool,
281            || {
282                let compiler = &self;
283                let target_machines = enum_iterator::all::<OptimizationStyle>()
284                    .map(|style| {
285                        (
286                            style,
287                            compiler.config().target_machine_with_opt(target, style),
288                        )
289                    })
290                    .collect();
291                let pointer_width = target.triple().pointer_width().unwrap().bytes();
292                FuncTranslator::new(
293                    target.triple().clone(),
294                    target_machines,
295                    binary_format,
296                    pointer_width,
297                    *target.cpu_features(),
298                    self.config.enable_non_volatile_memops,
299                    module
300                        .exports
301                        .get("__wasm_apply_data_relocs")
302                        .and_then(|export| {
303                            if let ExportIndex::Function(index) = export {
304                                Some(*index)
305                            } else {
306                                None
307                            }
308                        }),
309                )
310                .unwrap()
311            },
312            |func_translator, i, input| {
313                func_translator.translate(
314                    module,
315                    module_translation,
316                    signature_hashes,
317                    i,
318                    input,
319                    self.config(),
320                    memory_styles,
321                    table_styles,
322                    &symbol_registry,
323                    target.triple(),
324                    build_directory.path(),
325                )
326            },
327            progress.clone(),
328            &buckets,
329        )?;
330
331        let progress = progress.clone();
332        let function_call_trampolines = pool.install(|| {
333            module
334                .signatures
335                .iter()
336                .collect::<Vec<_>>()
337                .par_iter()
338                .map_init(
339                    || {
340                        let target_machine = self.config().target_machine(target);
341                        FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
342                            .unwrap()
343                    },
344                    |func_trampoline, (sig_index, sig)| {
345                        let kind = wasmer_compiler::misc::CompiledKind::FunctionCallTrampoline(
346                            *sig_index,
347                            (*sig).clone(),
348                        );
349                        let trampoline = func_trampoline.trampoline(
350                            sig,
351                            self.config(),
352                            &kind,
353                            compile_info,
354                            build_directory.path(),
355                        );
356                        if let Some(progress) = progress.as_ref() {
357                            progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
358                        }
359                        trampoline
360                    },
361                )
362                .collect::<Result<Vec<_>, _>>()
363        })?;
364
365        // TODO: I removed the parallel processing of dynamic trampolines because we're passing
366        // the sections bytes and relocations directly into the trampoline generation function.
367        // We can move that logic out and re-enable parallel processing. Hopefully, there aren't
368        // enough dynamic trampolines to actually cause a noticeable performance degradation.
369        let dynamic_function_trampolines = {
370            let progress = progress.clone();
371            let target_machine = self.config().target_machine(target);
372            let func_trampoline =
373                FuncTrampoline::new(target_machine, target.triple().clone(), binary_format)
374                    .unwrap();
375            module
376                .imported_function_types()
377                .collect::<Vec<_>>()
378                .into_iter()
379                .enumerate()
380                .map(|(index, func_type)| {
381                    let kind = wasmer_compiler::misc::CompiledKind::DynamicFunctionTrampoline(
382                        FunctionIndex::from_u32(index as u32),
383                        func_type.clone(),
384                    );
385                    let trampoline = func_trampoline.dynamic_trampoline(
386                        &func_type,
387                        self.config(),
388                        &kind,
389                        index as u32,
390                        &mut module_custom_sections,
391                        &mut eh_frame_section_bytes,
392                        &mut eh_frame_section_relocations,
393                        &mut compact_unwind_section_bytes,
394                        &mut compact_unwind_section_relocations,
395                        &module_hash,
396                        build_directory.path(),
397                    )?;
398                    if let Some(progress) = progress.as_ref() {
399                        progress.notify_steps(WASM_TRAMPOLINE_ESTIMATED_BODY_SIZE)?;
400                    }
401                    Ok(trampoline)
402                })
403                .collect::<Result<Vec<_>, CompileError>>()?
404        };
405
406        if cfg!(feature = "experimental-artifact") {
407            let object_files = functions
408                .into_iter()
409                .map(|compiled_function| match compiled_function {
410                    CompiledFunction::Elf(path) => path,
411                    CompiledFunction::Rkyv(_) => {
412                        unreachable!()
413                    }
414                })
415                .collect::<Vec<PathBuf>>();
416            let trampolines_objects = function_call_trampolines
417                .into_iter()
418                .map(|f| match f {
419                    CompiledFunctionBody::Elf(path) => path,
420                    CompiledFunctionBody::Rkyv(_) => {
421                        unreachable!()
422                    }
423                })
424                .collect::<Vec<PathBuf>>();
425            let dynamic_trampolines_objects = dynamic_function_trampolines
426                .into_iter()
427                .map(|f| match f {
428                    CompiledFunctionBody::Elf(path) => path,
429                    CompiledFunctionBody::Rkyv(_) => unreachable!(),
430                })
431                .collect::<Vec<PathBuf>>();
432
433            let module_file = NamedTempFile::new_in(build_directory.path()).map_err(|e| {
434                CompileError::Codegen(format!("cannot create temporary module file: {e}"))
435            })?;
436
437            let mut module_file = emit_metadata_and_link(
438                target,
439                compile_info_blob,
440                build_directory.path(),
441                module_file,
442                &CompiledObjects {
443                    object_files: &object_files,
444                    import_trampoline_object_files: &[],
445                    trampoline_object_files: &trampolines_objects,
446                    dynamic_trampoline_object_files: &dynamic_trampolines_objects,
447                },
448                self.config
449                    .callbacks
450                    .as_ref()
451                    .map(|callbacks| callbacks.debug_dir().clone()),
452                module.hash().map(|hash| hash.to_string()),
453            )?;
454
455            let mut elf_content = Vec::new();
456            module_file.read_to_end(&mut elf_content).map_err(|e| {
457                CompileError::Codegen(format!("cannot persist linked shared object: {e}"))
458            })?;
459            Ok(Compilation::Elf(elf_content))
460        } else {
461            let functions = functions
462                .into_iter()
463                .map(|compiled_function| {
464                    let CompiledFunction::Rkyv(mut compiled_function) = compiled_function else {
465                        unreachable!()
466                    };
467
468                    let first_section = module_custom_sections.len() as u32;
469                    for (section_index, custom_section) in compiled_function.custom_sections.iter()
470                    {
471                        // TODO: remove this call to clone()
472                        let mut custom_section = custom_section.clone();
473                        for reloc in &mut custom_section.relocations {
474                            if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
475                                reloc.reloc_target = RelocationTarget::CustomSection(
476                                    SectionIndex::from_u32(first_section + index.as_u32()),
477                                )
478                            }
479
480                            if reloc.kind.needs_got() {
481                                got_targets.insert(reloc.reloc_target);
482                            }
483                        }
484
485                        if compiled_function
486                            .eh_frame_section_indices
487                            .contains(&section_index)
488                        {
489                            let offset = eh_frame_section_bytes.len() as u32;
490                            for reloc in &mut custom_section.relocations {
491                                reloc.offset += offset;
492                            }
493                            eh_frame_section_bytes
494                                .extend_from_slice(custom_section.bytes.as_slice());
495                            // Terminate the eh_frame info with a zero-length CIE.
496                            eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
497                            eh_frame_section_relocations.extend(custom_section.relocations);
498                            // TODO: we do this to keep the count right, remove it.
499                            module_custom_sections.push(CustomSection {
500                                protection: CustomSectionProtection::Read,
501                                alignment: None,
502                                bytes: SectionBody::new_with_vec(vec![]),
503                                relocations: vec![],
504                            });
505                        } else if compiled_function
506                            .compact_unwind_section_indices
507                            .contains(&section_index)
508                        {
509                            let offset = compact_unwind_section_bytes.len() as u32;
510                            for reloc in &mut custom_section.relocations {
511                                reloc.offset += offset;
512                            }
513                            compact_unwind_section_bytes
514                                .extend_from_slice(custom_section.bytes.as_slice());
515                            compact_unwind_section_relocations.extend(custom_section.relocations);
516                            // TODO: we do this to keep the count right, remove it.
517                            module_custom_sections.push(CustomSection {
518                                protection: CustomSectionProtection::Read,
519                                alignment: None,
520                                bytes: SectionBody::new_with_vec(vec![]),
521                                relocations: vec![],
522                            });
523                        } else {
524                            module_custom_sections.push(custom_section);
525                        }
526                    }
527                    for reloc in &mut compiled_function.compiled_function.relocations {
528                        if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
529                            reloc.reloc_target = RelocationTarget::CustomSection(
530                                SectionIndex::from_u32(first_section + index.as_u32()),
531                            )
532                        }
533
534                        if reloc.kind.needs_got() {
535                            got_targets.insert(reloc.reloc_target);
536                        }
537                    }
538                    compiled_function.compiled_function
539                })
540                .collect::<PrimaryMap<LocalFunctionIndex, _>>();
541
542            let mut unwind_info = UnwindInfo::default();
543
544            if !eh_frame_section_bytes.is_empty() {
545                let eh_frame_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
546                module_custom_sections.push(CustomSection {
547                    protection: CustomSectionProtection::Read,
548                    alignment: None,
549                    bytes: SectionBody::new_with_vec(eh_frame_section_bytes),
550                    relocations: eh_frame_section_relocations,
551                });
552                unwind_info.eh_frame = Some(eh_frame_idx);
553            }
554
555            if !compact_unwind_section_bytes.is_empty() {
556                let cu_index = SectionIndex::from_u32(module_custom_sections.len() as u32);
557                module_custom_sections.push(CustomSection {
558                    protection: CustomSectionProtection::Read,
559                    alignment: None,
560                    bytes: SectionBody::new_with_vec(compact_unwind_section_bytes),
561                    relocations: compact_unwind_section_relocations,
562                });
563                unwind_info.compact_unwind = Some(cu_index);
564            }
565
566            let mut got = wasmer_compiler::types::function::GOT::empty();
567
568            if !got_targets.is_empty() {
569                let pointer_width = target.triple().pointer_width().map_err(|_| {
570                    CompileError::Codegen("Could not get pointer width".to_string())
571                })?;
572
573                let got_entry_size = match pointer_width {
574                    target_lexicon::PointerWidth::U64 => 8,
575                    target_lexicon::PointerWidth::U32 => 4,
576                    target_lexicon::PointerWidth::U16 => todo!(),
577                };
578
579                let got_entry_reloc_kind = match pointer_width {
580                    target_lexicon::PointerWidth::U64 => RelocationKind::Abs8,
581                    target_lexicon::PointerWidth::U32 => RelocationKind::Abs4,
582                    target_lexicon::PointerWidth::U16 => todo!(),
583                };
584
585                let got_data: Vec<u8> = vec![0; got_targets.len() * got_entry_size];
586                let mut got_relocs = vec![];
587
588                for (i, target) in got_targets.into_iter().enumerate() {
589                    got_relocs.push(wasmer_compiler::types::relocation::Relocation {
590                        kind: got_entry_reloc_kind,
591                        reloc_target: target,
592                        offset: (i * got_entry_size) as u32,
593                        addend: 0,
594                    });
595                }
596
597                let got_idx = SectionIndex::from_u32(module_custom_sections.len() as u32);
598                module_custom_sections.push(CustomSection {
599                    protection: CustomSectionProtection::Read,
600                    alignment: None,
601                    bytes: SectionBody::new_with_vec(got_data),
602                    relocations: got_relocs,
603                });
604                got.index = Some(got_idx);
605            };
606
607            let function_call_trampolines = function_call_trampolines
608                .into_iter()
609                .map(|f| {
610                    let CompiledFunctionBody::Rkyv(function) = f else {
611                        unreachable!()
612                    };
613                    function
614                })
615                .collect();
616            let dynamic_function_trampolines = dynamic_function_trampolines
617                .into_iter()
618                .map(|f| {
619                    let CompiledFunctionBody::Rkyv(function) = f else {
620                        unreachable!()
621                    };
622                    function
623                })
624                .collect();
625
626            Ok(Compilation::Rkyv(RkyvCompilation {
627                functions,
628                custom_sections: module_custom_sections,
629                function_call_trampolines,
630                dynamic_function_trampolines,
631                unwind_info,
632                got,
633            }))
634        }
635    }
636
637    fn with_opts(
638        &mut self,
639        _suggested_compiler_opts: &wasmer_types::target::UserCompilerOptimizations,
640    ) -> Result<(), CompileError> {
641        Ok(())
642    }
643}
wasmer_compiler_llvm/compiler.rs

wasmer_compiler_llvm/
compiler.rs
