wasmer_compiler/engine/

artifact.rs

//! Define `Artifact`, based on `ArtifactBuild`
//! to allow compiling and instantiating to be done as separate steps.

use std::{
    fs::File,
    io::BufReader,
    mem::size_of,
    path::{Path, PathBuf},
    sync::{
        Arc,
        atomic::{AtomicUsize, Ordering::SeqCst},
    },
};

#[cfg(feature = "compiler")]
use crate::ModuleEnvironment;
use crate::{
    ArtifactBuild, ArtifactBuildFromArchive, ArtifactCreate, Engine, EngineInner, Features,
    FrameInfosVariant, FunctionExtent, GlobalFrameInfoRegistration, InstantiationError, Tunables,
    WASMER_TRAP_FUNCTION_OFFSETS_SECTION_NAME, WASMER_TRAPS_SECTION_NAME,
    engine::{link::link_module, resolver::resolve_tags, trap::register_frame_info_source},
    resolve_imports,
    serialize::{MetadataHeader, SerializableCompilation, SerializableModule},
    types::relocation::{RelocationLike, RelocationTarget},
};
#[cfg(feature = "static-artifact-create")]
use crate::{Compiler, FunctionBodyData, ModuleTranslationState, types::module::CompileModuleInfo};
#[cfg(any(feature = "static-artifact-create", feature = "static-artifact-load"))]
use crate::{serialize::RkyvSerializableCompilation, types::symbols::ModuleMetadata};
use itertools::Itertools;
#[cfg(unix)]
use std::os::fd::AsRawFd;
use std::vec::IntoIter;
#[cfg(feature = "compiler")]
use wasmer_types::CompilationProgressCallback;

use enumset::EnumSet;
use object::{Object as _, ObjectSection as _, ReadCache, ReadRef};
use shared_buffer::OwnedBuffer;

use crate::engine::mapped_binary::DebugInfoSource;

#[cfg(any(feature = "static-artifact-create", feature = "static-artifact-load"))]
use std::mem;

#[cfg(feature = "static-artifact-create")]
use crate::object::{
    Object, ObjectMetadataBuilder, emit_compilation, emit_data, get_object_for_target,
};

use wasmer_types::{
    ArchivedDataInitializerLocation, ArchivedOwnedDataInitializer, CompileError, DataInitializer,
    DataInitializerLike, DataInitializerLocation, DataInitializerLocationLike, DeserializeError,
    FunctionIndex, LocalFunctionIndex, MemoryIndex, ModuleInfo, OwnedDataInitializer,
    SerializeError, SignatureIndex, TableIndex, TrapCode, TrapInformation,
    entity::{BoxedSlice, EntityRef, PrimaryMap},
    target::{CpuFeature, Target},
};

use wasmer_types::VMOffsets;
use wasmer_vm::{
    FunctionBodyPtr, InstanceAllocator, MemoryStyle, StoreObjects, TableStyle, TrapHandlerFn,
    VMConfig, VMExtern, VMInstance, VMSignatureHash, VMTrampoline,
};

#[cfg_attr(feature = "artifact-size", derive(loupe::MemoryUsage))]
pub struct AllocatedArtifact {
    // This shows if the frame info has been registered already or not.
    // Because the 'GlobalFrameInfoRegistration' ownership can be transferred to EngineInner
    // this bool is needed to track the status, as 'frame_info_registration' will be None
    // after the ownership is transferred.
    frame_info_registered: bool,
    // frame_info_registered is not staying there but transferred to CodeMemory from EngineInner
    // using 'Artifact::take_frame_info_registration' method
    // so the GloabelFrameInfo and MMap stays in sync and get dropped at the same time
    frame_info_registration: Option<GlobalFrameInfoRegistration>,
    finished_functions: BoxedSlice<LocalFunctionIndex, FunctionBodyPtr>,

    #[cfg_attr(feature = "artifact-size", loupe(skip))]
    finished_function_call_trampolines: BoxedSlice<SignatureIndex, VMTrampoline>,
    finished_dynamic_function_trampolines: BoxedSlice<FunctionIndex, FunctionBodyPtr>,
    signatures: BoxedSlice<SignatureIndex, VMSignatureHash>,
    finished_function_lengths: BoxedSlice<LocalFunctionIndex, usize>,
    // The maximum stack size used for each function (available only for the Singlepass compiler).
    function_max_stack_usage: BoxedSlice<LocalFunctionIndex, Option<usize>>,

    /// Precomputed `VMOffsets` for this artifact's module, cloned by
    /// `Artifact::instantiate` instead of recomputing on every call.
    ///
    /// Safe to cache because `VMOffsets::try_new(pointer_size, module_info)`
    /// is deterministic, `module_info` is immutable after compile (the
    /// only mutable field `name` is not a `VMOffsets` input), and the
    /// host's pointer size is a runtime constant.
    ///
    /// Built once in `from_parts` and in the deserialization path
    /// (`deserialize_object_native`); `VMOffsets::try_new` was ~9% of
    /// `Instance::new` time on profile traces of a per-request wasm
    /// host calling `Module::instantiate` in a tight loop.
    #[cfg_attr(feature = "artifact-size", loupe(skip))]
    vm_offsets: VMOffsets,

    /// The base address the module's code was loaded at, and the raw ELF
    /// image bytes it was loaded from, when this artifact was built from a
    /// native ELF image. Used to lazily build DWARF debug info for frame
    /// symbolication. `None` for non-ELF artifacts.
    #[cfg_attr(feature = "artifact-size", loupe(skip))]
    elf_image: Option<(usize, DebugInfoSource)>,
}

impl AllocatedArtifact {
    fn function_extents(&self) -> PrimaryMap<LocalFunctionIndex, FunctionExtent> {
        assert_eq!(
            self.finished_functions.len(),
            self.finished_function_lengths.len(),
            "finished_functions and finished_function_lengths must have equal length"
        );
        self.finished_functions
            .iter()
            .map(|(index, &ptr)| {
                let length = self.finished_function_lengths[index];
                FunctionExtent { ptr, length }
            })
            .collect()
    }
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
#[cfg_attr(feature = "artifact-size", derive(loupe::MemoryUsage))]
#[repr(transparent)]
/// A unique identifier for an Artifact.
pub struct ArtifactId {
    id: usize,
}

impl ArtifactId {
    /// Format this identifier as a string.
    pub fn id(&self) -> String {
        format!("{}", self.id)
    }
}

impl Clone for ArtifactId {
    fn clone(&self) -> Self {
        Self::default()
    }
}

impl Default for ArtifactId {
    fn default() -> Self {
        static NEXT_ID: AtomicUsize = AtomicUsize::new(0);
        Self {
            id: NEXT_ID.fetch_add(1, SeqCst),
        }
    }
}

/// A compiled wasm module, ready to be instantiated.
#[cfg_attr(feature = "artifact-size", derive(loupe::MemoryUsage))]
pub struct Artifact {
    id: ArtifactId,
    artifact: ArtifactBuildVariant,
    #[cfg_attr(feature = "artifact-size", loupe(skip))]
    module_file: Option<PathBuf>,
    // The artifact will only be allocated in memory in case we can execute it
    // (that means, if the target != host then this will be None).
    allocated: Option<AllocatedArtifact>,
}

/// Artifacts may be created as the result of the compilation of a wasm
/// module, corresponding to `ArtifactBuildVariant::Plain`, or loaded
/// from an archive, corresponding to `ArtifactBuildVariant::Archived`.
#[cfg_attr(feature = "artifact-size", derive(loupe::MemoryUsage))]
#[allow(clippy::large_enum_variant)]
pub enum ArtifactBuildVariant {
    Plain(ArtifactBuild),
    Archived(ArtifactBuildFromArchive),
}

impl Artifact {
    /// Compile a data buffer into a `ArtifactBuild`, which may then be instantiated.
    #[cfg(feature = "compiler")]
    pub fn new(
        engine: &Engine,
        data: &[u8],
        tunables: &dyn Tunables,
        progress_callback: Option<CompilationProgressCallback>,
    ) -> Result<Self, CompileError> {
        let mut inner_engine = engine.inner_mut();
        let environ = ModuleEnvironment::new();
        let translation = environ.translate(data).map_err(CompileError::Wasm)?;
        let module = translation.module;
        let memory_styles: PrimaryMap<MemoryIndex, MemoryStyle> = module
            .memories
            .values()
            .map(|memory_type| tunables.memory_style(memory_type))
            .collect();
        let table_styles: PrimaryMap<TableIndex, TableStyle> = module
            .tables
            .values()
            .map(|table_type| tunables.table_style(table_type))
            .collect();

        let artifact = ArtifactBuild::new(
            &mut inner_engine,
            data,
            engine.target(),
            memory_styles,
            table_styles,
            progress_callback.as_ref(),
        )?;

        Self::from_parts_with_module_file(
            &mut inner_engine,
            ArtifactBuildVariant::Plain(artifact),
            engine.target(),
            None,
        )
        .map_err(|e| match e {
            DeserializeError::Compiler(c) => c,
            DeserializeError::Generic(c) => CompileError::Resource(c),

            // `from_parts` only ever returns `CompileError`s when an
            // `ArtifactBuildVariant::Plain` is passed in. Other cases
            // of `DeserializeError` can only happen when an
            // `ArtifactBuildVariant::Archived` is passed in. We don't
            // wish to change the return type of this method because
            // a. it makes no sense and b. it would be a breaking change,
            // hence this match block and the other cases being
            // unreachable.
            _ => unreachable!(),
        })
    }

    /// This indicates if the Artifact is allocated and can be run by the current
    /// host. In case it can't be run (for example, if the artifact is cross compiled to
    /// other architecture), it will return false.
    pub fn allocated(&self) -> bool {
        self.allocated.is_some()
    }

    /// A unique identifier for this object.
    ///
    /// This exists to allow us to compare two Artifacts for equality. Otherwise,
    /// comparing two trait objects unsafely relies on implementation details
    /// of trait representation.
    pub fn id(&self) -> &ArtifactId {
        &self.id
    }

    /// Compile a data buffer into a `ArtifactBuild`, which may then be instantiated.
    #[cfg(not(feature = "compiler"))]
    pub fn new(_engine: &Engine, _data: &[u8]) -> Result<Self, CompileError> {
        Err(CompileError::Codegen(
            "Compilation is not enabled in the engine".to_string(),
        ))
    }

    /// Load a native artifact directly from a file-backed memory map.
    ///
    /// Unlike [`Self::deserialize`], this entrypoint only accepts ELF or Mach-O artifacts.
    ///
    /// # Safety
    /// This function loads executable code into memory. The file must be trusted
    /// and must not be modified while the returned artifact is in use.
    pub unsafe fn deserialize_file(
        engine: &Engine,
        path: impl AsRef<Path>,
    ) -> Result<Self, DeserializeError> {
        let path = path.as_ref().to_path_buf();
        let file = File::open(&path)?;
        let cache = ReadCache::new(BufReader::new(file));
        let image = object::File::parse(&cache)
            .map_err(|e| DeserializeError::CorruptedBinary(format!("cannot parse image: {e}")))?;
        if !matches!(
            image.format(),
            object::BinaryFormat::Elf | object::BinaryFormat::MachO
        ) {
            return Err(DeserializeError::Incompatible(
                "Artifact::deserialize_file only supports ELF or Mach-O artifacts".to_string(),
            ));
        }
        Self::check_native_image(&image)?;

        let module_info = image
            .section_by_name_bytes(crate::WASMER_MODULE_INFO_SECTION_NAME)
            .ok_or_else(|| {
                DeserializeError::CorruptedBinary("missing ModuleInfo section".to_string())
            })?
            .data()
            .map_err(|e| {
                DeserializeError::CorruptedBinary(format!(
                    "cannot load ModuleInfo section data: {e}"
                ))
            })?;
        let serializable = unsafe { SerializableModule::deserialize(module_info)? };
        if !matches!(serializable.compilation, SerializableCompilation::Elf(_)) {
            return Err(DeserializeError::Incompatible(
                "file does not contain an ELF artifact".to_string(),
            ));
        }

        let artifact = ArtifactBuildVariant::Plain(ArtifactBuild::from_serializable(serializable));
        let mut inner_engine = engine.inner_mut();
        Self::from_parts_with_module_file(&mut inner_engine, artifact, engine.target(), Some(path))
    }

    fn check_native_image<'a, R: ReadRef<'a>>(
        image: &object::File<'a, R>,
    ) -> Result<(), DeserializeError> {
        let host = crate::object::get_object_for_target(&target_lexicon::Triple::host())
            .map_err(|e| DeserializeError::Incompatible(format!("unsupported host target: {e}")))?;
        if image.format() != host.format() {
            return Err(DeserializeError::Incompatible(format!(
                "native artifact format {:?} does not match host format {:?}",
                image.format(),
                host.format()
            )));
        }
        if image.architecture() != host.architecture() {
            return Err(DeserializeError::Incompatible(format!(
                "native artifact architecture {:?} does not match host architecture {:?}",
                image.architecture(),
                host.architecture()
            )));
        }
        Ok(())
    }

    /// Deserialize an ELF artifact held in memory, if the bytes contain one.
    fn deserialize_elf(engine: &Engine, bytes: &[u8]) -> Result<Option<Self>, DeserializeError> {
        if !ArtifactBuild::has_native_header(bytes) {
            return Ok(None);
        }

        let image = object::File::parse(bytes)
            .map_err(|e| DeserializeError::CorruptedBinary(format!("cannot parse image: {e}")))?;
        Self::check_native_image(&image)?;

        let module_info = image
            .section_by_name_bytes(crate::WASMER_MODULE_INFO_SECTION_NAME)
            .ok_or_else(|| {
                DeserializeError::CorruptedBinary("missing ModuleInfo section".to_string())
            })?
            .data()
            .map_err(|e| {
                DeserializeError::CorruptedBinary(format!(
                    "cannot load ModuleInfo section data: {e}"
                ))
            })?;
        let mut serializable = unsafe { SerializableModule::deserialize(module_info)? };
        let SerializableCompilation::Elf(elf) = &mut serializable.compilation else {
            return Err(DeserializeError::Incompatible(
                "ELF image does not contain an ELF artifact".to_string(),
            ));
        };
        // The copy embedded in the image has an empty ELF placeholder to avoid
        // embedding the image in itself. Restore it for allocation and reserialization.
        *elf = bytes.to_vec();

        let artifact = ArtifactBuildVariant::Plain(ArtifactBuild::from_serializable(serializable));
        let mut inner_engine = engine.inner_mut();
        Self::from_parts(&mut inner_engine, artifact, engine.target()).map(Some)
    }

    /// Deserialize a serialized artifact.
    ///
    /// # Safety
    /// This function loads executable code into memory.
    /// You must trust the loaded bytes to be valid for the chosen engine and
    /// for the host CPU architecture.
    /// In contrast to [`Self::deserialize_unchecked`] the artifact layout is
    /// validated, which increases safety.
    pub unsafe fn deserialize(
        engine: &Engine,
        bytes: OwnedBuffer,
    ) -> Result<Self, DeserializeError> {
        unsafe {
            if !ArtifactBuild::has_rkyv_header(bytes.as_ref()) {
                if let Some(artifact) = Self::deserialize_elf(engine, bytes.as_ref())? {
                    return Ok(artifact);
                }

                let static_artifact = Self::deserialize_object(engine, bytes);
                match static_artifact {
                    Ok(v) => {
                        return Ok(v);
                    }
                    Err(e) => {
                        return Err(DeserializeError::Incompatible(format!(
                            "The provided bytes are not a Wasmer engine artifact: {e}"
                        )));
                    }
                }
            }

            let artifact = ArtifactBuildFromArchive::try_new(bytes, |bytes| {
                let bytes =
                    Self::get_byte_slice(bytes, ArtifactBuild::MAGIC_HEADER.len(), bytes.len())?;

                let metadata_len = MetadataHeader::parse(bytes)?;
                let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?;
                let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len)?;

                SerializableModule::archive_from_slice_checked(metadata_slice)
            })?;

            let mut inner_engine = engine.inner_mut();
            Self::from_parts(
                &mut inner_engine,
                ArtifactBuildVariant::Archived(artifact),
                engine.target(),
            )
        }
    }

    /// Deserialize a serialized artifact.
    ///
    /// NOTE: You should prefer [`Self::deserialize`].
    ///
    /// # Safety
    /// See [`Self::deserialize`].
    /// In contrast to the above, this function skips artifact layout validation,
    /// which increases the risk of loading invalid artifacts.
    pub unsafe fn deserialize_unchecked(
        engine: &Engine,
        bytes: OwnedBuffer,
    ) -> Result<Self, DeserializeError> {
        unsafe {
            if !ArtifactBuild::has_rkyv_header(bytes.as_ref()) {
                if let Some(artifact) = Self::deserialize_elf(engine, bytes.as_ref())? {
                    return Ok(artifact);
                }

                let static_artifact = Self::deserialize_object(engine, bytes);
                match static_artifact {
                    Ok(v) => {
                        return Ok(v);
                    }
                    Err(e) => {
                        return Err(DeserializeError::Incompatible(format!(
                            "The provided bytes are not a Wasmer engine artifact: {e}"
                        )));
                    }
                }
            }

            let artifact = ArtifactBuildFromArchive::try_new(bytes, |bytes| {
                let bytes =
                    Self::get_byte_slice(bytes, ArtifactBuild::MAGIC_HEADER.len(), bytes.len())?;

                let metadata_len = MetadataHeader::parse(bytes)?;
                let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?;
                let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len)?;

                SerializableModule::archive_from_slice(metadata_slice)
            })?;

            let mut inner_engine = engine.inner_mut();
            Self::from_parts(
                &mut inner_engine,
                ArtifactBuildVariant::Archived(artifact),
                engine.target(),
            )
        }
    }

    /// Construct a `ArtifactBuild` from component parts.
    pub fn from_parts(
        engine_inner: &mut EngineInner,
        artifact: ArtifactBuildVariant,
        target: &Target,
    ) -> Result<Self, DeserializeError> {
        Self::from_parts_with_module_file(engine_inner, artifact, target, None)
    }

    fn from_parts_with_module_file(
        engine_inner: &mut EngineInner,
        artifact: ArtifactBuildVariant,
        target: &Target,
        module_file: Option<PathBuf>,
    ) -> Result<Self, DeserializeError> {
        if !target.is_native() {
            return Ok(Self {
                id: Default::default(),
                artifact,
                module_file,
                allocated: None,
            });
        } else {
            // check if cpu features are compatible before anything else
            let cpu_features = artifact.cpu_features();
            if !target.cpu_features().is_superset(cpu_features) {
                return Err(DeserializeError::Incompatible(format!(
                    "Some CPU Features needed for the artifact are missing: {:?}",
                    cpu_features.difference(*target.cpu_features())
                )));
            }
        }
        let module_info = artifact.module_info();

        let elf_file_data = match &artifact {
            ArtifactBuildVariant::Plain(p) => {
                if let SerializableCompilation::Elf(data) = &p.serializable.compilation {
                    Some(data.as_ref())
                } else {
                    None
                }
            }
            ArtifactBuildVariant::Archived(a) => a.get_elf_file(),
        };
        let mut allocated = if let Some(module_file) = module_file.as_ref() {
            if elf_file_data.is_none() {
                return Err(DeserializeError::Incompatible(
                    "file-backed loading only supports ELF artifacts".to_string(),
                ));
            }
            Self::allocate_elf_artifact_from_path(engine_inner, module_info, module_file)?
        } else if let Some(elf_file_data) = elf_file_data {
            Self::allocate_elf_artifact(engine_inner, module_info, elf_file_data)?
        } else {
            let (
                finished_functions,
                finished_function_call_trampolines,
                finished_dynamic_function_trampolines,
                custom_sections,
            ) = match &artifact {
                ArtifactBuildVariant::Plain(p) => engine_inner.allocate(
                    module_info,
                    p.get_function_bodies_ref()
                        .expect("RKYV path expected")
                        .values(),
                    p.get_function_call_trampolines_ref()
                        .expect("RKYV path expected")
                        .values(),
                    p.get_dynamic_function_trampolines_ref()
                        .expect("RKYV path expected")
                        .values(),
                    p.get_custom_sections_ref()
                        .expect("RKYV path expected")
                        .values(),
                )?,

                ArtifactBuildVariant::Archived(a) => engine_inner.allocate(
                    module_info,
                    a.get_function_bodies_ref()
                        .expect("RKYV path expected")
                        .values(),
                    a.get_function_call_trampolines_ref()
                        .expect("RKYV path expected")
                        .values(),
                    a.get_dynamic_function_trampolines_ref()
                        .expect("RKYV path expected")
                        .values(),
                    a.get_custom_sections_ref()
                        .expect("RKYV path expected")
                        .values(),
                )?,
            };

            let get_got_address: Box<dyn Fn(RelocationTarget) -> Option<usize>> = match &artifact {
                ArtifactBuildVariant::Plain(p) => {
                    if let Some(got) = p.get_got_ref().expect("RKYV path expected").index {
                        let relocs: Vec<_> = p
                            .get_custom_section_relocations_ref()
                            .expect("RKYV path expected")[got]
                            .iter()
                            .map(|v| (v.reloc_target, v.offset))
                            .collect();
                        let got_base = custom_sections[got].0 as usize;
                        Box::new(move |t: RelocationTarget| {
                            relocs
                                .iter()
                                .find(|(v, _)| v == &t)
                                .map(|(_, o)| got_base + (*o as usize))
                        })
                    } else {
                        Box::new(|_: RelocationTarget| None)
                    }
                }

                ArtifactBuildVariant::Archived(p) => {
                    if let Some(got) = p.get_got_ref().expect("RKYV path expected").index {
                        let relocs: Vec<_> = p
                            .get_custom_section_relocations_ref()
                            .expect("RKYV path expected")[got]
                            .iter()
                            .map(|v| (v.reloc_target(), v.offset))
                            .collect();
                        let got_base = custom_sections[got].0 as usize;
                        Box::new(move |t: RelocationTarget| {
                            relocs
                                .iter()
                                .find(|(v, _)| v == &t)
                                .map(|(_, o)| got_base + (o.to_native() as usize))
                        })
                    } else {
                        Box::new(|_: RelocationTarget| None)
                    }
                }
            };
            let functions_max_stack_usage = match &artifact {
                ArtifactBuildVariant::Plain(p) => p
                    .get_function_max_stack_usage()
                    .expect("RKYV path expected")
                    .values()
                    .cloned()
                    .collect::<PrimaryMap<LocalFunctionIndex, _>>(),
                ArtifactBuildVariant::Archived(a) => a
                    .get_function_max_stack_usage()
                    .expect("RKYV path expected")
                    .values()
                    .cloned()
                    .collect::<PrimaryMap<LocalFunctionIndex, _>>(),
            };

            match &artifact {
                ArtifactBuildVariant::Plain(p) => link_module(
                    module_info,
                    &finished_functions,
                    &finished_dynamic_function_trampolines,
                    p.get_function_relocations()
                        .expect("RKYV path expected")
                        .iter()
                        .map(|(k, v)| (k, v.iter())),
                    &custom_sections,
                    p.get_custom_section_relocations_ref()
                        .expect("RKYV path expected")
                        .iter()
                        .map(|(k, v)| (k, v.iter())),
                    p.get_libcall_trampolines().expect("RKYV path expected"),
                    p.get_libcall_trampoline_len().expect("RKYV path expected"),
                    &get_got_address,
                ),
                ArtifactBuildVariant::Archived(a) => link_module(
                    module_info,
                    &finished_functions,
                    &finished_dynamic_function_trampolines,
                    a.get_function_relocations()
                        .expect("RKYV path expected")
                        .iter()
                        .map(|(k, v)| (k, v.iter())),
                    &custom_sections,
                    a.get_custom_section_relocations_ref()
                        .expect("RKYV path expected")
                        .iter()
                        .map(|(k, v)| (k, v.iter())),
                    a.get_libcall_trampolines().expect("RKYV path expected"),
                    a.get_libcall_trampoline_len().expect("RKYV path expected"),
                    &get_got_address,
                ),
            };

            // Compute indices into the shared signature table.
            let signatures = {
                let signature_registry = engine_inner.signatures();
                module_info
                    .signatures
                    .values()
                    .zip(module_info.signature_hashes.values())
                    .map(|(sig, sig_hash)| signature_registry.register(sig, *sig_hash))
                    .collect::<PrimaryMap<_, _>>()
            };

            #[allow(unused_variables)]
            let eh_frame = match &artifact {
                ArtifactBuildVariant::Plain(p) => p
                    .get_unwind_info()
                    .expect("RKYV path expected")
                    .eh_frame
                    .map(|v| unsafe {
                        std::slice::from_raw_parts(
                            *custom_sections[v],
                            p.get_custom_sections_ref().expect("RKYV path expected")[v]
                                .bytes
                                .len(),
                        )
                    }),
                ArtifactBuildVariant::Archived(a) => a
                    .get_unwind_info()
                    .expect("RKYV path expected")
                    .eh_frame
                    .map(|v| unsafe {
                        std::slice::from_raw_parts(
                            *custom_sections[v],
                            a.get_custom_sections_ref().expect("RKYV path expected")[v]
                                .bytes
                                .len(),
                        )
                    }),
            };
            #[allow(unused_variables)]
            let compact_unwind = match &artifact {
                ArtifactBuildVariant::Plain(p) => p
                    .get_unwind_info()
                    .expect("RKYV path expected")
                    .compact_unwind
                    .map(|v| unsafe {
                        std::slice::from_raw_parts(
                            *custom_sections[v],
                            p.get_custom_sections_ref().expect("RKYV path expected")[v]
                                .bytes
                                .len(),
                        )
                    }),
                ArtifactBuildVariant::Archived(a) => a
                    .get_unwind_info()
                    .expect("RKYV path expected")
                    .compact_unwind
                    .map(|v| unsafe {
                        std::slice::from_raw_parts(
                            *custom_sections[v],
                            a.get_custom_sections_ref().expect("RKYV path expected")[v]
                                .bytes
                                .len(),
                        )
                    }),
            };

            #[cfg(all(not(target_arch = "wasm32"), feature = "compiler"))]
            {
                engine_inner.register_perfmap(&finished_functions, module_info)?;
            }

            // Make all code compiled thus far executable.
            engine_inner.publish_compiled_code();

            #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
            if let Some(compact_unwind) = compact_unwind {
                engine_inner.publish_compact_unwind(
                    compact_unwind,
                    get_got_address(RelocationTarget::LibCall(wasmer_vm::LibCall::EHPersonality)),
                )?;
            } else {
                // Default RKYV artifacts (notably Singlepass) also need DWARF unwind
                // registration on macOS/aarch64 when compact unwind is absent.
                engine_inner.publish_eh_frame(eh_frame)?;
            }
            #[cfg(not(any(
                target_arch = "wasm32",
                all(target_os = "macos", target_arch = "aarch64")
            )))]
            engine_inner.publish_eh_frame(eh_frame)?;

            drop(get_got_address);

            let finished_function_lengths = finished_functions
                .values()
                .map(|extent| extent.length)
                .collect::<PrimaryMap<LocalFunctionIndex, usize>>()
                .into_boxed_slice();
            let finished_functions = finished_functions
                .values()
                .map(|extent| extent.ptr)
                .collect::<PrimaryMap<LocalFunctionIndex, FunctionBodyPtr>>()
                .into_boxed_slice();
            let finished_function_call_trampolines =
                finished_function_call_trampolines.into_boxed_slice();
            let finished_dynamic_function_trampolines =
                finished_dynamic_function_trampolines.into_boxed_slice();
            let signatures = signatures.into_boxed_slice();

            let vm_offsets = VMOffsets::try_new(std::mem::size_of::<usize>() as u8, module_info)
                .map_err(DeserializeError::Generic)?;

            AllocatedArtifact {
                frame_info_registered: false,
                frame_info_registration: None,
                finished_functions,
                finished_function_call_trampolines,
                finished_dynamic_function_trampolines,
                signatures,
                finished_function_lengths,
                vm_offsets,
                function_max_stack_usage: functions_max_stack_usage.into_boxed_slice(),
                elf_image: None,
            }
        };

        // ELF allocation recovers function addresses from the linked image, but
        // maximum stack usage is compile metadata rather than an ELF property.
        // Preserve it from the metadata embedded in the artifact, just as the
        // in-memory Rkyv path does above.
        if allocated.elf_image.is_some() {
            allocated.function_max_stack_usage = match &artifact {
                ArtifactBuildVariant::Plain(p) => p
                    .get_function_max_stack_usage()
                    .expect("function stack usage metadata expected")
                    .values()
                    .cloned()
                    .collect::<PrimaryMap<LocalFunctionIndex, _>>()
                    .into_boxed_slice(),
                ArtifactBuildVariant::Archived(a) => a
                    .get_function_max_stack_usage()
                    .expect("function stack usage metadata expected")
                    .values()
                    .cloned()
                    .collect::<PrimaryMap<LocalFunctionIndex, _>>()
                    .into_boxed_slice(),
            };
        }

        let mut artifact = Self {
            id: Default::default(),
            artifact,
            module_file,
            allocated: Some(allocated),
        };

        let is_elf = artifact
            .allocated
            .as_ref()
            .expect("It must be allocated")
            .elf_image
            .is_some();

        artifact
            .internal_register_frame_info()
            .map_err(|e| DeserializeError::CorruptedBinary(format!("{e:?}")))?;
        if let Some(frame_info) = artifact.internal_take_frame_info_registration() {
            if is_elf {
                engine_inner.register_elf_frame_info(frame_info);
            } else {
                engine_inner.register_frame_info(frame_info);
            }
        }

        Ok(artifact)
    }

    /// Build an [`AllocatedArtifact`] from a compiled native ELF image.
    ///
    /// Note that, unlike [`Self::allocate_elf_artifact_from_path`], no debugger
    /// command file is registered here: the image only exists in memory, so
    /// there is no path a debugger could load symbols from.
    fn allocate_elf_artifact(
        engine_inner: &mut EngineInner,
        module_info: &ModuleInfo,
        elf_file_data: &[u8],
    ) -> Result<AllocatedArtifact, DeserializeError> {
        let image = object::File::parse(elf_file_data)
            .map_err(|e| DeserializeError::CorruptedBinary(format!("cannot parse image: {e}")))?;
        let (base, map_size) = engine_inner.map_elf_binary(&image, elf_file_data)?;
        Self::allocate_elf_artifact_from_image(
            engine_inner,
            module_info,
            &image,
            base,
            DebugInfoSource::Bytes(Arc::from(elf_file_data)),
            map_size,
        )
    }

    #[cfg(unix)]
    fn allocate_elf_artifact_from_path(
        engine_inner: &mut EngineInner,
        module_info: &ModuleInfo,
        path: &Path,
    ) -> Result<AllocatedArtifact, DeserializeError> {
        use std::sync::Mutex;

        let file = File::open(path)?;
        let fd = file.as_raw_fd();
        let debug_file = file.try_clone()?;
        let cache = ReadCache::new(BufReader::new(file));
        let image = object::File::parse(&cache)
            .map_err(|e| DeserializeError::CorruptedBinary(format!("cannot parse image: {e}")))?;
        if !matches!(
            image.format(),
            object::BinaryFormat::Elf | object::BinaryFormat::MachO
        ) {
            return Err(DeserializeError::Incompatible(
                "file-backed Artifact is not ELF or Mach-O".to_string(),
            ));
        }
        let (base, map_size) = engine_inner.map_elf_binary_file(&image, fd)?;
        #[cfg(feature = "compiler")]
        if let Some(debugger) = engine_inner.debugger() {
            engine_inner.register_debugger(path, base, debugger)?;
        }
        Self::allocate_elf_artifact_from_image(
            engine_inner,
            module_info,
            &image,
            base,
            DebugInfoSource::File(Arc::new(Mutex::new(debug_file))),
            map_size,
        )
    }

    #[cfg(not(unix))]
    fn allocate_elf_artifact_from_path(
        _engine_inner: &mut EngineInner,
        _module_info: &ModuleInfo,
        _path: &Path,
    ) -> Result<AllocatedArtifact, DeserializeError> {
        Err(DeserializeError::Incompatible(
            "file-backed ELF artifacts are only supported on Unix".to_string(),
        ))
    }

    fn allocate_elf_artifact_from_image<'a, R: object::ReadRef<'a>>(
        engine_inner: &mut EngineInner,
        module_info: &ModuleInfo,
        image: &object::File<'a, R>,
        base: *mut std::ffi::c_void,
        debug_info: DebugInfoSource,
        map_size: usize,
    ) -> Result<AllocatedArtifact, DeserializeError> {
        const FN_OFFSET_SIZE: usize = std::mem::size_of::<usize>();
        let is_macho = image.format() == object::BinaryFormat::MachO;

        let mut function_offsets = None;
        for section in image.sections() {
            let Ok(section_name) = section.name_bytes() else {
                continue;
            };
            match section_name {
                crate::WASMER_FUNCTION_OFFSETS_SECTION_NAME => {
                    let data = section.data().map_err(|e| {
                        DeserializeError::CorruptedBinary(format!(
                            "cannot load image section data: {e}"
                        ))
                    })?;

                    function_offsets = Some(if is_macho {
                        // Mach-O stores chained pointers here, not plain offsets.
                        // Read the pointers after the mapper has applied the fixups.
                        let invalid_range = || {
                            DeserializeError::CorruptedBinary(
                                "Function offset section exceeds allocated range".to_string(),
                            )
                        };
                        let address =
                            usize::try_from(section.address()).map_err(|_| invalid_range())?;
                        if address
                            .checked_add(data.len())
                            .is_none_or(|end| end > map_size)
                        {
                            return Err(invalid_range());
                        }
                        let data = unsafe {
                            std::slice::from_raw_parts(base.cast::<u8>().add(address), data.len())
                        };
                        data.as_chunks::<FN_OFFSET_SIZE>()
                            .0
                            .iter()
                            .map(|chunk| usize::from_le_bytes(*chunk).wrapping_sub(base as usize))
                            .collect_vec()
                    } else {
                        data.as_chunks::<FN_OFFSET_SIZE>()
                            .0
                            .iter()
                            .map(|chunk| usize::from_le_bytes(*chunk))
                            .collect_vec()
                    });
                }
                crate::EH_FRAME_SECTION_NAME => {
                    engine_inner.publish_elf_eh_frame(section.address(), section.size())?;
                }
                _ => {}
            }
        }

        let Some(function_offsets) = function_offsets else {
            return Err(DeserializeError::CorruptedBinary(
                "missing function offset section in the image".to_string(),
            ));
        };

        let local_function_count = module_info.local_func_count();
        let corrupted_offsets = || {
            DeserializeError::CorruptedBinary(format!(
                "corrupted {} section",
                String::from_utf8_lossy(crate::WASMER_FUNCTION_OFFSETS_SECTION_NAME)
            ))
        };
        let signature_count = module_info.signatures.len();
        let dynamic_trampoline_count = module_info.imported_function_types().count();
        let expected_offset_count = local_function_count
            .checked_add(signature_count)
            .and_then(|count| count.checked_add(dynamic_trampoline_count))
            .ok_or_else(corrupted_offsets)?;
        if function_offsets.len() != expected_offset_count {
            return Err(corrupted_offsets());
        }

        let (local_fn_offsets, rest) = function_offsets.split_at(local_function_count);
        let (trampoline_offsets, dynamic_trampoline_offsets) = rest.split_at(signature_count);

        // Right now, we calculate function sizes as the difference from the next function.
        let local_fn_sizes = function_offsets
            .iter()
            .skip(1)
            .take(local_function_count)
            .zip(function_offsets.iter())
            .map(|(f1, f0)| f1.checked_sub(*f0).ok_or_else(corrupted_offsets))
            .collect::<Result<Vec<_>, _>>()?;

        let signatures = {
            let signature_registry = engine_inner.signatures();
            module_info
                .signatures
                .values()
                .zip(module_info.signature_hashes.values())
                .map(|(sig, sig_hash)| signature_registry.register(sig, *sig_hash))
                .collect::<PrimaryMap<_, _>>()
                .into_boxed_slice()
        };

        let finished_functions = local_fn_offsets
            .iter()
            .map(|&offset| FunctionBodyPtr(unsafe { base.add(offset) as _ }))
            .collect::<PrimaryMap<LocalFunctionIndex, _>>();

        #[cfg(all(not(target_arch = "wasm32"), feature = "compiler"))]
        let finished_function_extents = local_fn_offsets
            .iter()
            .zip(local_fn_sizes.iter())
            .map(|(&ptr, &length)| FunctionExtent {
                ptr: FunctionBodyPtr(unsafe { base.add(ptr) as _ }),
                length,
            })
            .collect::<PrimaryMap<LocalFunctionIndex, _>>();
        #[cfg(all(not(target_arch = "wasm32"), feature = "compiler"))]
        engine_inner.register_perfmap(&finished_function_extents, module_info)?;
        let finished_function_call_trampolines = trampoline_offsets
            .iter()
            .map(|&offset| unsafe {
                std::mem::transmute::<*mut std::ffi::c_void, VMTrampoline>(base.add(offset))
            })
            .collect::<PrimaryMap<SignatureIndex, _>>()
            .into_boxed_slice();
        let finished_dynamic_function_trampolines = dynamic_trampoline_offsets
            .iter()
            .map(|&offset| FunctionBodyPtr(unsafe { base.add(offset) as _ }))
            .collect::<PrimaryMap<FunctionIndex, _>>()
            .into_boxed_slice();
        let finished_function_lengths =
            PrimaryMap::<LocalFunctionIndex, _>::from_iter(local_fn_sizes).into_boxed_slice();
        let function_max_stack_usage = finished_functions
            .iter()
            .map(|_| None)
            .collect::<PrimaryMap<LocalFunctionIndex, Option<usize>>>()
            .into_boxed_slice();

        let vm_offsets = VMOffsets::try_new(std::mem::size_of::<usize>() as u8, module_info)
            .map_err(DeserializeError::Generic)?;

        Ok(AllocatedArtifact {
            frame_info_registered: false,
            frame_info_registration: None,
            finished_functions: finished_functions.into_boxed_slice(),
            finished_function_call_trampolines,
            finished_dynamic_function_trampolines,
            signatures,
            finished_function_lengths,
            vm_offsets,
            function_max_stack_usage,
            elf_image: Some((base as usize, debug_info)),
        })
    }

    /// Check if the provided bytes look like a serialized `ArtifactBuild`.
    pub fn is_deserializable(bytes: &[u8]) -> bool {
        ArtifactBuild::is_deserializable(bytes)
    }
}

impl PartialEq for Artifact {
    fn eq(&self, other: &Self) -> bool {
        self.id == other.id
    }
}
impl Eq for Artifact {}

impl std::fmt::Debug for Artifact {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.debug_struct("Artifact")
            .field("artifact_id", &self.id)
            .field("module_info", &self.module_info())
            .finish()
    }
}

impl<'a> ArtifactCreate<'a> for Artifact {
    type OwnedDataInitializer = <ArtifactBuildVariant as ArtifactCreate<'a>>::OwnedDataInitializer;
    type OwnedDataInitializerIterator =
        <ArtifactBuildVariant as ArtifactCreate<'a>>::OwnedDataInitializerIterator;

    fn set_module_info_name(&mut self, name: String) -> bool {
        self.artifact.set_module_info_name(name)
    }

    fn create_module_info(&self) -> Arc<ModuleInfo> {
        self.artifact.create_module_info()
    }

    fn module_info(&self) -> &ModuleInfo {
        self.artifact.module_info()
    }

    fn features(&self) -> &Features {
        self.artifact.features()
    }

    fn cpu_features(&self) -> EnumSet<CpuFeature> {
        self.artifact.cpu_features()
    }

    fn data_initializers(&'a self) -> Self::OwnedDataInitializerIterator {
        self.artifact.data_initializers()
    }

    fn memory_styles(&self) -> &PrimaryMap<MemoryIndex, MemoryStyle> {
        self.artifact.memory_styles()
    }

    fn table_styles(&self) -> &PrimaryMap<TableIndex, TableStyle> {
        self.artifact.table_styles()
    }

    fn serialize(&self) -> Result<Vec<u8>, SerializeError> {
        if let Some(module_file) = &self.module_file {
            return std::fs::read(module_file).map_err(|e| {
                SerializeError::Generic(format!("Failed to serialize Artifact file: {e}"))
            });
        }
        self.artifact.serialize()
    }
}

impl<'a> ArtifactCreate<'a> for ArtifactBuildVariant {
    type OwnedDataInitializer = OwnedDataInitializerVariant<'a>;
    type OwnedDataInitializerIterator = IntoIter<Self::OwnedDataInitializer>;

    fn create_module_info(&self) -> Arc<ModuleInfo> {
        match self {
            Self::Plain(artifact) => artifact.create_module_info(),
            Self::Archived(artifact) => artifact.create_module_info(),
        }
    }

    fn set_module_info_name(&mut self, name: String) -> bool {
        match self {
            Self::Plain(artifact) => artifact.set_module_info_name(name),
            Self::Archived(artifact) => artifact.set_module_info_name(name),
        }
    }

    fn module_info(&self) -> &ModuleInfo {
        match self {
            Self::Plain(artifact) => artifact.module_info(),
            Self::Archived(artifact) => artifact.module_info(),
        }
    }

    fn features(&self) -> &Features {
        match self {
            Self::Plain(artifact) => artifact.features(),
            Self::Archived(artifact) => artifact.features(),
        }
    }

    fn cpu_features(&self) -> EnumSet<CpuFeature> {
        match self {
            Self::Plain(artifact) => artifact.cpu_features(),
            Self::Archived(artifact) => artifact.cpu_features(),
        }
    }

    fn memory_styles(&self) -> &PrimaryMap<MemoryIndex, MemoryStyle> {
        match self {
            Self::Plain(artifact) => artifact.memory_styles(),
            Self::Archived(artifact) => artifact.memory_styles(),
        }
    }

    fn table_styles(&self) -> &PrimaryMap<TableIndex, TableStyle> {
        match self {
            Self::Plain(artifact) => artifact.table_styles(),
            Self::Archived(artifact) => artifact.table_styles(),
        }
    }

    fn data_initializers(&'a self) -> Self::OwnedDataInitializerIterator {
        match self {
            Self::Plain(artifact) => artifact
                .data_initializers()
                .map(OwnedDataInitializerVariant::Plain)
                .collect::<Vec<_>>()
                .into_iter(),
            Self::Archived(artifact) => artifact
                .data_initializers()
                .map(OwnedDataInitializerVariant::Archived)
                .collect::<Vec<_>>()
                .into_iter(),
        }
    }

    fn serialize(&self) -> Result<Vec<u8>, SerializeError> {
        match self {
            Self::Plain(artifact) => artifact.serialize(),
            Self::Archived(artifact) => artifact.serialize(),
        }
    }
}

#[derive(Clone, Copy)]
pub enum OwnedDataInitializerVariant<'a> {
    Plain(&'a OwnedDataInitializer),
    Archived(&'a ArchivedOwnedDataInitializer),
}

impl<'a> DataInitializerLike<'a> for OwnedDataInitializerVariant<'a> {
    type Location = DataInitializerLocationVariant<'a>;

    fn location(&self) -> Self::Location {
        match self {
            Self::Plain(plain) => DataInitializerLocationVariant::Plain(plain.location()),
            Self::Archived(archived) => {
                DataInitializerLocationVariant::Archived(archived.location())
            }
        }
    }

    fn data(&self) -> &'a [u8] {
        match self {
            Self::Plain(plain) => plain.data(),
            Self::Archived(archived) => archived.data(),
        }
    }
}

#[derive(Clone, Copy)]
pub enum DataInitializerLocationVariant<'a> {
    Plain(&'a DataInitializerLocation),
    Archived(&'a ArchivedDataInitializerLocation),
}

impl DataInitializerLocationVariant<'_> {
    pub fn clone_to_plain(&self) -> DataInitializerLocation {
        match self {
            Self::Plain(p) => (*p).clone(),
            Self::Archived(a) => DataInitializerLocation {
                memory_index: a.memory_index(),
                offset_expr: a.offset_expr(),
            },
        }
    }
}

impl DataInitializerLocationLike for DataInitializerLocationVariant<'_> {
    fn memory_index(&self) -> MemoryIndex {
        match self {
            Self::Plain(plain) => plain.memory_index(),
            Self::Archived(archived) => archived.memory_index(),
        }
    }

    fn offset_expr(&self) -> wasmer_types::InitExpr {
        match self {
            Self::Plain(plain) => plain.offset_expr(),
            Self::Archived(archived) => archived.offset_expr(),
        }
    }
}

impl Artifact {
    fn internal_register_frame_info(&mut self) -> Result<(), DeserializeError> {
        if self
            .allocated
            .as_ref()
            .expect("It must be allocated")
            .frame_info_registered
        {
            return Ok(()); // already done
        }

        // ELF artifacts don't carry the RKYV frame-info section (per-instruction
        // address maps); they get symbolicated from DWARF debug info instead,
        // lazily loaded from the ELF image (see `elf_image` below).
        let frame_infos = match &self.artifact {
            ArtifactBuildVariant::Plain(p) => p
                .get_frame_info_ref()
                .map(|f| FrameInfosVariant::Owned(f.clone())),
            ArtifactBuildVariant::Archived(a) => a
                .get_frame_info_ref()
                .map(|_| FrameInfosVariant::Archived(a.clone())),
        };

        let elf_image = self
            .allocated
            .as_ref()
            .expect("It must be allocated")
            .elf_image
            .clone();

        if frame_infos.is_some() || elf_image.is_some() {
            let finished_function_extents = self
                .allocated
                .as_ref()
                .expect("It must be allocated")
                .function_extents()
                .into_boxed_slice();

            let (image_base, elf_data) = match elf_image {
                Some((base, data)) => (base, Some(data)),
                None => (0, None),
            };

            let frame_info_registration = &mut self
                .allocated
                .as_mut()
                .expect("It must be allocated")
                .frame_info_registration;

            *frame_info_registration = register_frame_info_source(
                self.artifact.create_module_info(),
                &finished_function_extents,
                frame_infos,
                image_base,
                elf_data,
            );
        }

        self.allocated
            .as_mut()
            .expect("It must be allocated")
            .frame_info_registered = true;

        Ok(())
    }

    fn internal_take_frame_info_registration(&mut self) -> Option<GlobalFrameInfoRegistration> {
        let frame_info_registration = &mut self
            .allocated
            .as_mut()
            .expect("It must be allocated")
            .frame_info_registration;

        frame_info_registration.take()
    }

    /// Returns the functions allocated in memory or this `Artifact`
    /// ready to be run.
    pub fn finished_functions(&self) -> &BoxedSlice<LocalFunctionIndex, FunctionBodyPtr> {
        &self
            .allocated
            .as_ref()
            .expect("It must be allocated")
            .finished_functions
    }

    /// Returns the start address and byte length of each locally-defined
    /// function body in this artifact.
    ///
    /// Returns `None` for cross-compiled artifacts (where the artifact has not
    /// been allocated into the host process).
    ///
    /// # Security
    ///
    /// The returned addresses are host-process pointers. They are not stable
    /// across runs and must not be forwarded to untrusted parties, as they
    /// reveal ASLR layout information.
    pub fn finished_function_extents(&self) -> Option<Vec<(LocalFunctionIndex, FunctionExtent)>> {
        let allocated = self.allocated.as_ref()?;
        Some(allocated.function_extents().into_iter().collect())
    }

    /// Return the maximum stack size used for each function (available only for the Singlepass compiler).
    pub fn finished_functions_max_stack_usage(
        &self,
    ) -> Option<Vec<(LocalFunctionIndex, Option<usize>)>> {
        let allocated = self.allocated.as_ref()?;
        Some(
            allocated
                .function_max_stack_usage
                .into_iter()
                .map(|f| (f.0, *f.1))
                .collect(),
        )
    }

    /// Returns the function call trampolines allocated in memory of this
    /// `Artifact`, ready to be run.
    pub fn finished_function_call_trampolines(&self) -> &BoxedSlice<SignatureIndex, VMTrampoline> {
        &self
            .allocated
            .as_ref()
            .expect("It must be allocated")
            .finished_function_call_trampolines
    }

    /// Returns the dynamic function trampolines allocated in memory
    /// of this `Artifact`, ready to be run.
    pub fn finished_dynamic_function_trampolines(
        &self,
    ) -> &BoxedSlice<FunctionIndex, FunctionBodyPtr> {
        &self
            .allocated
            .as_ref()
            .expect("It must be allocated")
            .finished_dynamic_function_trampolines
    }

    /// Returns the associated VM signatures for this `Artifact`.
    pub fn signatures(&self) -> &BoxedSlice<SignatureIndex, VMSignatureHash> {
        &self
            .allocated
            .as_ref()
            .expect("It must be allocated")
            .signatures
    }

    /// Do preinstantiation logic that is executed before instantiating
    #[allow(clippy::result_large_err)]
    pub fn preinstantiate(&self) -> Result<(), InstantiationError> {
        Ok(())
    }

    /// Crate an `Instance` from this `Artifact`.
    ///
    /// # Safety
    ///
    /// See [`VMInstance::new`].
    #[allow(clippy::result_large_err)]
    pub unsafe fn instantiate(
        &self,
        tunables: &dyn Tunables,
        imports: &[VMExtern],
        context: &mut StoreObjects,
    ) -> Result<VMInstance, InstantiationError> {
        unsafe {
            // Validate the CPU features this module was compiled with against the
            // host CPU features.
            let host_cpu_features = CpuFeature::for_host();
            if !host_cpu_features.is_superset(self.cpu_features()) {
                return Err(InstantiationError::CpuFeature(format!(
                    "{:?}",
                    self.cpu_features().difference(host_cpu_features)
                )));
            }

            self.preinstantiate()?;

            let module = self.create_module_info();

            let tags = resolve_tags(&module, imports, context).map_err(InstantiationError::Link)?;

            let imports = resolve_imports(
                &module,
                imports,
                context,
                self.finished_dynamic_function_trampolines(),
                self.memory_styles(),
                self.table_styles(),
            )
            .map_err(InstantiationError::Link)?;

            // Get pointers to where metadata about local memories should live in VM memory.
            // Get pointers to where metadata about local tables should live in VM memory.

            let cached_offsets = self
                .allocated
                .as_ref()
                .map(|a| a.vm_offsets.clone())
                .expect("Artifact::instantiate called on a non-host artifact");

            let (
                allocator,
                memory_definition_locations,
                table_definition_locations,
                global_definition_locations,
            ) = InstanceAllocator::new_with_offsets(cached_offsets, &module);
            let finished_memories = tunables
                .create_memories(
                    context,
                    &module,
                    self.memory_styles(),
                    &memory_definition_locations,
                )
                .map_err(InstantiationError::Link)?
                .into_boxed_slice();
            let finished_tables = tunables
                .create_tables(
                    context,
                    &module,
                    self.table_styles(),
                    &table_definition_locations,
                )
                .map_err(InstantiationError::Link)?
                .into_boxed_slice();
            let finished_globals = tunables
                .create_globals(context, &module, &global_definition_locations)
                .map_err(InstantiationError::Link)?
                .into_boxed_slice();
            let initial_table_elements = module
                .tables
                .values()
                .skip(module.num_imported_tables)
                .fold(0u32, |total, ty| total.saturating_add(ty.minimum));
            let table_allocation_room = tunables
                .max_table_elements()
                .saturating_sub(initial_table_elements);

            let handle = VMInstance::new(
                allocator,
                module,
                context,
                self.finished_functions().clone(),
                self.finished_function_call_trampolines().clone(),
                finished_memories,
                finished_tables,
                table_allocation_room,
                finished_globals,
                tags,
                imports,
                self.signatures().clone(),
            )
            .map_err(InstantiationError::Start)?;
            Ok(handle)
        }
    }

    /// Finishes the instantiation of a just created `VMInstance`.
    ///
    /// # Safety
    ///
    /// See [`VMInstance::finish_instantiation`].
    #[allow(clippy::result_large_err)]
    pub unsafe fn finish_instantiation(
        &self,
        config: &VMConfig,
        trap_handler: Option<*const TrapHandlerFn<'static>>,
        handle: &mut VMInstance,
    ) -> Result<(), InstantiationError> {
        unsafe {
            let data_initializers = self
                .data_initializers()
                .map(|init| DataInitializer {
                    location: init.location().clone_to_plain(),
                    data: init.data(),
                })
                .collect::<Vec<_>>();
            handle
                .finish_instantiation(config, trap_handler, &data_initializers)
                .map_err(InstantiationError::Start)
        }
    }

    #[allow(clippy::type_complexity)]
    #[cfg(feature = "static-artifact-create")]
    /// Generate a compilation
    pub fn generate_metadata<'data>(
        data: &'data [u8],
        compiler: &dyn Compiler,
        tunables: &dyn Tunables,
        features: &Features,
    ) -> Result<
        (
            CompileModuleInfo,
            PrimaryMap<LocalFunctionIndex, FunctionBodyData<'data>>,
            Vec<DataInitializer<'data>>,
            Option<ModuleTranslationState>,
        ),
        CompileError,
    > {
        let environ = ModuleEnvironment::new();
        let translation = environ.translate(data).map_err(CompileError::Wasm)?;

        // We try to apply the middleware first
        use crate::translator::ModuleMiddlewareChain;
        let mut module = translation.module;
        let middlewares = compiler.get_middlewares();
        middlewares
            .apply_on_module_info(&mut module)
            .map_err(|e| CompileError::MiddlewareError(e.to_string()))?;

        let memory_styles: PrimaryMap<MemoryIndex, MemoryStyle> = module
            .memories
            .values()
            .map(|memory_type| tunables.memory_style(memory_type))
            .collect();
        let table_styles: PrimaryMap<TableIndex, TableStyle> = module
            .tables
            .values()
            .map(|table_type| tunables.table_style(table_type))
            .collect();

        let compile_info = CompileModuleInfo {
            module: Arc::new(module),
            features: features.clone(),
            memory_styles,
            table_styles,
            function_max_stack_usage: PrimaryMap::new(),
        };
        Ok((
            compile_info,
            translation.function_body_inputs,
            translation.data_initializers,
            translation.module_translation_state,
        ))
    }

    /// Generate the metadata object for the module
    #[cfg(feature = "static-artifact-create")]
    #[allow(clippy::type_complexity)]
    pub fn metadata<'a>(
        compiler: &dyn Compiler,
        data: &'a [u8],
        metadata_prefix: Option<&str>,
        target: &Target,
        tunables: &dyn Tunables,
        features: &Features,
    ) -> Result<
        (
            ModuleMetadata,
            Option<ModuleTranslationState>,
            PrimaryMap<LocalFunctionIndex, FunctionBodyData<'a>>,
        ),
        CompileError,
    > {
        #[allow(dead_code)]
        let (compile_info, function_body_inputs, data_initializers, module_translation) =
            Self::generate_metadata(data, compiler, tunables, features)?;

        let data_initializers = data_initializers
            .iter()
            .map(OwnedDataInitializer::new)
            .collect::<Vec<_>>()
            .into_boxed_slice();

        // TODO: we currently supply all-zero function body lengths.
        // We don't know the lengths until they're compiled, yet we have to
        // supply the metadata as an input to the compile.
        let function_body_lengths = function_body_inputs
            .keys()
            .map(|_function_body| 0u64)
            .collect::<PrimaryMap<LocalFunctionIndex, u64>>();

        let metadata = ModuleMetadata {
            compile_info,
            prefix: metadata_prefix.map(|s| s.to_string()).unwrap_or_default(),
            data_initializers,
            function_body_lengths,
            cpu_features: target.cpu_features().as_u64(),
        };

        Ok((metadata, module_translation, function_body_inputs))
    }

    /// Compile a module into an object file, which can be statically linked against.
    ///
    /// The `metadata_prefix` is an optional prefix for the object name to make the
    /// function names in the object file unique. When set, the function names will
    /// be `wasmer_function_{prefix}_{id}` and the object metadata will be addressable
    /// using `WASMER_METADATA_{prefix}_LENGTH` and `WASMER_METADATA_{prefix}_DATA`.
    ///
    #[cfg(feature = "static-artifact-create")]
    pub fn generate_object<'data>(
        compiler: &dyn Compiler,
        data: &[u8],
        metadata_prefix: Option<&str>,
        target: &'data Target,
        tunables: &dyn Tunables,
        features: &Features,
    ) -> Result<
        (
            ModuleInfo,
            Object<'data>,
            usize,
            Box<dyn crate::types::symbols::SymbolRegistry>,
        ),
        CompileError,
    > {
        use crate::types::{
            function::Compilation,
            symbols::{ModuleMetadataSymbolRegistry, SymbolRegistry},
        };

        fn to_compile_error(err: impl std::error::Error) -> CompileError {
            CompileError::Codegen(format!("{err}"))
        }

        let target_triple = target.triple();
        let (mut metadata, module_translation, function_body_inputs) =
            Self::metadata(compiler, data, metadata_prefix, target, tunables, features)
                .map_err(to_compile_error)?;

        /*
        In the C file we need:
        - imports
        - exports

        to construct an api::Module which is a Store (can be passed in via argument) and an
        Arc<dyn Artifact> which means this struct which includes:
        - CompileModuleInfo
        - Features
        - ModuleInfo
        - MemoryIndex -> MemoryStyle
        - TableIndex -> TableStyle
        - LocalFunctionIndex -> FunctionBodyPtr // finished functions
        - FunctionIndex -> FunctionBodyPtr // finished dynamic function trampolines
        - SignatureIndex -> VMSignatureHash // signatures
         */

        let compilation = compiler.compile_module(
            target,
            &metadata.compile_info,
            &[],
            module_translation.as_ref().unwrap(),
            function_body_inputs,
            None,
        )?;
        let Compilation::Rkyv {
            compilation,
            function_max_stack_usage,
        } = compilation
        else {
            return Err(CompileError::Codegen(
                "ELF compilation unsupported yet".to_string(),
            ));
        };
        metadata.compile_info.function_max_stack_usage = function_max_stack_usage;

        let mut metadata_builder =
            ObjectMetadataBuilder::new(&metadata, target_triple).map_err(to_compile_error)?;
        let (_compile_info, symbol_registry) = metadata.split();
        let mut obj = get_object_for_target(target_triple).map_err(to_compile_error)?;

        let object_name = ModuleMetadataSymbolRegistry {
            prefix: metadata_prefix.unwrap_or_default().to_string(),
        }
        .symbol_to_name(crate::types::symbols::Symbol::Metadata);

        let default_align = match target_triple.architecture {
            target_lexicon::Architecture::Aarch64(_) => {
                if matches!(
                    target_triple.operating_system,
                    target_lexicon::OperatingSystem::Darwin(_)
                ) {
                    8
                } else {
                    4
                }
            }
            _ => 1,
        };

        let offset = emit_data(
            &mut obj,
            object_name.as_bytes(),
            metadata_builder.placeholder_data(),
            std::cmp::max(MetadataHeader::ALIGN as u64, default_align),
        )
        .map_err(to_compile_error)?;
        metadata_builder.set_section_offset(offset);

        emit_compilation(
            &mut obj,
            compilation,
            &symbol_registry,
            target_triple,
            &metadata_builder,
        )
        .map_err(to_compile_error)?;
        Ok((
            Arc::try_unwrap(metadata.compile_info.module).unwrap(),
            obj,
            metadata_builder.placeholder_data().len(),
            Box::new(symbol_registry),
        ))
    }

    /// Deserialize a ArtifactBuild from an object file
    ///
    /// # Safety
    /// The object must be a valid static object generated by wasmer.
    #[cfg(not(feature = "static-artifact-load"))]
    pub unsafe fn deserialize_object(
        _engine: &Engine,
        _bytes: OwnedBuffer,
    ) -> Result<Self, DeserializeError> {
        Err(DeserializeError::Compiler(
            CompileError::UnsupportedFeature("static load is not compiled in".to_string()),
        ))
    }

    fn get_byte_slice(input: &[u8], start: usize, end: usize) -> Result<&[u8], DeserializeError> {
        if (start == end && input.len() > start)
            || (start < end && input.len() > start && input.len() >= end)
        {
            Ok(&input[start..end])
        } else {
            Err(DeserializeError::InvalidByteLength {
                expected: end - start,
                got: input.len(),
            })
        }
    }

    /// Deserialize a ArtifactBuild from an object file
    ///
    /// # Safety
    /// The object must be a valid static object generated by wasmer.
    #[cfg(feature = "static-artifact-load")]
    pub unsafe fn deserialize_object(
        engine: &Engine,
        bytes: OwnedBuffer,
    ) -> Result<Self, DeserializeError> {
        unsafe {
            use crate::serialize::SerializableCompilation;

            let bytes = bytes.as_slice();
            let metadata_len = MetadataHeader::parse(bytes)?;
            let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?;
            let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len)?;
            let metadata: ModuleMetadata = ModuleMetadata::deserialize(metadata_slice)?;

            const WORD_SIZE: usize = mem::size_of::<usize>();
            let mut byte_buffer = [0u8; WORD_SIZE];

            let mut cur_offset = MetadataHeader::LEN + metadata_len;

            let byte_buffer_slice =
                Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?;
            byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice);
            cur_offset += WORD_SIZE;

            let num_finished_functions = usize::from_ne_bytes(byte_buffer);
            let mut finished_functions: PrimaryMap<LocalFunctionIndex, FunctionBodyPtr> =
                PrimaryMap::new();

            let engine_inner = engine.inner();
            let signature_registry = engine_inner.signatures();

            // read finished functions in order now...
            for _i in 0..num_finished_functions {
                let byte_buffer_slice =
                    Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?;
                byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice);
                let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _);
                cur_offset += WORD_SIZE;

                // TODO: we can read back the length here if we serialize it. This will improve debug output.
                finished_functions.push(fp);
            }

            // We register all the signatures
            let signatures = {
                let module = &metadata.compile_info.module;
                module
                    .signatures
                    .values()
                    .zip(module.signature_hashes.values())
                    .map(|(sig, sig_hash)| signature_registry.register(sig, *sig_hash))
                    .collect::<PrimaryMap<_, _>>()
            };

            // read trampolines in order
            let mut finished_function_call_trampolines = PrimaryMap::new();

            let byte_buffer_slice =
                Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?;
            byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice);
            cur_offset += WORD_SIZE;
            let num_function_trampolines = usize::from_ne_bytes(byte_buffer);
            for _ in 0..num_function_trampolines {
                let byte_buffer_slice =
                    Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?;
                byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice);
                cur_offset += WORD_SIZE;
                let trampoline_ptr_bytes = usize::from_ne_bytes(byte_buffer);
                let trampoline = mem::transmute::<usize, VMTrampoline>(trampoline_ptr_bytes);
                finished_function_call_trampolines.push(trampoline);
                // TODO: we can read back the length here if we serialize it. This will improve debug output.
            }

            // read dynamic function trampolines in order now...
            let mut finished_dynamic_function_trampolines = PrimaryMap::new();
            let byte_buffer_slice =
                Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?;
            byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice);
            cur_offset += WORD_SIZE;
            let num_dynamic_trampoline_functions = usize::from_ne_bytes(byte_buffer);
            for _i in 0..num_dynamic_trampoline_functions {
                let byte_buffer_slice =
                    Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?;
                byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice);
                let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _);
                cur_offset += WORD_SIZE;

                // TODO: we can read back the length here if we serialize it. This will improve debug output.

                finished_dynamic_function_trampolines.push(fp);
            }

            let artifact = ArtifactBuild::from_serializable(SerializableModule {
                compilation: SerializableCompilation::Rkyv(RkyvSerializableCompilation::default()),
                compile_info: metadata.compile_info,
                data_initializers: metadata.data_initializers,
                cpu_features: metadata.cpu_features,
            });

            let finished_function_lengths = finished_functions
                .values()
                .map(|_| 0)
                .collect::<PrimaryMap<LocalFunctionIndex, usize>>()
                .into_boxed_slice();
            let function_max_stack_usage = finished_functions
                .iter()
                .map(|_| None)
                .collect::<PrimaryMap<LocalFunctionIndex, Option<usize>>>()
                .into_boxed_slice();

            // Variant is built first so its module_info is available for
            // the cached VMOffsets before it is moved into Self.
            let artifact_variant = ArtifactBuildVariant::Plain(artifact);
            let vm_offsets = VMOffsets::try_new(
                std::mem::size_of::<usize>() as u8,
                artifact_variant.module_info(),
            )
            .map_err(DeserializeError::Generic)?;

            Ok(Self {
                id: Default::default(),
                artifact: artifact_variant,
                module_file: None,
                allocated: Some(AllocatedArtifact {
                    frame_info_registered: false,
                    frame_info_registration: None,
                    finished_functions: finished_functions.into_boxed_slice(),
                    finished_function_call_trampolines: finished_function_call_trampolines
                        .into_boxed_slice(),
                    finished_dynamic_function_trampolines: finished_dynamic_function_trampolines
                        .into_boxed_slice(),
                    signatures: signatures.into_boxed_slice(),
                    finished_function_lengths,
                    vm_offsets,
                    function_max_stack_usage,
                    elf_image: None,
                }),
            })
        }
    }
}

/// On-demand reader of per-function trap information from an ELF artifact.
///
/// Trap lookups only happen when a trap fires, so the ELF trap sections are
/// parsed lazily instead of duplicating all trap tables in memory.
pub(crate) struct TrapReader {
    source: DebugInfoSource,
    image_base: usize,
}

impl TrapReader {
    pub(crate) fn new(source: DebugInfoSource, image_base: usize) -> Self {
        Self { source, image_base }
    }

    /// Looks up the trap information for `local_index` at `rel_pos`, the offset
    /// relative to the start of the function.
    pub(crate) fn lookup(
        &self,
        local_index: LocalFunctionIndex,
        rel_pos: u32,
    ) -> Option<TrapInformation> {
        match &self.source {
            DebugInfoSource::Bytes(data) => {
                let image = object::File::parse(&data[..]).ok()?;
                self.lookup_in_image(&image, local_index, rel_pos)
            }
            DebugInfoSource::File(file) => {
                let file = file.lock().unwrap();
                let cache = ReadCache::new(BufReader::new(file.try_clone().ok()?));
                let image = object::File::parse(&cache).ok()?;
                self.lookup_in_image(&image, local_index, rel_pos)
            }
        }
    }

    fn lookup_in_image<'data, R: ReadRef<'data>>(
        &self,
        image: &object::File<'data, R>,
        local_index: LocalFunctionIndex,
        rel_pos: u32,
    ) -> Option<TrapInformation> {
        let traps_section = image.section_by_name_bytes(WASMER_TRAPS_SECTION_NAME)?;
        let offsets_section =
            image.section_by_name_bytes(WASMER_TRAP_FUNCTION_OFFSETS_SECTION_NAME)?;
        let trap_offsets = offsets_section.data().ok()?;
        let slot = local_index.index().checked_mul(size_of::<usize>())?;
        let offset_bytes = trap_offsets.get(slot..slot + size_of::<usize>())?;

        // Mach-O slots contain chained pointers on disk. Read the mapped slot after fixups.
        let trap_address = if image.format() == object::BinaryFormat::MachO {
            let address = self
                .image_base
                .checked_add(offsets_section.address() as usize)?
                .checked_add(slot)?;
            unsafe { (address as *const usize).read_unaligned() }.checked_sub(self.image_base)?
        } else {
            usize::from_le_bytes(offset_bytes.try_into().ok()?)
        };
        let trap_offset = trap_address.checked_sub(traps_section.address() as usize)?;
        let traps = Self::parse_function_traps(traps_section.data().ok()?, trap_offset)?;
        traps
            .binary_search_by_key(&rel_pos, |info| info.code_offset)
            .ok()
            .map(|index| traps[index])
    }

    fn parse_function_traps(
        traps_section: &[u8],
        trap_offset: usize,
    ) -> Option<Vec<TrapInformation>> {
        const WORD_SIZE: usize = size_of::<u32>();
        const RECORD_SIZE: usize = 2 * WORD_SIZE;

        let data = traps_section.get(trap_offset..)?;
        let count = u32::from_le_bytes(data.get(..WORD_SIZE)?.try_into().ok()?) as usize;
        let records_len = count.checked_mul(RECORD_SIZE)?;
        let records = data.get(WORD_SIZE..WORD_SIZE.checked_add(records_len)?)?;

        records
            .as_chunks::<RECORD_SIZE>()
            .0
            .iter()
            .map(|record| {
                let code_offset = u32::from_le_bytes(record[..WORD_SIZE].try_into().ok()?);
                let code = u32::from_le_bytes(record[WORD_SIZE..].try_into().ok()?);
                let trap_code = TrapCode::try_from(code).ok()?;
                Some(TrapInformation {
                    code_offset,
                    trap_code,
                })
            })
            .collect()
    }
}