wasmer_vm/instance/

mod.rs

// This file contains code from external sources.
// Attributions: https://github.com/wasmerio/wasmer/blob/main/docs/ATTRIBUTIONS.md

//! An `Instance` contains all the runtime state used by execution of
//! a WebAssembly module (except its callstack and register state). An
//! `VMInstance` is a wrapper around `Instance` that manages
//! how it is allocated and deallocated.

mod allocator;

use crate::LinearMemory;
use crate::imports::Imports;
use crate::store::{InternalStoreHandle, StoreObjects};
use crate::table::TableElement;
use crate::trap::{Trap, TrapCode};
use crate::vmcontext::{
    VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionContext,
    VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition,
    VMMemoryImport, VMSharedTagIndex, VMSignatureHash, VMTableDefinition, VMTableImport,
    VMTrampoline, memory_copy, memory_fill, memory32_atomic_check_notify, memory32_atomic_check32,
    memory32_atomic_check64,
};
use crate::{FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMTag, wasmer_call_trampoline};
use crate::{VMConfig, VMFuncRef, VMFunction, VMGlobal, VMMemory, VMTable};
use crate::{export::VMExtern, threadconditions::ExpectedValue};
pub use allocator::InstanceAllocator;
use core::mem::offset_of;
use itertools::Itertools;
use more_asserts::assert_lt;
use std::alloc::Layout;
use std::cell::RefCell;
use std::collections::HashMap;
use std::convert::TryFrom;
use std::fmt;
use std::mem;
use std::ptr::{self, NonNull};
use std::slice;
use std::sync::Arc;
use wasmer_types::entity::{BoxedSlice, EntityRef, PrimaryMap, packed_option::ReservedValue};
use wasmer_types::{
    DataIndex, DataInitializer, ElemIndex, ExportIndex, FunctionIndex, GlobalIndex, GlobalInit,
    InitExpr, InitExprOp, LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex,
    MemoryError, MemoryIndex, ModuleInfo, Pages, RawValue, SignatureIndex, TableIndex, TagIndex,
    VMOffsets,
};

/// A WebAssembly instance.
///
/// The type is dynamically-sized. Indeed, the `vmctx` field can
/// contain various data. That's why the type has a C representation
/// to ensure that the `vmctx` field is last. See the documentation of
/// the `vmctx` field to learn more.
#[repr(C)]
#[allow(clippy::type_complexity)]
pub(crate) struct Instance {
    /// The `ModuleInfo` this `Instance` was instantiated from.
    module: Arc<ModuleInfo>,

    /// Pointer to the object store of the context owning this instance.
    context: *mut StoreObjects,

    /// Offsets in the `vmctx` region.
    offsets: VMOffsets,

    /// WebAssembly linear memory data.
    memories: BoxedSlice<LocalMemoryIndex, InternalStoreHandle<VMMemory>>,

    /// WebAssembly table data.
    tables: BoxedSlice<LocalTableIndex, InternalStoreHandle<VMTable>>,

    /// WebAssembly global data.
    globals: BoxedSlice<LocalGlobalIndex, InternalStoreHandle<VMGlobal>>,

    /// WebAssembly tag data. Notably, this stores *all* tags, not just local ones.
    tags: BoxedSlice<TagIndex, InternalStoreHandle<VMTag>>,

    /// Pointers to functions in executable memory.
    functions: BoxedSlice<LocalFunctionIndex, FunctionBodyPtr>,

    /// Pointers to function call trampolines in executable memory.
    function_call_trampolines: BoxedSlice<SignatureIndex, VMTrampoline>,

    /// Passive elements in this instantiation. As `elem.drop`s happen, these
    /// entries get removed.
    passive_elements: RefCell<HashMap<ElemIndex, Box<[Option<VMFuncRef>]>>>,

    /// Per-instance view of the module's passive data segments.
    ///
    /// A `None` entry (dropped) or a missing entry is treated as an empty slice.
    ///
    /// The bytes are shared with the module via `Arc` (no per-instance copy).
    /// `data.drop` replaces an entry's value with `None` to mark the segment
    /// unusable for subsequent `memory.init` on this instance, without
    /// affecting the shared module bytes or any other instance.
    passive_data: RefCell<HashMap<DataIndex, Option<Arc<[u8]>>>>,

    /// Mapping of function indices to their func ref backing data. `VMFuncRef`s
    /// will point to elements here for functions defined by this instance.
    funcrefs: BoxedSlice<LocalFunctionIndex, VMCallerCheckedAnyfunc>,

    /// Mapping of function indices to their func ref backing data. `VMFuncRef`s
    /// will point to elements here for functions imported by this instance.
    imported_funcrefs: BoxedSlice<FunctionIndex, NonNull<VMCallerCheckedAnyfunc>>,

    /// Additional context used by compiled WebAssembly code. This
    /// field is last, and represents a dynamically-sized array that
    /// extends beyond the nominal end of the struct (similar to a
    /// flexible array member).
    vmctx: VMContext,
}

impl fmt::Debug for Instance {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        formatter.debug_struct("Instance").finish()
    }
}

#[allow(clippy::cast_ptr_alignment)]
impl Instance {
    /// Helper function to access various locations offset from our `*mut
    /// VMContext` object.
    unsafe fn vmctx_plus_offset<T>(&self, offset: u32) -> *mut T {
        unsafe {
            (self.vmctx_ptr() as *mut u8)
                .add(usize::try_from(offset).unwrap())
                .cast()
        }
    }

    fn module(&self) -> &Arc<ModuleInfo> {
        &self.module
    }

    pub(crate) fn module_ref(&self) -> &ModuleInfo {
        &self.module
    }

    pub(crate) fn context(&self) -> &StoreObjects {
        unsafe { &*self.context }
    }

    pub(crate) fn context_mut(&mut self) -> &mut StoreObjects {
        unsafe { &mut *self.context }
    }

    /// Offsets in the `vmctx` region.
    fn offsets(&self) -> &VMOffsets {
        &self.offsets
    }

    /// Return the indexed `VMFunctionImport`.
    fn imported_function(&self, index: FunctionIndex) -> &VMFunctionImport {
        let index = usize::try_from(index.as_u32()).unwrap();
        unsafe { &*self.imported_functions_ptr().add(index) }
    }

    /// Return a pointer to the `VMFunctionImport`s.
    fn imported_functions_ptr(&self) -> *mut VMFunctionImport {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_functions_begin()) }
    }

    /// Return the index `VMTableImport`.
    fn imported_table(&self, index: TableIndex) -> &VMTableImport {
        let index = usize::try_from(index.as_u32()).unwrap();
        unsafe { &*self.imported_tables_ptr().add(index) }
    }

    /// Return a pointer to the `VMTableImports`s.
    fn imported_tables_ptr(&self) -> *mut VMTableImport {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_tables_begin()) }
    }

    /// Return the indexed `VMMemoryImport`.
    fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport {
        let index = usize::try_from(index.as_u32()).unwrap();
        unsafe { &*self.imported_memories_ptr().add(index) }
    }

    /// Return a pointer to the `VMMemoryImport`s.
    fn imported_memories_ptr(&self) -> *mut VMMemoryImport {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_memories_begin()) }
    }

    /// Return the indexed `VMGlobalImport`.
    fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport {
        let index = usize::try_from(index.as_u32()).unwrap();
        unsafe { &*self.imported_globals_ptr().add(index) }
    }

    /// Return a pointer to the `VMGlobalImport`s.
    fn imported_globals_ptr(&self) -> *mut VMGlobalImport {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_globals_begin()) }
    }

    /// Return the indexed `VMSharedTagIndex`.
    #[cfg_attr(target_os = "windows", allow(dead_code))]
    pub(crate) fn shared_tag_ptr(&self, index: TagIndex) -> &VMSharedTagIndex {
        let index = usize::try_from(index.as_u32()).unwrap();
        unsafe { &*self.shared_tags_ptr().add(index) }
    }

    /// Return a pointer to the `VMSharedTagIndex`s.
    pub(crate) fn shared_tags_ptr(&self) -> *mut VMSharedTagIndex {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_tag_ids_begin()) }
    }

    /// Return the indexed `VMTableDefinition`.
    #[allow(dead_code)]
    fn table(&self, index: LocalTableIndex) -> VMTableDefinition {
        unsafe { *self.table_ptr(index).as_ref() }
    }

    #[allow(dead_code)]
    /// Updates the value for a defined table to `VMTableDefinition`.
    fn set_table(&self, index: LocalTableIndex, table: &VMTableDefinition) {
        unsafe {
            *self.table_ptr(index).as_ptr() = *table;
        }
    }

    /// Return the indexed `VMTableDefinition`.
    fn table_ptr(&self, index: LocalTableIndex) -> NonNull<VMTableDefinition> {
        let index = usize::try_from(index.as_u32()).unwrap();
        NonNull::new(unsafe { self.tables_ptr().add(index) }).unwrap()
    }

    /// Return a pointer to the `VMTableDefinition`s.
    fn tables_ptr(&self) -> *mut VMTableDefinition {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_tables_begin()) }
    }

    fn fixed_funcref_table_ptr(
        &self,
        index: LocalTableIndex,
    ) -> Option<NonNull<VMCallerCheckedAnyfunc>> {
        let offset = self.offsets.vmctx_fixed_funcref_table_anyfuncs(index)?;
        Some(NonNull::new(unsafe { self.vmctx_plus_offset(offset) }).unwrap())
    }

    fn sync_fixed_funcref_table_element(
        &self,
        table_index: LocalTableIndex,
        index: u32,
        funcref: Option<VMFuncRef>,
    ) {
        let Some(base) = self.fixed_funcref_table_ptr(table_index) else {
            return;
        };
        unsafe {
            *base.as_ptr().add(index as usize) = anyfunc_from_funcref(funcref);
        }
    }

    fn sync_fixed_funcref_table(&self, table_index: LocalTableIndex) {
        let Some(base) = self.fixed_funcref_table_ptr(table_index) else {
            return;
        };
        let table = self.tables[table_index].get(self.context());
        for index in 0..table.size() {
            let TableElement::FuncRef(funcref) = table.get(index).unwrap() else {
                unreachable!("fixed funcref tables cannot contain externrefs");
            };
            unsafe {
                *base.as_ptr().add(index as usize) = anyfunc_from_funcref(funcref);
            }
        }
    }

    fn sync_fixed_funcref_table_by_index(&self, table_index: TableIndex) {
        if let Some(local_table_index) = self.module.local_table_index(table_index) {
            self.sync_fixed_funcref_table(local_table_index);
        }
    }

    /// Get a locally defined or imported memory.
    fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition {
        if let Some(local_index) = self.module.local_memory_index(index) {
            self.memory(local_index)
        } else {
            let import = self.imported_memory(index);
            unsafe { *import.definition.as_ref() }
        }
    }

    /// Return the indexed `VMMemoryDefinition`.
    fn memory(&self, index: LocalMemoryIndex) -> VMMemoryDefinition {
        unsafe { *self.memory_ptr(index).as_ref() }
    }

    #[allow(dead_code)]
    /// Set the indexed memory to `VMMemoryDefinition`.
    fn set_memory(&self, index: LocalMemoryIndex, mem: &VMMemoryDefinition) {
        unsafe {
            *self.memory_ptr(index).as_ptr() = *mem;
        }
    }

    /// Return the indexed `VMMemoryDefinition`.
    fn memory_ptr(&self, index: LocalMemoryIndex) -> NonNull<VMMemoryDefinition> {
        let index = usize::try_from(index.as_u32()).unwrap();
        NonNull::new(unsafe { self.memories_ptr().add(index) }).unwrap()
    }

    /// Return a pointer to the `VMMemoryDefinition`s.
    fn memories_ptr(&self) -> *mut VMMemoryDefinition {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_memories_begin()) }
    }

    /// Get a locally defined or imported memory.
    fn get_vmmemory(&self, index: MemoryIndex) -> &VMMemory {
        if let Some(local_index) = self.module.local_memory_index(index) {
            unsafe {
                self.memories
                    .get(local_index)
                    .unwrap()
                    .get(self.context.as_ref().unwrap())
            }
        } else {
            let import = self.imported_memory(index);
            unsafe { import.handle.get(self.context.as_ref().unwrap()) }
        }
    }

    /// Get a locally defined or imported memory.
    fn get_vmmemory_mut(&mut self, index: MemoryIndex) -> &mut VMMemory {
        if let Some(local_index) = self.module.local_memory_index(index) {
            unsafe {
                self.memories
                    .get_mut(local_index)
                    .unwrap()
                    .get_mut(self.context.as_mut().unwrap())
            }
        } else {
            let import = self.imported_memory(index);
            unsafe { import.handle.get_mut(self.context.as_mut().unwrap()) }
        }
    }

    /// Get a locally defined memory as mutable.
    fn get_local_vmmemory_mut(&mut self, local_index: LocalMemoryIndex) -> &mut VMMemory {
        unsafe {
            self.memories
                .get_mut(local_index)
                .unwrap()
                .get_mut(self.context.as_mut().unwrap())
        }
    }

    /// Return the indexed `VMGlobalDefinition`.
    fn global(&self, index: LocalGlobalIndex) -> VMGlobalDefinition {
        unsafe { self.global_ptr(index).as_ref().clone() }
    }

    /// Set the indexed global to `VMGlobalDefinition`.
    #[allow(dead_code)]
    fn set_global(&self, index: LocalGlobalIndex, global: &VMGlobalDefinition) {
        unsafe {
            *self.global_ptr(index).as_ptr() = global.clone();
        }
    }

    /// Return the indexed `VMGlobalDefinition`.
    fn global_ptr(&self, index: LocalGlobalIndex) -> NonNull<VMGlobalDefinition> {
        let index = usize::try_from(index.as_u32()).unwrap();
        NonNull::new(unsafe { self.globals_ptr().add(index) }).unwrap()
    }

    /// Return a pointer to the `VMGlobalDefinition`s.
    fn globals_ptr(&self) -> *mut VMGlobalDefinition {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_globals_begin()) }
    }

    /// Return a pointer to the `VMBuiltinFunctionsArray`.
    fn builtin_functions_ptr(&self) -> *mut VMBuiltinFunctionsArray {
        unsafe { self.vmctx_plus_offset(self.offsets.vmctx_builtin_functions_begin()) }
    }

    /// Return a reference to the vmctx used by compiled wasm code.
    fn vmctx(&self) -> &VMContext {
        &self.vmctx
    }

    /// Return a raw pointer to the vmctx used by compiled wasm code.
    fn vmctx_ptr(&self) -> *mut VMContext {
        self.vmctx() as *const VMContext as *mut VMContext
    }

    /// Invoke the WebAssembly start function of the instance, if one is present.
    fn invoke_start_function(
        &self,
        config: &VMConfig,
        trap_handler: Option<*const TrapHandlerFn<'static>>,
    ) -> Result<(), Trap> {
        let start_index = match self.module.start_function {
            Some(idx) => idx,
            None => return Ok(()),
        };

        let (callee_address, callee_vmctx) = match self.module.local_func_index(start_index) {
            Some(local_index) => {
                let body = self
                    .functions
                    .get(local_index)
                    .expect("function index is out of bounds")
                    .0;
                (
                    body as *const _,
                    VMFunctionContext {
                        vmctx: self.vmctx_ptr(),
                    },
                )
            }
            None => {
                assert_lt!(start_index.index(), self.module.num_imported_functions);
                let import = self.imported_function(start_index);
                (import.body, import.environment)
            }
        };

        let sig = self.module.functions[start_index];
        let trampoline = self.function_call_trampolines[sig];
        let mut values_vec = vec![];

        unsafe {
            // Even though we already know the type of the function we need to call, in certain
            // specific cases trampoline prepare callee arguments for specific optimizations, such
            // as passing g0 and m0_base_ptr as parameters.
            wasmer_call_trampoline(
                trap_handler,
                config,
                callee_vmctx,
                trampoline,
                callee_address,
                values_vec.as_mut_ptr(),
            )
        }
    }

    /// Return the offset from the vmctx pointer to its containing `Instance`.
    #[inline]
    pub(crate) fn vmctx_offset() -> isize {
        offset_of!(Self, vmctx) as isize
    }

    /// Return the table index for the given `VMTableDefinition`.
    pub(crate) fn table_index(&self, table: &VMTableDefinition) -> LocalTableIndex {
        let begin: *const VMTableDefinition = self.tables_ptr() as *const _;
        let end: *const VMTableDefinition = table;
        // TODO: Use `offset_from` once it stabilizes.
        let index = LocalTableIndex::new(
            (end as usize - begin as usize) / mem::size_of::<VMTableDefinition>(),
        );
        assert_lt!(index.index(), self.tables.len());
        index
    }

    /// Return the memory index for the given `VMMemoryDefinition`.
    pub(crate) fn memory_index(&self, memory: &VMMemoryDefinition) -> LocalMemoryIndex {
        let begin: *const VMMemoryDefinition = self.memories_ptr() as *const _;
        let end: *const VMMemoryDefinition = memory;
        // TODO: Use `offset_from` once it stabilizes.
        let index = LocalMemoryIndex::new(
            (end as usize - begin as usize) / mem::size_of::<VMMemoryDefinition>(),
        );
        assert_lt!(index.index(), self.memories.len());
        index
    }

    /// Grow memory by the specified amount of pages.
    ///
    /// Returns `None` if memory can't be grown by the specified amount
    /// of pages.
    pub(crate) fn memory_grow<IntoPages>(
        &mut self,
        memory_index: LocalMemoryIndex,
        delta: IntoPages,
    ) -> Result<Pages, MemoryError>
    where
        IntoPages: Into<Pages>,
    {
        let mem = *self
            .memories
            .get(memory_index)
            .unwrap_or_else(|| panic!("no memory for index {}", memory_index.index()));
        mem.get_mut(self.context_mut()).grow(delta.into())
    }

    /// Grow imported memory by the specified amount of pages.
    ///
    /// Returns `None` if memory can't be grown by the specified amount
    /// of pages.
    ///
    /// # Safety
    /// This and `imported_memory_size` are currently unsafe because they
    /// dereference the memory import's pointers.
    pub(crate) unsafe fn imported_memory_grow<IntoPages>(
        &mut self,
        memory_index: MemoryIndex,
        delta: IntoPages,
    ) -> Result<Pages, MemoryError>
    where
        IntoPages: Into<Pages>,
    {
        let import = self.imported_memory(memory_index);
        let mem = import.handle;
        mem.get_mut(self.context_mut()).grow(delta.into())
    }

    /// Returns the number of allocated wasm pages.
    pub(crate) fn memory_size(&self, memory_index: LocalMemoryIndex) -> Pages {
        let mem = *self
            .memories
            .get(memory_index)
            .unwrap_or_else(|| panic!("no memory for index {}", memory_index.index()));
        mem.get(self.context()).size()
    }

    /// Returns the number of allocated wasm pages in an imported memory.
    ///
    /// # Safety
    /// This and `imported_memory_grow` are currently unsafe because they
    /// dereference the memory import's pointers.
    pub(crate) unsafe fn imported_memory_size(&self, memory_index: MemoryIndex) -> Pages {
        let import = self.imported_memory(memory_index);
        let mem = import.handle;
        mem.get(self.context()).size()
    }

    /// Returns the number of elements in a given table.
    pub(crate) fn table_size(&self, table_index: LocalTableIndex) -> u32 {
        let table = self
            .tables
            .get(table_index)
            .unwrap_or_else(|| panic!("no table for index {}", table_index.index()));
        table.get(self.context()).size()
    }

    /// Returns the number of elements in a given imported table.
    ///
    /// # Safety
    /// `table_index` must be a valid, imported table index.
    pub(crate) unsafe fn imported_table_size(&self, table_index: TableIndex) -> u32 {
        let import = self.imported_table(table_index);
        let table = import.handle;
        table.get(self.context()).size()
    }

    /// Grow table by the specified amount of elements.
    ///
    /// Returns `None` if table can't be grown by the specified amount
    /// of elements.
    pub(crate) fn table_grow(
        &mut self,
        table_index: LocalTableIndex,
        delta: u32,
        init_value: TableElement,
    ) -> Option<u32> {
        let table = *self
            .tables
            .get(table_index)
            .unwrap_or_else(|| panic!("no table for index {}", table_index.index()));
        table.get_mut(self.context_mut()).grow(delta, init_value)
    }

    /// Grow table by the specified amount of elements.
    ///
    /// # Safety
    /// `table_index` must be a valid, imported table index.
    pub(crate) unsafe fn imported_table_grow(
        &mut self,
        table_index: TableIndex,
        delta: u32,
        init_value: TableElement,
    ) -> Option<u32> {
        let import = self.imported_table(table_index);
        let table = import.handle;
        table.get_mut(self.context_mut()).grow(delta, init_value)
    }

    /// Get table element by index.
    pub(crate) fn table_get(
        &self,
        table_index: LocalTableIndex,
        index: u32,
    ) -> Option<TableElement> {
        let table = self
            .tables
            .get(table_index)
            .unwrap_or_else(|| panic!("no table for index {}", table_index.index()));
        table.get(self.context()).get(index)
    }

    /// Returns the element at the given index.
    ///
    /// # Safety
    /// `table_index` must be a valid, imported table index.
    pub(crate) unsafe fn imported_table_get(
        &self,
        table_index: TableIndex,
        index: u32,
    ) -> Option<TableElement> {
        let import = self.imported_table(table_index);
        let table = import.handle;
        table.get(self.context()).get(index)
    }

    /// Set table element by index.
    pub(crate) fn table_set(
        &mut self,
        table_index: LocalTableIndex,
        index: u32,
        val: TableElement,
    ) -> Result<(), Trap> {
        let table = *self
            .tables
            .get(table_index)
            .unwrap_or_else(|| panic!("no table for index {}", table_index.index()));
        let funcref = match &val {
            TableElement::FuncRef(funcref) => Some(*funcref),
            TableElement::ExternRef(_) => None,
        };
        table.get_mut(self.context_mut()).set(index, val)?;
        if let Some(funcref) = funcref {
            self.sync_fixed_funcref_table_element(table_index, index, funcref);
        }
        Ok(())
    }

    /// Set table element by index for an imported table.
    ///
    /// # Safety
    /// `table_index` must be a valid, imported table index.
    pub(crate) unsafe fn imported_table_set(
        &mut self,
        table_index: TableIndex,
        index: u32,
        val: TableElement,
    ) -> Result<(), Trap> {
        let import = self.imported_table(table_index);
        let table = import.handle;
        table.get_mut(self.context_mut()).set(index, val)
    }

    /// Get a `VMFuncRef` for the given `FunctionIndex`.
    pub(crate) fn func_ref(&self, function_index: FunctionIndex) -> Option<VMFuncRef> {
        if function_index == FunctionIndex::reserved_value() {
            None
        } else if let Some(local_function_index) = self.module.local_func_index(function_index) {
            Some(VMFuncRef(NonNull::from(
                &self.funcrefs[local_function_index],
            )))
        } else {
            Some(VMFuncRef(self.imported_funcrefs[function_index]))
        }
    }

    /// The `table.init` operation: initializes a portion of a table with a
    /// passive element.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error when the range within the table is out of bounds
    /// or the range within the passive element is out of bounds.
    pub(crate) fn table_init(
        &mut self,
        table_index: TableIndex,
        elem_index: ElemIndex,
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init

        let table = self.get_table_handle(table_index);
        let table = unsafe { table.get_mut(&mut *self.context) };
        let passive_elements = self.passive_elements.borrow();
        let elem = passive_elements
            .get(&elem_index)
            .map_or::<&[Option<VMFuncRef>], _>(&[], |e| &**e);

        if src.checked_add(len).is_none_or(|n| n as usize > elem.len())
            || dst.checked_add(len).is_none_or(|m| m > table.size())
        {
            return Err(Trap::lib(TrapCode::TableAccessOutOfBounds));
        }

        for (dst, src) in (dst..dst + len).zip(src..src + len) {
            table
                .set(dst, TableElement::FuncRef(elem[src as usize]))
                .expect("should never panic because we already did the bounds check above");
        }

        self.sync_fixed_funcref_table_by_index(table_index);

        Ok(())
    }

    /// The `table.fill` operation: fills a portion of a table with a given value.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error when the range within the table is out of bounds
    pub(crate) fn table_fill(
        &mut self,
        table_index: TableIndex,
        start_index: u32,
        item: TableElement,
        len: u32,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init

        let table = self.get_table(table_index);
        let table_size = table.size() as usize;

        if start_index
            .checked_add(len)
            .is_none_or(|n| n as usize > table_size)
        {
            return Err(Trap::lib(TrapCode::TableAccessOutOfBounds));
        }

        for i in start_index..(start_index + len) {
            table
                .set(i, item.clone())
                .expect("should never panic because we already did the bounds check above");
        }

        self.sync_fixed_funcref_table_by_index(table_index);

        Ok(())
    }

    /// The `table.copy` operation.
    pub(crate) fn table_copy(
        &mut self,
        dst_table_index: TableIndex,
        src_table_index: TableIndex,
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        let result = if dst_table_index == src_table_index {
            let table = self.get_table(dst_table_index);
            table.copy_within(dst, src, len)
        } else {
            let dst_table = self.get_table_handle(dst_table_index);
            let src_table = self.get_table_handle(src_table_index);
            if dst_table == src_table {
                unsafe {
                    dst_table
                        .get_mut(&mut *self.context)
                        .copy_within(dst, src, len)
                }
            } else {
                unsafe {
                    dst_table.get_mut(&mut *self.context).copy(
                        src_table.get(&*self.context),
                        dst,
                        src,
                        len,
                    )
                }
            }
        };
        result?;
        self.sync_fixed_funcref_table_by_index(dst_table_index);

        Ok(())
    }

    /// Drop an element.
    pub(crate) fn elem_drop(&self, elem_index: ElemIndex) {
        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop

        let mut passive_elements = self.passive_elements.borrow_mut();
        passive_elements.remove(&elem_index);
        // Note that we don't check that we actually removed an element because
        // dropping a non-passive element is a no-op (not a trap).
    }

    /// Perform a `memory.copy` between two memories.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error when the source or destination range is out of
    /// bounds.
    pub(crate) fn memory_copy(
        &self,
        dst_memory_index: MemoryIndex,
        src_memory_index: MemoryIndex,
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        let dst_memory = self.get_memory(dst_memory_index);
        let src_memory = self.get_memory(src_memory_index);
        // The following memory copy is not synchronized and is not atomic.
        unsafe { memory_copy(&dst_memory, &src_memory, dst, src, len) }
    }

    /// Perform the `memory.fill` operation on a locally defined memory.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error if the memory range is out of bounds.
    pub(crate) fn local_memory_fill(
        &self,
        memory_index: LocalMemoryIndex,
        dst: u32,
        val: u32,
        len: u32,
    ) -> Result<(), Trap> {
        let memory = self.memory(memory_index);
        // The following memory fill is not synchronized and is not atomic:
        unsafe { memory_fill(&memory, dst, val, len) }
    }

    /// Perform the `memory.fill` operation on an imported memory.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error if the memory range is out of bounds.
    pub(crate) fn imported_memory_fill(
        &self,
        memory_index: MemoryIndex,
        dst: u32,
        val: u32,
        len: u32,
    ) -> Result<(), Trap> {
        let import = self.imported_memory(memory_index);
        let memory = unsafe { import.definition.as_ref() };
        // The following memory fill is not synchronized and is not atomic:
        unsafe { memory_fill(memory, dst, val, len) }
    }

    /// Performs the `memory.init` operation.
    ///
    /// # Errors
    ///
    /// Returns a `Trap` error if the destination range is out of this module's
    /// memory's bounds or if the source range is outside the data segment's
    /// bounds.
    pub(crate) fn memory_init(
        &self,
        memory_index: MemoryIndex,
        data_index: DataIndex,
        dst: u32,
        src: u32,
        len: u32,
    ) -> Result<(), Trap> {
        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init

        let memory = self.get_vmmemory(memory_index);
        let passive_data = self.passive_data.borrow();
        // A missing entry (never existed) or a dropped one (`None`) both behave
        // as a zero-length segment, so an in-bounds `memory.init` of non-zero
        // length traps below.
        let data = passive_data
            .get(&data_index)
            .and_then(|d| d.as_deref())
            .unwrap_or(&[]);

        let current_length = unsafe { memory.vmmemory().as_ref().current_length };
        if src.checked_add(len).is_none_or(|n| n as usize > data.len())
            || dst
                .checked_add(len)
                .is_none_or(|m| usize::try_from(m).unwrap() > current_length)
        {
            return Err(Trap::lib(TrapCode::HeapAccessOutOfBounds));
        }
        let src_slice = &data[src as usize..(src + len) as usize];
        unsafe { memory.initialize_with_data(dst as usize, src_slice) }
    }

    /// Drop the given data segment, truncating its length to zero.
    pub(crate) fn data_drop(&self, data_index: DataIndex) {
        let mut passive_data = self.passive_data.borrow_mut();
        // Release this instance's reference to the shared bytes and mark the
        // segment unusable. Other instances (and the module) are unaffected.
        if let Some(slot) = passive_data.get_mut(&data_index) {
            *slot = None;
        }
    }

    /// Get a table by index regardless of whether it is locally-defined or an
    /// imported, foreign table.
    pub(crate) fn get_table(&mut self, table_index: TableIndex) -> &mut VMTable {
        if let Some(local_table_index) = self.module.local_table_index(table_index) {
            self.get_local_table(local_table_index)
        } else {
            self.get_foreign_table(table_index)
        }
    }

    /// Get a locally-defined table.
    pub(crate) fn get_local_table(&mut self, index: LocalTableIndex) -> &mut VMTable {
        let table = self.tables[index];
        table.get_mut(self.context_mut())
    }

    /// Get an imported, foreign table.
    pub(crate) fn get_foreign_table(&mut self, index: TableIndex) -> &mut VMTable {
        let import = self.imported_table(index);
        let table = import.handle;
        table.get_mut(self.context_mut())
    }

    /// Get a table handle by index regardless of whether it is locally-defined
    /// or an imported, foreign table.
    pub(crate) fn get_table_handle(
        &mut self,
        table_index: TableIndex,
    ) -> InternalStoreHandle<VMTable> {
        if let Some(local_table_index) = self.module.local_table_index(table_index) {
            self.tables[local_table_index]
        } else {
            self.imported_table(table_index).handle
        }
    }

    /// # Safety
    /// See [`LinearMemory::do_wait`].
    unsafe fn memory_wait(
        memory: &mut VMMemory,
        dst: u32,
        expected: ExpectedValue,
        timeout: i64,
    ) -> Result<u32, Trap> {
        let timeout = if timeout < 0 {
            None
        } else {
            Some(std::time::Duration::from_nanos(timeout as u64))
        };
        match unsafe { memory.do_wait(dst, expected, timeout) } {
            Ok(count) => Ok(count),
            Err(_err) => Err(Trap::lib(TrapCode::HostInterrupt)),
        }
    }

    /// Perform an Atomic.Wait32
    pub(crate) fn local_memory_wait32(
        &mut self,
        memory_index: LocalMemoryIndex,
        dst: u32,
        val: u32,
        timeout: i64,
    ) -> Result<u32, Trap> {
        let memory = self.memory(memory_index);
        //if ! memory.shared {
        // We should trap according to spec, but official test rely on not trapping...
        //}

        // Do a fast-path check of the expected value, and also ensure proper alignment
        let ret = unsafe { memory32_atomic_check32(&memory, dst, val) };

        if let Ok(mut ret) = ret {
            if ret == 0 {
                let memory = self.get_local_vmmemory_mut(memory_index);
                // Safety: we have already checked alignment and bounds in memory32_atomic_check32
                ret = unsafe { Self::memory_wait(memory, dst, ExpectedValue::U32(val), timeout)? };
            }
            Ok(ret)
        } else {
            ret
        }
    }

    /// Perform an Atomic.Wait32
    pub(crate) fn imported_memory_wait32(
        &mut self,
        memory_index: MemoryIndex,
        dst: u32,
        val: u32,
        timeout: i64,
    ) -> Result<u32, Trap> {
        let import = self.imported_memory(memory_index);
        let memory = unsafe { import.definition.as_ref() };
        //if ! memory.shared {
        // We should trap according to spec, but official test rely on not trapping...
        //}

        // Do a fast-path check of the expected value, and also ensure proper alignment
        let ret = unsafe { memory32_atomic_check32(memory, dst, val) };

        if let Ok(mut ret) = ret {
            if ret == 0 {
                let memory = self.get_vmmemory_mut(memory_index);
                // Safety: we have already checked alignment and bounds in memory32_atomic_check32
                ret = unsafe { Self::memory_wait(memory, dst, ExpectedValue::U32(val), timeout)? };
            }
            Ok(ret)
        } else {
            ret
        }
    }

    /// Perform an Atomic.Wait64
    pub(crate) fn local_memory_wait64(
        &mut self,
        memory_index: LocalMemoryIndex,
        dst: u32,
        val: u64,
        timeout: i64,
    ) -> Result<u32, Trap> {
        let memory = self.memory(memory_index);
        //if ! memory.shared {
        // We should trap according to spec, but official test rely on not trapping...
        //}

        // Do a fast-path check of the expected value, and also ensure proper alignment
        let ret = unsafe { memory32_atomic_check64(&memory, dst, val) };

        if let Ok(mut ret) = ret {
            if ret == 0 {
                let memory = self.get_local_vmmemory_mut(memory_index);
                // Safety: we have already checked alignment and bounds in memory32_atomic_check64
                ret = unsafe { Self::memory_wait(memory, dst, ExpectedValue::U64(val), timeout)? };
            }
            Ok(ret)
        } else {
            ret
        }
    }

    /// Perform an Atomic.Wait64
    pub(crate) fn imported_memory_wait64(
        &mut self,
        memory_index: MemoryIndex,
        dst: u32,
        val: u64,
        timeout: i64,
    ) -> Result<u32, Trap> {
        let import = self.imported_memory(memory_index);
        let memory = unsafe { import.definition.as_ref() };
        //if ! memory.shared {
        // We should trap according to spec, but official test rely on not trapping...
        //}

        // Do a fast-path check of the expected value, and also ensure proper alignment
        let ret = unsafe { memory32_atomic_check64(memory, dst, val) };

        if let Ok(mut ret) = ret {
            if ret == 0 {
                let memory = self.get_vmmemory_mut(memory_index);
                // Safety: we have already checked alignment and bounds in memory32_atomic_check64
                ret = unsafe { Self::memory_wait(memory, dst, ExpectedValue::U64(val), timeout)? };
            }
            Ok(ret)
        } else {
            ret
        }
    }

    /// Perform an Atomic.Notify
    pub(crate) fn local_memory_notify(
        &mut self,
        memory_index: LocalMemoryIndex,
        dst: u32,
        count: u32,
    ) -> Result<u32, Trap> {
        let memory = self.memory(memory_index);
        memory32_atomic_check_notify(&memory, dst)?;
        let memory = self.get_local_vmmemory_mut(memory_index);
        Ok(memory.do_notify(dst, count))
    }

    /// Perform an Atomic.Notify
    pub(crate) fn imported_memory_notify(
        &mut self,
        memory_index: MemoryIndex,
        dst: u32,
        count: u32,
    ) -> Result<u32, Trap> {
        let import = self.imported_memory(memory_index);
        let memory = unsafe { import.definition.as_ref() };
        memory32_atomic_check_notify(memory, dst)?;
        let memory = self.get_vmmemory_mut(memory_index);
        Ok(memory.do_notify(dst, count))
    }
}

/// A handle holding an `Instance` of a WebAssembly module.
///
/// This is more or less a public facade of the private `Instance`,
/// providing useful higher-level API.
#[derive(Debug, Eq, PartialEq)]
pub struct VMInstance {
    /// The layout of `Instance` (which can vary).
    instance_layout: Layout,

    /// The `Instance` itself.
    ///
    /// `Instance` must not be dropped manually by Rust, because it's
    /// allocated manually with `alloc` and a specific layout (Rust
    /// would be able to drop `Instance` itself but it will imply a
    /// memory leak because of `alloc`).
    ///
    /// No one in the code has a copy of the `Instance`'s
    /// pointer. `Self` is the only one.
    instance: NonNull<Instance>,
}

/// VMInstance are created with an InstanceAllocator
/// and it will "consume" the memory
/// So the Drop here actually free it (else it would be leaked)
impl Drop for VMInstance {
    fn drop(&mut self) {
        let instance_ptr = self.instance.as_ptr();

        unsafe {
            // Need to drop all the actual Instance members
            instance_ptr.drop_in_place();
            // And then free the memory allocated for the Instance itself
            std::alloc::dealloc(instance_ptr as *mut u8, self.instance_layout);
        }
    }
}

impl VMInstance {
    /// Create a new `VMInstance` pointing at freshly allocated instance data.
    ///
    /// # Safety
    ///
    /// This method is not necessarily inherently unsafe to call, but in general
    /// the APIs of an `Instance` are quite unsafe and have not been really
    /// audited for safety that much. As a result the unsafety here on this
    /// method is a low-overhead way of saying “this is an extremely unsafe type
    /// to work with”.
    ///
    /// Extreme care must be taken when working with `VMInstance` and it's
    /// recommended to have relatively intimate knowledge of how it works
    /// internally if you'd like to do so. If possible it's recommended to use
    /// the `wasmer` crate API rather than this type since that is vetted for
    /// safety.
    ///
    /// However the following must be taken care of before calling this function:
    /// - The memory at `instance.tables_ptr()` must be initialized with data for
    ///   all the local tables.
    /// - The memory at `instance.memories_ptr()` must be initialized with data for
    ///   all the local memories.
    #[allow(clippy::too_many_arguments)]
    pub unsafe fn new(
        allocator: InstanceAllocator,
        module: Arc<ModuleInfo>,
        context: &mut StoreObjects,
        finished_functions: BoxedSlice<LocalFunctionIndex, FunctionBodyPtr>,
        finished_function_call_trampolines: BoxedSlice<SignatureIndex, VMTrampoline>,
        finished_memories: BoxedSlice<LocalMemoryIndex, InternalStoreHandle<VMMemory>>,
        finished_tables: BoxedSlice<LocalTableIndex, InternalStoreHandle<VMTable>>,
        finished_globals: BoxedSlice<LocalGlobalIndex, InternalStoreHandle<VMGlobal>>,
        tags: BoxedSlice<TagIndex, InternalStoreHandle<VMTag>>,
        imports: Imports,
        vmshared_signatures: BoxedSlice<SignatureIndex, VMSignatureHash>,
    ) -> Result<Self, Trap> {
        unsafe {
            let vmctx_tags = tags
                .values()
                .map(|m: &InternalStoreHandle<VMTag>| VMSharedTagIndex::new(m.index() as u32))
                .collect::<PrimaryMap<TagIndex, VMSharedTagIndex>>()
                .into_boxed_slice();
            // Share the module's passive data bytes via `Arc` (refcount bump)
            // rather than deep-copying them into every instance.
            let passive_data = RefCell::new(
                module
                    .passive_data
                    .iter()
                    .map(|(&idx, bytes)| (idx, Some(Arc::clone(bytes))))
                    .collect::<HashMap<_, _>>(),
            );

            let handle = {
                let offsets = allocator.offsets().clone();
                // use dummy value to create an instance so we can get the vmctx pointer
                let funcrefs = PrimaryMap::new().into_boxed_slice();
                let imported_funcrefs = PrimaryMap::new().into_boxed_slice();
                // Create the `Instance`. The unique, the One.
                let instance = Instance {
                    module,
                    context,
                    offsets,
                    memories: finished_memories,
                    tables: finished_tables,
                    tags,
                    globals: finished_globals,
                    functions: finished_functions,
                    function_call_trampolines: finished_function_call_trampolines,
                    passive_elements: Default::default(),
                    passive_data,
                    funcrefs,
                    imported_funcrefs,
                    vmctx: VMContext {},
                };

                let mut instance_handle = allocator.into_vminstance(instance);

                // Set the funcrefs after we've built the instance
                {
                    let instance = instance_handle.instance_mut();
                    let vmctx_ptr = instance.vmctx_ptr();
                    (instance.funcrefs, instance.imported_funcrefs) = build_funcrefs(
                        &instance.module,
                        context,
                        &imports,
                        &instance.functions,
                        &vmshared_signatures,
                        &instance.function_call_trampolines,
                        vmctx_ptr,
                    );
                    for local_table_index in instance.tables.keys() {
                        instance.sync_fixed_funcref_table(local_table_index);
                    }
                }

                instance_handle
            };
            let instance = handle.instance();

            ptr::copy(
                vmctx_tags.values().as_slice().as_ptr(),
                instance.shared_tags_ptr(),
                vmctx_tags.len(),
            );
            ptr::copy(
                imports.functions.values().as_slice().as_ptr(),
                instance.imported_functions_ptr(),
                imports.functions.len(),
            );
            ptr::copy(
                imports.tables.values().as_slice().as_ptr(),
                instance.imported_tables_ptr(),
                imports.tables.len(),
            );
            ptr::copy(
                imports.memories.values().as_slice().as_ptr(),
                instance.imported_memories_ptr(),
                imports.memories.len(),
            );
            ptr::copy(
                imports.globals.values().as_slice().as_ptr(),
                instance.imported_globals_ptr(),
                imports.globals.len(),
            );
            // these should already be set, add asserts here? for:
            // - instance.tables_ptr() as *mut VMTableDefinition
            // - instance.memories_ptr() as *mut VMMemoryDefinition
            ptr::write(
                instance.builtin_functions_ptr(),
                VMBuiltinFunctionsArray::initialized(),
            );

            // Perform infallible initialization in this constructor, while fallible
            // initialization is deferred to the `initialize` method.
            initialize_passive_elements(instance);
            initialize_globals(instance);

            Ok(handle)
        }
    }

    /// Return a reference to the contained `Instance`.
    pub(crate) fn instance(&self) -> &Instance {
        unsafe { self.instance.as_ref() }
    }

    /// Return a mutable reference to the contained `Instance`.
    pub(crate) fn instance_mut(&mut self) -> &mut Instance {
        unsafe { self.instance.as_mut() }
    }

    /// Finishes the instantiation process started by `Instance::new`.
    ///
    /// # Safety
    ///
    /// Only safe to call immediately after instantiation.
    pub unsafe fn finish_instantiation(
        &mut self,
        config: &VMConfig,
        trap_handler: Option<*const TrapHandlerFn<'static>>,
        data_initializers: &[DataInitializer<'_>],
    ) -> Result<(), Trap> {
        let instance = self.instance_mut();

        // Apply the initializers.
        initialize_tables(instance)?;
        initialize_memories(instance, data_initializers)?;

        // The WebAssembly spec specifies that the start function is
        // invoked automatically at instantiation time.
        instance.invoke_start_function(config, trap_handler)?;
        Ok(())
    }

    /// Return a reference to the vmctx used by compiled wasm code.
    pub fn vmctx(&self) -> &VMContext {
        self.instance().vmctx()
    }

    /// Return a raw pointer to the vmctx used by compiled wasm code.
    pub fn vmctx_ptr(&self) -> *mut VMContext {
        self.instance().vmctx_ptr()
    }

    /// Return a reference to the `VMOffsets` to get offsets in the
    /// `Self::vmctx_ptr` region. Be careful when doing pointer
    /// arithmetic!
    pub fn vmoffsets(&self) -> &VMOffsets {
        self.instance().offsets()
    }

    /// Return a reference-counting pointer to a module.
    pub fn module(&self) -> &Arc<ModuleInfo> {
        self.instance().module()
    }

    /// Return a reference to a module.
    pub fn module_ref(&self) -> &ModuleInfo {
        self.instance().module_ref()
    }

    /// Lookup an export with the given name.
    pub fn lookup(&mut self, field: &str) -> Option<VMExtern> {
        let export = *self.module_ref().exports.get(field)?;

        Some(self.lookup_by_declaration(export))
    }

    /// Lookup an export with the given export declaration.
    pub fn lookup_by_declaration(&mut self, export: ExportIndex) -> VMExtern {
        let instance = self.instance();

        match export {
            ExportIndex::Function(index) => {
                let sig_index = &instance.module.functions[index];
                let handle = if let Some(def_index) = instance.module.local_func_index(index) {
                    // A VMFunction is lazily created only for functions that are
                    // exported.
                    let signature = instance.module.signatures[*sig_index].clone();
                    let vm_function = VMFunction {
                        anyfunc: MaybeInstanceOwned::Instance(NonNull::from(
                            &instance.funcrefs[def_index],
                        )),
                        signature,
                        // Any function received is already static at this point as:
                        // 1. All locally defined functions in the Wasm have a static signature.
                        // 2. All the imported functions are already static (because
                        //    they point to the trampolines rather than the dynamic addresses).
                        kind: VMFunctionKind::Static,
                        host_data: Box::new(()),
                    };
                    InternalStoreHandle::new(self.instance_mut().context_mut(), vm_function)
                } else {
                    let import = instance.imported_function(index);
                    import.handle
                };

                VMExtern::Function(handle)
            }
            ExportIndex::Table(index) => {
                let handle = if let Some(def_index) = instance.module.local_table_index(index) {
                    instance.tables[def_index]
                } else {
                    let import = instance.imported_table(index);
                    import.handle
                };
                VMExtern::Table(handle)
            }
            ExportIndex::Memory(index) => {
                let handle = if let Some(def_index) = instance.module.local_memory_index(index) {
                    instance.memories[def_index]
                } else {
                    let import = instance.imported_memory(index);
                    import.handle
                };
                VMExtern::Memory(handle)
            }
            ExportIndex::Global(index) => {
                let handle = if let Some(def_index) = instance.module.local_global_index(index) {
                    instance.globals[def_index]
                } else {
                    let import = instance.imported_global(index);
                    import.handle
                };
                VMExtern::Global(handle)
            }

            ExportIndex::Tag(index) => {
                let handle = instance.tags[index];
                VMExtern::Tag(handle)
            }
        }
    }

    /// Return an iterator over the exports of this instance.
    ///
    /// Specifically, it provides access to the key-value pairs, where the keys
    /// are export names, and the values are export declarations which can be
    /// resolved `lookup_by_declaration`.
    pub fn exports(&self) -> indexmap::map::Iter<'_, String, ExportIndex> {
        self.module().exports.iter()
    }

    /// Return the memory index for the given `VMMemoryDefinition` in this instance.
    pub fn memory_index(&self, memory: &VMMemoryDefinition) -> LocalMemoryIndex {
        self.instance().memory_index(memory)
    }

    /// Grow memory in this instance by the specified amount of pages.
    ///
    /// Returns `None` if memory can't be grown by the specified amount
    /// of pages.
    pub fn memory_grow<IntoPages>(
        &mut self,
        memory_index: LocalMemoryIndex,
        delta: IntoPages,
    ) -> Result<Pages, MemoryError>
    where
        IntoPages: Into<Pages>,
    {
        self.instance_mut().memory_grow(memory_index, delta)
    }

    /// Return the table index for the given `VMTableDefinition` in this instance.
    pub fn table_index(&self, table: &VMTableDefinition) -> LocalTableIndex {
        self.instance().table_index(table)
    }

    /// Grow table in this instance by the specified amount of pages.
    ///
    /// Returns `None` if memory can't be grown by the specified amount
    /// of pages.
    pub fn table_grow(
        &mut self,
        table_index: LocalTableIndex,
        delta: u32,
        init_value: TableElement,
    ) -> Option<u32> {
        self.instance_mut()
            .table_grow(table_index, delta, init_value)
    }

    /// Get table element reference.
    ///
    /// Returns `None` if index is out of bounds.
    pub fn table_get(&self, table_index: LocalTableIndex, index: u32) -> Option<TableElement> {
        self.instance().table_get(table_index, index)
    }

    /// Set table element reference.
    ///
    /// Returns an error if the index is out of bounds
    pub fn table_set(
        &mut self,
        table_index: LocalTableIndex,
        index: u32,
        val: TableElement,
    ) -> Result<(), Trap> {
        self.instance_mut().table_set(table_index, index, val)
    }

    /// Get a table defined locally within this module.
    pub fn get_local_table(&mut self, index: LocalTableIndex) -> &mut VMTable {
        self.instance_mut().get_local_table(index)
    }
}

#[allow(clippy::mut_from_ref)]
#[allow(dead_code)]
/// Return a byte-slice view of a memory's data.
unsafe fn get_memory_slice<'instance>(
    init: &DataInitializer<'_>,
    instance: &'instance Instance,
) -> &'instance mut [u8] {
    unsafe {
        let memory = if let Some(local_memory_index) = instance
            .module
            .local_memory_index(init.location.memory_index)
        {
            instance.memory(local_memory_index)
        } else {
            let import = instance.imported_memory(init.location.memory_index);
            *import.definition.as_ref()
        };
        slice::from_raw_parts_mut(memory.base, memory.current_length)
    }
}

fn get_global(index: GlobalIndex, instance: &Instance) -> RawValue {
    unsafe {
        if let Some(local_global_index) = instance.module.local_global_index(index) {
            instance.global(local_global_index).val
        } else {
            instance.imported_global(index).definition.as_ref().val
        }
    }
}

enum EvaluatedInitExpr {
    I32(i32),
    I64(i64),
}

fn eval_init_expr(expr: &InitExpr, instance: &Instance) -> EvaluatedInitExpr {
    if expr
        .ops()
        .first()
        .expect("missing expression")
        .is_32bit_expression()
    {
        let mut stack = Vec::with_capacity(expr.ops().len());
        for op in expr.ops() {
            match *op {
                InitExprOp::I32Const(value) => stack.push(value),
                InitExprOp::GlobalGetI32(global) => {
                    stack.push(unsafe { get_global(global, instance).i32 })
                }
                InitExprOp::I32Add => {
                    let rhs = stack.pop().expect("invalid init expr stack for i32.add");
                    let lhs = stack.pop().expect("invalid init expr stack for i32.add");
                    stack.push(lhs.wrapping_add(rhs));
                }
                InitExprOp::I32Sub => {
                    let rhs = stack.pop().expect("invalid init expr stack for i32.sub");
                    let lhs = stack.pop().expect("invalid init expr stack for i32.sub");
                    stack.push(lhs.wrapping_sub(rhs));
                }
                InitExprOp::I32Mul => {
                    let rhs = stack.pop().expect("invalid init expr stack for i32.mul");
                    let lhs = stack.pop().expect("invalid init expr stack for i32.mul");
                    stack.push(lhs.wrapping_mul(rhs));
                }
                _ => {
                    panic!("unexpected init expr statement: {op:?}");
                }
            }
        }
        EvaluatedInitExpr::I32(
            stack
                .into_iter()
                .exactly_one()
                .expect("invalid init expr stack shape"),
        )
    } else {
        let mut stack = Vec::with_capacity(expr.ops().len());
        for op in expr.ops() {
            match *op {
                InitExprOp::I64Const(value) => stack.push(value),
                InitExprOp::GlobalGetI64(global) => {
                    stack.push(unsafe { get_global(global, instance).i64 })
                }
                InitExprOp::I64Add => {
                    let rhs = stack.pop().expect("invalid init expr stack for i64.add");
                    let lhs = stack.pop().expect("invalid init expr stack for i64.add");
                    stack.push(lhs.wrapping_add(rhs));
                }
                InitExprOp::I64Sub => {
                    let rhs = stack.pop().expect("invalid init expr stack for i64.sub");
                    let lhs = stack.pop().expect("invalid init expr stack for i64.sub");
                    stack.push(lhs.wrapping_sub(rhs));
                }
                InitExprOp::I64Mul => {
                    let rhs = stack.pop().expect("invalid init expr stack for i64.mul");
                    let lhs = stack.pop().expect("invalid init expr stack for i64.mul");
                    stack.push(lhs.wrapping_mul(rhs));
                }
                _ => {
                    panic!("unexpected init expr statement: {op:?}");
                }
            }
        }
        EvaluatedInitExpr::I64(
            stack
                .into_iter()
                .exactly_one()
                .expect("invalid init expr stack shape"),
        )
    }
}

/// Initialize the table memory from the provided initializers.
fn initialize_tables(instance: &mut Instance) -> Result<(), Trap> {
    let module = Arc::clone(&instance.module);
    for init in &module.table_initializers {
        let EvaluatedInitExpr::I32(start) = eval_init_expr(&init.offset_expr, instance) else {
            panic!("unexpected expression type, expected i32");
        };
        if start < 0 {
            return Err(Trap::lib(TrapCode::TableAccessOutOfBounds));
        }
        let start = start as usize;
        let table = instance.get_table_handle(init.table_index);
        let table = unsafe { table.get_mut(&mut *instance.context) };

        if start
            .checked_add(init.elements.len())
            .is_none_or(|end| end > table.size() as usize)
        {
            return Err(Trap::lib(TrapCode::TableAccessOutOfBounds));
        }

        if let wasmer_types::Type::FuncRef = table.ty().ty {
            for (i, func_idx) in init.elements.iter().enumerate() {
                let anyfunc = instance.func_ref(*func_idx);
                table
                    .set_with_construction(
                        u32::try_from(start + i).unwrap(),
                        TableElement::FuncRef(anyfunc),
                        true,
                    )
                    .unwrap();
            }
        } else {
            for i in 0..init.elements.len() {
                table
                    .set_with_construction(
                        u32::try_from(start + i).unwrap(),
                        TableElement::ExternRef(None),
                        true,
                    )
                    .unwrap();
            }
        }

        instance.sync_fixed_funcref_table_by_index(init.table_index);
    }

    Ok(())
}

/// Initialize the `Instance::passive_elements` map by resolving the
/// `ModuleInfo::passive_elements`'s `FunctionIndex`s into `VMCallerCheckedAnyfunc`s for
/// this instance.
fn initialize_passive_elements(instance: &Instance) {
    let mut passive_elements = instance.passive_elements.borrow_mut();
    debug_assert!(
        passive_elements.is_empty(),
        "should only be called once, at initialization time"
    );

    passive_elements.extend(instance.module.passive_elements.iter().filter_map(
        |(&idx, segments)| -> Option<(ElemIndex, Box<[Option<VMFuncRef>]>)> {
            if segments.is_empty() {
                None
            } else {
                Some((
                    idx,
                    segments
                        .iter()
                        .map(|s| instance.func_ref(*s))
                        .collect::<Box<[Option<VMFuncRef>]>>(),
                ))
            }
        },
    ));
}

/// Initialize the table memory from the provided initializers.
fn initialize_memories(
    instance: &mut Instance,
    data_initializers: &[DataInitializer<'_>],
) -> Result<(), Trap> {
    for init in data_initializers {
        let memory = instance.get_vmmemory(init.location.memory_index);

        let EvaluatedInitExpr::I32(start) = eval_init_expr(&init.location.offset_expr, instance)
        else {
            panic!("unexpected expression type, expected i32");
        };
        if start < 0 {
            return Err(Trap::lib(TrapCode::HeapAccessOutOfBounds));
        }
        let start = start as usize;
        unsafe {
            let current_length = memory.vmmemory().as_ref().current_length;
            if start
                .checked_add(init.data.len())
                .is_none_or(|end| end > current_length)
            {
                return Err(Trap::lib(TrapCode::HeapAccessOutOfBounds));
            }
            memory.initialize_with_data(start, init.data)?;
        }
    }

    Ok(())
}

fn initialize_globals(instance: &Instance) {
    let module = Arc::clone(&instance.module);
    for (index, initializer) in module.global_initializers.iter() {
        unsafe {
            let to = instance.global_ptr(index).as_ptr();
            match initializer {
                GlobalInit::I32Const(x) => (*to).val.i32 = *x,
                GlobalInit::I64Const(x) => (*to).val.i64 = *x,
                GlobalInit::F32Const(x) => (*to).val.f32 = *x,
                GlobalInit::F64Const(x) => (*to).val.f64 = *x,
                GlobalInit::V128Const(x) => (*to).val.bytes = *x.bytes(),
                GlobalInit::GetGlobal(x) => {
                    let from: VMGlobalDefinition =
                        if let Some(def_x) = module.local_global_index(*x) {
                            instance.global(def_x)
                        } else {
                            instance.imported_global(*x).definition.as_ref().clone()
                        };
                    *to = from;
                }
                GlobalInit::RefNullConst => (*to).val.funcref = 0,
                GlobalInit::RefFunc(func_idx) => {
                    let funcref = instance.func_ref(*func_idx).unwrap();
                    (*to).val = funcref.into_raw();
                }
                GlobalInit::Expr(expr) => match eval_init_expr(expr, instance) {
                    EvaluatedInitExpr::I32(value) => (*to).val.i32 = value,
                    EvaluatedInitExpr::I64(value) => (*to).val.i64 = value,
                },
            }
        }
    }
}

fn anyfunc_from_funcref(funcref: Option<VMFuncRef>) -> VMCallerCheckedAnyfunc {
    match funcref {
        Some(funcref) => unsafe { *funcref.0.as_ptr() },
        None => VMCallerCheckedAnyfunc::null(),
    }
}

/// Eagerly builds all the `VMFuncRef`s for imported and local functions so that all
/// future funcref operations are just looking up this data.
fn build_funcrefs(
    module_info: &ModuleInfo,
    ctx: &StoreObjects,
    imports: &Imports,
    finished_functions: &BoxedSlice<LocalFunctionIndex, FunctionBodyPtr>,
    vmshared_signatures: &BoxedSlice<SignatureIndex, VMSignatureHash>,
    function_call_trampolines: &BoxedSlice<SignatureIndex, VMTrampoline>,
    vmctx_ptr: *mut VMContext,
) -> (
    BoxedSlice<LocalFunctionIndex, VMCallerCheckedAnyfunc>,
    BoxedSlice<FunctionIndex, NonNull<VMCallerCheckedAnyfunc>>,
) {
    let mut func_refs =
        PrimaryMap::with_capacity(module_info.functions.len() - module_info.num_imported_functions);
    let mut imported_func_refs = PrimaryMap::with_capacity(module_info.num_imported_functions);

    // do imported functions
    for import in imports.functions.values() {
        imported_func_refs.push(import.handle.get(ctx).anyfunc.as_ptr());
    }

    // do local functions
    for (local_index, func_ptr) in finished_functions.iter() {
        let index = module_info.func_index(local_index);
        let sig_index = module_info.functions[index];
        let type_signature_hash = vmshared_signatures[sig_index];
        let call_trampoline = function_call_trampolines[sig_index];
        let anyfunc = VMCallerCheckedAnyfunc {
            func_ptr: func_ptr.0,
            type_signature_hash,
            vmctx: VMFunctionContext { vmctx: vmctx_ptr },
            call_trampoline,
        };
        func_refs.push(anyfunc);
    }
    (
        func_refs.into_boxed_slice(),
        imported_func_refs.into_boxed_slice(),
    )
}