wasmer_vm/
mmap.rs

1// This file contains code from external sources.
2// Attributions: https://github.com/wasmerio/wasmer/blob/main/docs/ATTRIBUTIONS.md
3
4//! Low-level abstraction for allocating and managing zero-filled pages
5//! of memory.
6
7use more_asserts::assert_le;
8use std::io;
9use std::ptr;
10use std::slice;
11
12/// A simple struct consisting of a page-aligned pointer to page-aligned
13/// and initially-zeroed memory and a length.
14#[derive(Debug)]
15pub struct Mmap {
16    // Note that this is stored as a `usize` instead of a `*const` or `*mut`
17    // pointer to allow this structure to be natively `Send` and `Sync` without
18    // `unsafe impl`. This type is sendable across threads and shareable since
19    // the coordination all happens at the OS layer.
20    ptr: usize,
21    total_size: usize,
22    accessible_size: usize,
23    #[cfg_attr(target_os = "windows", allow(dead_code))]
24    sync_on_drop: bool,
25}
26
27/// The type of mmap to create
28#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
29pub enum MmapType {
30    /// The memory is private to the process and not shared with other processes.
31    Private,
32    /// The memory is shared with other processes. This is only supported on Unix.
33    /// When the memory is flushed it will update the file data.
34    Shared,
35}
36
37impl Mmap {
38    /// Construct a new empty instance of `Mmap`.
39    pub fn new() -> Self {
40        // Rust's slices require non-null pointers, even when empty. `Vec`
41        // contains code to create a non-null dangling pointer value when
42        // constructed empty, so we reuse that here.
43        let empty = Vec::<u8>::new();
44        Self {
45            ptr: empty.as_ptr() as usize,
46            total_size: 0,
47            accessible_size: 0,
48            sync_on_drop: false,
49        }
50    }
51
52    /// Create a new `Mmap` pointing to at least `size` bytes of page-aligned accessible memory.
53    pub fn with_at_least(size: usize, hint_huge_pages: bool) -> Result<Self, String> {
54        let page_size = region::page::size();
55        let rounded_size = size.next_multiple_of(page_size);
56        Self::accessible_reserved(
57            rounded_size,
58            rounded_size,
59            None,
60            MmapType::Private,
61            hint_huge_pages,
62        )
63    }
64
65    /// Create a new `Mmap` pointing to `accessible_size` bytes of page-aligned accessible memory,
66    /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size`
67    /// must be native page-size multiples.
68    #[cfg(not(target_os = "windows"))]
69    pub fn accessible_reserved(
70        mut accessible_size: usize,
71        mapping_size: usize,
72        mut backing_file: Option<std::path::PathBuf>,
73        memory_type: MmapType,
74        hint_huge_pages: bool,
75    ) -> Result<Self, String> {
76        use std::os::fd::IntoRawFd;
77
78        let page_size = region::page::size();
79        assert_le!(accessible_size, mapping_size);
80        assert_eq!(mapping_size & (page_size - 1), 0);
81        assert_eq!(accessible_size & (page_size - 1), 0);
82
83        // Mmap may return EINVAL if the size is zero, so just
84        // special-case that.
85        if mapping_size == 0 {
86            return Ok(Self::new());
87        }
88
89        // If there is a backing file, resize the file so that its at least
90        // `mapping_size` bytes.
91        let mut memory_fd = -1;
92        if let Some(backing_file_path) = &mut backing_file {
93            let file = std::fs::OpenOptions::new()
94                .read(true)
95                .write(true)
96                .open(&backing_file_path)
97                .map_err(|e| e.to_string())?;
98
99            let mut backing_file_accessible = backing_file_path.clone();
100            backing_file_accessible.set_extension("accessible");
101
102            let len = file.metadata().map_err(|e| e.to_string())?.len() as usize;
103            if len < mapping_size {
104                std::fs::write(&backing_file_accessible, format!("{len}").as_bytes()).ok();
105
106                file.set_len(mapping_size as u64)
107                    .map_err(|e| e.to_string())?;
108            }
109
110            if backing_file_accessible.exists() {
111                let accessible = std::fs::read_to_string(&backing_file_accessible)
112                    .map_err(|e| e.to_string())?
113                    .parse::<usize>()
114                    .map_err(|e| e.to_string())?;
115                accessible_size = accessible_size.max(accessible);
116            } else {
117                accessible_size = accessible_size.max(len);
118            }
119
120            accessible_size = accessible_size.min(mapping_size);
121            memory_fd = file.into_raw_fd();
122        }
123
124        // Compute the flags
125        let mut flags = match memory_fd {
126            fd if fd < 0 => libc::MAP_ANON,
127            _ => libc::MAP_FILE,
128        };
129        flags |= match memory_type {
130            MmapType::Private => libc::MAP_PRIVATE,
131            MmapType::Shared => libc::MAP_SHARED,
132        };
133
134        Ok(if accessible_size == mapping_size {
135            // Allocate a single read-write region at once.
136            let ptr = unsafe {
137                libc::mmap(
138                    ptr::null_mut(),
139                    mapping_size,
140                    libc::PROT_READ | libc::PROT_WRITE,
141                    flags,
142                    memory_fd,
143                    0,
144                )
145            };
146            if ptr as isize == -1_isize {
147                return Err(io::Error::last_os_error().to_string());
148            }
149
150            if hint_huge_pages {
151                advise_huge_pages(ptr, mapping_size);
152            }
153
154            Self {
155                ptr: ptr as usize,
156                total_size: mapping_size,
157                accessible_size,
158                sync_on_drop: memory_fd != -1 && memory_type == MmapType::Shared,
159            }
160        } else {
161            // Reserve the mapping size.
162            let ptr = unsafe {
163                libc::mmap(
164                    ptr::null_mut(),
165                    mapping_size,
166                    libc::PROT_NONE,
167                    flags,
168                    memory_fd,
169                    0,
170                )
171            };
172            if ptr as isize == -1_isize {
173                return Err(io::Error::last_os_error().to_string());
174            }
175
176            if hint_huge_pages {
177                advise_huge_pages(ptr, mapping_size);
178            }
179
180            let mut result = Self {
181                ptr: ptr as usize,
182                total_size: mapping_size,
183                accessible_size,
184                sync_on_drop: memory_fd != -1 && memory_type == MmapType::Shared,
185            };
186
187            if accessible_size != 0 {
188                // Commit the accessible size.
189                result.make_accessible(0, accessible_size)?;
190            }
191
192            result
193        })
194    }
195
196    /// Create a new `Mmap` pointing to `accessible_size` bytes of page-aligned accessible memory,
197    /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size`
198    /// must be native page-size multiples.
199    #[cfg(target_os = "windows")]
200    pub fn accessible_reserved(
201        accessible_size: usize,
202        mapping_size: usize,
203        _backing_file: Option<std::path::PathBuf>,
204        _memory_type: MmapType,
205        _hint_huge_pages: bool,
206    ) -> Result<Self, String> {
207        use windows_sys::Win32::System::Memory::{
208            MEM_COMMIT, MEM_RESERVE, PAGE_NOACCESS, PAGE_READWRITE, VirtualAlloc,
209        };
210
211        let page_size = region::page::size();
212        assert_le!(accessible_size, mapping_size);
213        assert_eq!(mapping_size & (page_size - 1), 0);
214        assert_eq!(accessible_size & (page_size - 1), 0);
215
216        // VirtualAlloc may return ERROR_INVALID_PARAMETER if the size is zero,
217        // so just special-case that.
218        if mapping_size == 0 {
219            return Ok(Self::new());
220        }
221
222        Ok(if accessible_size == mapping_size {
223            // Allocate a single read-write region at once.
224            let ptr = unsafe {
225                VirtualAlloc(
226                    ptr::null_mut(),
227                    mapping_size,
228                    MEM_RESERVE | MEM_COMMIT,
229                    PAGE_READWRITE,
230                )
231            };
232            if ptr.is_null() {
233                return Err(io::Error::last_os_error().to_string());
234            }
235
236            Self {
237                ptr: ptr as usize,
238                total_size: mapping_size,
239                accessible_size,
240                sync_on_drop: false,
241            }
242        } else {
243            // Reserve the mapping size.
244            let ptr =
245                unsafe { VirtualAlloc(ptr::null_mut(), mapping_size, MEM_RESERVE, PAGE_NOACCESS) };
246            if ptr.is_null() {
247                return Err(io::Error::last_os_error().to_string());
248            }
249
250            let mut result = Self {
251                ptr: ptr as usize,
252                total_size: mapping_size,
253                accessible_size,
254                sync_on_drop: false,
255            };
256
257            if accessible_size != 0 {
258                // Commit the accessible size.
259                result.make_accessible(0, accessible_size)?;
260            }
261
262            result
263        })
264    }
265
266    /// Make the memory starting at `start` and extending for `len` bytes accessible.
267    /// `start` and `len` must be native page-size multiples and describe a range within
268    /// `self`'s reserved memory.
269    #[cfg(not(target_os = "windows"))]
270    pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> {
271        let page_size = region::page::size();
272        assert_eq!(start & (page_size - 1), 0);
273        assert_eq!(len & (page_size - 1), 0);
274        assert_le!(len, self.total_size);
275        assert_le!(start, self.total_size - len);
276
277        // Commit the accessible size.
278        let ptr = self.ptr as *const u8;
279        unsafe { region::protect(ptr.add(start), len, region::Protection::READ_WRITE) }
280            .map_err(|e| e.to_string())
281    }
282
283    /// Make the memory starting at `start` and extending for `len` bytes accessible.
284    /// `start` and `len` must be native page-size multiples and describe a range within
285    /// `self`'s reserved memory.
286    #[cfg(target_os = "windows")]
287    pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> {
288        use std::ffi::c_void;
289        use windows_sys::Win32::System::Memory::{MEM_COMMIT, PAGE_READWRITE, VirtualAlloc};
290        let page_size = region::page::size();
291        assert_eq!(start & (page_size - 1), 0);
292        assert_eq!(len & (page_size - 1), 0);
293        assert_le!(len, self.len());
294        assert_le!(start, self.len() - len);
295
296        // Commit the accessible size.
297        let ptr = self.ptr as *const u8;
298        if unsafe {
299            VirtualAlloc(
300                ptr.add(start) as *mut c_void,
301                len,
302                MEM_COMMIT,
303                PAGE_READWRITE,
304            )
305        }
306        .is_null()
307        {
308            return Err(io::Error::last_os_error().to_string());
309        }
310
311        Ok(())
312    }
313
314    /// Return the allocated memory as a slice of u8.
315    pub fn as_slice(&self) -> &[u8] {
316        unsafe { slice::from_raw_parts(self.ptr as *const u8, self.total_size) }
317    }
318
319    /// Return the allocated memory as a slice of u8.
320    pub fn as_slice_accessible(&self) -> &[u8] {
321        unsafe { slice::from_raw_parts(self.ptr as *const u8, self.accessible_size) }
322    }
323
324    /// Return the allocated memory as a slice of u8.
325    pub fn as_slice_arbitary(&self, size: usize) -> &[u8] {
326        let size = usize::min(size, self.total_size);
327        unsafe { slice::from_raw_parts(self.ptr as *const u8, size) }
328    }
329
330    /// Return the allocated memory as a mutable slice of u8.
331    pub fn as_mut_slice(&mut self) -> &mut [u8] {
332        unsafe { slice::from_raw_parts_mut(self.ptr as *mut u8, self.total_size) }
333    }
334
335    /// Return the allocated memory as a mutable slice of u8.
336    pub fn as_mut_slice_accessible(&mut self) -> &mut [u8] {
337        unsafe { slice::from_raw_parts_mut(self.ptr as *mut u8, self.accessible_size) }
338    }
339
340    /// Return the allocated memory as a mutable slice of u8.
341    pub fn as_mut_slice_arbitrary(&mut self, size: usize) -> &mut [u8] {
342        let size = usize::min(size, self.total_size);
343        unsafe { slice::from_raw_parts_mut(self.ptr as *mut u8, size) }
344    }
345
346    /// Return the allocated memory as a pointer to u8.
347    pub fn as_ptr(&self) -> *const u8 {
348        self.ptr as *const u8
349    }
350
351    /// Return the allocated memory as a mutable pointer to u8.
352    pub fn as_mut_ptr(&mut self) -> *mut u8 {
353        self.ptr as *mut u8
354    }
355
356    /// Return the length of the allocated memory.
357    pub fn len(&self) -> usize {
358        self.total_size
359    }
360
361    /// Return whether any memory has been allocated.
362    pub fn is_empty(&self) -> bool {
363        self.len() == 0
364    }
365
366    /// Duplicate in a new memory mapping.
367    #[deprecated = "use `copy` instead"]
368    pub fn duplicate(&self, size_hint: Option<usize>) -> Result<Self, String> {
369        self.copy(size_hint)
370    }
371
372    /// Duplicate in a new memory mapping.
373    pub fn copy(&self, size_hint: Option<usize>) -> Result<Self, String> {
374        // NOTE: accessible_size != used size as the value is not
375        //       automatically updated when the pre-provisioned space is used
376        let mut copy_size = self.accessible_size;
377        if let Some(size_hint) = size_hint {
378            copy_size = usize::max(copy_size, size_hint);
379        }
380
381        let mut new =
382            Self::accessible_reserved(copy_size, self.total_size, None, MmapType::Private, true)?;
383        new.as_mut_slice_arbitrary(copy_size)
384            .copy_from_slice(self.as_slice_arbitary(copy_size));
385        Ok(new)
386    }
387}
388
389#[cfg(not(target_os = "windows"))]
390fn advise_huge_pages(ptr: *mut libc::c_void, len: usize) {
391    #[cfg(target_os = "linux")]
392    unsafe {
393        // This is only a hint; allocation must still succeed if huge pages are unavailable.
394        let _ = libc::madvise(ptr, len, libc::MADV_HUGEPAGE);
395    }
396
397    // The system call is unavailable on macOS system.
398    #[cfg(not(target_os = "linux"))]
399    let _ = (ptr, len);
400}
401
402impl Drop for Mmap {
403    #[cfg(not(target_os = "windows"))]
404    fn drop(&mut self) {
405        if self.total_size != 0 {
406            if self.sync_on_drop {
407                let r = unsafe {
408                    libc::msync(
409                        self.ptr as *mut libc::c_void,
410                        self.total_size,
411                        libc::MS_SYNC | libc::MS_INVALIDATE,
412                    )
413                };
414                assert_eq!(r, 0, "msync failed: {}", io::Error::last_os_error());
415            }
416            let r = unsafe { libc::munmap(self.ptr as *mut libc::c_void, self.total_size) };
417            assert_eq!(r, 0, "munmap failed: {}", io::Error::last_os_error());
418        }
419    }
420
421    #[cfg(target_os = "windows")]
422    fn drop(&mut self) {
423        if !self.is_empty() {
424            use std::ffi::c_void;
425            use windows_sys::Win32::System::Memory::{MEM_RELEASE, VirtualFree};
426            let r = unsafe { VirtualFree(self.ptr as *mut c_void, 0, MEM_RELEASE) };
427            assert_ne!(r, 0);
428        }
429    }
430}
431
432fn _assert() {
433    fn _assert_send_sync<T: Send + Sync>() {}
434    _assert_send_sync::<Mmap>();
435}
wasmer_vm/mmap.rs

wasmer_vm/
mmap.rs
