wasmer_sys_utils/memory/fd_memory/
fd_mmap.rs

1// This file contains code from external sources.
2// Attributions: https://github.com/wasmerio/wasmer/blob/main/docs/ATTRIBUTIONS.md
3
4use std::{
5    io::{self, Read, Write},
6    ptr, slice,
7};
8
9/// A simple struct consisting of a page-aligned pointer to page-aligned
10/// and initially-zeroed memory and a length.
11#[derive(Debug)]
12pub struct FdMmap {
13    // Note that this is stored as a `usize` instead of a `*const` or `*mut`
14    // pointer to allow this structure to be natively `Send` and `Sync` without
15    // `unsafe impl`. This type is sendable across threads and shareable since
16    // the coordination all happens at the OS layer.
17    ptr: usize,
18    len: usize,
19    // Backing file that will be closed when the memory mapping goes out of scope
20    fd: FdGuard,
21}
22
23#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
24pub struct FdGuard(pub i32);
25
26impl Default for FdGuard {
27    fn default() -> Self {
28        Self(-1)
29    }
30}
31
32impl Clone for FdGuard {
33    fn clone(&self) -> Self {
34        unsafe { Self(libc::dup(self.0)) }
35    }
36}
37
38impl Drop for FdGuard {
39    fn drop(&mut self) {
40        if self.0 >= 0 {
41            unsafe {
42                libc::close(self.0);
43            }
44            self.0 = -1;
45        }
46    }
47}
48
49impl FdMmap {
50    /// Construct a new empty instance of `Mmap`.
51    pub fn new() -> Self {
52        // Rust's slices require non-null pointers, even when empty. `Vec`
53        // contains code to create a non-null dangling pointer value when
54        // constructed empty, so we reuse that here.
55        let empty = Vec::<u8>::new();
56        Self {
57            ptr: empty.as_ptr() as usize,
58            len: 0,
59            fd: FdGuard::default(),
60        }
61    }
62
63    /// Create a new `Mmap` pointing to `accessible_size` bytes of page-aligned accessible memory,
64    /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size`
65    /// must be native page-size multiples.
66    pub fn accessible_reserved(
67        accessible_size: usize,
68        mapping_size: usize,
69    ) -> Result<Self, String> {
70        let page_size = region::page::size();
71        assert!(accessible_size <= mapping_size);
72        assert_eq!(mapping_size & (page_size - 1), 0);
73        assert_eq!(accessible_size & (page_size - 1), 0);
74
75        // Mmap may return EINVAL if the size is zero, so just
76        // special-case that.
77        if mapping_size == 0 {
78            return Ok(Self::new());
79        }
80
81        // Open a temporary file (which is used for swapping)
82        let fd = unsafe {
83            let file = libc::tmpfile();
84            if file.is_null() {
85                return Err(format!(
86                    "failed to create temporary file - {}",
87                    io::Error::last_os_error()
88                ));
89            }
90            FdGuard(libc::fileno(file))
91        };
92
93        // First we initialize it with zeros
94        unsafe {
95            if libc::ftruncate(fd.0, mapping_size as libc::off_t) < 0 {
96                return Err("could not truncate tmpfile".to_string());
97            }
98        }
99
100        Ok(if accessible_size == mapping_size {
101            // Allocate a single read-write region at once.
102            let ptr = unsafe {
103                libc::mmap(
104                    ptr::null_mut(),
105                    mapping_size,
106                    libc::PROT_READ | libc::PROT_WRITE,
107                    libc::MAP_FILE | libc::MAP_SHARED,
108                    fd.0,
109                    0,
110                )
111            };
112            if ptr as isize == -1_isize {
113                return Err(io::Error::last_os_error().to_string());
114            }
115
116            Self {
117                ptr: ptr as usize,
118                len: mapping_size,
119                fd,
120            }
121        } else {
122            // Reserve the mapping size.
123            let ptr = unsafe {
124                libc::mmap(
125                    ptr::null_mut(),
126                    mapping_size,
127                    libc::PROT_NONE,
128                    libc::MAP_FILE | libc::MAP_SHARED,
129                    fd.0,
130                    0,
131                )
132            };
133            if ptr as isize == -1_isize {
134                return Err(io::Error::last_os_error().to_string());
135            }
136
137            let mut result = Self {
138                ptr: ptr as usize,
139                len: mapping_size,
140                fd,
141            };
142
143            if accessible_size != 0 {
144                // Commit the accessible size.
145                result.make_accessible(0, accessible_size)?;
146            }
147
148            result
149        })
150    }
151
152    /// Make the memory starting at `start` and extending for `len` bytes accessible.
153    /// `start` and `len` must be native page-size multiples and describe a range within
154    /// `self`'s reserved memory.
155    pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> {
156        let page_size = region::page::size();
157        assert_eq!(start & (page_size - 1), 0);
158        assert_eq!(len & (page_size - 1), 0);
159        assert!(len < self.len);
160        assert!(start < self.len - len);
161
162        // Commit the accessible size.
163        let ptr = self.ptr as *const u8;
164        unsafe { region::protect(ptr.add(start), len, region::Protection::READ_WRITE) }
165            .map_err(|e| e.to_string())
166    }
167
168    /// Return the allocated memory as a slice of u8.
169    pub fn as_slice(&self) -> &[u8] {
170        unsafe { slice::from_raw_parts(self.ptr as *const u8, self.len) }
171    }
172
173    /// Return the allocated memory as a mutable slice of u8.
174    pub fn as_mut_slice(&mut self) -> &mut [u8] {
175        unsafe { slice::from_raw_parts_mut(self.ptr as *mut u8, self.len) }
176    }
177
178    // /// Return the allocated memory as a pointer to u8.
179    // pub fn as_ptr(&self) -> *const u8 {
180    //     self.ptr as *const u8
181    // }
182
183    /// Return the allocated memory as a mutable pointer to u8.
184    pub fn as_mut_ptr(&mut self) -> *mut u8 {
185        self.ptr as *mut u8
186    }
187
188    /// Return the length of the allocated memory.
189    pub fn len(&self) -> usize {
190        self.len
191    }
192
193    // /// Return whether any memory has been allocated.
194    // pub fn is_empty(&self) -> bool {
195    //     self.len() == 0
196    // }
197
198    /// Copies the memory to a new swap file (using copy-on-write if available)
199    pub fn duplicate(&mut self, hint_used: Option<usize>) -> Result<Self, String> {
200        // Empty memory is an edge case
201
202        use std::os::unix::prelude::FromRawFd;
203        if self.len == 0 {
204            return Ok(Self::new());
205        }
206
207        // First we sync all the data to the backing file
208        unsafe {
209            libc::fsync(self.fd.0);
210        }
211
212        // Open a new temporary file (which is used for swapping for the forked memory)
213        let fd = unsafe {
214            let file = libc::tmpfile();
215            if file.is_null() {
216                return Err(format!(
217                    "failed to create temporary file - {}",
218                    io::Error::last_os_error()
219                ));
220            }
221            FdGuard(libc::fileno(file))
222        };
223
224        // Attempt to do a shallow copy (needs a backing file system that supports it)
225        unsafe {
226            if libc::ioctl(fd.0, 0x94, 9, self.fd.0) != 0
227            // FICLONE
228            {
229                #[cfg(feature = "tracing")]
230                trace!("memory copy started");
231
232                // Determine host much to copy
233                let len = match hint_used {
234                    Some(a) => a,
235                    None => self.len,
236                };
237
238                // The shallow copy failed so we have to do it the hard way
239
240                let mut source = std::fs::File::from_raw_fd(self.fd.0);
241                let mut out = std::fs::File::from_raw_fd(fd.0);
242                copy_file_range(&mut source, 0, &mut out, 0, len)
243                    .map_err(|err| format!("Could not copy memory: {err}"))?;
244
245                #[cfg(feature = "tracing")]
246                trace!("memory copy finished (size={})", len);
247            }
248        }
249
250        // Compute the flags
251        let flags = libc::MAP_FILE | libc::MAP_SHARED;
252
253        // Allocate a single read-write region at once.
254        let ptr = unsafe {
255            libc::mmap(
256                ptr::null_mut(),
257                self.len,
258                libc::PROT_READ | libc::PROT_WRITE,
259                flags,
260                fd.0,
261                0,
262            )
263        };
264        if ptr as isize == -1_isize {
265            return Err(io::Error::last_os_error().to_string());
266        }
267
268        Ok(Self {
269            ptr: ptr as usize,
270            len: self.len,
271            fd,
272        })
273    }
274}
275
276impl Drop for FdMmap {
277    fn drop(&mut self) {
278        if self.len != 0 {
279            let r = unsafe { libc::munmap(self.ptr as *mut libc::c_void, self.len) };
280            assert_eq!(r, 0, "munmap failed: {}", io::Error::last_os_error());
281        }
282    }
283}
284
285/// Copy a range of a file to another file.
286// We could also use libc::copy_file_range on some systems, but it's
287// hard to do this because it is not available on many libc implementations.
288// (not on Mac OS, musl, ...)
289#[cfg(target_family = "unix")]
290fn copy_file_range(
291    source: &mut std::fs::File,
292    source_offset: u64,
293    out: &mut std::fs::File,
294    out_offset: u64,
295    len: usize,
296) -> Result<(), std::io::Error> {
297    use std::io::{Seek, SeekFrom};
298
299    let source_original_pos = source.stream_position()?;
300    source.seek(SeekFrom::Start(source_offset))?;
301
302    // TODO: don't cast with as
303
304    let out_original_pos = out.stream_position()?;
305    out.seek(SeekFrom::Start(out_offset))?;
306
307    // TODO: don't do this horrible "triple buffering" below".
308    // let mut reader = std::io::BufReader::new(source);
309
310    // TODO: larger buffer?
311    let mut buffer = vec![0u8; 4096];
312
313    let mut to_read = len;
314    while to_read > 0 {
315        let chunk_size = std::cmp::min(to_read, buffer.len());
316        let read = source.read(&mut buffer[0..chunk_size])?;
317        out.write_all(&buffer[0..read])?;
318        to_read -= read;
319    }
320
321    // Need to read the last chunk.
322    out.flush()?;
323
324    // Restore files to original position.
325    source.seek(SeekFrom::Start(source_original_pos))?;
326    out.flush()?;
327    out.sync_data()?;
328    out.seek(SeekFrom::Start(out_original_pos))?;
329
330    Ok(())
331}
332
333#[cfg(test)]
334mod tests {
335    use super::*;
336
337    #[cfg(target_family = "unix")]
338    #[test]
339    fn test_copy_file_range() -> Result<(), std::io::Error> {
340        // I know tempfile:: exists, but this doesn't bring in an extra
341        // dependency.
342
343        use std::{fs::OpenOptions, io::Seek};
344
345        let dir = std::env::temp_dir().join("wasmer/copy_file_range");
346        if dir.is_dir() {
347            std::fs::remove_dir_all(&dir).unwrap()
348        }
349        std::fs::create_dir_all(&dir).unwrap();
350
351        let pa = dir.join("a");
352        let pb = dir.join("b");
353
354        let data: Vec<u8> = (0..100).collect();
355        let mut a = OpenOptions::new()
356            .read(true)
357            .write(true)
358            .create_new(true)
359            .open(pa)
360            .unwrap();
361        a.write_all(&data).unwrap();
362
363        let datb: Vec<u8> = (100..200).collect();
364        let mut b = OpenOptions::new()
365            .read(true)
366            .write(true)
367            .create_new(true)
368            .open(pb)
369            .unwrap();
370        b.write_all(&datb).unwrap();
371
372        a.seek(io::SeekFrom::Start(30)).unwrap();
373        b.seek(io::SeekFrom::Start(99)).unwrap();
374        copy_file_range(&mut a, 10, &mut b, 40, 15).unwrap();
375
376        assert_eq!(a.stream_position().unwrap(), 30);
377        assert_eq!(b.stream_position().unwrap(), 99);
378
379        b.seek(io::SeekFrom::Start(0)).unwrap();
380        let mut out = Vec::new();
381        let len = b.read_to_end(&mut out).unwrap();
382        assert_eq!(len, 100);
383        assert_eq!(out[0..40], datb[0..40]);
384        assert_eq!(out[40..55], data[10..25]);
385        assert_eq!(out[55..100], datb[55..100]);
386
387        // TODO: needs more variant tests, but this is enough for now.
388
389        Ok(())
390    }
391}
wasmer_sys_utils/memory/fd_memory/fd_mmap.rs

wasmer_sys_utils/memory/fd_memory/
fd_mmap.rs
