wasmer_package/package/volume/
fs.rs

1use std::{
2    collections::{BTreeMap, BTreeSet},
3    fmt::Debug,
4    fs::File,
5    io::Read,
6    path::{Path, PathBuf},
7};
8
9use anyhow::{Context, Error};
10use shared_buffer::OwnedBuffer;
11
12use webc::{
13    AbstractVolume, Metadata, PathSegment, PathSegments, Timestamps, ToPathSegments, sanitize_path,
14    v3::{
15        self,
16        write::{DirEntry, Directory, FileEntry},
17    },
18};
19
20use crate::package::{Strictness, WalkBuilderFactory};
21
22use super::WasmerPackageVolume;
23
24/// A lazily loaded volume in a Wasmer package.
25///
26/// Note that it is the package resolver's role to interpret a package's
27/// filesystem mappings. A volume contains directories as they were when the
28/// package was published.
29pub struct FsVolume {
30    /// Name of the volume
31    name: String,
32    /// A pre-computed set of intermediate directories that are needed to allow
33    /// access to the whitelisted files and directories.
34    intermediate_directories: BTreeSet<PathBuf>,
35    /// Specific files that this volume has access to.
36    metadata_files: BTreeSet<PathBuf>,
37    /// Directories that allow the user to access anything inside them.
38    mapped_directories: BTreeSet<PathBuf>,
39    /// The base directory all [`PathSegments`] will be resolved relative to.
40    base_dir: PathBuf,
41    /// The walker builder factory to use when reading directories.
42    walker_factory: WalkBuilderFactory,
43}
44
45impl Debug for FsVolume {
46    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
47        f.debug_struct("FsVolume")
48            .field("name", &self.name)
49            .field("intermediate_directories", &self.intermediate_directories)
50            .field("metadata_files", &self.metadata_files)
51            .field("mapped_directories", &self.mapped_directories)
52            .field("base_dir", &self.base_dir)
53            .finish()
54    }
55}
56
57impl FsVolume {
58    /// The name of the volume used to store metadata files.
59    pub(crate) const METADATA: &'static str = "metadata";
60
61    /// Create a new metadata volume.
62    pub(crate) fn new_metadata(
63        manifest: &wasmer_config::package::Manifest,
64        base_dir: impl Into<PathBuf>,
65    ) -> Result<Self, Error> {
66        let base_dir = base_dir.into();
67        let mut files = BTreeSet::new();
68
69        // check if manifest.package is None
70        if let Some(package) = &manifest.package {
71            if let Some(license_file) = &package.license_file {
72                files.insert(base_dir.join(license_file));
73            }
74
75            if let Some(readme) = &package.readme {
76                files.insert(base_dir.join(readme));
77            }
78        }
79
80        for module in &manifest.modules {
81            if let Some(bindings) = &module.bindings {
82                let bindings_files = bindings.referenced_files(&base_dir)?;
83                files.extend(bindings_files);
84            }
85        }
86
87        Ok(FsVolume::new_with_intermediate_dirs(
88            FsVolume::METADATA.to_string(),
89            base_dir,
90            files,
91            BTreeSet::new(),
92            crate::package::include_everything_walker(),
93        ))
94    }
95
96    pub(crate) fn new_assets(
97        manifest: &wasmer_config::package::Manifest,
98        base_dir: &Path,
99        walker_factory: crate::package::WalkBuilderFactory,
100    ) -> Result<BTreeMap<String, Self>, Error> {
101        // Create asset volumes
102        let dirs: BTreeSet<_> = manifest
103            .fs
104            .values()
105            .map(|path| base_dir.join(path))
106            .collect();
107
108        for path in &dirs {
109            // Perform a basic sanity check to make sure the directories exist.
110            let _ = std::fs::metadata(path).with_context(|| {
111                format!("Unable to get the metadata for \"{}\"", path.display())
112            })?;
113        }
114
115        let mut volumes = BTreeMap::new();
116        for entry in manifest.fs.values() {
117            let name = entry
118                .to_str()
119                .ok_or_else(|| anyhow::anyhow!("Failed to convert path to str"))?;
120
121            let name = sanitize_path(name);
122
123            let mut dirs = BTreeSet::new();
124            let dir = base_dir.join(entry);
125            dirs.insert(dir);
126
127            volumes.insert(
128                name.clone(),
129                FsVolume::new(
130                    name.to_string(),
131                    base_dir.to_path_buf(),
132                    BTreeSet::new(),
133                    dirs,
134                    walker_factory,
135                ),
136            );
137        }
138
139        Ok(volumes)
140    }
141
142    pub(crate) fn new_with_intermediate_dirs(
143        name: String,
144        base_dir: PathBuf,
145        whitelisted_files: BTreeSet<PathBuf>,
146        whitelisted_directories: BTreeSet<PathBuf>,
147        walker_factory: crate::package::WalkBuilderFactory,
148    ) -> Self {
149        let mut intermediate_directories: BTreeSet<PathBuf> = whitelisted_files
150            .iter()
151            .filter_map(|p| p.parent())
152            .chain(whitelisted_directories.iter().map(|p| p.as_path()))
153            .flat_map(|dir| dir.ancestors())
154            .filter(|dir| dir.starts_with(&base_dir))
155            .map(|dir| dir.to_path_buf())
156            .collect();
157
158        // The base directory is always accessible (even if its contents isn't)
159        intermediate_directories.insert(base_dir.clone());
160
161        FsVolume {
162            name,
163            intermediate_directories,
164            metadata_files: whitelisted_files,
165            mapped_directories: whitelisted_directories,
166            base_dir,
167            walker_factory,
168        }
169    }
170
171    pub(crate) fn new(
172        name: String,
173        base_dir: PathBuf,
174        whitelisted_files: BTreeSet<PathBuf>,
175        whitelisted_directories: BTreeSet<PathBuf>,
176        walker_factory: crate::package::WalkBuilderFactory,
177    ) -> Self {
178        FsVolume {
179            name,
180            intermediate_directories: BTreeSet::new(),
181            metadata_files: whitelisted_files,
182            mapped_directories: whitelisted_directories,
183            base_dir,
184            walker_factory,
185        }
186    }
187
188    fn is_accessible(&self, path: &Path) -> bool {
189        self.intermediate_directories.contains(path)
190            || self.metadata_files.contains(path)
191            || self
192                .mapped_directories
193                .iter()
194                .any(|dir| path.starts_with(dir))
195    }
196
197    fn resolve(&self, path: &PathSegments) -> Option<PathBuf> {
198        let resolved = if let Some(dir) = &self.mapped_directories.first() {
199            resolve(dir, path)
200        } else {
201            resolve(&self.base_dir, path)
202        };
203
204        let accessible = self.is_accessible(&resolved);
205        accessible.then_some(resolved)
206    }
207
208    /// Returns the name of the volume
209    pub fn name(&self) -> &str {
210        self.name.as_str()
211    }
212
213    /// Read a file from the volume.
214    pub fn read_file(&self, path: &PathSegments) -> Option<OwnedBuffer> {
215        let path = self.resolve(path)?;
216        let mut f = File::open(path).ok()?;
217
218        // First we try to mmap it
219        if let Ok(mmapped) = OwnedBuffer::from_file(&f) {
220            return Some(mmapped);
221        }
222
223        // otherwise, fall back to reading the file's contents into memory
224        let mut buffer = Vec::new();
225        f.read_to_end(&mut buffer).ok()?;
226        Some(OwnedBuffer::from_bytes(buffer))
227    }
228
229    /// Read the contents of a directory.
230    #[allow(clippy::type_complexity)]
231    pub fn read_dir(
232        &self,
233        path: &PathSegments,
234    ) -> Option<Vec<(PathSegment, Option<[u8; 32]>, Metadata)>> {
235        let resolved = self.resolve(path)?;
236
237        let mut walker_builder = self.walker_factory.create_walk_builder(&resolved);
238        walker_builder.max_depth(Some(1));
239        let walker = walker_builder.build();
240
241        let mut entries = Vec::new();
242
243        for entry in walker {
244            let entry = entry.ok()?;
245            // Walk returns the root dir as well, we don't want to process it
246            if entry.depth() == 0 {
247                continue;
248            }
249
250            let entry = entry.path();
251
252            if !self.is_accessible(entry) {
253                continue;
254            }
255
256            let segment: PathSegment = entry.file_name()?.to_str()?.parse().ok()?;
257
258            let path = path.join(segment.clone());
259            let metadata = self.metadata(&path)?;
260            entries.push((segment, None, metadata));
261        }
262
263        entries.sort_by_key(|k| k.0.clone());
264
265        Some(entries)
266    }
267
268    /// Get the metadata for a particular item.
269    pub fn metadata(&self, path: &PathSegments) -> Option<Metadata> {
270        let path = self.resolve(path)?;
271        let meta = path.metadata().ok()?;
272
273        let timestamps = Timestamps::from_metadata(&meta).unwrap();
274
275        if meta.is_dir() {
276            Some(Metadata::Dir {
277                timestamps: Some(timestamps),
278            })
279        } else if meta.is_file() {
280            Some(Metadata::File {
281                length: meta.len().try_into().ok()?,
282                timestamps: Some(timestamps),
283            })
284        } else {
285            None
286        }
287    }
288
289    pub(crate) fn as_directory_tree(&self, strictness: Strictness) -> Result<Directory<'_>, Error> {
290        if self.name() == "metadata" {
291            let mut root = Directory::default();
292
293            for file_path in self.metadata_files.iter() {
294                if !file_path.exists() || !file_path.is_file() {
295                    if strictness.is_strict() {
296                        anyhow::bail!("{} does not exist", file_path.display());
297                    }
298
299                    // ignore missing metadata
300                    continue;
301                }
302                let path = file_path.strip_prefix(&self.base_dir)?;
303                let path = PathBuf::from("/").join(path);
304                let segments = path.to_path_segments()?;
305                let segments: Vec<_> = segments.iter().collect();
306
307                let file_entry = DirEntry::File(FileEntry::from_path(file_path)?);
308
309                let mut curr_dir = &mut root;
310                for (index, segment) in segments.iter().enumerate() {
311                    if segments.len() == 1 {
312                        curr_dir.children.insert((*segment).clone(), file_entry);
313                        break;
314                    } else {
315                        if index == segments.len() - 1 {
316                            curr_dir.children.insert((*segment).clone(), file_entry);
317                            break;
318                        }
319
320                        let curr_entry = curr_dir
321                            .children
322                            .entry((*segment).clone())
323                            .or_insert(DirEntry::Dir(Directory::default()));
324                        let DirEntry::Dir(dir) = curr_entry else {
325                            unreachable!()
326                        };
327
328                        curr_dir = dir;
329                    }
330                }
331            }
332
333            Ok(root)
334        } else {
335            let paths: Vec<_> = self.mapped_directories.iter().cloned().collect();
336            directory_tree(paths, &self.base_dir, self.walker_factory)
337        }
338    }
339}
340
341impl AbstractVolume for FsVolume {
342    fn read_file(&self, path: &PathSegments) -> Option<(OwnedBuffer, Option<[u8; 32]>)> {
343        self.read_file(path).map(|c| (c, None))
344    }
345
346    fn read_dir(
347        &self,
348        path: &PathSegments,
349    ) -> Option<Vec<(PathSegment, Option<[u8; 32]>, Metadata)>> {
350        self.read_dir(path)
351    }
352
353    fn metadata(&self, path: &PathSegments) -> Option<Metadata> {
354        self.metadata(path)
355    }
356}
357
358impl WasmerPackageVolume for FsVolume {
359    fn as_directory_tree(&self, strictness: Strictness) -> Result<Directory<'_>, Error> {
360        self.as_directory_tree(strictness)
361    }
362}
363
364/// Resolve a [`PathSegments`] to its equivalent path on disk.
365fn resolve(base_dir: &Path, path: &PathSegments) -> PathBuf {
366    let mut resolved = base_dir.to_path_buf();
367    for segment in path.iter() {
368        resolved.push(segment.as_str());
369    }
370
371    resolved
372}
373
374/// Given a list of absolute paths, create a directory tree relative to some
375/// base directory.
376fn directory_tree(
377    paths: impl IntoIterator<Item = PathBuf>,
378    base_dir: &Path,
379    walker_factory: crate::package::WalkBuilderFactory,
380) -> Result<Directory<'static>, Error> {
381    let paths: Vec<_> = paths.into_iter().collect();
382    let mut root = Directory::default();
383
384    for path in paths {
385        if path.is_file() {
386            let dir_entry = v3::write::DirEntry::File(v3::write::FileEntry::from_path(&path)?);
387            let path = path.strip_prefix(base_dir)?;
388            let path_segment = PathSegment::try_from(path.as_os_str())?;
389
390            if root.children.insert(path_segment, dir_entry).is_some() {
391                println!("Warning: {path:?} already exists. Overriding the old entry");
392            }
393        } else {
394            let dir = webc::v3::write::Directory::from_path_with_walker(
395                &path,
396                walker_factory.create_walk_builder(&path).build(),
397            )
398            .map_err(|e| {
399                Error::from(e).context(format!(
400                    "Unable to add \"{}\" to the directory tree",
401                    path.display()
402                ))
403            })?;
404            for (path, child) in dir.children {
405                root.children.insert(path.clone(), child);
406            }
407        }
408    }
409
410    Ok(root)
411}
412
413#[cfg(test)]
414mod tests {
415    use tempfile::TempDir;
416    use wasmer_config::package::Manifest;
417
418    use super::*;
419
420    #[test]
421    fn metadata_volume() {
422        let temp = TempDir::new().unwrap();
423        let wasmer_toml = r#"
424            [package]
425            name = "some/package"
426            version = "0.0.0"
427            description = ""
428            license-file = "./path/to/LICENSE"
429            readme = "README.md"
430
431            [[module]]
432            name = "asdf"
433            source = "asdf.wasm"
434            abi = "none"
435            bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] }
436        "#;
437        let wasmer_toml_path = temp.path().join("wasmer.toml");
438        std::fs::write(&wasmer_toml_path, wasmer_toml.as_bytes()).unwrap();
439        let license_dir = temp.path().join("path").join("to");
440        std::fs::create_dir_all(&license_dir).unwrap();
441        std::fs::write(license_dir.join("LICENSE"), "license").unwrap();
442        std::fs::write(temp.path().join("README.md"), "readme").unwrap();
443        std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap();
444        std::fs::write(temp.path().join("browser.wai"), "imports").unwrap();
445        let manifest: Manifest = toml::from_str(wasmer_toml).unwrap();
446
447        let volume = FsVolume::new_metadata(&manifest, temp.path().to_path_buf()).unwrap();
448
449        let entries = volume.read_dir(&PathSegments::ROOT).unwrap();
450        let expected = [
451            PathSegment::parse("README.md").unwrap(),
452            PathSegment::parse("asdf.wai").unwrap(),
453            PathSegment::parse("browser.wai").unwrap(),
454            PathSegment::parse("path").unwrap(),
455        ];
456
457        for i in 0..expected.len() {
458            assert_eq!(entries[i].0, expected[i]);
459            assert!(entries[i].2.timestamps().is_some());
460        }
461
462        let license: PathSegments = "/path/to/LICENSE".parse().unwrap();
463        assert_eq!(
464            String::from_utf8(volume.read_file(&license).unwrap().into()).unwrap(),
465            "license"
466        );
467    }
468
469    #[test]
470    fn asset_volume() {
471        let temp = TempDir::new().unwrap();
472        let wasmer_toml = r#"
473            [package]
474            name = "some/package"
475            version = "0.0.0"
476            description = ""
477            license_file = "./path/to/LICENSE"
478            readme = "README.md"
479
480            [[module]]
481            name = "asdf"
482            source = "asdf.wasm"
483            abi = "none"
484            bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] }
485
486            [fs]
487            "/etc" = "etc"
488        "#;
489        let license_dir = temp.path().join("path").join("to");
490        std::fs::create_dir_all(&license_dir).unwrap();
491        std::fs::write(license_dir.join("LICENSE"), "license").unwrap();
492        std::fs::write(temp.path().join("README.md"), "readme").unwrap();
493        std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap();
494        std::fs::write(temp.path().join("browser.wai"), "imports").unwrap();
495
496        let etc = temp.path().join("etc");
497        let share = etc.join("share");
498        std::fs::create_dir_all(&share).unwrap();
499
500        std::fs::write(etc.join(".wasmerignore"), b"ignore_me").unwrap();
501        std::fs::write(etc.join(".hidden"), "anything, really").unwrap();
502        std::fs::write(etc.join("ignore_me"), "I should be ignored").unwrap();
503        std::fs::write(share.join("package.1"), "man page").unwrap();
504        std::fs::write(share.join("ignore_me"), "I should be ignored too").unwrap();
505
506        let manifest: Manifest = toml::from_str(wasmer_toml).unwrap();
507
508        let volume = FsVolume::new_assets(
509            &manifest,
510            temp.path(),
511            crate::package::wasmer_ignore_walker(),
512        )
513        .unwrap();
514
515        let volume = &volume["/etc"];
516
517        let entries = volume.read_dir(&PathSegments::ROOT).unwrap();
518        let expected = [
519            PathSegment::parse(".hidden").unwrap(),
520            PathSegment::parse(".wasmerignore").unwrap(),
521            PathSegment::parse("share").unwrap(),
522        ];
523
524        for i in 0..expected.len() {
525            assert_eq!(entries[i].0, expected[i]);
526            assert!(entries[i].2.timestamps().is_some());
527        }
528
529        let man_page: PathSegments = "/share/package.1".parse().unwrap();
530        assert_eq!(
531            String::from_utf8(volume.read_file(&man_page).unwrap().into()).unwrap(),
532            "man page"
533        );
534    }
535
536    #[test]
537    fn directory_tree_propagates_errors() {
538        let temp = TempDir::new().unwrap();
539
540        // Create a directory that will be used as the base directory
541        let base_dir = temp.path().to_path_buf();
542
543        // Create a non-existent path that will cause `from_path_with_walker` to fail
544        let non_existent_path = base_dir.join("non_existent_dir");
545
546        // Try to create a directory tree with a non-existent path
547        let result = directory_tree(
548            std::iter::once(non_existent_path.clone()),
549            &base_dir,
550            crate::package::wasmer_ignore_walker(),
551        );
552
553        // Verify that the error is propagated
554        assert!(result.is_err());
555        let error = result.unwrap_err();
556        assert!(
557            error.to_string().contains("Unable to add"),
558            "Error message should indicate failure to add path: {}",
559            error
560        );
561        assert!(
562            error.to_string().contains("non_existent_dir"),
563            "Error message should include the problematic path: {}",
564            error
565        );
566    }
567}
wasmer_package/package/volume/fs.rs

wasmer_package/package/volume/
fs.rs
