wasmer_package/package/volume/
fs.rs

1use std::{
2    collections::{BTreeMap, BTreeSet},
3    fmt::Debug,
4    fs::{self, File},
5    io::Read,
6    path::{Path, PathBuf},
7};
8
9use anyhow::{Context, Error};
10use shared_buffer::OwnedBuffer;
11
12use webc::{
13    AbstractVolume, Metadata, PathSegment, PathSegments, Timestamps, ToPathSegments, sanitize_path,
14    v3::{
15        self,
16        write::{DirEntry, Directory, FileEntry, SymlinkEntry},
17    },
18};
19
20use crate::package::{Strictness, WalkBuilderFactory};
21
22use super::WasmerPackageVolume;
23
24/// A lazily loaded volume in a Wasmer package.
25///
26/// Note that it is the package resolver's role to interpret a package's
27/// filesystem mappings. A volume contains directories as they were when the
28/// package was published.
29pub struct FsVolume {
30    /// Name of the volume
31    name: String,
32    /// A pre-computed set of intermediate directories that are needed to allow
33    /// access to the whitelisted files and directories.
34    intermediate_directories: BTreeSet<PathBuf>,
35    /// Specific files that this volume has access to.
36    metadata_files: BTreeSet<PathBuf>,
37    /// Directories that allow the user to access anything inside them.
38    mapped_directories: BTreeSet<PathBuf>,
39    /// The base directory all [`PathSegments`] will be resolved relative to.
40    base_dir: PathBuf,
41    /// The walker builder factory to use when reading directories.
42    walker_factory: WalkBuilderFactory,
43}
44
45impl Debug for FsVolume {
46    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
47        f.debug_struct("FsVolume")
48            .field("name", &self.name)
49            .field("intermediate_directories", &self.intermediate_directories)
50            .field("metadata_files", &self.metadata_files)
51            .field("mapped_directories", &self.mapped_directories)
52            .field("base_dir", &self.base_dir)
53            .finish()
54    }
55}
56
57impl FsVolume {
58    /// The name of the volume used to store metadata files.
59    pub(crate) const METADATA: &'static str = "metadata";
60
61    /// Create a new metadata volume.
62    pub(crate) fn new_metadata(
63        manifest: &wasmer_config::package::Manifest,
64        base_dir: impl Into<PathBuf>,
65    ) -> Result<Self, Error> {
66        let base_dir = base_dir.into();
67        let mut files = BTreeSet::new();
68
69        // check if manifest.package is None
70        if let Some(package) = &manifest.package {
71            if let Some(license_file) = &package.license_file {
72                files.insert(base_dir.join(license_file));
73            }
74
75            if let Some(readme) = &package.readme {
76                files.insert(base_dir.join(readme));
77            }
78        }
79
80        for module in &manifest.modules {
81            if let Some(bindings) = &module.bindings {
82                let bindings_files = bindings.referenced_files(&base_dir)?;
83                files.extend(bindings_files);
84            }
85        }
86
87        Ok(FsVolume::new_with_intermediate_dirs(
88            FsVolume::METADATA.to_string(),
89            base_dir,
90            files,
91            BTreeSet::new(),
92            crate::package::include_everything_walker(),
93        ))
94    }
95
96    pub(crate) fn new_assets(
97        manifest: &wasmer_config::package::Manifest,
98        base_dir: &Path,
99        walker_factory: crate::package::WalkBuilderFactory,
100    ) -> Result<BTreeMap<String, Self>, Error> {
101        // Create asset volumes
102        let dirs: BTreeSet<_> = manifest
103            .fs
104            .values()
105            .map(|path| base_dir.join(path))
106            .collect();
107
108        for path in &dirs {
109            // Perform a basic sanity check without following symlinks.
110            let _ = std::fs::symlink_metadata(path).with_context(|| {
111                format!("Unable to get the metadata for \"{}\"", path.display())
112            })?;
113        }
114
115        let mut volumes = BTreeMap::new();
116        for entry in manifest.fs.values() {
117            let name = entry
118                .to_str()
119                .ok_or_else(|| anyhow::anyhow!("Failed to convert path to str"))?;
120
121            let name = sanitize_path(name);
122
123            let mut dirs = BTreeSet::new();
124            let dir = base_dir.join(entry);
125            dirs.insert(dir);
126
127            volumes.insert(
128                name.clone(),
129                FsVolume::new(
130                    name.to_string(),
131                    base_dir.to_path_buf(),
132                    BTreeSet::new(),
133                    dirs,
134                    walker_factory,
135                ),
136            );
137        }
138
139        Ok(volumes)
140    }
141
142    pub(crate) fn new_with_intermediate_dirs(
143        name: String,
144        base_dir: PathBuf,
145        whitelisted_files: BTreeSet<PathBuf>,
146        whitelisted_directories: BTreeSet<PathBuf>,
147        walker_factory: crate::package::WalkBuilderFactory,
148    ) -> Self {
149        let mut intermediate_directories: BTreeSet<PathBuf> = whitelisted_files
150            .iter()
151            .filter_map(|p| p.parent())
152            .chain(whitelisted_directories.iter().map(|p| p.as_path()))
153            .flat_map(|dir| dir.ancestors())
154            .filter(|dir| dir.starts_with(&base_dir))
155            .map(|dir| dir.to_path_buf())
156            .collect();
157
158        // The base directory is always accessible (even if its contents isn't)
159        intermediate_directories.insert(base_dir.clone());
160
161        FsVolume {
162            name,
163            intermediate_directories,
164            metadata_files: whitelisted_files,
165            mapped_directories: whitelisted_directories,
166            base_dir,
167            walker_factory,
168        }
169    }
170
171    pub(crate) fn new(
172        name: String,
173        base_dir: PathBuf,
174        whitelisted_files: BTreeSet<PathBuf>,
175        whitelisted_directories: BTreeSet<PathBuf>,
176        walker_factory: crate::package::WalkBuilderFactory,
177    ) -> Self {
178        FsVolume {
179            name,
180            intermediate_directories: BTreeSet::new(),
181            metadata_files: whitelisted_files,
182            mapped_directories: whitelisted_directories,
183            base_dir,
184            walker_factory,
185        }
186    }
187
188    fn is_accessible(&self, path: &Path) -> bool {
189        self.intermediate_directories.contains(path)
190            || self.metadata_files.contains(path)
191            || self
192                .mapped_directories
193                .iter()
194                .any(|dir| path.starts_with(dir))
195    }
196
197    fn resolve(&self, path: &PathSegments) -> Option<PathBuf> {
198        let resolved = if let Some(dir) = &self.mapped_directories.first() {
199            resolve(dir, path)
200        } else {
201            resolve(&self.base_dir, path)
202        };
203
204        let accessible = self.is_accessible(&resolved);
205        accessible.then_some(resolved)
206    }
207
208    /// Returns the name of the volume
209    pub fn name(&self) -> &str {
210        self.name.as_str()
211    }
212
213    /// Read a file from the volume.
214    pub fn read_file(&self, path: &PathSegments) -> Option<OwnedBuffer> {
215        let path = self.resolve(path)?;
216        if !path.symlink_metadata().ok()?.is_file() {
217            return None;
218        }
219
220        let mut f = File::open(path).ok()?;
221
222        // First we try to mmap it
223        if let Ok(mmapped) = OwnedBuffer::from_file(&f) {
224            return Some(mmapped);
225        }
226
227        // otherwise, fall back to reading the file's contents into memory
228        let mut buffer = Vec::new();
229        f.read_to_end(&mut buffer).ok()?;
230        Some(OwnedBuffer::from_bytes(buffer))
231    }
232
233    /// Read the contents of a directory.
234    #[allow(clippy::type_complexity)]
235    pub fn read_dir(
236        &self,
237        path: &PathSegments,
238    ) -> Option<Vec<(PathSegment, Option<[u8; 32]>, Metadata)>> {
239        let resolved = self.resolve(path)?;
240        if !resolved.symlink_metadata().ok()?.is_dir() {
241            return None;
242        }
243
244        let mut walker_builder = self.walker_factory.create_walk_builder(&resolved);
245        walker_builder.max_depth(Some(1));
246        let walker = walker_builder.build();
247
248        let mut entries = Vec::new();
249
250        for entry in walker {
251            let entry = entry.ok()?;
252            // Walk returns the root dir as well, we don't want to process it
253            if entry.depth() == 0 {
254                continue;
255            }
256
257            let entry = entry.path();
258
259            if !self.is_accessible(entry) {
260                continue;
261            }
262
263            let segment: PathSegment = entry.file_name()?.to_str()?.parse().ok()?;
264
265            let path = path.join(segment.clone());
266            let metadata = self.metadata(&path)?;
267            entries.push((segment, None, metadata));
268        }
269
270        entries.sort_by_key(|k| k.0.clone());
271
272        Some(entries)
273    }
274
275    /// Get the metadata for a particular item.
276    pub fn metadata(&self, path: &PathSegments) -> Option<Metadata> {
277        let path = self.resolve(path)?;
278        let meta = path.symlink_metadata().ok()?;
279
280        let timestamps = Timestamps::from_metadata(&meta).ok()?;
281
282        if meta.file_type().is_symlink() {
283            let target = fs::read_link(&path).ok()?;
284            let target = target.to_str()?;
285
286            Some(Metadata::Symlink {
287                target_length: target.len(),
288                timestamps: Some(timestamps),
289            })
290        } else if meta.is_dir() {
291            Some(Metadata::Dir {
292                timestamps: Some(timestamps),
293            })
294        } else if meta.is_file() {
295            Some(Metadata::File {
296                length: meta.len().try_into().ok()?,
297                timestamps: Some(timestamps),
298            })
299        } else {
300            None
301        }
302    }
303
304    pub(crate) fn as_directory_tree(&self, strictness: Strictness) -> Result<Directory<'_>, Error> {
305        if self.name() == "metadata" {
306            let mut root = Directory::default();
307
308            for file_path in self.metadata_files.iter() {
309                let meta = match file_path.symlink_metadata() {
310                    Ok(meta) => meta,
311                    Err(_) if strictness.is_strict() => {
312                        anyhow::bail!("{} does not exist", file_path.display());
313                    }
314                    Err(_) => {
315                        // ignore missing metadata
316                        continue;
317                    }
318                };
319
320                if !meta.is_file() && !meta.file_type().is_symlink() {
321                    if strictness.is_strict() {
322                        anyhow::bail!("{} is not a file", file_path.display());
323                    }
324
325                    continue;
326                }
327
328                let path = file_path.strip_prefix(&self.base_dir)?;
329                let path = PathBuf::from("/").join(path);
330                let segments = path.to_path_segments()?;
331                let segments: Vec<_> = segments.iter().collect();
332
333                let file_entry = if meta.file_type().is_symlink() {
334                    DirEntry::Symlink(SymlinkEntry::from_path(file_path)?)
335                } else {
336                    DirEntry::File(FileEntry::from_path(file_path)?)
337                };
338
339                let mut curr_dir = &mut root;
340                for (index, segment) in segments.iter().enumerate() {
341                    if segments.len() == 1 {
342                        curr_dir.children.insert((*segment).clone(), file_entry);
343                        break;
344                    } else {
345                        if index == segments.len() - 1 {
346                            curr_dir.children.insert((*segment).clone(), file_entry);
347                            break;
348                        }
349
350                        let curr_entry = curr_dir
351                            .children
352                            .entry((*segment).clone())
353                            .or_insert(DirEntry::Dir(Directory::default()));
354                        let DirEntry::Dir(dir) = curr_entry else {
355                            unreachable!()
356                        };
357
358                        curr_dir = dir;
359                    }
360                }
361            }
362
363            Ok(root)
364        } else {
365            let paths: Vec<_> = self.mapped_directories.iter().cloned().collect();
366            directory_tree(paths, &self.base_dir, self.walker_factory)
367        }
368    }
369}
370
371impl AbstractVolume for FsVolume {
372    fn read_file(&self, path: &PathSegments) -> Option<(OwnedBuffer, Option<[u8; 32]>)> {
373        self.read_file(path).map(|c| (c, None))
374    }
375
376    fn read_dir(
377        &self,
378        path: &PathSegments,
379    ) -> Option<Vec<(PathSegment, Option<[u8; 32]>, Metadata)>> {
380        self.read_dir(path)
381    }
382
383    fn metadata(&self, path: &PathSegments) -> Option<Metadata> {
384        self.metadata(path)
385    }
386
387    fn read_link(&self, path: &PathSegments) -> Option<(String, Option<[u8; 32]>)> {
388        let path = self.resolve(path)?;
389        if !path.symlink_metadata().ok()?.file_type().is_symlink() {
390            return None;
391        }
392
393        let target = fs::read_link(path).ok()?;
394        Some((target.to_str()?.to_owned(), None))
395    }
396}
397
398impl WasmerPackageVolume for FsVolume {
399    fn as_directory_tree(&self, strictness: Strictness) -> Result<Directory<'_>, Error> {
400        self.as_directory_tree(strictness)
401    }
402}
403
404/// Resolve a [`PathSegments`] to its equivalent path on disk.
405fn resolve(base_dir: &Path, path: &PathSegments) -> PathBuf {
406    let mut resolved = base_dir.to_path_buf();
407    for segment in path.iter() {
408        resolved.push(segment.as_str());
409    }
410
411    resolved
412}
413
414/// Given a list of absolute paths, create a directory tree relative to some
415/// base directory.
416fn directory_tree(
417    paths: impl IntoIterator<Item = PathBuf>,
418    base_dir: &Path,
419    walker_factory: crate::package::WalkBuilderFactory,
420) -> Result<Directory<'static>, Error> {
421    let paths: Vec<_> = paths.into_iter().collect();
422    let mut root = Directory::default();
423
424    for path in paths {
425        let meta = path.symlink_metadata().map_err(|e| {
426            Error::from(e).context(format!(
427                "Unable to add \"{}\" to the directory tree",
428                path.display()
429            ))
430        })?;
431        let file_type = meta.file_type();
432
433        if file_type.is_symlink() {
434            let dir_entry =
435                v3::write::DirEntry::Symlink(v3::write::SymlinkEntry::from_path(&path)?);
436            let path = path.strip_prefix(base_dir)?;
437            let path_segment = PathSegment::try_from(path.as_os_str())?;
438
439            if root.children.insert(path_segment, dir_entry).is_some() {
440                println!("Warning: {path:?} already exists. Overriding the old entry");
441            }
442        } else if file_type.is_file() {
443            let dir_entry = v3::write::DirEntry::File(v3::write::FileEntry::from_path(&path)?);
444            let path = path.strip_prefix(base_dir)?;
445            let path_segment = PathSegment::try_from(path.as_os_str())?;
446
447            if root.children.insert(path_segment, dir_entry).is_some() {
448                println!("Warning: {path:?} already exists. Overriding the old entry");
449            }
450        } else {
451            let dir = webc::v3::write::Directory::from_path_with_walker(
452                &path,
453                walker_factory.create_walk_builder(&path).build(),
454            )
455            .map_err(|e| {
456                Error::from(e).context(format!(
457                    "Unable to add \"{}\" to the directory tree",
458                    path.display()
459                ))
460            })?;
461            for (path, child) in dir.children {
462                root.children.insert(path.clone(), child);
463            }
464        }
465    }
466
467    Ok(root)
468}
469
470#[cfg(test)]
471mod tests {
472    use tempfile::TempDir;
473    use wasmer_config::package::Manifest;
474
475    use super::*;
476
477    #[test]
478    fn metadata_volume() {
479        let temp = TempDir::new().unwrap();
480        let wasmer_toml = r#"
481            [package]
482            name = "some/package"
483            version = "0.0.0"
484            description = ""
485            license-file = "./path/to/LICENSE"
486            readme = "README.md"
487
488            [[module]]
489            name = "asdf"
490            source = "asdf.wasm"
491            abi = "none"
492            bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] }
493        "#;
494        let wasmer_toml_path = temp.path().join("wasmer.toml");
495        std::fs::write(&wasmer_toml_path, wasmer_toml.as_bytes()).unwrap();
496        let license_dir = temp.path().join("path").join("to");
497        std::fs::create_dir_all(&license_dir).unwrap();
498        std::fs::write(license_dir.join("LICENSE"), "license").unwrap();
499        std::fs::write(temp.path().join("README.md"), "readme").unwrap();
500        std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap();
501        std::fs::write(temp.path().join("browser.wai"), "imports").unwrap();
502        let manifest: Manifest = toml::from_str(wasmer_toml).unwrap();
503
504        let volume = FsVolume::new_metadata(&manifest, temp.path().to_path_buf()).unwrap();
505
506        let entries = volume.read_dir(&PathSegments::ROOT).unwrap();
507        let expected = [
508            PathSegment::parse("README.md").unwrap(),
509            PathSegment::parse("asdf.wai").unwrap(),
510            PathSegment::parse("browser.wai").unwrap(),
511            PathSegment::parse("path").unwrap(),
512        ];
513
514        for i in 0..expected.len() {
515            assert_eq!(entries[i].0, expected[i]);
516            assert!(entries[i].2.timestamps().is_some());
517        }
518
519        let license: PathSegments = "/path/to/LICENSE".parse().unwrap();
520        assert_eq!(
521            String::from_utf8(volume.read_file(&license).unwrap().into()).unwrap(),
522            "license"
523        );
524    }
525
526    #[test]
527    fn asset_volume() {
528        let temp = TempDir::new().unwrap();
529        let wasmer_toml = r#"
530            [package]
531            name = "some/package"
532            version = "0.0.0"
533            description = ""
534            license_file = "./path/to/LICENSE"
535            readme = "README.md"
536
537            [[module]]
538            name = "asdf"
539            source = "asdf.wasm"
540            abi = "none"
541            bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] }
542
543            [fs]
544            "/etc" = "etc"
545        "#;
546        let license_dir = temp.path().join("path").join("to");
547        std::fs::create_dir_all(&license_dir).unwrap();
548        std::fs::write(license_dir.join("LICENSE"), "license").unwrap();
549        std::fs::write(temp.path().join("README.md"), "readme").unwrap();
550        std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap();
551        std::fs::write(temp.path().join("browser.wai"), "imports").unwrap();
552
553        let etc = temp.path().join("etc");
554        let share = etc.join("share");
555        std::fs::create_dir_all(&share).unwrap();
556
557        std::fs::write(etc.join(".wasmerignore"), b"ignore_me").unwrap();
558        std::fs::write(etc.join(".hidden"), "anything, really").unwrap();
559        std::fs::write(etc.join("ignore_me"), "I should be ignored").unwrap();
560        std::fs::write(share.join("package.1"), "man page").unwrap();
561        std::fs::write(share.join("ignore_me"), "I should be ignored too").unwrap();
562
563        let manifest: Manifest = toml::from_str(wasmer_toml).unwrap();
564
565        let volume = FsVolume::new_assets(
566            &manifest,
567            temp.path(),
568            crate::package::wasmer_ignore_walker(),
569        )
570        .unwrap();
571
572        let volume = &volume["/etc"];
573
574        let entries = volume.read_dir(&PathSegments::ROOT).unwrap();
575        let expected = [
576            PathSegment::parse(".hidden").unwrap(),
577            PathSegment::parse(".wasmerignore").unwrap(),
578            PathSegment::parse("share").unwrap(),
579        ];
580
581        for i in 0..expected.len() {
582            assert_eq!(entries[i].0, expected[i]);
583            assert!(entries[i].2.timestamps().is_some());
584        }
585
586        let man_page: PathSegments = "/share/package.1".parse().unwrap();
587        assert_eq!(
588            String::from_utf8(volume.read_file(&man_page).unwrap().into()).unwrap(),
589            "man page"
590        );
591    }
592
593    #[cfg(unix)]
594    #[test]
595    fn fs_volume_preserves_symlinks() {
596        let temp = TempDir::new().unwrap();
597        let target_file = temp.path().join("target.txt");
598        let target_dir = temp.path().join("target-dir");
599
600        std::fs::write(&target_file, "target").unwrap();
601        std::fs::create_dir(&target_dir).unwrap();
602        std::os::unix::fs::symlink("target.txt", temp.path().join("file-link")).unwrap();
603        std::os::unix::fs::symlink("target-dir", temp.path().join("dir-link")).unwrap();
604        std::os::unix::fs::symlink("subdir/../target.txt", temp.path().join("nested-link"))
605            .unwrap();
606        std::os::unix::fs::symlink("missing.txt", temp.path().join("broken-link")).unwrap();
607
608        let volume = FsVolume::new(
609            "/assets".to_string(),
610            temp.path().to_path_buf(),
611            BTreeSet::new(),
612            BTreeSet::from([temp.path().to_path_buf()]),
613            crate::package::include_everything_walker(),
614        );
615
616        let file_link: PathSegments = "/file-link".parse().unwrap();
617        let file_link_meta = volume.metadata(&file_link).unwrap();
618        assert!(file_link_meta.is_symlink());
619        assert_eq!(
620            match file_link_meta {
621                Metadata::Symlink { target_length, .. } => target_length,
622                _ => unreachable!(),
623            },
624            "target.txt".len()
625        );
626        assert_eq!(
627            volume.read_link(&file_link).unwrap().0,
628            "target.txt".to_string()
629        );
630        assert!(volume.read_file(&file_link).is_none());
631
632        let broken_link: PathSegments = "/broken-link".parse().unwrap();
633        let broken_link_meta = volume.metadata(&broken_link).unwrap();
634        assert!(broken_link_meta.is_symlink());
635        assert_eq!(
636            volume.read_link(&broken_link).unwrap().0,
637            "missing.txt".to_string()
638        );
639
640        let nested_link: PathSegments = "/nested-link".parse().unwrap();
641        assert_eq!(
642            volume.read_link(&nested_link).unwrap().0,
643            "subdir/../target.txt".to_string()
644        );
645
646        let entries = volume.read_dir(&PathSegments::ROOT).unwrap();
647        assert!(
648            entries
649                .iter()
650                .any(|(name, _, meta)| name.as_str() == "broken-link" && meta.is_symlink())
651        );
652
653        let dir = volume.as_directory_tree(Strictness::Strict).unwrap();
654        for link in ["broken-link", "dir-link", "file-link", "nested-link"] {
655            assert!(matches!(
656                dir.children.get(&PathSegment::parse(link).unwrap()),
657                Some(v3::write::DirEntry::Symlink(_))
658            ));
659        }
660    }
661
662    #[test]
663    fn directory_tree_propagates_errors() {
664        let temp = TempDir::new().unwrap();
665
666        // Create a directory that will be used as the base directory
667        let base_dir = temp.path().to_path_buf();
668
669        // Create a non-existent path that will cause `from_path_with_walker` to fail
670        let non_existent_path = base_dir.join("non_existent_dir");
671
672        // Try to create a directory tree with a non-existent path
673        let result = directory_tree(
674            std::iter::once(non_existent_path.clone()),
675            &base_dir,
676            crate::package::wasmer_ignore_walker(),
677        );
678
679        // Verify that the error is propagated
680        assert!(result.is_err());
681        let error = result.unwrap_err();
682        assert!(
683            error.to_string().contains("Unable to add"),
684            "Error message should indicate failure to add path: {error}",
685        );
686        assert!(
687            error.to_string().contains("non_existent_dir"),
688            "Error message should include the problematic path: {error}",
689        );
690    }
691}
wasmer_package/package/volume/fs.rs

wasmer_package/package/volume/
fs.rs
