virtual_fs/
overlay_fs.rs

1use std::{
2    collections::HashSet,
3    fmt::Debug,
4    io::{self, SeekFrom},
5    path::{Path, PathBuf},
6    pin::Pin,
7    sync::Arc,
8    task::{Context, Poll},
9};
10
11use futures::future::BoxFuture;
12use replace_with::replace_with_or_abort;
13use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf};
14
15use crate::{
16    FileOpener, FileSystem, FileSystems, FsError, Metadata, OpenOptions, OpenOptionsConfig,
17    ReadDir, VirtualFile, ops,
18};
19
20/// A primary filesystem and chain of secondary filesystems that are overlayed
21/// on top of each other.
22///
23/// # Precedence
24///
25/// The [`OverlayFileSystem`] will execute operations based on precedence.
26///
27///
28/// Most importantly, this means earlier filesystems can shadow files and
29/// directories that have a lower precedence.
30///
31///# Examples
32///
33/// Something useful to know is that the [`FileSystems`] trait is implemented
34/// for both arrays and tuples.
35///
36/// For example, if you want to create a [`crate::FileSystem`] which will
37/// create files in-memory while still being able to read from the host, you
38/// might do something like this:
39///
40/// ```rust
41/// use virtual_fs::{
42///     mem_fs::FileSystem as MemFS,
43///     host_fs::FileSystem as HostFS,
44///     OverlayFileSystem,
45/// };
46///
47/// let runtime = tokio::runtime::Builder::new_current_thread()
48///     .enable_all()
49///     .build()
50///     .unwrap();
51/// let _guard = runtime.enter();
52///
53/// let fs = OverlayFileSystem::new(MemFS::default(), [HostFS::new(tokio::runtime::Handle::current(), "/").unwrap()]);
54///
55/// // This also has the benefit of storing the two values in-line with no extra
56/// // overhead or indirection.
57/// assert_eq!(
58///     std::mem::size_of_val(&fs),
59///     std::mem::size_of::<(MemFS, HostFS)>(),
60/// );
61/// ```
62///
63/// A more complex example is
64#[derive(Clone, PartialEq, Eq)]
65pub struct OverlayFileSystem<P, S> {
66    primary: Arc<P>,
67    secondaries: S,
68}
69
70impl<P, S> OverlayFileSystem<P, S>
71where
72    P: FileSystem + Send + Sync + 'static,
73    S: for<'a> FileSystems<'a> + Send + Sync + 'static,
74{
75    /// Create a new [`FileSystem`] using a primary [`crate::FileSystem`] and a
76    /// chain of secondary [`FileSystems`].
77    pub fn new(primary: P, secondaries: S) -> Self {
78        OverlayFileSystem {
79            primary: Arc::new(primary),
80            secondaries,
81        }
82    }
83
84    /// Get a reference to the primary filesystem.
85    pub fn primary(&self) -> &P {
86        &self.primary
87    }
88
89    /// Get a reference to the secondary filesystems.
90    pub fn secondaries(&self) -> &S {
91        &self.secondaries
92    }
93
94    /// Get a mutable reference to the secondary filesystems.
95    pub fn secondaries_mut(&mut self) -> &mut S {
96        &mut self.secondaries
97    }
98
99    fn permission_error_or_not_found(&self, path: &Path) -> Result<(), FsError> {
100        for fs in self.secondaries.filesystems() {
101            if ops::exists(fs, path) {
102                return Err(FsError::PermissionDenied);
103            }
104        }
105
106        Err(FsError::EntryNotFound)
107    }
108}
109
110impl<P, S> FileSystem for OverlayFileSystem<P, S>
111where
112    P: FileSystem + Send + 'static,
113    S: for<'a> FileSystems<'a> + Send + Sync + 'static,
114    for<'a> <<S as FileSystems<'a>>::Iter as IntoIterator>::IntoIter: Send,
115{
116    fn readlink(&self, path: &Path) -> crate::Result<PathBuf> {
117        // Whiteout files can not be read, they are just markers
118        if ops::is_white_out(path).is_some() {
119            return Err(FsError::EntryNotFound);
120        }
121
122        // Check if the file is in the primary
123        match self.primary.readlink(path) {
124            Ok(meta) => return Ok(meta),
125            Err(e) if should_continue(e) => {}
126            Err(e) => return Err(e),
127        }
128
129        // There might be a whiteout, search for this
130        if ops::has_white_out(&self.primary, path) {
131            return Err(FsError::EntryNotFound);
132        }
133
134        // Otherwise scan the secondaries
135        for fs in self.secondaries.filesystems() {
136            match fs.readlink(path) {
137                Err(e) if should_continue(e) => continue,
138                other => return other,
139            }
140        }
141
142        Err(FsError::EntryNotFound)
143    }
144
145    fn read_dir(&self, path: &Path) -> Result<ReadDir, FsError> {
146        let mut entries = Vec::new();
147        let mut had_at_least_one_success = false;
148        let mut white_outs = HashSet::new();
149
150        let filesystems = std::iter::once(&self.primary as &(dyn FileSystem + Send))
151            .chain(self.secondaries().filesystems());
152
153        for fs in filesystems {
154            match fs.read_dir(path) {
155                Ok(r) => {
156                    for entry in r {
157                        let entry = entry?;
158
159                        // White out entries block any later entries in the secondaries
160                        // unless the entry has comes before the white out, thus the order
161                        // that the file systems are parsed is important to this logic.
162                        if let Some(path) = entry.is_white_out() {
163                            tracing::trace!(
164                                path=%path.display(),
165                                "Found whiteout file",
166                            );
167                            white_outs.insert(path);
168                            continue;
169                        } else if white_outs.contains(&entry.path) {
170                            tracing::trace!(
171                                path=%path.display(),
172                                "Skipping path because a whiteout exists",
173                            );
174                            continue;
175                        }
176
177                        entries.push(entry);
178                    }
179                    had_at_least_one_success = true;
180                }
181                Err(e) if should_continue(e) => continue,
182                Err(e) => return Err(e),
183            }
184        }
185
186        if had_at_least_one_success {
187            // Make sure later entries are removed in favour of earlier ones.
188            // Note: this sort is guaranteed to be stable, meaning filesystems
189            // "higher up" the chain will be further towards the start and kept
190            // when deduplicating.
191            entries.sort_by(|a, b| a.path.cmp(&b.path));
192            entries.dedup_by(|a, b| a.path == b.path);
193
194            Ok(ReadDir::new(entries))
195        } else {
196            Err(FsError::BaseNotDirectory)
197        }
198    }
199
200    fn create_dir(&self, path: &Path) -> Result<(), FsError> {
201        // You can not create directories that use the whiteout prefix
202        if ops::is_white_out(path).is_some() {
203            return Err(FsError::InvalidInput);
204        }
205
206        // It could be the case that the directory was earlier hidden in the secondaries
207        // by a whiteout file, hence we need to make sure those are cleared out.
208        ops::remove_white_out(self.primary.as_ref(), path);
209
210        // Make sure the parent tree is in place on the primary, this is to cover the
211        // scenario where the secondaries has a parent structure that is not yet in the
212        // primary and the primary needs it to create a sub-directory
213        if let Some(parent) = path.parent() {
214            if self.read_dir(parent).is_ok() {
215                ops::create_dir_all(&self.primary, parent).ok();
216            }
217        }
218
219        // Create the directory in the primary
220        match self.primary.create_dir(path) {
221            Err(e) if should_continue(e) => {}
222            other => return other,
223        }
224
225        self.permission_error_or_not_found(path)
226    }
227
228    fn remove_dir(&self, path: &Path) -> Result<(), FsError> {
229        // Whiteout files can not be removed, instead the original directory
230        // must be removed or recreated.
231        if ops::is_white_out(path).is_some() {
232            tracing::trace!(
233                path=%path.display(),
234                "Unable to remove a whited out directory",
235            );
236            return Err(FsError::EntryNotFound);
237        }
238
239        // If the directory is contained in a secondary file system then we need to create a
240        // whiteout file so that it is suppressed and is no longer returned in `readdir` calls.
241
242        let had_at_least_one_success = self.secondaries.filesystems().into_iter().any(|fs| {
243            fs.read_dir(path).is_ok() && ops::create_white_out(&self.primary, path).is_ok()
244        });
245
246        // Attempt to remove it from the primary, if this succeeds then we may have also
247        // added the whiteout file in the earlier step, but are required in this case to
248        // properly delete the directory.
249        match self.primary.remove_dir(path) {
250            Err(e) if should_continue(e) => {}
251            other => return other,
252        }
253
254        if had_at_least_one_success {
255            return Ok(());
256        }
257        self.permission_error_or_not_found(path)
258    }
259
260    fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, Result<(), FsError>> {
261        let from = from.to_owned();
262        let to = to.to_owned();
263        Box::pin(async move {
264            // Whiteout files can not be renamed
265            if ops::is_white_out(&from).is_some() {
266                tracing::trace!(
267                    from=%from.display(),
268                    to=%to.display(),
269                    "Attempting to rename a file that was whited out"
270                );
271                return Err(FsError::EntryNotFound);
272            }
273            // You can not rename a file into a whiteout file
274            if ops::is_white_out(&to).is_some() {
275                tracing::trace!(
276                    from=%from.display(),
277                    to=%to.display(),
278                    "Attempting to rename a file into a whiteout file"
279                );
280                return Err(FsError::InvalidInput);
281            }
282
283            // We attempt to rename the file or directory in the primary
284            // if this succeeds then we also need to ensure the white out
285            // files are created where we need them, so we do not immediately
286            // return until that is done
287            let mut had_at_least_one_success = false;
288            match self.primary.rename(&from, &to).await {
289                Err(e) if should_continue(e) => {}
290                Ok(()) => {
291                    had_at_least_one_success = true;
292                }
293                other => return other,
294            }
295
296            // If we have not yet renamed the file it may still reside in
297            // the secondaries, in which case we need to copy it to the
298            // primary rather than rename it
299            if !had_at_least_one_success {
300                for fs in self.secondaries.filesystems() {
301                    if fs.metadata(&from).is_ok() {
302                        ops::copy_reference_ext(fs, &self.primary, &from, &to).await?;
303                        had_at_least_one_success = true;
304                        break;
305                    }
306                }
307            }
308
309            // If the rename operation was a success then we need to update any
310            // whiteout files on the primary before we return success.
311            if had_at_least_one_success {
312                for fs in self.secondaries.filesystems() {
313                    if fs.metadata(&from).is_ok() {
314                        tracing::trace!(
315                            path=%from.display(),
316                            "Creating a whiteout for the file that was renamed",
317                        );
318                        ops::create_white_out(&self.primary, &from).ok();
319                        break;
320                    }
321                }
322                ops::remove_white_out(&self.primary, &to);
323                return Ok(());
324            }
325
326            // Otherwise we are in a failure scenario
327            self.permission_error_or_not_found(&from)
328        })
329    }
330
331    fn metadata(&self, path: &Path) -> Result<Metadata, FsError> {
332        // Whiteout files can not be read, they are just markers
333        if ops::is_white_out(path).is_some() {
334            return Err(FsError::EntryNotFound);
335        }
336
337        // Check if the file is in the primary
338        match self.primary.metadata(path) {
339            Ok(meta) => return Ok(meta),
340            Err(e) if should_continue(e) => {}
341            Err(e) => return Err(e),
342        }
343
344        // There might be a whiteout, search for this
345        if ops::has_white_out(&self.primary, path) {
346            return Err(FsError::EntryNotFound);
347        }
348
349        // Otherwise scan the secondaries
350        for fs in self.secondaries.filesystems() {
351            match fs.metadata(path) {
352                Err(e) if should_continue(e) => continue,
353                other => return other,
354            }
355        }
356
357        Err(FsError::EntryNotFound)
358    }
359
360    fn symlink_metadata(&self, path: &Path) -> crate::Result<Metadata> {
361        // Whiteout files can not be read, they are just markers
362        if ops::is_white_out(path).is_some() {
363            return Err(FsError::EntryNotFound);
364        }
365
366        // Check if the file is in the primary
367        match self.primary.symlink_metadata(path) {
368            Ok(meta) => return Ok(meta),
369            Err(e) if should_continue(e) => {}
370            Err(e) => return Err(e),
371        }
372
373        // There might be a whiteout, search for this
374        if ops::has_white_out(&self.primary, path) {
375            return Err(FsError::EntryNotFound);
376        }
377
378        // Otherwise scan the secondaries
379        for fs in self.secondaries.filesystems() {
380            match fs.symlink_metadata(path) {
381                Err(e) if should_continue(e) => continue,
382                other => return other,
383            }
384        }
385
386        Err(FsError::EntryNotFound)
387    }
388
389    fn remove_file(&self, path: &Path) -> Result<(), FsError> {
390        // It is not possible to delete whiteout files directly, instead
391        // one must delete the original file
392        if ops::is_white_out(path).is_some() {
393            return Err(FsError::InvalidInput);
394        }
395
396        // If the file is contained in a secondary then then we need to create a
397        // whiteout file so that it is suppressed.
398        let had_at_least_one_success = self.secondaries.filesystems().into_iter().any(|fs| {
399            fs.metadata(path).is_ok() && ops::create_white_out(&self.primary, path).is_ok()
400        });
401
402        // Attempt to remove it from the primary
403        match self.primary.remove_file(path) {
404            Err(e) if should_continue(e) => {}
405            other => return other,
406        }
407
408        if had_at_least_one_success {
409            return Ok(());
410        }
411        self.permission_error_or_not_found(path)
412    }
413
414    fn new_open_options(&self) -> OpenOptions<'_> {
415        OpenOptions::new(self)
416    }
417
418    fn mount(
419        &self,
420        _name: String,
421        _path: &Path,
422        _fs: Box<dyn FileSystem + Send + Sync>,
423    ) -> Result<(), FsError> {
424        Err(FsError::Unsupported)
425    }
426}
427
428impl<P, S> FileOpener for OverlayFileSystem<P, S>
429where
430    P: FileSystem + Send + 'static,
431    S: for<'a> FileSystems<'a> + Send + Sync + 'static,
432    for<'a> <<S as FileSystems<'a>>::Iter as IntoIterator>::IntoIter: Send,
433{
434    fn open(
435        &self,
436        path: &Path,
437        conf: &OpenOptionsConfig,
438    ) -> Result<Box<dyn VirtualFile + Send + Sync + 'static>, FsError> {
439        // Whiteout files can not be read, they are just markers
440        if ops::is_white_out(path).is_some() {
441            tracing::trace!(
442                path=%path.display(),
443                options=?conf,
444                "Whiteout files can't be opened",
445            );
446            return Err(FsError::InvalidInput);
447        }
448
449        // Check if the file is in the primary (without actually creating it) as
450        // when the file is in the primary it takes preference over any of file
451        {
452            let mut conf = conf.clone();
453            conf.create = false;
454            conf.create_new = false;
455            match self.primary.new_open_options().options(conf).open(path) {
456                Err(e) if should_continue(e) => {}
457                other => return other,
458            }
459        }
460
461        // In the scenario that we are creating the file then there is
462        // special handling that will ensure its setup correctly
463        if conf.create_new {
464            // When the secondary has the directory structure but the primary
465            // does not then we need to make sure we create all the structure
466            // in the primary
467            if let Some(parent) = path.parent() {
468                if ops::exists(self, parent) {
469                    // We create the directory structure on the primary so that
470                    // the new file can be created, this will make it override
471                    // whatever is in the primary
472                    ops::create_dir_all(&self.primary, parent)?;
473                } else {
474                    return Err(FsError::EntryNotFound);
475                }
476            }
477
478            // Remove any whiteout
479            ops::remove_white_out(&self.primary, path);
480
481            // Create the file in the primary
482            return self
483                .primary
484                .new_open_options()
485                .options(conf.clone())
486                .open(path);
487        }
488
489        // There might be a whiteout, search for this and if its found then
490        // we are done as the secondary file or directory has been earlier
491        // deleted via a white out (when the create flag is set then
492        // the white out marker is ignored)
493        if !conf.create && ops::has_white_out(&self.primary, path) {
494            tracing::trace!(
495                path=%path.display(),
496                "The file has been whited out",
497            );
498            return Err(FsError::EntryNotFound);
499        }
500
501        // Determine if a mutation will be possible with the opened file
502        let require_mutations = conf.append || conf.write || conf.create_new | conf.truncate;
503
504        // If the file is on a secondary then we should open it
505        if !ops::has_white_out(&self.primary, path) {
506            for fs in self.secondaries.filesystems() {
507                let mut sub_conf = conf.clone();
508                sub_conf.create = false;
509                sub_conf.create_new = false;
510                sub_conf.append = false;
511                sub_conf.truncate = false;
512                match fs.new_open_options().options(sub_conf.clone()).open(path) {
513                    Err(e) if should_continue(e) => continue,
514                    Ok(file) if require_mutations => {
515                        // If the file was opened with the ability to mutate then we need
516                        // to return a copy on write emulation so that the file can be
517                        // copied from the secondary to the primary in the scenario that
518                        // it is edited
519                        return open_copy_on_write(path, conf, &self.primary, file);
520                    }
521                    other => return other,
522                }
523            }
524        }
525
526        // If we are creating the file then do so
527        if conf.create {
528            // Create the parent structure and remove any whiteouts
529            if let Some(parent) = path.parent() {
530                if ops::exists(self, parent) {
531                    ops::create_dir_all(&self.primary, parent)?;
532                }
533            }
534            ops::remove_white_out(&self.primary, path);
535
536            // Create the file in the primary
537            return self
538                .primary
539                .new_open_options()
540                .options(conf.clone())
541                .open(path);
542        }
543
544        // The file does not exist anywhere
545        Err(FsError::EntryNotFound)
546    }
547}
548
549fn open_copy_on_write<P>(
550    path: &Path,
551    conf: &OpenOptionsConfig,
552    primary: &Arc<P>,
553    file: Box<dyn VirtualFile + Send + Sync>,
554) -> Result<Box<dyn VirtualFile + Send + Sync>, FsError>
555where
556    P: FileSystem,
557{
558    struct CopyOnWriteFile<P> {
559        path: PathBuf,
560        primary: Arc<P>,
561        state: CowState,
562        readable: bool,
563        append: bool,
564        new_size: Option<u64>,
565    }
566    enum CowState {
567        // The original file is still open and can be accessed for all
568        // read operations
569        ReadOnly(Box<dyn VirtualFile + Send + Sync>),
570        // The copy has started but first we need to get the cursor
571        // position within the source file so that it can be restored
572        SeekingGet(Box<dyn VirtualFile + Send + Sync>),
573        // Now we have the original starting cursor location we need
574        // to move the position of the read to the start of the source
575        // file
576        SeekingSet {
577            original_offset: u64,
578            src: Box<dyn VirtualFile + Send + Sync>,
579        },
580        // We are now copying the data in parts held in the buffer piece
581        // by piece until the original file is completely copied
582        Copying {
583            original_offset: u64,
584            buf: Vec<u8>,
585            buf_pos: usize,
586            dst: Box<dyn VirtualFile + Send + Sync>,
587            src: Box<dyn VirtualFile + Send + Sync>,
588        },
589        // After copying the file we need to seek the position back
590        // to its original location on the newly copied file
591        SeekingRestore {
592            dst: Box<dyn VirtualFile + Send + Sync>,
593        },
594        // We have copied the file and can use all the normal operations
595        Copied(Box<dyn VirtualFile + Send + Sync>),
596        // An error occurred during the copy operation and we are now in a
597        // failed state, after the error is consumed it will reset back
598        // to the original file
599        Error {
600            err: io::Error,
601            src: Box<dyn VirtualFile + Send + Sync>,
602        },
603    }
604    impl CowState {
605        fn as_ref(&self) -> &(dyn VirtualFile + Send + Sync) {
606            match self {
607                Self::ReadOnly(inner) => inner.as_ref(),
608                Self::SeekingGet(inner) => inner.as_ref(),
609                Self::SeekingSet { src, .. } => src.as_ref(),
610                Self::Copying { src, .. } => src.as_ref(),
611                Self::SeekingRestore { dst, .. } => dst.as_ref(),
612                Self::Copied(inner) => inner.as_ref(),
613                Self::Error { src, .. } => src.as_ref(),
614            }
615        }
616        fn as_mut(&mut self) -> &mut (dyn VirtualFile + Send + Sync) {
617            match self {
618                Self::ReadOnly(inner) => inner.as_mut(),
619                Self::SeekingGet(inner) => inner.as_mut(),
620                Self::SeekingSet { src, .. } => src.as_mut(),
621                Self::Copying { src, .. } => src.as_mut(),
622                Self::SeekingRestore { dst, .. } => dst.as_mut(),
623                Self::Copied(inner) => inner.as_mut(),
624                Self::Error { src, .. } => src.as_mut(),
625            }
626        }
627    }
628
629    impl<P> CopyOnWriteFile<P>
630    where
631        P: FileSystem + 'static,
632    {
633        fn poll_copy_progress(&mut self, cx: &mut Context) -> Poll<io::Result<()>> {
634            // Enter a loop until we go pending
635            let mut again = true;
636            while again {
637                again = false;
638
639                // The state machine is updated during the poll operation
640                replace_with_or_abort(&mut self.state, |state| match state {
641                    // We record the current position of the file so that it can be
642                    // restored after the copy-on-write is finished
643                    CowState::SeekingGet(mut src) => {
644                        match Pin::new(src.as_mut()).poll_complete(cx) {
645                            Poll::Ready(Ok(offset)) => {
646                                if let Err(err) =
647                                    Pin::new(src.as_mut()).start_seek(SeekFrom::Start(0))
648                                {
649                                    return CowState::Error { err, src };
650                                }
651                                again = true;
652                                CowState::SeekingSet {
653                                    original_offset: offset,
654                                    src,
655                                }
656                            }
657                            Poll::Ready(Err(err)) => CowState::Error { err, src },
658                            Poll::Pending => CowState::SeekingGet(src),
659                        }
660                    }
661
662                    // We complete the seek operation to the start of the source file
663                    CowState::SeekingSet {
664                        original_offset,
665                        mut src,
666                    } => {
667                        match Pin::new(src.as_mut()).poll_complete(cx).map_ok(|_| ()) {
668                            Poll::Ready(Ok(())) => {
669                                // Remove the whiteout, create the parent structure and open
670                                // the new file on the primary
671                                if let Some(parent) = self.path.parent() {
672                                    ops::create_dir_all(&self.primary, parent).ok();
673                                }
674                                let mut had_white_out = false;
675                                if ops::has_white_out(&self.primary, &self.path) {
676                                    ops::remove_white_out(&self.primary, &self.path);
677                                    had_white_out = true;
678                                }
679                                let dst = self
680                                    .primary
681                                    .new_open_options()
682                                    .create(true)
683                                    .read(self.readable)
684                                    .write(true)
685                                    .truncate(true)
686                                    .open(&self.path);
687                                match dst {
688                                    Ok(dst) if had_white_out => {
689                                        again = true;
690                                        CowState::Copied(dst)
691                                    }
692                                    Ok(dst) => {
693                                        again = true;
694                                        CowState::Copying {
695                                            original_offset,
696                                            buf: Vec::new(),
697                                            buf_pos: 0,
698                                            src,
699                                            dst,
700                                        }
701                                    }
702                                    Err(err) => CowState::Error {
703                                        err: err.into(),
704                                        src,
705                                    },
706                                }
707                            }
708                            Poll::Ready(Err(err)) => CowState::Error { err, src },
709                            Poll::Pending => CowState::SeekingSet {
710                                original_offset,
711                                src,
712                            },
713                        }
714                    }
715                    // We are now copying all the data on blocks
716                    CowState::Copying {
717                        mut src,
718                        mut dst,
719                        mut buf,
720                        mut buf_pos,
721                        original_offset,
722                    } => {
723                        loop {
724                            // We are either copying more data from the source
725                            // or we are copying the data to the destination
726                            if buf_pos < buf.len() {
727                                let dst_pinned = Pin::new(dst.as_mut());
728                                match dst_pinned.poll_write(cx, &buf[buf_pos..]) {
729                                    Poll::Ready(Ok(0)) => {}
730                                    Poll::Ready(Ok(amt)) => {
731                                        buf_pos += amt;
732                                        continue;
733                                    }
734                                    Poll::Ready(Err(err)) => {
735                                        return CowState::Error { err, src };
736                                    }
737                                    Poll::Pending => {}
738                                }
739                            } else {
740                                buf.resize_with(8192, || 0);
741                                buf_pos = 8192;
742                                let mut read_buf = ReadBuf::new(&mut buf);
743                                match Pin::new(src.as_mut()).poll_read(cx, &mut read_buf) {
744                                    Poll::Ready(Ok(())) if read_buf.filled().is_empty() => {
745                                        again = true;
746
747                                        if self.append {
748                                            // When we append then we leave the cursor at the
749                                            // end of the file
750                                            return CowState::Copied(dst);
751                                        } else {
752                                            // No more data exists to be read so we now move on to
753                                            // restoring the cursor back to the original position
754                                            if let Err(err) = Pin::new(dst.as_mut())
755                                                .start_seek(SeekFrom::Start(original_offset))
756                                            {
757                                                return CowState::Error { err, src };
758                                            }
759                                            return CowState::SeekingRestore { dst };
760                                        }
761                                    }
762                                    Poll::Ready(Ok(())) => {
763                                        // There is more data to be processed
764                                        let new_len = read_buf.filled().len();
765                                        unsafe { buf.set_len(new_len) };
766                                        buf_pos = 0;
767                                        continue;
768                                    }
769                                    Poll::Ready(Err(err)) => return CowState::Error { err, src },
770                                    Poll::Pending => {}
771                                }
772                            }
773                            return CowState::Copying {
774                                original_offset,
775                                buf,
776                                buf_pos,
777                                src,
778                                dst,
779                            };
780                        }
781                    }
782                    // Now once the restoration of the seek position completes we set the copied state
783                    CowState::SeekingRestore { mut dst } => {
784                        match Pin::new(dst.as_mut()).poll_complete(cx) {
785                            Poll::Ready(_) => {
786                                // If we have changed the length then set it
787                                if let Some(new_size) = self.new_size.take() {
788                                    dst.set_len(new_size).ok();
789                                }
790                                CowState::Copied(dst)
791                            }
792                            Poll::Pending => CowState::SeekingRestore { dst },
793                        }
794                    }
795                    s => s,
796                });
797            }
798
799            // Determine what response to give based off the state, when an error occurs
800            // this will be returned and the copy-on-write will be reset
801            let mut ret = Poll::Pending;
802            replace_with_or_abort(&mut self.state, |state| match state {
803                CowState::ReadOnly(src) => {
804                    ret = Poll::Ready(Ok(()));
805                    CowState::ReadOnly(src)
806                }
807                CowState::Copied(src) => {
808                    ret = Poll::Ready(Ok(()));
809                    CowState::Copied(src)
810                }
811                CowState::Error { err, src } => {
812                    ret = Poll::Ready(Err(err));
813                    CowState::ReadOnly(src)
814                }
815                state => {
816                    ret = Poll::Pending;
817                    state
818                }
819            });
820            ret
821        }
822
823        fn poll_copy_start_and_progress(&mut self, cx: &mut Context) -> Poll<io::Result<()>> {
824            replace_with_or_abort(&mut self.state, |state| match state {
825                CowState::ReadOnly(inner) => {
826                    tracing::trace!("COW file touched, starting file clone");
827                    CowState::SeekingGet(inner)
828                }
829                state => state,
830            });
831            self.poll_copy_progress(cx)
832        }
833    }
834
835    impl<P> Debug for CopyOnWriteFile<P>
836    where
837        P: FileSystem + 'static,
838    {
839        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
840            f.debug_struct("CopyOnWriteFile").finish()
841        }
842    }
843
844    impl<P> VirtualFile for CopyOnWriteFile<P>
845    where
846        P: FileSystem + 'static,
847    {
848        fn last_accessed(&self) -> u64 {
849            self.state.as_ref().last_accessed()
850        }
851
852        fn last_modified(&self) -> u64 {
853            self.state.as_ref().last_modified()
854        }
855
856        fn created_time(&self) -> u64 {
857            self.state.as_ref().created_time()
858        }
859
860        fn size(&self) -> u64 {
861            self.state.as_ref().size()
862        }
863
864        fn set_len(&mut self, new_size: u64) -> crate::Result<()> {
865            self.new_size = Some(new_size);
866            replace_with_or_abort(&mut self.state, |state| match state {
867                CowState::Copied(mut file) => {
868                    file.set_len(new_size).ok();
869                    CowState::Copied(file)
870                }
871                state => {
872                    // in the scenario where the length is set but the file is not
873                    // polled then we need to make sure we create a file properly
874                    if let Some(parent) = self.path.parent() {
875                        ops::create_dir_all(&self.primary, parent).ok();
876                    }
877                    let dst = self
878                        .primary
879                        .new_open_options()
880                        .create(true)
881                        .write(true)
882                        .open(&self.path);
883                    if let Ok(mut file) = dst {
884                        file.set_len(new_size).ok();
885                    }
886                    state
887                }
888            });
889            Ok(())
890        }
891
892        fn unlink(&mut self) -> crate::Result<()> {
893            let primary = self.primary.clone();
894            let path = self.path.clone();
895
896            // Create the whiteout file in the primary
897            let mut had_at_least_one_success = false;
898            if ops::create_white_out(&primary, &path).is_ok() {
899                had_at_least_one_success = true;
900            }
901
902            // Attempt to remove it from the primary first
903            match primary.remove_file(&path) {
904                Err(e) if should_continue(e) => {}
905                other => return other,
906            }
907
908            if had_at_least_one_success {
909                return Ok(());
910            }
911            Err(FsError::PermissionDenied)
912        }
913
914        fn poll_read_ready(
915            mut self: Pin<&mut Self>,
916            cx: &mut std::task::Context<'_>,
917        ) -> Poll<std::io::Result<usize>> {
918            match self.poll_copy_progress(cx) {
919                Poll::Ready(Ok(())) => {}
920                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
921                Poll::Pending => return Poll::Pending,
922            }
923            Pin::new(self.state.as_mut()).poll_read_ready(cx)
924        }
925
926        fn poll_write_ready(
927            mut self: Pin<&mut Self>,
928            cx: &mut std::task::Context<'_>,
929        ) -> Poll<std::io::Result<usize>> {
930            match self.poll_copy_progress(cx) {
931                Poll::Ready(Ok(())) => {}
932                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
933                Poll::Pending => return Poll::Pending,
934            }
935            Pin::new(self.state.as_mut()).poll_write_ready(cx)
936        }
937    }
938
939    impl<P> AsyncWrite for CopyOnWriteFile<P>
940    where
941        P: FileSystem + 'static,
942    {
943        fn poll_write(
944            mut self: Pin<&mut Self>,
945            cx: &mut std::task::Context<'_>,
946            buf: &[u8],
947        ) -> Poll<Result<usize, std::io::Error>> {
948            match self.poll_copy_start_and_progress(cx) {
949                Poll::Pending => return Poll::Pending,
950                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
951                Poll::Ready(Ok(())) => {}
952            }
953            Pin::new(self.state.as_mut()).poll_write(cx, buf)
954        }
955
956        fn poll_write_vectored(
957            mut self: Pin<&mut Self>,
958            cx: &mut Context<'_>,
959            bufs: &[io::IoSlice<'_>],
960        ) -> Poll<Result<usize, io::Error>> {
961            match self.poll_copy_start_and_progress(cx) {
962                Poll::Pending => return Poll::Pending,
963                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
964                Poll::Ready(Ok(())) => {}
965            }
966            Pin::new(self.state.as_mut()).poll_write_vectored(cx, bufs)
967        }
968
969        fn poll_flush(
970            mut self: Pin<&mut Self>,
971            cx: &mut std::task::Context<'_>,
972        ) -> Poll<Result<(), std::io::Error>> {
973            match self.poll_copy_progress(cx) {
974                Poll::Ready(Ok(())) => {}
975                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
976                Poll::Pending => return Poll::Pending,
977            }
978            // The file may actually be read-only and not support flush operations
979            // at all, and there's nothing to flush in read-only state anyway.
980            match self.state {
981                CowState::ReadOnly(_) => Poll::Ready(Ok(())),
982                _ => Pin::new(self.state.as_mut()).poll_flush(cx),
983            }
984        }
985
986        fn poll_shutdown(
987            mut self: Pin<&mut Self>,
988            cx: &mut std::task::Context<'_>,
989        ) -> Poll<Result<(), std::io::Error>> {
990            match self.poll_copy_progress(cx) {
991                Poll::Ready(Ok(())) => {}
992                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
993                Poll::Pending => return Poll::Pending,
994            }
995            // Same deal as flush above
996            match self.state {
997                CowState::ReadOnly(_) => Poll::Ready(Ok(())),
998                _ => Pin::new(self.state.as_mut()).poll_shutdown(cx),
999            }
1000        }
1001    }
1002
1003    impl<P> AsyncRead for CopyOnWriteFile<P>
1004    where
1005        P: FileSystem + 'static,
1006    {
1007        fn poll_read(
1008            mut self: Pin<&mut Self>,
1009            cx: &mut std::task::Context<'_>,
1010            buf: &mut tokio::io::ReadBuf<'_>,
1011        ) -> Poll<std::io::Result<()>> {
1012            match self.poll_copy_progress(cx) {
1013                Poll::Ready(Ok(())) => {}
1014                p => return p,
1015            }
1016            Pin::new(self.state.as_mut()).poll_read(cx, buf)
1017        }
1018    }
1019
1020    impl<P> AsyncSeek for CopyOnWriteFile<P>
1021    where
1022        P: FileSystem + 'static,
1023    {
1024        fn start_seek(
1025            mut self: Pin<&mut Self>,
1026            position: std::io::SeekFrom,
1027        ) -> std::io::Result<()> {
1028            match &mut self.state {
1029                CowState::ReadOnly(file)
1030                | CowState::SeekingGet(file)
1031                | CowState::Error { src: file, .. }
1032                | CowState::Copied(file)
1033                | CowState::SeekingRestore { dst: file, .. } => {
1034                    Pin::new(file.as_mut()).start_seek(position)
1035                }
1036                CowState::SeekingSet {
1037                    original_offset,
1038                    src,
1039                    ..
1040                }
1041                | CowState::Copying {
1042                    original_offset,
1043                    src,
1044                    ..
1045                } => {
1046                    *original_offset = match position {
1047                        SeekFrom::Current(delta) => original_offset
1048                            .checked_add_signed(delta)
1049                            .unwrap_or(*original_offset),
1050                        SeekFrom::Start(pos) => pos,
1051                        SeekFrom::End(pos) => src
1052                            .size()
1053                            .checked_add_signed(pos)
1054                            .unwrap_or(*original_offset),
1055                    };
1056                    Ok(())
1057                }
1058            }
1059        }
1060
1061        fn poll_complete(
1062            mut self: Pin<&mut Self>,
1063            cx: &mut std::task::Context<'_>,
1064        ) -> Poll<std::io::Result<u64>> {
1065            match self.poll_copy_progress(cx) {
1066                Poll::Ready(Ok(())) => {}
1067                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
1068                Poll::Pending => return Poll::Pending,
1069            }
1070            Pin::new(self.state.as_mut()).poll_complete(cx)
1071        }
1072    }
1073
1074    tracing::trace!(
1075        path=%path.display(),
1076        options=?conf,
1077        "Opening the file in copy-on-write mode",
1078    );
1079    Ok(Box::new(CopyOnWriteFile::<P> {
1080        path: path.to_path_buf(),
1081        primary: primary.clone(),
1082        state: CowState::ReadOnly(file),
1083        readable: conf.read,
1084        append: conf.append,
1085        new_size: None,
1086    }))
1087}
1088
1089impl<P, S> Debug for OverlayFileSystem<P, S>
1090where
1091    P: FileSystem,
1092    S: for<'a> FileSystems<'a>,
1093{
1094    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1095        struct IterFilesystems<'a, S>(&'a S);
1096        impl<S> Debug for IterFilesystems<'_, S>
1097        where
1098            S: for<'b> FileSystems<'b>,
1099        {
1100            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1101                let mut f = f.debug_list();
1102
1103                for fs in self.0.filesystems() {
1104                    f.entry(&fs);
1105                }
1106
1107                f.finish()
1108            }
1109        }
1110
1111        f.debug_struct("OverlayFileSystem")
1112            .field("primary", &self.primary)
1113            .field("secondaries", &IterFilesystems(&self.secondaries))
1114            .finish()
1115    }
1116}
1117
1118fn should_continue(e: FsError) -> bool {
1119    // HACK: We shouldn't really be ignoring FsError::BaseNotDirectory, but
1120    // it's needed because the mem_fs::FileSystem doesn't return
1121    // FsError::EntryNotFound when an intermediate directory doesn't exist
1122    // (i.e. the "/path/to" in "/path/to/file.txt").
1123    matches!(
1124        e,
1125        FsError::EntryNotFound | FsError::InvalidInput | FsError::BaseNotDirectory
1126    )
1127}
1128
1129#[cfg(test)]
1130mod tests {
1131    use std::path::PathBuf;
1132
1133    use super::*;
1134    use crate::mem_fs::FileSystem as MemFS;
1135    use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
1136
1137    #[tokio::test]
1138    async fn remove_directory() {
1139        let primary = MemFS::default();
1140        let secondary = MemFS::default();
1141        let first = Path::new("/first");
1142        let second = Path::new("/second");
1143        let file_txt = second.join("file.txt");
1144        let third = Path::new("/third");
1145        primary.create_dir(first).unwrap();
1146        primary.create_dir(second).unwrap();
1147        primary
1148            .new_open_options()
1149            .create(true)
1150            .write(true)
1151            .open(&file_txt)
1152            .unwrap()
1153            .write_all(b"Hello, World!")
1154            .await
1155            .unwrap();
1156        secondary.create_dir(third).unwrap();
1157
1158        let overlay = OverlayFileSystem::new(primary, [secondary]);
1159
1160        // Delete a folder on the primary filesystem
1161        overlay.remove_dir(first).unwrap();
1162        assert_eq!(
1163            overlay.primary().metadata(first).unwrap_err(),
1164            FsError::EntryNotFound,
1165            "Deleted from primary"
1166        );
1167        assert!(!ops::exists(&overlay.secondaries[0], second));
1168
1169        // Directory on the primary fs isn't empty
1170        assert_eq!(
1171            overlay.remove_dir(second).unwrap_err(),
1172            FsError::DirectoryNotEmpty,
1173        );
1174
1175        // Try to remove something on one of the overlay filesystems
1176        assert_eq!(overlay.remove_dir(third), Ok(()));
1177
1178        // It should no longer exist
1179        assert_eq!(overlay.metadata(third).unwrap_err(), FsError::EntryNotFound);
1180
1181        assert!(ops::exists(&overlay.secondaries[0], third));
1182    }
1183
1184    #[tokio::test]
1185    async fn open_files() {
1186        let primary = MemFS::default();
1187        let secondary = MemFS::default();
1188        ops::create_dir_all(&primary, "/primary").unwrap();
1189        ops::touch(&primary, "/primary/read.txt").unwrap();
1190        ops::touch(&primary, "/primary/write.txt").unwrap();
1191        ops::create_dir_all(&secondary, "/secondary").unwrap();
1192        ops::touch(&secondary, "/secondary/read.txt").unwrap();
1193        ops::touch(&secondary, "/secondary/write.txt").unwrap();
1194        ops::create_dir_all(&secondary, "/primary").unwrap();
1195        ops::write(&secondary, "/primary/read.txt", "This is shadowed")
1196            .await
1197            .unwrap();
1198
1199        let fs = OverlayFileSystem::new(primary, [secondary]);
1200
1201        // Any new files will be created on the primary fs
1202        let _ = fs
1203            .new_open_options()
1204            .create(true)
1205            .write(true)
1206            .open("/new.txt")
1207            .unwrap();
1208        assert!(ops::exists(&fs.primary, "/new.txt"));
1209        assert!(!ops::exists(&fs.secondaries[0], "/new.txt"));
1210
1211        // You can open a file for reading and writing on the primary fs
1212        let _ = fs
1213            .new_open_options()
1214            .create(false)
1215            .write(true)
1216            .read(true)
1217            .open("/primary/write.txt")
1218            .unwrap();
1219
1220        // Files on the primary should always shadow the secondary
1221        let content = ops::read_to_string(&fs, "/primary/read.txt").await.unwrap();
1222        assert_ne!(content, "This is shadowed");
1223    }
1224
1225    #[tokio::test]
1226    async fn create_file_that_looks_like_it_is_in_a_secondary_filesystem_folder() {
1227        let primary = MemFS::default();
1228        let secondary = MemFS::default();
1229        ops::create_dir_all(&secondary, "/path/to/").unwrap();
1230        assert!(!ops::is_dir(&primary, "/path/to/"));
1231        let fs = OverlayFileSystem::new(primary, [secondary]);
1232
1233        ops::touch(&fs, "/path/to/file.txt").unwrap();
1234
1235        assert!(ops::is_dir(&fs.primary, "/path/to/"));
1236        assert!(ops::is_file(&fs.primary, "/path/to/file.txt"));
1237        assert!(!ops::is_file(&fs.secondaries[0], "/path/to/file.txt"));
1238    }
1239
1240    #[tokio::test]
1241    async fn listed_files_appear_overlayed() {
1242        let primary = MemFS::default();
1243        let secondary = MemFS::default();
1244        let secondary_overlayed = MemFS::default();
1245        ops::create_dir_all(&primary, "/primary").unwrap();
1246        ops::touch(&primary, "/primary/read.txt").unwrap();
1247        ops::touch(&primary, "/primary/write.txt").unwrap();
1248        ops::create_dir_all(&secondary, "/secondary").unwrap();
1249        ops::touch(&secondary, "/secondary/read.txt").unwrap();
1250        ops::touch(&secondary, "/secondary/write.txt").unwrap();
1251        // This second "secondary" filesystem should share the same folders as
1252        // the first one.
1253        ops::create_dir_all(&secondary_overlayed, "/secondary").unwrap();
1254        ops::touch(&secondary_overlayed, "/secondary/overlayed.txt").unwrap();
1255
1256        let fs = OverlayFileSystem::new(primary, [secondary, secondary_overlayed]);
1257
1258        let paths: Vec<_> = ops::walk(&fs, "/").map(|entry| entry.path()).collect();
1259        assert_eq!(
1260            paths,
1261            vec![
1262                PathBuf::from("/secondary"),
1263                PathBuf::from("/secondary/write.txt"),
1264                PathBuf::from("/secondary/read.txt"),
1265                PathBuf::from("/secondary/overlayed.txt"),
1266                PathBuf::from("/primary"),
1267                PathBuf::from("/primary/write.txt"),
1268                PathBuf::from("/primary/read.txt"),
1269            ]
1270        );
1271    }
1272
1273    #[tokio::test]
1274    async fn open_secondary_fs_files_in_write_mode() {
1275        let primary = MemFS::default();
1276        let secondary = MemFS::default();
1277        ops::create_dir_all(&secondary, "/secondary").unwrap();
1278        ops::write(&secondary, "/secondary/file.txt", b"Hello, World!")
1279            .await
1280            .unwrap();
1281
1282        let fs = OverlayFileSystem::new(primary, [secondary]);
1283
1284        let mut f = fs
1285            .new_open_options()
1286            .write(true)
1287            .read(true)
1288            .open("/secondary/file.txt")
1289            .unwrap();
1290        // reading is fine
1291        let mut buf = String::new();
1292        f.read_to_string(&mut buf).await.unwrap();
1293        assert_eq!(buf, "Hello, World!");
1294        f.seek(SeekFrom::Start(0)).await.unwrap();
1295        // next we will write a new set of bytes
1296        f.set_len(0).unwrap();
1297        assert_eq!(f.write(b"Hi").await.unwrap(), 2);
1298        // Same with flushing
1299        assert_eq!(f.flush().await.unwrap(), ());
1300
1301        // if we now read it then the data should be different
1302        buf = String::new();
1303        f.seek(SeekFrom::Start(0)).await.unwrap();
1304        f.read_to_string(&mut buf).await.unwrap();
1305        assert_eq!(buf, "Hi");
1306        drop(f);
1307
1308        // including if we open it again
1309        let mut f = fs
1310            .new_open_options()
1311            .read(true)
1312            .open("/secondary/file.txt")
1313            .unwrap();
1314        buf = String::new();
1315        f.read_to_string(&mut buf).await.unwrap();
1316        assert_eq!(buf, "Hi");
1317    }
1318
1319    #[tokio::test]
1320    async fn open_secondary_fs_files_unlink() {
1321        let primary = MemFS::default();
1322        let secondary = MemFS::default();
1323        ops::create_dir_all(&secondary, "/secondary").unwrap();
1324        ops::write(&secondary, "/secondary/file.txt", b"Hello, World!")
1325            .await
1326            .unwrap();
1327
1328        let fs = OverlayFileSystem::new(primary, [secondary]);
1329
1330        fs.metadata(Path::new("/secondary/file.txt")).unwrap();
1331
1332        // Now delete the file and make sure its not found
1333        fs.remove_file(Path::new("/secondary/file.txt")).unwrap();
1334        assert_eq!(
1335            fs.metadata(Path::new("/secondary/file.txt")).unwrap_err(),
1336            FsError::EntryNotFound
1337        )
1338    }
1339
1340    #[tokio::test]
1341    async fn open_secondary_fs_without_cow() {
1342        let primary = MemFS::default();
1343        let secondary = MemFS::default();
1344        ops::create_dir_all(&secondary, "/secondary").unwrap();
1345        ops::write(&secondary, "/secondary/file.txt", b"Hello, World!")
1346            .await
1347            .unwrap();
1348
1349        let fs = OverlayFileSystem::new(primary, [secondary]);
1350
1351        let mut f = fs
1352            .new_open_options()
1353            .create(true)
1354            .read(true)
1355            .open(Path::new("/secondary/file.txt"))
1356            .unwrap();
1357        assert_eq!(f.size() as usize, 13);
1358
1359        let mut buf = String::new();
1360        f.read_to_string(&mut buf).await.unwrap();
1361        assert_eq!(buf, "Hello, World!");
1362
1363        // it should not be in the primary and nor should the secondary folder
1364        assert!(!ops::is_dir(&fs.primary, "/secondary"));
1365        assert!(!ops::is_file(&fs.primary, "/secondary/file.txt"));
1366        assert!(ops::is_dir(&fs.secondaries[0], "/secondary"));
1367        assert!(ops::is_file(&fs.secondaries[0], "/secondary/file.txt"));
1368    }
1369
1370    #[tokio::test]
1371    async fn create_and_append_secondary_fs_with_cow() {
1372        let primary = MemFS::default();
1373        let secondary = MemFS::default();
1374        ops::create_dir_all(&secondary, "/secondary").unwrap();
1375        ops::write(&secondary, "/secondary/file.txt", b"Hello, World!")
1376            .await
1377            .unwrap();
1378
1379        let fs = OverlayFileSystem::new(primary, [secondary]);
1380
1381        let mut f = fs
1382            .new_open_options()
1383            .create(true)
1384            .append(true)
1385            .read(true)
1386            .open(Path::new("/secondary/file.txt"))
1387            .unwrap();
1388        assert_eq!(f.size() as usize, 13);
1389
1390        f.write_all(b"asdf").await.unwrap();
1391        assert_eq!(f.size() as usize, 17);
1392
1393        f.seek(SeekFrom::Start(0)).await.unwrap();
1394
1395        let mut buf = String::new();
1396        f.read_to_string(&mut buf).await.unwrap();
1397        assert_eq!(buf, "Hello, World!asdf");
1398
1399        // Now lets check the file systems under
1400        let f = fs
1401            .primary
1402            .new_open_options()
1403            .create(true)
1404            .append(true)
1405            .read(true)
1406            .open(Path::new("/secondary/file.txt"))
1407            .unwrap();
1408        assert_eq!(f.size() as usize, 17);
1409        let f = fs.secondaries[0]
1410            .new_open_options()
1411            .create(true)
1412            .append(true)
1413            .read(true)
1414            .open(Path::new("/secondary/file.txt"))
1415            .unwrap();
1416        assert_eq!(f.size() as usize, 13);
1417
1418        // it should now exist in both the primary and secondary
1419        assert!(ops::is_dir(&fs.primary, "/secondary"));
1420        assert!(ops::is_file(&fs.primary, "/secondary/file.txt"));
1421        assert!(ops::is_dir(&fs.secondaries[0], "/secondary"));
1422        assert!(ops::is_file(&fs.secondaries[0], "/secondary/file.txt"));
1423    }
1424
1425    #[tokio::test]
1426    async fn unlink_file_from_secondary_fs() {
1427        let primary = MemFS::default();
1428        let secondary = MemFS::default();
1429        ops::create_dir_all(&secondary, "/secondary").unwrap();
1430        ops::write(&secondary, "/secondary/file.txt", b"Hello, World!")
1431            .await
1432            .unwrap();
1433
1434        let fs = OverlayFileSystem::new(primary, [secondary]);
1435
1436        fs.remove_file(Path::new("/secondary/file.txt")).unwrap();
1437        assert_eq!(ops::exists(&fs, Path::new("/secondary/file.txt")), false);
1438
1439        assert!(ops::is_file(&fs.primary, "/secondary/.wh.file.txt"));
1440        assert!(ops::is_file(&fs.secondaries[0], "/secondary/file.txt"));
1441
1442        // Now create the file again after the unlink
1443        let mut f = fs
1444            .new_open_options()
1445            .create(true)
1446            .write(true)
1447            .read(true)
1448            .open(Path::new("/secondary/file.txt"))
1449            .unwrap();
1450        assert_eq!(f.size() as usize, 0);
1451        f.write_all(b"asdf").await.unwrap();
1452        assert_eq!(f.size() as usize, 4);
1453
1454        // The whiteout should be gone and new file exist
1455        assert!(!ops::is_file(&fs.primary, "/secondary/.wh.file.txt"));
1456        assert!(ops::is_file(&fs.primary, "/secondary/file.txt"));
1457        assert!(ops::is_file(&fs.secondaries[0], "/secondary/file.txt"));
1458    }
1459
1460    #[tokio::test]
1461    async fn rmdir_from_secondary_fs() {
1462        let primary = MemFS::default();
1463        let secondary = MemFS::default();
1464        ops::create_dir_all(&secondary, "/secondary").unwrap();
1465
1466        let fs = OverlayFileSystem::new(primary, [secondary]);
1467
1468        assert!(ops::is_dir(&fs, "/secondary"));
1469        fs.remove_dir(Path::new("/secondary")).unwrap();
1470
1471        assert!(!ops::is_dir(&fs, "/secondary"));
1472        assert!(ops::is_file(&fs.primary, "/.wh.secondary"));
1473        assert!(ops::is_dir(&fs.secondaries[0], "/secondary"));
1474
1475        fs.create_dir(Path::new("/secondary")).unwrap();
1476        assert!(ops::is_dir(&fs, "/secondary"));
1477        assert!(ops::is_dir(&fs.primary, "/secondary"));
1478        assert!(!ops::is_file(&fs.primary, "/.wh.secondary"));
1479        assert!(ops::is_dir(&fs.secondaries[0], "/secondary"));
1480    }
1481
1482    #[tokio::test]
1483    async fn rmdir_sub_from_secondary_fs() {
1484        let primary = MemFS::default();
1485        let secondary = MemFS::default();
1486        ops::create_dir_all(&secondary, "/first/secondary").unwrap();
1487
1488        let fs = OverlayFileSystem::new(primary, [secondary]);
1489
1490        assert!(ops::is_dir(&fs, "/first/secondary"));
1491        fs.remove_dir(Path::new("/first/secondary")).unwrap();
1492
1493        assert!(!ops::is_dir(&fs, "/first/secondary"));
1494        assert!(ops::is_file(&fs.primary, "/first/.wh.secondary"));
1495        assert!(ops::is_dir(&fs.secondaries[0], "/first/secondary"));
1496
1497        fs.create_dir(Path::new("/first/secondary")).unwrap();
1498        assert!(ops::is_dir(&fs, "/first/secondary"));
1499        assert!(ops::is_dir(&fs.primary, "/first/secondary"));
1500        assert!(!ops::is_file(&fs.primary, "/first/.wh.secondary"));
1501        assert!(ops::is_dir(&fs.secondaries[0], "/first/secondary"));
1502    }
1503
1504    #[tokio::test]
1505    async fn create_new_secondary_fs_without_cow() {
1506        let primary = MemFS::default();
1507        let secondary = MemFS::default();
1508        ops::create_dir_all(&secondary, "/secondary").unwrap();
1509        ops::write(&secondary, "/secondary/file.txt", b"Hello, World!")
1510            .await
1511            .unwrap();
1512
1513        let fs = OverlayFileSystem::new(primary, [secondary]);
1514
1515        let mut f = fs
1516            .new_open_options()
1517            .create_new(true)
1518            .read(true)
1519            .open(Path::new("/secondary/file.txt"))
1520            .unwrap();
1521        assert_eq!(f.size() as usize, 0);
1522
1523        let mut buf = String::new();
1524        f.read_to_string(&mut buf).await.unwrap();
1525        assert_eq!(buf, "");
1526
1527        // it should now exist in both the primary and secondary
1528        assert!(ops::is_dir(&fs.primary, "/secondary"));
1529        assert!(ops::is_file(&fs.primary, "/secondary/file.txt"));
1530        assert!(ops::is_dir(&fs.secondaries[0], "/secondary"));
1531        assert!(ops::is_file(&fs.secondaries[0], "/secondary/file.txt"));
1532    }
1533
1534    #[tokio::test]
1535    async fn open_secondary_fs_files_remove_dir() {
1536        let primary = MemFS::default();
1537        let secondary = MemFS::default();
1538        ops::create_dir_all(&secondary, "/secondary").unwrap();
1539
1540        let fs = OverlayFileSystem::new(primary, [secondary]);
1541
1542        fs.metadata(Path::new("/secondary")).unwrap();
1543
1544        // Now delete the file and make sure its not found
1545        fs.remove_dir(Path::new("/secondary")).unwrap();
1546        assert_eq!(
1547            fs.metadata(Path::new("/secondary")).unwrap_err(),
1548            FsError::EntryNotFound
1549        )
1550    }
1551
1552    /// Make sure files that are never written are not copied to the primary,
1553    /// even when opened with write permissions.
1554    /// Regression test for https://github.com/wasmerio/wasmer/issues/5445
1555    #[tokio::test]
1556    async fn test_overlayfs_readonly_files_not_copied() {
1557        let primary = MemFS::default();
1558        let secondary = MemFS::default();
1559        ops::create_dir_all(&secondary, "/secondary").unwrap();
1560        ops::write(&secondary, "/secondary/file.txt", b"Hello, World!")
1561            .await
1562            .unwrap();
1563
1564        let fs = OverlayFileSystem::new(primary, [secondary]);
1565
1566        {
1567            let mut f = fs
1568                .new_open_options()
1569                .read(true)
1570                .write(true)
1571                .open(Path::new("/secondary/file.txt"))
1572                .unwrap();
1573            let mut s = String::new();
1574            f.read_to_string(&mut s).await.unwrap();
1575            assert_eq!(s, "Hello, World!");
1576
1577            f.flush().await.unwrap();
1578            f.shutdown().await.unwrap();
1579        }
1580
1581        // Primary should not have the file
1582        assert!(!ops::is_file(&fs.primary, "/secondary/file.txt"));
1583    }
1584
1585    // OLD tests that used WebcFileSystem.
1586    // Should be re-implemented with WebcVolumeFs
1587    // #[tokio::test]
1588    // async fn wasi_runner_use_case() {
1589    //     // Set up some dummy files on the host
1590    //     let temp = TempDir::new().unwrap();
1591    //     let first = temp.path().join("first");
1592    //     let file_txt = first.join("file.txt");
1593    //     let second = temp.path().join("second");
1594    //     std::fs::create_dir_all(&first).unwrap();
1595    //     std::fs::write(&file_txt, b"First!").unwrap();
1596    //     std::fs::create_dir_all(&second).unwrap();
1597    //     // configure the union FS so things are saved in memory by default
1598    //     // (initialized with a set of unix-like folders), but certain folders
1599    //     // are first to the host.
1600    //     let primary = RootFileSystemBuilder::new().build();
1601    //     let host_fs: Arc<dyn FileSystem + Send + Sync> =
1602    //         Arc::new(crate::host_fs::FileSystem::default());
1603    //     let first_dirs = [(&first, "/first"), (&second, "/second")];
1604    //     for (host, guest) in first_dirs {
1605    //         primary
1606    //             .mount(PathBuf::from(guest), &host_fs, host.clone())
1607    //             .unwrap();
1608    //     }
1609    //     // Set up the secondary file systems
1610    //     let webc = WebCOwned::parse(Bytes::from_static(PYTHON), &ParseOptions::default()).unwrap();
1611    //     let webc = WebcFileSystem::init_all(Arc::new(webc));
1612    //
1613    //     let fs = OverlayFileSystem::new(primary, [webc]);
1614    //
1615    //     // We should get all the normal directories from rootfs (primary)
1616    //     assert!(ops::is_dir(&fs, "/lib"));
1617    //     assert!(ops::is_dir(&fs, "/bin"));
1618    //     assert!(ops::is_file(&fs, "/dev/stdin"));
1619    //     assert!(ops::is_file(&fs, "/dev/stdout"));
1620    //     // We also want to see files from the WEBC volumes (secondary)
1621    //     assert!(ops::is_dir(&fs, "/lib/python3.6"));
1622    //     assert!(ops::is_file(&fs, "/lib/python3.6/collections/__init__.py"));
1623    //     #[cfg(never)]
1624    //     {
1625    //         // files on a secondary fs aren't writable
1626    //         // TODO(Michael-F-Bryan): re-enable this if/when we fix
1627    //         // open_readonly_file_hack()
1628    //         assert_eq!(
1629    //             fs.new_open_options()
1630    //                 .append(true)
1631    //                 .open("/lib/python3.6/collections/__init__.py")
1632    //                 .unwrap_err(),
1633    //             FsError::PermissionDenied,
1634    //         );
1635    //     }
1636    //     // you are allowed to create files that look like they are in a secondary
1637    //     // folder, though
1638    //     ops::touch(&fs, "/lib/python3.6/collections/something-else.py").unwrap();
1639    //     // But it'll be on the primary filesystem, not the secondary one
1640    //     assert!(ops::is_file(
1641    //         &fs.primary,
1642    //         "/lib/python3.6/collections/something-else.py"
1643    //     ));
1644    //     assert!(!ops::is_file(
1645    //         &fs.secondaries[0],
1646    //         "/lib/python3.6/collections/something-else.py"
1647    //     ));
1648    //     // You can do the same thing with folders
1649    //     fs.create_dir("/lib/python3.6/something-else".as_ref())
1650    //         .unwrap();
1651    //     assert!(ops::is_dir(&fs.primary, "/lib/python3.6/something-else"));
1652    //     assert!(!ops::is_dir(
1653    //         &fs.secondaries[0],
1654    //         "/lib/python3.6/something-else"
1655    //     ));
1656    //     // It only works when you are directly inside an existing directory
1657    //     // on the secondary filesystem, though
1658    //     assert_eq!(
1659    //         ops::touch(&fs, "/lib/python3.6/collections/this/doesnt/exist.txt").unwrap_err(),
1660    //         FsError::EntryNotFound
1661    //     );
1662    //     // you should also be able to read files mounted from the host
1663    //     assert!(ops::is_dir(&fs, "/first"));
1664    //     assert!(ops::is_file(&fs, "/first/file.txt"));
1665    //     assert_eq!(
1666    //         ops::read_to_string(&fs, "/first/file.txt").await.unwrap(),
1667    //         "First!"
1668    //     );
1669    //     // Overwriting them is fine and we'll see the changes on the host
1670    //     ops::write(&fs, "/first/file.txt", "Updated").await.unwrap();
1671    //     assert_eq!(std::fs::read_to_string(&file_txt).unwrap(), "Updated");
1672    //     // The filesystem will see changes on the host that happened after it was
1673    //     // set up
1674    //     let another = second.join("another.txt");
1675    //     std::fs::write(&another, "asdf").unwrap();
1676    //     assert_eq!(
1677    //         ops::read_to_string(&fs, "/second/another.txt")
1678    //             .await
1679    //             .unwrap(),
1680    //         "asdf"
1681    //     );
1682    // }
1683    //
1684    // #[tokio::test]
1685    // async fn absolute_and_relative_paths_are_passed_through() {
1686    //     let python = Arc::new(load_webc(PYTHON));
1687    //
1688    //     // The underlying filesystem doesn't care about absolute/relative paths
1689    //     assert_eq!(python.read_dir("/lib".as_ref()).unwrap().count(), 4);
1690    //     assert_eq!(python.read_dir("lib".as_ref()).unwrap().count(), 4);
1691    //
1692    //     // read_dir() should be passed through to the primary
1693    //     let webc_primary =
1694    //         OverlayFileSystem::new(Arc::clone(&python), [crate::EmptyFileSystem::default()]);
1695    //     assert_same_directory_contents(&python, "/lib", &webc_primary);
1696    //     assert_same_directory_contents(&python, "lib", &webc_primary);
1697    //
1698    //     // read_dir() should also be passed through to the secondary
1699    //     let webc_secondary =
1700    //         OverlayFileSystem::new(crate::EmptyFileSystem::default(), [Arc::clone(&python)]);
1701    //     assert_same_directory_contents(&python, "/lib", &webc_secondary);
1702    //     assert_same_directory_contents(&python, "lib", &webc_secondary);
1703    //
1704    //     // It should be fine to overlay the root fs on top of our webc file
1705    //     let overlay_rootfs = OverlayFileSystem::new(
1706    //         RootFileSystemBuilder::default().build(),
1707    //         [Arc::clone(&python)],
1708    //     );
1709    //     assert_same_directory_contents(&python, "/lib", &overlay_rootfs);
1710    //     assert_same_directory_contents(&python, "lib", &overlay_rootfs);
1711    // }
1712    // #[track_caller]
1713    // fn assert_same_directory_contents(
1714    //     original: &dyn FileSystem,
1715    //     path: impl AsRef<Path>,
1716    //     candidate: &dyn FileSystem,
1717    // ) {
1718    //     let path = path.as_ref();
1719    //
1720    //     let original_entries: Vec<_> = original
1721    //         .read_dir(path)
1722    //         .unwrap()
1723    //         .map(|r| r.unwrap())
1724    //         .collect();
1725    //     let candidate_entries: Vec<_> = candidate
1726    //         .read_dir(path)
1727    //         .unwrap()
1728    //         .map(|r| r.unwrap())
1729    //         .collect();
1730    //
1731    //     assert_eq!(original_entries, candidate_entries);
1732    // }
1733}
virtual_fs/overlay_fs.rs

virtual_fs/
overlay_fs.rs
