use std::{
borrow::Cow,
collections::BTreeMap,
fmt::Debug,
fs::File,
io::{BufRead, BufReader},
path::{Path, PathBuf},
};
use anyhow::{Context, Error};
use bytes::Bytes;
use flate2::bufread::GzDecoder;
use shared_buffer::OwnedBuffer;
use tar::Archive;
use tempfile::TempDir;
use wasmer_toml::Manifest as WasmerManifest;
use crate::{
metadata::Manifest as WebcManifest,
v2::{
write::{FileEntry, Writer},
ChecksumAlgorithm,
},
wasmer_package::{manifest::ManifestError, Strictness, Volume},
PathSegment,
};
#[derive(Debug, thiserror::Error)]
#[allow(clippy::result_large_err)]
#[non_exhaustive]
pub enum WasmerPackageError {
#[error("Unable to create a temporary directory")]
TempDir(#[source] std::io::Error),
#[error("Unable to open \"{}\"", path.display())]
FileOpen {
path: PathBuf,
#[source]
error: std::io::Error,
},
#[error("Unable to read \"{}\"", path.display())]
FileRead {
path: PathBuf,
#[source]
error: std::io::Error,
},
#[error("Unable to extract the tarball")]
Tarball(#[source] std::io::Error),
#[error("Unable to deserialize \"{}\"", path.display())]
DeserializeWasmerToml {
path: PathBuf,
#[source]
error: toml::de::Error,
},
#[error("Unable to find the \"wasmer.toml\"")]
MissingManifest,
#[error("Unable to get the absolute path for \"{}\"", path.display())]
Canonicalize {
path: PathBuf,
#[source]
error: std::io::Error,
},
#[error("Unable to load the \"wasmer.toml\" manifest")]
Manifest(#[from] ManifestError),
#[error("The manifest is invalid")]
Validation(#[from] wasmer_toml::ValidationError),
}
#[derive(Debug)]
pub struct Package {
manifest: WebcManifest,
original_manifest: WasmerManifest,
atoms: BTreeMap<String, OwnedBuffer>,
base_dir: BaseDir,
strictness: Strictness,
}
impl Package {
pub fn from_tarball_file(path: impl AsRef<Path>) -> Result<Self, WasmerPackageError> {
Package::from_tarball_file_with_strictness(path.as_ref(), Strictness::default())
}
pub fn from_tarball_file_with_strictness(
path: impl AsRef<Path>,
strictness: Strictness,
) -> Result<Self, WasmerPackageError> {
let path = path.as_ref();
let f = File::open(path).map_err(|error| WasmerPackageError::FileOpen {
path: path.to_path_buf(),
error,
})?;
Package::from_tarball_with_strictness(BufReader::new(f), strictness)
}
pub fn from_tarball(tarball: impl BufRead) -> Result<Self, WasmerPackageError> {
Package::from_tarball_with_strictness(tarball, Strictness::default())
}
pub fn from_tarball_with_strictness(
tarball: impl BufRead,
strictness: Strictness,
) -> Result<Self, WasmerPackageError> {
let tarball = GzDecoder::new(tarball);
let temp = tempdir().map_err(WasmerPackageError::TempDir)?;
let archive = Archive::new(tarball);
unpack_archive(archive, temp.path()).map_err(WasmerPackageError::Tarball)?;
let manifest = read_manifest(temp.path())?;
Package::load(manifest, temp, strictness)
}
pub fn from_manifest(wasmer_toml: impl AsRef<Path>) -> Result<Self, WasmerPackageError> {
Package::from_manifest_with_strictness(wasmer_toml, Strictness::default())
}
pub fn from_manifest_with_strictness(
wasmer_toml: impl AsRef<Path>,
strictness: Strictness,
) -> Result<Self, WasmerPackageError> {
let path = wasmer_toml.as_ref();
let path = path
.canonicalize()
.map_err(|error| WasmerPackageError::Canonicalize {
path: path.to_path_buf(),
error,
})?;
let wasmer_toml =
std::fs::read_to_string(&path).map_err(|error| WasmerPackageError::FileRead {
path: path.to_path_buf(),
error,
})?;
let wasmer_toml = toml::from_str(&wasmer_toml).map_err(|error| {
WasmerPackageError::DeserializeWasmerToml {
path: path.to_path_buf(),
error,
}
})?;
let base_dir = path
.parent()
.expect("Canonicalizing should always result in a file with a parent directory")
.to_path_buf();
Package::load(wasmer_toml, base_dir, strictness)
}
fn load(
wasmer_toml: WasmerManifest,
base_dir: impl Into<BaseDir>,
strictness: Strictness,
) -> Result<Self, WasmerPackageError> {
let base_dir = base_dir.into();
if strictness.is_strict() {
wasmer_toml.validate()?;
}
let (manifest, atoms) = crate::wasmer_package::manifest::wasmer_manifest_to_webc(
&wasmer_toml,
base_dir.path(),
strictness,
)?;
Ok(Package {
manifest,
original_manifest: wasmer_toml,
atoms,
base_dir,
strictness,
})
}
pub fn manifest(&self) -> &WebcManifest {
&self.manifest
}
pub fn atoms(&self) -> &BTreeMap<String, OwnedBuffer> {
&self.atoms
}
pub fn serialize(&self) -> Result<Bytes, Error> {
let metadata_volume = self.metadata_volume()?;
let asset_volume = self.asset_volume()?;
let w = Writer::new(ChecksumAlgorithm::Sha256)
.write_manifest(self.manifest())?
.write_atoms(self.atom_entries()?)?
.with_volume(
Volume::METADATA,
metadata_volume.as_directory_tree(self.strictness)?,
)?
.with_volume(
Volume::ASSET,
asset_volume.as_directory_tree(self.strictness)?,
)?;
let serialized = w.finish(crate::v2::SignatureAlgorithm::None)?;
Ok(serialized)
}
fn atom_entries(&self) -> Result<BTreeMap<PathSegment, FileEntry<'_>>, Error> {
self.atoms()
.iter()
.map(|(key, value)| {
let filename = PathSegment::parse(key)
.with_context(|| format!("\"{key}\" isn't a valid atom name"))?;
Ok((filename, FileEntry::Borrowed(value)))
})
.collect()
}
pub fn metadata_volume(&self) -> Result<Volume, Error> {
Volume::new_metadata(&self.original_manifest, self.base_dir().to_path_buf())
}
pub fn asset_volume(&self) -> Result<Volume, Error> {
Volume::new_asset(&self.original_manifest, self.base_dir())
}
pub(crate) fn get_volume(&self, name: &str) -> Option<Volume> {
match name {
Volume::METADATA => Volume::new_metadata(&self.original_manifest, self.base_dir()).ok(),
Volume::ASSET => Volume::new_asset(&self.original_manifest, self.base_dir()).ok(),
_ => None,
}
}
pub(crate) fn volume_names(&self) -> Vec<Cow<'static, str>> {
vec![
Cow::Borrowed(Volume::METADATA),
Cow::Borrowed(Volume::ASSET),
]
}
fn base_dir(&self) -> &Path {
self.base_dir.path()
}
}
const IS_WASI: bool = cfg!(all(target_family = "wasm", target_os = "wasi"));
fn tempdir() -> Result<TempDir, std::io::Error> {
if !IS_WASI {
return TempDir::new();
}
let temp_dir: PathBuf = std::env::var("TMPDIR")
.unwrap_or_else(|_| "/tmp".to_string())
.into();
if temp_dir.exists() {
TempDir::new_in(temp_dir)
} else {
if let Ok(current_exe) = std::env::current_exe() {
if let Some(parent) = current_exe.parent() {
if let Ok(temp) = TempDir::new_in(parent) {
return Ok(temp);
}
}
}
std::fs::create_dir_all(&temp_dir)?;
TempDir::new_in(temp_dir)
}
}
fn unpack_archive(
mut archive: Archive<impl std::io::Read>,
dest: &Path,
) -> Result<(), std::io::Error> {
if !IS_WASI {
return archive.unpack(dest);
}
for entry in archive.entries()? {
let mut entry = entry?;
let item_path = entry.path()?;
let full_path = resolve_archive_path(dest, &item_path);
match entry.header().entry_type() {
tar::EntryType::Directory => {
std::fs::create_dir_all(&full_path)?;
}
tar::EntryType::Regular => {
if let Some(parent) = full_path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut f = File::create(&full_path)?;
std::io::copy(&mut entry, &mut f)?;
}
_ => {}
}
}
Ok(())
}
fn resolve_archive_path(base_dir: &Path, path: &Path) -> PathBuf {
let mut buffer = base_dir.to_path_buf();
for component in path.components() {
match component {
std::path::Component::Prefix(_)
| std::path::Component::RootDir
| std::path::Component::CurDir => continue,
std::path::Component::ParentDir => {
buffer.pop();
}
std::path::Component::Normal(segment) => {
buffer.push(segment);
}
}
}
buffer
}
fn read_manifest(base_dir: &Path) -> Result<WasmerManifest, WasmerPackageError> {
for path in ["wasmer.toml", "wapm.toml"] {
let path = base_dir.join(path);
match std::fs::read_to_string(&path) {
Ok(s) => {
return toml::from_str(&s)
.map_err(|error| WasmerPackageError::DeserializeWasmerToml { path, error });
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => {
return Err(WasmerPackageError::FileRead { path, error });
}
}
}
Err(WasmerPackageError::MissingManifest)
}
#[derive(Debug)]
enum BaseDir {
Path(PathBuf),
Temp(TempDir),
}
impl BaseDir {
fn path(&self) -> &Path {
match self {
BaseDir::Path(p) => p.as_path(),
BaseDir::Temp(t) => t.path(),
}
}
}
impl From<TempDir> for BaseDir {
fn from(v: TempDir) -> Self {
Self::Temp(v)
}
}
impl From<PathBuf> for BaseDir {
fn from(v: PathBuf) -> Self {
Self::Path(v)
}
}
#[cfg(test)]
mod tests {
use crate::{
metadata::{
annotations::{FileSystemMapping, VolumeSpecificPath},
Binding, BindingsExtended, WaiBindings, WitBindings,
},
Container,
};
use flate2::{write::GzEncoder, Compression};
use super::*;
#[test]
fn nonexistent_files() {
let temp = TempDir::new().unwrap();
assert!(Package::from_manifest(temp.path().join("nonexistent.toml")).is_err());
assert!(Package::from_tarball_file(temp.path().join("nonexistent.tar.gz")).is_err());
}
#[test]
fn load_a_tarball() {
let coreutils = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("wapm-targz-to-pirita")
.join("fixtures")
.join("coreutils-1.0.11.tar.gz");
assert!(coreutils.exists());
let package = Package::from_tarball_file(coreutils).unwrap();
let wapm = package.manifest().wapm().unwrap().unwrap();
assert_eq!(wapm.name, "sharrattj/coreutils");
assert_eq!(wapm.version, "1.0.11");
}
#[test]
fn tarball_with_no_manifest() {
let temp = TempDir::new().unwrap();
let empty_tarball = temp.path().join("empty.tar.gz");
let mut f = File::create(&empty_tarball).unwrap();
tar::Builder::new(GzEncoder::new(&mut f, Compression::fast()))
.finish()
.unwrap();
assert!(Package::from_tarball_file(&empty_tarball).is_err());
}
#[test]
fn empty_package_on_disk() {
let temp = TempDir::new().unwrap();
let manifest = temp.path().join("wasmer.toml");
std::fs::write(
&manifest,
r#"
[package]
name = "some/package"
version = "0.0.0"
description = "A dummy package"
"#,
)
.unwrap();
let package = Package::from_manifest(&manifest).unwrap();
let wapm = package.manifest().wapm().unwrap().unwrap();
assert_eq!(wapm.name, "some/package");
assert_eq!(wapm.version, "0.0.0");
}
#[test]
fn load_old_cowsay() {
let tarball = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("wapm-to-webc")
.join("test")
.join("fixtures")
.join("cowsay-0.3.0.tar.gz");
let pkg = Package::from_tarball_file(tarball).unwrap();
insta::assert_yaml_snapshot!(pkg.manifest());
assert_eq!(
pkg.manifest.commands.keys().collect::<Vec<_>>(),
["cowsay", "cowthink"],
);
}
#[test]
fn serialize_package_with_bundled_directories() {
let temp = TempDir::new().unwrap();
let wasmer_toml = r#"
[package]
name = "some/package"
version = "0.0.0"
description = "Test package"
[fs]
"/first" = "./first"
second = "nested/dir"
"second/child" = "./third"
empty = "empty"
"#;
let manifest = temp.path().join("wasmer.toml");
std::fs::write(&manifest, wasmer_toml).unwrap();
let first = temp.path().join("first");
std::fs::create_dir_all(&first).unwrap();
std::fs::write(first.join("file.txt"), "File").unwrap();
let second = temp.path().join("nested").join("dir");
std::fs::create_dir_all(&second).unwrap();
std::fs::write(second.join("README.md"), "please").unwrap();
let another_dir = temp.path().join("nested").join("dir").join("another-dir");
std::fs::create_dir_all(&another_dir).unwrap();
std::fs::write(another_dir.join("empty.txt"), "").unwrap();
let third = temp.path().join("third");
std::fs::create_dir_all(&third).unwrap();
std::fs::write(third.join("file.txt"), "Hello, World!").unwrap();
let empty_dir = temp.path().join("empty");
std::fs::create_dir_all(empty_dir).unwrap();
let package = Package::from_manifest(manifest).unwrap();
let webc = package.serialize().unwrap();
let webc = Container::from_bytes(webc).unwrap();
let manifest = webc.manifest();
let wapm_metadata = manifest.wapm().unwrap().unwrap();
assert_eq!(wapm_metadata.name, "some/package");
let fs_table = manifest.filesystem().unwrap().unwrap();
assert_eq!(
fs_table,
[
FileSystemMapping {
from: None,
volume_name: "atom".to_string(),
original_path: "/first".to_string(),
mount_path: "/first".to_string(),
},
FileSystemMapping {
from: None,
volume_name: "atom".to_string(),
original_path: "/nested/dir".to_string(),
mount_path: "/second".to_string(),
},
FileSystemMapping {
from: None,
volume_name: "atom".to_string(),
original_path: "/third".to_string(),
mount_path: "/second/child".to_string(),
},
FileSystemMapping {
from: None,
volume_name: "atom".to_string(),
original_path: "/empty".to_string(),
mount_path: "/empty".to_string(),
},
]
);
let atom_volume = webc.get_volume("atom").unwrap();
assert_eq!(atom_volume.read_file("/first/file.txt").unwrap(), b"File");
assert_eq!(
atom_volume.read_file("/nested/dir/README.md").unwrap(),
b"please"
);
assert_eq!(
atom_volume
.read_file("/nested/dir/another-dir/empty.txt")
.unwrap(),
b""
);
assert_eq!(
atom_volume.read_file("/third/file.txt").unwrap(),
b"Hello, World!"
);
assert_eq!(
atom_volume.read_dir("/empty").unwrap().len(),
0,
"Directories should be included, even if empty"
);
}
#[test]
fn serialize_package_with_metadata_files() {
let temp = TempDir::new().unwrap();
let wasmer_toml = r#"
[package]
name = "some/package"
version = "0.0.0"
description = "Test package"
readme = "README.md"
license-file = "LICENSE"
"#;
let manifest = temp.path().join("wasmer.toml");
std::fs::write(&manifest, wasmer_toml).unwrap();
std::fs::write(temp.path().join("README.md"), "readme").unwrap();
std::fs::write(temp.path().join("LICENSE"), "license").unwrap();
let serialized = Package::from_manifest(manifest)
.unwrap()
.serialize()
.unwrap();
let webc = Container::from_bytes(serialized).unwrap();
let metadata_volume = webc.get_volume("metadata").unwrap();
assert_eq!(metadata_volume.read_file("/README.md").unwrap(), b"readme");
assert_eq!(metadata_volume.read_file("/LICENSE").unwrap(), b"license");
}
#[test]
fn load_package_with_wit_bindings() {
let temp = TempDir::new().unwrap();
let wasmer_toml = r#"
[package]
name = "some/package"
version = "0.0.0"
description = ""
[[module]]
name = "my-lib"
source = "./my-lib.wasm"
abi = "none"
bindings = { wit-bindgen = "0.1.0", wit-exports = "./file.wit" }
"#;
std::fs::write(temp.path().join("wasmer.toml"), wasmer_toml).unwrap();
std::fs::write(temp.path().join("file.wit"), "file").unwrap();
std::fs::write(temp.path().join("my-lib.wasm"), b"\0asm...").unwrap();
let package = Package::from_manifest(temp.path().join("wasmer.toml"))
.unwrap()
.serialize()
.unwrap();
let webc = Container::from_bytes(package).unwrap();
assert_eq!(
webc.manifest().bindings,
vec![Binding {
name: "library-bindings".to_string(),
kind: "wit@0.1.0".to_string(),
annotations: serde_cbor::value::to_value(BindingsExtended::Wit(WitBindings {
exports: "metadata://file.wit".to_string(),
module: "my-lib".to_string(),
}))
.unwrap(),
}]
);
let metadata_volume = webc.get_volume("metadata").unwrap();
assert_eq!(metadata_volume.read_file("/file.wit").unwrap(), b"file");
insta::with_settings! {
{ description => wasmer_toml },
{ insta::assert_yaml_snapshot!(webc.manifest()); }
}
}
#[test]
fn load_package_with_wai_bindings() {
let temp = TempDir::new().unwrap();
let wasmer_toml = r#"
[package]
name = "some/package"
version = "0.0.0"
description = ""
[[module]]
name = "my-lib"
source = "./my-lib.wasm"
abi = "none"
bindings = { wai-version = "0.2.0", exports = "./file.wai", imports = ["a.wai", "b.wai"] }
"#;
std::fs::write(temp.path().join("wasmer.toml"), wasmer_toml).unwrap();
std::fs::write(temp.path().join("file.wai"), "file").unwrap();
std::fs::write(temp.path().join("a.wai"), "a").unwrap();
std::fs::write(temp.path().join("b.wai"), "b").unwrap();
std::fs::write(temp.path().join("my-lib.wasm"), b"\0asm...").unwrap();
let package = Package::from_manifest(temp.path().join("wasmer.toml"))
.unwrap()
.serialize()
.unwrap();
let webc = Container::from_bytes(package).unwrap();
assert_eq!(
webc.manifest().bindings,
vec![Binding {
name: "library-bindings".to_string(),
kind: "wai@0.2.0".to_string(),
annotations: serde_cbor::value::to_value(BindingsExtended::Wai(WaiBindings {
exports: Some("metadata://file.wai".to_string()),
module: "my-lib".to_string(),
imports: vec![
"metadata://a.wai".to_string(),
"metadata://b.wai".to_string(),
]
}))
.unwrap(),
}]
);
let metadata_volume = webc.get_volume("metadata").unwrap();
assert_eq!(metadata_volume.read_file("/file.wai").unwrap(), b"file");
assert_eq!(metadata_volume.read_file("/a.wai").unwrap(), b"a");
assert_eq!(metadata_volume.read_file("/b.wai").unwrap(), b"b");
insta::with_settings! {
{ description => wasmer_toml },
{ insta::assert_yaml_snapshot!(webc.manifest()); }
}
}
#[test]
fn absolute_paths_in_wasmer_toml_issue_105() {
let temp = TempDir::new().unwrap();
let base_dir = temp.path().canonicalize().unwrap();
let sep = std::path::MAIN_SEPARATOR;
let wasmer_toml = format!(
r#"
[package]
name = 'some/package'
version = '0.0.0'
description = 'Test package'
readme = '{BASE_DIR}{sep}README.md'
license-file = '{BASE_DIR}{sep}LICENSE'
[[module]]
name = 'first'
source = '{BASE_DIR}{sep}target{sep}debug{sep}package.wasm'
bindings = {{ wai-version = '0.2.0', exports = '{BASE_DIR}{sep}bindings{sep}file.wai', imports = ['{BASE_DIR}{sep}bindings{sep}a.wai'] }}
"#,
BASE_DIR = base_dir.display(),
);
let manifest = temp.path().join("wasmer.toml");
std::fs::write(&manifest, &wasmer_toml).unwrap();
std::fs::write(temp.path().join("README.md"), "readme").unwrap();
std::fs::write(temp.path().join("LICENSE"), "license").unwrap();
let bindings = temp.path().join("bindings");
std::fs::create_dir_all(&bindings).unwrap();
std::fs::write(bindings.join("file.wai"), "file.wai").unwrap();
std::fs::write(bindings.join("a.wai"), "a.wai").unwrap();
let target = temp.path().join("target").join("debug");
std::fs::create_dir_all(&target).unwrap();
std::fs::write(target.join("package.wasm"), b"\0asm...").unwrap();
let serialized = Package::from_manifest(manifest)
.unwrap()
.serialize()
.unwrap();
let webc = Container::from_bytes(serialized).unwrap();
let manifest = webc.manifest();
let wapm = manifest.wapm().unwrap().unwrap();
let lookup = |item: VolumeSpecificPath| {
let volume = webc.get_volume(&item.volume).unwrap();
let contents = volume.read_file(&item.path).unwrap();
String::from_utf8(contents.into()).unwrap()
};
assert_eq!(lookup(wapm.license_file.unwrap()), "license");
assert_eq!(lookup(wapm.readme.unwrap()), "readme");
let lookup = |item: &str| {
let (volume, path) = item.split_once(":/").unwrap();
let volume = webc.get_volume(volume).unwrap();
let content = volume.read_file(path).unwrap();
String::from_utf8(content.into()).unwrap()
};
let bindings = manifest.bindings[0].get_wai_bindings().unwrap();
assert_eq!(lookup(&bindings.imports[0]), "a.wai");
assert_eq!(lookup(bindings.exports.unwrap().as_str()), "file.wai");
let mut settings = insta::Settings::clone_current();
let base_dir = base_dir.display().to_string();
settings.set_description(wasmer_toml.replace(&base_dir, "[BASE_DIR]"));
let filter = regex::escape(&base_dir);
settings.add_filter(&filter, "[BASE_DIR]");
settings.bind(|| {
insta::assert_yaml_snapshot!(webc.manifest());
});
}
#[test]
fn serializing_will_skip_missing_metadata_by_default() {
let temp = TempDir::new().unwrap();
let wasmer_toml = r#"
[package]
name = 'some/package'
version = '0.0.0'
description = 'Test package'
readme = '/this/does/not/exist/README.md'
license-file = 'LICENSE.wtf'
"#;
let manifest = temp.path().join("wasmer.toml");
std::fs::write(&manifest, wasmer_toml).unwrap();
let pkg = Package::from_manifest(manifest).unwrap();
let serialized = pkg.serialize().unwrap();
let webc = Container::from_bytes(serialized).unwrap();
let manifest = webc.manifest();
let wapm = manifest.wapm().unwrap().unwrap();
assert!(wapm.license_file.is_none());
assert!(wapm.readme.is_none());
let pkg = Package {
strictness: Strictness::Strict,
..pkg
};
assert!(pkg.serialize().is_err());
}
}