wasi_test_generator/
wasitests.rs

1//! This file will run at build time to autogenerate the WASI regression tests
2//! It will compile the files indicated in TESTS, to:executable and .wasm
3//! - Compile with the native rust target to get the expected output
4//! - Compile with the latest WASI target to get the wasm
5//! - Generate the test that will compare the output of running the .wasm file
6//!   with wasmer with the expected output
7
8use glob::glob;
9use std::fs;
10use std::path::{Path, PathBuf};
11use std::process::{Command, Stdio};
12
13use std::io;
14use std::io::prelude::*;
15
16use super::util;
17use super::wasi_version::*;
18
19#[derive(Debug, Clone, PartialEq, Eq)]
20pub struct NativeOutput {
21    stdout: String,
22    stderr: String,
23    result: i64,
24}
25
26/// Compile and execute the test file as native code, saving the results to be
27/// compared against later.
28///
29/// This function attempts to clean up its output after it executes it.
30fn generate_native_output(
31    temp_dir: &Path,
32    file: &str,
33    normalized_name: &str,
34    args: &[String],
35    options: &WasiOptions,
36) -> io::Result<NativeOutput> {
37    let executable_path = temp_dir.join(normalized_name);
38    println!(
39        "Compiling program {} to native at {}",
40        file,
41        executable_path.to_string_lossy()
42    );
43    let native_out = Command::new("rustc")
44        .arg(file)
45        .arg("-o")
46        .args(args)
47        .arg(&executable_path)
48        .output()
49        .expect("Failed to compile program to native code");
50    util::print_info_on_error(&native_out, "COMPILATION FAILED");
51
52    #[cfg(unix)]
53    {
54        use std::os::unix::fs::PermissionsExt;
55        let mut perm = executable_path
56            .metadata()
57            .expect("native executable")
58            .permissions();
59        perm.set_mode(0o766);
60        println!(
61            "Setting execute permissions on {}",
62            executable_path.to_string_lossy()
63        );
64        fs::set_permissions(&executable_path, perm)?;
65    }
66
67    println!(
68        "Executing native program at {}",
69        executable_path.to_string_lossy()
70    );
71    // workspace root
72    const EXECUTE_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/wasi");
73    let mut native_command = Command::new(&executable_path)
74        .current_dir(EXECUTE_DIR)
75        .stdin(Stdio::piped())
76        .stdout(Stdio::piped())
77        .stderr(Stdio::piped())
78        .spawn()
79        .unwrap();
80
81    if let Some(stdin_str) = &options.stdin {
82        write!(native_command.stdin.as_ref().unwrap(), "{stdin_str}").unwrap();
83    }
84
85    let result = native_command
86        .wait()
87        .expect("Failed to execute native program");
88
89    let stdout_str = {
90        let mut stdout = native_command.stdout.unwrap();
91        let mut s = String::new();
92        stdout.read_to_string(&mut s).unwrap();
93        s
94    };
95    let stderr_str = {
96        let mut stderr = native_command.stderr.unwrap();
97        let mut s = String::new();
98        stderr.read_to_string(&mut s).unwrap();
99        s
100    };
101    if !result.success() {
102        println!("NATIVE PROGRAM FAILED");
103        println!("stdout:\n{stdout_str}");
104        eprintln!("stderr:\n{stderr_str}");
105    }
106
107    let result = result.code().unwrap() as i64;
108    Ok(NativeOutput {
109        stdout: stdout_str,
110        stderr: stderr_str,
111        result,
112    })
113}
114
115/// compile the Wasm file for the given version of WASI
116///
117/// returns the path of where the wasm file is
118fn compile_wasm_for_version(
119    temp_dir: &Path,
120    file: &str,
121    out_dir: &Path,
122    rs_mod_name: &str,
123    version: WasiVersion,
124) -> io::Result<PathBuf> {
125    //let out_dir = base_dir; //base_dir.join("..").join(version.get_directory_name());
126    if !out_dir.exists() {
127        fs::create_dir(out_dir)?;
128    }
129    let wasm_out_name = {
130        let mut wasm_out_name = out_dir.join(rs_mod_name);
131        wasm_out_name.set_extension("wasm");
132        wasm_out_name
133    };
134    println!("Reading contents from file `{file}`");
135    let file_contents: String = {
136        let mut fc = String::new();
137        let mut f = fs::OpenOptions::new().read(true).open(file)?;
138        f.read_to_string(&mut fc)?;
139        fc
140    };
141
142    let temp_wasi_rs_file_name = temp_dir.join(format!("wasi_modified_version_{rs_mod_name}.rs"));
143    {
144        let mut actual_file = fs::OpenOptions::new()
145            .write(true)
146            .truncate(true)
147            .create(true)
148            .open(&temp_wasi_rs_file_name)
149            .unwrap();
150        actual_file.write_all(file_contents.as_bytes()).unwrap();
151    }
152
153    println!(
154        "Compiling wasm module `{}` with toolchain `{}`",
155        &wasm_out_name.to_string_lossy(),
156        version.get_compiler_toolchain()
157    );
158    let mut command = Command::new("rustc");
159
160    command
161        .arg(format!("+{}", version.get_compiler_toolchain()))
162        .arg("--target=wasm32-wasip1")
163        .arg("-C")
164        .arg("opt-level=z")
165        .arg(&temp_wasi_rs_file_name)
166        .arg("-o")
167        .arg(&wasm_out_name);
168    println!("Command {command:?}");
169
170    let wasm_compilation_out = command.output().expect("Failed to compile program to wasm");
171    util::print_info_on_error(&wasm_compilation_out, "WASM COMPILATION");
172    println!(
173        "Removing file `{}`",
174        &temp_wasi_rs_file_name.to_string_lossy()
175    );
176
177    // to prevent commiting huge binary blobs forever
178    let wasm_strip_out = Command::new("wasm-strip")
179        .arg(&wasm_out_name)
180        .output()
181        .expect("Failed to strip compiled wasm module");
182    util::print_info_on_error(&wasm_strip_out, "STRIPPING WASM");
183    let wasm_opt_out = Command::new("wasm-opt")
184        .arg("-Oz")
185        .arg(&wasm_out_name)
186        .arg("-o")
187        .arg(&wasm_out_name)
188        .output()
189        .expect("Failed to optimize compiled wasm module with wasm-opt!");
190    util::print_info_on_error(&wasm_opt_out, "OPTIMIZING WASM");
191
192    Ok(wasm_out_name)
193}
194
195/// Returns the a Vec of the test modules created
196fn compile(temp_dir: &Path, file: &str, wasi_versions: &[WasiVersion]) {
197    let src_code: String = fs::read_to_string(file).unwrap();
198    let options: WasiOptions = extract_args_from_source_file(&src_code).unwrap_or_default();
199
200    assert!(file.ends_with(".rs"));
201    let rs_mod_name = {
202        Path::new(&file.to_lowercase())
203            .file_stem()
204            .unwrap()
205            .to_string_lossy()
206            .to_string()
207    };
208    let base_dir = Path::new(file).parent().unwrap();
209    let NativeOutput {
210        stdout,
211        stderr,
212        result,
213    } = generate_native_output(temp_dir, file, &rs_mod_name, &options.args, &options)
214        .expect("Generate native output");
215
216    let test = WasiTest {
217        wasm_prog_name: format!("{rs_mod_name}.wasm"),
218        stdout,
219        stderr,
220        result,
221        options,
222    };
223    let test_serialized = test.into_wasi_wast();
224    println!("Generated test output: {}", &test_serialized);
225
226    wasi_versions
227        .iter()
228        .map(|&version| {
229            let out_dir = base_dir.join("..").join(version.get_directory_name());
230            if !out_dir.exists() {
231                fs::create_dir(&out_dir).unwrap();
232            }
233            let wasm_out_name = {
234                let mut wasm_out_name = out_dir.join(rs_mod_name.clone());
235                wasm_out_name.set_extension("wast");
236                wasm_out_name
237            };
238            println!("Writing test output to {}", wasm_out_name.to_string_lossy());
239            fs::write(&wasm_out_name, test_serialized.clone()).unwrap();
240
241            println!("Compiling wasm version {version:?}");
242            compile_wasm_for_version(temp_dir, file, &out_dir, &rs_mod_name, version)
243                .unwrap_or_else(|_| panic!("Could not compile Wasm to WASI version {:?}, perhaps you need to install the `{}` rust toolchain", version, version.get_compiler_toolchain()));
244        }).for_each(drop); // Do nothing with it, but let the iterator be consumed/iterated.
245}
246
247const WASI_TEST_SRC_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/wasi/tests/*.rs");
248pub fn build(wasi_versions: &[WasiVersion], specific_tests: &[&str]) {
249    let temp_dir = tempfile::TempDir::new().unwrap();
250    for entry in glob(WASI_TEST_SRC_DIR).unwrap() {
251        match entry {
252            Ok(path) => {
253                let test = path.to_str().unwrap();
254                if !specific_tests.is_empty() {
255                    if let Some(filename) = path.file_stem().and_then(|f| f.to_str()) {
256                        if specific_tests.contains(&filename) {
257                            compile(temp_dir.path(), test, wasi_versions);
258                        }
259                    }
260                } else {
261                    compile(temp_dir.path(), test, wasi_versions);
262                }
263            }
264            Err(e) => println!("{e:?}"),
265        }
266    }
267    println!("All modules generated.");
268}
269
270/// This is the structure of the `.wast` file
271#[derive(Debug, Default, Serialize, Deserialize)]
272pub struct WasiTest {
273    /// The name of the wasm module to run
274    pub wasm_prog_name: String,
275    /// The program expected output on stdout
276    pub stdout: String,
277    /// The program expected output on stderr
278    pub stderr: String,
279    /// The program expected result
280    pub result: i64,
281    /// The program options
282    pub options: WasiOptions,
283}
284
285impl WasiTest {
286    fn into_wasi_wast(self) -> String {
287        use std::fmt::Write;
288
289        let mut out = format!(
290            ";; This file was generated by https://github.com/wasmerio/wasi-tests\n
291(wasi_test \"{}\"",
292            self.wasm_prog_name
293        );
294        if !self.options.env.is_empty() {
295            let envs = self
296                .options
297                .env
298                .iter()
299                .map(|(name, value)| format!("\"{name}={value}\""))
300                .collect::<Vec<String>>()
301                .join(" ");
302            let _ = write!(out, "\n  (envs {envs})");
303        }
304        if !self.options.args.is_empty() {
305            let args = self
306                .options
307                .args
308                .iter()
309                .map(|v| format!("\"{v}\""))
310                .collect::<Vec<String>>()
311                .join(" ");
312            let _ = write!(out, "\n  (args {args})");
313        }
314
315        if !self.options.dir.is_empty() {
316            let preopens = self
317                .options
318                .dir
319                .iter()
320                .map(|v| format!("\"{v}\""))
321                .collect::<Vec<String>>()
322                .join(" ");
323            let _ = write!(out, "\n  (preopens {preopens})");
324        }
325        if !self.options.mapdir.is_empty() {
326            let map_dirs = self
327                .options
328                .mapdir
329                .iter()
330                .map(|(a, b)| format!("\"{a}:{b}\""))
331                .collect::<Vec<String>>()
332                .join(" ");
333            let _ = write!(out, "\n  (map_dirs {map_dirs})");
334        }
335        if !self.options.tempdir.is_empty() {
336            let temp_dirs = self
337                .options
338                .tempdir
339                .iter()
340                .map(|td| format!("\"{td}\""))
341                .collect::<Vec<String>>()
342                .join(" ");
343            let _ = write!(out, "\n  (temp_dirs {temp_dirs})");
344        }
345
346        let _ = write!(out, "\n  (assert_return (i64.const {}))", self.result);
347        if let Some(stdin) = &self.options.stdin {
348            let _ = write!(out, "\n  (stdin {stdin:?})");
349        }
350
351        if !self.stdout.is_empty() {
352            let _ = write!(out, "\n  (assert_stdout {:?})", self.stdout);
353        }
354        if !self.stderr.is_empty() {
355            let _ = write!(out, "\n  (assert_stderr {:?})", self.stderr);
356        }
357
358        let _ = write!(out, "\n)\n");
359
360        out
361    }
362}
363
364/// The options provied when executed a WASI Wasm program
365#[derive(Debug, Default, Serialize, Deserialize)]
366pub struct WasiOptions {
367    /// Mapped pre-opened dirs
368    pub mapdir: Vec<(String, String)>,
369    /// Environment vars
370    pub env: Vec<(String, String)>,
371    /// Program arguments
372    pub args: Vec<String>,
373    /// Pre-opened directories
374    pub dir: Vec<String>,
375    /// The alias of the temporary directory to use
376    pub tempdir: Vec<String>,
377    /// Stdin to give to the native program and WASI program.
378    pub stdin: Option<String>,
379}
380
381/// Pulls args to the program out of a comment at the top of the file starting with "// WasiOptions:"
382fn extract_args_from_source_file(source_code: &str) -> Option<WasiOptions> {
383    if source_code.starts_with("// WASI:") {
384        let mut args = WasiOptions::default();
385        for arg_line in source_code
386            .lines()
387            .skip(1)
388            .take_while(|line| line.starts_with("// "))
389        {
390            let arg_line = arg_line.strip_prefix("// ").unwrap();
391            let arg_line = arg_line.trim();
392            let colon_idx = arg_line
393                .find(':')
394                .expect("directives provided at the top must be separated by a `:`");
395
396            let (command_name, value) = arg_line.split_at(colon_idx);
397            let value = value.strip_prefix(':').unwrap();
398            let value = value.trim();
399
400            match command_name {
401                "mapdir" =>
402                // We try first splitting by `::`
403                {
404                    if let [alias, real_dir] = value.split("::").collect::<Vec<&str>>()[..] {
405                        args.mapdir.push((alias.to_string(), real_dir.to_string()));
406                    } else if let [alias, real_dir] = value.split(':').collect::<Vec<&str>>()[..] {
407                        // And then we try splitting by `:` (for compatibility with previous API)
408                        args.mapdir.push((alias.to_string(), real_dir.to_string()));
409                    } else {
410                        eprintln!("Parse error in mapdir {value} not parsed correctly");
411                    }
412                }
413                "env" => {
414                    if let [name, val] = value.split('=').collect::<Vec<&str>>()[..] {
415                        args.env.push((name.to_string(), val.to_string()));
416                    } else {
417                        eprintln!("Parse error in env {value} not parsed correctly");
418                    }
419                }
420                "dir" => {
421                    args.dir.push(value.to_string());
422                }
423                "arg" => {
424                    args.args.push(value.to_string());
425                }
426                "tempdir" => {
427                    args.tempdir.push(value.to_string());
428                }
429                "stdin" => {
430                    assert!(
431                        args.stdin.is_none(),
432                        "Only the first `stdin` directive is used! Please correct this or update this code"
433                    );
434                    let s = value;
435                    let s = s.strip_prefix('"').expect("expected leading '\"' in stdin");
436                    let s = s
437                        .trim_end()
438                        .strip_suffix('\"')
439                        .expect("expected trailing '\"' in stdin");
440                    args.stdin = Some(s.to_string());
441                }
442                e => {
443                    eprintln!("WARN: comment arg: `{e}` is not supported");
444                }
445            }
446        }
447        return Some(args);
448    }
449    None
450}
wasi_test_generator/wasitests.rs

wasi_test_generator/
wasitests.rs
