From 0dfa9a8bfaa02743e6569db92c077492601a5ec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 18 Nov 2022 13:54:30 +0100 Subject: [PATCH 01/88] Add wasmer init --- Cargo.lock | 1 + lib/cli/src/cli.rs | 5 + lib/cli/src/commands.rs | 1 + lib/cli/src/commands/init.rs | 260 +++++++++++++++++++++++++++++++++++ 4 files changed, 267 insertions(+) create mode 100644 lib/cli/src/commands/init.rs diff --git a/Cargo.lock b/Cargo.lock index 8ffacca1335..25a5505e2be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4074,6 +4074,7 @@ dependencies = [ "prettytable-rs", "regex", "reqwest", + "semver 1.0.14", "serde", "serde_json", "spinoff", diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 3b9b8cea6f3..7450a39e50a 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -135,6 +135,10 @@ enum WasmerCLIOptions { /// Inspect a WebAssembly file Inspect(Inspect), + /// Initializes a new wapm.toml file + #[clap(name = "init")] + Init(Init), + /// Run spec testsuite #[cfg(feature = "wast")] Wast(Wast), @@ -165,6 +169,7 @@ impl WasmerCLIOptions { Self::CreateObj(create_obj) => create_obj.execute(), Self::Config(config) => config.execute(), Self::Inspect(inspect) => inspect.execute(), + Self::Init(init) => init.execute(), Self::List(list) => list.execute(), Self::Login(login) => login.execute(), #[cfg(feature = "wast")] diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index d59d5c78cf4..6fbb720f493 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -10,6 +10,7 @@ mod config; mod create_exe; #[cfg(feature = "static-artifact-create")] mod create_obj; +mod init; mod inspect; mod list; mod login; diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs new file mode 100644 index 00000000000..02c6a872257 --- /dev/null +++ b/lib/cli/src/commands/init.rs @@ -0,0 +1,260 @@ +use clap::Parser; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; + +/// CLI args for the `wasmer init` command +#[derive(Debug, Parser)] +pub struct Init { + /// Name of the package, defaults to the name of the init directory + #[clap(name = "PACKAGE_NAME")] + pub package_name: Option, + /// Path to the directory to init the wapm.toml in + #[clap(long, name = "dir", env = "DIR", parse(from_os_str))] + pub dir: Option, + /// Initialize wapm.toml for a library package + #[clap(long, name = "lib")] + pub lib: Option, + /// Initialize wapm.toml for a binary package + #[clap(long, name = "bin")] + pub bin: Option, + /// Add default dependencies for common packages (currently supported: `python`, `js`) + #[clap(long, name = "template")] + pub template: Option, + /// Include file paths into the target container filesystem + #[clap(long)] + #[clap(long, name = "include")] + pub include: Vec, +} + +#[derive(PartialEq, Copy, Clone)] +enum BinOrLib { + Bin, + Lib, +} + +impl Init { + /// `wasmer init` execution + pub fn execute(&self) -> Result<(), anyhow::Error> { + let package_name = match self.package_name.as_ref() { + None => std::env::current_dir()? + .file_stem() + .ok_or_else(|| anyhow!("current dir has no file stem"))? + .to_str() + .ok_or_else(|| anyhow!("current dir has no file stem"))? + .to_string(), + Some(s) => s.to_string(), + }; + + let target_file = match self.dir.as_ref() { + None => std::env::current_dir()?.join("wapm.toml"), + Some(s) => { + if !s.exists() { + let _ = std::fs::create_dir_all(s); + } + s.join("wapm.toml") + } + }; + + // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. + let cargo_toml = target_file.parent().and_then(|p| { + let file = std::fs::read_to_string(p.join("Cargo.toml")).ok()?; + file.parse::().ok() + }); + let version = cargo_toml + .as_ref() + .and_then(|toml| { + semver::Version::parse(toml.get("package")?.as_table()?.get("version")?.as_str()?) + .ok() + }) + .unwrap_or(semver::Version::parse("0.1.0").unwrap()); + + let license = cargo_toml.as_ref().and_then(|toml| { + Some( + toml.get("package")? + .as_table()? + .get("license")? + .as_str()? + .to_string(), + ) + }); + + let license_file = cargo_toml.as_ref().and_then(|toml| { + Some( + Path::new( + toml.get("package")? + .as_table()? + .get("license_file")? + .as_str()?, + ) + .to_path_buf(), + ) + }); + + let readme = cargo_toml.as_ref().and_then(|toml| { + Some(Path::new(toml.get("package")?.as_table()?.get("readme")?.as_str()?).to_path_buf()) + }); + + let repository = cargo_toml.as_ref().and_then(|toml| { + Some( + toml.get("package")? + .as_table()? + .get("repository")? + .as_str()? + .to_string(), + ) + }); + + let homepage = cargo_toml.as_ref().and_then(|toml| { + Some( + toml.get("package")? + .as_table()? + .get("homepage")? + .as_str()? + .to_string(), + ) + }); + + let description = cargo_toml + .as_ref() + .and_then(|toml| { + Some( + toml.get("package")? + .as_table()? + .get("description")? + .as_str()? + .to_string(), + ) + }) + .unwrap_or_else(|| format!("Description of the package {package_name}")); + + let bin_or_lib = match (self.bin, self.lib) { + (Some(true), Some(true)) => { + return Err(anyhow::anyhow!( + "cannot initialize a wapm manifest with both --bin and --lib, pick one" + )) + } + (Some(true), _) => BinOrLib::Bin, + (_, Some(true)) => BinOrLib::Lib, + _ => BinOrLib::Bin, + }; + + let module_name = package_name.split("/").next().unwrap_or(&package_name); + + let modules = vec![wapm_toml::Module { + name: module_name.to_string(), + source: if cargo_toml.is_some() { + Path::new(&format!("target/release/wasm32-wasi/{module_name}.wasm")).to_path_buf() + } else { + Path::new(&format!("{module_name}.wasm")).to_path_buf() + }, + kind: None, + abi: wapm_toml::Abi::Wasi, + bindings: match bin_or_lib { + BinOrLib::Bin => None, + BinOrLib::Lib => target_file.parent().and_then(|parent| { + walkdir::WalkDir::new(parent) + .min_depth(1) + .max_depth(3) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| { + e.path().extension().and_then(|s| s.to_str()) == Some(".wit") + || e.path().extension().and_then(|s| s.to_str()) == Some(".wai") + }) + .map(|e| wapm_toml::Bindings { + wit_exports: e.path().to_path_buf(), + wit_bindgen: semver::Version::parse("0.1.0").unwrap(), + }) + .next() + }), + }, + interfaces: Some({ + let mut map = HashMap::new(); + map.insert("wasi".to_string(), "0.1.0-unstable".to_string()); + map + }), + }]; + + let default_manifest = wapm_toml::Manifest { + package: wapm_toml::Package { + name: package_name.clone(), + version: version, + description, + license: license, + license_file: license_file, + readme: readme, + repository: repository, + homepage: homepage, + wasmer_extra_flags: None, + disable_command_rename: false, + rename_commands_to_raw_command_name: false, + }, + dependencies: Some({ + match self.template.as_deref() { + Some("js") => { + let mut map = HashMap::default(); + map.insert("python".to_string(), "quickjs/quickjs@latest".to_string()); + map + } + Some("python") => { + let mut map = HashMap::default(); + map.insert("python".to_string(), "python/python@latest".to_string()); + map + } + _ => HashMap::default(), + } + }), + command: match bin_or_lib { + BinOrLib::Bin => Some( + modules + .iter() + .map(|m| { + wapm_toml::Command::V1(wapm_toml::CommandV1 { + name: m.name.clone(), + module: m.name.clone(), + main_args: None, + package: None, + }) + }) + .collect(), + ), + BinOrLib::Lib => None, + }, + module: Some(modules), + fs: if self.include.is_empty() { + None + } else { + Some( + self.include + .iter() + .filter_map(|path| { + let path = Path::new(path); + if !path.exists() { + return None; + } + Some(( + format!("{}", path.display()), + Path::new(&format!("/{}", path.display())).to_path_buf(), + )) + }) + .collect(), + ) + }, + base_directory_path: target_file + .parent() + .map(|o| o.to_path_buf()) + .unwrap_or(target_file.clone()), + }; + + println!( + "package: {package_name:?} target: {} - {:?}, wapm_toml = {}", + target_file.display(), + self, + toml::to_string_pretty(&default_manifest).unwrap_or_default(), + ); + + Ok(()) + } +} From 41a311df52b25537be660fa35a749112a0ec8e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 18 Nov 2022 15:25:49 +0100 Subject: [PATCH 02/88] Fix wapm.toml definition --- .gitignore | 1 + lib/cli/src/commands/init.rs | 118 +++++++++++++++++++++-------------- 2 files changed, 73 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 24affcea4df..da8d8e39bb9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ api-docs-repo/ /src/windows-installer/WasmerInstaller.exe /lib/c-api/wasmer.h .xwin-cache +wapm.toml # Generated by tests on Android /avd /core diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 02c6a872257..c346502b319 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -6,56 +6,64 @@ use std::path::PathBuf; /// CLI args for the `wasmer init` command #[derive(Debug, Parser)] pub struct Init { - /// Name of the package, defaults to the name of the init directory - #[clap(name = "PACKAGE_NAME")] - pub package_name: Option, - /// Path to the directory to init the wapm.toml in - #[clap(long, name = "dir", env = "DIR", parse(from_os_str))] - pub dir: Option, /// Initialize wapm.toml for a library package #[clap(long, name = "lib")] - pub lib: Option, + pub lib: bool, /// Initialize wapm.toml for a binary package #[clap(long, name = "bin")] - pub bin: Option, + pub bin: bool, + /// Initialize an empty wapm.toml + #[clap(long, name = "empty")] + pub empty: bool, + /// Path to the directory to init the wapm.toml in + #[clap(long, name = "dir", env = "DIR", parse(from_os_str))] + pub dir: Option, /// Add default dependencies for common packages (currently supported: `python`, `js`) #[clap(long, name = "template")] pub template: Option, /// Include file paths into the target container filesystem - #[clap(long)] #[clap(long, name = "include")] - pub include: Vec, + pub include: Vec, + /// Name of the package, defaults to the name of the init directory + #[clap(name = "PACKAGE_NAME")] + pub package_name: Option, } -#[derive(PartialEq, Copy, Clone)] +#[derive(Debug, PartialEq, Copy, Clone)] enum BinOrLib { Bin, Lib, + Empty, } impl Init { /// `wasmer init` execution pub fn execute(&self) -> Result<(), anyhow::Error> { - let package_name = match self.package_name.as_ref() { - None => std::env::current_dir()? - .file_stem() - .ok_or_else(|| anyhow!("current dir has no file stem"))? - .to_str() - .ok_or_else(|| anyhow!("current dir has no file stem"))? - .to_string(), - Some(s) => s.to_string(), - }; let target_file = match self.dir.as_ref() { None => std::env::current_dir()?.join("wapm.toml"), Some(s) => { - if !s.exists() { - let _ = std::fs::create_dir_all(s); - } + let _ = std::fs::create_dir_all(s); s.join("wapm.toml") } }; + let package_name = match self.package_name.as_ref() { + None => { + if let Some(parent) = target_file.parent().and_then(|p| Some(p.file_stem()?.to_str()?.to_string())) { + parent + } else { + std::env::current_dir()? + .file_stem() + .ok_or_else(|| anyhow!("current dir has no file stem"))? + .to_str() + .ok_or_else(|| anyhow!("current dir has no file stem"))? + .to_string() + } + }, + Some(s) => s.to_string(), + }; + // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. let cargo_toml = target_file.parent().and_then(|p| { let file = std::fs::read_to_string(p.join("Cargo.toml")).ok()?; @@ -128,14 +136,21 @@ impl Init { }) .unwrap_or_else(|| format!("Description of the package {package_name}")); - let bin_or_lib = match (self.bin, self.lib) { - (Some(true), Some(true)) => { + let bin_or_lib = match (self.empty, self.bin, self.lib) { + (true, true, _) | + (true, _, true) => { + return Err(anyhow::anyhow!( + "cannot combine --empty with --bin or --lib" + )) + }, + (true, false, false) => BinOrLib::Empty, + (_, true, true) => { return Err(anyhow::anyhow!( "cannot initialize a wapm manifest with both --bin and --lib, pick one" )) - } - (Some(true), _) => BinOrLib::Bin, - (_, Some(true)) => BinOrLib::Lib, + }, + (false, true, _) => BinOrLib::Bin, + (false, _, true) => BinOrLib::Lib, _ => BinOrLib::Bin, }; @@ -144,14 +159,14 @@ impl Init { let modules = vec![wapm_toml::Module { name: module_name.to_string(), source: if cargo_toml.is_some() { - Path::new(&format!("target/release/wasm32-wasi/{module_name}.wasm")).to_path_buf() + Path::new(&format!("target/wasm32-wasi/release/{module_name}.wasm")).to_path_buf() } else { Path::new(&format!("{module_name}.wasm")).to_path_buf() }, kind: None, abi: wapm_toml::Abi::Wasi, bindings: match bin_or_lib { - BinOrLib::Bin => None, + BinOrLib::Bin | BinOrLib::Empty => None, BinOrLib::Lib => target_file.parent().and_then(|parent| { walkdir::WalkDir::new(parent) .min_depth(1) @@ -220,24 +235,30 @@ impl Init { }) .collect(), ), - BinOrLib::Lib => None, + BinOrLib::Lib | BinOrLib::Empty => None, + }, + module: match bin_or_lib { + BinOrLib::Empty => None, + _ => Some(modules), }, - module: Some(modules), fs: if self.include.is_empty() { None } else { Some( self.include .iter() - .filter_map(|path| { - let path = Path::new(path); - if !path.exists() { - return None; + .map(|path| { + if path == "." || path == "/" { + ( + "/".to_string(), + Path::new("/").to_path_buf(), + ) + } else { + ( + format!("./{path}"), + Path::new(&format!("/{path}")).to_path_buf(), + ) } - Some(( - format!("{}", path.display()), - Path::new(&format!("/{}", path.display())).to_path_buf(), - )) }) .collect(), ) @@ -248,12 +269,17 @@ impl Init { .unwrap_or(target_file.clone()), }; - println!( - "package: {package_name:?} target: {} - {:?}, wapm_toml = {}", - target_file.display(), - self, - toml::to_string_pretty(&default_manifest).unwrap_or_default(), - ); + if let Some(parent) = target_file.parent() { + let _ = std::fs::create_dir_all(parent); + } + + let toml_string = toml::to_string_pretty(&default_manifest)? + .replace("[dependencies]", "# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest\r\n\r\n[dependencies]") + .lines() + .collect::>() + .join("\r\n"); + + std::fs::write(target_file, &toml_string)?; Ok(()) } From e2126b593b4dccaa457ecc679dc798406b452129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 18 Nov 2022 16:08:29 +0100 Subject: [PATCH 03/88] Detect cargo wapm and add metadata to Cargo.toml, if applicable --- lib/cli/src/commands/init.rs | 106 +++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 23 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index c346502b319..905554560f2 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -12,7 +12,7 @@ pub struct Init { /// Initialize wapm.toml for a binary package #[clap(long, name = "bin")] pub bin: bool, - /// Initialize an empty wapm.toml + /// Initialize an empty wapm.toml #[clap(long, name = "empty")] pub empty: bool, /// Path to the directory to init the wapm.toml in @@ -39,7 +39,6 @@ enum BinOrLib { impl Init { /// `wasmer init` execution pub fn execute(&self) -> Result<(), anyhow::Error> { - let target_file = match self.dir.as_ref() { None => std::env::current_dir()?.join("wapm.toml"), Some(s) => { @@ -48,19 +47,29 @@ impl Init { } }; + if target_file.exists() { + return Err(anyhow::anyhow!( + "wapm project already initialized in {}", + target_file.display() + )); + } + let package_name = match self.package_name.as_ref() { None => { - if let Some(parent) = target_file.parent().and_then(|p| Some(p.file_stem()?.to_str()?.to_string())) { + if let Some(parent) = target_file + .parent() + .and_then(|p| Some(p.file_stem()?.to_str()?.to_string())) + { parent } else { std::env::current_dir()? - .file_stem() - .ok_or_else(|| anyhow!("current dir has no file stem"))? - .to_str() - .ok_or_else(|| anyhow!("current dir has no file stem"))? - .to_string() + .file_stem() + .ok_or_else(|| anyhow!("current dir has no file stem"))? + .to_str() + .ok_or_else(|| anyhow!("current dir has no file stem"))? + .to_string() } - }, + } Some(s) => s.to_string(), }; @@ -137,18 +146,17 @@ impl Init { .unwrap_or_else(|| format!("Description of the package {package_name}")); let bin_or_lib = match (self.empty, self.bin, self.lib) { - (true, true, _) | - (true, _, true) => { + (true, true, _) | (true, _, true) => { return Err(anyhow::anyhow!( "cannot combine --empty with --bin or --lib" )) - }, + } (true, false, false) => BinOrLib::Empty, (_, true, true) => { return Err(anyhow::anyhow!( "cannot initialize a wapm manifest with both --bin and --lib, pick one" )) - }, + } (false, true, _) => BinOrLib::Bin, (false, _, true) => BinOrLib::Lib, _ => BinOrLib::Bin, @@ -249,10 +257,7 @@ impl Init { .iter() .map(|path| { if path == "." || path == "/" { - ( - "/".to_string(), - Path::new("/").to_path_buf(), - ) + ("/".to_string(), Path::new("/").to_path_buf()) } else { ( format!("./{path}"), @@ -273,13 +278,68 @@ impl Init { let _ = std::fs::create_dir_all(parent); } - let toml_string = toml::to_string_pretty(&default_manifest)? - .replace("[dependencies]", "# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest\r\n\r\n[dependencies]") - .lines() - .collect::>() - .join("\r\n"); + let cargo_wapm_stdout = std::process::Command::new("cargo") + .arg("wapm") + .arg("--version") + .output() + .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) + .unwrap_or_default(); + + let cargo_wapm_present = + cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); + let should_add_to_cargo_toml = cargo_toml.is_some() && cargo_wapm_present; - std::fs::write(target_file, &toml_string)?; + if !cargo_wapm_present && cargo_toml.is_some() { + eprintln!( + "Note: you seem to have a Cargo.toml file, but you haven't installed `cargo wapm`." + ); + eprintln!("You can build and release Rust projects directly with `cargo wapm publish`: https://crates.io/crates/cargo-wapm"); + eprintln!("Install it with:"); + eprintln!(" cargo install cargo-wapm"); + eprintln!(); + } + + if should_add_to_cargo_toml { + let toml_string = toml::to_string_pretty(&default_manifest)?; + let mut value = toml_string.parse::().unwrap(); + value.as_table_mut().unwrap().remove("package"); + + let mut new_table = toml::value::Table::new(); + new_table.insert("wapm".to_string(), value); + + let mut table_2 = toml::value::Table::new(); + table_2.insert("metadata".to_string(), new_table.into()); + + let toml_string = toml::to_string_pretty(&table_2)?; + + let cargo_toml_path = target_file.parent().unwrap().join("Cargo.toml"); + + let old_cargo = std::fs::read_to_string(&cargo_toml_path).unwrap(); + + if old_cargo.contains("metadata.wapm") { + return Err(anyhow::anyhow!( + "wapm project already initialized in Cargo.toml file" + )); + } else { + eprintln!("You have cargo-wapm installed, added metadata to Cargo.toml instead of wapm.toml"); + eprintln!("Build and publish your package with:"); + eprintln!(); + eprintln!(" cargo wapm publish"); + eprintln!(); + std::fs::write( + &cargo_toml_path, + &format!("{old_cargo}\r\n\r\n# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest\r\n\r\n{toml_string}"), + )?; + } + } else { + let toml_string = toml::to_string_pretty(&default_manifest)? + .replace("[dependencies]", "# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest\r\n\r\n[dependencies]") + .lines() + .collect::>() + .join("\r\n"); + + std::fs::write(target_file, &toml_string)?; + } Ok(()) } From 248b2f20f89d2ecc56c88edbfc4ae1623b55b16e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 18 Nov 2022 16:59:12 +0100 Subject: [PATCH 04/88] Fix make lint --- lib/cli/src/commands/init.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 905554560f2..d7202a750ca 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -84,7 +84,7 @@ impl Init { semver::Version::parse(toml.get("package")?.as_table()?.get("version")?.as_str()?) .ok() }) - .unwrap_or(semver::Version::parse("0.1.0").unwrap()); + .unwrap_or_else(|| semver::Version::parse("0.1.0").unwrap()); let license = cargo_toml.as_ref().and_then(|toml| { Some( @@ -162,7 +162,7 @@ impl Init { _ => BinOrLib::Bin, }; - let module_name = package_name.split("/").next().unwrap_or(&package_name); + let module_name = package_name.split('/').next().unwrap_or(&package_name); let modules = vec![wapm_toml::Module { name: module_name.to_string(), @@ -203,13 +203,13 @@ impl Init { let default_manifest = wapm_toml::Manifest { package: wapm_toml::Package { name: package_name.clone(), - version: version, + version, description, - license: license, - license_file: license_file, - readme: readme, - repository: repository, - homepage: homepage, + license, + license_file, + readme, + repository, + homepage, wasmer_extra_flags: None, disable_command_rename: false, rename_commands_to_raw_command_name: false, @@ -271,7 +271,7 @@ impl Init { base_directory_path: target_file .parent() .map(|o| o.to_path_buf()) - .unwrap_or(target_file.clone()), + .unwrap_or_else(|| target_file.clone()), }; if let Some(parent) = target_file.parent() { From 73c63873361a2c1f8aff5b6978aecd04d3cd7776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 11:51:41 +0100 Subject: [PATCH 05/88] Fix wapm init compilation --- Cargo.lock | 41 +++++++++++++++++++++++++++++++++--- lib/cli/Cargo.toml | 2 +- lib/cli/src/commands/init.rs | 25 ++++++++++++++++------ 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25a5505e2be..32390666e24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3008,6 +3008,19 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "serde_yaml" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d232d893b10de3eb7258ff01974d6ee20663d8e833263c99409d4b13a0209da" +dependencies = [ + "indexmap", + "itoa 1.0.4", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serial_test" version = "0.5.1" @@ -3695,6 +3708,12 @@ dependencies = [ "void", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e5fa573d8ac5f1a856f8d7be41d390ee973daf97c806b2c1a465e4e1406e68" + [[package]] name = "untrusted" version = "0.7.1" @@ -3773,7 +3792,23 @@ dependencies = [ "serde_cbor", "serde_derive", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", + "thiserror", + "toml", +] + +[[package]] +name = "wapm-toml" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a52a44eb6e8ede075e12c491bbe0422beee61febb23b86dbafda4badcf8db576" +dependencies = [ + "anyhow", + "semver 1.0.14", + "serde", + "serde_cbor", + "serde_json", + "serde_yaml 0.9.14", "thiserror", "toml", ] @@ -4086,7 +4121,7 @@ dependencies = [ "unix_mode", "url", "walkdir", - "wapm-toml", + "wapm-toml 0.3.2", "wasmer", "wasmer-cache", "wasmer-compiler", @@ -4328,7 +4363,7 @@ dependencies = [ "tokio", "toml", "url", - "wapm-toml", + "wapm-toml 0.2.2", "webc", "whoami", ] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index d927cedc2cd..85a7e293e26 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -62,7 +62,7 @@ dirs = { version = "4.0" } serde_json = { version = "1.0" } target-lexicon = { version = "0.12", features = ["std"] } prettytable-rs = "0.9.0" -wapm-toml = "0.2.0" +wapm-toml = "0.3.2" walkdir = "2.3.2" regex = "1.6.0" toml = "0.5.9" diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index d7202a750ca..ee6b03ed9d2 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -182,13 +182,24 @@ impl Init { .follow_links(false) .into_iter() .filter_map(|e| e.ok()) - .filter(|e| { - e.path().extension().and_then(|s| s.to_str()) == Some(".wit") - || e.path().extension().and_then(|s| s.to_str()) == Some(".wai") - }) - .map(|e| wapm_toml::Bindings { - wit_exports: e.path().to_path_buf(), - wit_bindgen: semver::Version::parse("0.1.0").unwrap(), + .filter_map(|e| { + let is_wit = + e.path().extension().and_then(|s| s.to_str()) == Some(".wit"); + let is_wai = + e.path().extension().and_then(|s| s.to_str()) == Some(".wai"); + if is_wit { + Some(wapm_toml::Bindings::Wit(wapm_toml::WitBindings { + wit_exports: e.path().to_path_buf(), + wit_bindgen: semver::Version::parse("0.1.0").unwrap(), + })) + } else if is_wai { + Some(wapm_toml::Bindings::Wit(wapm_toml::WitBindings { + wit_exports: e.path().to_path_buf(), + wit_bindgen: semver::Version::parse("0.1.0").unwrap(), + })) + } else { + None + } }) .next() }), From a29cc7dedec156c06fec507c48271610447a60ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 13:44:01 +0100 Subject: [PATCH 06/88] wasmer init: migrate to cargo_metadata --- Cargo.lock | 33 +++++++ lib/cli/src/cli.rs | 4 +- lib/cli/src/commands/init.rs | 179 ++++++++++++++++++----------------- 3 files changed, 127 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32390666e24..43d6225860f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,6 +264,38 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" +[[package]] +name = "camino" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ad0e1e3e88dd237a156ab9f571021b8a158caa0ae44b1968a241efb5144c1e" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "406c859255d568f4f742b3146d51851f3bfd49f734a2c289d9107c4395ee0062" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.14", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cast" version = "0.3.0" @@ -4092,6 +4124,7 @@ dependencies = [ "anyhow", "atty", "bytesize", + "cargo_metadata", "cfg-if 1.0.0", "chrono", "clap 3.2.23", diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 7450a39e50a..1679fb3d4c5 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -229,8 +229,8 @@ fn wasmer_main_inner() -> Result<(), anyhow::Error> { WasmerCLIOptions::Run(Run::from_binfmt_args()) } else { match command.unwrap_or(&"".to_string()).as_ref() { - "add" | "cache" | "compile" | "config" | "create-exe" | "help" | "inspect" | "run" - | "self-update" | "validate" | "wast" | "binfmt" | "list" | "login" => { + "add" | "cache" | "compile" | "config" | "create-exe" | "help" | "inspect" | "init" + | "run" | "self-update" | "validate" | "wast" | "binfmt" | "list" | "login" => { WasmerCLIOptions::parse() } _ => { diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index ee6b03ed9d2..78c081d40ab 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -1,3 +1,4 @@ +use cargo_metadata::{CargoOpt, MetadataCommand}; use clap::Parser; use std::collections::HashMap; use std::path::Path; @@ -15,9 +16,18 @@ pub struct Init { /// Initialize an empty wapm.toml #[clap(long, name = "empty")] pub empty: bool, - /// Path to the directory to init the wapm.toml in - #[clap(long, name = "dir", env = "DIR", parse(from_os_str))] - pub dir: Option, + /// If the `manifest-dir` contains a Cargo.toml, use that file to initialize the wapm.toml + #[clap(long, name = "manifest-dir")] + pub manifest_dir: Option, + /// Directory of the output file name. wasmer init will error in the target dir already contains a wapm.toml + #[clap(long, short = 'o', name = "out")] + pub out: Option, + /// Force overwriting the wapm.toml, even if it already exists + #[clap(long, name = "overwrite")] + pub overwrite: bool, + /// Don't display debug output + #[clap(long, name = "quiet")] + pub quiet: bool, /// Add default dependencies for common packages (currently supported: `python`, `js`) #[clap(long, name = "template")] pub template: Option, @@ -36,10 +46,26 @@ enum BinOrLib { Empty, } +// minimal version of the Cargo.toml [package] section +struct MiniCargoTomlPackage { + name: String, + version: semver::Version, + description: Option, + homepage: Option, + repository: Option, + license: Option, + readme: Option, + license_file: Option, + #[allow(dead_code)] + workspace_root: PathBuf, + #[allow(dead_code)] + build_dir: PathBuf, +} + impl Init { /// `wasmer init` execution pub fn execute(&self) -> Result<(), anyhow::Error> { - let target_file = match self.dir.as_ref() { + let target_file = match self.out.as_ref() { None => std::env::current_dir()?.join("wapm.toml"), Some(s) => { let _ = std::fs::create_dir_all(s); @@ -47,14 +73,46 @@ impl Init { } }; - if target_file.exists() { + if target_file.exists() && !self.overwrite { return Err(anyhow::anyhow!( "wapm project already initialized in {}", target_file.display() )); } - let package_name = match self.package_name.as_ref() { + // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. + let cargo_toml = MetadataCommand::new() + .manifest_path(match self.manifest_dir.as_ref() { + Some(s) => { + if format!("{}", s.display()).ends_with("Cargo.toml") { + s.clone() + } else { + s.join("Cargo.toml") + } + } + None => Path::new("./Cargo.toml").to_path_buf(), + }) + .features(CargoOpt::AllFeatures) + .exec() + .ok(); + + let cargo_toml = cargo_toml.and_then(|metadata| { + let package = metadata.root_package()?; + Some(MiniCargoTomlPackage { + name: package.name.clone(), + version: package.version.clone(), + description: package.description.clone(), + homepage: package.homepage.clone(), + repository: package.repository.clone(), + license: package.license.clone(), + readme: package.readme.clone().map(|s| s.into_std_path_buf()), + license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), + workspace_root: metadata.workspace_root.into_std_path_buf(), + build_dir: metadata.target_directory.into_std_path_buf(), + }) + }); + + let package_name = match cargo_toml.as_ref() { None => { if let Some(parent) = target_file .parent() @@ -70,80 +128,23 @@ impl Init { .to_string() } } - Some(s) => s.to_string(), + Some(s) => s.name.clone(), }; - // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. - let cargo_toml = target_file.parent().and_then(|p| { - let file = std::fs::read_to_string(p.join("Cargo.toml")).ok()?; - file.parse::().ok() - }); + let module_name = package_name.split('/').next().unwrap_or(&package_name); let version = cargo_toml .as_ref() - .and_then(|toml| { - semver::Version::parse(toml.get("package")?.as_table()?.get("version")?.as_str()?) - .ok() - }) + .map(|t| t.version.clone()) .unwrap_or_else(|| semver::Version::parse("0.1.0").unwrap()); - - let license = cargo_toml.as_ref().and_then(|toml| { - Some( - toml.get("package")? - .as_table()? - .get("license")? - .as_str()? - .to_string(), - ) - }); - - let license_file = cargo_toml.as_ref().and_then(|toml| { - Some( - Path::new( - toml.get("package")? - .as_table()? - .get("license_file")? - .as_str()?, - ) - .to_path_buf(), - ) - }); - - let readme = cargo_toml.as_ref().and_then(|toml| { - Some(Path::new(toml.get("package")?.as_table()?.get("readme")?.as_str()?).to_path_buf()) - }); - - let repository = cargo_toml.as_ref().and_then(|toml| { - Some( - toml.get("package")? - .as_table()? - .get("repository")? - .as_str()? - .to_string(), - ) - }); - - let homepage = cargo_toml.as_ref().and_then(|toml| { - Some( - toml.get("package")? - .as_table()? - .get("homepage")? - .as_str()? - .to_string(), - ) - }); - + let license = cargo_toml.as_ref().and_then(|t| t.license.clone()); + let license_file = cargo_toml.as_ref().and_then(|t| t.license_file.clone()); + let readme = cargo_toml.as_ref().and_then(|t| t.readme.clone()); + let repository = cargo_toml.as_ref().and_then(|t| t.repository.clone()); + let homepage = cargo_toml.as_ref().and_then(|t| t.homepage.clone()); let description = cargo_toml .as_ref() - .and_then(|toml| { - Some( - toml.get("package")? - .as_table()? - .get("description")? - .as_str()? - .to_string(), - ) - }) - .unwrap_or_else(|| format!("Description of the package {package_name}")); + .and_then(|t| t.description.clone()) + .unwrap_or(format!("Description for package {module_name}")); let bin_or_lib = match (self.empty, self.bin, self.lib) { (true, true, _) | (true, _, true) => { @@ -162,15 +163,16 @@ impl Init { _ => BinOrLib::Bin, }; - let module_name = package_name.split('/').next().unwrap_or(&package_name); - let modules = vec![wapm_toml::Module { name: module_name.to_string(), - source: if cargo_toml.is_some() { - Path::new(&format!("target/wasm32-wasi/release/{module_name}.wasm")).to_path_buf() - } else { - Path::new(&format!("{module_name}.wasm")).to_path_buf() - }, + source: cargo_toml + .as_ref() + .map(|p| { + p.build_dir + .join("release") + .join(&format!("{module_name}.wasm")) + }) + .unwrap_or_else(|| Path::new(&format!("{module_name}.wasm")).to_path_buf()), kind: None, abi: wapm_toml::Abi::Wasi, bindings: match bin_or_lib { @@ -300,12 +302,13 @@ impl Init { cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); let should_add_to_cargo_toml = cargo_toml.is_some() && cargo_wapm_present; - if !cargo_wapm_present && cargo_toml.is_some() { + if !cargo_wapm_present && cargo_toml.is_some() && !self.quiet { eprintln!( "Note: you seem to have a Cargo.toml file, but you haven't installed `cargo wapm`." ); eprintln!("You can build and release Rust projects directly with `cargo wapm publish`: https://crates.io/crates/cargo-wapm"); eprintln!("Install it with:"); + eprintln!(); eprintln!(" cargo install cargo-wapm"); eprintln!(); } @@ -327,16 +330,18 @@ impl Init { let old_cargo = std::fs::read_to_string(&cargo_toml_path).unwrap(); - if old_cargo.contains("metadata.wapm") { + if old_cargo.contains("metadata.wapm") && !self.overwrite { return Err(anyhow::anyhow!( "wapm project already initialized in Cargo.toml file" )); } else { - eprintln!("You have cargo-wapm installed, added metadata to Cargo.toml instead of wapm.toml"); - eprintln!("Build and publish your package with:"); - eprintln!(); - eprintln!(" cargo wapm publish"); - eprintln!(); + if !self.quiet { + eprintln!("You have cargo-wapm installed, added metadata to Cargo.toml instead of wapm.toml"); + eprintln!("Build and publish your package with:"); + eprintln!(); + eprintln!(" cargo wapm publish"); + eprintln!(); + } std::fs::write( &cargo_toml_path, &format!("{old_cargo}\r\n\r\n# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest\r\n\r\n{toml_string}"), From 507b4f1a7a710e7da751227fffea859b120d912c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 13:55:47 +0100 Subject: [PATCH 07/88] Correctly recognize .wai bindings --- lib/cli/src/commands/init.rs | 42 +++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 78c081d40ab..dcf8c0a7649 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -65,6 +65,24 @@ struct MiniCargoTomlPackage { impl Init { /// `wasmer init` execution pub fn execute(&self) -> Result<(), anyhow::Error> { + + let bin_or_lib = match (self.empty, self.bin, self.lib) { + (true, true, _) | (true, _, true) => { + return Err(anyhow::anyhow!( + "cannot combine --empty with --bin or --lib" + )) + } + (true, false, false) => BinOrLib::Empty, + (_, true, true) => { + return Err(anyhow::anyhow!( + "cannot initialize a wapm manifest with both --bin and --lib, pick one" + )) + } + (false, true, _) => BinOrLib::Bin, + (false, _, true) => BinOrLib::Lib, + _ => BinOrLib::Bin, + }; + let target_file = match self.out.as_ref() { None => std::env::current_dir()?.join("wapm.toml"), Some(s) => { @@ -146,23 +164,6 @@ impl Init { .and_then(|t| t.description.clone()) .unwrap_or(format!("Description for package {module_name}")); - let bin_or_lib = match (self.empty, self.bin, self.lib) { - (true, true, _) | (true, _, true) => { - return Err(anyhow::anyhow!( - "cannot combine --empty with --bin or --lib" - )) - } - (true, false, false) => BinOrLib::Empty, - (_, true, true) => { - return Err(anyhow::anyhow!( - "cannot initialize a wapm manifest with both --bin and --lib, pick one" - )) - } - (false, true, _) => BinOrLib::Bin, - (false, _, true) => BinOrLib::Lib, - _ => BinOrLib::Bin, - }; - let modules = vec![wapm_toml::Module { name: module_name.to_string(), source: cargo_toml @@ -195,9 +196,10 @@ impl Init { wit_bindgen: semver::Version::parse("0.1.0").unwrap(), })) } else if is_wai { - Some(wapm_toml::Bindings::Wit(wapm_toml::WitBindings { - wit_exports: e.path().to_path_buf(), - wit_bindgen: semver::Version::parse("0.1.0").unwrap(), + Some(wapm_toml::Bindings::Wai(wapm_toml::WaiBindings { + exports: None, + imports: vec![e.path().to_path_buf()], + wai_version: semver::Version::parse("0.1.0").unwrap(), })) } else { None From bb3de1fd0efb0aa0a27430128f5771e363c240d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 14:03:16 +0100 Subject: [PATCH 08/88] cargo fmt --- lib/cli/src/commands/init.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index dcf8c0a7649..c52904f2275 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -65,7 +65,6 @@ struct MiniCargoTomlPackage { impl Init { /// `wasmer init` execution pub fn execute(&self) -> Result<(), anyhow::Error> { - let bin_or_lib = match (self.empty, self.bin, self.lib) { (true, true, _) | (true, _, true) => { return Err(anyhow::anyhow!( @@ -82,7 +81,7 @@ impl Init { (false, _, true) => BinOrLib::Lib, _ => BinOrLib::Bin, }; - + let target_file = match self.out.as_ref() { None => std::env::current_dir()?.join("wapm.toml"), Some(s) => { From 83a4334c7cebf436c36deada7971c53b2655a5cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 15:35:35 +0100 Subject: [PATCH 09/88] cargo update --- Cargo.lock | 6 +++--- lib/cli/src/commands/init.rs | 16 +++++----------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43d6225860f..b61b1a2bd97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3831,9 +3831,9 @@ dependencies = [ [[package]] name = "wapm-toml" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a52a44eb6e8ede075e12c491bbe0422beee61febb23b86dbafda4badcf8db576" +checksum = "bde48580008d83994e59db51dc23d200d91c06d6b82f59a955717d589dac29b5" dependencies = [ "anyhow", "semver 1.0.14", @@ -4154,7 +4154,7 @@ dependencies = [ "unix_mode", "url", "walkdir", - "wapm-toml 0.3.2", + "wapm-toml 0.3.3", "wasmer", "wasmer-cache", "wasmer-compiler", diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index c52904f2275..8a3177d37ca 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -16,9 +16,9 @@ pub struct Init { /// Initialize an empty wapm.toml #[clap(long, name = "empty")] pub empty: bool, - /// If the `manifest-dir` contains a Cargo.toml, use that file to initialize the wapm.toml - #[clap(long, name = "manifest-dir")] - pub manifest_dir: Option, + /// If the `manifest-path` is a Cargo.toml, use that file to initialize the wapm.toml + #[clap(long, name = "manifest-path")] + pub manifest_path: Option, /// Directory of the output file name. wasmer init will error in the target dir already contains a wapm.toml #[clap(long, short = 'o', name = "out")] pub out: Option, @@ -99,14 +99,8 @@ impl Init { // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. let cargo_toml = MetadataCommand::new() - .manifest_path(match self.manifest_dir.as_ref() { - Some(s) => { - if format!("{}", s.display()).ends_with("Cargo.toml") { - s.clone() - } else { - s.join("Cargo.toml") - } - } + .manifest_path(match self.manifest_path.as_ref() { + Some(s) => s.clone(), None => Path::new("./Cargo.toml").to_path_buf(), }) .features(CargoOpt::AllFeatures) From e69ea73c2903df8632b90db949aa13bc97db4c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 15:38:17 +0100 Subject: [PATCH 10/88] Update wapm toml version --- Cargo.lock | 50 +++-------------------------------------- lib/cli/Cargo.toml | 2 +- lib/registry/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b61b1a2bd97..1e251328ae9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1834,12 +1834,6 @@ dependencies = [ "cc", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "llvm-sys" version = "120.2.5" @@ -3028,18 +3022,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" -dependencies = [ - "indexmap", - "ryu", - "serde", - "yaml-rust", -] - [[package]] name = "serde_yaml" version = "0.9.14" @@ -3812,23 +3794,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wapm-toml" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61b6d3b6a2fc171198e6378b3a9b38650e114298775a9e63401613abb6a10b3" -dependencies = [ - "anyhow", - "semver 1.0.14", - "serde", - "serde_cbor", - "serde_derive", - "serde_json", - "serde_yaml 0.8.26", - "thiserror", - "toml", -] - [[package]] name = "wapm-toml" version = "0.3.3" @@ -3840,7 +3805,7 @@ dependencies = [ "serde", "serde_cbor", "serde_json", - "serde_yaml 0.9.14", + "serde_yaml", "thiserror", "toml", ] @@ -4154,7 +4119,7 @@ dependencies = [ "unix_mode", "url", "walkdir", - "wapm-toml 0.3.3", + "wapm-toml", "wasmer", "wasmer-cache", "wasmer-compiler", @@ -4396,7 +4361,7 @@ dependencies = [ "tokio", "toml", "url", - "wapm-toml 0.2.2", + "wapm-toml", "webc", "whoami", ] @@ -5051,15 +5016,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "yansi" version = "0.5.1" diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 85a7e293e26..0185237f1fc 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -62,7 +62,7 @@ dirs = { version = "4.0" } serde_json = { version = "1.0" } target-lexicon = { version = "0.12", features = ["std"] } prettytable-rs = "0.9.0" -wapm-toml = "0.3.2" +wapm-toml = "0.3.3" walkdir = "2.3.2" regex = "1.6.0" toml = "0.5.9" diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index 1213735f6dd..2df56fcdd13 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -20,7 +20,7 @@ serde_json = "1.0.85" url = "2.3.1" thiserror = "1.0.37" toml = "0.5.9" -wapm-toml = "0.2.0" +wapm-toml = "0.3.2" tar = "0.4.38" flate2 = "1.0.24" semver = "1.0.14" From 53a08b47ff30be04b598e92e19eaf92ba59e099a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 19:02:21 +0100 Subject: [PATCH 11/88] Update wapm-toml version again --- Cargo.lock | 4 ++-- lib/cli/Cargo.toml | 2 +- lib/registry/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e251328ae9..66c3f51dbce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3796,9 +3796,9 @@ dependencies = [ [[package]] name = "wapm-toml" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde48580008d83994e59db51dc23d200d91c06d6b82f59a955717d589dac29b5" +checksum = "994ef26447f3158955d2e3fca96021d1f1c47b830e2053569177673dca1447a9" dependencies = [ "anyhow", "semver 1.0.14", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 0185237f1fc..293e91dde14 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -62,7 +62,7 @@ dirs = { version = "4.0" } serde_json = { version = "1.0" } target-lexicon = { version = "0.12", features = ["std"] } prettytable-rs = "0.9.0" -wapm-toml = "0.3.3" +wapm-toml = "0.4.0" walkdir = "2.3.2" regex = "1.6.0" toml = "0.5.9" diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index 2df56fcdd13..8ad0763502c 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -20,7 +20,7 @@ serde_json = "1.0.85" url = "2.3.1" thiserror = "1.0.37" toml = "0.5.9" -wapm-toml = "0.3.2" +wapm-toml = "0.4.0" tar = "0.4.38" flate2 = "1.0.24" semver = "1.0.14" From c4e1422456f8a134957dde4a3e102f54b4fe9986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 21 Nov 2022 19:49:36 +0100 Subject: [PATCH 12/88] Refactor wasmer init with comments --- lib/cli/src/commands/init.rs | 393 ++++++++++++++++++++--------------- 1 file changed, 223 insertions(+), 170 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 8a3177d37ca..f591dc1a860 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -65,22 +65,7 @@ struct MiniCargoTomlPackage { impl Init { /// `wasmer init` execution pub fn execute(&self) -> Result<(), anyhow::Error> { - let bin_or_lib = match (self.empty, self.bin, self.lib) { - (true, true, _) | (true, _, true) => { - return Err(anyhow::anyhow!( - "cannot combine --empty with --bin or --lib" - )) - } - (true, false, false) => BinOrLib::Empty, - (_, true, true) => { - return Err(anyhow::anyhow!( - "cannot initialize a wapm manifest with both --bin and --lib, pick one" - )) - } - (false, true, _) => BinOrLib::Bin, - (false, _, true) => BinOrLib::Lib, - _ => BinOrLib::Bin, - }; + let bin_or_lib = self.get_bin_or_lib()?; let target_file = match self.out.as_ref() { None => std::env::current_dir()?.join("wapm.toml"), @@ -98,51 +83,62 @@ impl Init { } // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. + let manifest_path = match self.manifest_path.as_ref() { + Some(s) => s.clone(), + None => Path::new("./Cargo.toml").to_path_buf(), + }; + let cargo_toml = MetadataCommand::new() - .manifest_path(match self.manifest_path.as_ref() { - Some(s) => s.clone(), - None => Path::new("./Cargo.toml").to_path_buf(), - }) + .manifest_path(&manifest_path) .features(CargoOpt::AllFeatures) .exec() - .ok(); - - let cargo_toml = cargo_toml.and_then(|metadata| { - let package = metadata.root_package()?; - Some(MiniCargoTomlPackage { - name: package.name.clone(), - version: package.version.clone(), - description: package.description.clone(), - homepage: package.homepage.clone(), - repository: package.repository.clone(), - license: package.license.clone(), - readme: package.readme.clone().map(|s| s.into_std_path_buf()), - license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), - workspace_root: metadata.workspace_root.into_std_path_buf(), - build_dir: metadata.target_directory.into_std_path_buf(), - }) - }); + .ok() + .and_then(|metadata| { + let package = metadata.root_package()?; + Some(MiniCargoTomlPackage { + name: package.name.clone(), + version: package.version.clone(), + description: package.description.clone(), + homepage: package.homepage.clone(), + repository: package.repository.clone(), + license: package.license.clone(), + readme: package.readme.clone().map(|s| s.into_std_path_buf()), + license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), + workspace_root: metadata.workspace_root.into_std_path_buf(), + build_dir: metadata.target_directory.into_std_path_buf(), + }) + }); - let package_name = match cargo_toml.as_ref() { - None => { - if let Some(parent) = target_file + let package_name = self + .package_name + .clone() + .or_else(|| cargo_toml.as_ref().map(|p| p.name.clone())) + .or_else(|| { + target_file .parent() .and_then(|p| Some(p.file_stem()?.to_str()?.to_string())) - { - parent - } else { - std::env::current_dir()? - .file_stem() - .ok_or_else(|| anyhow!("current dir has no file stem"))? - .to_str() - .ok_or_else(|| anyhow!("current dir has no file stem"))? - .to_string() - } - } - Some(s) => s.name.clone(), - }; + }) + .or_else(|| { + Some( + std::env::current_dir() + .ok()? + .file_stem()? + .to_str()? + .to_string(), + ) + }) + .ok_or_else(|| anyhow!("current dir has no file stem"))?; - let module_name = package_name.split('/').next().unwrap_or(&package_name); + let namespace = package_name + .split('/') + .next() + .unwrap_or(&package_name) + .to_string(); + let module_name = package_name + .split('/') + .last() + .unwrap_or(&package_name) + .to_string(); let version = cargo_toml .as_ref() .map(|t| t.version.clone()) @@ -157,6 +153,8 @@ impl Init { .and_then(|t| t.description.clone()) .unwrap_or(format!("Description for package {module_name}")); + let default_abi = wapm_toml::Abi::Wasi; + let bindings = Self::get_bindings(&target_file, bin_or_lib); let modules = vec![wapm_toml::Module { name: module_name.to_string(), source: cargo_toml @@ -168,39 +166,8 @@ impl Init { }) .unwrap_or_else(|| Path::new(&format!("{module_name}.wasm")).to_path_buf()), kind: None, - abi: wapm_toml::Abi::Wasi, - bindings: match bin_or_lib { - BinOrLib::Bin | BinOrLib::Empty => None, - BinOrLib::Lib => target_file.parent().and_then(|parent| { - walkdir::WalkDir::new(parent) - .min_depth(1) - .max_depth(3) - .follow_links(false) - .into_iter() - .filter_map(|e| e.ok()) - .filter_map(|e| { - let is_wit = - e.path().extension().and_then(|s| s.to_str()) == Some(".wit"); - let is_wai = - e.path().extension().and_then(|s| s.to_str()) == Some(".wai"); - if is_wit { - Some(wapm_toml::Bindings::Wit(wapm_toml::WitBindings { - wit_exports: e.path().to_path_buf(), - wit_bindgen: semver::Version::parse("0.1.0").unwrap(), - })) - } else if is_wai { - Some(wapm_toml::Bindings::Wai(wapm_toml::WaiBindings { - exports: None, - imports: vec![e.path().to_path_buf()], - wai_version: semver::Version::parse("0.1.0").unwrap(), - })) - } else { - None - } - }) - .next() - }), - }, + abi: default_abi, + bindings: bindings.clone(), interfaces: Some({ let mut map = HashMap::new(); map.insert("wasi".to_string(), "0.1.0-unstable".to_string()); @@ -208,7 +175,7 @@ impl Init { }), }]; - let default_manifest = wapm_toml::Manifest { + let constructed_manifest = wapm_toml::Manifest { package: wapm_toml::Package { name: package_name.clone(), version, @@ -222,60 +189,13 @@ impl Init { disable_command_rename: false, rename_commands_to_raw_command_name: false, }, - dependencies: Some({ - match self.template.as_deref() { - Some("js") => { - let mut map = HashMap::default(); - map.insert("python".to_string(), "quickjs/quickjs@latest".to_string()); - map - } - Some("python") => { - let mut map = HashMap::default(); - map.insert("python".to_string(), "python/python@latest".to_string()); - map - } - _ => HashMap::default(), - } - }), - command: match bin_or_lib { - BinOrLib::Bin => Some( - modules - .iter() - .map(|m| { - wapm_toml::Command::V1(wapm_toml::CommandV1 { - name: m.name.clone(), - module: m.name.clone(), - main_args: None, - package: None, - }) - }) - .collect(), - ), - BinOrLib::Lib | BinOrLib::Empty => None, - }, + dependencies: self.get_dependencies(), + command: Self::get_command(&modules, bin_or_lib), module: match bin_or_lib { BinOrLib::Empty => None, _ => Some(modules), }, - fs: if self.include.is_empty() { - None - } else { - Some( - self.include - .iter() - .map(|path| { - if path == "." || path == "/" { - ("/".to_string(), Path::new("/").to_path_buf()) - } else { - ( - format!("./{path}"), - Path::new(&format!("/{path}")).to_path_buf(), - ) - } - }) - .collect(), - ) - }, + fs: self.get_filesystem_mapping(), base_directory_path: target_file .parent() .map(|o| o.to_path_buf()) @@ -286,6 +206,7 @@ impl Init { let _ = std::fs::create_dir_all(parent); } + // Test if cargo wapm is installed let cargo_wapm_stdout = std::process::Command::new("cargo") .arg("wapm") .arg("--version") @@ -295,8 +216,13 @@ impl Init { let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); + + // if Cargo.toml is present and cargo wapm is installed, add the + // generated manifest to the Cargo.toml instead of creating a new wapm.toml let should_add_to_cargo_toml = cargo_toml.is_some() && cargo_wapm_present; + // If the Cargo.toml is present, but cargo wapm is not installed, + // generate a wapm.toml, but notify the user about installing cargo-wapm if !cargo_wapm_present && cargo_toml.is_some() && !self.quiet { eprintln!( "Note: you seem to have a Cargo.toml file, but you haven't installed `cargo wapm`." @@ -308,50 +234,177 @@ impl Init { eprintln!(); } - if should_add_to_cargo_toml { - let toml_string = toml::to_string_pretty(&default_manifest)?; - let mut value = toml_string.parse::().unwrap(); - value.as_table_mut().unwrap().remove("package"); + let note = + "# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest"; - let mut new_table = toml::value::Table::new(); - new_table.insert("wapm".to_string(), value); + // generate the wapm.toml and exit + if !should_add_to_cargo_toml { + let toml_string = toml::to_string_pretty(&constructed_manifest)? + .replace("[dependencies]", &format!("{note}\r\n\r\n[dependencies]")) + .lines() + .collect::>() + .join("\r\n"); - let mut table_2 = toml::value::Table::new(); - table_2.insert("metadata".to_string(), new_table.into()); + std::fs::write(target_file, &toml_string)?; - let toml_string = toml::to_string_pretty(&table_2)?; + return Ok(()); + } - let cargo_toml_path = target_file.parent().unwrap().join("Cargo.toml"); + // add the manifest to the Cargo.toml + let old_cargo = std::fs::read_to_string(&manifest_path).unwrap(); - let old_cargo = std::fs::read_to_string(&cargo_toml_path).unwrap(); + // if the Cargo.toml already contains a [metadata.wapm] section, don't generate it again + if old_cargo.contains("metadata.wapm") && !self.overwrite { + return Err(anyhow::anyhow!( + "wapm project already initialized in Cargo.toml file" + )); + } - if old_cargo.contains("metadata.wapm") && !self.overwrite { - return Err(anyhow::anyhow!( - "wapm project already initialized in Cargo.toml file" - )); - } else { - if !self.quiet { - eprintln!("You have cargo-wapm installed, added metadata to Cargo.toml instead of wapm.toml"); - eprintln!("Build and publish your package with:"); - eprintln!(); - eprintln!(" cargo wapm publish"); - eprintln!(); - } - std::fs::write( - &cargo_toml_path, - &format!("{old_cargo}\r\n\r\n# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest\r\n\r\n{toml_string}"), - )?; - } - } else { - let toml_string = toml::to_string_pretty(&default_manifest)? - .replace("[dependencies]", "# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest\r\n\r\n[dependencies]") + // generate the Wapm struct for the [metadata.wapm] table + // and add it to the end of the file + let metadata_wapm = wapm_toml::rust::Wapm { + namespace, + package: Some(module_name), + wasmer_extra_flags: None, + abi: default_abi, + fs: constructed_manifest.fs, + bindings: bindings, + }; + + let toml_string = toml::to_string_pretty(&metadata_wapm)? + .replace("[dependencies]", &format!("{note}\r\n\r\n[dependencies]")) .lines() .collect::>() .join("\r\n"); - std::fs::write(target_file, &toml_string)?; + if !self.quiet { + eprintln!( + "You have cargo-wapm installed, added metadata to Cargo.toml instead of wapm.toml" + ); + eprintln!("Build and publish your package with:"); + eprintln!(); + eprintln!(" cargo wapm publish"); + eprintln!(); } + std::fs::write(&manifest_path, &format!("{old_cargo}\r\n\r\n{toml_string}"))?; Ok(()) } + + fn get_filesystem_mapping(&self) -> Option> { + if self.include.is_empty() { + return None; + } + + let include = self + .include + .iter() + .map(|path| { + if path == "." || path == "/" { + return ("/".to_string(), Path::new("/").to_path_buf()); + } + + let key = format!("./{path}"); + let value = Path::new(&format!("/{path}")).to_path_buf(); + + (key, value) + }) + .collect(); + + Some(include) + } + + fn get_command( + modules: &[wapm_toml::Module], + bin_or_lib: BinOrLib, + ) -> Option> { + match bin_or_lib { + BinOrLib::Bin => Some( + modules + .iter() + .map(|m| { + wapm_toml::Command::V1(wapm_toml::CommandV1 { + name: m.name.clone(), + module: m.name.clone(), + main_args: None, + package: None, + }) + }) + .collect(), + ), + BinOrLib::Lib | BinOrLib::Empty => None, + } + } + + /// Returns the dependencies based on the `--template` flag + fn get_dependencies(&self) -> Option> { + Some({ + match self.template.as_deref() { + Some("js") => { + let mut map = HashMap::default(); + map.insert("python".to_string(), "quickjs/quickjs@latest".to_string()); + map + } + Some("python") => { + let mut map = HashMap::default(); + map.insert("python".to_string(), "python/python@latest".to_string()); + map + } + _ => HashMap::default(), + } + }) + } + + // Returns whether the template for the wapm.toml should be a binary, a library or an empty file + fn get_bin_or_lib(&self) -> Result { + match (self.empty, self.bin, self.lib) { + (true, true, _) | (true, _, true) => { + return Err(anyhow::anyhow!( + "cannot combine --empty with --bin or --lib" + )) + } + (true, false, false) => Ok(BinOrLib::Empty), + (_, true, true) => { + return Err(anyhow::anyhow!( + "cannot initialize a wapm manifest with both --bin and --lib, pick one" + )) + } + (false, true, _) => Ok(BinOrLib::Bin), + (false, _, true) => Ok(BinOrLib::Lib), + _ => Ok(BinOrLib::Bin), + } + } + + fn get_bindings(target_file: &PathBuf, bin_or_lib: BinOrLib) -> Option { + match bin_or_lib { + BinOrLib::Bin | BinOrLib::Empty => None, + BinOrLib::Lib => target_file.parent().and_then(|parent| { + walkdir::WalkDir::new(parent) + .min_depth(1) + .max_depth(3) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + .filter_map(|e| { + let is_wit = e.path().extension().and_then(|s| s.to_str()) == Some(".wit"); + let is_wai = e.path().extension().and_then(|s| s.to_str()) == Some(".wai"); + if is_wit { + Some(wapm_toml::Bindings::Wit(wapm_toml::WitBindings { + wit_exports: e.path().to_path_buf(), + wit_bindgen: semver::Version::parse("0.1.0").unwrap(), + })) + } else if is_wai { + Some(wapm_toml::Bindings::Wai(wapm_toml::WaiBindings { + exports: None, + imports: vec![e.path().to_path_buf()], + wai_version: semver::Version::parse("0.1.0").unwrap(), + })) + } else { + None + } + }) + .next() + }), + } + } } From 7ba13bc77a9e70f8f5f1178f0b94a7c2b46ecea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 15:23:43 +0100 Subject: [PATCH 13/88] Fix make lint --- lib/cli/src/commands/init.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index f591dc1a860..65dfbc01d3a 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -177,7 +177,7 @@ impl Init { let constructed_manifest = wapm_toml::Manifest { package: wapm_toml::Package { - name: package_name.clone(), + name: package_name, version, description, license, @@ -268,7 +268,7 @@ impl Init { wasmer_extra_flags: None, abi: default_abi, fs: constructed_manifest.fs, - bindings: bindings, + bindings, }; let toml_string = toml::to_string_pretty(&metadata_wapm)? @@ -358,24 +358,20 @@ impl Init { // Returns whether the template for the wapm.toml should be a binary, a library or an empty file fn get_bin_or_lib(&self) -> Result { match (self.empty, self.bin, self.lib) { - (true, true, _) | (true, _, true) => { - return Err(anyhow::anyhow!( - "cannot combine --empty with --bin or --lib" - )) - } + (true, true, _) | (true, _, true) => Err(anyhow::anyhow!( + "cannot combine --empty with --bin or --lib" + )), (true, false, false) => Ok(BinOrLib::Empty), - (_, true, true) => { - return Err(anyhow::anyhow!( - "cannot initialize a wapm manifest with both --bin and --lib, pick one" - )) - } + (_, true, true) => Err(anyhow::anyhow!( + "cannot initialize a wapm manifest with both --bin and --lib, pick one" + )), (false, true, _) => Ok(BinOrLib::Bin), (false, _, true) => Ok(BinOrLib::Lib), _ => Ok(BinOrLib::Bin), } } - fn get_bindings(target_file: &PathBuf, bin_or_lib: BinOrLib) -> Option { + fn get_bindings(target_file: &Path, bin_or_lib: BinOrLib) -> Option { match bin_or_lib { BinOrLib::Bin | BinOrLib::Empty => None, BinOrLib::Lib => target_file.parent().and_then(|parent| { From b8d1d4e2928c61bfcb9487fb69e06e4dde884c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 15:42:41 +0100 Subject: [PATCH 14/88] Add integration tests for wasmer init --- .../integration/cli/tests/fixtures/init1.toml | 20 ++++++ .../integration/cli/tests/fixtures/init2.toml | 6 ++ .../integration/cli/tests/fixtures/init3.toml | 8 +++ tests/integration/cli/tests/init.rs | 67 +++++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 tests/integration/cli/tests/fixtures/init1.toml create mode 100644 tests/integration/cli/tests/fixtures/init2.toml create mode 100644 tests/integration/cli/tests/fixtures/init3.toml create mode 100644 tests/integration/cli/tests/init.rs diff --git a/tests/integration/cli/tests/fixtures/init1.toml b/tests/integration/cli/tests/fixtures/init1.toml new file mode 100644 index 00000000000..3e6bb52a003 --- /dev/null +++ b/tests/integration/cli/tests/fixtures/init1.toml @@ -0,0 +1,20 @@ +[package] +name = 'testfirstproject' +version = '0.1.0' +description = 'Description for package testfirstproject' + +# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest + +[dependencies] + +[[module]] +name = 'testfirstproject' +source = 'testfirstproject.wasm' +abi = 'wasi' + +[module.interfaces] +wasi = '0.1.0-unstable' + +[[command]] +name = 'testfirstproject' +module = 'testfirstproject' \ No newline at end of file diff --git a/tests/integration/cli/tests/fixtures/init2.toml b/tests/integration/cli/tests/fixtures/init2.toml new file mode 100644 index 00000000000..04d48853105 --- /dev/null +++ b/tests/integration/cli/tests/fixtures/init2.toml @@ -0,0 +1,6 @@ +[package] +name = "wasmer" +version = "0.5.6" +description = "hello" + +[dependencies] \ No newline at end of file diff --git a/tests/integration/cli/tests/fixtures/init3.toml b/tests/integration/cli/tests/fixtures/init3.toml new file mode 100644 index 00000000000..f20cd0ccc0f --- /dev/null +++ b/tests/integration/cli/tests/fixtures/init3.toml @@ -0,0 +1,8 @@ +[package] +name = "wasmer" +version = "0.5.6" +description = "hello" + +[dependencies] + +[metadata.package.wapm] \ No newline at end of file diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs new file mode 100644 index 00000000000..e565eddca7f --- /dev/null +++ b/tests/integration/cli/tests/init.rs @@ -0,0 +1,67 @@ +use anyhow::bail; +use std::path::PathBuf; +use std::process::Command; +use wasmer_integration_tests_cli::{get_repo_root_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; + +macro_rules! check_output { + ($output:expr) => { + let stdout_output = std::str::from_utf8(&$output.stdout).unwrap(); + let stderr_output = std::str::from_utf8(&$output.stdout).unwrap(); + if !$output.status.success() { + bail!("wasmer init failed with: stdout: {stdout_output}\n\nstderr: {stderr_output}"); + } + }; +} + +// Test that wasmer init without arguments works +#[test] +fn wasmer_init_works_1() -> anyhow::Result<()> { + let tempdir = tempfile::tempdir()?; + let path = tempdir.path(); + let path = path.join("testfirstproject"); + std::fs::create_dir_all(&path)?; + let output = Command::new(get_wasmer_path()) + .arg("init") + .current_dir(&path) + .output()?; + check_output!(output); + + let read = std::fs::read_to_string(path.join("wapm.toml")).unwrap().lines().collect::>().join("\n"); + let target = include_str!("./fixtures/init1.toml").lines().collect::>().join("\n"); + if read.trim() != target.trim() { + println!("expected:"); + println!("{target}"); + println!("got:"); + println!("{read}"); + panic!("read != target"); + } + Ok(()) +} + +// Test that wasmer init adds to a Cargo.toml +// instead of creating a new wapm.toml +#[test] +fn wasmer_init_works_2() -> anyhow::Result<()> { + let tempdir = tempfile::tempdir()?; + let path = tempdir.path(); + let path = path.join("testfirstproject"); + std::fs::create_dir_all(&path)?; + std::fs::write(path.join("Cargo.toml"), include_bytes!("./fixtures/init2.toml")); + let output = Command::new(get_wasmer_path()) + .arg("init") + .current_dir(&path) + .output()?; + check_output!(output); + + assert!(!path.join("wapm.toml").exists()); + let read = std::fs::read_to_string(path.join("Cargo.toml")).unwrap().lines().collect::>().join("\n"); + let target = include_str!("./fixtures/init3.toml").lines().collect::>().join("\n"); + if read.trim() != target.trim() { + println!("expected:"); + println!("{target}"); + println!("got:"); + println!("{read}"); + panic!("read != target"); + } + Ok(()) +} From 1c754a0131ada0426aa90b557355b4b2d91596bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 15:56:21 +0100 Subject: [PATCH 15/88] Fix second integration test for wasmer init --- Cargo.lock | 1 + .../integration/cli/tests/fixtures/init2.toml | 4 +- .../integration/cli/tests/fixtures/init4.toml | 20 +++++++++ tests/integration/cli/tests/init.rs | 42 +++++++++++-------- 4 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 tests/integration/cli/tests/fixtures/init4.toml diff --git a/Cargo.lock b/Cargo.lock index 66c3f51dbce..f637ea7f6a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4306,6 +4306,7 @@ version = "3.0.2" dependencies = [ "anyhow", "flate2", + "pretty_assertions", "rand 0.8.5", "tar", "target-lexicon 0.12.5", diff --git a/tests/integration/cli/tests/fixtures/init2.toml b/tests/integration/cli/tests/fixtures/init2.toml index 04d48853105..52310b228f9 100644 --- a/tests/integration/cli/tests/fixtures/init2.toml +++ b/tests/integration/cli/tests/fixtures/init2.toml @@ -1,6 +1,6 @@ [package] name = "wasmer" -version = "0.5.6" -description = "hello" +version = "0.1.0" +edition = "2021" [dependencies] \ No newline at end of file diff --git a/tests/integration/cli/tests/fixtures/init4.toml b/tests/integration/cli/tests/fixtures/init4.toml new file mode 100644 index 00000000000..06aa15af38d --- /dev/null +++ b/tests/integration/cli/tests/fixtures/init4.toml @@ -0,0 +1,20 @@ +[package] +name = 'wasmer' +version = '0.1.0' +description = 'Description for package wasmer' + +# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest + +[dependencies] + +[[module]] +name = 'wasmer' +source = '/target/release/wasmer.wasm' +abi = 'wasi' + +[module.interfaces] +wasi = '0.1.0-unstable' + +[[command]] +name = 'wasmer' +module = 'wasmer' \ No newline at end of file diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index e565eddca7f..654d0a899c7 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -28,13 +28,7 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { let read = std::fs::read_to_string(path.join("wapm.toml")).unwrap().lines().collect::>().join("\n"); let target = include_str!("./fixtures/init1.toml").lines().collect::>().join("\n"); - if read.trim() != target.trim() { - println!("expected:"); - println!("{target}"); - println!("got:"); - println!("{read}"); - panic!("read != target"); - } + pretty_assertions::assert_eq!(read.trim(), target.trim()); Ok(()) } @@ -46,22 +40,36 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { let path = tempdir.path(); let path = path.join("testfirstproject"); std::fs::create_dir_all(&path)?; - std::fs::write(path.join("Cargo.toml"), include_bytes!("./fixtures/init2.toml")); + std::fs::write(path.join("Cargo.toml"), include_bytes!("./fixtures/init2.toml"))?; + std::fs::create_dir_all(path.join("src"))?; + std::fs::write(path.join("src").join("main.rs"), b"fn main() { }")?; let output = Command::new(get_wasmer_path()) .arg("init") .current_dir(&path) .output()?; check_output!(output); - assert!(!path.join("wapm.toml").exists()); - let read = std::fs::read_to_string(path.join("Cargo.toml")).unwrap().lines().collect::>().join("\n"); - let target = include_str!("./fixtures/init3.toml").lines().collect::>().join("\n"); - if read.trim() != target.trim() { - println!("expected:"); - println!("{target}"); - println!("got:"); - println!("{read}"); - panic!("read != target"); + let cargo_wapm_stdout = std::process::Command::new("cargo") + .arg("wapm") + .arg("--version") + .output() + .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) + .unwrap_or_default(); + + let cargo_wapm_present = + cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); + + if cargo_wapm_present { + assert!(!path.join("wapm.toml").exists()); + let read = std::fs::read_to_string(path.join("Cargo.toml")).unwrap().lines().collect::>().join("\n"); + let target = include_str!("./fixtures/init3.toml").lines().collect::>().join("\n"); + pretty_assertions::assert_eq!(read.trim(), target.trim()); + } else { + pretty_assertions::assert_eq!(std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), include_str!("./fixtures/init2.toml")); + let read = std::fs::read_to_string(path.join("wapm.toml")).unwrap().lines().collect::>().join("\n"); + let target = include_str!("./fixtures/init4.toml").lines().collect::>().join("\n"); + pretty_assertions::assert_eq!(read.trim(), target.trim()); } + Ok(()) } From ac9d392664d5454e1fb84c12591437a6544abc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 18:26:25 +0100 Subject: [PATCH 16/88] Fix relative path in source of module --- lib/cli/src/commands/init.rs | 15 ++++++-- tests/integration/cli/tests/init.rs | 55 +++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 65dfbc01d3a..96cbd8101fd 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -160,9 +160,20 @@ impl Init { source: cargo_toml .as_ref() .map(|p| { - p.build_dir + // Normalize the path to /target/release to be relative to the parent of the Cargo.toml + let outpath = p + .build_dir .join("release") - .join(&format!("{module_name}.wasm")) + .join(&format!("{module_name}.wasm")); + let canonicalized_outpath = outpath.canonicalize().unwrap_or(outpath); + let outpath_str = format!("{}", canonicalized_outpath.display()); + let manifest_canonicalized = manifest_path + .parent() + .and_then(|p| p.canonicalize().ok()) + .unwrap_or(manifest_path.clone()); + let manifest_str = format!("{}", manifest_canonicalized.display()); + let relative_str = outpath_str.replacen(&manifest_str, "", 1); + Path::new(&relative_str).to_path_buf() }) .unwrap_or_else(|| Path::new(&format!("{module_name}.wasm")).to_path_buf()), kind: None, diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index 654d0a899c7..e5bd08548e6 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -26,13 +26,20 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { .output()?; check_output!(output); - let read = std::fs::read_to_string(path.join("wapm.toml")).unwrap().lines().collect::>().join("\n"); - let target = include_str!("./fixtures/init1.toml").lines().collect::>().join("\n"); + let read = std::fs::read_to_string(path.join("wapm.toml")) + .unwrap() + .lines() + .collect::>() + .join("\n"); + let target = include_str!("./fixtures/init1.toml") + .lines() + .collect::>() + .join("\n"); pretty_assertions::assert_eq!(read.trim(), target.trim()); Ok(()) } -// Test that wasmer init adds to a Cargo.toml +// Test that wasmer init adds to a Cargo.toml // instead of creating a new wapm.toml #[test] fn wasmer_init_works_2() -> anyhow::Result<()> { @@ -40,7 +47,10 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { let path = tempdir.path(); let path = path.join("testfirstproject"); std::fs::create_dir_all(&path)?; - std::fs::write(path.join("Cargo.toml"), include_bytes!("./fixtures/init2.toml"))?; + std::fs::write( + path.join("Cargo.toml"), + include_bytes!("./fixtures/init2.toml"), + )?; std::fs::create_dir_all(path.join("src"))?; std::fs::write(path.join("src").join("main.rs"), b"fn main() { }")?; let output = Command::new(get_wasmer_path()) @@ -50,24 +60,41 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { check_output!(output); let cargo_wapm_stdout = std::process::Command::new("cargo") - .arg("wapm") - .arg("--version") - .output() - .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) - .unwrap_or_default(); + .arg("wapm") + .arg("--version") + .output() + .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) + .unwrap_or_default(); let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); if cargo_wapm_present { assert!(!path.join("wapm.toml").exists()); - let read = std::fs::read_to_string(path.join("Cargo.toml")).unwrap().lines().collect::>().join("\n"); - let target = include_str!("./fixtures/init3.toml").lines().collect::>().join("\n"); + let read = std::fs::read_to_string(path.join("Cargo.toml")) + .unwrap() + .lines() + .collect::>() + .join("\n"); + let target = include_str!("./fixtures/init3.toml") + .lines() + .collect::>() + .join("\n"); pretty_assertions::assert_eq!(read.trim(), target.trim()); } else { - pretty_assertions::assert_eq!(std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), include_str!("./fixtures/init2.toml")); - let read = std::fs::read_to_string(path.join("wapm.toml")).unwrap().lines().collect::>().join("\n"); - let target = include_str!("./fixtures/init4.toml").lines().collect::>().join("\n"); + pretty_assertions::assert_eq!( + std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), + include_str!("./fixtures/init2.toml") + ); + let read = std::fs::read_to_string(path.join("wapm.toml")) + .unwrap() + .lines() + .collect::>() + .join("\n"); + let target = include_str!("./fixtures/init4.toml") + .lines() + .collect::>() + .join("\n"); pretty_assertions::assert_eq!(read.trim(), target.trim()); } From 2b09fc4011111517ad473a615ccb139e302b602b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 19:07:13 +0100 Subject: [PATCH 17/88] Add flow test for wasmer init + cargo wapm publish --- .../integration/cli/tests/fixtures/init5.toml | 6 ++ tests/integration/cli/tests/init.rs | 79 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 tests/integration/cli/tests/fixtures/init5.toml diff --git a/tests/integration/cli/tests/fixtures/init5.toml b/tests/integration/cli/tests/fixtures/init5.toml new file mode 100644 index 00000000000..46ea5b98a33 --- /dev/null +++ b/tests/integration/cli/tests/fixtures/init5.toml @@ -0,0 +1,6 @@ +[package] +name = "test-wasmer-init" +version = "RANDOMVERSION1.RANDOMVERSION2.RANDOMVERSION3" +edition = "2021" + +[dependencies] \ No newline at end of file diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index e5bd08548e6..0f377c515fb 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -39,6 +39,76 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { Ok(()) } +// Test that wasmer init works with cargo wapm +#[cfg(not(target_os = "macos"))] +#[test] +fn wasmer_init_works_3() -> anyhow::Result<()> { + // running test locally: should always pass since + // developers don't have access to WAPM_DEV_TOKEN + if std::env::var("GITHUB_TOKEN").is_err() { + return Ok(()); + } + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").expect("WAPM_DEV_TOKEN env var not set"); + + let cargo_wapm_stdout = std::process::Command::new("cargo") + .arg("wapm") + .arg("--version") + .output() + .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) + .unwrap_or_default(); + + let cargo_wapm_present = + cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); + + if !cargo_wapm_present { + // Install cargo wapm if not installed + let output = Command::new("cargo") + .arg("install") + .arg("cargo-wapm") + .output()?; + + check_output!(output); + } + + let tempdir = tempfile::tempdir()?; + let path = tempdir.path(); + let path = path.join("testfirstproject"); + std::fs::create_dir_all(&path)?; + std::fs::write( + path.join("Cargo.toml"), + include_str!("./fixtures/init5.toml") + .replace("RANDOMVERSION1", &format!("{}", rand::random::())) + .replace("RANDOMVERSION2", &format!("{}", rand::random::())) + .replace("RANDOMVERSION3", &format!("{}", rand::random::())), + )?; + std::fs::create_dir_all(path.join("src"))?; + std::fs::write(path.join("src").join("main.rs"), b"fn main() { }")?; + + let output = Command::new(get_wasmer_path()) + .arg("init") + .current_dir(&path) + .output()?; + check_output!(output); + + // login to wapm.dev, prepare for publish + let output = Command::new(get_wasmer_path()) + .arg("login") + .arg("--registry") + .arg("wapm.dev") + .arg(wapm_dev_token) + .output()?; + + let output = Command::new("cargo") + .arg("wapm") + .arg("publish") + .current_dir(&path) + .output()?; + + check_output!(output); + + Ok(()) +} + // Test that wasmer init adds to a Cargo.toml // instead of creating a new wapm.toml #[test] @@ -81,6 +151,15 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { .collect::>() .join("\n"); pretty_assertions::assert_eq!(read.trim(), target.trim()); + + // Install cargo wapm if not installed + let output = Command::new("cargo") + .arg("install") + .arg("cargo-wapm") + .current_dir(&path) + .output()?; + + check_output!(output); } else { pretty_assertions::assert_eq!( std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), From c325ce69f0bdb6db5fbe541652006e6844496add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 20:18:28 +0100 Subject: [PATCH 18/88] Use current logged in user as namespace in wasmer init --- lib/cli/src/commands/init.rs | 15 +++++++-------- tests/integration/cli/tests/fixtures/init4.toml | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 96cbd8101fd..93b01d608b6 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -129,11 +129,10 @@ impl Init { }) .ok_or_else(|| anyhow!("current dir has no file stem"))?; - let namespace = package_name - .split('/') - .next() - .unwrap_or(&package_name) - .to_string(); + let username = wasmer_registry::whoami(None).ok().map(|o| o.1); + let namespace = username + .or_else(|| package_name.split('/').next().map(|s| s.to_string())) + .unwrap_or(package_name.clone()); let module_name = package_name .split('/') .last() @@ -170,8 +169,8 @@ impl Init { let manifest_canonicalized = manifest_path .parent() .and_then(|p| p.canonicalize().ok()) - .unwrap_or(manifest_path.clone()); - let manifest_str = format!("{}", manifest_canonicalized.display()); + .unwrap_or_else(|| manifest_path.clone()); + let manifest_str = format!("{}/", manifest_canonicalized.display()); let relative_str = outpath_str.replacen(&manifest_str, "", 1); Path::new(&relative_str).to_path_buf() }) @@ -188,7 +187,7 @@ impl Init { let constructed_manifest = wapm_toml::Manifest { package: wapm_toml::Package { - name: package_name, + name: format!("{namespace}/{module_name}"), version, description, license, diff --git a/tests/integration/cli/tests/fixtures/init4.toml b/tests/integration/cli/tests/fixtures/init4.toml index 06aa15af38d..d8ff32c6b38 100644 --- a/tests/integration/cli/tests/fixtures/init4.toml +++ b/tests/integration/cli/tests/fixtures/init4.toml @@ -9,7 +9,7 @@ description = 'Description for package wasmer' [[module]] name = 'wasmer' -source = '/target/release/wasmer.wasm' +source = 'target/release/wasmer.wasm' abi = 'wasi' [module.interfaces] From 3eaa5850a07edcd5d9b2faae1704436a06f26673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 20:24:55 +0100 Subject: [PATCH 19/88] Add debugging to integration test --- lib/cli/src/commands/init.rs | 4 ++-- tests/integration/cli/tests/init.rs | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 93b01d608b6..e524e1667ac 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -132,7 +132,7 @@ impl Init { let username = wasmer_registry::whoami(None).ok().map(|o| o.1); let namespace = username .or_else(|| package_name.split('/').next().map(|s| s.to_string())) - .unwrap_or(package_name.clone()); + .unwrap_or_else(|| package_name.clone()); let module_name = package_name .split('/') .last() @@ -150,7 +150,7 @@ impl Init { let description = cargo_toml .as_ref() .and_then(|t| t.description.clone()) - .unwrap_or(format!("Description for package {module_name}")); + .unwrap_or_else(|| format!("Description for package {module_name}")); let default_abi = wapm_toml::Abi::Wasi; let bindings = Self::get_bindings(&target_file, bin_or_lib); diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index 0f377c515fb..fd03f79161a 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -1,6 +1,6 @@ use anyhow::bail; use std::path::PathBuf; -use std::process::Command; +use std::process::{Command, Stdio}; use wasmer_integration_tests_cli::{get_repo_root_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; macro_rules! check_output { @@ -43,16 +43,21 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { #[cfg(not(target_os = "macos"))] #[test] fn wasmer_init_works_3() -> anyhow::Result<()> { + println!("starting test..."); // running test locally: should always pass since // developers don't have access to WAPM_DEV_TOKEN if std::env::var("GITHUB_TOKEN").is_err() { return Ok(()); } let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").expect("WAPM_DEV_TOKEN env var not set"); + println!("wapm dev token ok..."); let cargo_wapm_stdout = std::process::Command::new("cargo") .arg("wapm") .arg("--version") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .output() .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) .unwrap_or_default(); @@ -61,10 +66,15 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); if !cargo_wapm_present { + println!("cargo wapm not present"); + // Install cargo wapm if not installed let output = Command::new("cargo") .arg("install") .arg("cargo-wapm") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .output()?; check_output!(output); @@ -84,12 +94,16 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { std::fs::create_dir_all(path.join("src"))?; std::fs::write(path.join("src").join("main.rs"), b"fn main() { }")?; + println!("project created"); + let output = Command::new(get_wasmer_path()) .arg("init") .current_dir(&path) .output()?; check_output!(output); + println!("wasmer init ok!"); + // login to wapm.dev, prepare for publish let output = Command::new(get_wasmer_path()) .arg("login") @@ -98,6 +112,8 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { .arg(wapm_dev_token) .output()?; + println!("wasmer login ok!"); + let output = Command::new("cargo") .arg("wapm") .arg("publish") @@ -106,6 +122,8 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { check_output!(output); + println!("cargo wapm publish ok! test done."); + Ok(()) } From 6ac97afa030e1e9d2805ae0b7861be065867f6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 20:27:09 +0100 Subject: [PATCH 20/88] Update wapm version in Makefile --- .github/workflows/test-sys.yaml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index f440ede6fda..50472623b81 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -206,7 +206,7 @@ jobs: if: matrix.run_test && matrix.os != 'windows-2019' shell: bash run: | - make build-wasmer && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli + make build-wasmer && make build-wapm && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli env: TARGET: ${{ matrix.target }} TARGET_DIR: target/${{ matrix.target }}/release diff --git a/Makefile b/Makefile index 0123ebf7c97..3f14b775254 100644 --- a/Makefile +++ b/Makefile @@ -401,7 +401,7 @@ else strip --strip-unneeded target/$(HOST_TARGET)/release/wasmer-headless endif -WAPM_VERSION = v0.5.3 +WAPM_VERSION = v0.5.9 get-wapm: [ -d "wapm-cli" ] || git clone --branch $(WAPM_VERSION) https://github.com/wasmerio/wapm-cli.git From 9a4d21b64a7a362ebacbe39aa00309ea6a7137d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 20:29:12 +0100 Subject: [PATCH 21/88] Install wapm before running integration tests --- .github/workflows/test-sys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index 50472623b81..81fff0a3eb1 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -206,7 +206,7 @@ jobs: if: matrix.run_test && matrix.os != 'windows-2019' shell: bash run: | - make build-wasmer && make build-wapm && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli + curl https://get.wasmer.io -sSfL | sh && make build-wasmer && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli env: TARGET: ${{ matrix.target }} TARGET_DIR: target/${{ matrix.target }}/release From 589dd1bfc12f64eb4bc99cb76ecd90eebc094038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 24 Nov 2022 20:58:26 +0100 Subject: [PATCH 22/88] Add login for cargo init tests --- .../integration/cli/tests/fixtures/init1.toml | 2 +- .../integration/cli/tests/fixtures/init4.toml | 4 +- .../integration/cli/tests/fixtures/init5.toml | 2 +- tests/integration/cli/tests/init.rs | 47 +++++++++++++++++++ 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/tests/integration/cli/tests/fixtures/init1.toml b/tests/integration/cli/tests/fixtures/init1.toml index 3e6bb52a003..8848e3ad3bd 100644 --- a/tests/integration/cli/tests/fixtures/init1.toml +++ b/tests/integration/cli/tests/fixtures/init1.toml @@ -1,5 +1,5 @@ [package] -name = 'testfirstproject' +name = 'ciuser/testfirstproject' version = '0.1.0' description = 'Description for package testfirstproject' diff --git a/tests/integration/cli/tests/fixtures/init4.toml b/tests/integration/cli/tests/fixtures/init4.toml index d8ff32c6b38..bf4d54708f3 100644 --- a/tests/integration/cli/tests/fixtures/init4.toml +++ b/tests/integration/cli/tests/fixtures/init4.toml @@ -1,7 +1,7 @@ [package] -name = 'wasmer' +name = 'ciuser/wasmer' version = '0.1.0' -description = 'Description for package wasmer' +description = 'Description for package ciuser/wasmer' # See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest diff --git a/tests/integration/cli/tests/fixtures/init5.toml b/tests/integration/cli/tests/fixtures/init5.toml index 46ea5b98a33..db4746c2b05 100644 --- a/tests/integration/cli/tests/fixtures/init5.toml +++ b/tests/integration/cli/tests/fixtures/init5.toml @@ -1,5 +1,5 @@ [package] -name = "test-wasmer-init" +name = "ciuser/test-wasmer-init" version = "RANDOMVERSION1.RANDOMVERSION2.RANDOMVERSION3" edition = "2021" diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index fd03f79161a..3c3cb9e9b83 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -20,6 +20,25 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { let path = tempdir.path(); let path = path.join("testfirstproject"); std::fs::create_dir_all(&path)?; + + if std::env::var("GITHUB_TOKEN").is_err() { + return Ok(()); + } + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").expect("WAPM_DEV_TOKEN env var not set"); + println!("wapm dev token ok..."); + + let output = Command::new(get_wasmer_path()) + .arg("login") + .arg("--registry") + .arg("wapm.dev") + .arg(wapm_dev_token) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) + .output()?; + + println!("wasmer login ok!"); + let output = Command::new(get_wasmer_path()) .arg("init") .current_dir(&path) @@ -98,6 +117,9 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { let output = Command::new(get_wasmer_path()) .arg("init") + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) .current_dir(&path) .output()?; check_output!(output); @@ -110,6 +132,9 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { .arg("--registry") .arg("wapm.dev") .arg(wapm_dev_token) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) .output()?; println!("wasmer login ok!"); @@ -117,6 +142,9 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { let output = Command::new("cargo") .arg("wapm") .arg("publish") + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) .current_dir(&path) .output()?; @@ -141,6 +169,25 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { )?; std::fs::create_dir_all(path.join("src"))?; std::fs::write(path.join("src").join("main.rs"), b"fn main() { }")?; + + if std::env::var("GITHUB_TOKEN").is_err() { + return Ok(()); + } + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").expect("WAPM_DEV_TOKEN env var not set"); + println!("wapm dev token ok..."); + + let output = Command::new(get_wasmer_path()) + .arg("login") + .arg("--registry") + .arg("wapm.dev") + .arg(wapm_dev_token) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) + .output()?; + + println!("wasmer login ok!"); + let output = Command::new(get_wasmer_path()) .arg("init") .current_dir(&path) From faf18522c8b0043b83b2f2288261855661ce7b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 25 Nov 2022 15:52:24 +0100 Subject: [PATCH 23/88] Remove wrong argument from cargo wapm --- tests/integration/cli/tests/init.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index 3c3cb9e9b83..c8a70ffb76f 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -141,7 +141,6 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { let output = Command::new("cargo") .arg("wapm") - .arg("publish") .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) .stdin(Stdio::null()) From 6892965e367d9d0effdfb8b39c181078288ebf4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 19:03:17 +0100 Subject: [PATCH 24/88] wapm.toml -> wasmer.toml --- .../assets/{qjs-wapm.toml => qjs-wasmer.toml} | 0 lib/cli/src/commands/init.rs | 8 ++-- lib/registry/src/lib.rs | 8 ++-- tests/integration/cli/tests/init.rs | 40 +++++++++---------- tests/integration/cli/tests/run.rs | 6 +-- 5 files changed, 32 insertions(+), 30 deletions(-) rename lib/c-api/examples/assets/{qjs-wapm.toml => qjs-wasmer.toml} (100%) diff --git a/lib/c-api/examples/assets/qjs-wapm.toml b/lib/c-api/examples/assets/qjs-wasmer.toml similarity index 100% rename from lib/c-api/examples/assets/qjs-wapm.toml rename to lib/c-api/examples/assets/qjs-wasmer.toml diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index e524e1667ac..bb1bee0711b 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -62,16 +62,18 @@ struct MiniCargoTomlPackage { build_dir: PathBuf, } +static WASMER_TOML_NAME: &str = "wasmer.toml"; + impl Init { /// `wasmer init` execution pub fn execute(&self) -> Result<(), anyhow::Error> { let bin_or_lib = self.get_bin_or_lib()?; let target_file = match self.out.as_ref() { - None => std::env::current_dir()?.join("wapm.toml"), + None => std::env::current_dir()?.join(WASMER_TOML_NAME), Some(s) => { let _ = std::fs::create_dir_all(s); - s.join("wapm.toml") + s.join(WASMER_TOML_NAME) } }; @@ -289,7 +291,7 @@ impl Init { if !self.quiet { eprintln!( - "You have cargo-wapm installed, added metadata to Cargo.toml instead of wapm.toml" + "You have cargo-wapm installed, added metadata to Cargo.toml instead of wasmer.toml" ); eprintln!("Build and publish your package with:"); eprintln!(); diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index 4a7c18a2fe0..ccbfa03df92 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -94,7 +94,7 @@ impl LocalPackage { let path = self.get_path()?; #[cfg(test)] let path = self.get_path(test_name)?; - let toml_path = path.join("wapm.toml"); + let toml_path = path.join(GLOBAL_CONFIG_FILE_NAME); let toml = std::fs::read_to_string(&toml_path) .map_err(|e| format!("error reading {}: {e}", toml_path.display()))?; let toml_parsed = toml::from_str::(&toml) @@ -113,8 +113,8 @@ pub fn get_executable_file_from_path( package_dir: &PathBuf, command: Option<&str>, ) -> Result<(wapm_toml::Manifest, PathBuf), anyhow::Error> { - let wapm_toml = std::fs::read_to_string(package_dir.join("wapm.toml")) - .map_err(|_| anyhow::anyhow!("Package {package_dir:?} has no wapm.toml"))?; + let wapm_toml = std::fs::read_to_string(package_dir.join(GLOBAL_CONFIG_FILE_NAME)) + .map_err(|_| anyhow::anyhow!("Package {package_dir:?} has no {GLOBAL_CONFIG_FILE_NAME}"))?; let wapm_toml = toml::from_str::(&wapm_toml) .map_err(|e| anyhow::anyhow!("Could not parse toml for {package_dir:?}: {e}"))?; @@ -150,7 +150,7 @@ pub fn get_executable_file_from_path( .find(|m| m.name == module_name) .ok_or_else(|| { anyhow::anyhow!( - "Cannot run {name}@{version}: module {module_name} not found in wapm.toml" + "Cannot run {name}@{version}: module {module_name} not found in {GLOBAL_CONFIG_FILE_NAME}" ) })?; diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index c8a70ffb76f..ce3c956e76f 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -28,14 +28,14 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { println!("wapm dev token ok..."); let output = Command::new(get_wasmer_path()) - .arg("login") - .arg("--registry") - .arg("wapm.dev") - .arg(wapm_dev_token) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .stdin(Stdio::null()) - .output()?; + .arg("login") + .arg("--registry") + .arg("wapm.dev") + .arg(wapm_dev_token) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) + .output()?; println!("wasmer login ok!"); @@ -45,7 +45,7 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { .output()?; check_output!(output); - let read = std::fs::read_to_string(path.join("wapm.toml")) + let read = std::fs::read_to_string(path.join("wasmer.toml")) .unwrap() .lines() .collect::>() @@ -176,17 +176,17 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { println!("wapm dev token ok..."); let output = Command::new(get_wasmer_path()) - .arg("login") - .arg("--registry") - .arg("wapm.dev") - .arg(wapm_dev_token) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .stdin(Stdio::null()) - .output()?; + .arg("login") + .arg("--registry") + .arg("wapm.dev") + .arg(wapm_dev_token) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) + .output()?; println!("wasmer login ok!"); - + let output = Command::new(get_wasmer_path()) .arg("init") .current_dir(&path) @@ -204,7 +204,7 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); if cargo_wapm_present { - assert!(!path.join("wapm.toml").exists()); + assert!(!path.join("wasmer.toml").exists()); let read = std::fs::read_to_string(path.join("Cargo.toml")) .unwrap() .lines() @@ -229,7 +229,7 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), include_str!("./fixtures/init2.toml") ); - let read = std::fs::read_to_string(path.join("wapm.toml")) + let read = std::fs::read_to_string(path.join("wasmer.toml")) .unwrap() .lines() .collect::>() diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index bcd1cbec0ee..a506cf4923a 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -352,12 +352,12 @@ fn test_wasmer_run_works_with_dir() -> anyhow::Result<()> { std::fs::copy(wasi_test_wasm_path(), &qjs_path)?; std::fs::copy( - format!("{}/{}", C_ASSET_PATH, "qjs-wapm.toml"), - temp_dir.path().join("wapm.toml"), + format!("{}/{}", C_ASSET_PATH, "qjs-wasmer.toml"), + temp_dir.path().join("wasmer.toml"), )?; assert!(temp_dir.path().exists()); - assert!(temp_dir.path().join("wapm.toml").exists()); + assert!(temp_dir.path().join("wasmer.toml").exists()); assert!(temp_dir.path().join("qjs.wasm").exists()); // test with "wasmer qjs.wasm" From 7fb5ad7d73c12f714b71f0c97dea03305578a215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 19:11:05 +0100 Subject: [PATCH 25/88] Debug why package.metadata.wapm has the wrong format --- lib/cli/src/commands/init.rs | 2 ++ tests/integration/cli/tests/fixtures/init4.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index bb1bee0711b..244b88dd967 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -283,6 +283,8 @@ impl Init { bindings, }; + println!("{:#?}", metadata_wapm); + let toml_string = toml::to_string_pretty(&metadata_wapm)? .replace("[dependencies]", &format!("{note}\r\n\r\n[dependencies]")) .lines() diff --git a/tests/integration/cli/tests/fixtures/init4.toml b/tests/integration/cli/tests/fixtures/init4.toml index bf4d54708f3..fff30418ac8 100644 --- a/tests/integration/cli/tests/fixtures/init4.toml +++ b/tests/integration/cli/tests/fixtures/init4.toml @@ -1,7 +1,7 @@ [package] name = 'ciuser/wasmer' version = '0.1.0' -description = 'Description for package ciuser/wasmer' +description = 'Description for package wasmer' # See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest From 2dec532388bb752d0a41b9b7587591c8eec0cc9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 20:53:23 +0100 Subject: [PATCH 26/88] Fix unit tests --- tests/integration/cli/tests/fixtures/init5.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cli/tests/fixtures/init5.toml b/tests/integration/cli/tests/fixtures/init5.toml index db4746c2b05..46ea5b98a33 100644 --- a/tests/integration/cli/tests/fixtures/init5.toml +++ b/tests/integration/cli/tests/fixtures/init5.toml @@ -1,5 +1,5 @@ [package] -name = "ciuser/test-wasmer-init" +name = "test-wasmer-init" version = "RANDOMVERSION1.RANDOMVERSION2.RANDOMVERSION3" edition = "2021" From b993321409c252a4376e69579309d6d9c36899c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 18 Nov 2022 18:33:16 +0100 Subject: [PATCH 27/88] Add wasmer publish command --- lib/cli/src/cli.rs | 5 +++++ lib/cli/src/commands.rs | 1 + lib/cli/src/commands/publish.rs | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 lib/cli/src/commands/publish.rs diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 1679fb3d4c5..652bbf6dd85 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -46,6 +46,10 @@ enum WasmerCLIOptions { /// Login into a wapm.io-like registry Login(Login), + /// Login into a wapm.io-like registry + #[clap(name = "publish")] + Publish(Publish), + /// Wasmer cache #[clap(subcommand)] Cache(Cache), @@ -172,6 +176,7 @@ impl WasmerCLIOptions { Self::Init(init) => init.execute(), Self::List(list) => list.execute(), Self::Login(login) => login.execute(), + Self::Publish(publish) => publish.execute(), #[cfg(feature = "wast")] Self::Wast(wast) => wast.execute(), #[cfg(target_os = "linux")] diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 6fbb720f493..b1ae79461b9 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -14,6 +14,7 @@ mod init; mod inspect; mod list; mod login; +mod publish; mod run; mod self_update; mod validate; diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs new file mode 100644 index 00000000000..489c194d1ce --- /dev/null +++ b/lib/cli/src/commands/publish.rs @@ -0,0 +1,20 @@ +use clap::Parser; + +/// CLI options for the `wasmer publish` command +#[derive(Debug, Parser)] +pub struct Publish { + /// Directory containing the `wapm.toml` (defaults to current root dir) + #[clap(long, name = "dir", env = "DIR")] + pub dir: Option, + /// Registry to publish to + #[clap(long, name = "registry")] + pub registry: Option, +} + +impl Publish { + /// Executes `wasmer publish` + pub fn execute(&self) -> Result<(), anyhow::Error> { + println!("{:?}", self); + Ok(()) + } +} From 874cb75164411706beab681bee5cec0fd3ae236c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 18 Nov 2022 18:52:53 +0100 Subject: [PATCH 28/88] Add uploading function to wasmer-registry --- Cargo.lock | 29 +++ lib/cli/src/commands/publish.rs | 6 + .../graphql/queries/get_signed_url.graphql | 5 + .../queries/publish_package_chunked.graphql | 22 ++ lib/registry/src/graphql.rs | 54 ++++ lib/registry/src/lib.rs | 1 + lib/registry/src/publish.rs | 242 ++++++++++++++++++ 7 files changed, 359 insertions(+) create mode 100644 lib/registry/graphql/queries/get_signed_url.graphql create mode 100644 lib/registry/graphql/queries/publish_package_chunked.graphql create mode 100644 lib/registry/src/publish.rs diff --git a/Cargo.lock b/Cargo.lock index f637ea7f6a2..e76131c1ae8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1644,6 +1644,18 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inkwell" version = "0.1.0-beta.4" @@ -2119,6 +2131,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.28.4" @@ -2282,6 +2300,12 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "portable-atomic" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15eb2c6e362923af47e13c23ca5afb859e83d54452c55b0b9ac763b8f7c1ac16" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4340,13 +4364,18 @@ name = "wasmer-registry" version = "3.0.2" dependencies = [ "anyhow", +<<<<<<< HEAD "dirs", "filetime", "flate2", "fs_extra", "futures-util", "graphql_client", +<<<<<<< HEAD "hex", +======= + "indicatif", +>>>>>>> af652c4e64 (Add uploading function to wasmer-registry) "log", "lzma-rs", "rand 0.8.5", diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index 489c194d1ce..08db0341c3d 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -9,6 +9,12 @@ pub struct Publish { /// Registry to publish to #[clap(long, name = "registry")] pub registry: Option, + /// Run the publish logic without sending anything to the registry server + #[clap(long, name = "dry-run")] + dry_run: bool, + /// Run the publish command without any output + #[clap(long, name = "quiet")] + quiet: bool, } impl Publish { diff --git a/lib/registry/graphql/queries/get_signed_url.graphql b/lib/registry/graphql/queries/get_signed_url.graphql new file mode 100644 index 00000000000..057277b645f --- /dev/null +++ b/lib/registry/graphql/queries/get_signed_url.graphql @@ -0,0 +1,5 @@ +query GetSignedUrl ($name:String!, $version:String!,$expiresAfterSeconds:Int) { + url: getSignedUrlForPackageUpload(name:$name, version:$version,expiresAfterSeconds:$expiresAfterSeconds) { + url + } +} \ No newline at end of file diff --git a/lib/registry/graphql/queries/publish_package_chunked.graphql b/lib/registry/graphql/queries/publish_package_chunked.graphql new file mode 100644 index 00000000000..cc63bb72268 --- /dev/null +++ b/lib/registry/graphql/queries/publish_package_chunked.graphql @@ -0,0 +1,22 @@ +mutation PublishPackageMutationChunked($name: String!, $version: String!, $description: String!, $manifest: String!, $license: String, $licenseFile: String, $readme: String, $fileName:String, $repository:String, $homepage:String, $signature: InputSignature, $signedUrl:String) { + publishPackage(input: { + name: $name, + version: $version, + description: $description, + manifest: $manifest, + license: $license, + licenseFile: $licenseFile, + readme: $readme, + file: $fileName, + signedUrl: $signedUrl, + repository: $repository, + homepage: $homepage, + signature: $signature, + clientMutationId: "" + }) { + success + packageVersion { + version + } + } +} \ No newline at end of file diff --git a/lib/registry/src/graphql.rs b/lib/registry/src/graphql.rs index 86c4d01d818..d90b131cc5f 100644 --- a/lib/registry/src/graphql.rs +++ b/lib/registry/src/graphql.rs @@ -99,6 +99,60 @@ pub(crate) mod proxy { } } +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "graphql/schema.graphql", + query_path = "graphql/queries/get_package_version.graphql", + response_derives = "Debug" +)] +pub(crate) struct GetPackageVersionQuery; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "graphql/schema.graphql", + query_path = "graphql/queries/get_package_by_command.graphql", + response_derives = "Debug" +)] +pub(crate) struct GetPackageByCommandQuery; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "graphql/schema.graphql", + query_path = "graphql/queries/test_if_registry_present.graphql", + response_derives = "Debug" +)] +pub(crate) struct TestIfRegistryPresent; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "graphql/schema.graphql", + query_path = "graphql/queries/get_bindings.graphql", + response_derives = "Debug,Clone,PartialEq,Eq" +)] +pub(crate) struct GetBindingsQuery; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "graphql/schema.graphql", + query_path = "graphql/queries/publish_package_chunked.graphql", + response_derives = "Debug" +)] +pub(crate) struct PublishPackageMutationChunked; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "graphql/schema.graphql", + query_path = "graphql/queries/get_signed_url.graphql", + response_derives = "Debug, Clone" +)] +pub(crate) struct GetSignedUrl; + +#[cfg(target_os = "wasi")] +pub fn whoami_distro() -> String { + whoami::os().to_lowercase() +} + +#[cfg(not(target_os = "wasi"))] pub fn whoami_distro() -> String { whoami::distro().to_lowercase() } diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index ccbfa03df92..e3dafe7e387 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -23,6 +23,7 @@ pub mod graphql; pub mod login; pub mod package; pub mod queries; +pub mod publish; pub mod utils; pub use crate::{ diff --git a/lib/registry/src/publish.rs b/lib/registry/src/publish.rs new file mode 100644 index 00000000000..7eb6e517d9a --- /dev/null +++ b/lib/registry/src/publish.rs @@ -0,0 +1,242 @@ +use crate::graphql::{execute_query_modifier_inner, get_signed_url, GetSignedUrl}; +use crate::graphql::{publish_package_mutation_chunked, PublishPackageMutationChunked}; +use crate::{format_graphql, PartialWapmConfig}; +use console::{style, Emoji}; +use graphql_client::GraphQLQuery; +use indicatif::{ProgressBar, ProgressState, ProgressStyle}; + +use std::collections::BTreeMap; +use std::fmt::Write; +use std::io::BufRead; + +static UPLOAD: Emoji<'_, '_> = Emoji("⬆️ ", ""); +static PACKAGE: Emoji<'_, '_> = Emoji("📦 ", ""); + +#[derive(Debug, Clone)] +pub enum SignArchiveResult { + Ok { + public_key_id: String, + signature: String, + }, + NoKeyRegistered, +} + +#[allow(clippy::too_many_arguments)] +pub fn try_chunked_uploading( + registry: Option, + token: Option, + package: &wapm_toml::Package, + manifest_string: &String, + license_file: &Option, + readme: &Option, + archive_name: &String, + archive_path: &PathBuf, + maybe_signature_data: &SignArchiveResult, + archived_data_size: u64, + quiet: bool, +) -> Result<(), anyhow::Error> { + let registry = match registry.as_ref() { + Some(s) => format_graphql(s), + None => { + #[cfg(not(test))] + let config = PartialWapmConfig::from_file(); + #[cfg(test)] + let config = PartialWapmConfig::from_file("publish"); + + config + .map_err(|e| anyhow::anyhow!("{e}"))? + .registry + .get_current_registry() + } + }; + + let token = match token.as_ref() { + Some(s) => s.to_string(), + None => { + #[cfg(not(test))] + let config = PartialWapmConfig::from_file(); + #[cfg(test)] + let config = PartialWapmConfig::from_file("publish"); + + config + .map_err(|e| anyhow::anyhow!("{e}"))? + .registry + .get_login_token_for_registry(®istry) + .ok_or_else(|| { + anyhow::anyhow!("cannot publish package: not logged into registry {registry:?}") + })? + } + }; + + let maybe_signature_data = match maybe_signature_data { + SignArchiveResult::Ok { + public_key_id, + signature, + } => { + log::info!( + "Package successfully signed with public key: \"{}\"!", + &public_key_id + ); + Some(publish_package_mutation_chunked::InputSignature { + public_key_key_id: public_key_id.to_string(), + data: signature.to_string(), + }) + } + SignArchiveResult::NoKeyRegistered => { + // TODO: uncomment this when we actually want users to start using it + //warn!("Publishing package without a verifying signature. Consider registering a key pair with wapm"); + None + } + }; + + if !quiet { + println!("{} {} Uploading...", style("[1/2]").bold().dim(), UPLOAD); + } + + let get_google_signed_url = GetSignedUrl::build_query(get_signed_url::Variables { + name: package.name.to_string(), + version: package.version.to_string(), + expires_after_seconds: Some(60 * 30), + }); + + let _response: get_signed_url::ResponseData = + execute_query_modifier_inner(®istry, &token, &get_google_signed_url, None, |f| f)?; + + let url = _response.url.ok_or_else(|| { + anyhow::anyhow!( + "could not get signed url for package {}@{}", + package.name, + package.version + ) + })?; + + let signed_url = url.url; + let url = url::Url::parse(&signed_url).unwrap(); + let client = reqwest::blocking::Client::builder() + .default_headers(reqwest::header::HeaderMap::default()) + .build() + .unwrap(); + + let res = client + .post(url) + .header(reqwest::header::CONTENT_LENGTH, "0") + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header("x-goog-resumable", "start"); + + let result = res.send().unwrap(); + + if result.status() != reqwest::StatusCode::from_u16(201).unwrap() { + return Err(anyhow::anyhow!( + "Uploading package failed: got HTTP {:?} when uploading", + result.status() + )); + } + + let headers = result + .headers() + .into_iter() + .filter_map(|(k, v)| { + let k = k.to_string(); + let v = v.to_str().ok()?.to_string(); + Some((k.to_lowercase(), v)) + }) + .collect::>(); + + let session_uri = headers.get("location").unwrap().clone(); + + let total = archived_data_size; + + // archive_path + let mut file = std::fs::OpenOptions::new() + .read(true) + .open(archive_path) + .map_err(|e| anyhow::anyhow!("cannot open archive {}: {e}", archive_path.display()))?; + + let pb = ProgressBar::new(archived_data_size); + pb.set_style(ProgressStyle::with_template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})") + .unwrap() + .with_key("eta", |state: &ProgressState, w: &mut dyn Write| { + write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap() + }) + .progress_chars("#>-")); + + let chunk_size = 1_048_576; // 1MB - 315s / 100MB + let mut file_pointer = 0; + + let mut reader = std::io::BufReader::with_capacity(chunk_size, &mut file); + + let client = reqwest::blocking::Client::builder() + .default_headers(reqwest::header::HeaderMap::default()) + .build() + .unwrap(); + + while let Some(chunk) = reader.fill_buf().ok().map(|s| s.to_vec()) { + let n = chunk.len(); + + if chunk.is_empty() { + break; + } + + let start = file_pointer; + let end = file_pointer + chunk.len().saturating_sub(1); + let content_range = format!("bytes {start}-{end}/{total}"); + + let res = client + .put(&session_uri) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header(reqwest::header::CONTENT_LENGTH, format!("{}", chunk.len())) + .header("Content-Range".to_string(), content_range) + .body(chunk.to_vec()); + + pb.set_position(file_pointer as u64); + + res.send() + .map(|response| response.error_for_status()) + .map_err(|e| { + anyhow::anyhow!( + "cannot send request to {session_uri} (chunk {}..{}): {e}", + file_pointer, + file_pointer + chunk_size + ) + })??; + + if n < chunk_size { + break; + } + + reader.consume(n); + file_pointer += n; + } + + pb.finish_and_clear(); + + if !quiet { + println!("{} {}Publishing...", style("[2/2]").bold().dim(), PACKAGE); + } + + let q = + PublishPackageMutationChunked::build_query(publish_package_mutation_chunked::Variables { + name: package.name.to_string(), + version: package.version.to_string(), + description: package.description.clone(), + manifest: manifest_string.to_string(), + license: package.license.clone(), + license_file: license_file.to_owned(), + readme: readme.to_owned(), + repository: package.repository.clone(), + homepage: package.homepage.clone(), + file_name: Some(archive_name.to_string()), + signature: maybe_signature_data, + signed_url: Some(signed_url), + }); + + let _response: publish_package_mutation_chunked::ResponseData = + crate::graphql::execute_query(®istry, &token, &q)?; + + println!( + "Successfully published package `{}@{}`", + package.name, package.version + ); + + Ok(()) +} From a462bc868c28bf989bd402e64804c1e48d171286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 21:13:28 +0100 Subject: [PATCH 29/88] Fix merge errors --- Cargo.lock | 47 +++++++++++++++++-------------------- lib/registry/src/lib.rs | 1 + lib/registry/src/publish.rs | 1 + 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e76131c1ae8..2ca4fe352d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,9 +101,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", @@ -284,9 +284,9 @@ dependencies = [ [[package]] name = "cargo_metadata" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406c859255d568f4f742b3146d51851f3bfd49f734a2c289d9107c4395ee0062" +checksum = "982a0cf6a99c350d7246035613882e376d58cebe571785abc5da4f648d53ac0a" dependencies = [ "camino", "cargo-platform", @@ -2056,9 +2056,9 @@ checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" [[package]] name = "nix" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -2068,9 +2068,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags", @@ -2246,9 +2246,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f400b0f7905bf702f9f3dc3df5a121b16c54e9e8012c082905fdf09a931861a" +checksum = "cc8bed3549e0f9b0a2a78bf7c0018237a2cdf085eecbbc048e52612438e4e9d0" dependencies = [ "thiserror", "ucd-trie", @@ -2314,9 +2314,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "predicates" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6bd09a7f7e68f3f0bf710fb7ab9c4615a488b58b5f653382a687701e458c92" +checksum = "f54fc5dc63ed3bbf19494623db4f3af16842c0d975818e469022d09e53f0aa05" dependencies = [ "difflib", "float-cmp", @@ -2975,9 +2975,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" dependencies = [ "serde_derive", ] @@ -3014,9 +3014,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -3634,9 +3634,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "trybuild" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea496675d71016e9bc76aa42d87f16aefd95447cc5818e671e12b2d7e269075d" +checksum = "db29f438342820400f2d9acfec0d363e987a38b2950bdb50a7069ed17b2148ee" dependencies = [ "glob", "once_cell", @@ -4371,11 +4371,8 @@ dependencies = [ "fs_extra", "futures-util", "graphql_client", -<<<<<<< HEAD "hex", -======= "indicatif", ->>>>>>> af652c4e64 (Add uploading function to wasmer-registry) "log", "lzma-rs", "rand 0.8.5", @@ -4503,7 +4500,7 @@ name = "wasmer-wasi-experimental-io-devices" version = "3.0.2" dependencies = [ "minifb", - "nix 0.25.0", + "nix 0.25.1", "ref_thread_local", "serde", "tracing", @@ -4725,7 +4722,7 @@ dependencies = [ "bitflags", "downcast-rs", "libc", - "nix 0.24.2", + "nix 0.24.3", "scoped-tls", "wayland-commons", "wayland-scanner", @@ -4738,7 +4735,7 @@ version = "0.29.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" dependencies = [ - "nix 0.24.2", + "nix 0.24.3", "once_cell", "smallvec", "wayland-sys", @@ -4750,7 +4747,7 @@ version = "0.29.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6865c6b66f13d6257bef1cd40cbfe8ef2f150fb8ebbdb1e8e873455931377661" dependencies = [ - "nix 0.24.2", + "nix 0.24.3", "wayland-client", "xcursor", ] diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index e3dafe7e387..6af1789a196 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -24,6 +24,7 @@ pub mod login; pub mod package; pub mod queries; pub mod publish; +pub mod queries; pub mod utils; pub use crate::{ diff --git a/lib/registry/src/publish.rs b/lib/registry/src/publish.rs index 7eb6e517d9a..6c67405ec12 100644 --- a/lib/registry/src/publish.rs +++ b/lib/registry/src/publish.rs @@ -8,6 +8,7 @@ use indicatif::{ProgressBar, ProgressState, ProgressStyle}; use std::collections::BTreeMap; use std::fmt::Write; use std::io::BufRead; +use std::path::PathBuf; static UPLOAD: Emoji<'_, '_> = Emoji("⬆️ ", ""); static PACKAGE: Emoji<'_, '_> = Emoji("📦 ", ""); From 364d1a445ed5dad7c07a982f7f039106721ec950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Sat, 3 Dec 2022 07:42:19 +0100 Subject: [PATCH 30/88] Port wapm publish to wasmer publish --- Cargo.lock | 166 +++++++++++++- lib/cli/Cargo.toml | 3 +- lib/cli/sql/migrations/0000.sql | 27 +++ lib/cli/sql/migrations/0001.sql | 9 + lib/cli/sql/migrations/0002.sql | 26 +++ lib/cli/src/commands/publish.rs | 372 +++++++++++++++++++++++++++++++- lib/registry/src/config.rs | 11 + 7 files changed, 602 insertions(+), 12 deletions(-) create mode 100644 lib/cli/sql/migrations/0000.sql create mode 100644 lib/cli/sql/migrations/0001.sql create mode 100644 lib/cli/sql/migrations/0002.sql diff --git a/Cargo.lock b/Cargo.lock index 2ca4fe352d6..7cb0d967e46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -356,6 +356,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "cipher" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "2.34.0" @@ -1087,6 +1097,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "1.8.0" @@ -1310,7 +1326,7 @@ dependencies = [ "cfg-if 1.0.0", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1463,6 +1479,15 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashlink" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "heck" version = "0.3.3" @@ -1493,6 +1518,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "0.2.8" @@ -1710,6 +1744,15 @@ dependencies = [ "rustc_version 0.3.3", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -1837,6 +1880,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libsqlite3-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "link-cplusplus" version = "1.0.7" @@ -2009,6 +2062,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "minisign" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f331f58ac7746ecb6c65843e5c8962103d0254e87ff0858be13a529988ea0333" +dependencies = [ + "getrandom", + "rpassword", + "scrypt", +] + [[package]] name = "miniz_oxide" version = "0.5.4" @@ -2035,7 +2099,7 @@ checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.42.0", ] @@ -2238,6 +2302,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -2762,6 +2835,41 @@ dependencies = [ "syn", ] +[[package]] +name = "rpassword" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" +dependencies = [ + "libc", + "rtoolbox", + "winapi", +] + +[[package]] +name = "rtoolbox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "rusqlite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -2853,6 +2961,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2880,6 +2997,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +[[package]] +name = "scrypt" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f9e24d2b632954ded8ab2ef9fea0a0c769ea56ea98bddbafbad22caeeadf45d" +dependencies = [ + "hmac", + "pbkdf2", + "salsa20", + "sha2", +] + [[package]] name = "sct" version = "0.6.1" @@ -3433,6 +3562,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "time" version = "0.2.27" @@ -3770,6 +3910,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.1.1" @@ -3834,6 +3980,12 @@ dependencies = [ "toml", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -4127,17 +4279,23 @@ dependencies = [ "isatty", "libc", "log", + "minisign", "nuke-dir", "prettytable-rs", "regex", "reqwest", + "rpassword", + "rusqlite", "semver 1.0.14", "serde", "serde_json", "spinoff", + "tar", "target-lexicon 0.12.5", "tempdir", "tempfile", + "thiserror", + "time 0.1.45", "tldextract", "toml", "unix_mode", @@ -4291,7 +4449,7 @@ dependencies = [ "lazy_static", "libc", "log", - "time", + "time 0.2.27", "wasmer", "wasmer-types", ] @@ -4525,7 +4683,7 @@ dependencies = [ "byteorder", "pretty_assertions", "serde", - "time", + "time 0.2.27", "wasmer", "wasmer-derive", "wasmer-types", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 293e91dde14..378183024d7 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -52,7 +52,6 @@ bytesize = "1.0" cfg-if = "1.0" # For debug feature fern = { version = "0.6", features = ["colored"], optional = true } -log = { version = "0.4", optional = true } tempfile = "3" tempdir = "0.3.7" http_req = { version="^0.8", default-features = false, features = ["rust-tls"] } @@ -144,7 +143,7 @@ llvm = [ "wasmer-compiler-llvm", "compiler", ] -debug = ["fern", "log", "wasmer-wasi/logging"] +debug = ["fern", "wasmer-wasi/logging"] disable-all-logging = ["wasmer-wasi/disable-all-logging"] headless = [] headless-minimal = ["headless", "disable-all-logging", "wasi"] diff --git a/lib/cli/sql/migrations/0000.sql b/lib/cli/sql/migrations/0000.sql new file mode 100644 index 00000000000..a527dbc4f29 --- /dev/null +++ b/lib/cli/sql/migrations/0000.sql @@ -0,0 +1,27 @@ +CREATE TABLE personal_keys +( + id integer primary key, + active integer not null, + public_key_id text not null UNIQUE, + public_key_value text not null UNIQUE, + private_key_location text UNIQUE, + key_type_identifier text not null, + date_added text not null +); + +CREATE TABLE wapm_users +( + id integer primary key, + name text not null UNIQUE +); + +CREATE TABLE wapm_public_keys +( + id integer primary key, + public_key_id text not null UNIQUE, + user_key integer not null, + public_key_value text not null UNIQUE, + key_type_identifier text not null, + date_added text not null, + FOREIGN KEY(user_key) REFERENCES wapm_users(id) +); diff --git a/lib/cli/sql/migrations/0001.sql b/lib/cli/sql/migrations/0001.sql new file mode 100644 index 00000000000..42587319156 --- /dev/null +++ b/lib/cli/sql/migrations/0001.sql @@ -0,0 +1,9 @@ +CREATE TABLE wasm_contracts +( + id integer primary key, + contract_name text not null, + version text not null, + date_added text not null, + content text not null, + CONSTRAINT name_version_unique UNIQUE (contract_name, version) +); diff --git a/lib/cli/sql/migrations/0002.sql b/lib/cli/sql/migrations/0002.sql new file mode 100644 index 00000000000..cc57ff9b8ee --- /dev/null +++ b/lib/cli/sql/migrations/0002.sql @@ -0,0 +1,26 @@ +PRAGMA foreign_keys=off; + +CREATE TABLE wasm_interfaces +( + id integer primary key, + interface_name text not null, + version text not null, + date_added text not null, + content text not null, + CONSTRAINT name_version_unique UNIQUE (interface_name, version) +); + +INSERT INTO wasm_interfaces +( + id, + interface_name, + version, + date_added, + content +) + SELECT id, contract_name, version, date_added, content + FROM wasm_contracts; + +DROP TABLE wasm_contracts; + +PRAGMA foreign_keys=on; diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index 08db0341c3d..7f6e2712b04 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -1,4 +1,17 @@ use clap::Parser; +use flate2::{write::GzEncoder, Compression}; +use rusqlite::{params, Connection, OpenFlags, TransactionBehavior}; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use tar::Builder; +use thiserror::Error; +use time::{self, Timespec}; +use wasmer_registry::publish::SignArchiveResult; +use wasmer_registry::PartialWapmConfig; + +const CURRENT_DATA_VERSION: i32 = 3; +const RFC3339_FORMAT_STRING: &str = "%Y-%m-%dT%H:%M:%S-%f"; /// CLI options for the `wasmer publish` command #[derive(Debug, Parser)] @@ -7,20 +20,367 @@ pub struct Publish { #[clap(long, name = "dir", env = "DIR")] pub dir: Option, /// Registry to publish to - #[clap(long, name = "registry")] + #[clap(long)] pub registry: Option, /// Run the publish logic without sending anything to the registry server #[clap(long, name = "dry-run")] - dry_run: bool, + pub dry_run: bool, /// Run the publish command without any output - #[clap(long, name = "quiet")] - quiet: bool, + #[clap(long)] + pub quiet: bool, + /// Override the namespace of the uploaded package in the wapm.toml + pub namespace: Option, + /// Override the token (by default, it will use the current logged in user) + pub token: Option, +} + +#[derive(Debug, Error)] +enum PublishError { + #[error("Cannot publish without a module.")] + NoModule, + #[error("Unable to publish the \"{module}\" module because \"{}\" is not a file", path.display())] + SourceMustBeFile { module: String, path: PathBuf }, + #[error("Unable to load the bindings for \"{module}\" because \"{}\" doesn't exist", path.display())] + MissingBindings { module: String, path: PathBuf }, + #[error("Error building package when parsing module \"{0}\".")] + ErrorBuildingPackage(String), + #[error( + "Path \"{0}\", specified in the manifest as part of the package file system does not exist.", + )] + MissingManifestFsPath(String), + #[error("When processing the package filesystem, found path \"{0}\" which is not a directory")] + PackageFileSystemEntryMustBeDirectory(String), } impl Publish { /// Executes `wasmer publish` pub fn execute(&self) -> Result<(), anyhow::Error> { - println!("{:?}", self); - Ok(()) + // First, check for the wasmer.toml and see if the user is logged in + // and has authorization to publish the package under the correct namespace. + let mut builder = Builder::new(Vec::new()); + + let cwd = match self.dir.as_ref() { + Some(s) => Path::new(&s).to_path_buf(), + None => std::env::current_dir()?, + }; + + // TODO: implement validation + // validate::validate_directory(cwd.clone())?; + + let manifest = wapm_toml::Manifest::find_in_directory(&cwd)?; + + let manifest_path_buf = cwd.join("wasmer.toml"); + builder.append_path_with_name(&manifest_path_buf, "wapm.toml")?; + + let package = &manifest.package; + let modules = manifest.module.as_ref().ok_or(PublishError::NoModule)?; + let manifest_string = toml::to_string(&manifest)?; + + let readme = package.readme.as_ref().and_then(|readme_path| { + let normalized_path = normalize_path(&manifest.base_directory_path, readme_path); + if builder.append_path(&normalized_path).is_err() { + // TODO: Maybe do something here + } + fs::read_to_string(normalized_path).ok() + }); + + let license_file = package.license_file.as_ref().and_then(|license_file_path| { + let normalized_path = normalize_path(&manifest.base_directory_path, license_file_path); + if builder.append_path(&normalized_path).is_err() { + // TODO: Maybe do something here + } + fs::read_to_string(normalized_path).ok() + }); + + for module in modules { + let normalized_path = normalize_path(&manifest.base_directory_path, &module.source); + normalized_path + .metadata() + .map_err(|_| PublishError::SourceMustBeFile { + module: module.name.clone(), + path: normalized_path.clone(), + })?; + builder + .append_path(normalized_path) + .map_err(|_| PublishError::ErrorBuildingPackage(module.name.clone()))?; + + if let Some(bindings) = &module.bindings { + for path in bindings.referenced_files(&manifest.base_directory_path)? { + let normalized_path = normalize_path(&manifest.base_directory_path, &path); + normalized_path + .metadata() + .map_err(|_| PublishError::MissingBindings { + module: module.name.clone(), + path: normalized_path.clone(), + })?; + builder + .append_path(normalized_path) + .map_err(|_| PublishError::ErrorBuildingPackage(module.name.clone()))?; + } + } + } + + // bundle the package filesystem + for (_alias, path) in manifest.fs.unwrap_or_default().iter() { + let normalized_path = normalize_path(&cwd, path); + let path_metadata = normalized_path.metadata().map_err(|_| { + PublishError::MissingManifestFsPath(normalized_path.to_string_lossy().to_string()) + })?; + if path_metadata.is_dir() { + builder.append_dir_all(path, &normalized_path) + } else { + return Err(PublishError::PackageFileSystemEntryMustBeDirectory( + path.to_string_lossy().to_string(), + ) + .into()); + } + .map_err(|_| { + PublishError::MissingManifestFsPath(normalized_path.to_string_lossy().to_string()) + })?; + } + + builder.finish().ok(); + let tar_archive_data = builder.into_inner().map_err(|_| PublishError::NoModule)?; + let archive_name = "package.tar.gz".to_string(); + let archive_dir = tempfile::TempDir::new()?; + let archive_dir_path: &std::path::Path = archive_dir.as_ref(); + fs::create_dir(archive_dir_path.join("wapm_package"))?; + let archive_path = archive_dir_path.join("wapm_package").join(&archive_name); + let mut compressed_archive = fs::File::create(&archive_path).unwrap(); + let mut gz_enc = GzEncoder::new(&mut compressed_archive, Compression::best()); + + gz_enc.write_all(&tar_archive_data).unwrap(); + let _compressed_archive = gz_enc.finish().unwrap(); + let mut compressed_archive_reader = fs::File::open(&archive_path)?; + + let maybe_signature_data = sign_compressed_archive(&mut compressed_archive_reader)?; + let archived_data_size = archive_path.metadata()?.len(); + + assert!(archive_path.exists()); + assert!(archive_path.is_file()); + + if self.dry_run { + // dry run: publish is done here + + println!( + "Successfully published package `{}@{}`", + package.name, package.version + ); + + log::info!( + "Publish succeeded, but package was not published because it was run in dry-run mode" + ); + + return Ok(()); + } + + wasmer_registry::publish::try_chunked_uploading( + self.registry.clone(), + self.token.clone(), + package, + &manifest_string, + &license_file, + &readme, + &archive_name, + &archive_path, + &maybe_signature_data, + archived_data_size, + self.quiet, + ) + .map_err(on_error) + } +} + +fn on_error(e: anyhow::Error) -> anyhow::Error { + #[cfg(feature = "telemetry")] + sentry::integrations::anyhow::capture_anyhow(&e); + + e +} + +fn normalize_path(cwd: &Path, path: &Path) -> PathBuf { + let mut out = PathBuf::from(cwd); + let mut components = path.components(); + if path.is_absolute() { + log::warn!( + "Interpreting absolute path {} as a relative path", + path.to_string_lossy() + ); + components.next(); + } + for comp in components { + out.push(comp); + } + out +} + +/// Takes the package archive as a File and attempts to sign it using the active key +/// returns the public key id used to sign it and the signature string itself +pub fn sign_compressed_archive( + compressed_archive: &mut fs::File, +) -> anyhow::Result { + let key_db = open_db()?; + let personal_key = if let Ok(v) = get_active_personal_key(&key_db) { + v + } else { + return Ok(SignArchiveResult::NoKeyRegistered); + }; + let password = rpassword::prompt_password(format!( + "Please enter your password for the key pair {}:", + &personal_key.public_key_id + )) + .ok(); + let private_key = if let Some(priv_key_location) = personal_key.private_key_location { + match minisign::SecretKey::from_file(&priv_key_location, password) { + Ok(priv_key_data) => priv_key_data, + Err(e) => { + log::error!( + "Could not read private key from location {}: {}", + priv_key_location, + e + ); + return Err(e.into()); + } + } + } else { + // TODO: add more info about why this might have happened and what the user can do about it + log::warn!("Active key does not have a private key location registered with it!"); + return Err(anyhow!("Cannot sign package, no private key")); + }; + Ok(SignArchiveResult::Ok { + public_key_id: personal_key.public_key_id, + signature: (minisign::sign( + Some(&minisign::PublicKey::from_base64( + &personal_key.public_key_value, + )?), + &private_key, + compressed_archive, + None, + None, + )? + .to_string()), + }) +} + +/// Opens an exclusive read/write connection to the database, creating it if it does not exist +pub fn open_db() -> anyhow::Result { + let db_path = + PartialWapmConfig::get_database_file_path().map_err(|e| anyhow::anyhow!("{e}"))?; + let mut conn = Connection::open_with_flags( + db_path, + OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_FULL_MUTEX, + )?; + + apply_migrations(&mut conn)?; + Ok(conn) +} + +/// Applies migrations to the database +pub fn apply_migrations(conn: &mut Connection) -> anyhow::Result<()> { + let user_version = conn.pragma_query_value(None, "user_version", |val| val.get(0))?; + for data_version in user_version..CURRENT_DATA_VERSION { + log::debug!("Applying migration {}", data_version); + apply_migration(conn, data_version)?; + } + Ok(()) +} + +#[derive(Debug, Error)] +enum MigrationError { + #[error( + "Critical internal error: the data version {0} is not handleded; current data version: {1}" + )] + MigrationNumberDoesNotExist(i32, i32), + #[error("Critical internal error: failed to commit trasaction migrating to data version {0}")] + CommitFailed(i32), + #[error("Critical internal error: transaction failed on migration number {0}: {1}")] + TransactionFailed(i32, String), +} + +/// Applies migrations to the database and updates the `user_version` pragma. +/// Every migration must leave the database in a valid state. +fn apply_migration(conn: &mut Connection, migration_number: i32) -> Result<(), MigrationError> { + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| MigrationError::TransactionFailed(migration_number, format!("{}", e)))?; + match migration_number { + 0 => { + tx.execute_batch(include_str!("../../sql/migrations/0000.sql")) + .map_err(|e| { + MigrationError::TransactionFailed(migration_number, format!("{}", e)) + })?; + } + 1 => { + tx.execute_batch(include_str!("../../sql/migrations/0001.sql")) + .map_err(|e| { + MigrationError::TransactionFailed(migration_number, format!("{}", e)) + })?; + } + 2 => { + tx.execute_batch(include_str!("../../sql/migrations/0002.sql")) + .map_err(|e| { + MigrationError::TransactionFailed(migration_number, format!("{}", e)) + })?; + } + _ => { + return Err(MigrationError::MigrationNumberDoesNotExist( + migration_number, + CURRENT_DATA_VERSION, + )); + } + } + tx.pragma_update(None, "user_version", &(migration_number + 1)) + .map_err(|e| MigrationError::TransactionFailed(migration_number, format!("{}", e)))?; + tx.commit() + .map_err(|_| MigrationError::CommitFailed(migration_number)) +} + +/// Information about one of the user's keys +#[derive(Debug)] +pub struct PersonalKey { + /// Flag saying if the key will be used (there can only be one active key at a time) + pub active: bool, + /// The public key's tag. Used to identify the key pair + pub public_key_id: String, + /// The raw value of the public key in base64 + pub public_key_value: String, + /// The location in the file system of the private key + pub private_key_location: Option, + /// The type of private/public key this is + pub key_type_identifier: String, + /// The time at which the key was registered with wapm + pub date_created: Timespec, +} + +fn get_active_personal_key(conn: &Connection) -> anyhow::Result { + let mut stmt = conn.prepare( + "SELECT active, public_key_value, private_key_location, date_added, key_type_identifier, public_key_id FROM personal_keys + where active = 1", + )?; + + let result = stmt + .query_map(params![], |row| { + Ok(PersonalKey { + active: row.get(0)?, + public_key_value: row.get(1)?, + private_key_location: row.get(2)?, + date_created: { + let time_str: String = row.get(3)?; + time::strptime(&time_str, RFC3339_FORMAT_STRING) + .unwrap_or_else(|_| panic!("Failed to parse time string {}", &time_str)) + .to_timespec() + }, + key_type_identifier: row.get(4)?, + public_key_id: row.get(5)?, + }) + })? + .next(); + + if let Some(res) = result { + Ok(res?) + } else { + Err(anyhow!("No active key found")) } } diff --git a/lib/registry/src/config.rs b/lib/registry/src/config.rs index 422da94b4e4..53ea0320dac 100644 --- a/lib/registry/src/config.rs +++ b/lib/registry/src/config.rs @@ -4,6 +4,8 @@ use serde::Serialize; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +pub static GLOBAL_CONFIG_DATABASE_FILE_NAME: &str = "wapm.sqlite"; + #[derive(Deserialize, Default, Serialize, Debug, PartialEq, Eq)] pub struct PartialWapmConfig { /// The number of seconds to wait before checking the registry for a new @@ -303,6 +305,15 @@ impl PartialWapmConfig { pub fn get_file_location() -> Result { Ok(Self::get_folder()?.join(crate::GLOBAL_CONFIG_FILE_NAME)) } + + pub fn get_database_file_path(#[cfg(test)] test_name: &str) -> Result { + #[cfg(test)] + let f = Self::get_folder(test_name); + #[cfg(not(test))] + let f = Self::get_folder(); + + f.map(|config_folder| config_folder.join(GLOBAL_CONFIG_DATABASE_FILE_NAME)) + } } #[derive(GraphQLQuery)] From 445e9382e8d07fb430bb9f7c507b108fb3abb63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 6 Dec 2022 13:05:48 +0100 Subject: [PATCH 31/88] Debug cargo wapm integration not working --- lib/cli/src/commands/init.rs | 81 +++++++++++++++++++---------- tests/integration/cli/tests/init.rs | 8 +-- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 244b88dd967..4a2472e481f 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -47,6 +47,7 @@ enum BinOrLib { } // minimal version of the Cargo.toml [package] section +#[derive(Debug, Clone)] struct MiniCargoTomlPackage { name: String, version: semver::Version, @@ -87,29 +88,50 @@ impl Init { // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. let manifest_path = match self.manifest_path.as_ref() { Some(s) => s.clone(), - None => Path::new("./Cargo.toml").to_path_buf(), + None => { + let cargo_toml_path = std::env::current_dir().unwrap().join("Cargo.toml"); + cargo_toml_path + .canonicalize() + .unwrap_or_else(|_| cargo_toml_path.clone()) + } }; - let cargo_toml = MetadataCommand::new() - .manifest_path(&manifest_path) - .features(CargoOpt::AllFeatures) - .exec() - .ok() - .and_then(|metadata| { - let package = metadata.root_package()?; - Some(MiniCargoTomlPackage { - name: package.name.clone(), - version: package.version.clone(), - description: package.description.clone(), - homepage: package.homepage.clone(), - repository: package.repository.clone(), - license: package.license.clone(), - readme: package.readme.clone().map(|s| s.into_std_path_buf()), - license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), - workspace_root: metadata.workspace_root.into_std_path_buf(), - build_dir: metadata.target_directory.into_std_path_buf(), - }) - }); + let cargo_toml = if manifest_path.exists() { + use anyhow::Context; + + let metadata = MetadataCommand::new() + .manifest_path(&manifest_path) + .features(CargoOpt::AllFeatures) + .exec(); + + let metadata = match metadata { + Ok(o) => o, + Err(e) => { + return Err(anyhow::anyhow!("failed to load metadata: {e}") + .context(anyhow::anyhow!("{}", manifest_path.display()))); + } + }; + + let package = metadata + .root_package() + .ok_or_else(|| anyhow::anyhow!("no root package found in cargo metadata")) + .context(anyhow::anyhow!("{}", manifest_path.display()))?; + + Some(MiniCargoTomlPackage { + name: package.name.clone(), + version: package.version.clone(), + description: package.description.clone(), + homepage: package.homepage.clone(), + repository: package.repository.clone(), + license: package.license.clone(), + readme: package.readme.clone().map(|s| s.into_std_path_buf()), + license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), + workspace_root: metadata.workspace_root.into_std_path_buf(), + build_dir: metadata.target_directory.into_std_path_buf(), + }) + } else { + None + }; let package_name = self .package_name @@ -226,8 +248,9 @@ impl Init { .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) .unwrap_or_default(); - let cargo_wapm_present = - cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); + let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 + && (cargo_wapm_stdout.contains("cargo wapm") + || cargo_wapm_stdout.contains("cargo-wapm")); // if Cargo.toml is present and cargo wapm is installed, add the // generated manifest to the Cargo.toml instead of creating a new wapm.toml @@ -249,7 +272,7 @@ impl Init { let note = "# See more keys and definitions at https://docs.wasmer.io/ecosystem/wapm/manifest"; - // generate the wapm.toml and exit + // generate the wasmer.toml and exit if !should_add_to_cargo_toml { let toml_string = toml::to_string_pretty(&constructed_manifest)? .replace("[dependencies]", &format!("{note}\r\n\r\n[dependencies]")) @@ -283,8 +306,6 @@ impl Init { bindings, }; - println!("{:#?}", metadata_wapm); - let toml_string = toml::to_string_pretty(&metadata_wapm)? .replace("[dependencies]", &format!("{note}\r\n\r\n[dependencies]")) .lines() @@ -297,10 +318,14 @@ impl Init { ); eprintln!("Build and publish your package with:"); eprintln!(); - eprintln!(" cargo wapm publish"); + eprintln!(" cargo wapm"); eprintln!(); } - std::fs::write(&manifest_path, &format!("{old_cargo}\r\n\r\n{toml_string}"))?; + + std::fs::write( + &manifest_path, + &format!("{old_cargo}\r\n\r\n[metadata.package.wapm]\r\n{toml_string}"), + )?; Ok(()) } diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index ce3c956e76f..57a3fee3f8a 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -81,8 +81,8 @@ fn wasmer_init_works_3() -> anyhow::Result<()> { .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) .unwrap_or_default(); - let cargo_wapm_present = - cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); + let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 + && (cargo_wapm_stdout.contains("cargo wapm") || cargo_wapm_stdout.contains("cargo-wapm")); if !cargo_wapm_present { println!("cargo wapm not present"); @@ -200,8 +200,8 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) .unwrap_or_default(); - let cargo_wapm_present = - cargo_wapm_stdout.lines().count() == 1 && cargo_wapm_stdout.contains("cargo wapm"); + let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 + && (cargo_wapm_stdout.contains("cargo wapm") || cargo_wapm_stdout.contains("cargo-wapm")); if cargo_wapm_present { assert!(!path.join("wasmer.toml").exists()); From 3c66829b0dd1708db715ee7f7524dace7ad13557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 12:39:11 +0100 Subject: [PATCH 32/88] Finish wasmer publish & verify it to work manually --- Makefile | 2 +- lib/cli/src/cli.rs | 5 ++-- lib/cli/src/commands/publish.rs | 43 ++++++++++++++++++++++----------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 3f14b775254..86579a67ee7 100644 --- a/Makefile +++ b/Makefile @@ -367,7 +367,7 @@ check-capi: capi-setup --no-default-features --features wat,compiler,wasi,middlewares $(capi_compiler_features) build-wasmer: - $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --features="webc_runner" --bin wasmer + $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml --features cranelift --features="webc_runner" --bin wasmer build-wasmer-debug: $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "webc_runner,debug" --bin wasmer diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 652bbf6dd85..b208f661956 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -235,9 +235,8 @@ fn wasmer_main_inner() -> Result<(), anyhow::Error> { } else { match command.unwrap_or(&"".to_string()).as_ref() { "add" | "cache" | "compile" | "config" | "create-exe" | "help" | "inspect" | "init" - | "run" | "self-update" | "validate" | "wast" | "binfmt" | "list" | "login" => { - WasmerCLIOptions::parse() - } + | "run" | "self-update" | "validate" | "wast" | "binfmt" | "list" | "login" + | "publish" => WasmerCLIOptions::parse(), _ => { WasmerCLIOptions::try_parse_from(args.iter()).unwrap_or_else(|e| { match e.kind() { diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index 7f6e2712b04..d3db08681e5 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -1,8 +1,9 @@ +use anyhow::Context; use clap::Parser; use flate2::{write::GzEncoder, Compression}; use rusqlite::{params, Connection, OpenFlags, TransactionBehavior}; use std::fs; -use std::io::Write; +use std::io::{self, Write}; use std::path::{Path, PathBuf}; use tar::Builder; use thiserror::Error; @@ -16,7 +17,7 @@ const RFC3339_FORMAT_STRING: &str = "%Y-%m-%dT%H:%M:%S-%f"; /// CLI options for the `wasmer publish` command #[derive(Debug, Parser)] pub struct Publish { - /// Directory containing the `wapm.toml` (defaults to current root dir) + /// Directory containing the `wasmer.toml` (defaults to current root dir) #[clap(long, name = "dir", env = "DIR")] pub dir: Option, /// Registry to publish to @@ -28,9 +29,11 @@ pub struct Publish { /// Run the publish command without any output #[clap(long)] pub quiet: bool, - /// Override the namespace of the uploaded package in the wapm.toml + /// Override the namespace of the uploaded package in the wasmer.toml + #[clap(long)] pub namespace: Option, /// Override the token (by default, it will use the current logged in user) + #[clap(long)] pub token: Option, } @@ -42,8 +45,8 @@ enum PublishError { SourceMustBeFile { module: String, path: PathBuf }, #[error("Unable to load the bindings for \"{module}\" because \"{}\" doesn't exist", path.display())] MissingBindings { module: String, path: PathBuf }, - #[error("Error building package when parsing module \"{0}\".")] - ErrorBuildingPackage(String), + #[error("Error building package when parsing module \"{0}\": {1}.")] + ErrorBuildingPackage(String, io::Error), #[error( "Path \"{0}\", specified in the manifest as part of the package file system does not exist.", )] @@ -60,16 +63,20 @@ impl Publish { let mut builder = Builder::new(Vec::new()); let cwd = match self.dir.as_ref() { - Some(s) => Path::new(&s).to_path_buf(), + Some(s) => std::env::current_dir()?.join(s), None => std::env::current_dir()?, }; // TODO: implement validation // validate::validate_directory(cwd.clone())?; - let manifest = wapm_toml::Manifest::find_in_directory(&cwd)?; - let manifest_path_buf = cwd.join("wasmer.toml"); + let manifest = std::fs::read_to_string(&manifest_path_buf) + .map_err(|e| anyhow::anyhow!("could not find manifest: {e}")) + .with_context(|| anyhow::anyhow!("{}", manifest_path_buf.display()))?; + let mut manifest = wapm_toml::Manifest::parse(&manifest)?; + manifest.base_directory_path = cwd.clone(); + builder.append_path_with_name(&manifest_path_buf, "wapm.toml")?; let package = &manifest.package; @@ -78,7 +85,10 @@ impl Publish { let readme = package.readme.as_ref().and_then(|readme_path| { let normalized_path = normalize_path(&manifest.base_directory_path, readme_path); - if builder.append_path(&normalized_path).is_err() { + if builder + .append_path_with_name(&normalized_path, readme_path) + .is_err() + { // TODO: Maybe do something here } fs::read_to_string(normalized_path).ok() @@ -86,7 +96,10 @@ impl Publish { let license_file = package.license_file.as_ref().and_then(|license_file_path| { let normalized_path = normalize_path(&manifest.base_directory_path, license_file_path); - if builder.append_path(&normalized_path).is_err() { + if builder + .append_path_with_name(&normalized_path, license_file_path) + .is_err() + { // TODO: Maybe do something here } fs::read_to_string(normalized_path).ok() @@ -101,8 +114,10 @@ impl Publish { path: normalized_path.clone(), })?; builder - .append_path(normalized_path) - .map_err(|_| PublishError::ErrorBuildingPackage(module.name.clone()))?; + .append_path_with_name(&normalized_path, &module.source) + .map_err(|e| { + PublishError::ErrorBuildingPackage(format!("{}", normalized_path.display()), e) + })?; if let Some(bindings) = &module.bindings { for path in bindings.referenced_files(&manifest.base_directory_path)? { @@ -114,8 +129,8 @@ impl Publish { path: normalized_path.clone(), })?; builder - .append_path(normalized_path) - .map_err(|_| PublishError::ErrorBuildingPackage(module.name.clone()))?; + .append_path_with_name(&normalized_path, &module.source) + .map_err(|e| PublishError::ErrorBuildingPackage(module.name.clone(), e))?; } } } From 20428fca9688939a88c377090b4d82811a221f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 13:06:25 +0100 Subject: [PATCH 33/88] Fix error: metadata.package -> package.metadata --- lib/cli/src/commands/init.rs | 2 +- tests/integration/cli/tests/fixtures/init3.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 4a2472e481f..7fdfc6fe90f 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -324,7 +324,7 @@ impl Init { std::fs::write( &manifest_path, - &format!("{old_cargo}\r\n\r\n[metadata.package.wapm]\r\n{toml_string}"), + &format!("{old_cargo}\r\n\r\n[package.metadata.wapm]\r\n{toml_string}"), )?; Ok(()) diff --git a/tests/integration/cli/tests/fixtures/init3.toml b/tests/integration/cli/tests/fixtures/init3.toml index f20cd0ccc0f..e09ce1cdc68 100644 --- a/tests/integration/cli/tests/fixtures/init3.toml +++ b/tests/integration/cli/tests/fixtures/init3.toml @@ -5,4 +5,4 @@ description = "hello" [dependencies] -[metadata.package.wapm] \ No newline at end of file +[package.metadata.wapm] \ No newline at end of file From 60323484acd455e7095f07562d7255148943df53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 14:53:37 +0100 Subject: [PATCH 34/88] Fix merge issues --- Cargo.lock | 78 ++++++++++++++++++++--------------------- lib/cli/Cargo.toml | 10 ++++++ lib/cli/src/cli.rs | 2 +- lib/cli/src/commands.rs | 4 +-- lib/registry/Cargo.toml | 2 ++ lib/registry/src/lib.rs | 1 - 6 files changed, 54 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cb0d967e46..6f85c3dda0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,9 +782,9 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" [[package]] name = "cxx" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" dependencies = [ "cc", "cxxbridge-flags", @@ -794,9 +794,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" dependencies = [ "cc", "codespan-reporting", @@ -809,15 +809,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" +checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" [[package]] name = "cxxbridge-macro" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" dependencies = [ "proc-macro2", "quote", @@ -871,9 +871,9 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4903dff04948f22033ca30232ab8eca2c3fc4c913a8b6a34ee5199699814817f" +checksum = "f8a16495aeb28047bb1185fca837baf755e7d71ed3aeed7f8504654ffa927208" dependencies = [ "proc-macro2", "quote", @@ -1599,9 +1599,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59df7c4e19c950e6e0e868dcc0a300b09a9b88e9ec55bd879ca819087a77355d" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", @@ -1777,9 +1777,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.5.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" +checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "isatty" @@ -1855,9 +1855,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.137" +version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "libfuzzer-sys" @@ -1886,6 +1886,7 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" dependencies = [ + "cc", "pkg-config", "vcpkg", ] @@ -2064,9 +2065,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "minisign" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f331f58ac7746ecb6c65843e5c8962103d0254e87ff0858be13a529988ea0333" +checksum = "ce49953dd06a44e1034590bb619bfe8900c29500053c0c0f83e9260a34466aa5" dependencies = [ "getrandom", "rpassword", @@ -2236,9 +2237,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "orbclient" -version = "0.3.39" +version = "0.3.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13aaa572be9c80bf827187cd4a188746bf2edb95368362d8ff3c785ff38c3ff" +checksum = "ba683f1641c11041c59d5d93689187abcab3c1349dc6d9d70c550c9f9360802f" dependencies = [ "cfg-if 1.0.0", "libc", @@ -2375,9 +2376,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15eb2c6e362923af47e13c23ca5afb859e83d54452c55b0b9ac763b8f7c1ac16" +checksum = "ac662b3a6490de378b0ee15cf2dfff7127aebfe0b19acc65e7fbca3d299c3788" [[package]] name = "ppv-lite86" @@ -2623,11 +2624,10 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] @@ -2790,7 +2790,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.22.5", + "webpki-roots 0.22.6", "winreg", ] @@ -3104,9 +3104,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.148" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" +checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" dependencies = [ "serde_derive", ] @@ -3143,9 +3143,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.148" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" +checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" dependencies = [ "proc-macro2", "quote", @@ -3652,9 +3652,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" dependencies = [ "autocfg", "bytes", @@ -3664,7 +3664,7 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -3789,9 +3789,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "typetag" @@ -4274,6 +4274,7 @@ dependencies = [ "dirs", "distance", "fern", + "flate2", "hex", "http_req", "isatty", @@ -4488,7 +4489,6 @@ version = "3.0.2" dependencies = [ "anyhow", "flate2", - "pretty_assertions", "rand 0.8.5", "tar", "target-lexicon 0.12.5", @@ -4522,7 +4522,7 @@ name = "wasmer-registry" version = "3.0.2" dependencies = [ "anyhow", -<<<<<<< HEAD + "console", "dirs", "filetime", "flate2", @@ -5008,9 +5008,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ "webpki 0.22.0", ] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 378183024d7..1c6bdd77d89 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -73,6 +73,16 @@ isatty = "0.1.9" dialoguer = "0.10.2" tldextract = "0.6.0" hex = "0.4.3" +flate2 = "1.0.25" +cargo_metadata = "0.15.2" +rusqlite = { version = "0.28.0", features = ["bundled"] } +tar = "0.4.38" +thiserror = "1.0.37" +time = "0.1.45" +log = "0.4.17" +minisign = "0.7.2" +semver = "1.0.14" +rpassword = "7.2.0" [build-dependencies] chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] } diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index b208f661956..b9f6a55ab51 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -11,7 +11,7 @@ use crate::commands::CreateObj; #[cfg(feature = "wast")] use crate::commands::Wast; use crate::commands::{ - Add, Cache, Config, Inspect, List, Login, Run, SelfUpdate, Validate, Whoami, + Add, Cache, Config, Init, Inspect, List, Login, Publish, Run, SelfUpdate, Validate, Whoami, }; use crate::error::PrettyError; use clap::{CommandFactory, ErrorKind, Parser}; diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index b1ae79461b9..00b814d4df5 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -33,8 +33,8 @@ pub use create_obj::*; #[cfg(feature = "wast")] pub use wast::*; pub use { - add::*, cache::*, config::*, inspect::*, list::*, login::*, run::*, self_update::*, - validate::*, whoami::*, + add::*, cache::*, config::*, init::*, inspect::*, list::*, login::*, publish::*, run::*, + self_update::*, validate::*, whoami::*, }; /// The kind of object format to emit. diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index 8ad0763502c..cc349c3162e 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -34,3 +34,5 @@ regex = "1.7.0" fs_extra = "1.2.0" filetime = "0.2.19" tldextract = "0.6.0" +console = "0.15.2" +indicatif = "0.17.2" \ No newline at end of file diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index 6af1789a196..9005112c9dd 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -22,7 +22,6 @@ pub mod config; pub mod graphql; pub mod login; pub mod package; -pub mod queries; pub mod publish; pub mod queries; pub mod utils; From 45102105bd02aec86520ba1f81ba636a4dd331de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 15:25:38 +0100 Subject: [PATCH 35/88] Add integration test for wasmer publish --- .../integration/cli/tests/fixtures/init6.toml | 13 ++++ tests/integration/cli/tests/publish.rs | 59 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 tests/integration/cli/tests/fixtures/init6.toml create mode 100644 tests/integration/cli/tests/publish.rs diff --git a/tests/integration/cli/tests/fixtures/init6.toml b/tests/integration/cli/tests/fixtures/init6.toml new file mode 100644 index 00000000000..fc044af9822 --- /dev/null +++ b/tests/integration/cli/tests/fixtures/init6.toml @@ -0,0 +1,13 @@ +[package] +name = "WAPMUSERNAME/largewasmfile" +version = "RANDOMVERSION1.RANDOMVERSION2.RANDOMVERSION3" +description = "published from wasmer largewasmfile" + +[[module]] +name = "largewasmfile" +source = "largewasmfile.wasm" +abi = "wasi" + +[[command]] +name = "largewasmfile" +module = "largewasmfile" diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs new file mode 100644 index 00000000000..fdcd4ef6927 --- /dev/null +++ b/tests/integration/cli/tests/publish.rs @@ -0,0 +1,59 @@ +use anyhow::bail; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use wasmer_integration_tests_cli::{get_repo_root_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; + +fn create_exe_test_wasm_path() -> String { + format!("{}/{}", C_ASSET_PATH, "qjs.wasm") +} + +#[test] +fn wasmer_publish() -> anyhow::Result<()> { + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").ok(); + let tempdir = tempfile::tempdir()?; + let path = tempdir.path(); + let username = "ciuser"; + + let random1 = format!("{}", rand::random::()); + let random2 = format!("{}", rand::random::()); + let random3 = format!("{}", rand::random::()); + + std::fs::copy(create_exe_test_wasm_path(), path.join("largewasmfile.wasm")).unwrap(); + std::fs::write( + path.join("wasmer.toml"), + include_str!("./fixtures/init6.toml") + .replace("WAPMUSERNAME", username) // <-- TODO! + .replace("RANDOMVERSION1", &random1) + .replace("RANDOMVERSION2", &random2) + .replace("RANDOMVERSION3", &random3), + )?; + + let mut cmd = std::process::Command::new(get_wasmer_path()); + cmd.arg("publish"); + cmd.arg("--dir"); + cmd.arg(path); + cmd.arg("--quiet"); + cmd.arg("--registry"); + cmd.arg("wapm.dev"); + + if let Some(token) = wapm_dev_token { + cmd.arg("--token"); + cmd.arg(token); + } + + let output = cmd + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .unwrap(); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert_eq!(stdout, format!("Successfully published package `{username}/largewasmfile@{random1}.{random2}.{random3}`\n"), "failed to publish: {cmd:?}: {stderr}"); + + println!("wasmer publish ok! test done."); + + Ok(()) +} From 9000d7995127f1651e0a4cfbcf6bf7fbf5e4747d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 19:54:28 +0100 Subject: [PATCH 36/88] Add pretty_assertions to integration test deps --- Cargo.lock | 1 + tests/integration/cli/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 6f85c3dda0c..ee9f0dbb15f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4489,6 +4489,7 @@ version = "3.0.2" dependencies = [ "anyhow", "flate2", + "pretty_assertions", "rand 0.8.5", "tar", "target-lexicon 0.12.5", diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index 6b769b32d55..f304657e3d6 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -12,6 +12,7 @@ rand = "0.8.5" tar = "0.4.38" flate2 = "1.0.24" target-lexicon = "0.12.4" +pretty_assertions = "1.3.0" [dependencies] anyhow = "1" From 5e1ee452dcf73cb3f244f86140862d00146e6d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 20:31:45 +0100 Subject: [PATCH 37/88] Upgrade to time v3 --- Cargo.lock | 50 +++++++++++++++++++-------------- lib/cli/Cargo.toml | 2 +- lib/cli/src/commands/publish.rs | 9 +++--- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee9f0dbb15f..924e5bcc73a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1326,7 +1326,7 @@ dependencies = [ "cfg-if 1.0.0", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -2100,7 +2100,7 @@ checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.42.0", ] @@ -3562,17 +3562,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "time" version = "0.2.27" @@ -3583,11 +3572,27 @@ dependencies = [ "libc", "standback", "stdweb", - "time-macros", + "time-macros 0.1.1", "version_check", "winapi", ] +[[package]] +name = "time" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +dependencies = [ + "time-core", + "time-macros 0.2.6", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "time-macros" version = "0.1.1" @@ -3598,6 +3603,15 @@ dependencies = [ "time-macros-impl", ] +[[package]] +name = "time-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] + [[package]] name = "time-macros-impl" version = "0.1.2" @@ -3980,12 +3994,6 @@ dependencies = [ "toml", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -4296,7 +4304,7 @@ dependencies = [ "tempdir", "tempfile", "thiserror", - "time 0.1.45", + "time 0.3.17", "tldextract", "toml", "unix_mode", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 1c6bdd77d89..9322ac9206a 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -78,7 +78,7 @@ cargo_metadata = "0.15.2" rusqlite = { version = "0.28.0", features = ["bundled"] } tar = "0.4.38" thiserror = "1.0.37" -time = "0.1.45" +time = { version = "0.3.17", default-features = false, features = ["parsing"] } log = "0.4.17" minisign = "0.7.2" semver = "1.0.14" diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index d3db08681e5..b612cde421b 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -7,12 +7,11 @@ use std::io::{self, Write}; use std::path::{Path, PathBuf}; use tar::Builder; use thiserror::Error; -use time::{self, Timespec}; +use time::{self, OffsetDateTime}; use wasmer_registry::publish::SignArchiveResult; use wasmer_registry::PartialWapmConfig; const CURRENT_DATA_VERSION: i32 = 3; -const RFC3339_FORMAT_STRING: &str = "%Y-%m-%dT%H:%M:%S-%f"; /// CLI options for the `wasmer publish` command #[derive(Debug, Parser)] @@ -366,7 +365,7 @@ pub struct PersonalKey { /// The type of private/public key this is pub key_type_identifier: String, /// The time at which the key was registered with wapm - pub date_created: Timespec, + pub date_created: OffsetDateTime, } fn get_active_personal_key(conn: &Connection) -> anyhow::Result { @@ -382,10 +381,10 @@ fn get_active_personal_key(conn: &Connection) -> anyhow::Result { public_key_value: row.get(1)?, private_key_location: row.get(2)?, date_created: { + use time::format_description::well_known::Rfc3339; let time_str: String = row.get(3)?; - time::strptime(&time_str, RFC3339_FORMAT_STRING) + OffsetDateTime::parse(&time_str, &Rfc3339) .unwrap_or_else(|_| panic!("Failed to parse time string {}", &time_str)) - .to_timespec() }, key_type_identifier: row.get(4)?, public_key_id: row.get(5)?, From eb3a217fabd59e12718ee398ec1a6d0c46f411ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 20:45:44 +0100 Subject: [PATCH 38/88] Fix registry code for wasmer.toml --- lib/registry/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index 9005112c9dd..c1a9c46addd 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -115,6 +115,7 @@ pub fn get_executable_file_from_path( command: Option<&str>, ) -> Result<(wapm_toml::Manifest, PathBuf), anyhow::Error> { let wapm_toml = std::fs::read_to_string(package_dir.join(GLOBAL_CONFIG_FILE_NAME)) + .or_else(|_| std::fs::read_to_string(package_dir.join("wasmer.toml"))) .map_err(|_| anyhow::anyhow!("Package {package_dir:?} has no {GLOBAL_CONFIG_FILE_NAME}"))?; let wapm_toml = toml::from_str::(&wapm_toml) From ba5004ad287549afea45e2f13488fb262b161067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 21:13:54 +0100 Subject: [PATCH 39/88] Add description to test-wasmer-init --- tests/integration/cli/tests/fixtures/init5.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/cli/tests/fixtures/init5.toml b/tests/integration/cli/tests/fixtures/init5.toml index 46ea5b98a33..7bd654df849 100644 --- a/tests/integration/cli/tests/fixtures/init5.toml +++ b/tests/integration/cli/tests/fixtures/init5.toml @@ -2,5 +2,6 @@ name = "test-wasmer-init" version = "RANDOMVERSION1.RANDOMVERSION2.RANDOMVERSION3" edition = "2021" +description = "description of package test-wasmer-init" [dependencies] \ No newline at end of file From 88172d873352de21e36783f3951b3c4f00d04bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 21:15:40 +0100 Subject: [PATCH 40/88] Fix Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 86579a67ee7..3f14b775254 100644 --- a/Makefile +++ b/Makefile @@ -367,7 +367,7 @@ check-capi: capi-setup --no-default-features --features wat,compiler,wasi,middlewares $(capi_compiler_features) build-wasmer: - $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml --features cranelift --features="webc_runner" --bin wasmer + $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --features="webc_runner" --bin wasmer build-wasmer-debug: $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "webc_runner,debug" --bin wasmer From fb6ab1388c8eebd6e880f9262e9e121ebc493bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 22:20:21 +0100 Subject: [PATCH 41/88] Add target wasm32-wasi --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 3f14b775254..55a2bc43234 100644 --- a/Makefile +++ b/Makefile @@ -549,6 +549,7 @@ test-examples: $(CARGO_BINARY) test $(CARGO_TARGET) --release $(compiler_features) --features wasi --examples test-integration-cli: + rustup target add wasm32-wasi $(CARGO_BINARY) test $(CARGO_TARGET) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture --test-threads=1 test-integration-ios: From eb0799a7585478ccecd93f7004008a30bc43a701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 22:49:42 +0100 Subject: [PATCH 42/88] source wapm in bash --- .github/workflows/test-sys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index 81fff0a3eb1..b664424b9d1 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -206,7 +206,7 @@ jobs: if: matrix.run_test && matrix.os != 'windows-2019' shell: bash run: | - curl https://get.wasmer.io -sSfL | sh && make build-wasmer && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli + curl https://get.wasmer.io -sSfL | sh && source ~/.wasmer/wasmer.sh && make build-wasmer && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli env: TARGET: ${{ matrix.target }} TARGET_DIR: target/${{ matrix.target }}/release From 0e3ef9e89873bc1a4f1a747b0d96318ced428f81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Mon, 12 Dec 2022 23:24:08 +0100 Subject: [PATCH 43/88] Remove cargo wapm test (tested manually) --- .github/workflows/test-sys.yaml | 2 +- tests/integration/cli/tests/init.rs | 147 +++---------------------- tests/integration/cli/tests/publish.rs | 5 + 3 files changed, 20 insertions(+), 134 deletions(-) diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index b664424b9d1..f440ede6fda 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -206,7 +206,7 @@ jobs: if: matrix.run_test && matrix.os != 'windows-2019' shell: bash run: | - curl https://get.wasmer.io -sSfL | sh && source ~/.wasmer/wasmer.sh && make build-wasmer && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli + make build-wasmer && make build-capi && make package-capi && make package && export WASMER_DIR=`pwd`/package && make test-integration-cli env: TARGET: ${{ matrix.target }} TARGET_DIR: target/${{ matrix.target }}/release diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index 57a3fee3f8a..848901e36fe 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -58,102 +58,6 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { Ok(()) } -// Test that wasmer init works with cargo wapm -#[cfg(not(target_os = "macos"))] -#[test] -fn wasmer_init_works_3() -> anyhow::Result<()> { - println!("starting test..."); - // running test locally: should always pass since - // developers don't have access to WAPM_DEV_TOKEN - if std::env::var("GITHUB_TOKEN").is_err() { - return Ok(()); - } - let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").expect("WAPM_DEV_TOKEN env var not set"); - println!("wapm dev token ok..."); - - let cargo_wapm_stdout = std::process::Command::new("cargo") - .arg("wapm") - .arg("--version") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) - .unwrap_or_default(); - - let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 - && (cargo_wapm_stdout.contains("cargo wapm") || cargo_wapm_stdout.contains("cargo-wapm")); - - if !cargo_wapm_present { - println!("cargo wapm not present"); - - // Install cargo wapm if not installed - let output = Command::new("cargo") - .arg("install") - .arg("cargo-wapm") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output()?; - - check_output!(output); - } - - let tempdir = tempfile::tempdir()?; - let path = tempdir.path(); - let path = path.join("testfirstproject"); - std::fs::create_dir_all(&path)?; - std::fs::write( - path.join("Cargo.toml"), - include_str!("./fixtures/init5.toml") - .replace("RANDOMVERSION1", &format!("{}", rand::random::())) - .replace("RANDOMVERSION2", &format!("{}", rand::random::())) - .replace("RANDOMVERSION3", &format!("{}", rand::random::())), - )?; - std::fs::create_dir_all(path.join("src"))?; - std::fs::write(path.join("src").join("main.rs"), b"fn main() { }")?; - - println!("project created"); - - let output = Command::new(get_wasmer_path()) - .arg("init") - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .stdin(Stdio::null()) - .current_dir(&path) - .output()?; - check_output!(output); - - println!("wasmer init ok!"); - - // login to wapm.dev, prepare for publish - let output = Command::new(get_wasmer_path()) - .arg("login") - .arg("--registry") - .arg("wapm.dev") - .arg(wapm_dev_token) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .stdin(Stdio::null()) - .output()?; - - println!("wasmer login ok!"); - - let output = Command::new("cargo") - .arg("wapm") - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .stdin(Stdio::null()) - .current_dir(&path) - .output()?; - - check_output!(output); - - println!("cargo wapm publish ok! test done."); - - Ok(()) -} - // Test that wasmer init adds to a Cargo.toml // instead of creating a new wapm.toml #[test] @@ -203,43 +107,20 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 && (cargo_wapm_stdout.contains("cargo wapm") || cargo_wapm_stdout.contains("cargo-wapm")); - if cargo_wapm_present { - assert!(!path.join("wasmer.toml").exists()); - let read = std::fs::read_to_string(path.join("Cargo.toml")) - .unwrap() - .lines() - .collect::>() - .join("\n"); - let target = include_str!("./fixtures/init3.toml") - .lines() - .collect::>() - .join("\n"); - pretty_assertions::assert_eq!(read.trim(), target.trim()); - - // Install cargo wapm if not installed - let output = Command::new("cargo") - .arg("install") - .arg("cargo-wapm") - .current_dir(&path) - .output()?; - - check_output!(output); - } else { - pretty_assertions::assert_eq!( - std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), - include_str!("./fixtures/init2.toml") - ); - let read = std::fs::read_to_string(path.join("wasmer.toml")) - .unwrap() - .lines() - .collect::>() - .join("\n"); - let target = include_str!("./fixtures/init4.toml") - .lines() - .collect::>() - .join("\n"); - pretty_assertions::assert_eq!(read.trim(), target.trim()); - } + pretty_assertions::assert_eq!( + std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), + include_str!("./fixtures/init2.toml") + ); + let read = std::fs::read_to_string(path.join("wasmer.toml")) + .unwrap() + .lines() + .collect::>() + .join("\n"); + let target = include_str!("./fixtures/init4.toml") + .lines() + .collect::>() + .join("\n"); + pretty_assertions::assert_eq!(read.trim(), target.trim()); Ok(()) } diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs index fdcd4ef6927..8c6d67bafc7 100644 --- a/tests/integration/cli/tests/publish.rs +++ b/tests/integration/cli/tests/publish.rs @@ -9,6 +9,11 @@ fn create_exe_test_wasm_path() -> String { #[test] fn wasmer_publish() -> anyhow::Result<()> { + // Only run this test in the CI + if std::env::var("GITHUB_TOKEN").is_err() { + return Ok(()); + } + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").ok(); let tempdir = tempfile::tempdir()?; let path = tempdir.path(); From 60bb15070bd937d9c4ebdfb1b2be3f9d9265d5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 13 Dec 2022 12:45:33 +0100 Subject: [PATCH 44/88] Add integration test to make sure that wasmer init + publish works --- lib/cli/src/commands/init.rs | 137 +++++++++++++++---------- lib/cli/src/commands/publish.rs | 42 ++++++-- lib/cli/src/commands/whoami.rs | 2 +- lib/registry/src/lib.rs | 7 +- tests/integration/cli/tests/publish.rs | 102 +++++++++++++++++- 5 files changed, 223 insertions(+), 67 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 7fdfc6fe90f..cfc2bbe5344 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -16,27 +16,34 @@ pub struct Init { /// Initialize an empty wapm.toml #[clap(long, name = "empty")] pub empty: bool, - /// If the `manifest-path` is a Cargo.toml, use that file to initialize the wapm.toml - #[clap(long, name = "manifest-path")] - pub manifest_path: Option, - /// Directory of the output file name. wasmer init will error in the target dir already contains a wapm.toml - #[clap(long, short = 'o', name = "out")] - pub out: Option, /// Force overwriting the wapm.toml, even if it already exists #[clap(long, name = "overwrite")] pub overwrite: bool, /// Don't display debug output #[clap(long, name = "quiet")] pub quiet: bool, + /// Ignore the existence of cargo wapm / cargo wasmer + #[clap(long, name = "no-cargo-wapm")] + pub no_cargo_wapm: bool, + /// Namespace to init with, default = current logged in user or _ + #[clap(long, name = "namespace")] + pub namespace: Option, + /// Version of the initialized package + #[clap(long, name = "version")] + pub version: Option, + /// If the `manifest-path` is a Cargo.toml, use that file to initialize the wapm.toml + #[clap(long, name = "manifest-path")] + pub manifest_path: Option, /// Add default dependencies for common packages (currently supported: `python`, `js`) #[clap(long, name = "template")] pub template: Option, /// Include file paths into the target container filesystem #[clap(long, name = "include")] pub include: Vec, - /// Name of the package, defaults to the name of the init directory - #[clap(name = "PACKAGE_NAME")] - pub package_name: Option, + /// Directory of the output file name. wasmer init will error in the target dir + /// already contains a wasmer.toml. Also sets the package name. + #[clap(name = "PACKAGE_PATH")] + pub out: Option, } #[derive(Debug, PartialEq, Copy, Clone)] @@ -70,10 +77,26 @@ impl Init { pub fn execute(&self) -> Result<(), anyhow::Error> { let bin_or_lib = self.get_bin_or_lib()?; + let package_name; let target_file = match self.out.as_ref() { - None => std::env::current_dir()?.join(WASMER_TOML_NAME), + None => { + let current_dir = std::env::current_dir()?; + package_name = current_dir + .canonicalize()? + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .ok_or_else(|| anyhow::anyhow!("no current dir name"))?; + current_dir.join(WASMER_TOML_NAME) + } Some(s) => { let _ = std::fs::create_dir_all(s); + package_name = s + .canonicalize()? + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .ok_or_else(|| anyhow::anyhow!("no dir name"))?; s.join(WASMER_TOML_NAME) } }; @@ -89,7 +112,11 @@ impl Init { let manifest_path = match self.manifest_path.as_ref() { Some(s) => s.clone(), None => { - let cargo_toml_path = std::env::current_dir().unwrap().join("Cargo.toml"); + let cargo_toml_path = self + .out + .clone() + .unwrap_or_else(|| std::env::current_exe().unwrap()) + .join("Cargo.toml"); cargo_toml_path .canonicalize() .unwrap_or_else(|_| cargo_toml_path.clone()) @@ -99,10 +126,14 @@ impl Init { let cargo_toml = if manifest_path.exists() { use anyhow::Context; - let metadata = MetadataCommand::new() - .manifest_path(&manifest_path) - .features(CargoOpt::AllFeatures) - .exec(); + let mut metadata = MetadataCommand::new(); + metadata.manifest_path(&manifest_path); + metadata.no_deps(); + metadata.features(CargoOpt::AllFeatures); + + println!("{:#?}", metadata.cargo_command()); + + let metadata = metadata.exec(); let metadata = match metadata { Ok(o) => o, @@ -127,45 +158,37 @@ impl Init { readme: package.readme.clone().map(|s| s.into_std_path_buf()), license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), workspace_root: metadata.workspace_root.into_std_path_buf(), - build_dir: metadata.target_directory.into_std_path_buf(), + build_dir: metadata + .target_directory + .into_std_path_buf() + .join("wasm32-wasi"), }) } else { None }; - let package_name = self - .package_name - .clone() - .or_else(|| cargo_toml.as_ref().map(|p| p.name.clone())) - .or_else(|| { - target_file - .parent() - .and_then(|p| Some(p.file_stem()?.to_str()?.to_string())) - }) - .or_else(|| { - Some( - std::env::current_dir() - .ok()? - .file_stem()? - .to_str()? - .to_string(), - ) - }) - .ok_or_else(|| anyhow!("current dir has no file stem"))?; - - let username = wasmer_registry::whoami(None).ok().map(|o| o.1); - let namespace = username - .or_else(|| package_name.split('/').next().map(|s| s.to_string())) - .unwrap_or_else(|| package_name.clone()); + let package_name = cargo_toml + .as_ref() + .map(|p| &p.name) + .unwrap_or(&package_name); + + let namespace = self.namespace.clone().unwrap_or_else(|| { + let username = wasmer_registry::whoami(None, None).ok().map(|o| o.1); + username + .or_else(|| package_name.split('/').next().map(|s| s.to_string())) + .unwrap_or_else(|| "_".to_string()) + }); let module_name = package_name .split('/') .last() .unwrap_or(&package_name) .to_string(); - let version = cargo_toml - .as_ref() - .map(|t| t.version.clone()) - .unwrap_or_else(|| semver::Version::parse("0.1.0").unwrap()); + let version = self.version.clone().unwrap_or_else(|| { + cargo_toml + .as_ref() + .map(|t| t.version.clone()) + .unwrap_or_else(|| semver::Version::parse("0.1.0").unwrap()) + }); let license = cargo_toml.as_ref().and_then(|t| t.license.clone()); let license_file = cargo_toml.as_ref().and_then(|t| t.license_file.clone()); let readme = cargo_toml.as_ref().and_then(|t| t.readme.clone()); @@ -241,16 +264,20 @@ impl Init { } // Test if cargo wapm is installed - let cargo_wapm_stdout = std::process::Command::new("cargo") - .arg("wapm") - .arg("--version") - .output() - .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) - .unwrap_or_default(); - - let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 - && (cargo_wapm_stdout.contains("cargo wapm") - || cargo_wapm_stdout.contains("cargo-wapm")); + let cargo_wapm_present = if self.no_cargo_wapm { + false + } else { + let cargo_wapm_stdout = std::process::Command::new("cargo") + .arg("wapm") + .arg("--version") + .output() + .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) + .unwrap_or_default(); + + cargo_wapm_stdout.lines().count() == 1 + && (cargo_wapm_stdout.contains("cargo wapm") + || cargo_wapm_stdout.contains("cargo-wapm")) + }; // if Cargo.toml is present and cargo wapm is installed, add the // generated manifest to the Cargo.toml instead of creating a new wapm.toml @@ -258,7 +285,7 @@ impl Init { // If the Cargo.toml is present, but cargo wapm is not installed, // generate a wapm.toml, but notify the user about installing cargo-wapm - if !cargo_wapm_present && cargo_toml.is_some() && !self.quiet { + if !cargo_wapm_present && !self.no_cargo_wapm && cargo_toml.is_some() && !self.quiet { eprintln!( "Note: you seem to have a Cargo.toml file, but you haven't installed `cargo wapm`." ); diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index b612cde421b..83726837ddc 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -16,9 +16,6 @@ const CURRENT_DATA_VERSION: i32 = 3; /// CLI options for the `wasmer publish` command #[derive(Debug, Parser)] pub struct Publish { - /// Directory containing the `wasmer.toml` (defaults to current root dir) - #[clap(long, name = "dir", env = "DIR")] - pub dir: Option, /// Registry to publish to #[clap(long)] pub registry: Option, @@ -34,6 +31,9 @@ pub struct Publish { /// Override the token (by default, it will use the current logged in user) #[clap(long)] pub token: Option, + /// Directory containing the `wasmer.toml` (defaults to current root dir) + #[clap(name = "PACKAGE_PATH")] + pub package_path: Option, } #[derive(Debug, Error)] @@ -57,11 +57,9 @@ enum PublishError { impl Publish { /// Executes `wasmer publish` pub fn execute(&self) -> Result<(), anyhow::Error> { - // First, check for the wasmer.toml and see if the user is logged in - // and has authorization to publish the package under the correct namespace. let mut builder = Builder::new(Vec::new()); - let cwd = match self.dir.as_ref() { + let cwd = match self.package_path.as_ref() { Some(s) => std::env::current_dir()?.join(s), None => std::env::current_dir()?, }; @@ -188,6 +186,38 @@ impl Publish { return Ok(()); } + // See if the user is logged in and has authorization to publish the package + // under the correct namespace before trying to upload. + let (registry, username) = + wasmer_registry::whoami(self.registry.as_deref(), self.token.as_deref()).with_context( + || { + anyhow::anyhow!( + "could not find username / registry for registry = {:?}, token = {}", + self.registry, + self.token.as_deref().unwrap_or_default() + ) + }, + )?; + + let registry_present = + wasmer_registry::test_if_registry_present(®istry).unwrap_or(false); + if !registry_present { + return Err(anyhow::anyhow!( + "registry {} is currently unavailable", + registry + )); + } + + let namespace = self + .namespace + .as_deref() + .or_else(|| package.name.split('/').next()) + .unwrap_or("") + .to_string(); + if username != namespace { + return Err(anyhow::anyhow!("trying to publish package under the namespace {namespace:?}, but logged in as user {username:?}")); + } + wasmer_registry::publish::try_chunked_uploading( self.registry.clone(), self.token.clone(), diff --git a/lib/cli/src/commands/whoami.rs b/lib/cli/src/commands/whoami.rs index 275467fc524..836bf87ce11 100644 --- a/lib/cli/src/commands/whoami.rs +++ b/lib/cli/src/commands/whoami.rs @@ -11,7 +11,7 @@ pub struct Whoami { impl Whoami { /// Execute `wasmer whoami` pub fn execute(&self) -> Result<(), anyhow::Error> { - let (registry, username) = wasmer_registry::whoami(self.registry.as_deref())?; + let (registry, username) = wasmer_registry::whoami(self.registry.as_deref(), None)?; println!("logged into registry {registry:?} as user {username:?}"); Ok(()) } diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index c1a9c46addd..456baf80a71 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -604,6 +604,7 @@ pub fn install_package(#[cfg(test)] test_name: &str, url: &Url) -> Result, + token: Option<&str>, ) -> Result<(String, String), anyhow::Error> { use crate::queries::{who_am_i_query, WhoAmIQuery}; use graphql_client::GraphQLQuery; @@ -622,9 +623,9 @@ pub fn whoami( None => config.registry.get_current_registry(), }; - let login_token = config - .registry - .get_login_token_for_registry(®istry) + let login_token = token + .map(|s| s.to_string()) + .or_else(|| config.registry.get_login_token_for_registry(®istry)) .ok_or_else(|| anyhow::anyhow!("not logged into registry {:?}", registry))?; let q = WhoAmIQuery::build_query(who_am_i_query::Variables {}); diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs index 8c6d67bafc7..822632b29e1 100644 --- a/tests/integration/cli/tests/publish.rs +++ b/tests/integration/cli/tests/publish.rs @@ -48,8 +48,8 @@ fn wasmer_publish() -> anyhow::Result<()> { let output = cmd .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) .output() .unwrap(); @@ -62,3 +62,101 @@ fn wasmer_publish() -> anyhow::Result<()> { Ok(()) } + +// Runs a full integration test to test that the flow wasmer init - cargo build - +// wasmer publish is working +#[test] +fn wasmer_init_publish() -> anyhow::Result<()> { + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").ok(); + let tempdir = tempfile::tempdir()?; + let path = tempdir.path(); + let username = "ciuser"; + + let random1 = format!("{}", rand::random::()); + let random2 = format!("{}", rand::random::()); + let random3 = format!("{}", rand::random::()); + + let mut cmd = std::process::Command::new("rustup"); + cmd.arg("target"); + cmd.arg("add"); + cmd.arg("wasm32-wasi"); + + let _ = cmd + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + + let mut cmd = std::process::Command::new("cargo"); + cmd.arg("init"); + cmd.arg("--bin"); + cmd.arg(path.join("randomversion")); + + let _ = cmd + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + + let mut cmd = std::process::Command::new("cargo"); + cmd.arg("build"); + cmd.arg("--release"); + cmd.arg("--target"); + cmd.arg("wasm32-wasi"); + cmd.arg("--manifest-path"); + cmd.arg(path.join("randomversion").join("Cargo.toml")); + + let _ = cmd + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + + // generate the wasmer.toml + let mut cmd = std::process::Command::new(get_wasmer_path()); + cmd.arg("init"); + cmd.arg("--no-cargo-wapm"); + cmd.arg("--namespace"); + cmd.arg(username); + cmd.arg("--version"); + cmd.arg(format!("{random1}.{random2}.{random3}")); + cmd.arg(path.join("randomversion")); + + let _ = cmd + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + + let s = std::fs::read_to_string(path.join("randomversion").join("wasmer.toml")).unwrap(); + + println!("{s}"); + + // publish + let mut cmd = std::process::Command::new(get_wasmer_path()); + cmd.arg("publish"); + cmd.arg("--quiet"); + cmd.arg("--registry"); + cmd.arg("wapm.dev"); + cmd.arg(path.join("randomversion")); + + if let Some(token) = wapm_dev_token { + cmd.arg("--token"); + cmd.arg(token); + } + + let output = cmd.stdin(Stdio::null()).output().unwrap(); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert_eq!(stdout, format!("Successfully published package `{username}/randomversion@{random1}.{random2}.{random3}`\n"), "failed to publish: {cmd:?}: {stderr}"); + + println!("wasmer init publish ok! test done."); + + Ok(()) +} From e4cd6e87d14888d1dc0ea945f98f43a325a18d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 13 Dec 2022 12:57:07 +0100 Subject: [PATCH 45/88] Fix make lint --- lib/cli/src/commands/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index cfc2bbe5344..e1224d4168b 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -181,7 +181,7 @@ impl Init { let module_name = package_name .split('/') .last() - .unwrap_or(&package_name) + .unwrap_or(package_name) .to_string(); let version = self.version.clone().unwrap_or_else(|| { cargo_toml From 050324b0dc70738ee9b711c3ad2b4aba6893f619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 13 Dec 2022 12:58:04 +0100 Subject: [PATCH 46/88] Only run wasmer_init_publish in the CI --- tests/integration/cli/tests/publish.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs index 822632b29e1..6776bff6464 100644 --- a/tests/integration/cli/tests/publish.rs +++ b/tests/integration/cli/tests/publish.rs @@ -67,6 +67,12 @@ fn wasmer_publish() -> anyhow::Result<()> { // wasmer publish is working #[test] fn wasmer_init_publish() -> anyhow::Result<()> { + + // Only run this test in the CI + if std::env::var("GITHUB_TOKEN").is_err() { + return Ok(()); + } + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").ok(); let tempdir = tempfile::tempdir()?; let path = tempdir.path(); From 41a54b606b9cf8beb128fee7a53aca21d062ca86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 13 Dec 2022 13:26:10 +0100 Subject: [PATCH 47/88] Add validation when publishing + checking the publishing username --- Cargo.lock | 62 +- Cargo.toml | 1 + lib/cli/Cargo.toml | 4 +- lib/cli/src/commands/publish.rs | 267 +++++++- lib/cli/src/commands/sql/get_interface.sql | 4 + lib/cli/src/commands/sql/insert_interface.sql | 3 + .../sql/wasm_interface_existence_check.sql | 5 + .../queries/get_interface_version.graphql | 9 + lib/registry/src/interface.rs | 47 ++ lib/registry/src/lib.rs | 1 + lib/wasm-interface/Cargo.toml | 24 + lib/wasm-interface/README.md | 88 +++ lib/wasm-interface/src/interface.rs | 205 ++++++ lib/wasm-interface/src/interface_matcher.rs | 29 + lib/wasm-interface/src/lib.rs | 13 + lib/wasm-interface/src/parser.rs | 596 ++++++++++++++++++ lib/wasm-interface/src/validate.rs | 465 ++++++++++++++ tests/integration/cli/tests/publish.rs | 3 +- 18 files changed, 1796 insertions(+), 30 deletions(-) create mode 100644 lib/cli/src/commands/sql/get_interface.sql create mode 100644 lib/cli/src/commands/sql/insert_interface.sql create mode 100644 lib/cli/src/commands/sql/wasm_interface_existence_check.sql create mode 100644 lib/registry/graphql/queries/get_interface_version.graphql create mode 100644 lib/registry/src/interface.rs create mode 100644 lib/wasm-interface/Cargo.toml create mode 100644 lib/wasm-interface/README.md create mode 100644 lib/wasm-interface/src/interface.rs create mode 100644 lib/wasm-interface/src/interface_matcher.rs create mode 100644 lib/wasm-interface/src/lib.rs create mode 100644 lib/wasm-interface/src/parser.rs create mode 100644 lib/wasm-interface/src/validate.rs diff --git a/Cargo.lock b/Cargo.lock index 9e9d650be0c..67506f457d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,6 +73,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "arrayvec" version = "0.7.2" @@ -176,7 +182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" dependencies = [ "arrayref", - "arrayvec", + "arrayvec 0.7.2", "cc", "cfg-if 1.0.0", "constant_time_eq", @@ -1844,6 +1850,19 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +[[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec 0.5.2", + "bitflags", + "cfg-if 1.0.0", + "ryu", + "static_assertions", +] + [[package]] name = "lexical-sort" version = "0.3.1" @@ -2145,6 +2164,17 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "lexical-core", + "memchr", + "version_check", +] + [[package]] name = "nom" version = "7.1.1" @@ -3309,6 +3339,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stdweb" version = "0.4.20" @@ -3583,6 +3619,8 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ + "itoa 1.0.4", + "serde", "time-core", "time-macros 0.2.6", ] @@ -4325,7 +4363,9 @@ dependencies = [ "wasmer-vm", "wasmer-wasi", "wasmer-wasi-experimental-io-devices", + "wasmer-wasm-interface", "wasmer-wast", + "wasmparser 0.51.4", "webc", ] @@ -4702,6 +4742,18 @@ dependencies = [ "wasmer-wit-parser", ] +[[package]] +name = "wasmer-wasm-interface" +version = "3.1.0" +dependencies = [ + "bincode", + "either", + "nom 5.1.2", + "serde", + "wasmparser 0.51.4", + "wat", +] + [[package]] name = "wasmer-wast" version = "3.1.0" @@ -4815,6 +4867,12 @@ dependencies = [ "wasmer-wast", ] +[[package]] +name = "wasmparser" +version = "0.51.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb1956b19469d1c5e63e459d29e7b5aa0f558d9f16fcef09736f8a265e6c10a" + [[package]] name = "wasmparser" version = "0.83.0" @@ -5201,7 +5259,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "463705a63313cd4301184381c5e8042f0a7e9b4bb63653f216311d4ae74690b7" dependencies = [ - "nom", + "nom 7.1.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 931d979465e..25dbdf20a83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ members = [ "lib/wasi-types", "lib/wasi-experimental-io-devices", "lib/wasi-local-networking", + "lib/wasm-interface", "lib/c-api/tests/wasmer-c-api-test-runner", "lib/c-api/examples/wasmer-capi-examples-runner", "lib/types", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 4cc5cfa88ed..1f528b04faf 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -40,6 +40,8 @@ wasmer-types = { version = "=3.1.0", path = "../types" } wasmer-registry = { version = "=3.1.0", path = "../registry" } wasmer-object = { version = "=3.1.0", path = "../object", optional = true } wasmer-vfs = { version = "=3.1.0", path = "../vfs", default-features = false, features = ["host-fs"] } +wasmer-wasm-interface = { version = "3.0.1", path = "../wasm-interface" } +wasmparser = "0.51.4" atty = "0.2" colored = "2.0" anyhow = "1.0" @@ -78,7 +80,7 @@ cargo_metadata = "0.15.2" rusqlite = { version = "0.28.0", features = ["bundled"] } tar = "0.4.38" thiserror = "1.0.37" -time = { version = "0.3.17", default-features = false, features = ["parsing"] } +time = { version = "0.3.17", default-features = false, features = ["parsing", "std", "formatting"] } log = "0.4.17" minisign = "0.7.2" semver = "1.0.14" diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index 83726837ddc..e0991b2e45c 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -64,9 +64,6 @@ impl Publish { None => std::env::current_dir()?, }; - // TODO: implement validation - // validate::validate_directory(cwd.clone())?; - let manifest_path_buf = cwd.join("wasmer.toml"); let manifest = std::fs::read_to_string(&manifest_path_buf) .map_err(|e| anyhow::anyhow!("could not find manifest: {e}")) @@ -74,6 +71,30 @@ impl Publish { let mut manifest = wapm_toml::Manifest::parse(&manifest)?; manifest.base_directory_path = cwd.clone(); + // See if the user is logged in and has authorization to publish the package + // under the correct namespace before trying to upload. + let (registry, username) = + wasmer_registry::whoami(self.registry.as_deref(), self.token.as_deref()).with_context( + || { + anyhow::anyhow!( + "could not find username / registry for registry = {:?}, token = {}", + self.registry, + self.token.as_deref().unwrap_or_default() + ) + }, + )?; + + let registry_present = + wasmer_registry::test_if_registry_present(®istry).unwrap_or(false); + if !registry_present { + return Err(anyhow::anyhow!( + "registry {} is currently unavailable", + registry + )); + } + + validate::validate_directory(&manifest, ®istry, cwd.clone())?; + builder.append_path_with_name(&manifest_path_buf, "wapm.toml")?; let package = &manifest.package; @@ -186,28 +207,6 @@ impl Publish { return Ok(()); } - // See if the user is logged in and has authorization to publish the package - // under the correct namespace before trying to upload. - let (registry, username) = - wasmer_registry::whoami(self.registry.as_deref(), self.token.as_deref()).with_context( - || { - anyhow::anyhow!( - "could not find username / registry for registry = {:?}, token = {}", - self.registry, - self.token.as_deref().unwrap_or_default() - ) - }, - )?; - - let registry_present = - wasmer_registry::test_if_registry_present(®istry).unwrap_or(false); - if !registry_present { - return Err(anyhow::anyhow!( - "registry {} is currently unavailable", - registry - )); - } - let namespace = self .namespace .as_deref() @@ -428,3 +427,221 @@ fn get_active_personal_key(conn: &Connection) -> anyhow::Result { Err(anyhow!("No active key found")) } } + +mod interfaces { + + use rusqlite::{params, Connection, TransactionBehavior}; + + pub const WASM_INTERFACE_EXISTENCE_CHECK: &str = + include_str!("./sql/wasm_interface_existence_check.sql"); + pub const INSERT_WASM_INTERFACE: &str = include_str!("./sql/insert_interface.sql"); + pub const GET_WASM_INTERFACE: &str = include_str!("./sql/get_interface.sql"); + + pub fn interface_exists( + conn: &mut Connection, + interface_name: &str, + version: &str, + ) -> anyhow::Result { + let mut stmt = conn.prepare(WASM_INTERFACE_EXISTENCE_CHECK)?; + Ok(stmt.exists(params![interface_name, version])?) + } + + pub fn load_interface_from_db( + conn: &mut Connection, + interface_name: &str, + version: &str, + ) -> anyhow::Result { + let mut stmt = conn.prepare(GET_WASM_INTERFACE)?; + let interface_string: String = + stmt.query_row(params![interface_name, version], |row| row.get(0))?; + + wasmer_wasm_interface::parser::parse_interface(&interface_string).map_err(|e| { + anyhow!( + "Failed to parse interface {} version {} in database: {}", + interface_name, + version, + e + ) + }) + } + + pub fn import_interface( + conn: &mut Connection, + interface_name: &str, + version: &str, + content: &str, + ) -> anyhow::Result<()> { + // fail if we already have this interface + { + let mut key_check = conn.prepare(WASM_INTERFACE_EXISTENCE_CHECK)?; + let result = key_check.exists(params![interface_name, version])?; + + if result { + return Err(anyhow!( + "Interface {}, version {} already exists", + interface_name, + version + )); + } + } + + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let time_string = get_current_time_in_format().expect("Could not get current time"); + + log::debug!("Adding interface {:?} {:?}", interface_name, version); + tx.execute( + INSERT_WASM_INTERFACE, + params![interface_name, version, time_string, content], + )?; + + tx.commit()?; + Ok(()) + } + + /// Gets the current time in our standard format + pub fn get_current_time_in_format() -> Option { + use time::format_description::well_known::Rfc3339; + let cur_time = time::OffsetDateTime::now_utc(); + cur_time.format(&Rfc3339).ok() + } +} + +mod validate { + use super::interfaces; + use std::{ + fs, + io::Read, + path::{Path, PathBuf}, + }; + use thiserror::Error; + use wasmer_registry::interface::InterfaceFromServer; + use wasmer_wasm_interface::{validate, Interface}; + + pub fn validate_directory( + manifest: &wapm_toml::Manifest, + registry: &str, + pkg_path: PathBuf, + ) -> anyhow::Result<()> { + // validate as dir + if let Some(modules) = manifest.module.as_ref() { + for module in modules.iter() { + let source_path = if module.source.is_relative() { + manifest.base_directory_path.join(&module.source) + } else { + module.source.clone() + }; + let source_path_string = source_path.to_string_lossy().to_string(); + let mut wasm_file = + fs::File::open(&source_path).map_err(|_| ValidationError::MissingFile { + file: source_path_string.clone(), + })?; + let mut wasm_buffer = Vec::new(); + wasm_file.read_to_end(&mut wasm_buffer).map_err(|err| { + ValidationError::MiscCannotRead { + file: source_path_string.clone(), + error: format!("{}", err), + } + })?; + + if let Some(bindings) = &module.bindings { + validate_bindings(bindings, &manifest.base_directory_path)?; + } + + // hack, short circuit if no interface for now + if module.interfaces.is_none() { + return validate_wasm_and_report_errors_old( + &wasm_buffer[..], + source_path_string, + ); + } + + let mut conn = super::open_db()?; + let mut interface: Interface = Default::default(); + for (interface_name, interface_version) in + module.interfaces.clone().unwrap_or_default().into_iter() + { + if !interfaces::interface_exists( + &mut conn, + &interface_name, + &interface_version, + )? { + // download interface and store it if we don't have it locally + let interface_data_from_server = InterfaceFromServer::get( + registry, + interface_name.clone(), + interface_version.clone(), + )?; + interfaces::import_interface( + &mut conn, + &interface_name, + &interface_version, + &interface_data_from_server.content, + )?; + } + let sub_interface = interfaces::load_interface_from_db( + &mut conn, + &interface_name, + &interface_version, + )?; + interface = interface.merge(sub_interface).map_err(|e| { + anyhow!("Failed to merge interface {}: {}", &interface_name, e) + })?; + } + validate::validate_wasm_and_report_errors(&wasm_buffer, &interface).map_err( + |e| ValidationError::InvalidWasm { + file: source_path_string, + error: format!("{:?}", e), + }, + )?; + } + } + log::debug!("package at path {:#?} validated", &pkg_path); + + Ok(()) + } + + fn validate_bindings( + bindings: &wapm_toml::Bindings, + base_directory_path: &Path, + ) -> Result<(), ValidationError> { + // Note: checking for referenced files will make sure they all exist. + let _ = bindings.referenced_files(base_directory_path)?; + + Ok(()) + } + + #[derive(Debug, Error)] + pub enum ValidationError { + #[error("WASM file \"{file}\" detected as invalid because {error}")] + InvalidWasm { file: String, error: String }, + #[error("Could not find file {file}")] + MissingFile { file: String }, + #[error("Failed to read file {file}; {error}")] + MiscCannotRead { file: String, error: String }, + #[error(transparent)] + Imports(#[from] wapm_toml::ImportsError), + } + + // legacy function, validates wasm. TODO: clean up + pub fn validate_wasm_and_report_errors_old( + wasm: &[u8], + file_name: String, + ) -> anyhow::Result<()> { + use wasmparser::WasmDecoder; + let mut parser = wasmparser::ValidatingParser::new(wasm, None); + loop { + let state = parser.read(); + match state { + wasmparser::ParserState::EndWasm => return Ok(()), + wasmparser::ParserState::Error(e) => { + return Err(ValidationError::InvalidWasm { + file: file_name, + error: format!("{}", e), + } + .into()); + } + _ => {} + } + } + } +} diff --git a/lib/cli/src/commands/sql/get_interface.sql b/lib/cli/src/commands/sql/get_interface.sql new file mode 100644 index 00000000000..8dffff7ece1 --- /dev/null +++ b/lib/cli/src/commands/sql/get_interface.sql @@ -0,0 +1,4 @@ +SELECT content +FROM wasm_interfaces +WHERE interface_name = (?1) + AND version = (?2) diff --git a/lib/cli/src/commands/sql/insert_interface.sql b/lib/cli/src/commands/sql/insert_interface.sql new file mode 100644 index 00000000000..b1f99678ff4 --- /dev/null +++ b/lib/cli/src/commands/sql/insert_interface.sql @@ -0,0 +1,3 @@ +INSERT INTO wasm_interfaces +(interface_name, version, date_added, content) +VALUES (?1, ?2, ?3, ?4) diff --git a/lib/cli/src/commands/sql/wasm_interface_existence_check.sql b/lib/cli/src/commands/sql/wasm_interface_existence_check.sql new file mode 100644 index 00000000000..d00f31e2d93 --- /dev/null +++ b/lib/cli/src/commands/sql/wasm_interface_existence_check.sql @@ -0,0 +1,5 @@ +SELECT 1 +FROM wasm_interfaces +WHERE interface_name = (?1) + AND version = (?2) +LIMIT 1 diff --git a/lib/registry/graphql/queries/get_interface_version.graphql b/lib/registry/graphql/queries/get_interface_version.graphql new file mode 100644 index 00000000000..4262e26508e --- /dev/null +++ b/lib/registry/graphql/queries/get_interface_version.graphql @@ -0,0 +1,9 @@ +query GetInterfaceVersionQuery ($name: String!, $version: String!) { + interface: getInterfaceVersion(name: $name, version: $version) { + version, + content, + interface { + name, + } + } +} \ No newline at end of file diff --git a/lib/registry/src/interface.rs b/lib/registry/src/interface.rs new file mode 100644 index 00000000000..4913c49d3c1 --- /dev/null +++ b/lib/registry/src/interface.rs @@ -0,0 +1,47 @@ +#![cfg_attr( + not(feature = "full"), + allow(dead_code, unused_imports, unused_variables) +)] +use crate::graphql::execute_query; +use graphql_client::*; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "graphql/schema.graphql", + query_path = "graphql/queries/get_interface_version.graphql", + response_derives = "Debug" +)] +struct GetInterfaceVersionQuery; + +#[derive(Debug)] +pub struct InterfaceFromServer { + pub name: String, + pub version: String, + pub content: String, +} + +impl InterfaceFromServer { + fn get_response( + registry: &str, + name: String, + version: String, + ) -> anyhow::Result { + let q = GetInterfaceVersionQuery::build_query(get_interface_version_query::Variables { + name, + version, + }); + execute_query(registry, "", &q) + } + + pub fn get(registry: &str, name: String, version: String) -> anyhow::Result { + let response = Self::get_response(registry, name, version)?; + let response_val = response + .interface + .ok_or_else(|| anyhow::anyhow!("Error downloading Interface from the server"))?; + Ok(Self { + name: response_val.interface.name, + version: response_val.version, + content: response_val.content, + }) + } +} diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index 456baf80a71..b25c6de6ca7 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -20,6 +20,7 @@ use url::Url; pub mod config; pub mod graphql; +pub mod interface; pub mod login; pub mod package; pub mod publish; diff --git a/lib/wasm-interface/Cargo.toml b/lib/wasm-interface/Cargo.toml new file mode 100644 index 00000000000..0c780394c0d --- /dev/null +++ b/lib/wasm-interface/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "wasmer-wasm-interface" +version = "3.1.0" +authors = ["The Wasmer Engineering Team "] +edition = "2018" +repository = "https://github.com/wasmerio/wapm-cli" +description = "WASM Interface definition and parser" +readme = "README.md" +license = "MIT" + +[dependencies] +bincode = { version = "1", optional = true } +either = "1.5" +nom = "5" +serde = { version = "1", features = ["derive"] } +wasmparser = { version = "0.51.4", optional = true } + +[dev-dependencies] +wat = "1.0" + +[features] +validation = ["wasmparser"] +binary_encode = ["bincode"] +default = ["validation"] diff --git a/lib/wasm-interface/README.md b/lib/wasm-interface/README.md new file mode 100644 index 00000000000..c3526b64310 --- /dev/null +++ b/lib/wasm-interface/README.md @@ -0,0 +1,88 @@ +# Wasm Interface + +This is an experimental crate for validating the imports and exports of a WebAssembly module. + +For the time being, Wasm Interface provides: + +- a convenient text format for specifying the requirements of Wasm modules +- a convenient way to compose interfaces safely (it ensures no conflicts (duplicates are allowed but they must agree)) +- validation that the modules meet the requirements + +## Syntax example + +Here's the interface for the current version of [WASI](https://github.com/WebAssembly/WASI): + +```lisp +(interface "wasi_unstable" + ;; Here's a bunch of function imports! + (func (import "wasi_unstable" "args_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "args_sizes_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "clock_res_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "clock_time_get") (param i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "environ_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "environ_sizes_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_advise") (param i32 i64 i64 i32) (result i32)) + (func (import "wasi_unstable" "fd_allocate") (param i32 i64 i64) (result i32)) + (func (import "wasi_unstable" "fd_close") (param i32) (result i32)) + (func (import "wasi_unstable" "fd_datasync") (param i32) (result i32)) + (func (import "wasi_unstable" "fd_fdstat_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_fdstat_set_flags") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_fdstat_set_rights") (param i32 i64 i64) (result i32)) + (func (import "wasi_unstable" "fd_filestat_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_filestat_set_size") (param i32 i64) (result i32)) + (func (import "wasi_unstable" "fd_filestat_set_times") (param i32 i64 i64 i32) (result i32)) + (func (import "wasi_unstable" "fd_pread") (param i32 i32 i32 i64 i32) (result i32)) + (func (import "wasi_unstable" "fd_prestat_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_prestat_dir_name") (param i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_pwrite") (param i32 i32 i32 i64 i32) (result i32)) + (func (import "wasi_unstable" "fd_read") (param i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_readdir") (param i32 i32 i32 i64 i32) (result i32)) + (func (import "wasi_unstable" "fd_renumber") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_seek") (param i32 i64 i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_sync") (param i32) (result i32)) + (func (import "wasi_unstable" "fd_tell") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "fd_write") (param i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_create_directory") (param i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_filestat_get") (param i32 i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_filestat_set_times") (param i32 i32 i32 i32 i64 i64 i32) (result i32)) + (func (import "wasi_unstable" "path_link") (param i32 i32 i32 i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_open") (param i32 i32 i32 i32 i32 i64 i64 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_readlink") (param i32 i32 i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_remove_directory") (param i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_rename") (param i32 i32 i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_symlink") (param i32 i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "path_unlink_file") (param i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "poll_oneoff") (param i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "proc_exit") (param i32)) + (func (import "wasi_unstable" "proc_raise") (param i32) (result i32)) + (func (import "wasi_unstable" "random_get") (param i32 i32) (result i32)) + (func (import "wasi_unstable" "sched_yield") (result i32)) + (func (import "wasi_unstable" "sock_recv") (param i32 i32 i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "sock_send") (param i32 i32 i32 i32 i32) (result i32)) + (func (import "wasi_unstable" "sock_shutdown") (param i32 i32) (result i32)) +) +``` + + +Notes: +- multiple `assert-import` and `assert-export` declarations are allowed. +- comments (starts with `;` and ends with a newline) and whitespace are valid between any tokens + +## Semantics + +All imports used by the module must be specified in the interface. + +All exports in the interface must be exported by the module. + +Thus the module may have additional exports than the interface or fewer imports than the interface specifies and be considered valid. + + +## Misc + +Wasm Interface serves a slightly different purpose than the proposed WebIDL for Wasm standard, but may be replaced by it in the future if things change. + +Due to an issue with nested closures in Rust, `wasm-interface` can't both compile on stable and have good error reporting. This is being fixed and `wasm-interface` will be updated to have better error handling. + +See the `parser.rs` file for a comment containing the grammar in a BNF style. + +Suggestions, contributions, and thoughts welcome! This is an experiment in the early stages, but we hope to work with the wider community and develop this in cooperation with all interested parties. diff --git a/lib/wasm-interface/src/interface.rs b/lib/wasm-interface/src/interface.rs new file mode 100644 index 00000000000..12a4d1f7554 --- /dev/null +++ b/lib/wasm-interface/src/interface.rs @@ -0,0 +1,205 @@ +//! The definition of a WASM interface + +use crate::interface_matcher::InterfaceMatcher; +use serde::{Deserialize, Serialize}; +use std::collections::{hash_map::Entry, HashMap, HashSet}; + +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Interface { + /// The name the interface gave itself + pub name: Option, + /// Things that the module can import + pub imports: HashMap<(String, String), Import>, + /// Things that the module must export + pub exports: HashMap, +} + +impl Interface { + pub fn merge(&self, other: Interface) -> Result { + let mut base = self.clone(); + + for (key, val) in other.imports { + match base.imports.entry(key) { + Entry::Occupied(e) if *e.get() != val => { + let (namespace, name) = e.key(); + let original_value = e.get(); + return Err(format!("Conflict detected: the import \"{namespace}\" \"{name}\" was found but the definitions were different: {original_value:?} {val:?}")); + } + Entry::Occupied(_) => { + // it's okay for the imported items to be the same. + } + Entry::Vacant(e) => { + e.insert(val); + } + }; + } + + for (key, val) in other.exports { + match base.exports.entry(key) { + Entry::Occupied(e) if *e.get() != val => { + let name = e.key(); + let original_value = e.get(); + return Err(format!("Conflict detected: the key \"{name}\" was found in exports but the definitions were different: {original_value:?} {val:?}")); + } + Entry::Occupied(_) => { + // it's okay for the exported items to be the same. + } + Entry::Vacant(e) => { + e.insert(val); + } + }; + } + + Ok(base) + } + + pub fn create_interface_matcher(&self) -> InterfaceMatcher { + let mut namespaces = HashSet::new(); + let mut namespace_imports: HashMap> = + HashMap::with_capacity(self.imports.len()); + let mut exports = HashSet::with_capacity(self.exports.len()); + + for (_, import) in self.imports.iter() { + match import { + Import::Func { namespace, .. } | Import::Global { namespace, .. } => { + if !namespaces.contains(namespace) { + namespaces.insert(namespace.clone()); + } + let ni = namespace_imports.entry(namespace.clone()).or_default(); + ni.insert(import.clone()); + } + } + } + for (_, export) in self.exports.iter() { + exports.insert(export.clone()); + } + InterfaceMatcher { + namespaces, + namespace_imports, + exports, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Import { + Func { + namespace: String, + name: String, + params: Vec, + result: Vec, + }, + Global { + namespace: String, + name: String, + var_type: WasmType, + }, +} + +impl Import { + pub fn format_key(ns: &str, name: &str) -> (String, String) { + (ns.to_string(), name.to_string()) + } + + /// Get the key used to look this import up in the Interface's import hashmap + pub fn get_key(&self) -> (String, String) { + match self { + Import::Func { + namespace, name, .. + } + | Import::Global { + namespace, name, .. + } => Self::format_key(namespace, name), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Export { + Func { + name: String, + params: Vec, + result: Vec, + }, + Global { + name: String, + var_type: WasmType, + }, +} + +impl Export { + pub fn format_key(name: &str) -> String { + name.to_string() + } + + /// Get the key used to look this export up in the Interface's export hashmap + pub fn get_key(&self) -> String { + match self { + Export::Func { name, .. } | Export::Global { name, .. } => Self::format_key(name), + } + } +} + +/// Primitive wasm type +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum WasmType { + I32, + I64, + F32, + F64, +} + +impl std::fmt::Display for WasmType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + match self { + WasmType::I32 => "i32", + WasmType::I64 => "i64", + WasmType::F32 => "f32", + WasmType::F64 => "f64", + } + ) + } +} + +#[cfg(test)] +mod test { + use crate::parser; + + #[test] + fn merging_works() { + let interface1_src = + r#"(interface (func (import "env" "plus_one") (param i32) (result i32)))"#; + let interface2_src = + r#"(interface (func (import "env" "plus_one") (param i64) (result i64)))"#; + let interface3_src = + r#"(interface (func (import "env" "times_two") (param i64) (result i64)))"#; + let interface4_src = + r#"(interface (func (import "env" "times_two") (param i64 i64) (result i64)))"#; + let interface5_src = r#"(interface (func (export "empty_bank_account") (param) (result)))"#; + let interface6_src = + r#"(interface (func (export "empty_bank_account") (param) (result i64)))"#; + + let interface1 = parser::parse_interface(interface1_src).unwrap(); + let interface2 = parser::parse_interface(interface2_src).unwrap(); + let interface3 = parser::parse_interface(interface3_src).unwrap(); + let interface4 = parser::parse_interface(interface4_src).unwrap(); + let interface5 = parser::parse_interface(interface5_src).unwrap(); + let interface6 = parser::parse_interface(interface6_src).unwrap(); + + assert!(interface1.merge(interface2.clone()).is_err()); + assert!(interface2.merge(interface1.clone()).is_err()); + assert!(interface1.merge(interface3.clone()).is_ok()); + assert!(interface2.merge(interface3.clone()).is_ok()); + assert!(interface3.merge(interface2).is_ok()); + assert!( + interface1.merge(interface1.clone()).is_ok(), + "exact matches are accepted" + ); + assert!(interface3.merge(interface4).is_err()); + assert!(interface5.merge(interface5.clone()).is_ok()); + assert!(interface5.merge(interface6).is_err()); + } +} diff --git a/lib/wasm-interface/src/interface_matcher.rs b/lib/wasm-interface/src/interface_matcher.rs new file mode 100644 index 00000000000..c9bf62b06fc --- /dev/null +++ b/lib/wasm-interface/src/interface_matcher.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +use crate::interface::{Export, Import}; + +/// A struct containing data for more efficient matching. +/// +/// An ideal use case for this is to parse [`Interface`]s at compile time, +/// create [`InterfaceMatcher`]s, and store them as bytes so that they +/// can be efficiently loaded at runtime for matching. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct InterfaceMatcher { + pub namespaces: HashSet, + pub namespace_imports: HashMap>, + pub exports: HashSet, +} + +#[cfg(feature = "binary_encode")] +impl InterfaceMatcher { + /// Store the matcher as bytes to avoid reparsing + fn into_bytes(&self) -> Vec { + bincode::serialize(self).expect("Could not serialize InterfaceMatcher") + } + + /// Load the matcher from bytes to avoid reparsing + fn from_bytes(bytes: &[u8]) -> Option { + bincode::deserialize(bytes).ok() + } +} diff --git a/lib/wasm-interface/src/lib.rs b/lib/wasm-interface/src/lib.rs new file mode 100644 index 00000000000..06fab2d3731 --- /dev/null +++ b/lib/wasm-interface/src/lib.rs @@ -0,0 +1,13 @@ +#![type_length_limit = "5795522"] +//! Definition and parsing of wasm interfaces +//! +//! wasm interfaces ensure wasm modules conform to a specific shape +//! they do this by asserting on the imports and exports of the module. + +pub mod interface; +pub mod interface_matcher; +pub mod parser; +#[cfg(feature = "validation")] +pub mod validate; + +pub use interface::*; diff --git a/lib/wasm-interface/src/parser.rs b/lib/wasm-interface/src/parser.rs new file mode 100644 index 00000000000..03475a9b303 --- /dev/null +++ b/lib/wasm-interface/src/parser.rs @@ -0,0 +1,596 @@ +//! Parsers to get a wasm interface from text +//! +//! The grammar of the text format is: +//! interface = "(" interface name? interface-entry* ")" +//! interface-entry = func | global +//! +//! func = import-fn | export-fn +//! global = import-global | export-global +//! +//! import-fn = "(" "func" import-id param-list? result-list? ")" +//! import-global = "(" "global" import-id type-decl ")" +//! import-id = "(" "import" namespace name ")" +//! +//! export-fn = "(" "func" export-id param-list? result-list? ")" +//! export-global = "(" "global" export-id type-decl ")" +//! export-id = "(" export name ")" +//! +//! param-list = "(" param type* ")" +//! result-list = "(" result type* ")" +//! type-decl = "(" "type" type ")" +//! namespace = "\"" identifier "\"" +//! name = "\"" identifier "\"" +//! identifier = any character that's not a whitespace character or an open or close parenthesis +//! type = "i32" | "i64" | "f32" | "f64" +//! +//! + means 1 or more +//! * means 0 or more +//! ? means 0 or 1 +//! | means "or" +//! "\"" means one `"` character +//! +//! comments start with a `;` character and go until a newline `\n` character is reached +//! comments and whitespace are valid between any tokens + +use either::Either; +use nom::{ + branch::*, + bytes::complete::{escaped, is_not, tag}, + character::complete::{char, multispace0, multispace1, one_of}, + combinator::*, + error::context, + multi::many0, + sequence::{delimited, preceded, tuple}, + IResult, +}; + +use crate::interface::*; + +/// Some example input: +/// (interface "example_interface" +/// (func (import "ns" "name") (param f64 i32) (result f64 i32)) +/// (func (export "name") (param f64 i32) (result f64 i32)) +/// (global (import "ns" "name") (type f64))) +pub fn parse_interface(mut input: &str) -> Result { + let mut interface = Interface::default(); + let interface_inner = preceded( + tag("interface"), + tuple(( + opt(preceded(space_comments, identifier)), + many0(parse_func_or_global), + )), + ); + let interface_parser = preceded(space_comments, s_exp(interface_inner)); + + if let Result::Ok((inp, (sig_id, out))) = interface_parser(input) { + interface.name = sig_id.map(|s_id| s_id.to_string()); + + for entry in out.into_iter() { + match entry { + Either::Left(import) => { + if let Some(dup) = interface.imports.insert(import.get_key(), import) { + return Err(format!("Duplicate import found {:?}", dup)); + } + } + Either::Right(export) => { + if let Some(dup) = interface.exports.insert(export.get_key(), export) { + return Err(format!("Duplicate export found {:?}", dup)); + } + } + } + } + input = inp; + } + // catch trailing comments and spaces + if let Ok((inp, _)) = space_comments(input) { + input = inp; + } + if !input.is_empty() { + Err(format!("Could not parse remaining input: {}", input)) + } else { + Ok(interface) + } +} + +fn parse_comment(input: &str) -> IResult<&str, ()> { + map( + preceded(multispace0, preceded(char(';'), many0(is_not("\n")))), + |_| (), + )(input) +} + +/// Consumes spaces and comments +/// comments must terminate with a new line character +fn space_comments<'a>(mut input: &'a str) -> IResult<&'a str, ()> { + let mut space_found = true; + let mut comment_found = true; + while space_found || comment_found { + let space: IResult<&'a str, _> = multispace1(input); + space_found = if let Result::Ok((inp, _)) = space { + input = inp; + true + } else { + false + }; + comment_found = if let Result::Ok((inp, _)) = parse_comment(input) { + input = inp; + true + } else { + false + }; + } + Ok((input, ())) +} + +/// A quoted identifier, must be valid UTF8 +fn identifier(input: &str) -> IResult<&str, &str> { + let name_inner = escaped(is_not("\"\\"), '\\', one_of("\"n\\")); + context("identifier", delimited(char('"'), name_inner, char('"')))(input) +} + +/// Parses a wasm primitive type +fn wasm_type(input: &str) -> IResult<&str, WasmType> { + let i32_tag = map(tag("i32"), |_| WasmType::I32); + let i64_tag = map(tag("i64"), |_| WasmType::I64); + let f32_tag = map(tag("f32"), |_| WasmType::F32); + let f64_tag = map(tag("f64"), |_| WasmType::F64); + + alt((i32_tag, i64_tag, f32_tag, f64_tag))(input) +} + +/// Parses an S-expression +fn s_exp<'a, O1, F>(inner: F) -> impl Fn(&'a str) -> IResult<&'a str, O1> +where + F: Fn(&'a str) -> IResult<&'a str, O1>, +{ + delimited( + char('('), + preceded(space_comments, inner), + preceded(space_comments, char(')')), + ) +} + +fn parse_func_or_global(input: &str) -> IResult<&str, Either> { + preceded(space_comments, alt((func, global)))(input) +} + +/// (func (import "ns" "name") (param f64 i32) (result f64 i32)) +/// (func (export "name") (param f64 i32) (result f64 i32)) +fn func(input: &str) -> IResult<&str, Either> { + let param_list_inner = preceded(tag("param"), many0(preceded(space_comments, wasm_type))); + let param_list = opt(s_exp(param_list_inner)); + let result_list_inner = preceded(tag("result"), many0(preceded(space_comments, wasm_type))); + let result_list = opt(s_exp(result_list_inner)); + let import_id_inner = preceded( + tag("import"), + tuple(( + preceded(space_comments, identifier), + preceded(space_comments, identifier), + )), + ); + let export_id_inner = preceded(tag("export"), preceded(space_comments, identifier)); + let func_id_inner = alt(( + map(import_id_inner, |(ns, name)| { + Either::Left((ns.to_string(), name.to_string())) + }), + map(export_id_inner, |name| Either::Right(name.to_string())), + )); + let func_id = s_exp(func_id_inner); + let func_import_inner = context( + "func import inner", + preceded( + tag("func"), + map( + tuple(( + preceded(space_comments, func_id), + preceded(space_comments, param_list), + preceded(space_comments, result_list), + )), + |(func_id, pl, rl)| match func_id { + Either::Left((ns, name)) => Either::Left(Import::Func { + namespace: ns, + name, + params: pl.unwrap_or_default(), + result: rl.unwrap_or_default(), + }), + Either::Right(name) => Either::Right(Export::Func { + name, + params: pl.unwrap_or_default(), + result: rl.unwrap_or_default(), + }), + }, + ), + ), + ); + s_exp(func_import_inner)(input) +} + +/// (global (import "ns" "name") (type f64)) +/// (global (export "name") (type f64)) +fn global(input: &str) -> IResult<&str, Either> { + let global_type_inner = preceded(tag("type"), preceded(space_comments, wasm_type)); + let type_s_exp = s_exp(global_type_inner); + let export_inner = preceded(tag("export"), preceded(space_comments, identifier)); + let import_inner = preceded( + tag("import"), + tuple(( + preceded(space_comments, identifier), + preceded(space_comments, identifier), + )), + ); + let global_id_inner = alt(( + map(import_inner, |(ns, name)| { + Either::Left(Import::Global { + namespace: ns.to_string(), + name: name.to_string(), + // placeholder type, overwritten in `global_inner` + var_type: WasmType::I32, + }) + }), + map(export_inner, |name| { + Either::Right(Export::Global { + name: name.to_string(), + // placeholder type, overwritten in `global_inner` + var_type: WasmType::I32, + }) + }), + )); + let global_id = s_exp(global_id_inner); + let global_inner = context( + "global inner", + preceded( + tag("global"), + map( + tuple(( + preceded(space_comments, global_id), + preceded(space_comments, type_s_exp), + )), + |(import_or_export, var_type)| match import_or_export { + Either::Left(Import::Global { + namespace, name, .. + }) => Either::Left(Import::Global { + namespace, + name, + var_type, + }), + Either::Right(Export::Global { name, .. }) => { + Either::Right(Export::Global { name, var_type }) + } + _ => unreachable!("Invalid value interonally in parse global function"), + }, + ), + ), + ); + s_exp(global_inner)(input) +} + +#[cfg(test)] +mod test { + use super::*; + use std::collections::HashMap; + + #[test] + fn parse_wasm_type() { + let i32_res = wasm_type("i32").unwrap(); + assert_eq!(i32_res, ("", WasmType::I32)); + let i64_res = wasm_type("i64").unwrap(); + assert_eq!(i64_res, ("", WasmType::I64)); + let f32_res = wasm_type("f32").unwrap(); + assert_eq!(f32_res, ("", WasmType::F32)); + let f64_res = wasm_type("f64").unwrap(); + assert_eq!(f64_res, ("", WasmType::F64)); + + assert!(wasm_type("i128").is_err()); + } + + #[test] + fn parse_identifier() { + let inner_str = "柴は可愛すぎるだと思います"; + let input = format!("\"{}\"", &inner_str); + let parse_res = identifier(&input).unwrap(); + assert_eq!(parse_res, ("", inner_str)) + } + + #[test] + fn parse_global_import() { + let parse_res = global(r#"(global (import "env" "length") (type i32))"#) + .ok() + .and_then(|(a, b)| Some((a, b.left()?))) + .unwrap(); + assert_eq!( + parse_res, + ( + "", + Import::Global { + namespace: "env".to_string(), + name: "length".to_string(), + var_type: WasmType::I32, + } + ) + ); + } + + #[test] + fn parse_global_export() { + let parse_res = global(r#"(global (export "length") (type i32))"#) + .ok() + .and_then(|(a, b)| Some((a, b.right()?))) + .unwrap(); + assert_eq!( + parse_res, + ( + "", + Export::Global { + name: "length".to_string(), + var_type: WasmType::I32, + } + ) + ); + } + + #[test] + fn parse_func_import() { + let parse_res = func(r#"(func (import "ns" "name") (param f64 i32) (result f64 i32))"#) + .ok() + .and_then(|(a, b)| Some((a, b.left()?))) + .unwrap(); + assert_eq!( + parse_res, + ( + "", + Import::Func { + namespace: "ns".to_string(), + name: "name".to_string(), + params: vec![WasmType::F64, WasmType::I32], + result: vec![WasmType::F64, WasmType::I32], + } + ) + ); + } + + #[test] + fn parse_func_export() { + let parse_res = func(r#"(func (export "name") (param f64 i32) (result f64 i32))"#) + .ok() + .and_then(|(a, b)| Some((a, b.right()?))) + .unwrap(); + assert_eq!( + parse_res, + ( + "", + Export::Func { + name: "name".to_string(), + params: vec![WasmType::F64, WasmType::I32], + result: vec![WasmType::F64, WasmType::I32], + } + ) + ); + + let parse_res = func(r#"(func (export "name"))"#) + .ok() + .and_then(|(a, b)| Some((a, b.right()?))) + .unwrap(); + assert_eq!( + parse_res, + ( + "", + Export::Func { + name: "name".to_string(), + params: vec![], + result: vec![], + } + ) + ) + } + + #[test] + fn parse_imports_test() { + let parse_imports = |in_str| { + many0(parse_func_or_global)(in_str) + .map(|(a, b)| { + ( + a, + b.into_iter().filter_map(|x| x.left()).collect::>(), + ) + }) + .unwrap() + }; + let parse_res = + parse_imports(r#"(func (import "ns" "name") (param f64 i32) (result f64 i32))"#); + assert_eq!( + parse_res, + ( + "", + vec![Import::Func { + namespace: "ns".to_string(), + name: "name".to_string(), + params: vec![WasmType::F64, WasmType::I32], + result: vec![WasmType::F64, WasmType::I32], + }] + ) + ); + + let parse_res = parse_imports( + r#"(func (import "ns" "name") + (param f64 i32) (result f64 i32)) + ( global ( import "env" "length" ) ( type + ;; i32 is the best type + i32 ) + ) + (func (import "ns" "name2") (param f32 + i64) + ;; The return value comes next + ( + result + f64 + i32 + ) + )"#, + ); + assert_eq!( + parse_res, + ( + "", + vec![ + Import::Func { + namespace: "ns".to_string(), + name: "name".to_string(), + params: vec![WasmType::F64, WasmType::I32], + result: vec![WasmType::F64, WasmType::I32], + }, + Import::Global { + namespace: "env".to_string(), + name: "length".to_string(), + var_type: WasmType::I32, + }, + Import::Func { + namespace: "ns".to_string(), + name: "name2".to_string(), + params: vec![WasmType::F32, WasmType::I64], + result: vec![WasmType::F64, WasmType::I32], + }, + ] + ) + ); + } + + #[test] + fn top_level_test() { + let parse_res = parse_interface( + r#" (interface + (func (import "ns" "name") (param f64 i32) (result f64 i32)) + (func (export "name2") (param) (result i32)) + (global (import "env" "length") (type f64)))"#, + ) + .unwrap(); + + let imports = vec![ + Import::Func { + namespace: "ns".to_string(), + name: "name".to_string(), + params: vec![WasmType::F64, WasmType::I32], + result: vec![WasmType::F64, WasmType::I32], + }, + Import::Global { + namespace: "env".to_string(), + name: "length".to_string(), + var_type: WasmType::F64, + }, + ]; + let exports = vec![Export::Func { + name: "name2".to_string(), + params: vec![], + result: vec![WasmType::I32], + }]; + let import_map = imports + .into_iter() + .map(|entry| (entry.get_key(), entry)) + .collect::>(); + let export_map = exports + .into_iter() + .map(|entry| (entry.get_key(), entry)) + .collect::>(); + assert_eq!( + parse_res, + Interface { + name: None, + imports: import_map, + exports: export_map, + } + ); + } + + #[test] + fn duplicates_not_allowed() { + let parse_res = parse_interface( + r#" (interface "sig_name" (func (import "ns" "name") (param f64 i32) (result f64 i32)) +; test comment + ;; hello + (func (import "ns" "name") (param) (result i32)) + (global (export "length") (type f64))) + +"#, + ); + + assert!(parse_res.is_err()); + } + + #[test] + fn test_comment_space_parsing() { + let parse_res = space_comments(" ").unwrap(); + assert_eq!(parse_res, ("", ())); + let parse_res = space_comments("").unwrap(); + assert_eq!(parse_res, ("", ())); + let parse_res = space_comments("; hello\n").unwrap(); + assert_eq!(parse_res, ("", ())); + let parse_res = space_comments("abc").unwrap(); + assert_eq!(parse_res, ("abc", ())); + let parse_res = space_comments("\n ; hello\n ").unwrap(); + assert_eq!(parse_res, ("", ())); + let parse_res = space_comments("\n ; hello\n ; abc\n\n ; hello\n").unwrap(); + assert_eq!(parse_res, ("", ())); + } + + #[test] + fn test_param_elision() { + let parse_res = parse_interface( + r#" (interface "interface_name" (func (import "ns" "name") (result f64 i32)) +(func (export "name"))) +"#, + ) + .unwrap(); + + let imports = vec![Import::Func { + namespace: "ns".to_string(), + name: "name".to_string(), + params: vec![], + result: vec![WasmType::F64, WasmType::I32], + }]; + let exports = vec![Export::Func { + name: "name".to_string(), + params: vec![], + result: vec![], + }]; + let import_map = imports + .into_iter() + .map(|entry| (entry.get_key(), entry)) + .collect::>(); + let export_map = exports + .into_iter() + .map(|entry| (entry.get_key(), entry)) + .collect::>(); + assert_eq!( + parse_res, + Interface { + name: Some("interface_name".to_string()), + imports: import_map, + exports: export_map, + } + ); + } + + #[test] + fn typo_gets_caught() { + let interface_src = r#" +(interface "interface_id" +(func (import "env" "do_panic") (params i32 i64)) +(global (import "length") (type i32)))"#; + let result = parse_interface(interface_src); + assert!(result.is_err()); + } + + #[test] + fn parse_trailing_spaces_on_interface() { + let parse_res = parse_interface( + r#" (interface "really_good_interface" (func (import "ns" "name") (param f64 i32) (result f64 i32)) +; test comment + ;; hello + (global (import "ns" "length") (type f64)) +) + +"#, + ); + + assert!(parse_res.is_ok()); + } +} diff --git a/lib/wasm-interface/src/validate.rs b/lib/wasm-interface/src/validate.rs new file mode 100644 index 00000000000..23eac282115 --- /dev/null +++ b/lib/wasm-interface/src/validate.rs @@ -0,0 +1,465 @@ +//! Validate a wasm module given a interface. +//! +//! This checks that all imports are specified in the interface and that their types +//! are correct, as well as that all exports that the interface expects are exported +//! by the module and that their types are correct. + +use crate::{Export, Import, Interface, WasmType}; +use std::collections::HashMap; +use wasmparser::{ExternalKind, FuncType, GlobalType, ImportSectionEntryType}; + +pub fn validate_wasm_and_report_errors( + wasm: &[u8], + interface: &Interface, +) -> Result<(), WasmValidationError> { + use wasmparser::WasmDecoder; + + let mut errors: Vec = vec![]; + let mut import_fns: HashMap<(String, String), u32> = HashMap::new(); + let mut export_fns: HashMap = HashMap::new(); + let mut export_globals: HashMap = HashMap::new(); + let mut type_defs: Vec = vec![]; + let mut global_types: Vec = vec![]; + let mut fn_sigs: Vec = vec![]; + + let mut parser = wasmparser::ValidatingParser::new(wasm, None); + loop { + let state = parser.read(); + match state { + wasmparser::ParserState::EndWasm => break, + wasmparser::ParserState::Error(e) => { + return Err(WasmValidationError::InvalidWasm { + error: format!("{}", e), + }); + } + wasmparser::ParserState::ImportSectionEntry { + module, + field, + ref ty, + } => match ty { + ImportSectionEntryType::Function(idx) => { + import_fns.insert(Import::format_key(module, field), *idx); + fn_sigs.push(*idx); + } + ImportSectionEntryType::Global(GlobalType { content_type, .. }) => { + let global_type = + wasmparser_type_into_wasm_type(*content_type).map_err(|err| { + WasmValidationError::UnsupportedType { + error: format!( + "Invalid type found in import \"{}\" \"{}\": {}", + module, field, err + ), + } + })?; + if let Some(val) = interface.imports.get(&Import::format_key(module, field)) { + if let Import::Global { var_type, .. } = val { + if *var_type != global_type { + errors.push(format!( + "Invalid type on Global \"{}\". Expected {} found {}", + field, var_type, global_type + )); + } + } else { + errors.push(format!( + "Invalid import type. Expected Global, found {:?}", + val + )); + } + } else { + errors.push(format!( + "Global import \"{}\" not found in the specified interface", + field + )); + } + } + _ => (), + }, + wasmparser::ParserState::ExportSectionEntry { + field, + index, + ref kind, + } => match kind { + ExternalKind::Function => { + export_fns.insert(Export::format_key(field), *index); + } + ExternalKind::Global => { + export_globals.insert(Export::format_key(field), *index); + } + _ => (), + }, + wasmparser::ParserState::BeginGlobalSectionEntry(gt) => { + global_types.push(*gt); + } + wasmparser::ParserState::TypeSectionEntry(ft) => { + type_defs.push(ft.clone()); + } + wasmparser::ParserState::FunctionSectionEntry(n) => { + fn_sigs.push(*n); + } + _ => {} + } + } + + validate_imports(&import_fns, &type_defs, interface, &mut errors); + validate_export_fns(&export_fns, &type_defs, &fn_sigs, interface, &mut errors); + validate_export_globals(&export_globals, &global_types, interface, &mut errors); + + if errors.is_empty() { + Ok(()) + } else { + Err(WasmValidationError::InterfaceViolated { errors }) + } +} + +/// Validates the import functions, checking the name and type against the given +/// `Interface` +fn validate_imports( + import_fns: &HashMap<(String, String), u32>, + type_defs: &[FuncType], + interface: &Interface, + errors: &mut Vec, +) { + for (key, val) in import_fns.iter() { + if let Some(interface_def) = interface.imports.get(key) { + let type_sig = if let Some(v) = type_defs.get(*val as usize) { + v + } else { + errors.push(format!( + "Use of undeclared function reference \"{}\" in import function \"{}\" \"{}\"", + val, key.0, key.1 + )); + continue; + }; + if let Import::Func { params, result, .. } = interface_def { + debug_assert!(type_sig.form == wasmparser::Type::Func); + for (i, param) in type_sig + .params + .iter() + .cloned() + .map(wasmparser_type_into_wasm_type) + .enumerate() + { + match param { + Ok(t) => { + if params.get(i).is_none() { + errors.push(format!("Found {} args but the interface only expects {} for imported function \"{}\" \"{}\"", i, params.len(), &key.0, &key.1)); + continue; + } + if t != params[i] { + errors.push(format!( + "Type mismatch in params in imported func \"{}\" \"{}\": argument {}, expected {} found {}", + &key.0, &key.1, i + 1, params[i], t + )); + } + } + Err(e) => errors.push(format!( + "Invalid type in func \"{}\" \"{}\": {}", + &key.0, &key.1, e + )), + } + } + for (i, ret) in type_sig + .returns + .iter() + .cloned() + .map(wasmparser_type_into_wasm_type) + .enumerate() + { + match ret { + Ok(t) => { + if result.get(i).is_none() { + errors.push(format!("Found {} returns but the interface only expects {} for imported function \"{}\" \"{}\"", i, params.len(), &key.0, &key.1)); + continue; + } + + if t != result[i] { + errors.push(format!( + "Type mismatch in returns in func \"{}\" \"{}\", return {}, expected {} found {}", + &key.0, &key.1, i + 1, params[i], t + )); + } + } + Err(e) => errors.push(format!( + "Invalid type in func \"{}\" \"{}\": {}", + &key.0, &key.1, e + )), + } + } + } + } else { + // we didn't find the import at all in the interface + // TODO: improve error messages by including type information + errors.push(format!("Missing import \"{}\" \"{}\"", key.0, key.1)); + } + } +} + +/// Validates the export functions, checking the name and type against the given +/// `Interface` +fn validate_export_fns( + export_fns: &HashMap, + type_defs: &[FuncType], + fn_sigs: &Vec, + interface: &Interface, + errors: &mut Vec, +) { + 'export_loop: for (key, val) in export_fns.iter() { + if let Some(interface_def) = interface.exports.get(key) { + let type_sig = if let Some(type_idx) = fn_sigs.get(*val as usize) { + if let Some(v) = type_defs.get(*type_idx as usize) { + v + } else { + errors.push(format!( + "Export \"{}\" refers to type \"{}\" but only {} types were found", + &key, + type_idx, + fn_sigs.len() + )); + continue; + } + } else { + errors.push(format!( + "Use of undeclared function reference \"{}\" in export \"{}\"", + val, &key + )); + continue; + }; + if let Export::Func { params, result, .. } = interface_def { + debug_assert!(type_sig.form == wasmparser::Type::Func); + for (i, param) in type_sig + .params + .iter() + .cloned() + .map(wasmparser_type_into_wasm_type) + .enumerate() + { + match param { + Ok(t) => { + if params.get(i).is_none() { + errors.push(format!("Found {} args but the interface only expects {} for exported function \"{}\"", type_sig.params.len(), params.len(), &key)); + continue 'export_loop; + } + if t != params[i] { + errors.push(format!( + "Type mismatch in params in exported func \"{}\": in argument {}, expected {} found {}", + &key, i + 1, params[i], t + )); + } + } + Err(e) => errors + .push(format!("Invalid type in exported func \"{}\": {}", &key, e)), + } + } + for (i, ret) in type_sig + .returns + .iter() + .cloned() + .map(wasmparser_type_into_wasm_type) + .enumerate() + { + match ret { + Ok(t) => { + if result.get(i).is_none() { + errors.push(format!("Found {} returns but the interface only expects {} for exported function \"{}\"", i, params.len(), &key)); + continue 'export_loop; + } + + if t != result[i] { + errors.push(format!( + "Type mismatch in returns in exported func \"{}\": in return {}, expected {} found {}", + &key, i + 1, result[i], t + )); + } + } + Err(e) => errors + .push(format!("Invalid type in exported func \"{}\": {}", &key, e)), + } + } + } + } + } +} + +/// Validates the export globals, checking the name and type against the given +/// `Interface` +fn validate_export_globals( + export_globals: &HashMap, + global_types: &Vec, + interface: &Interface, + errors: &mut Vec, +) { + for (key, val) in export_globals.iter() { + if let Some(Export::Global { var_type, .. }) = interface.exports.get(key) { + if global_types.get(*val as usize).is_none() { + errors.push(format!( + "Invalid wasm, expected {} global types, found {}", + val, + global_types.len() + )); + } + match wasmparser_type_into_wasm_type(global_types[*val as usize].content_type) { + Ok(t) => { + if *var_type != t { + errors.push(format!( + "Type mismatch in global export {}: expected {} found {}", + &key, var_type, t + )); + } + } + Err(e) => errors.push(format!("In global export {}: {}", &key, e)), + } + } + } +} + +/// Converts Wasmparser's type enum into wasm-interface's type enum +/// wasmparser's enum contains things which are invalid in many situations +/// +/// Additionally wasmerparser containers more advanced types like references that +/// wasm-interface does not yet support +fn wasmparser_type_into_wasm_type(ty: wasmparser::Type) -> Result { + use wasmparser::Type; + Ok(match ty { + Type::I32 => WasmType::I32, + Type::I64 => WasmType::I64, + Type::F32 => WasmType::F32, + Type::F64 => WasmType::F64, + e => { + return Err(format!("Invalid type found: {:?}", e)); + } + }) +} + +#[cfg(test)] +mod validation_tests { + use super::*; + use crate::parser; + + #[test] + fn global_imports() { + const WAT: &str = r#"(module +(type $t0 (func (param i32 i64))) +(global $length (import "env" "length") i32) +(import "env" "do_panic" (func $do_panic (type $t0))) +)"#; + let wasm = wat::parse_str(WAT).unwrap(); + + let interface_src = r#" +(interface +(func (import "env" "do_panic") (param i32 i64)) +(global (import "env" "length") (type i32)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!(result.is_ok()); + + // Now set the global import type to mismatch the wasm + let interface_src = r#" +(interface +(func (import "env" "do_panic") (param i32 i64)) +(global (import "env" "length") (type i64)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!( + result.is_err(), + "global import type mismatch causes an error" + ); + + // Now set the function import type to mismatch the wasm + let interface_src = r#" +(interface +(func (import "env" "do_panic") (param i64)) +(global (import "env" "length") (type i32)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!( + result.is_err(), + "function import type mismatch causes an error" + ); + + // Now try with a module that has an import that the interface doesn't have + let interface_src = r#" +(interface +(func (import "env" "do_panic") (param i64)) +(global (import "env" "length_plus_plus") (type i32)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!( + result.is_err(), + "all imports must be covered by the interface" + ); + } + + #[test] + fn global_exports() { + const WAT: &str = r#"(module +(func (export "as-set_local-first") (param i32) (result i32) + (nop) (i32.const 2) (set_local 0) (get_local 0)) +(global (export "num_tries") i64 (i64.const 0)) +)"#; + let wasm = wat::parse_str(WAT).unwrap(); + + let interface_src = r#" +(interface +(func (export "as-set_local-first") (param i32) (result i32)) +(global (export "num_tries") (type i64)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!(result.is_ok()); + + // Now set the global export type to mismatch the wasm + let interface_src = r#" +(interface +(func (export "as-set_local-first") (param i32) (result i32)) +(global (export "num_tries") (type f32)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!( + result.is_err(), + "global export type mismatch causes an error" + ); + + // Now set the function export type to mismatch the wasm + let interface_src = r#" +(interface +(func (export "as-set_local-first") (param i64) (result i64)) +(global (export "num_tries") (type i64)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!( + result.is_err(), + "function export type mismatch causes an error" + ); + + // Now try a interface that requires an export that the module doesn't have + let interface_src = r#" +(interface +(func (export "as-set_local-first") (param i64) (result i64)) +(global (export "numb_trees") (type i64)))"#; + let interface = parser::parse_interface(interface_src).unwrap(); + + let result = validate_wasm_and_report_errors(&wasm[..], &interface); + + assert!(result.is_err(), "missing a required export is an error"); + } +} + +#[derive(Debug)] +pub enum WasmValidationError { + InvalidWasm { error: String }, + InterfaceViolated { errors: Vec }, + UnsupportedType { error: String }, +} diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs index 6776bff6464..7b8b6fcafc0 100644 --- a/tests/integration/cli/tests/publish.rs +++ b/tests/integration/cli/tests/publish.rs @@ -67,12 +67,11 @@ fn wasmer_publish() -> anyhow::Result<()> { // wasmer publish is working #[test] fn wasmer_init_publish() -> anyhow::Result<()> { - // Only run this test in the CI if std::env::var("GITHUB_TOKEN").is_err() { return Ok(()); } - + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").ok(); let tempdir = tempfile::tempdir()?; let path = tempdir.path(); From 9553c50a85d40c9ab8b6a6d9ca546fae5d695365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 13 Dec 2022 15:00:51 +0100 Subject: [PATCH 48/88] Fix integration tests --- lib/cli/src/commands/init.rs | 4 +- .../integration/cli/tests/fixtures/init4.toml | 2 +- tests/integration/cli/tests/init.rs | 66 ++++++++++--------- tests/integration/cli/tests/publish.rs | 10 +-- 4 files changed, 38 insertions(+), 44 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index e1224d4168b..e17fd6a3b91 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -115,7 +115,7 @@ impl Init { let cargo_toml_path = self .out .clone() - .unwrap_or_else(|| std::env::current_exe().unwrap()) + .unwrap_or_else(|| std::env::current_dir().unwrap()) .join("Cargo.toml"); cargo_toml_path .canonicalize() @@ -131,8 +131,6 @@ impl Init { metadata.no_deps(); metadata.features(CargoOpt::AllFeatures); - println!("{:#?}", metadata.cargo_command()); - let metadata = metadata.exec(); let metadata = match metadata { diff --git a/tests/integration/cli/tests/fixtures/init4.toml b/tests/integration/cli/tests/fixtures/init4.toml index fff30418ac8..8a672b131f3 100644 --- a/tests/integration/cli/tests/fixtures/init4.toml +++ b/tests/integration/cli/tests/fixtures/init4.toml @@ -9,7 +9,7 @@ description = 'Description for package wasmer' [[module]] name = 'wasmer' -source = 'target/release/wasmer.wasm' +source = 'target/wasm32-wasi/release/wasmer.wasm' abi = 'wasi' [module.interfaces] diff --git a/tests/integration/cli/tests/init.rs b/tests/integration/cli/tests/init.rs index 848901e36fe..b1185c2b0ea 100644 --- a/tests/integration/cli/tests/init.rs +++ b/tests/integration/cli/tests/init.rs @@ -24,18 +24,22 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { if std::env::var("GITHUB_TOKEN").is_err() { return Ok(()); } - let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").expect("WAPM_DEV_TOKEN env var not set"); + + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").ok(); println!("wapm dev token ok..."); - let output = Command::new(get_wasmer_path()) - .arg("login") - .arg("--registry") - .arg("wapm.dev") - .arg(wapm_dev_token) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .stdin(Stdio::null()) - .output()?; + if let Some(token) = wapm_dev_token { + let output = Command::new(get_wasmer_path()) + .arg("login") + .arg("--registry") + .arg("wapm.dev") + .arg(token) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .stdin(Stdio::null()) + .output()?; + check_output!(output); + } println!("wasmer login ok!"); @@ -58,8 +62,6 @@ fn wasmer_init_works_1() -> anyhow::Result<()> { Ok(()) } -// Test that wasmer init adds to a Cargo.toml -// instead of creating a new wapm.toml #[test] fn wasmer_init_works_2() -> anyhow::Result<()> { let tempdir = tempfile::tempdir()?; @@ -76,41 +78,41 @@ fn wasmer_init_works_2() -> anyhow::Result<()> { if std::env::var("GITHUB_TOKEN").is_err() { return Ok(()); } - let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").expect("WAPM_DEV_TOKEN env var not set"); + + let wapm_dev_token = std::env::var("WAPM_DEV_TOKEN").ok(); println!("wapm dev token ok..."); - let output = Command::new(get_wasmer_path()) - .arg("login") - .arg("--registry") - .arg("wapm.dev") - .arg(wapm_dev_token) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .stdin(Stdio::null()) - .output()?; + if let Some(token) = wapm_dev_token.as_ref() { + let mut cmd = Command::new(get_wasmer_path()); + cmd.arg("login"); + cmd.arg("--registry"); + cmd.arg("wapm.dev"); + cmd.arg(token); + cmd.stdout(Stdio::inherit()); + cmd.stderr(Stdio::inherit()); + cmd.stdin(Stdio::null()); + let output = cmd.output()?; + check_output!(output); + } println!("wasmer login ok!"); let output = Command::new(get_wasmer_path()) .arg("init") + .arg("--no-cargo-wapm") + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) .current_dir(&path) .output()?; check_output!(output); - let cargo_wapm_stdout = std::process::Command::new("cargo") - .arg("wapm") - .arg("--version") - .output() - .map(|s| String::from_utf8_lossy(&s.stdout).to_string()) - .unwrap_or_default(); - - let cargo_wapm_present = cargo_wapm_stdout.lines().count() == 1 - && (cargo_wapm_stdout.contains("cargo wapm") || cargo_wapm_stdout.contains("cargo-wapm")); - pretty_assertions::assert_eq!( std::fs::read_to_string(path.join("Cargo.toml")).unwrap(), include_str!("./fixtures/init2.toml") ); + + println!("ok 1"); + let read = std::fs::read_to_string(path.join("wasmer.toml")) .unwrap() .lines() diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs index 7b8b6fcafc0..c1c7cb8a6d0 100644 --- a/tests/integration/cli/tests/publish.rs +++ b/tests/integration/cli/tests/publish.rs @@ -35,23 +35,17 @@ fn wasmer_publish() -> anyhow::Result<()> { let mut cmd = std::process::Command::new(get_wasmer_path()); cmd.arg("publish"); - cmd.arg("--dir"); - cmd.arg(path); cmd.arg("--quiet"); cmd.arg("--registry"); cmd.arg("wapm.dev"); + cmd.arg(path); if let Some(token) = wapm_dev_token { cmd.arg("--token"); cmd.arg(token); } - let output = cmd - .stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .output() - .unwrap(); + let output = cmd.stdin(Stdio::null()).output().unwrap(); let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); From 7c8607ffbeaa7f5dc646de592a614e7ce2bc2869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 14 Dec 2022 13:07:21 +0100 Subject: [PATCH 49/88] Use rust-toolchain --- rust-toolchain | 6 +++++- tests/integration/cli/tests/publish.rs | 12 ------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/rust-toolchain b/rust-toolchain index 58e4eb6b299..9670e8a4bcf 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1,5 @@ -1.63 +# rust-toolchain.toml +[toolchain] +channel = "1.63" +components = [ "rustfmt", "clippy" ] +targets = [ "wasm32-unknown-unknown", "wasm32-wasi" ] \ No newline at end of file diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs index c1c7cb8a6d0..e5850051ff6 100644 --- a/tests/integration/cli/tests/publish.rs +++ b/tests/integration/cli/tests/publish.rs @@ -75,18 +75,6 @@ fn wasmer_init_publish() -> anyhow::Result<()> { let random2 = format!("{}", rand::random::()); let random3 = format!("{}", rand::random::()); - let mut cmd = std::process::Command::new("rustup"); - cmd.arg("target"); - cmd.arg("add"); - cmd.arg("wasm32-wasi"); - - let _ = cmd - .stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - let mut cmd = std::process::Command::new("cargo"); cmd.arg("init"); cmd.arg("--bin"); From 2c7ee61d39633e8a583fc1383f3a0241816f8248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <12084016+fschutt@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:09:35 +0100 Subject: [PATCH 50/88] Remove name = "..." on CLI options Co-authored-by: Michael Bryan --- lib/cli/src/commands/init.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index e17fd6a3b91..dfaaeed8591 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -8,37 +8,37 @@ use std::path::PathBuf; #[derive(Debug, Parser)] pub struct Init { /// Initialize wapm.toml for a library package - #[clap(long, name = "lib")] + #[clap(long)] pub lib: bool, /// Initialize wapm.toml for a binary package - #[clap(long, name = "bin")] + #[clap(long)] pub bin: bool, /// Initialize an empty wapm.toml - #[clap(long, name = "empty")] + #[clap(long)] pub empty: bool, /// Force overwriting the wapm.toml, even if it already exists - #[clap(long, name = "overwrite")] + #[clap(long")] pub overwrite: bool, /// Don't display debug output - #[clap(long, name = "quiet")] + #[clap(long)] pub quiet: bool, /// Ignore the existence of cargo wapm / cargo wasmer - #[clap(long, name = "no-cargo-wapm")] + #[clap(long)] pub no_cargo_wapm: bool, /// Namespace to init with, default = current logged in user or _ - #[clap(long, name = "namespace")] + #[clap(long)] pub namespace: Option, /// Version of the initialized package - #[clap(long, name = "version")] + #[clap(long)] pub version: Option, /// If the `manifest-path` is a Cargo.toml, use that file to initialize the wapm.toml - #[clap(long, name = "manifest-path")] + #[clap(long)] pub manifest_path: Option, /// Add default dependencies for common packages (currently supported: `python`, `js`) - #[clap(long, name = "template")] + #[clap(long)] pub template: Option, /// Include file paths into the target container filesystem - #[clap(long, name = "include")] + #[clap(long)] pub include: Vec, /// Directory of the output file name. wasmer init will error in the target dir /// already contains a wasmer.toml. Also sets the package name. From ddeb9f77c646ee1ad9245710170ead195722047c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 14 Dec 2022 13:14:36 +0100 Subject: [PATCH 51/88] Fix migration number selection --- lib/cli/src/commands/publish.rs | 49 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index e0991b2e45c..8d1109dcfac 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -348,32 +348,29 @@ fn apply_migration(conn: &mut Connection, migration_number: i32) -> Result<(), M let tx = conn .transaction_with_behavior(TransactionBehavior::Immediate) .map_err(|e| MigrationError::TransactionFailed(migration_number, format!("{}", e)))?; - match migration_number { - 0 => { - tx.execute_batch(include_str!("../../sql/migrations/0000.sql")) - .map_err(|e| { - MigrationError::TransactionFailed(migration_number, format!("{}", e)) - })?; - } - 1 => { - tx.execute_batch(include_str!("../../sql/migrations/0001.sql")) - .map_err(|e| { - MigrationError::TransactionFailed(migration_number, format!("{}", e)) - })?; - } - 2 => { - tx.execute_batch(include_str!("../../sql/migrations/0002.sql")) - .map_err(|e| { - MigrationError::TransactionFailed(migration_number, format!("{}", e)) - })?; - } - _ => { - return Err(MigrationError::MigrationNumberDoesNotExist( - migration_number, - CURRENT_DATA_VERSION, - )); - } - } + + let migrations = &[ + (0, include_str!("../../sql/migrations/0000.sql")), + (1, include_str!("../../sql/migrations/0001.sql")), + (2, include_str!("../../sql/migrations/0002.sql")), + ]; + + let migration_to_apply = migrations + .iter() + .find_map(|(number, sql)| { + if *number == migration_number { + Some(sql) + } else { + None + } + }) + .ok_or_else(|| { + MigrationError::MigrationNumberDoesNotExist(migration_number, CURRENT_DATA_VERSION) + })?; + + tx.execute_batch(&migration_to_apply) + .map_err(|e| MigrationError::TransactionFailed(migration_number, format!("{}", e)))?; + tx.pragma_update(None, "user_version", &(migration_number + 1)) .map_err(|e| MigrationError::TransactionFailed(migration_number, format!("{}", e)))?; tx.commit() From c7967666de3d38624a842fad4d941b2d824c6f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <12084016+fschutt@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:16:28 +0100 Subject: [PATCH 52/88] Fix typo Co-authored-by: Michael Bryan --- lib/cli/src/commands/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index dfaaeed8591..59c870512b3 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -40,7 +40,7 @@ pub struct Init { /// Include file paths into the target container filesystem #[clap(long)] pub include: Vec, - /// Directory of the output file name. wasmer init will error in the target dir + /// Directory of the output file name. wasmer init will error if the target dir /// already contains a wasmer.toml. Also sets the package name. #[clap(name = "PACKAGE_PATH")] pub out: Option, From 9bb8eef18992157a96c3147fee47dcf82d8e879e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 14 Dec 2022 13:18:49 +0100 Subject: [PATCH 53/88] Use crate-type group --- lib/cli/src/commands/init.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 59c870512b3..d2a39acb6dd 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -8,16 +8,16 @@ use std::path::PathBuf; #[derive(Debug, Parser)] pub struct Init { /// Initialize wapm.toml for a library package - #[clap(long)] + #[clap(long, group = "crate-type")] pub lib: bool, /// Initialize wapm.toml for a binary package - #[clap(long)] + #[clap(long, group = "crate-type")] pub bin: bool, /// Initialize an empty wapm.toml - #[clap(long)] + #[clap(long, group = "crate-type")] pub empty: bool, /// Force overwriting the wapm.toml, even if it already exists - #[clap(long")] + #[clap(long)] pub overwrite: bool, /// Don't display debug output #[clap(long)] From 912b14026cd3329a948bc4497022095e719259be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <12084016+fschutt@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:20:04 +0100 Subject: [PATCH 54/88] Use anyhow::bail Co-authored-by: Michael Bryan --- lib/cli/src/commands/init.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index d2a39acb6dd..1e91ffb80be 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -102,10 +102,10 @@ impl Init { }; if target_file.exists() && !self.overwrite { - return Err(anyhow::anyhow!( + anyhow::bail!( "wapm project already initialized in {}", - target_file.display() - )); + target_file.display(), + ); } // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. From 3c099b7df0386af88dd0f6ccb67d272a37bff61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <12084016+fschutt@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:21:56 +0100 Subject: [PATCH 55/88] Show file name in error message Co-authored-by: Michael Bryan --- lib/cli/src/commands/init.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 1e91ffb80be..ef47d605244 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -305,7 +305,8 @@ impl Init { .collect::>() .join("\r\n"); - std::fs::write(target_file, &toml_string)?; + std::fs::write(&target_file, &toml_string) + .with_context(|| format!("Unable to write to \"{}\"", target_file.display()))?; return Ok(()); } From 590afd132745e36f79f87ce503847a264bd2b937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <12084016+fschutt@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:29:58 +0100 Subject: [PATCH 56/88] Use wai 0.2.0 Co-authored-by: Michael Bryan --- lib/cli/src/commands/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index ef47d605244..3c7c2094194 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -458,7 +458,7 @@ impl Init { Some(wapm_toml::Bindings::Wai(wapm_toml::WaiBindings { exports: None, imports: vec![e.path().to_path_buf()], - wai_version: semver::Version::parse("0.1.0").unwrap(), + wai_version: semver::Version::parse("0.2.0").unwrap(), })) } else { None From 359f91d7a3f51db4f8b8df48639e634b9b1ade92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 14 Dec 2022 13:34:17 +0100 Subject: [PATCH 57/88] Refactor parsing Cargo.toml in wasmer init --- lib/cli/src/commands/init.rs | 117 ++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 3c7c2094194..81d76683a43 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use cargo_metadata::{CargoOpt, MetadataCommand}; use clap::Parser; use std::collections::HashMap; @@ -77,6 +78,27 @@ impl Init { pub fn execute(&self) -> Result<(), anyhow::Error> { let bin_or_lib = self.get_bin_or_lib()?; + // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. + let manifest_path = match self.manifest_path.as_ref() { + Some(s) => s.clone(), + None => { + let cargo_toml_path = self + .out + .clone() + .unwrap_or_else(|| std::env::current_dir().unwrap()) + .join("Cargo.toml"); + cargo_toml_path + .canonicalize() + .unwrap_or_else(|_| cargo_toml_path.clone()) + } + }; + + let cargo_toml = if manifest_path.exists() { + Some(parse_cargo_toml(&manifest_path)?) + } else { + None + }; + let package_name; let target_file = match self.out.as_ref() { None => { @@ -108,63 +130,6 @@ impl Init { ); } - // See if the directory has a Cargo.toml file, if yes, copy the license / readme, etc. - let manifest_path = match self.manifest_path.as_ref() { - Some(s) => s.clone(), - None => { - let cargo_toml_path = self - .out - .clone() - .unwrap_or_else(|| std::env::current_dir().unwrap()) - .join("Cargo.toml"); - cargo_toml_path - .canonicalize() - .unwrap_or_else(|_| cargo_toml_path.clone()) - } - }; - - let cargo_toml = if manifest_path.exists() { - use anyhow::Context; - - let mut metadata = MetadataCommand::new(); - metadata.manifest_path(&manifest_path); - metadata.no_deps(); - metadata.features(CargoOpt::AllFeatures); - - let metadata = metadata.exec(); - - let metadata = match metadata { - Ok(o) => o, - Err(e) => { - return Err(anyhow::anyhow!("failed to load metadata: {e}") - .context(anyhow::anyhow!("{}", manifest_path.display()))); - } - }; - - let package = metadata - .root_package() - .ok_or_else(|| anyhow::anyhow!("no root package found in cargo metadata")) - .context(anyhow::anyhow!("{}", manifest_path.display()))?; - - Some(MiniCargoTomlPackage { - name: package.name.clone(), - version: package.version.clone(), - description: package.description.clone(), - homepage: package.homepage.clone(), - repository: package.repository.clone(), - license: package.license.clone(), - readme: package.readme.clone().map(|s| s.into_std_path_buf()), - license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), - workspace_root: metadata.workspace_root.into_std_path_buf(), - build_dir: metadata - .target_directory - .into_std_path_buf() - .join("wasm32-wasi"), - }) - } else { - None - }; - let package_name = cargo_toml .as_ref() .map(|p| &p.name) @@ -469,3 +434,41 @@ impl Init { } } } + +fn parse_cargo_toml(manifest_path: &PathBuf) -> Result { + let mut metadata = MetadataCommand::new(); + metadata.manifest_path(&manifest_path); + metadata.no_deps(); + metadata.features(CargoOpt::AllFeatures); + + let metadata = metadata.exec(); + + let metadata = match metadata { + Ok(o) => o, + Err(e) => { + return Err(anyhow::anyhow!("failed to load metadata: {e}") + .context(anyhow::anyhow!("{}", manifest_path.display()))); + } + }; + + let package = metadata + .root_package() + .ok_or_else(|| anyhow::anyhow!("no root package found in cargo metadata")) + .context(anyhow::anyhow!("{}", manifest_path.display()))?; + + Ok(MiniCargoTomlPackage { + name: package.name.clone(), + version: package.version.clone(), + description: package.description.clone(), + homepage: package.homepage.clone(), + repository: package.repository.clone(), + license: package.license.clone(), + readme: package.readme.clone().map(|s| s.into_std_path_buf()), + license_file: package.license_file.clone().map(|f| f.into_std_path_buf()), + workspace_root: metadata.workspace_root.into_std_path_buf(), + build_dir: metadata + .target_directory + .into_std_path_buf() + .join("wasm32-wasi"), + }) +} From 443ed12200fc29d5818f509584a5333e098344ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <12084016+fschutt@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:36:05 +0100 Subject: [PATCH 58/88] Simplify get_dependencies Co-authored-by: Michael Bryan --- lib/cli/src/commands/init.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index 81d76683a43..a7f6ebf8cbf 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -368,21 +368,19 @@ impl Init { /// Returns the dependencies based on the `--template` flag fn get_dependencies(&self) -> Option> { - Some({ - match self.template.as_deref() { - Some("js") => { - let mut map = HashMap::default(); - map.insert("python".to_string(), "quickjs/quickjs@latest".to_string()); - map - } - Some("python") => { - let mut map = HashMap::default(); - map.insert("python".to_string(), "python/python@latest".to_string()); - map - } - _ => HashMap::default(), + let mut map = HashMap::new(); + + match self.template.as_deref() { + Some("js") => { + map.insert("python".to_string(), "quickjs/quickjs@latest".to_string()); + } + Some("python") => { + map.insert("python".to_string(), "python/python@latest".to_string()); + } + _ => {}, } - }) + + Some(map) } // Returns whether the template for the wapm.toml should be a binary, a library or an empty file From a91a9ab3d47d505e9316df53263c27193a16d472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 14 Dec 2022 13:44:03 +0100 Subject: [PATCH 59/88] Use Template with clap(ValueEnum) --- lib/cli/src/commands/init.rs | 46 +++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index a7f6ebf8cbf..3ec007b6450 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -4,6 +4,7 @@ use clap::Parser; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; +use std::str::FromStr; /// CLI args for the `wasmer init` command #[derive(Debug, Parser)] @@ -37,7 +38,7 @@ pub struct Init { pub manifest_path: Option, /// Add default dependencies for common packages (currently supported: `python`, `js`) #[clap(long)] - pub template: Option, + pub template: Option