From 1dd9eb7d04a257855e60e27c04d26085f0502f75 Mon Sep 17 00:00:00 2001 From: Yuval Kogman Date: Sun, 21 May 2023 00:07:24 +0300 Subject: [PATCH] carlo: implied command dependencies Given build options (cargo workspace options, e.g. `-p my_crate`), the `upload` command will build before uploading, and given build or upload options `create` will build or upload before creating a machine. These primary options are mutually exclusive, but may require nested ArgGroups in the future which are supposed to be supported, perhaps aided by clap-rs/clap#2621 if resolved. The current constraints and reuse of shared options is rather crude (see in particular UploadOptsWrapper). --- crates/carlo/src/main.rs | 382 +++++++++++++++++++++++++-------------- 1 file changed, 244 insertions(+), 138 deletions(-) diff --git a/crates/carlo/src/main.rs b/crates/carlo/src/main.rs index c5e5292..7bda104 100644 --- a/crates/carlo/src/main.rs +++ b/crates/carlo/src/main.rs @@ -1,15 +1,31 @@ use anyhow::{anyhow, Context}; use cargo_metadata::camino::Utf8PathBuf; use cargo_metadata::Message; -use carol_core::BinaryId; +use carol_core::{BinaryId, MachineId}; use carol_host::Executor; -use clap::{Parser, Subcommand}; +use clap::{Args, Parser, Subcommand}; use clap_cargo::Workspace; use std::process::{Command, Stdio}; use wit_component::ComponentEncoder; mod client; +use client::Client; +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + + // TODO return a proper enum, of output types, and serialize them in a + // consistent manner. for only paths and SHA256 hashes are output. + match &cli.command { + Commands::Build(opts) => println!("{}", opts.run()?), + Commands::Upload(opts) => println!("{}", opts.run()?), + Commands::Create(opts) => println!("{}", opts.run()?), + }; + + Ok(()) +} + +/// carlo: command line interface for Carol #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Cli { @@ -19,150 +35,240 @@ struct Cli { #[derive(Subcommand)] enum Commands { - Build { - #[arg(short, long, value_name = "SPEC", group = "build")] - /// Package to compile to a Carol WASM component (see `cargo help pkgid`) - package: Option, // real one has Vec - }, - Upload { - #[arg(long)] - carol_url: String, - #[arg(long)] - binary: Utf8PathBuf, - }, - Create { - #[arg(long)] - carol_url: String, - #[arg(long)] - binary_id: BinaryId, - }, + Build(BuildOpts), + Upload(UploadOptsWrapper), + Create(CreateOpts), } -fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); +#[derive(Args, Debug)] +/// Compile a Carol WASM component binary from a Rust crate +struct BuildOpts { + #[arg(short, long, value_name = "SPEC", group = "build")] + /// Package to compile to a Carol WASM component (see `cargo help pkgid`) + pub package: Option, // real one has Vec +} - match &cli.command { - Commands::Build { package } => { - let mut cmd = Command::new("cargo"); - - // Find the crate package to compile - let metadata = cargo_metadata::MetadataCommand::new() - .exec() - .context("Couldn't build Carol WASM component")?; - - let mut ws = Workspace::default(); - ws.package = package.iter().cloned().collect(); - let (included, _) = ws.partition_packages(&metadata); - if included.is_empty() { - return Err(anyhow!( - "package ID specification {:?} did not match any packages", - ws.package - )); - } - if included.len() != 1 { - return Err(anyhow!("Carol WASM components must be built from a single crate, but package ID specification {:?} resulted in {} packages (did you forget to specify -p in a workspace?)", ws.package, included.len())); - } - let package = &included[0].name; - - cmd.env("RUSTFLAGS", "-C opt-level=z") - .args([ - "rustc", - "--package", - package, - "--message-format=json-render-diagnostics", - "--target", - "wasm32-unknown-unknown", - "--release", - "--crate-type=cdylib", - ]) - .stdout(Stdio::piped()); - - let mut proc = cmd.spawn().expect("things to compile"); - - let reader = std::io::BufReader::new(proc.stdout.take().unwrap()); - - let messages = cargo_metadata::Message::parse_stream(reader) - .collect::, _>>() - .context("Reading cargo output")?; - - let final_artifact_message = messages - .into_iter() - .rev() - .find_map(|message| match message { - Message::CompilerArtifact(artifact) => Some(artifact), - _ => None, - }) - .ok_or(anyhow!( - "No compiler artifact messages in output, could not find wasm output file" - ))?; - - if final_artifact_message.filenames.len() != 1 { - return Err(anyhow!( - "Expected a single wasm artifact in files, but got {}", - final_artifact_message.filenames.len() - )); - } - - let final_wasm_artifact = final_artifact_message.filenames[0].clone(); - - proc.wait().expect("Couldn't get cargo's exit status"); - - let mut component_target = final_wasm_artifact.clone(); - // FIXME horrible jankyness, but it works - component_target.set_extension(""); - component_target - .set_file_name(component_target.file_name().unwrap().to_owned() + "-component"); - component_target.set_extension("wasm"); - - let wasm = std::fs::read(&final_wasm_artifact) - .context(format!("Reading compiled WASM file {final_wasm_artifact}"))?; - - let encoder = ComponentEncoder::default().validate(true).module(&wasm)?; - - let bytes = encoder - .encode() - .context("failed to encode a component from module")?; - - std::fs::write(&component_target, bytes) - .context(format!("Writing WASM component {component_target}"))?; - - // TODO remove or (after careful consideration) convert to a - // warning before release, as this strongly assumes the client side - // carlo binary and server side carol host exactly agree on the - // definition of Executor::load_binary_from_wasm_file. - _ = Executor::new() - .load_binary_from_wasm_file(&component_target) - .context("Ensuring WASM component {component_target} is loadable")?; - - println!("{component_target}"); - } - Commands::Upload { binary, carol_url } => { - let client = client::Client::new(carol_url.clone()); +#[derive(Args, Debug)] +/// Upload a component binary to a Carol server +struct UploadOpts { + /// The binary (WASM component) to upload (implied by --package) + #[arg( + long, + value_name = "WASM_FILE", + group = "upload", + conflicts_with = "build" + )] + binary: Option, + + #[clap(flatten)] + implied_build: BuildOpts, +} + +#[derive(Args, Debug)] +/// Create a machine from a component binary on a Carol server +struct CreateOpts { + #[clap(flatten)] + server: ServerOpts, + + /// The ID of the compiled binary from which to create a machine (implied by --binary) + #[arg( + long, + value_name = "SHA256", + group = "create", + conflicts_with = "upload", + conflicts_with = "build" + )] + binary_id: Option, + + #[clap(flatten)] + implied_upload: UploadOpts, +} + +#[derive(Args, Debug)] +struct ServerOpts { + /// The Carol server's URL (e.g. http://localhost:8000, see README.md) + #[arg(long)] // , default_value = "http://localhost:8000")] ? + carol_url: String, +} - // Validate and derive BinaryId - let binary_id = Executor::new() - .load_binary_from_wasm_file(binary) - .context("Loading compiled binary")? - .binary_id(); +// This wrapper is here because otherwise UploadOpts and CreateOpts specify +// duplicate ServerOpts/carol URL, which shouldn't be global because it doesn't +// actually apply to all commands +#[derive(Args, Debug)] +struct UploadOptsWrapper { + #[clap(flatten)] + server: ServerOpts, - let file = std::fs::File::open(binary) - .context(format!("Reading compiled WASM file {}", binary))?; + #[clap(flatten)] + internal: UploadOpts, +} - let response = client.upload_binary(&binary_id, file)?; - let binary_id = response.id; - println!("{binary_id}"); +impl BuildOpts { + fn run(&self) -> anyhow::Result { + // Find the crate package to compile + let metadata = cargo_metadata::MetadataCommand::new() + .exec() + .context("Couldn't build Carol WASM component")?; + let mut ws = Workspace::default(); + ws.package = self.package.iter().cloned().collect(); + let (included, _) = ws.partition_packages(&metadata); + if included.is_empty() { + return Err(anyhow!( + "package ID specification {:?} did not match any packages", + ws.package + )); } - Commands::Create { - binary_id, - carol_url, - } => { - let client = client::Client::new(carol_url.clone()); - - let response = client.create_machine(binary_id)?; - let machine_id = response.id; - print!("{machine_id}"); + if included.len() != 1 { + return Err(anyhow!("Carol WASM components must be built from a single crate, but package ID specification {:?} resulted in {} packages (did you forget to specify -p in a workspace?)", ws.package, included.len())); } + let package = &included[0].name; + + // Compile to WASM target + // TODO use cargo::ops::compile instead of invoking cargo CLI? + let mut cmd = Command::new("cargo"); + cmd.env("RUSTFLAGS", "-C opt-level=z") + .args([ + "rustc", + "--package", + package, + "--message-format=json-render-diagnostics", + "--target", + "wasm32-unknown-unknown", + "--release", + "--crate-type=cdylib", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut proc = cmd.spawn().context("Couldn't spawn cargo rustc")?; + + let reader = std::io::BufReader::new(proc.stdout.take().unwrap()); + let messages = cargo_metadata::Message::parse_stream(reader) + .collect::, _>>() + .context("Couldn't read cargo output")?; + + let output = proc + .wait_with_output() + .context("Couldn't read cargo rustc exit status or output")?; + + // Find the last compiler artifact message + let final_artifact_message = messages + .into_iter() + .rev() + .find_map(|message| match message { + Message::CompilerArtifact(artifact) => Some(artifact), + // Message::CompilerMessage(message) => Some(Err(anyhow!("{message}"))), // FIXME is this appropriate, with ?? at the end? + _ => None, + }) + .ok_or_else(|| { + let err = anyhow!("{}", String::from_utf8_lossy(&output.stderr)); + err.context( + "No compiler artifact messages in output, could not find wasm output file.", + ) + })?; + + if final_artifact_message.filenames.len() != 1 { + return Err(anyhow!( + "Expected a single wasm artifact in files, but got {}", + final_artifact_message.filenames.len() + )); + } + + let final_wasm_artifact = final_artifact_message.filenames[0].clone(); + + let component_target = append_to_basename(&final_wasm_artifact, "-component")?; + + // Encode the component and write artifcat + let wasm = std::fs::read(&final_wasm_artifact).context(format!( + "Couldn't read compiled WASM file {final_wasm_artifact}" + ))?; + + let encoder = ComponentEncoder::default().validate(true).module(&wasm)?; + + let bytes = encoder + .encode() + .context("Failed to encode a component from module")?; + + std::fs::write(&component_target, bytes) + .context(format!("Couldn't write WASM component {component_target}"))?; + + // TODO remove or (after careful consideration) convert to a + // warning before release, as this strongly assumes the client side + // carlo binary and server side carol host exactly agree on the + // definition of Executor::load_binary_from_wasm_file. + _ = Executor::new() + .load_binary_from_wasm_file(&component_target) + .context(format!( + "Compiled WASM component {component_target} was invalid" + ))?; + + Ok(component_target) } +} - Ok(()) +impl UploadOptsWrapper { + fn run(&self) -> anyhow::Result { + let client = Client::new(self.server.carol_url.clone()); + self.internal.run(&client) + } +} + +impl UploadOpts { + fn run(&self, client: &Client) -> anyhow::Result { + let binary = match &self.binary { + Some(binary) => binary.clone(), + None => self + .implied_build + .run() + .context("Failed to build crate for upload")?, + }; + + // Validate and derive BinaryId + let binary_id = Executor::new() + .load_binary_from_wasm_file(&binary) + .context("Couldn't load compiled binary")? + .binary_id(); + + let file = + std::fs::File::open(&binary).context(format!("Couldn't read file {}", binary))?; + + let response = client.upload_binary(&binary_id, file)?; + let binary_id = response.id; + Ok(binary_id) + } +} + +impl CreateOpts { + fn run(&self) -> anyhow::Result { + let client = Client::new(self.server.carol_url.clone()); + + let binary_id = match self.binary_id { + Some(binary_id) => binary_id, + None => self + .implied_upload + .run(&client) + .context("Failed to upload binary for machine creation")?, + }; + + let response = client.create_machine(&binary_id)?; + let machine_id = response.id; + Ok(machine_id) + } +} + +/// Helper function for rewriting filenames while retaining extension +fn append_to_basename(path: &Utf8PathBuf, suffix: &str) -> anyhow::Result { + let ext = path + .extension() + .context("Expected path to contain an extension")? + .to_string(); + + let basename = path + .file_stem() + .context("Expected path to contain a file basename component")?; + + let mut path = path.clone(); + path.set_file_name(format!("{basename}{suffix}")); + path.set_extension(ext); + Ok(path) }