Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

port(turborepo): Run stub #4752

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ once_cell = "1.17.1"
owo-colors = "3.5.0"
parking_lot = "0.12.1"
pathdiff = "0.2.1"
petgraph = "0.6.3"
pin-project-lite = "0.2.9"
port_scanner = "0.1.5"
postcard = "1.0.4"
Expand Down
3 changes: 3 additions & 0 deletions crates/turborepo-lib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ license = "MPL-2.0"
default = ["rustls-tls"]
native-tls = ["turborepo-api-client/native-tls", "turbo-updater/native-tls"]
rustls-tls = ["turborepo-api-client/rustls-tls", "turbo-updater/rustls-tls"]
run-stub = []

# serve the daemon over a port (useful for testing)
http = ["tonic-reflection"]
Expand Down Expand Up @@ -56,6 +57,7 @@ itertools = { workspace = true }
lazy_static = { workspace = true }
libc = "0.2.140"
notify = "5.1"
petgraph = { workspace = true }
pidlock = { path = "../turborepo-pidlock" }
prost = "0.11.6"
reqwest = { workspace = true, default_features = false, features = ["json"] }
Expand Down Expand Up @@ -83,6 +85,7 @@ const_format = "0.2.30"
go-parse-duration = "0.1.1"
is-terminal = "0.4.7"
node-semver = "2.1.0"
num_cpus = "1.15.0"
owo-colors.workspace = true
regex.workspace = true
tracing-appender = "0.2.2"
Expand Down
21 changes: 21 additions & 0 deletions crates/turborepo-lib/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ use serde::Serialize;
use tracing::{debug, error};
use turbopath::AbsoluteSystemPathBuf;

#[cfg(feature = "run-stub")]
use crate::commands::run;
use crate::{
commands::{bin, daemon, generate, link, login, logout, unlink, CommandBase},
get_version,
Expand Down Expand Up @@ -230,6 +232,17 @@ impl Args {

Ok(clap_args)
}

pub fn get_tasks(&self) -> &[String] {
match &self.command {
Some(Command::Run(box RunArgs { tasks, .. })) => tasks,
_ => self
.run_args
.as_ref()
.map(|run_args| run_args.tasks.as_slice())
.unwrap_or(&[]),
}
}
}

/// Defines the subcommands for CLI. NOTE: If we change the commands in Go,
Expand Down Expand Up @@ -679,6 +692,14 @@ pub async fn run(

Ok(Payload::Rust(Ok(0)))
}
#[cfg(feature = "run-stub")]
Command::Run(args) => {
let base = CommandBase::new(cli_args, repo_root, version, ui)?;
run::run(base).await?;

Ok(Payload::Rust(Ok(0)))
}
#[cfg(not(feature = "run-stub"))]
Command::Run(args) => {
if args.tasks.is_empty() {
return Err(anyhow!("at least one task must be specified"));
Expand Down
2 changes: 2 additions & 0 deletions crates/turborepo-lib/src/commands/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@ pub(crate) mod generate;
pub(crate) mod link;
pub(crate) mod login;
pub(crate) mod logout;
pub(crate) mod run;
pub(crate) mod unlink;

#[derive(Debug)]
pub struct CommandBase {
pub repo_root: AbsoluteSystemPathBuf,
pub ui: UI,
Expand Down
19 changes: 19 additions & 0 deletions crates/turborepo-lib/src/commands/run.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
use anyhow::Result;
use tracing::{error, info};

use crate::{commands::CommandBase, run::Run};

#[allow(dead_code)]
pub async fn run(base: CommandBase) -> Result<()> {
info!("Executing run stub");
let mut run = Run::new(base);
info!("configured run struct: {:?}", run);

match run.run().await {
Ok(_) => Ok(()),
Err(err) => {
error!("run failed: {}", err);
Err(err)
}
}
}
7 changes: 6 additions & 1 deletion crates/turborepo-lib/src/config/turbo.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use serde::{Deserialize, Serialize};

use crate::{opts::RemoteCacheOpts, run::pipeline::Pipeline};

#[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
pub struct SpacesJson {
Expand All @@ -8,11 +10,14 @@ pub struct SpacesJson {
pub other: Option<serde_json::Value>,
}

#[derive(Serialize, Deserialize, Debug)]
#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct TurboJson {
#[serde(flatten)]
other: serde_json::Value,
pub(crate) remote_cache_opts: Option<RemoteCacheOpts>,
pub space_id: Option<String>,
pub pipeline: Pipeline,
#[serde(skip_serializing_if = "Option::is_none")]
pub experimental_spaces: Option<SpacesJson>,
}
8 changes: 6 additions & 2 deletions crates/turborepo-lib/src/execution_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,12 @@ impl<'a> TryFrom<&'a CommandBase> for ExecutionState<'a> {
type Error = anyhow::Error;

fn try_from(base: &'a CommandBase) -> Result<Self, Self::Error> {
let root_package_json =
PackageJson::load(&base.repo_root.join_component("package.json")).ok();
let root_package_json = PackageJson::load(
base.repo_root
.join_component("package.json")
.as_absolute_path(),
)
.ok();

let package_manager =
PackageManager::get_package_manager(base, root_package_json.as_ref())?;
Expand Down
4 changes: 4 additions & 0 deletions crates/turborepo-lib/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#![feature(assert_matches)]
#![feature(box_patterns)]

mod child;
mod cli;
Expand All @@ -7,8 +8,11 @@ mod config;
mod daemon;
mod execution_state;
pub(crate) mod globwatcher;
mod manager;
mod opts;
mod package_json;
mod package_manager;
mod run;
mod shim;
mod tracing;
mod ui;
Expand Down
10 changes: 10 additions & 0 deletions crates/turborepo-lib/src/manager.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#[derive(Debug)]
pub struct Manager {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this the child process manager? Can you add a comment?

// TODO
}

impl Manager {
pub fn new() -> Self {
Self {}
}
}
155 changes: 155 additions & 0 deletions crates/turborepo-lib/src/opts.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
#![allow(dead_code)]
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};

use crate::{
cli::{Command, DryRunMode, EnvMode, LogPrefix, RunArgs},
daemon::{DaemonClient, DaemonConnector},
Args,
};

#[derive(Debug)]
pub struct Opts<'a> {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we care about the opts having a lifetime? Would it make it easier if they owned their data? It's not a ton of data...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now it can go either way. I think in the future I'd like a single 'run lifetime that's tied to either CommandBase or a Run struct. And that can own stuff like Args that then in turn get transformed to Opts. For now I don't think we have to make that call until this is used.

pub cache_opts: CacheOpts<'a>,
pub run_opts: RunOpts<'a>,
pub runcache_opts: RunCacheOpts,
pub scope_opts: ScopeOpts,
}

#[derive(Debug, Default)]
pub struct CacheOpts<'a> {
override_dir: Option<&'a str>,
skip_remote: bool,
skip_filesystem: bool,
workers: u32,
pub(crate) remote_cache_opts: Option<RemoteCacheOpts>,
}

impl<'a> From<&'a RunArgs> for CacheOpts<'a> {
fn from(run_args: &'a RunArgs) -> Self {
CacheOpts {
override_dir: run_args.cache_dir.as_deref(),
skip_filesystem: run_args.remote_only,
workers: run_args.cache_workers,
..CacheOpts::default()
}
}
}

#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct RemoteCacheOpts {
team_id: String,
signature: bool,
}

impl<'a> TryFrom<&'a Args> for Opts<'a> {
type Error = anyhow::Error;

fn try_from(args: &'a Args) -> std::result::Result<Self, Self::Error> {
let Some(Command::Run(run_args)) = &args.command else {
return Err(anyhow!("Expected run command"))
};
let run_opts = RunOpts::try_from(run_args.as_ref())?;
let cache_opts = CacheOpts::from(run_args.as_ref());

Ok(Self {
run_opts,
cache_opts,
scope_opts: ScopeOpts::default(),
runcache_opts: RunCacheOpts::default(),
})
}
}

#[derive(Debug, Default)]
pub struct RunCacheOpts {
pub(crate) output_watcher: Option<DaemonClient<DaemonConnector>>,
}

#[derive(Debug)]
pub struct RunOpts<'a> {
tasks: &'a [String],
concurrency: u32,
parallel: bool,
env_mode: EnvMode,
profile: Option<&'a str>,
continue_on_error: bool,
passthrough_args: &'a [String],
only: bool,
dry_run: bool,
pub(crate) dry_run_json: bool,
pub graph_dot: bool,
graph_file: Option<&'a str>,
pub(crate) no_daemon: bool,
pub(crate) single_package: bool,
log_prefix: Option<LogPrefix>,
summarize: Option<Option<bool>>,
pub(crate) experimental_space_id: Option<String>,
}

const DEFAULT_CONCURRENCY: u32 = 10;

impl<'a> TryFrom<&'a RunArgs> for RunOpts<'a> {
type Error = anyhow::Error;

fn try_from(args: &'a RunArgs) -> Result<Self> {
let concurrency = args
.concurrency
.as_deref()
.map(parse_concurrency)
.transpose()?
.unwrap_or(DEFAULT_CONCURRENCY);

let (graph_dot, graph_file) = match &args.graph {
Some(file) if file.is_empty() => (true, None),
Some(file) => (false, Some(file.as_str())),
None => (false, None),
};

Ok(Self {
tasks: args.tasks.as_slice(),
log_prefix: args.log_prefix,
summarize: args.summarize,
experimental_space_id: args.experimental_space_id.clone(),
env_mode: args.env_mode,
concurrency,
parallel: args.parallel,
profile: args.profile.as_deref(),
continue_on_error: args.continue_execution,
passthrough_args: args.pass_through_args.as_ref(),
only: args.only,
no_daemon: args.no_daemon,
single_package: args.single_package,
graph_dot,
graph_file,
dry_run_json: matches!(args.dry_run, Some(DryRunMode::Json)),
dry_run: args.dry_run.is_some(),
})
}
}

fn parse_concurrency(concurrency_raw: &str) -> Result<u32> {
if let Some(percent) = concurrency_raw.strip_suffix('%') {
let percent = percent.parse::<f64>()?;
return if percent > 0.0 && percent.is_finite() {
Ok((num_cpus::get() as f64 * percent / 100.0).max(1.0) as u32)
} else {
Err(anyhow!(
"invalid percentage value for --concurrency CLI flag. This should be a percentage \
of CPU cores, between 1% and 100% : {}",
percent
))
};
}
match concurrency_raw.parse::<u32>() {
Ok(concurrency) if concurrency > 1 => Ok(concurrency),
Ok(_) | Err(_) => Err(anyhow!(
"invalid value for --concurrency CLI flag. This should be a positive integer greater \
than or equal to 1: {}",
concurrency_raw
)),
}
}

#[derive(Debug, Default)]
pub struct ScopeOpts {}
4 changes: 2 additions & 2 deletions crates/turborepo-lib/src/package_json.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use turbopath::AbsoluteSystemPathBuf;
use turbopath::AbsoluteSystemPath;

#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
Expand All @@ -9,7 +9,7 @@ pub struct PackageJson {
}

impl PackageJson {
pub fn load(path: &AbsoluteSystemPathBuf) -> Result<PackageJson> {
pub fn load(path: &AbsoluteSystemPath) -> Result<PackageJson> {
let contents = std::fs::read_to_string(path)?;
let package_json: PackageJson = serde_json::from_str(&contents)?;
Ok(package_json)
Expand Down