Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf(turbopack): Use Arc<String> instead of String #7772

Open
wants to merge 78 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
78 commits
Select commit Hold shift + click to select a range
37c4173
print clone
kdy1 Mar 20, 2024
e120c39
Arc
kdy1 Mar 20, 2024
526162a
Arc: turbo-tasks-memory
kdy1 Mar 20, 2024
6baa89c
Fix turbo-tasks
kdy1 Mar 20, 2024
dae98a6
Fix turbo-tasks-fs
kdy1 Mar 20, 2024
0be3a1f
Fix turbo-tasks-memory
kdy1 Mar 20, 2024
c0f784f
Fix turbo-tasks-fs
kdy1 Mar 20, 2024
ba22df6
Fix turbo-tasks-env
kdy1 Mar 20, 2024
978a3bb
Fix turbopack-core
kdy1 Mar 20, 2024
64a7934
Fix turbopack-core
kdy1 Mar 20, 2024
1e86606
Fix turbopack-core
kdy1 Mar 20, 2024
83fff2a
Fix turbopack-core
kdy1 Mar 20, 2024
8336f71
Fix turbopack-core
kdy1 Mar 20, 2024
288d125
Fix turbopack-core
kdy1 Mar 20, 2024
5a47811
Fix turbopack-core
kdy1 Mar 21, 2024
6e5b302
Fix turbopack-core
kdy1 Mar 21, 2024
82943b5
Fix turbopack-core
kdy1 Mar 21, 2024
8e86d0b
Fix turbopack-core
kdy1 Mar 21, 2024
4f9f27e
Fix turbo-tasks-fetch
kdy1 Mar 21, 2024
4bb748e
Fix turbopack-resolve
kdy1 Mar 21, 2024
53654b1
Fix turbopack-resolve
kdy1 Mar 21, 2024
f9bed38
Fix turbopack-ecmascript
kdy1 Mar 21, 2024
434c8ab
Remove println
kdy1 Mar 21, 2024
b2f5565
Fix turbopack-css
kdy1 Mar 21, 2024
e19bc94
Fix turbopack-css
kdy1 Mar 21, 2024
7ad012f
Fix turbopack-env
kdy1 Mar 21, 2024
787fec3
Fix turbopack-css
kdy1 Mar 21, 2024
02b2df6
More wokr
kdy1 Mar 21, 2024
a4187af
More work
kdy1 Mar 21, 2024
02b1fb2
Fix turbopack-dev-server
kdy1 Mar 21, 2024
ad0ea3f
Fix turbopack-dev-server
kdy1 Mar 21, 2024
b73802f
Fix turbopack-dev-server
kdy1 Mar 21, 2024
979bb08
Fix turbopack-dev-server
kdy1 Mar 21, 2024
eb44251
Fix turbopack-dev-server
kdy1 Mar 21, 2024
e909d55
Fix turbopack-dev-server
kdy1 Mar 21, 2024
8400a10
Fix turbo-tasks-fs
kdy1 Mar 21, 2024
fc61018
Fix turbopack-node
kdy1 Mar 21, 2024
1f2dd7b
Fix turbopack-node
kdy1 Mar 21, 2024
244323b
Fix turbopack-node
kdy1 Mar 21, 2024
221a6c5
Fix turbopack-node
kdy1 Mar 21, 2024
4231d82
Fix turbopack-node
kdy1 Mar 21, 2024
13116cf
Fix turbopack-node
kdy1 Mar 21, 2024
33adb0a
Fix turbopack
kdy1 Mar 21, 2024
eb87c72
Fix turbopack
kdy1 Mar 21, 2024
2309dd7
Fix node-file-trace
kdy1 Mar 21, 2024
bbd9591
Fix node-file-trace
kdy1 Mar 21, 2024
83b20da
fix more
kdy1 Mar 21, 2024
30b408e
fix more
kdy1 Mar 21, 2024
3d335b7
fix more
kdy1 Mar 21, 2024
c98db0a
fix more
kdy1 Mar 21, 2024
69630ac
more work
kdy1 Mar 21, 2024
05eaea2
fix more
kdy1 Mar 21, 2024
65e77b9
fix more
kdy1 Mar 22, 2024
1102e6a
fix more
kdy1 Mar 22, 2024
7728ab3
fix more
kdy1 Mar 22, 2024
dd06114
fix more
kdy1 Mar 22, 2024
3900b81
fix more
kdy1 Mar 22, 2024
ab86162
fix more
kdy1 Mar 22, 2024
47c0a9b
fix more
kdy1 Mar 25, 2024
c05ab77
fix more
kdy1 Mar 25, 2024
8519e73
fix more
kdy1 Mar 25, 2024
425f603
fix more
kdy1 Mar 25, 2024
da945bd
fix more
kdy1 Apr 2, 2024
b65400e
fix more
kdy1 Apr 2, 2024
88a306d
fix more
kdy1 Apr 2, 2024
2a25f97
fix more
kdy1 Apr 2, 2024
1bb4878
fix more
kdy1 Apr 2, 2024
5c63a9f
fix more
kdy1 Apr 2, 2024
23d86d4
`RcStr`
kdy1 May 28, 2024
30cd742
fixup: RcStr
kdy1 May 28, 2024
ccf5bb4
reexport
kdy1 May 28, 2024
7d21901
Implement more `From` to `RcStr`
kdy1 May 28, 2024
46f9efd
Fix turbo-tasks
kdy1 May 28, 2024
e552404
impl AsRef<Path> for RcStr
kdy1 May 28, 2024
3704937
Fix turbo-tasks-fs
kdy1 May 28, 2024
4442b04
impl TraceRawVcs for RcStr
kdy1 May 28, 2024
d1a6b45
Fix turbo-tasks-fs
kdy1 May 28, 2024
87f2c94
Fix turbo-tasks-memory
kdy1 May 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
44 changes: 25 additions & 19 deletions crates/node-file-trace/src/lib.rs
Expand Up @@ -188,7 +188,7 @@ impl Args {
}

async fn create_fs(name: &str, root: &str, watch: bool) -> Result<Vc<Box<dyn FileSystem>>> {
let fs = DiskFileSystem::new(name.to_string(), root.to_string(), vec![]);
let fs = DiskFileSystem::new(name.to_string().into(), root.to_string().into(), vec![]);
if watch {
fs.await?.start_watching()?;
} else {
Expand Down Expand Up @@ -232,17 +232,18 @@ async fn add_glob_results(
#[turbo_tasks::function]
async fn input_to_modules(
fs: Vc<Box<dyn FileSystem>>,
input: Vec<String>,
input: Vec<Arc<String>>,
exact: bool,
process_cwd: Option<String>,
context_directory: String,
process_cwd: Option<Arc<String>>,
context_directory: Arc<String>,
module_options: TransientInstance<ModuleOptionsContext>,
resolve_options: TransientInstance<ResolveOptionsContext>,
) -> Result<Vc<Modules>> {
let root = fs.root();
let process_cwd = process_cwd
.clone()
.map(|p| format!("/ROOT{}", p.trim_start_matches(&context_directory)));
.map(|p| format!("/ROOT{}", p.trim_start_matches(&*context_directory)))
.map(Arc::new);

let asset_context: Vc<Box<dyn AssetContext>> = Vc::upcast(create_module_asset(
root,
Expand Down Expand Up @@ -283,7 +284,7 @@ fn process_context(dir: &Path, context_directory: Option<&String>) -> Result<Str
.to_string())
}

fn make_relative_path(dir: &Path, context_directory: &str, input: &str) -> Result<String> {
fn make_relative_path(dir: &Path, context_directory: &str, input: &str) -> Result<Arc<String>> {
let mut input = PathBuf::from(input);
if !input.is_absolute() {
input = dir.join(input);
Expand All @@ -299,10 +300,15 @@ fn make_relative_path(dir: &Path, context_directory: &str, input: &str) -> Resul
Ok(input
.to_str()
.ok_or_else(|| anyhow!("input contains invalid characters"))?
.replace('\\', "/"))
.replace('\\', "/")
.into())
}

fn process_input(dir: &Path, context_directory: &str, input: &[String]) -> Result<Vec<String>> {
fn process_input(
dir: &Path,
context_directory: &str,
input: &[String],
) -> Result<Vec<Arc<String>>> {
input
.iter()
.map(|input| make_relative_path(dir, context_directory, input))
Expand Down Expand Up @@ -537,8 +543,8 @@ async fn main_operation(
fs,
input,
exact,
process_cwd.clone(),
context_directory,
process_cwd.clone().map(Arc::new),
context_directory.into(),
module_options,
resolve_options,
)
Expand All @@ -563,8 +569,8 @@ async fn main_operation(
fs,
input,
exact,
process_cwd.clone(),
context_directory,
process_cwd.clone().map(Arc::new),
context_directory.into(),
module_options,
resolve_options,
)
Expand All @@ -573,7 +579,7 @@ async fn main_operation(
{
let nft_asset = NftJsonAsset::new(*module);
let path = nft_asset.ident().path().await?.path.clone();
output_nft_assets.push(path);
output_nft_assets.push((*path).clone());
emits.push(emit_asset(Vc::upcast(nft_asset)));
}
// Wait for all files to be emitted
Expand All @@ -596,8 +602,8 @@ async fn main_operation(
fs,
input,
exact,
process_cwd.clone(),
context_directory,
process_cwd.clone().map(Arc::new),
context_directory.into(),
module_options,
resolve_options,
)
Expand All @@ -620,13 +626,13 @@ async fn main_operation(
#[turbo_tasks::function]
async fn create_module_asset(
root: Vc<FileSystemPath>,
process_cwd: Option<String>,
process_cwd: Option<Arc<String>>,
module_options: TransientInstance<ModuleOptionsContext>,
resolve_options: TransientInstance<ResolveOptionsContext>,
) -> Result<Vc<ModuleAssetContext>> {
let env = Environment::new(Value::new(ExecutionEnvironment::NodeJsLambda(
NodeJsEnvironment {
cwd: Vc::cell(process_cwd),
cwd: Vc::cell(process_cwd.as_deref().cloned()),
..Default::default()
}
.into(),
Expand All @@ -635,12 +641,12 @@ async fn create_module_asset(
let glob_mappings = vec![
(
root,
Glob::new("**/*/next/dist/server/next.js".to_string()),
Glob::new("**/*/next/dist/server/next.js".to_string().into()),
ImportMapping::Ignore.into(),
),
(
root,
Glob::new("**/*/next/dist/bin/next".to_string()),
Glob::new("**/*/next/dist/bin/next".to_string().into()),
ImportMapping::Ignore.into(),
),
];
Expand Down
4 changes: 3 additions & 1 deletion crates/node-file-trace/src/nft_json.rs
Expand Up @@ -29,7 +29,9 @@ impl OutputAsset for NftJsonAsset {
async fn ident(&self) -> Result<Vc<AssetIdent>> {
let path = self.entry.ident().path().await?;
Ok(AssetIdent::from_path(
path.fs.root().join(format!("{}.nft.json", path.path)),
path.fs
.root()
.join(format!("{}.nft.json", path.path).into()),
))
}
}
Expand Down
4 changes: 3 additions & 1 deletion crates/turbo-tasks-env/src/custom.rs
@@ -1,3 +1,5 @@
use std::sync::Arc;

use anyhow::Result;
use turbo_tasks::Vc;

Expand Down Expand Up @@ -32,7 +34,7 @@ impl ProcessEnv for CustomProcessEnv {
}

#[turbo_tasks::function]
async fn read(&self, name: String) -> Result<Vc<Option<String>>> {
async fn read(&self, name: Arc<String>) -> Result<Vc<Option<String>>> {
let custom = case_insensitive_read(self.custom, name.clone());
match &*custom.await? {
Some(_) => Ok(custom),
Expand Down
6 changes: 4 additions & 2 deletions crates/turbo-tasks-env/src/filter.rs
@@ -1,3 +1,5 @@
use std::sync::Arc;

use anyhow::Result;
use indexmap::IndexMap;
use turbo_tasks::Vc;
Expand All @@ -15,7 +17,7 @@ pub struct FilterProcessEnv {
#[turbo_tasks::value_impl]
impl FilterProcessEnv {
#[turbo_tasks::function]
pub fn new(prior: Vc<Box<dyn ProcessEnv>>, filters: Vec<String>) -> Vc<Self> {
pub fn new(prior: Vc<Box<dyn ProcessEnv>>, filters: Vec<Arc<String>>) -> Vc<Self> {
FilterProcessEnv {
prior,
filters: filters.into_iter().map(|f| f.to_uppercase()).collect(),
Expand Down Expand Up @@ -43,7 +45,7 @@ impl ProcessEnv for FilterProcessEnv {
}

#[turbo_tasks::function]
fn read(&self, name: String) -> Vc<Option<String>> {
fn read(&self, name: Arc<String>) -> Vc<Option<String>> {
for filter in &self.filters {
if name.to_uppercase().starts_with(filter) {
return self.prior.read(name);
Expand Down
14 changes: 10 additions & 4 deletions crates/turbo-tasks-env/src/lib.rs
Expand Up @@ -5,7 +5,10 @@ mod custom;
mod dotenv;
mod filter;

use std::{env, sync::Mutex};
use std::{
env,
sync::{Arc, Mutex},
};

use anyhow::Result;
use indexmap::IndexMap;
Expand Down Expand Up @@ -35,7 +38,7 @@ impl ProcessEnv for EnvMap {
}

#[turbo_tasks::function]
async fn read(self: Vc<Self>, name: String) -> Vc<Option<String>> {
async fn read(self: Vc<Self>, name: Arc<String>) -> Vc<Option<String>> {
case_insensitive_read(self, name)
}
}
Expand All @@ -51,7 +54,7 @@ pub trait ProcessEnv {
fn read_all(self: Vc<Self>) -> Vc<EnvMap>;

/// Reads a single env variable. Ignores casing.
fn read(self: Vc<Self>, name: String) -> Vc<Option<String>> {
fn read(self: Vc<Self>, name: Arc<String>) -> Vc<Option<String>> {
case_insensitive_read(self.read_all(), name)
}
}
Expand All @@ -63,7 +66,10 @@ pub fn sorted_env_vars() -> IndexMap<String, String> {
}

#[turbo_tasks::function]
pub async fn case_insensitive_read(map: Vc<EnvMap>, name: String) -> Result<Vc<Option<String>>> {
pub async fn case_insensitive_read(
map: Vc<EnvMap>,
name: Arc<String>,
) -> Result<Vc<Option<String>>> {
Ok(Vc::cell(
to_uppercase_map(map)
.await?
Expand Down
2 changes: 1 addition & 1 deletion crates/turbo-tasks-fetch/tests/fetch.rs
Expand Up @@ -141,5 +141,5 @@ async fn errors_on_404() {
}

fn get_issue_context() -> Vc<FileSystemPath> {
DiskFileSystem::new("root".to_owned(), "/".to_owned(), vec![]).root()
DiskFileSystem::new("root".to_owned().into(), "/".to_owned().into(), vec![]).root()
}
4 changes: 2 additions & 2 deletions crates/turbo-tasks-fs/examples/hash_directory.rs
Expand Up @@ -30,12 +30,12 @@ async fn main() -> Result<()> {
let task = tt.spawn_root_task(|| {
Box::pin(async {
let root = current_dir().unwrap().to_str().unwrap().to_string();
let disk_fs = DiskFileSystem::new("project".to_string(), root, vec![]);
let disk_fs = DiskFileSystem::new("project".to_string().into(), root.into(), vec![]);
disk_fs.await?.start_watching()?;

// Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
let input = fs.root().join("demo".to_string());
let input = fs.root().join("demo".to_string().into());
let dir_hash = hash_directory(input);
print_hash(dir_hash).await?;
Ok::<Vc<()>, _>(Default::default())
Expand Down
6 changes: 3 additions & 3 deletions crates/turbo-tasks-fs/examples/hash_glob.rs
Expand Up @@ -27,13 +27,13 @@ async fn main() -> Result<()> {
let task = tt.spawn_root_task(|| {
Box::pin(async {
let root = current_dir().unwrap().to_str().unwrap().to_string();
let disk_fs = DiskFileSystem::new("project".to_string(), root, vec![]);
let disk_fs = DiskFileSystem::new("project".to_string().into(), root.into(), vec![]);
disk_fs.await?.start_watching()?;

// Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
let input = fs.root().join("crates".to_string());
let glob = Glob::new("**/*.rs".to_string());
let input = fs.root().join("crates".to_string().into());
let glob = Glob::new("**/*.rs".to_string().into());
let glob_result = input.read_glob(glob, true);
let dir_hash = hash_glob_result(glob_result);
print_hash(dir_hash).await?;
Expand Down
6 changes: 4 additions & 2 deletions crates/turbo-tasks-fs/src/attach.rs
@@ -1,3 +1,5 @@
use std::sync::Arc;

use anyhow::{bail, Result};
use auto_hash_map::AutoMap;
use turbo_tasks::{Completion, ValueToString, Vc};
Expand All @@ -16,7 +18,7 @@ pub struct AttachedFileSystem {
root_fs: Vc<Box<dyn FileSystem>>,
// we turn this into a string because creating a FileSystemPath requires the filesystem which
// we are creating (circular reference)
child_path: String,
child_path: Arc<String>,
child_fs: Vc<Box<dyn FileSystem>>,
}

Expand Down Expand Up @@ -112,7 +114,7 @@ impl AttachedFileSystem {
.root()
.resolve()
.await?
.join(inner_path.to_string())
.join(inner_path.to_string().into())
} else {
this.root_fs.root().resolve().await?.join(path.path.clone())
})
Expand Down
10 changes: 5 additions & 5 deletions crates/turbo-tasks-fs/src/embed/dir.rs
Expand Up @@ -2,14 +2,14 @@ pub use ::include_dir::{
include_dir, {self},
};
use anyhow::Result;
use turbo_tasks::{TransientInstance, Vc};
use turbo_tasks::{RcStr, TransientInstance, Vc};

use crate::{embed::EmbeddedFileSystem, DiskFileSystem, FileSystem};

#[turbo_tasks::function]
pub async fn directory_from_relative_path(
name: String,
path: String,
name: RcStr,
path: RcStr,
) -> Result<Vc<Box<dyn FileSystem>>> {
let disk_fs = DiskFileSystem::new(name, path, vec![]);
disk_fs.await?.start_watching()?;
Expand All @@ -19,7 +19,7 @@ pub async fn directory_from_relative_path(

#[turbo_tasks::function]
pub async fn directory_from_include_dir(
name: String,
name: RcStr,
dir: TransientInstance<&'static include_dir::Dir<'static>>,
) -> Result<Vc<Box<dyn FileSystem>>> {
Ok(Vc::upcast(EmbeddedFileSystem::new(name, dir)))
Expand Down Expand Up @@ -72,7 +72,7 @@ macro_rules! embed_directory_internal {
static dir: include_dir::Dir<'static> = turbo_tasks_fs::embed::include_dir!($path);

turbo_tasks_fs::embed::directory_from_include_dir(
$name.to_string(),
$name.to_string().into(),
turbo_tasks::TransientInstance::new(&dir),
)
}};
Expand Down
27 changes: 13 additions & 14 deletions crates/turbo-tasks-fs/src/embed/file.rs
@@ -1,37 +1,34 @@
use std::path::PathBuf;
use std::{path::Path, sync::Arc};

use anyhow::{Context, Result};
use dunce::canonicalize;
use turbo_tasks::Vc;
use turbo_tasks::{RcStr, Vc};

use crate::{DiskFileSystem, File, FileContent, FileSystem};

#[turbo_tasks::function]
pub async fn content_from_relative_path(

Check failure on line 10 in crates/turbo-tasks-fs/src/embed/file.rs

View workflow job for this annotation

GitHub Actions / Turbopack rust check

the trait bound `fn(Arc<std::string::String>, Arc<std::string::String>) -> impl futures::Future<Output = Result<Vc<FileContent>, anyhow::Error>> {content_from_relative_path_inline_function}: IntoTaskFn<_, _>` is not satisfied

Check failure on line 10 in crates/turbo-tasks-fs/src/embed/file.rs

View workflow job for this annotation

GitHub Actions / Turbopack rust check

the trait bound `fn(Arc<std::string::String>, Arc<std::string::String>) -> impl futures::Future<Output = Result<Vc<FileContent>, anyhow::Error>> {content_from_relative_path_inline_function}: IntoTaskFn<_, _>` is not satisfied
package_path: String,
path: String,
package_path: Arc<String>,
path: Arc<String>,
) -> Result<Vc<FileContent>> {
let package_path = PathBuf::from(package_path);
let resolved_path = package_path.join(path);
let package_path = Path::new(&**package_path);
let resolved_path = package_path.join(&**path);
let resolved_path =
canonicalize(&resolved_path).context("failed to canonicalize embedded file path")?;
let root_path = resolved_path.parent().unwrap();
let path = resolved_path.file_name().unwrap().to_str().unwrap();

let disk_fs = DiskFileSystem::new(
root_path.to_string_lossy().to_string(),
root_path.to_string_lossy().to_string(),
vec![],
);
let root_path_str = Arc::new(root_path.to_string_lossy().to_string());
let disk_fs = DiskFileSystem::new(root_path_str.clone(), root_path_str, vec![]);

Check failure on line 22 in crates/turbo-tasks-fs/src/embed/file.rs

View workflow job for this annotation

GitHub Actions / Turbopack rust check

arguments to this function are incorrect

Check failure on line 22 in crates/turbo-tasks-fs/src/embed/file.rs

View workflow job for this annotation

GitHub Actions / Turbopack rust check

arguments to this function are incorrect
disk_fs.await?.start_watching()?;

let fs_path = disk_fs.root().join(path.to_string());
let fs_path = disk_fs.root().join(path.to_string().into());
Ok(fs_path.read())
}

#[turbo_tasks::function]
pub async fn content_from_str(string: String) -> Result<Vc<FileContent>> {
pub async fn content_from_str(string: RcStr) -> Result<Vc<FileContent>> {
Ok(File::from(string).into())

Check failure on line 31 in crates/turbo-tasks-fs/src/embed/file.rs

View workflow job for this annotation

GitHub Actions / Turbopack rust check

the trait bound `File: std::convert::From<RcStr>` is not satisfied

Check failure on line 31 in crates/turbo-tasks-fs/src/embed/file.rs

View workflow job for this annotation

GitHub Actions / Turbopack rust check

the trait bound `File: std::convert::From<RcStr>` is not satisfied
}

/// Loads a file's content from disk and invalidates on change (debug builds).
Expand All @@ -57,7 +54,9 @@
macro_rules! embed_file {
($path:expr) => {
turbo_tasks_fs::embed::content_from_str(
include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/", $path)).to_string(),
include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/", $path))
.to_string()
.into(),
)
};
}