feat(core): introduce workspace file archive (#20471)

packages/nx/Cargo.toml

@@ -10,7 +10,7 @@ crossbeam-channel = '0.5'
 dashmap = { version = "5.5.3", features = ["rayon"] }
 fs_extra = "1.3.0"
 globset = "0.4.10"
-hashbrown = { version = "0.14.0", features = ["rayon"] }
+hashbrown = { version = "0.14.3", features = ["rayon", "rkyv"] }
 ignore = '0.4'
 ignore-files = "1.3.0"
 itertools = "0.10.5"
@@ -25,11 +25,11 @@ napi-derive = '2.9.3'
 nom = '7.1.3'
 regex = "1.9.1"
 rayon = "1.7.0"
+rkyv = { version = "0.7", features = ["validation"] }
 thiserror = "1.0.40"
 tokio = { version = "1.28.2", features = ["fs"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
-tsconfig = "0.2"
 walkdir = '2.3.3'
 watchexec = "2.3.0"
 watchexec-events = "1.0.0"

packages/nx/src/native/index.d.ts

@@ -169,7 +169,7 @@ export class Watcher {
 }
 export class WorkspaceContext {
   workspaceRoot: string
-  constructor(workspaceRoot: string)
+  constructor(workspaceRoot: string, cacheDir: string)
   getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
   glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
   hashFilesMatchingGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): string

packages/nx/src/native/plugins/js/ts_import_locators.rs

@@ -666,7 +666,6 @@ fn find_imports(
 mod find_imports {
     use super::*;
     use crate::native::glob::build_glob_set;
-    use crate::native::utils::Normalize;
     use crate::native::walker::nx_walker;
     use assert_fs::prelude::*;
     use assert_fs::TempDir;
@@ -1363,8 +1362,8 @@ import('./dynamic-import.vue')
 
         let glob = build_glob_set(&["**/*.[jt]s"]).unwrap();
         let files = nx_walker(root.clone())
-            .filter(|(full_path, _)| glob.is_match(full_path))
-            .map(|(full_path, _)| full_path.to_normalized_string())
+            .filter(|file| glob.is_match(&file.full_path))
+            .map(|file| file.full_path)
             .collect::<Vec<_>>();
 
         let results: HashMap<_, _> =

packages/nx/src/native/tests/workspace_files.spec.ts

@@ -3,6 +3,7 @@ import { TempFs } from '../../internal-testing-utils/temp-fs';
 import { NxJsonConfiguration } from '../../config/nx-json';
 import { dirname, join } from 'path';
 import { readJsonFile } from '../../utils/fileutils';
+import { cacheDirectoryForWorkspace } from '../../utils/cache-directory';
 
 describe('workspace files', () => {
   function createParseConfigurationsFunction(tempDir: string) {
@@ -49,16 +50,17 @@ describe('workspace files', () => {
       './nested/non-project/file.txt': '',
     });
 
-    const context = new WorkspaceContext(fs.tempDir);
-    let { projectFileMap, globalFiles } = await context.getWorkspaceFiles(
-      {
-        'libs/project1': 'project1',
-        'libs/project2': 'project2',
-        'libs/project3': 'project3',
-        'libs/nested/project': 'nested-project',
-        'libs/package-project': 'package-project'
-      }
+    const context = new WorkspaceContext(
+      fs.tempDir,
+      cacheDirectoryForWorkspace(fs.tempDir)
     );
+    let { projectFileMap, globalFiles } = await context.getWorkspaceFiles({
+      'libs/project1': 'project1',
+      'libs/project2': 'project2',
+      'libs/project3': 'project3',
+      'libs/nested/project': 'nested-project',
+      'libs/package-project': 'package-project',
+    });
 
     expect(projectFileMap).toMatchInlineSnapshot(`
       {
@@ -149,14 +151,15 @@ describe('workspace files', () => {
       './jest.config.js': '',
     });
 
-    const context = new WorkspaceContext(fs.tempDir);
-
-    const { globalFiles, projectFileMap } = await context.getWorkspaceFiles(
-      {
-        '.': 'repo-name'
-      }
+    const context = new WorkspaceContext(
+      fs.tempDir,
+      cacheDirectoryForWorkspace(fs.tempDir)
     );
 
+    const { globalFiles, projectFileMap } = await context.getWorkspaceFiles({
+      '.': 'repo-name',
+    });
+
     expect(globalFiles).toEqual([]);
     expect(projectFileMap['repo-name']).toMatchInlineSnapshot(`
       [

packages/nx/src/native/utils/get_mod_time.rs

@@ -0,0 +1,19 @@
+use std::fs::Metadata;
+
+#[cfg(target_os = "macos")]
+pub fn get_mod_time(metadata: &Metadata) -> i64 {
+    use std::os::macos::fs::MetadataExt;
+    metadata.st_mtime()
+}
+
+#[cfg(target_os = "windows")]
+pub fn get_mod_time(metadata: &Metadata) -> i64 {
+    use std::os::windows::fs::MetadataExt;
+    metadata.last_write_time() as i64
+}
+
+#[cfg(target_os = "linux")]
+pub fn get_mod_time(metadata: &Metadata) -> i64 {
+    use std::os::unix::fs::MetadataExt;
+    metadata.mtime()
+}

packages/nx/src/native/utils/mod.rs

@@ -1,6 +1,8 @@
 mod find_matching_projects;
+mod get_mod_time;
 mod normalize_trait;
 pub mod path;
 
 pub use find_matching_projects::*;
+pub use get_mod_time::*;
 pub use normalize_trait::Normalize;

packages/nx/src/native/utils/path.rs

@@ -3,17 +3,17 @@ use std::path::{Path, PathBuf};
 
 impl Normalize for Path {
     fn to_normalized_string(&self) -> String {
-        normalize_path(self)
+        normalize_nx_path(self)
     }
 }
 
 impl Normalize for PathBuf {
     fn to_normalized_string(&self) -> String {
-        normalize_path(self)
+        normalize_nx_path(self)
     }
 }
 
-fn normalize_path<P>(path: P) -> String
+fn normalize_nx_path<P>(path: P) -> String
 where
     P: AsRef<Path>,
 {

packages/nx/src/native/walker.rs

@@ -8,8 +8,16 @@ use tracing::trace;
 
 use crate::native::glob::build_glob_set;
 
+use crate::native::utils::{get_mod_time, Normalize};
 use walkdir::WalkDir;
 
+#[derive(PartialEq, Debug, Ord, PartialOrd, Eq, Clone)]
+pub struct NxFile {
+    pub full_path: String,
+    pub normalized_path: String,
+    pub mod_time: i64,
+}
+
 /// Walks the directory in a single thread and does not ignore any files
 /// Should only be used for small directories, and not traversing the whole workspace
 pub fn nx_walker_sync<'a, P>(directory: P) -> impl Iterator<Item = PathBuf>
@@ -36,7 +44,7 @@ where
 }
 
 /// Walk the directory and ignore files from .gitignore and .nxignore
-pub fn nx_walker<P>(directory: P) -> impl Iterator<Item = (PathBuf, PathBuf)>
+pub fn nx_walker<P>(directory: P) -> impl Iterator<Item = NxFile>
 where
     P: AsRef<Path>,
 {
@@ -80,8 +88,16 @@ where
                 return Continue;
             };
 
-            tx.send((dir_entry.path().to_owned(), file_path.to_owned()))
-                .ok();
+            let Ok(metadata) = dir_entry.metadata() else {
+                return Continue;
+            };
+
+            tx.send(NxFile {
+                full_path: String::from(dir_entry.path().to_string_lossy()),
+                normalized_path: file_path.to_normalized_string(),
+                mod_time: get_mod_time(&metadata),
+            })
+            .ok();
 
             Continue
         })
@@ -100,8 +116,6 @@ mod test {
     use assert_fs::prelude::*;
     use assert_fs::TempDir;
 
-    use crate::native::utils::Normalize;
-
     use super::*;
 
     ///
@@ -133,6 +147,10 @@ mod test {
 
         let mut content = nx_walker(&temp_dir).collect::<Vec<_>>();
         content.sort();
+        let content = content
+            .into_iter()
+            .map(|f| (f.full_path.into(), f.normalized_path.into()))
+            .collect::<Vec<_>>();
         assert_eq!(
             content,
             vec![
@@ -173,7 +191,12 @@ nested/child-two/
 
         let mut file_names = nx_walker(temp_dir)
             .into_iter()
-            .map(|(_, p)| p.to_normalized_string())
+            .map(
+                |NxFile {
+                     normalized_path: relative_path,
+                     ..
+                 }| relative_path,
+            )
             .collect::<Vec<_>>();
 
         file_names.sort();

packages/nx/src/native/workspace/context.rs

@@ -1,26 +1,26 @@
 use napi::bindgen_prelude::External;
 use std::collections::HashMap;
 
-use crate::native::hasher::{hash, hash_file_path};
+use crate::native::hasher::hash;
 use crate::native::utils::Normalize;
 use rayon::prelude::*;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::thread::available_parallelism;
-use std::{cmp, thread};
+use std::thread;
 
 use crate::native::logger::enable_logger;
 use crate::native::project_graph::utils::{find_project_for_path, ProjectRootMappings};
 use crate::native::types::FileData;
 use parking_lot::{Condvar, Mutex};
 use tracing::{trace, warn};
 
-use crate::native::walker::nx_walker;
+use crate::native::workspace::files_archive::{read_files_archive, write_files_archive};
+use crate::native::workspace::files_hashing::{full_files_hash, selective_files_hash};
 use crate::native::workspace::types::{
     FileMap, NxWorkspaceFilesExternals, ProjectFiles, UpdatedWorkspaceFiles,
 };
-use crate::native::workspace::{config_files, workspace_files, types::NxWorkspaceFiles};
+use crate::native::workspace::{config_files, types::NxWorkspaceFiles, workspace_files};
 
 #[napi]
 pub struct WorkspaceContext {
@@ -33,7 +33,7 @@ type Files = Vec<(PathBuf, String)>;
 
 struct FilesWorker(Option<Arc<(Mutex<Files>, Condvar)>>);
 impl FilesWorker {
-    fn gather_files(workspace_root: &Path) -> Self {
+    fn gather_files(workspace_root: &Path, cache_dir: String) -> Self {
         if !workspace_root.exists() {
             warn!(
                 "workspace root does not exist: {}",
@@ -42,6 +42,8 @@ impl FilesWorker {
             return FilesWorker(None);
         }
 
+        let archived_files = read_files_archive(&cache_dir);
+
         let files_lock = Arc::new((Mutex::new(Vec::new()), Condvar::new()));
         let files_lock_clone = Arc::clone(&files_lock);
         let workspace_root = workspace_root.to_owned();
@@ -50,38 +52,27 @@ impl FilesWorker {
             trace!("locking files");
             let (lock, cvar) = &*files_lock_clone;
             let mut workspace_files = lock.lock();
-
-            let files = nx_walker(workspace_root).collect::<Vec<_>>();
-            let num_parallelism = cmp::max(available_parallelism().map_or(2, |n| n.get()) / 3, 2);
-            let chunks = files.len() / num_parallelism;
-
             let now = std::time::Instant::now();
-
-            let mut files = if chunks < num_parallelism {
-                files
-                    .iter()
-                    .filter_map(|(full_path, path)| {
-                        hash_file_path(full_path).map(|hash| (path.to_owned(), hash))
-                    })
-                    .collect::<Vec<_>>()
+            let file_hashes = if let Some(archived_files) = archived_files {
+                selective_files_hash(&workspace_root, archived_files)
             } else {
-                files
-                    .par_chunks(chunks)
-                    .flat_map_iter(|chunks| {
-                        chunks.iter().filter_map(|(full_path, path)| {
-                            hash_file_path(full_path).map(|hash| (path.to_owned(), hash))
-                        })
-                    })
-                    .collect::<Vec<_>>()
+                full_files_hash(&workspace_root)
             };
 
+            let mut files = file_hashes
+                .iter()
+                .map(|(path, file_hashed)| (PathBuf::from(path), file_hashed.0.to_owned()))
+                .collect::<Vec<_>>();
             files.par_sort();
-            trace!("hashed and sorted workspace files in {:?}", now.elapsed());
+            trace!("hashed and sorted files in {:?}", now.elapsed());
+
             *workspace_files = files;
             let files_len = workspace_files.len();
             trace!(?files_len, "files retrieved");
 
             cvar.notify_all();
+
+            write_files_archive(&cache_dir, file_hashes);
         });
 
         FilesWorker(Some(files_lock))
@@ -162,15 +153,15 @@ impl FilesWorker {
 #[napi]
 impl WorkspaceContext {
     #[napi(constructor)]
-    pub fn new(workspace_root: String) -> Self {
+    pub fn new(workspace_root: String, cache_dir: String) -> Self {
         enable_logger();
 
         trace!(?workspace_root);
 
         let workspace_root_path = PathBuf::from(&workspace_root);
 
         WorkspaceContext {
-            files_worker: FilesWorker::gather_files(&workspace_root_path),
+            files_worker: FilesWorker::gather_files(&workspace_root_path, cache_dir),
             workspace_root,
             workspace_root_path,
         }
@@ -180,8 +171,7 @@ impl WorkspaceContext {
     pub fn get_workspace_files(
         &self,
         project_root_map: HashMap<String, String>,
-    ) -> anyhow::Result<NxWorkspaceFiles>
-    {
+    ) -> anyhow::Result<NxWorkspaceFiles> {
         workspace_files::get_files(project_root_map, self.all_file_data())
             .map_err(anyhow::Error::from)
     }