Skip to content

Commit

Permalink
fix(core): handle errors in the deamon when recomputing file map incr…
Browse files Browse the repository at this point in the history 
…ementally
  • Loading branch information
@vsavkin
vsavkin committed Dec 5, 2021
1 parent a4e97c7 commit 4935049
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 155 deletions.
22 changes: 11 additions & 11 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,19 +61,19 @@
"@ngrx/schematics": "~13.0.0",
"@ngrx/store": "~13.0.0",
"@ngrx/store-devtools": "~13.0.0",
"@nrwl/cli": "13.3.0-beta.12",
"@nrwl/cypress": "13.3.0-beta.12",
"@nrwl/eslint-plugin-nx": "13.3.0-beta.12",
"@nrwl/jest": "13.3.0-beta.12",
"@nrwl/cli": "13.3.0-beta.16",
"@nrwl/cypress": "13.3.0-beta.16",
"@nrwl/eslint-plugin-nx": "13.3.0-beta.16",
"@nrwl/jest": "13.3.0-beta.16",
"@nrwl/js": "13.3.0-beta.2",
"@nrwl/linter": "13.3.0-beta.12",
"@nrwl/next": "13.3.0-beta.12",
"@nrwl/node": "13.3.0-beta.12",
"@nrwl/linter": "13.3.0-beta.16",
"@nrwl/next": "13.3.0-beta.16",
"@nrwl/node": "13.3.0-beta.16",
"@nrwl/nx-cloud": "13.0.0-beta.17",
"@nrwl/react": "13.3.0-beta.12",
"@nrwl/tao": "13.3.0-beta.12",
"@nrwl/web": "13.3.0-beta.12",
"@nrwl/workspace": "13.3.0-beta.12",
"@nrwl/react": "13.3.0-beta.16",
"@nrwl/tao": "13.3.0-beta.16",
"@nrwl/web": "13.3.0-beta.16",
"@nrwl/workspace": "13.3.0-beta.16",
"@parcel/watcher": "2.0.4",
"@phenomnomnominal/tsquery": "4.1.1",
"@pmmmwh/react-refresh-webpack-plugin": "^0.5.1",
Expand Down
2 changes: 2 additions & 0 deletions packages/workspace/src/core/hasher/file-hasher.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export class FileHasher {
this.fileHashes = {};
this.workspaceFiles = new Set<string>();
this.usesGitForHashing = false;
this.isInitialized = false;
}

/**
Expand Down Expand Up @@ -48,6 +49,7 @@ export class FileHasher {
updatedFiles: Map<string, string>,
deletedFiles: string[] = []
): void {
this.ensureInitialized();
performance.mark('incremental hashing:start');

updatedFiles.forEach((hash, filename) => {
Expand Down
109 changes: 68 additions & 41 deletions ...workspace/src/core/project-graph/daemon/server/project-graph-incremental-recomputation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ export async function getCachedSerializedProjectGraphPromise() {
resetInternalStateIfNxDepsMissing();
if (collectedUpdatedFiles.size == 0 && collectedDeletedFiles.size == 0) {
if (!cachedSerializedProjectGraphPromise) {
processCollectedUpdatedAndDeletedFiles(); // this creates a project graph
cachedSerializedProjectGraphPromise = createAndSerializeProjectGraph();
cachedSerializedProjectGraphPromise =
processFilesAndCreateAndSerializeProjectGraph();
}
} else {
processCollectedUpdatedAndDeletedFiles();
cachedSerializedProjectGraphPromise = createAndSerializeProjectGraph();
cachedSerializedProjectGraphPromise =
processFilesAndCreateAndSerializeProjectGraph();
}
return await cachedSerializedProjectGraphPromise;
} catch (e) {
Expand Down Expand Up @@ -79,50 +79,76 @@ export function addUpdatedAndDeletedFiles(
if (waitPeriod < 4000) {
waitPeriod = waitPeriod * 2;
}
processCollectedUpdatedAndDeletedFiles();
cachedSerializedProjectGraphPromise = createAndSerializeProjectGraph();
cachedSerializedProjectGraphPromise =
processFilesAndCreateAndSerializeProjectGraph();
}, waitPeriod);
}
}

function processCollectedUpdatedAndDeletedFiles() {
performance.mark('hash-watched-changes-start');
const updatedFiles = getGitHashForFiles(
[...collectedUpdatedFiles.values()],
appRootPath
);
const deletedFiles = [...collectedDeletedFiles.values()];
performance.mark('hash-watched-changes-end');
performance.measure(
'hash changed files from watcher',
'hash-watched-changes-start',
'hash-watched-changes-end'
);
defaultFileHasher.incrementalUpdate(updatedFiles, deletedFiles);
const workspaceJson = readWorkspaceJson();
serverLogger.requestLog(
`Updated file-hasher based on watched changes, recomputing project graph...`
);
// when workspace.json changes we cannot be sure about the correctness of the project file map
if (
collectedUpdatedFiles.has(configName) ||
collectedDeletedFiles.has(configName)
) {
projectFileMapWithFiles = createProjectFileMap(workspaceJson);
} else {
projectFileMapWithFiles = projectFileMapWithFiles
? updateProjectFileMap(
workspaceJson,
projectFileMapWithFiles.projectFileMap,
projectFileMapWithFiles.allWorkspaceFiles,
updatedFiles,
deletedFiles
)
: createProjectFileMap(workspaceJson);
try {
performance.mark('hash-watched-changes-start');
const updatedFiles = getGitHashForFiles(
[...collectedUpdatedFiles.values()],
appRootPath
);
const deletedFiles = [...collectedDeletedFiles.values()];
performance.mark('hash-watched-changes-end');
performance.measure(
'hash changed files from watcher',
'hash-watched-changes-start',
'hash-watched-changes-end'
);
defaultFileHasher.incrementalUpdate(updatedFiles, deletedFiles);
const workspaceJson = readWorkspaceJson();
serverLogger.requestLog(
`Updated file-hasher based on watched changes, recomputing project graph...`
);
// when workspace.json changes we cannot be sure about the correctness of the project file map
if (
collectedUpdatedFiles.has(configName) ||
collectedDeletedFiles.has(configName)
) {
projectFileMapWithFiles = createProjectFileMap(workspaceJson);
} else {
projectFileMapWithFiles = projectFileMapWithFiles
? updateProjectFileMap(
workspaceJson,
projectFileMapWithFiles.projectFileMap,
projectFileMapWithFiles.allWorkspaceFiles,
updatedFiles,
deletedFiles
)
: createProjectFileMap(workspaceJson);
}

collectedUpdatedFiles.clear();
collectedDeletedFiles.clear();
} catch (e) {
// this is expected
// for instance, workspace.json can be incorrect etc
// we are resetting internal state to start from scratch next time a file changes
// given the user the opportunity to fix the error
// if Nx requests the project graph prior to the error being fixed,
// the error will be propagated
serverLogger.log(
`Error detected when recomputing project file map: ${e.message}`
);
resetInternalState();
return e;
}
}

collectedUpdatedFiles.clear();
collectedDeletedFiles.clear();
function processFilesAndCreateAndSerializeProjectGraph() {
const err = processCollectedUpdatedAndDeletedFiles();
if (err) {
return Promise.resolve({
error: err,
serializedProjectGraph: null,
});
} else {
return createAndSerializeProjectGraph();
}
}

async function createAndSerializeProjectGraph() {
Expand Down Expand Up @@ -173,6 +199,7 @@ export function resetInternalState() {
currentProjectGraphCache = undefined;
collectedUpdatedFiles.clear();
collectedDeletedFiles.clear();
defaultFileHasher.clear();
waitPeriod = 100;
}

Expand Down
7 changes: 4 additions & 3 deletions packages/workspace/src/core/project-graph/daemon/server/watcher.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,10 @@ export async function subscribeToWorkspaceChanges(
cachedIgnoreGlobs = getIgnoredGlobs(appRootPath);
}

const nonIgnoredEvents = workspaceRelativeEvents.filter(
({ path }) => !cachedIgnoreGlobs.ignores(path)
);
const nonIgnoredEvents = workspaceRelativeEvents
.filter(({ path }) => !!path)
.filter(({ path }) => !cachedIgnoreGlobs.ignores(path));

if (!nonIgnoredEvents || !nonIgnoredEvents.length) {
return;
}
Expand Down

1 comment on commit 4935049

@vercel
Copy link

@vercel vercel bot commented on 4935049 Dec 5, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

Please sign in to comment.