Skip to content

Commit

Permalink
Cache documents, schema loading + os.cpus for concurrency limit + asy…
Browse files Browse the repository at this point in the history 
…nc fs (#5707)

* Cache documents and schema loading

* changelog

* Use os.cpus to calculate concurrency limit

* fs async

* changes

* revert config on cache (already introduced at codegen level)

* refactor(cli): use native node fs/promises

* fix(cli): imports issue

* fix(changeset): remove useless changeset

* fix(cli): remove duplicated code..

* fix(cli): remove useless async/await

Co-authored-by: Charly POLY <cpoly55@gmail.com>
  • Loading branch information
@kamilkisiela @charlypoly
kamilkisiela and charlypoly committed Feb 9, 2022
1 parent a01a741 commit 35566a0
Show file tree
Hide file tree
Showing 7 changed files with 211 additions and 198 deletions.
5 changes: 5 additions & 0 deletions .changeset/serious-months-enjoy.md
View file
@@ -0,0 +1,5 @@
---
'@graphql-codegen/cli': minor
---

Use os.cpus to calculate concurrency limit
5 changes: 5 additions & 0 deletions .changeset/two-bears-poke.md
View file
@@ -0,0 +1,5 @@
---
'@graphql-codegen/cli': minor
---

Async File System
4 changes: 2 additions & 2 deletions packages/graphql-codegen-cli/src/codegen.ts
View file
Expand Up @@ -20,6 +20,7 @@ import { debugLog } from './utils/debugging';
import { CodegenContext, ensureContext } from './config';
import fs from 'fs';
import path from 'path';
import { cpus } from 'os';
// eslint-disable-next-line
import { createRequire } from 'module';
import Listr from 'listr';
Expand Down Expand Up @@ -381,8 +382,7 @@ export async function executeCodegen(input: CodegenContext | Types.Config): Prom
{
// it doesn't stop when one of tasks failed, to finish at least some of outputs
exitOnError: false,
// run 4 at once
concurrent: 4,
concurrent: cpus().length,
}
);
},
Expand Down
23 changes: 11 additions & 12 deletions packages/graphql-codegen-cli/src/generate-and-save.ts
View file
Expand Up @@ -2,7 +2,7 @@ import { lifecycleHooks } from './hooks';
import { Types } from '@graphql-codegen/plugin-helpers';
import { executeCodegen } from './codegen';
import { createWatcher } from './utils/watcher';
import { fileExists, readSync, writeSync, unlinkFile } from './utils/file-system';
import { fileExists, readFile, writeFile, unlinkFile } from './utils/file-system';
import mkdirp from 'mkdirp';
import { dirname, join, isAbsolute } from 'path';
import { debugLog } from './utils/debugging';
Expand All @@ -26,7 +26,7 @@ export async function generate(
const staleFilenames = previouslyGeneratedFilenames.filter(f => !filenames.includes(f));
staleFilenames.forEach(filename => {
if (shouldOverwrite(config, filename)) {
unlinkFile(filename, err => {
return unlinkFile(filename, err => {
const prettyFilename = filename.replace(`${input.cwd || process.cwd()}/`, '');
if (err) {
debugLog(`Cannot remove stale file: ${prettyFilename}\n${err}`);
Expand All @@ -49,16 +49,15 @@ export async function generate(
removeStaleFiles(config, generationResult);
}

await context.profiler.run(
() => lifecycleHooks(config.hooks).beforeAllFileWrite(generationResult.map(r => r.filename)),
'Lifecycle: beforeAllFileWrite'
);
await context.profiler.run(async () => {
await lifecycleHooks(config.hooks).beforeAllFileWrite(generationResult.map(r => r.filename));
}, 'Lifecycle: beforeAllFileWrite');

await context.profiler.run(
() =>
Promise.all(
generationResult.map(async (result: Types.FileOutput) => {
const exists = fileExists(result.filename);
const exists = await fileExists(result.filename);

if (!shouldOverwrite(config, result.filename) && exists) {
return;
Expand All @@ -69,11 +68,11 @@ export async function generate(
let previousHash = recentOutputHash.get(result.filename);

if (!previousHash && exists) {
previousHash = hash(readSync(result.filename));
previousHash = hash(await readFile(result.filename));
}

if (previousHash && currentHash === previousHash) {
debugLog(`Skipping file (${result.filename}) writing due to identical hash...`);
debugLog(`Skipping file (${result.filename}) writing due to indentical hash...`);

return;
}
Expand All @@ -86,11 +85,11 @@ export async function generate(
const basedir = dirname(result.filename);
await lifecycleHooks(result.hooks).beforeOneFileWrite(result.filename);
await lifecycleHooks(config.hooks).beforeOneFileWrite(result.filename);
mkdirp.sync(basedir);
await mkdirp(basedir);
const absolutePath = isAbsolute(result.filename)
? result.filename
: join(input.cwd || process.cwd(), result.filename);
writeSync(absolutePath, result.content);
await writeFile(absolutePath, result.content);
await lifecycleHooks(result.hooks).afterOneFileWrite(result.filename);
await lifecycleHooks(config.hooks).afterOneFileWrite(result.filename);
})
Expand All @@ -117,7 +116,7 @@ export async function generate(
await context.profiler.run(() => lifecycleHooks(config.hooks).beforeDone(), 'Lifecycle: beforeDone');

if (context.profilerOutput) {
writeSync(join(context.cwd, context.profilerOutput), JSON.stringify(context.profiler.collect()));
await writeFile(join(context.cwd, context.profilerOutput), JSON.stringify(context.profiler.collect()));
}

return outputFiles;
Expand Down
17 changes: 9 additions & 8 deletions packages/graphql-codegen-cli/src/utils/file-system.ts
View file
@@ -1,21 +1,22 @@
import { writeFileSync, statSync, readFileSync, unlink } from 'fs';
import { unlink as fsUnlink, promises } from 'fs';
const { writeFile: fsWriteFile, readFile: fsReadFile, stat: fsStat } = promises;

export function writeSync(filepath: string, content: string) {
return writeFileSync(filepath, content);
export function writeFile(filepath: string, content: string) {
return fsWriteFile(filepath, content);
}

export function readSync(filepath: string) {
return readFileSync(filepath, 'utf-8');
export function readFile(filepath: string) {
return fsReadFile(filepath, 'utf-8');
}

export function fileExists(filePath: string): boolean {
export async function fileExists(filePath: string): Promise<boolean> {
try {
return statSync(filePath).isFile();
return (await fsStat(filePath)).isFile();
} catch (err) {
return false;
}
}

export function unlinkFile(filePath: string, cb?: (err?: Error) => any): void {
unlink(filePath, cb);
fsUnlink(filePath, cb);
}
22 changes: 11 additions & 11 deletions packages/graphql-codegen-cli/tests/generate-and-save.spec.ts
View file
Expand Up @@ -21,7 +21,7 @@ describe('generate-and-save', () => {

test('allow to specify overwrite for specific output (should write file)', async () => {
const filename = 'overwrite.ts';
const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();

const output = await generate(
{
Expand All @@ -47,10 +47,10 @@ describe('generate-and-save', () => {

test('allow to specify overwrite for specific output (should not write file)', async () => {
const filename = 'overwrite.ts';
const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
// forces file to exist
const fileExistsSpy = jest.spyOn(fs, 'fileExists');
fileExistsSpy.mockImplementation(file => file === filename);
fileExistsSpy.mockImplementation(async file => file === filename);

const output = await generate(
{
Expand Down Expand Up @@ -78,7 +78,7 @@ describe('generate-and-save', () => {

test('should use global overwrite option and write a file', async () => {
const filename = 'overwrite.ts';
const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();

const output = await generate(
{
Expand All @@ -103,10 +103,10 @@ describe('generate-and-save', () => {

test('should use global overwrite option and not write a file', async () => {
const filename = 'overwrite.ts';
const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
// forces file to exist
const fileExistsSpy = jest.spyOn(fs, 'fileExists');
fileExistsSpy.mockImplementation(file => file === filename);
fileExistsSpy.mockImplementation(async file => file === filename);

const output = await generate(
{
Expand All @@ -133,12 +133,12 @@ describe('generate-and-save', () => {

test('should overwrite a file by default', async () => {
const filename = 'overwrite.ts';
const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
const readSpy = jest.spyOn(fs, 'readSync').mockImplementation();
readSpy.mockImplementation(_f => '');
const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();
const readSpy = jest.spyOn(fs, 'readFile').mockImplementation();
readSpy.mockImplementation(async _f => '');
// forces file to exist
const fileExistsSpy = jest.spyOn(fs, 'fileExists');
fileExistsSpy.mockImplementation(file => file === filename);
fileExistsSpy.mockImplementation(async file => file === filename);

const output = await generate(
{
Expand Down Expand Up @@ -197,7 +197,7 @@ describe('generate-and-save', () => {
});
test('should extract a document from the gql tag (imported from apollo-server)', async () => {
const filename = 'overwrite.ts';
const writeSpy = jest.spyOn(fs, 'writeSync').mockImplementation();
const writeSpy = jest.spyOn(fs, 'writeFile').mockImplementation();

const output = await generate(
{
Expand Down

1 comment on commit 35566a0

@vercel
Copy link

@vercel vercel bot commented on 35566a0 Feb 9, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

Please sign in to comment.