feat: support to run multi script commands specified with regex selector

.changeset/silly-flies-care.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/plugin-commands-script-runners": minor
+---
+
+feat: support wildcard selector to specify multiple scripts to execute"

exec/plugin-commands-script-runners/src/run.ts

@@ -1,4 +1,5 @@
 import path from 'path'
+import pLimit from 'p-limit'
 import {
   docsUrl,
   readProjectManifestOnly,
@@ -43,13 +44,22 @@ export const RESUME_FROM_OPTION_HELP = {
   name: '--resume-from',
 }
 
+export const SEQUENTIAL_OPTION_HELP = {
+  description: 'Run a multiple scripts specified with workspace selector or a wildcard script selector sequential.\
+This is the preferred flag for tasks which has dependency on execution order.',
+  name: '--sequential',
+}
+
 export const shorthands = {
   parallel: [
     '--workspace-concurrency=Infinity',
     '--no-sort',
     '--stream',
     '--recursive',
   ],
+  sequential: [
+    '--workspace-concurrency=1',
+  ],
 }
 
 export function rcOptionsTypes () {
@@ -112,6 +122,7 @@ For options that may be used with `-r`, see "pnpm help recursive"',
           PARALLEL_OPTION_HELP,
           RESUME_FROM_OPTION_HELP,
           ...UNIVERSAL_OPTIONS,
+          SEQUENTIAL_OPTION_HELP,
         ],
       },
       FILTERING,
@@ -159,7 +170,13 @@ export async function handler (
       : undefined
     return printProjectCommands(manifest, rootManifest ?? undefined)
   }
-  if (scriptName !== 'start' && !manifest.scripts?.[scriptName]) {
+
+  const multiScriptSelectorSpecified = scriptName.slice(-2) === ':*'
+  // extract XXX:YYY: from XXX:YYY:ZZZ
+  const multiScriptSelectorRegExp = new RegExp(scriptName)
+  const specifiedScriptsWithWildcardSelector = Object.keys(manifest.scripts ?? {}).filter(script => script.match(multiScriptSelectorRegExp))
+
+  if (scriptName !== 'start' && !manifest.scripts?.[scriptName] && !(multiScriptSelectorSpecified && specifiedScriptsWithWildcardSelector.length > 0)) {
     if (opts.ifPresent) return
     if (opts.fallbackCommandUsed) {
       if (opts.argv == null) throw new Error('Could not fallback because opts.argv.original was not passed to the script runner')
@@ -170,7 +187,7 @@ export async function handler (
     }
     if (opts.workspaceDir) {
       const { manifest: rootManifest } = await tryReadProjectManifest(opts.workspaceDir, opts)
-      if (rootManifest?.scripts?.[scriptName]) {
+      if (rootManifest?.scripts?.[scriptName] && !(multiScriptSelectorSpecified && specifiedScriptsWithWildcardSelector.length > 0)) {
         throw new PnpmError('NO_SCRIPT', `Missing script: ${scriptName}`, {
           hint: `But ${scriptName} is present in the root of the workspace,
 so you may run "pnpm -w run ${scriptName}"`,
@@ -203,20 +220,12 @@ so you may run "pnpm -w run ${scriptName}"`,
     }
   }
   try {
-    if (
-      opts.enablePrePostScripts &&
-      manifest.scripts?.[`pre${scriptName}`] &&
-      !manifest.scripts[scriptName].includes(`pre${scriptName}`)
-    ) {
-      await runLifecycleHook(`pre${scriptName}`, manifest, lifecycleOpts)
-    }
-    await runLifecycleHook(scriptName, manifest, { ...lifecycleOpts, args: passedThruArgs })
-    if (
-      opts.enablePrePostScripts &&
-      manifest.scripts?.[`post${scriptName}`] &&
-      !manifest.scripts[scriptName].includes(`post${scriptName}`)
-    ) {
-      await runLifecycleHook(`post${scriptName}`, manifest, lifecycleOpts)
+    if (multiScriptSelectorSpecified) {
+      const limitRun = pLimit(opts.workspaceConcurrency ?? 4)
+
+      await Promise.all(specifiedScriptsWithWildcardSelector.map(script => limitRun(() => runScript(script, manifest, lifecycleOpts, { enablePrePostScripts: opts.enablePrePostScripts ?? false }, passedThruArgs))))
+    } else {
+      await runScript(scriptName, manifest, lifecycleOpts, { enablePrePostScripts: opts.enablePrePostScripts ?? false }, passedThruArgs)
     }
   } catch (err: any) { // eslint-disable-line
     if (opts.bail !== false) {
@@ -300,6 +309,28 @@ ${renderCommands(rootScripts)}`
   return output
 }
 
+export interface RunScriptOptions {
+  enablePrePostScripts: boolean
+}
+
+export const runScript: (scriptName: string, manifest: ProjectManifest, lifecycleOpts: RunLifecycleHookOptions, runScriptOptions: RunScriptOptions, passedThruArgs: string[]) => Promise<void> = async function (scriptName, manifest, lifecycleOpts, runScriptOptions, passedThruArgs) {
+  if (
+    runScriptOptions.enablePrePostScripts &&
+    manifest.scripts?.[`pre${scriptName}`] &&
+    !manifest.scripts[scriptName].includes(`pre${scriptName}`)
+  ) {
+    await runLifecycleHook(`pre${scriptName}`, manifest, lifecycleOpts)
+  }
+  await runLifecycleHook(scriptName, manifest, { ...lifecycleOpts, args: passedThruArgs })
+  if (
+    runScriptOptions.enablePrePostScripts &&
+    manifest.scripts?.[`post${scriptName}`] &&
+    !manifest.scripts[scriptName].includes(`post${scriptName}`)
+  ) {
+    await runLifecycleHook(`post${scriptName}`, manifest, lifecycleOpts)
+  }
+}
+
 function renderCommands (commands: string[][]) {
   return commands.map(([scriptName, script]) => `  ${scriptName}\n    ${script}`).join('\n')
 }

exec/plugin-commands-script-runners/src/runRecursive.ts

@@ -3,7 +3,6 @@ import { RecursiveSummary, throwOnCommandFail } from '@pnpm/cli-utils'
 import { Config } from '@pnpm/config'
 import { PnpmError } from '@pnpm/error'
 import {
-  runLifecycleHook,
   makeNodeRequireOption,
   RunLifecycleHookOptions,
 } from '@pnpm/lifecycle'
@@ -13,6 +12,7 @@ import pLimit from 'p-limit'
 import realpathMissing from 'realpath-missing'
 import { existsInDir } from './existsInDir'
 import { getResumedPackageChunks } from './exec'
+import { runScript } from './run'
 
 export type RecursiveRunOpts = Pick<Config,
 | 'enablePrePostScripts'
@@ -68,20 +68,37 @@ export async function runRecursive (
   const existsPnp = existsInDir.bind(null, '.pnp.cjs')
   const workspacePnpPath = opts.workspaceDir && await existsPnp(opts.workspaceDir)
 
+  const multiScriptSelectorSpecified = scriptName.slice(-2) === ':*'
+  const multiScriptSelectorRegExp = new RegExp(scriptName)
+
   const requiredScripts = opts.rootProjectManifest?.pnpm?.requiredScripts ?? []
   if (requiredScripts.includes(scriptName)) {
     const missingScriptPackages: string[] = packageChunks
       .flat()
       .map((prefix) => opts.selectedProjectsGraph[prefix])
-      .filter((pkg) => !pkg.package.manifest.scripts?.[scriptName])
+      .filter((pkg) => multiScriptSelectorSpecified ? !Object.keys(pkg.package.manifest.scripts ?? {}).some(script => script.match(multiScriptSelectorRegExp)) : !pkg.package.manifest.scripts?.[scriptName])
       .map((pkg) => pkg.package.manifest.name ?? pkg.package.dir)
     if (missingScriptPackages.length) {
       throw new PnpmError('RECURSIVE_RUN_NO_SCRIPT', `Missing script "${scriptName}" in packages: ${missingScriptPackages.join(', ')}`)
     }
   }
 
   for (const chunk of packageChunks) {
-    await Promise.all(chunk.map(async (prefix: string) =>
+    const selectedScripts = chunk.map(prefix => {
+      if (!multiScriptSelectorSpecified) {
+        return {
+          prefix,
+          scriptName,
+        }
+      }
+
+      const pkg = opts.selectedProjectsGraph[prefix]
+      const specifiedScriptsWithSelector = Object.keys(pkg.package.manifest.scripts ?? {}).filter(script => script.match(multiScriptSelectorRegExp))
+
+      return specifiedScriptsWithSelector.map(script => ({ prefix, scriptName: script }))
+    }).flat()
+
+    await Promise.all(selectedScripts.map(async ({ prefix, scriptName }) =>
       limitRun(async () => {
         const pkg = opts.selectedProjectsGraph[prefix]
         if (
@@ -113,21 +130,8 @@ export async function runRecursive (
               ...makeNodeRequireOption(pnpPath),
             }
           }
-          if (
-            opts.enablePrePostScripts &&
-            pkg.package.manifest.scripts?.[`pre${scriptName}`] &&
-            !pkg.package.manifest.scripts[scriptName].includes(`pre${scriptName}`)
-          ) {
-            await runLifecycleHook(`pre${scriptName}`, pkg.package.manifest, lifecycleOpts)
-          }
-          await runLifecycleHook(scriptName, pkg.package.manifest, { ...lifecycleOpts, args: passedThruArgs })
-          if (
-            opts.enablePrePostScripts &&
-            pkg.package.manifest.scripts?.[`post${scriptName}`] &&
-            !pkg.package.manifest.scripts[scriptName].includes(`post${scriptName}`)
-          ) {
-            await runLifecycleHook(`post${scriptName}`, pkg.package.manifest, lifecycleOpts)
-          }
+
+          await runScript(scriptName, pkg.package.manifest, lifecycleOpts, { enablePrePostScripts: opts.enablePrePostScripts ?? false }, passedThruArgs)
           result.passes++
         } catch (err: any) { // eslint-disable-line
           logger.info(err)

exec/plugin-commands-script-runners/test/index.ts

@@ -465,3 +465,91 @@ test('pnpm run with custom shell', async () => {
 
   expect((await import(path.resolve('shell-input.json'))).default).toStrictEqual(['-c', 'foo bar'])
 })
+
+test('pnpm run with multiple script selector should work', async () => {
+  prepare({
+    scripts: {
+      'build:a': 'node -e "require(\'fs\').writeFileSync(\'./output-a.txt\', \'a\', \'utf8\')"',
+      'build:b': 'node -e "require(\'fs\').writeFileSync(\'./output-b.txt\', \'b\', \'utf8\')"',
+      'build:c': 'node -e "require(\'fs\').writeFileSync(\'./output-c.txt\', \'c\', \'utf8\')"',
+    },
+  })
+
+  await run.handler({
+    dir: process.cwd(),
+    extraBinPaths: [],
+    extraEnv: {},
+    rawConfig: {},
+  }, ['build:*'])
+
+  expect(await fs.readFile('output-a.txt', { encoding: 'utf-8' })).toEqual('a')
+  expect(await fs.readFile('output-b.txt', { encoding: 'utf-8' })).toEqual('b')
+  expect(await fs.readFile('output-c.txt', { encoding: 'utf-8' })).toEqual('c')
+})
+
+test('pnpm run with multiple script selector should work also for pre/post script', async () => {
+  prepare({
+    scripts: {
+      'build:a': 'node -e "require(\'fs\').writeFileSync(\'./output-a.txt\', \'a\', \'utf8\')"',
+      'prebuild:a': 'node -e "require(\'fs\').writeFileSync(\'./output-pre-a.txt\', \'pre-a\', \'utf8\')"',
+    },
+  })
+
+  await run.handler({
+    dir: process.cwd(),
+    extraBinPaths: [],
+    extraEnv: {},
+    rawConfig: {},
+    enablePrePostScripts: true,
+  }, ['build:*'])
+
+  expect(await fs.readFile('output-a.txt', { encoding: 'utf-8' })).toEqual('a')
+  expect(await fs.readFile('output-pre-a.txt', { encoding: 'utf-8' })).toEqual('pre-a')
+})
+
+test('pnpm run with multiple script selector should work parallel as a default behavior (parallel execution limits number is four)', async () => {
+  prepare({
+    scripts: {
+      'build:a': 'node -e "let i = 20;setInterval(() => {if (!--i) process.exit(0); require(\'json-append\').append(Date.now(),\'./output-a.json\');},50)"',
+      'build:b': 'node -e "let i = 40;setInterval(() => {if (!--i) process.exit(0); require(\'json-append\').append(Date.now(),\'./output-b.json\');},25)"',
+    },
+  })
+
+  await execa('pnpm', ['add', 'json-append@1'])
+
+  await run.handler({
+    dir: process.cwd(),
+    extraBinPaths: [],
+    extraEnv: {},
+    rawConfig: {},
+  }, ['build:*'])
+
+  const { default: outputsA } = await import(path.resolve('output-a.json'))
+  const { default: outputsB } = await import(path.resolve('output-b.json'))
+
+  expect(Math.max(outputsA[0], outputsB[0]) < Math.min(outputsA[outputsA.length - 1], outputsB[outputsB.length - 1])).toBeTruthy()
+})
+
+test('pnpm run with multiple script selector should work sequentially with --workspace-concurrency=1', async () => {
+  prepare({
+    scripts: {
+      'build:a': 'node -e "let i = 2;setInterval(() => {if (!i--) process.exit(0); require(\'json-append\').append(Date.now(),\'./output-a.json\');},16)"',
+      'build:b': 'node -e "let i = 2;setInterval(() => {if (!i--) process.exit(0); require(\'json-append\').append(Date.now(),\'./output-b.json\');},16)"',
+    },
+  })
+
+  await execa('pnpm', ['add', 'json-append@1'])
+
+  await run.handler({
+    dir: process.cwd(),
+    extraBinPaths: [],
+    extraEnv: {},
+    rawConfig: {},
+    workspaceConcurrency: 1,
+  }, ['build:*'])
+
+  const { default: outputsA } = await import(path.resolve('output-a.json'))
+  const { default: outputsB } = await import(path.resolve('output-b.json'))
+
+  expect(outputsA[0] < outputsB[0] && outputsA[1] < outputsB[1]).toBeTruthy()
+})

exec/plugin-commands-script-runners/test/runRecursive.ts

@@ -1,3 +1,4 @@
+import { promises as fs } from 'fs'
 import path from 'path'
 import { preparePackages } from '@pnpm/prepare'
 import { run } from '@pnpm/plugin-commands-script-runners'
@@ -890,3 +891,64 @@ test('`pnpm -r --resume-from run` should executed from given package', async ()
   expect(output1).toContain('project-2')
   expect(output1).toContain('project-3')
 })
+
+test('pnpm run with multiple script selector should work on recursive', async () => {
+  preparePackages([
+    {
+      name: 'project-1',
+      version: '1.0.0',
+      scripts: {
+        'build:a': 'node -e "require(\'fs\').writeFileSync(\'../output-1-a.txt\', \'1-a\', \'utf8\')"',
+        'build:b': 'node -e "require(\'fs\').writeFileSync(\'../output-1-b.txt\', \'1-b\', \'utf8\')"',
+        'build:c': 'node -e "require(\'fs\').writeFileSync(\'../output-1-c.txt\', \'1-c\', \'utf8\')"',
+      },
+    },
+    {
+      name: 'project-2',
+      version: '1.0.0',
+      scripts: {
+        'build:a': 'node -e "require(\'fs\').writeFileSync(\'../output-2-a.txt\', \'2-a\', \'utf8\')"',
+        'build:b': 'node -e "require(\'fs\').writeFileSync(\'../output-2-b.txt\', \'2-b\', \'utf8\')"',
+        'build:c': 'node -e "require(\'fs\').writeFileSync(\'../output-2-c.txt\', \'2-c\', \'utf8\')"',
+      },
+    },
+    {
+      name: 'project-3',
+      version: '1.0.0',
+      scripts: {
+        'build:a': 'node -e "require(\'fs\').writeFileSync(\'../output-3-a.txt\', \'3-a\', \'utf8\')"',
+        'build:b': 'node -e "require(\'fs\').writeFileSync(\'../output-3-b.txt\', \'3-b\', \'utf8\')"',
+        'build:c': 'node -e "require(\'fs\').writeFileSync(\'../output-3-c.txt\', \'3-c\', \'utf8\')"',
+      },
+    },
+  ])
+
+  await execa(pnpmBin, [
+    'install',
+    '-r',
+    '--registry',
+    REGISTRY_URL,
+    '--store-dir',
+    path.resolve(DEFAULT_OPTS.storeDir),
+  ])
+  await run.handler({
+    ...DEFAULT_OPTS,
+    ...await readProjects(process.cwd(), [{ namePattern: '*' }]),
+    dir: process.cwd(),
+    recursive: true,
+    rootProjectManifest: {
+      name: 'test-workspaces',
+      private: true,
+    },
+    workspaceDir: process.cwd(),
+  }, ['build:*'])
+  expect(await fs.readFile('output-1-a.txt', { encoding: 'utf-8' })).toEqual('1-a')
+  expect(await fs.readFile('output-1-b.txt', { encoding: 'utf-8' })).toEqual('1-b')
+  expect(await fs.readFile('output-1-c.txt', { encoding: 'utf-8' })).toEqual('1-c')
+  expect(await fs.readFile('output-2-a.txt', { encoding: 'utf-8' })).toEqual('2-a')
+  expect(await fs.readFile('output-2-b.txt', { encoding: 'utf-8' })).toEqual('2-b')
+  expect(await fs.readFile('output-2-c.txt', { encoding: 'utf-8' })).toEqual('2-c')
+  expect(await fs.readFile('output-3-a.txt', { encoding: 'utf-8' })).toEqual('3-a')
+  expect(await fs.readFile('output-3-b.txt', { encoding: 'utf-8' })).toEqual('3-b')
+  expect(await fs.readFile('output-3-c.txt', { encoding: 'utf-8' })).toEqual('3-c')
+})