feat(compiler): allow ignore pattern for copy task (#5899)

* feat: allow ignore pattern for copy task

* add tests to pipeline

* fix code docs

* remove unnecessary files

* add workflow to pipeline

* add missing script

* fix end-to-end tests

* prettier

* revert some changes

* set absolute flag

* use rimraf for support in Windows

* chore(deps): install Jest dependencies via TypeScript and Node

* fix task for windows

* make it relative to source path

* tweak

Author: christian-bromann

.github/workflows/main.yml

@@ -38,6 +38,11 @@ jobs:
     needs: [build_core]
     uses: ./.github/workflows/test-bundlers.yml
 
+  copytask_tests:
+    name: Copy Task Tests
+    needs: [build_core]
+    uses: ./.github/workflows/test-copytask.yml
+
   component_starter_tests:
     name: Component Starter Smoke Test
     needs: [build_core]

.github/workflows/test-copytask.yml

@@ -0,0 +1,31 @@
+name: Copy Task Tests
+
+on:
+  workflow_call:
+    # Make this a reusable workflow, no value needed
+    # https://docs.github.com/en/actions/using-workflows/reusing-workflows
+
+jobs:
+  bundler_tests:
+    name: Verify Copy Task
+    runs-on: 'ubuntu-22.04'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+
+      - name: Get Core Dependencies
+        uses: ./.github/workflows/actions/get-core-dependencies
+
+      - name: Download Build Archive
+        uses: ./.github/workflows/actions/download-archive
+        with:
+          name: stencil-core
+          path: .
+          filename: stencil-core-build.zip
+
+      - name: Bundler Tests
+        run: npm run test.copytask
+        shell: bash
+
+      - name: Check Git Context
+        uses: ./.github/workflows/actions/check-git-context

package-lock.json

@@ -96,9 +96,9 @@
     "build": "npm run clean && npm run tsc.prod && npm run ts scripts/index.ts -- --prod --ci",
     "build.watch": "npm run build -- --watch",
     "build.updateSelectorEngine": "npm run ts scripts/updateSelectorEngine.ts",
-    "clean": "rm -rf build/ cli/ compiler/ dev-server/ internal/ mock-doc/ sys/ testing/ && npm run clean:scripts && npm run clean.screenshots",
-    "clean.screenshots": "rm -rf test/end-to-end/screenshot/builds test/end-to-end/screenshot/images",
-    "clean:scripts": "rm -rf scripts/build",
+    "clean": "rimraf build/ cli/ compiler/ dev-server/ internal/ mock-doc/ sys/ testing/ && npm run clean:scripts && npm run clean.screenshots",
+    "clean.screenshots": "rimraf test/end-to-end/screenshot/builds test/end-to-end/screenshot/images",
+    "clean:scripts": "rimraf scripts/build",
     "lint": "eslint 'bin/*' 'scripts/*.ts' 'scripts/**/*.ts' 'src/*.ts' 'src/**/*.ts' 'src/**/*.tsx' 'test/wdio/**/*.tsx'",
     "install.jest": "npx tsx ./src/testing/jest/install-dependencies.mts",
     "prettier": "npm run prettier.base -- --write",
@@ -112,6 +112,7 @@
     "test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js --coverage",
     "test.analysis": "cd test && npm run analysis.build-and-analyze",
     "test.bundlers": "cd test && npm run bundlers",
+    "test.copytask": "cd test/copy-task && npm ci && npm run test",
     "test.dist": "npm run ts scripts/index.ts -- --validate-build",
     "test.end-to-end": "cd test/end-to-end && npm ci && npm test && npm run test.dist",
     "test.jest": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js",
@@ -184,6 +185,7 @@
     "prettier": "3.3.1",
     "prompts": "2.4.2",
     "puppeteer": "^21.0.0",
+    "rimraf": "^6.0.1",
     "rollup": "2.56.3",
     "semver": "^7.3.7",
     "terser": "5.31.1",

package.json

@@ -27,6 +27,7 @@ export const getComponentAssetsCopyTasks = (
             src: assetsMeta.absolutePath,
             dest: join(dest, assetsMeta.cmpRelativePath),
             warn: false,
+            ignore: undefined,
             keepDirStructure: false,
           });
         });
@@ -37,6 +38,7 @@ export const getComponentAssetsCopyTasks = (
             src: assetsMeta.absolutePath,
             dest: collectionDirDestination,
             warn: false,
+            ignore: undefined,
             keepDirStructure: false,
           });
         });

src/compiler/output-targets/copy/assets-copy-tasks.ts

@@ -5,6 +5,17 @@ import type * as d from '../../../declarations';
 import { canSkipAssetsCopy, getComponentAssetsCopyTasks } from './assets-copy-tasks';
 import { getDestAbsPath, getSrcAbsPath } from './local-copy-tasks';
 
+const DEFAULT_IGNORE = [
+  '**/__mocks__/**',
+  '**/__fixtures__/**',
+  '**/dist/**',
+  '**/.{idea,git,cache,output,temp}/**',
+  '**/.ds_store',
+  '**/.gitignore',
+  '**/desktop.ini',
+  '**/thumbs.db',
+];
+
 export const outputCopy = async (config: d.ValidatedConfig, compilerCtx: d.CompilerCtx, buildCtx: d.BuildCtx) => {
   const outputTargets = config.outputTargets.filter(isOutputTargetCopy);
   if (outputTargets.length === 0) {
@@ -83,6 +94,7 @@ const transformToAbs = (copyTask: d.CopyTask, dest: string): Required<d.CopyTask
   return {
     src: copyTask.src,
     dest: getDestAbsPath(copyTask.src, dest, copyTask.dest),
+    ignore: copyTask.ignore || DEFAULT_IGNORE,
     keepDirStructure:
       typeof copyTask.keepDirStructure === 'boolean' ? copyTask.keepDirStructure : copyTask.dest == null,
     warn: copyTask.warn !== false,

src/compiler/output-targets/copy/output-copy.ts

@@ -1647,6 +1647,11 @@ export interface CopyTask {
    * the output target for which this copy operation is configured.
    */
   dest?: string;
+  /**
+   * An optional array of glob patterns to exclude from the copy operation.
+   * @default ['**\/__mocks__/**', '**\/__fixtures__/**', '**\/dist/**', '**\/.{idea,git,cache,output,temp}/**', '**\/.ds_store', '**\/.gitignore', '**\/desktop.ini', '**\/thumbs.db']
+   */
+  ignore?: string[];
   /**
    * Whether or not Stencil should issue warnings if it cannot find the
    * specified source files or directories. Defaults to `false`.

src/declarations/stencil-public-compiler.ts

@@ -1,5 +1,5 @@
 import { buildError, catchError, flatOne, isGlob, normalizePath } from '@utils';
-import { glob } from 'glob';
+import { glob, type GlobOptions } from 'glob';
 import path from 'path';
 
 import type * as d from '../../declarations';
@@ -13,7 +13,7 @@ export async function nodeCopyTasks(copyTasks: Required<d.CopyTask>[], srcDir: s
   };
 
   try {
-    copyTasks = flatOne(await Promise.all(copyTasks.map((task) => processGlobs(task, srcDir))));
+    copyTasks = flatOne(await Promise.all(copyTasks.map((task) => processGlobTask(task, srcDir))));
 
     const allCopyTasks: d.CopyTask[] = [];
 
@@ -44,30 +44,22 @@ export async function nodeCopyTasks(copyTasks: Required<d.CopyTask>[], srcDir: s
   return results;
 }
 
-async function processGlobs(copyTask: Required<d.CopyTask>, srcDir: string): Promise<Required<d.CopyTask>[]> {
-  return isGlob(copyTask.src)
-    ? await processGlobTask(copyTask, srcDir)
-    : [
-        {
-          src: getSrcAbsPath(srcDir, copyTask.src),
-          dest: copyTask.keepDirStructure ? path.join(copyTask.dest, copyTask.src) : copyTask.dest,
-          warn: copyTask.warn,
-          keepDirStructure: copyTask.keepDirStructure,
-        },
-      ];
-}
-
-function getSrcAbsPath(srcDir: string, src: string) {
-  if (path.isAbsolute(src)) {
-    return src;
-  }
-  return path.join(srcDir, src);
-}
-
 async function processGlobTask(copyTask: Required<d.CopyTask>, srcDir: string): Promise<Required<d.CopyTask>[]> {
-  const files = await asyncGlob(copyTask.src, {
+  /**
+   * To properly match all files within a certain directory we have to ensure to attach a `/**` to
+   * the end of the pattern. However we only want to do this if the `src` entry is not a glob pattern
+   * already or a file with an extension.
+   */
+  const pattern =
+    isGlob(copyTask.src) || path.extname(copyTask.src).length > 0
+      ? copyTask.src
+      : './' + path.relative(srcDir, path.join(path.resolve(srcDir, copyTask.src), '**')).replaceAll(path.sep, '/');
+
+  const files = await asyncGlob(pattern, {
     cwd: srcDir,
     nodir: true,
+    absolute: false,
+    ignore: copyTask.ignore,
   });
   return files.map((globRelPath) => createGlobCopyTask(copyTask, srcDir, globRelPath));
 }
@@ -77,6 +69,7 @@ function createGlobCopyTask(copyTask: Required<d.CopyTask>, srcDir: string, glob
   return {
     src: path.join(srcDir, globRelPath),
     dest,
+    ignore: copyTask.ignore,
     warn: copyTask.warn,
     keepDirStructure: copyTask.keepDirStructure,
   };
@@ -96,7 +89,7 @@ async function processCopyTask(results: d.CopyResults, allCopyTasks: d.CopyTask[
       }
 
       await processCopyTaskDirectory(results, allCopyTasks, copyTask);
-    } else if (!shouldIgnore(copyTask.src)) {
+    } else {
       // this is a file we should copy
       if (!results.filePaths.includes(copyTask.dest)) {
         results.filePaths.push(copyTask.dest);
@@ -169,13 +162,6 @@ function addMkDir(mkDirs: string[], destDir: string) {
 
 const ROOT_DIR = normalizePath(path.resolve('/'));
 
-function shouldIgnore(filePath: string) {
-  filePath = filePath.trim().toLowerCase();
-  return IGNORE.some((ignoreFile) => filePath.endsWith(ignoreFile));
-}
-
-const IGNORE = ['.ds_store', '.gitignore', 'desktop.ini', 'thumbs.db'];
-
-export function asyncGlob(pattern: string, opts: any) {
-  return glob(pattern, opts);
+export async function asyncGlob(pattern: string, opts: GlobOptions): Promise<string[]> {
+  return glob(pattern, { ...opts, withFileTypes: false });
 }

src/sys/node/node-copy-tasks.ts

@@ -0,0 +1,25 @@
+Copy Task Tests
+===============
+
+This directory aims to test and validate the behavior for Stencils [Copy Task for Output Targets](https://stenciljs.com/docs/copy-tasks#copy-tasks-for-output-targets). It has a copy task defined in `test/copy-task/stencil.config.ts` and builds this starter projects to then validate if the right files where copies.
+
+## Given
+
+We have a copy task defined as part of an output target, e.g. 
+
+```ts
+{
+  type: 'dist-custom-elements',
+  copy: [{
+    src: './utils',
+    dest: './dist/utilsExtra',
+  }]
+}
+```
+
+I expect that a `utilsExtra` directory is created that does __not__ copy the following entries:
+
+- files in `__fixtures__` and `__mocks__` directories
+- as well as files named `desktop.ini`
+
+Furthermore I expect that no JS files are copied over within the collection directory.

test/copy-task/README.md

@@ -0,0 +1,41 @@
+{
+  "name": "copytask",
+  "version": "0.0.1",
+  "description": "Stencil Component Starter",
+  "main": "dist/index.cjs.js",
+  "module": "dist/index.js",
+  "types": "dist/types/index.d.ts",
+  "collection": "dist/collection/collection-manifest.json",
+  "collection:main": "dist/collection/index.js",
+  "unpkg": "dist/copytask/copytask.esm.js",
+  "exports": {
+    ".": {
+      "import": "./dist/copytask/copytask.esm.js",
+      "require": "./dist/copytask/copytask.cjs.js"
+    },
+    "./my-component": {
+      "import": "./dist/components/my-component.js",
+      "types": "./dist/components/my-component.d.ts"
+    },
+    "./loader": {
+      "import": "./loader/index.js",
+      "require": "./loader/index.cjs",
+      "types": "./loader/index.d.ts"
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ionic-team/stencil-component-starter.git"
+  },
+  "files": [
+    "dist/"
+  ],
+  "scripts": {
+    "build": "rimraf ./dist && node ../../bin/stencil build",
+    "test": "npm run build && tsx validate.mts"
+  },
+  "devDependencies": {
+    "rimraf": "^6.0.1"
+  },
+  "license": "MIT"
+}

test/copy-task/package-lock.json

@@ -0,0 +1,24 @@
+/* eslint-disable */
+/* tslint:disable */
+/**
+ * This is an autogenerated file created by the Stencil compiler.
+ * It contains typing information for all components that exist in this project.
+ */
+import { HTMLStencilElement, JSXBase } from "@stencil/core/internal";
+export namespace Components {
+}
+declare global {
+    interface HTMLElementTagNameMap {
+    }
+}
+declare namespace LocalJSX {
+    interface IntrinsicElements {
+    }
+}
+export { LocalJSX as JSX };
+declare module "@stencil/core" {
+    export namespace JSX {
+        interface IntrinsicElements {
+        }
+    }
+}

test/copy-task/package.json

@@ -0,0 +1 @@
+{}

test/copy-task/src/components.d.ts

@@ -0,0 +1 @@
+export const copyFile = jest.fn().mockImplementation(() => 'JavaScript file content');

test/copy-task/src/utils/__fixtures__/foobar.json

@@ -0,0 +1 @@
+Hello

test/copy-task/src/utils/__mocks__/foo.js

@@ -0,0 +1,19 @@
+import { format } from './utils';
+
+describe('format', () => {
+  it('returns empty string for no names defined', () => {
+    expect(format(undefined, undefined, undefined)).toEqual('');
+  });
+
+  it('formats just first names', () => {
+    expect(format('Joseph', undefined, undefined)).toEqual('Joseph');
+  });
+
+  it('formats first and last names', () => {
+    expect(format('Joseph', undefined, 'Publique')).toEqual('Joseph Publique');
+  });
+
+  it('formats first, middle and last names', () => {
+    expect(format('Joseph', 'Quincy', 'Publique')).toEqual('Joseph Quincy Publique');
+  });
+});

test/copy-task/src/utils/desktop.ini

@@ -0,0 +1,3 @@
+export function format(first: string, middle: string, last: string): string {
+  return (first || '') + (middle ? ` ${middle}` : '') + (last ? ` ${last}` : '');
+}

test/copy-task/src/utils/utils.spec.ts

@@ -0,0 +1,19 @@
+import { Config } from '@stencil/core';
+
+export const config: Config = {
+  namespace: 'copytask',
+  outputTargets: [
+    {
+      type: 'dist-custom-elements',
+      copy: [
+        {
+          src: './utils',
+          dest: './dist/utilsExtra',
+        },
+      ],
+    },
+    {
+      type: 'dist',
+    },
+  ],
+};

test/copy-task/src/utils/utils.ts

@@ -0,0 +1,39 @@
+{
+  "compilerOptions": {
+    "allowSyntheticDefaultImports": true,
+    "allowUnreachableCode": false,
+    "declaration": false,
+    "experimentalDecorators": true,
+    "lib": [
+      "dom",
+      "es2017"
+    ],
+    "moduleResolution": "node",
+    "module": "esnext",
+    "target": "es2017",
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "skipLibCheck": true,
+    "jsx": "react",
+    "jsxFactory": "h",
+    "paths": {
+      "@stencil/core": [
+        "../../internal"
+      ],
+      "@stencil/core/compiler": [
+        "../../compiler"
+      ],
+      "@stencil/core/internal": [
+        "../../internal"
+      ]
+    }
+  },
+  "include": [
+    "src"
+  ],
+  "exclude": [
+    "node_modules",
+    "src/__mocks__"
+  ]
+
+}

test/copy-task/stencil.config.ts

@@ -0,0 +1,20 @@
+import assert from 'node:assert';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import url from 'node:url';
+
+const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
+
+console.log('Running copy-task validate script');
+
+const utilsExtraFiles = await fs.readdir(path.resolve(__dirname, 'dist', 'utilsExtra'));
+assert.equal(
+  JSON.stringify(utilsExtraFiles),
+  JSON.stringify(['utils.spec.ts', 'utils.ts'])
+);
+
+const copiesMockDirIntoCollection = await fs.access(path.resolve(__dirname, 'dist', 'collection', '__mocks__'))
+  .then(() => true, () => false);
+assert(!copiesMockDirIntoCollection);
+
+console.log(`✅ All assertions passed`);

test/copy-task/tsconfig.json

@@ -10,7 +10,7 @@
     "jsx": "react",
     "jsxFactory": "h",
     "jsxFragmentFactory": "Fragment",
-    "lib": ["dom", "es2019"],
+    "lib": ["dom", "es2021"],
     "module": "esnext",
     "moduleResolution": "node",
     "noImplicitAny": true,