Merge pull request #153 from PHPCSStandards/feature/add-initial-integ…

…ration-tests Add initial integration test setup and first few tests
PHPCSStandards · Mar 5, 2022 · 8ab9bb5 · 8ab9bb5
2 parents 5227b24 + 97ba038
commit 8ab9bb5
Show file tree

Hide file tree

Showing 20 changed files with 2,076 additions and 199 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -6,4 +6,8 @@
 .github                   export-ignore
 
 CONTRIBUTING.md           export-ignore
-phpcs.xml.dist            export-ignore
+phpcs.xml.dist            export-ignore
+phpunit.xml.dist          export-ignore
+
+# Ignore directories for distribution archives.
+/tests/                   export-ignore
diff --git a/.github/workflows/integrationtest.yml b/.github/workflows/integrationtest.yml
@@ -18,126 +18,32 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: "${{ matrix.os }}"
 
     strategy:
       matrix:
-        php: ['5.4', '5.5', '5.6', '7.0', '7.1', '7.2', '7.3', '7.4', '8.0', '8.1']
-        phpcs_version: ['dev-master']
-        phpcompat: ['composer']
-        experimental: [false]
-
-        include:
-          # Ensure a "highest" PHP/PHPCS build combination for PHPCS 2.x is included.
-          - php: '5.4'
-            phpcs_version: '2.9.2'
-            phpcompat: 'composer'
-            experimental: false
-
-          # Complement the matrix with build against the lowest supported PHPCS version
-          # for each PHP version.
-          - php: '8.1'
-            # Lowest PHPCS version on which PHP 8.1 is supported.
-            phpcs_version: '3.6.1'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '8.0'
-            # Lowest PHPCS version on which PHP 8.0 is supported.
-            phpcs_version: '3.5.7'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.4'
-            # Lowest PHPCS version on which PHP 7.4 is supported.
-            phpcs_version: '3.5.0'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.3'
-            # Lowest PHPCS version on which PHP 7.3 is supported.
-            phpcs_version: '3.3.1'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.2'
-            # Lowest PHPCS version on which PHP 7.2 is supported.
-            phpcs_version: '2.9.2'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.1'
-            phpcs_version: '2.0.0'
-            phpcompat: '^7.0'
-            experimental: false
-          - php: '7.0'
-            phpcs_version: '2.0.0'
-            phpcompat: '^7.0'
-            experimental: false
-          - php: '5.6'
-            phpcs_version: '2.0.0'
-            phpcompat: '^7.0'
-            experimental: false
-          - php: '5.5'
-            phpcs_version: '2.0.0'
-            phpcompat: '^7.0'
-            experimental: false
-          - php: '5.4'
-            phpcs_version: '2.0.0'
-            phpcompat: '^7.0'
-            experimental: false
-
-          # Additional builds against arbitrary interim PHPCS versions.
-          - php: '7.3'
-            phpcs_version: '3.5.3'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.2'
-            phpcs_version: '3.2.3'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.1'
-            phpcs_version: '3.1.1'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.0'
-            phpcs_version: '3.4.2'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '7.0'
-            phpcs_version: '2.2.0'
-            phpcompat: '^8.0'
-            experimental: false
-          - php: '5.6'
-            phpcs_version: '3.0.2'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '5.6'
-            phpcs_version: '2.4.0'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '5.5'
-            phpcs_version: '2.6.1'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '5.4'
-            phpcs_version: '3.5.3'
-            phpcompat: 'composer'
-            experimental: false
-          - php: '5.4'
-            phpcs_version: '2.8.1'
-            phpcompat: 'composer'
-            experimental: false
-
-          # Experimental builds. These are allowed to fail.
-          - php: '8.2'
-            phpcs_version: 'dev-master'
-            phpcompat: 'composer'
-            experimental: true
-
-          - php: '8.1'
-            phpcs_version: '4.0.x-dev as 3.9.99'
-            phpcompat: 'composer'
-            experimental: true
-
-    name: "Integration test: PHP ${{ matrix.php }} - PHPCS ${{ matrix.phpcs_version }}"
-
-    continue-on-error: ${{ matrix.experimental }}
+        php:
+          - '5.4'
+          - '5.5'
+          - '5.6'
+          - '7.0'
+          - '7.1'
+          - '7.2'
+          - '7.3'
+          - '7.4'
+          - '8.0'
+          - '8.1'
+          - '8.2'
+        composer:
+          - 'v1'
+          - 'v2'
+        os:
+          - 'ubuntu-latest'
+          - 'windows-latest'
+
+    name: "Integration test"
+
+    continue-on-error: ${{ matrix.php == '8.2' }}
 
     steps:
       - name: Checkout code
@@ -147,41 +53,27 @@ jobs:
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ matrix.php }}
-          ini-values: error_reporting=-1, display_errors=On
+          ini-values: zend.assertions=1, error_reporting=-1, display_errors=On
+          tools: "composer:${{ matrix.composer }}"
           coverage: none
 
-      - name: 'Composer: set PHPCS version for tests'
-        run: composer require --no-update --no-scripts squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-interaction
-
-      # Install PHPCompatibility 7.x/8.x for PHPCS < 2.3.
-      - name: 'Composer: set PHPCompatibility version for tests (PHPCS < 2.3)'
-        if: ${{ matrix.phpcompat != 'composer' }}
-        run: composer require --dev --no-update --no-scripts phpcompatibility/php-compatibility:"${{ matrix.phpcompat }}" --no-interaction
+      - name: "Conditionally disable tls (Composer 1.x/Windows/PHP 5.4)"
+        if: ${{ matrix.os == 'windows-latest' && matrix.composer == 'v1' && matrix.php == '5.4' }}
+        run: composer config -- disable-tls true
 
       # Install dependencies and handle caching in one go.
       # @link https://github.com/marketplace/actions/install-composer-dependencies
-      - name: 'Install Composer dependencies'
+      - name: Install Composer dependencies
+        if: ${{ matrix.php != '8.2' }}
         uses: "ramsey/composer-install@v2"
         with:
-          composer-options: --no-scripts --optimize-autoloader
-
-      # Rename the PHPCompatibility directory as PHPCompatibility 7.x wasn't fully compatible with Composer yet.
-      - name: 'Rename the PHPCompatibility directory (PHPCS < 2.2)'
-        if: ${{ matrix.phpcompat == '^7.0' }}
-        run: mv ./vendor/phpcompatibility/php-compatibility ./vendor/phpcompatibility/PHPCompatibility
-
-      - name: 'Install standards'
-        run: composer install-codestandards
+          composer-options: '--optimize-autoloader'
 
-      - name: 'Show installed standards'
-        run: vendor/bin/phpcs -i
-
-      # Test that an external standard has been registered correctly by running it against the codebase on PHPCS < 2.3.
-      - name: 'Test the PHPCompatibility standard was installed succesfully (PHPCS < 2.3)'
-        if: ${{ matrix.phpcompat != 'composer' }}
-        run: ./vendor/bin/phpcs -ps ./src/ --standard=PHPCompatibility --sniffs=PHPCompatibility.PHP.DeprecatedFunctions --runtime-set testVersion ${{ matrix.php }}
+      - name: Install Composer dependencies
+        if: ${{ matrix.php == '8.2' }}
+        uses: "ramsey/composer-install@v2"
+        with:
+          composer-options: '--ignore-platform-reqs --optimize-autoloader'
 
-      # Test that an external standard has been registered correctly by running it against the codebase.
-      - name: 'Test the PHPCompatibility standard was installed succesfully (PHPCS >= 2.3)'
-        if: ${{ matrix.phpcompat == 'composer' }}
-        run: ./vendor/bin/phpcs -ps ./src/ --standard=PHPCompatibility --sniffs=PHPCompatibility.FunctionUse.RemovedFunctions --runtime-set testVersion ${{ matrix.php }}
+      - name: Run integration tests
+        run: vendor/bin/phpunit --no-coverage
diff --git a/.github/workflows/quicktest.yml b/.github/workflows/quicktest.yml
@@ -20,28 +20,22 @@ jobs:
   # This is a much quicker test which only runs the integration tests against a limited set of
   # supported PHP/PHPCS combinations.
   quicktest:
-    runs-on: ubuntu-latest
+    runs-on: "${{ matrix.os }}"
 
     strategy:
       matrix:
-        include:
-          - php: 'latest'
-            phpcs_version: 'dev-master'
-            phpcompat: 'composer'
-          - php: '7.3'
-            phpcs_version: '2.9.2'
-            phpcompat: 'composer'
-          - php: '7.1'
-            phpcs_version: '3.3.1'
-            phpcompat: 'composer'
-          - php: '5.6'
-            phpcs_version: '2.6.0'
-            phpcompat: 'composer'
-          - php: '5.4'
-            phpcs_version: '2.0.0'
-            phpcompat: '^7.0'
+        php:
+          - '5.4'
+          - '7.2'
+          - 'latest'
+        composer:
+          - 'v1'
+          - 'v2'
+        os:
+          - 'ubuntu-latest'
+          - 'windows-latest'
 
-    name: "Quick test: PHP ${{ matrix.php }} - PHPCS ${{ matrix.phpcs_version }}"
+    name: "Quick test"
 
     steps:
       - name: Checkout code
@@ -51,41 +45,20 @@ jobs:
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ matrix.php }}
-          ini-values: error_reporting=-1, display_errors=On
+          ini-values: zend.assertions=1, error_reporting=-1, display_errors=On
+          tools: "composer:${{ matrix.composer }}"
           coverage: none
 
-      - name: 'Composer: set PHPCS version for tests'
-        run: composer require --no-update --no-scripts squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-interaction
-
-      # Install PHPCompatibility 7.x/8.x for PHPCS < 2.3.
-      - name: 'Composer: set PHPCompatibility version for tests (PHPCS < 2.3)'
-        if: ${{ matrix.phpcompat != 'composer' }}
-        run: composer require --dev --no-update --no-scripts phpcompatibility/php-compatibility:"${{ matrix.phpcompat }}" --no-interaction
+      - name: "Conditionally disable tls (Composer 1.x/Windows/PHP 5.4)"
+        if: ${{ matrix.os == 'windows-latest' && matrix.composer == 'v1' && matrix.php == '5.4' }}
+        run: composer config -- disable-tls true
 
       # Install dependencies and handle caching in one go.
       # @link https://github.com/marketplace/actions/install-composer-dependencies
-      - name: 'Install Composer dependencies'
+      - name: Install Composer dependencies
         uses: "ramsey/composer-install@v2"
         with:
-          composer-options: --no-scripts --optimize-autoloader
-
-      # Rename the PHPCompatibility directory as PHPCompatibility 7.x wasn't fully compatible with Composer yet.
-      - name: 'Rename the PHPCompatibility directory (PHPCS < 2.2)'
-        if: ${{ matrix.phpcompat == '^7.0' }}
-        run: mv ./vendor/phpcompatibility/php-compatibility ./vendor/phpcompatibility/PHPCompatibility
-
-      - name: 'Install standards'
-        run: composer install-codestandards
-
-      - name: 'Show installed standards'
-        run: vendor/bin/phpcs -i
-
-      # Test that an external standard has been registered correctly by running it against the codebase on PHPCS < 2.3.
-      - name: 'Test the PHPCompatibility standard was installed succesfully (PHPCS < 2.3)'
-        if: ${{ matrix.phpcompat != 'composer' }}
-        run: ./vendor/bin/phpcs -ps ./src/ --standard=PHPCompatibility --sniffs=PHPCompatibility.PHP.DeprecatedFunctions --runtime-set testVersion ${{ matrix.php }}
+          composer-options: '--optimize-autoloader'
 
-      # Test that an external standard has been registered correctly by running it against the codebase.
-      - name: 'Test the PHPCompatibility standard was installed succesfully (PHPCS >= 2.3)'
-        if: ${{ matrix.phpcompat == 'composer' }}
-        run: ./vendor/bin/phpcs -ps ./src/ --standard=PHPCompatibility --sniffs=PHPCompatibility.FunctionUse.RemovedFunctions --runtime-set testVersion ${{ matrix.php }}
+      - name: Run integration tests
+        run: vendor/bin/phpunit --no-coverage
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,7 @@ vendor/
 composer.lock
 .phpcs.xml
 phpcs.xml
+.phpunit.result.cache
+phpunit.xml
+/build/
+/tests/artifact/*.zip
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -23,7 +23,9 @@ Even better: You could submit a pull request with a fix / new feature!
 1. Search our repository for open or closed [pull requests][prs] that relate
    to your submission. You don't want to duplicate effort.
 
-2. You may merge the pull request in once you have the sign-off of two other
+2. All pull requests are expected to be accompanied by tests which cover the change.
+
+3. You may merge the pull request in once you have the sign-off of two other
    developers, or if you do not have permission to do that, you may request
    the second reviewer to merge it for you.
 
@@ -39,6 +41,7 @@ These tools fall into two categories: PHP and non-PHP.
 
 The PHP specific tools used by this build are:
 
+- [PHPUnit][] and the [PHPUnit Polyfills][] for the integration tests.
 - [PHP_CodeSniffer][] to verify PHP code complies with the [PSR12][] standard.
 - [PHPCompatibility][] to verify that code is written in a PHP cross-version compatible manner.
 - [PHP-Parallel-Lint][] to check against parse errors in PHP files.
@@ -57,13 +60,48 @@ can be downloaded suitable for your operating system from their [releases page][
 Alternatively, these tools can be run using `docker run`, through the Docker
 images provided by [Pipeline-Component][].
 
+[PHPUnit]: https://phpunit.de/
+[PHPUnit Polyfills]: https://github.com/Yoast/PHPUnit-Polyfills/
 [PHP_CodeSniffer]: https://github.com/squizlabs/PHP_CodeSniffer
 [PHPCompatibility]: https://github.com/PHPCompatibility/PHPCompatibility
 [PHP-Parallel-Lint]: https://github.com/php-parallel-lint/PHP-Parallel-Lint
 [PHP-Security-Checker]: https://github.com/fabpot/local-php-security-checker
 [PSR12]: https://www.php-fig.org/psr/psr-12/
 [releases page]: https://github.com/fabpot/local-php-security-checker/releases/
 
+#### Automated testing
+
+This package includes a test setup for automated testing on all supported PHP versions
+using [PHPUnit][] with the [PHPUnit Polyfills][].
+This means that tests can be written for the latest version of PHPUnit
+(9.x at the time of writing) and still be run on all PHPUnit versions needed to test
+all supported PHP versions (PHPUnit 4.x - 9.x).
+
+The tests can be run both via a Composer installed version of PHPUnit, as well as using
+a PHPUnit PHAR file, however, whichever way you run the tests, you will always need to
+make sure that `composer install` has been run on the repository to make sure the
+PHPUnit Polyfills are available.
+
+**Note**: _as these tests run Composer and other CLI commands they will be slow to run._
+
+To run the tests locally:
+1. Run `composer install`
+2. Run the tests either using a PHPUnit PHAR file or by calling `composer test`.
+
+In case the test setup has trouble locating your `composer.phar` file:
+
+1. Copy the `phpunit.xml.dist` file to `phpunit.xml`.
+
+2. Edit the `phpunit.xml` file and add the following, replacing the value with the applicable path to Composer for your local machine:
+    ```xml
+    <php>
+        <env name="COMPOSER_PHAR" value="path/to/composer.phar"/>
+    </php>
+    ```
+    **Note**: this setting also allows for locally testing with different versions of Composer.
+    You could, for instance, have multiple Composer Phar files locally, `composer1.phar`, `composer2.1.phar`, `composer2.2.phar`.
+    By changing the path in the value of this `env` setting, you can switch which version will be used in the tests.
+
 ### Non-PHP
 
 The non-PHP specific tools used by this build are: