fix(ci): pin importlib_metadata<5.0.0 (#4257) (#4287)

CI jobs `profile-windows-*`, `kombu` and `celery` are failing due to the release of `importlib_metadata==5.0.0` which removed deprecated shims, resulting in the following exception: ``` AttributeError: 'EntryPoints' object has no attribute 'get' ``` We fix this by pinning the dependency for kombu and celery. In the case of the Windows tests, we address it by upgrading the Windows image to a version that includes Python 3.10 rather than Python 3.7 which removes the need to install the dependency when installing tox. - [Example log of failing job](https://app.circleci.com/pipelines/github/DataDog/dd-trace-py/21679/workflows/2a7606b5-9ca8-4212-9b02-053665de0198/jobs/1477147) - python/importlib_metadata#409 (cherry picked from commit c2b416e) Co-authored-by: Tahir H. Butt <tahir.butt@datadoghq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
DataDog · Oct 12, 2022 · 6a3a7a8 · 6a3a7a8
1 parent 8d87692
commit 6a3a7a8
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 8 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -17,7 +17,7 @@ vertica_image: &vertica_image sumitchawla/vertica:latest
 rabbitmq_image: &rabbitmq_image rabbitmq:3.7-alpine
 
 orbs:
-  win: circleci/windows@4.1
+  win: circleci/windows@5.0
 
 machine_executor: &machine_executor
   machine:
@@ -438,7 +438,7 @@ jobs:
       name: win/default
       shell: bash.exe
     steps:
-      - run: choco install python --version=3.5.4
+      - run: choco install -y python --version=3.5.4 --side-by-side
       - run_tox_scenario:
           store_coverage: false
           pattern: '^py35-profile'
@@ -448,7 +448,7 @@ jobs:
       name: win/default
       shell: bash.exe
     steps:
-      - run: choco install python --version=3.6.8
+      - run: choco install -y python --version=3.6.8 --side-by-side
       - run_tox_scenario:
           store_coverage: false
           pattern: '^py36-profile'
@@ -469,7 +469,7 @@ jobs:
       name: win/default
       shell: bash.exe
     steps:
-      - run: choco install python --version=3.8.10
+      - run: choco install -y python --version=3.8.10 --side-by-side
       - run_tox_scenario:
           store_coverage: false
           pattern: '^py38-profile'
@@ -479,7 +479,7 @@ jobs:
       name: win/default
       shell: bash.exe
     steps:
-      - run: choco install python --version=3.9.12
+      - run: choco install -y python --version=3.9.12 --side-by-side
       - run_tox_scenario:
           store_coverage: false
           pattern: '^py39-profile'
@@ -489,7 +489,7 @@ jobs:
       name: win/default
       shell: bash.exe
     steps:
-      - run: choco install python --version=3.10.4
+      # circleci/windows@5.0 orb includes python 3.10.6
       - run_tox_scenario:
           store_coverage: false
           pattern: '^py310-profile'

diff --git a/riotfile.py b/riotfile.py
@@ -409,6 +409,7 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                         ],
                         "redis": "~=2.10.6",
                         "kombu": "~=4.3.0",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib-metadata>=5.0
                     },
                 ),
                 # Celery 4.2 is now limited to Kombu 4.3
@@ -420,11 +421,27 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                         "celery": "~=4.2.2",
                         "redis": "~=2.10.6",
                         "kombu": "~=4.3.0",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib_metadata 5.0
                     },
                 ),
                 # Celery 4.3 wants Kombu >= 4.4 and Redis >= 3.2
+                # Split into <3.8 and >=3.8 to pin importlib_metadata dependency for kombu
                 Venv(
-                    pys=select_pys(max_version="3.9"),
+                    pys=select_pys(max_version="3.7"),
+                    pkgs={
+                        "pytest": "~=3.10",
+                        "celery": [
+                            "~=4.3.1",
+                            "~=4.4.7",
+                            "~=4.4",  # most recent 4.x
+                        ],
+                        "redis": "~=3.5",
+                        "kombu": "~=4.4",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib_metadata 5.0
+                    },
+                ),
+                Venv(
+                    pys=select_pys(min_version="3.8", max_version="3.9"),
                     pkgs={
                         "pytest": "~=3.10",
                         "celery": [
@@ -437,8 +454,25 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                     },
                 ),
                 # Celery 5.x wants Python 3.6+
+                # Split into <3.8 and >=3.8 to pin importlib_metadata dependency for kombu
                 Venv(
-                    pys=select_pys(min_version="3.6"),
+                    pys=select_pys(min_version="3.6", max_version="3.7"),
+                    env={
+                        # https://docs.celeryproject.org/en/v5.0.5/userguide/testing.html#enabling
+                        "PYTEST_PLUGINS": "celery.contrib.pytest",
+                    },
+                    pkgs={
+                        "celery": [
+                            "~=5.0.5",
+                            "~=5.0",  # most recent 5.x
+                            latest,
+                        ],
+                        "redis": "~=3.5",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib_metadata 5.0
+                    },
+                ),
+                Venv(
+                    pys=select_pys(min_version="3.8"),
                     env={
                         # https://docs.celeryproject.org/en/v5.0.5/userguide/testing.html#enabling
                         "PYTEST_PLUGINS": "celery.contrib.pytest",

diff --git a/tox.ini b/tox.ini
@@ -161,6 +161,8 @@ deps =
     gevent2012: gevent>=20.12,<20.13
     gevent211: gevent>=21.1,<21.2
     gevent218: gevent>=21.8,<21.9
+    # kombu using deprecated shims removed in importlib-metadata 5.0
+    kombu{40,41,42,43,44,45,46,}: importlib_metadata<5.0; python_version<'3.8'
     kombu: kombu
     kombu40: kombu>=4.0,<4.1
     kombu41: kombu>=4.1,<4.2