Fix ci (#767)

* fix flakiness in publish retry tests not clear how this ever passed. it's unhelpfully tied to the kombu implementation which probably means it's a bad test, but just fixing it up for now * upgrade isort * rename whitelist -> allowlist * upgrade moto and pyjwt * bump requests to resolve conflict with boto * pyjwt requires you to specify algorithm now * upgrade urllib to resolve dependency tree * expected retries isn't flakey; depends on version of kombu * update import following deprecation, removes warning * pytest arg --strict renamed to --strict-markers * import sort * no cover pragmas for kombu version conditionals * pyjwt and moto require older versions on py2 * run docs on ubuntu 20.04; drop 3.6 from matrix * put 3.6 back and try all on 20.04 * drop 3.6 again * sort 5.12.0 not available on py3.7 * pin down jinja to a releaase that is still compatible with our old version of sphinx. it's not worth trying to upgrade sphinx because the new docs use mkdocs * importlib-metadata 5.0 is not compatible with latest kombu on py3.7 importlib-metadata is not on later versions of python, so we can use a conditional pin. see # celery/celery#7783 * upgrade pinned version of eventlet fixes https://stackoverflow.com/questions/75137717/eventlet-dns-python-attribute-error-module-dns-rdtypes-has-no-attribute-any * whoops; fix syntax in setup.py * oldest eventlet on py3.10 needs updating too * match "pinned" requests to the one in dev reqs also choose a version that works on all currently supported pythons * refactor conditional to avoid another pragma * copy/paste snafu * try running tests on ubuntu latest
nameko · Feb 9, 2023 · 03ce391 · 03ce391
1 parent 6c7c959
commit 03ce391
Show file tree

Hide file tree

Showing 12 changed files with 64 additions and 37 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -82,7 +82,6 @@ jobs:
           - 3.9
           - 3.8
           - 3.7
-          - 3.6
           - 2.7
         dependency-set:
           - pinned
@@ -138,7 +137,7 @@ jobs:
         TOXENV: py${{ matrix.python-version }}-${{ matrix.dependency-set }}-${{ matrix.command }}
 
   docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - name: checkout
       uses: actions/checkout@v2

diff --git a/Makefile b/Makefile
@@ -17,14 +17,14 @@ pylint:
 	pylint --rcfile=pylintrc nameko -E
 
 imports:
-	isort -rc $(autofix) nameko test
+	isort $(autofix) nameko test
 
 test_lib:
-	BRANCH=$(ENABLE_BRANCH_COVERAGE) coverage run -m nameko test test -v --strict --timeout 30
+	BRANCH=$(ENABLE_BRANCH_COVERAGE) coverage run -m nameko test test -v --strict-markers --timeout 30
 	BRANCH=$(ENABLE_BRANCH_COVERAGE) coverage report
 
 test_examples:
-	BRANCH=$(ENABLE_BRANCH_COVERAGE) nameko test docs/examples/test --strict --timeout 30 --cov=docs/examples --cov-config=$(CURDIR)/.coveragerc
+	BRANCH=$(ENABLE_BRANCH_COVERAGE) nameko test docs/examples/test --strict-markers --timeout 30 --cov=docs/examples --cov-config=$(CURDIR)/.coveragerc
 	nameko test docs/examples/testing
 
 test_docs: docs spelling #linkcheck

diff --git a/docs/examples/auth.py b/docs/examples/auth.py
@@ -33,7 +33,7 @@ def authenticate(self, username, password):
                 'username': username,
                 'roles': self.users[username]['roles']
             }
-            token = jwt.encode(payload, key=JWT_SECRET).decode('utf-8')
+            token = jwt.encode(payload, key=JWT_SECRET, algorithm="HS256")
             self.worker_ctx.context_data['auth'] = token
             return token
 
@@ -43,7 +43,9 @@ def has_role(self, role):
                 raise Unauthenticated()
 
             try:
-                payload = jwt.decode(token, key=JWT_SECRET, verify=True)
+                payload = jwt.decode(
+                    token, key=JWT_SECRET, verify=True, algorithms=["HS256"]
+                )
                 if role in payload['roles']:
                     return True
             except Exception:

diff --git a/docs/examples/test/test_examples.py b/docs/examples/test/test_examples.py
@@ -368,7 +368,7 @@ def test_authenticate(self, db):
         worker_ctx = Mock(context_data={})
         dep = Auth.Api(db, worker_ctx)
         token = dep.authenticate("matt", "secret")
-        jwt.decode(token, key=JWT_SECRET, verify=True)
+        jwt.decode(token, key=JWT_SECRET, verify=True, algorithms=["HS256"])
         assert worker_ctx.context_data['auth'] == token
 
     def test_authenticate_bad_username(self, db):
@@ -482,7 +482,7 @@ def test_sensitive_arguments(self, container_factory):
 
         with ServiceRpcClient("service") as client:
             token = client.login("matt", "secret")
-            jwt.decode(token, key=JWT_SECRET, verify=True)
+            jwt.decode(token, key=JWT_SECRET, verify=True, algorithms=["HS256"])
             with pytest.raises(RemoteError) as exc:
                 client.login("matt", "incorrect")
             assert exc.value.exc_type == "Unauthenticated"

diff --git a/docs/examples/testing/alternative_dependency_unit_test.py b/docs/examples/testing/alternative_dependency_unit_test.py
@@ -3,8 +3,7 @@
 
 import pytest
 from sqlalchemy import Column, Integer, String, create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import declarative_base, sessionmaker
 
 from nameko.rpc import rpc
 from nameko.testing.services import worker_factory

diff --git a/nameko/cli/__init__.py b/nameko/cli/__init__.py
@@ -3,15 +3,12 @@
 from nameko import config
 from nameko.exceptions import CommandError, ConfigurationError
 
-from .utils import setup_config
 from .click_arguments import argument_services
 from .click_options import (
-    option_broker,
-    option_config_file,
-    option_define,
-    option_backdoor_port,
-    option_interface,
+    option_backdoor_port, option_broker, option_config_file, option_define,
+    option_interface
 )
+from .utils import setup_config
 
 
 # main nameko command
@@ -93,9 +90,10 @@ def test(args):
     import eventlet
     eventlet.monkey_patch()  # noqa (code before rest of imports)
 
-    import pytest
     import sys
 
+    import pytest
+
     args = list(args)
     args.extend(
         ["-W", "ignore:Module already imported:_pytest.warning_types.PytestWarning"]

diff --git a/nameko/cli/utils/__init__.py b/nameko/cli/utils/__init__.py
@@ -1,5 +1,5 @@
 """Importing all relevant utility functions.
 """
-from .config import setup_config  # noqa
 from .code import interact  # noqa
+from .config import setup_config  # noqa
 from .import_services import import_services  # noqa
diff --git a/nameko/rpc.py b/nameko/rpc.py
@@ -17,8 +17,7 @@
 from nameko.constants import (
     AMQP_SSL_CONFIG_KEY, AMQP_URI_CONFIG_KEY, DEFAULT_AMQP_URI,
     DEFAULT_HEARTBEAT, DEFAULT_PREFETCH_COUNT, HEARTBEAT_CONFIG_KEY,
-    LOGIN_METHOD_CONFIG_KEY, PREFETCH_COUNT_CONFIG_KEY,
-    RPC_EXCHANGE_CONFIG_KEY
+    LOGIN_METHOD_CONFIG_KEY, PREFETCH_COUNT_CONFIG_KEY, RPC_EXCHANGE_CONFIG_KEY
 )
 from nameko.exceptions import (
     ContainerBeingKilled, MalformedRequest, MethodNotFound,

diff --git a/nameko/testing/pytest.py b/nameko/testing/pytest.py
@@ -125,7 +125,8 @@ def empty_config():
 @pytest.fixture
 def mock_container(request):
     from mock import create_autospec
-    from nameko.constants import SERIALIZER_CONFIG_KEY, DEFAULT_SERIALIZER
+
+    from nameko.constants import DEFAULT_SERIALIZER, SERIALIZER_CONFIG_KEY
     from nameko.containers import ServiceContainer
 
     container = create_autospec(ServiceContainer)
@@ -151,10 +152,12 @@ def vhost_pipeline(request, rabbit_manager):
     except ImportError:  # pragma: no cover
         # py2 compatibility
         from collections import Iterable  # pylint: disable=E0611
-    from six.moves.urllib.parse import urlparse  # pylint: disable=E0401
     import random
     import string
+
     from kombu.pools import connections
+    from six.moves.urllib.parse import urlparse  # pylint: disable=E0401
+
     from nameko.testing.utils import ResourcePipeline
     from nameko.utils.retry import retry
 
@@ -216,6 +219,7 @@ def rabbit_config(rabbit_uri):
 @pytest.fixture
 def rabbit_ssl_options(request):
     import os
+
     from test import on_travis
 
     ssl_options = request.config.getoption('AMQP_SSL_OPTIONS')
@@ -327,6 +331,7 @@ def make_container(service_cls, config=None):
 @pytest.fixture
 def runner_factory():
     import sys
+
     import nameko
     from nameko.runners import ServiceRunner
 
@@ -369,6 +374,7 @@ def make_runner(*service_classes):
 @pytest.fixture
 def predictable_call_ids(request):
     import itertools
+
     from mock import patch
 
     with patch('nameko.standalone.rpc.uuid') as client_uuid:
@@ -414,6 +420,7 @@ def request(self, method, url, *args, **kwargs):
 @pytest.fixture()
 def websocket(web_config_port):
     import eventlet
+
     from nameko.testing.websocket import make_virtual_socket
 
     active_sockets = []

diff --git a/setup.py b/setup.py
@@ -30,6 +30,7 @@
         "eventlet>=0.33.0 ; python_version>='3.10'",
         "kombu>=4.2.0",
         "kombu>=5.2.0 ; python_version>='3.10'",
+        "importlib-metadata<5 ; python_version<='3.7'",  # https://github.com/celery/celery/issues/7783
         "mock>=1.2",
         "path.py>=6.2",
         "pyyaml>=5.1",
@@ -43,15 +44,16 @@
         'dev': [
             "coverage==5.5",
             "flake8==3.9.2",
-            "isort==4.2.15",
+            "isort==4.3.21 ; python_version<'3'",
+            "isort==5.11.5 ; python_version>'3'",
             "pylint==1.9.5 ; python_version<'3'",
             "pylint==2.11.1 ; python_version>'3'",
             "pytest==4.6.11 ; python_version<'3'",
             "pytest==6.2.5 ; python_version>'3'",
             "pytest-cov==2.5.1",
             "pytest-timeout==1.3.3",
-            "requests==2.19.1",
-            "urllib3==1.23",
+            "requests==2.27.1",
+            "urllib3==1.26.4",
             "websocket-client==0.48.0",
         ],
         'docs': [
@@ -60,11 +62,14 @@
             "sphinxcontrib-spelling==4.2.1",
             "sphinx-nameko-theme==0.0.3",
             "docutils<0.18",  # https://github.com/sphinx-doc/sphinx/issues/9788
+            "jinja2<3.1.0",  # https://github.com/readthedocs/readthedocs.org/issues/9037
         ],
         'examples': [
             "nameko-sqlalchemy==0.0.1",
-            "PyJWT==1.5.2",
-            "moto==1.3.6",
+            "PyJWT==1.7.1 ; python_version<'3'",
+            "PyJWT==2.6.0 ; python_version>'3'",
+            "moto==1.3.6 ; python_version<'3'",
+            "moto==4.1.2 ; python_version>'3'",
             "bcrypt==3.1.3",
             "regex==2018.2.21"
         ],

diff --git a/test/amqp/test_publish.py b/test/amqp/test_publish.py
@@ -2,6 +2,7 @@
 
 from time import time
 
+import kombu
 import pytest
 from amqp.exceptions import (
     NotFound, PreconditionFailed, RecoverableConnectionError
@@ -13,6 +14,7 @@
 from kombu.messaging import Exchange, Producer, Queue
 from kombu.serialization import registry
 from mock import ANY, MagicMock, Mock, call, patch
+from packaging import version
 from six.moves import queue
 from six.moves.urllib.parse import urlparse
 
@@ -23,6 +25,9 @@
 from nameko.constants import AMQP_URI_CONFIG_KEY
 
 
+IS_LEGACY_KOMBU = version.parse(kombu.__version__) < version.Version("5.2.4")
+
+
 def test_get_connection(amqp_uri):
     connection_ids = []
 
@@ -379,13 +384,19 @@ def test_retry(
         mock_publish = MagicMock(__name__="", __doc__="", __module__="")
         mock_publish.side_effect = RecoverableConnectionError("error")
 
-        expected_retries = publisher.retry_policy['max_retries'] + 1
-
         # with retry
         with patch.object(Producer, '_publish', new=mock_publish):
             with pytest.raises(OperationalError):
                 publisher.publish("payload", retry=True)
-        assert mock_publish.call_count == 1 + expected_retries
+
+        expected_publish_calls = publisher.retry_policy['max_retries'] + (
+            # plus two because the first publish doesn't count as a "retry",
+            # and older versions of kombu allowed one extra attempt. see
+            # https://github.com/celery/kombu/commit/5bed2a8f983a3bf61c12443e7704ffd89991ef9a
+            2 if IS_LEGACY_KOMBU else 1   # pragma: no cover (for branches)
+        )
+
+        assert mock_publish.call_count == expected_publish_calls
 
         mock_publish.reset_mock()
 
@@ -404,12 +415,19 @@ def test_retry_policy(
         retry_policy = {
             'max_retries': 5
         }
-        expected_retries = retry_policy['max_retries'] + 1
+
+        expected_publish_calls = retry_policy['max_retries'] + (
+            # plus two because the first publish doesn't count as a "retry",
+            # and older versions of kombu allowed one extra attempt. see
+            # https://github.com/celery/kombu/commit/5bed2a8f983a3bf61c12443e7704ffd89991ef9a
+            2 if IS_LEGACY_KOMBU else 1  # pragma: no cover (for branches)
+        )
 
         with patch.object(Producer, '_publish', new=mock_publish):
             with pytest.raises(OperationalError):
                 publisher.publish("payload", retry_policy=retry_policy)
-        assert mock_publish.call_count == 1 + expected_retries
+
+        assert mock_publish.call_count == expected_publish_calls
 
 
 class TestDefaults(object):

diff --git a/tox.ini b/tox.ini
@@ -8,7 +8,7 @@ deps =
     {py2.7,py3.5}-oldest: eventlet==0.20.1
     {py3.6}-oldest: eventlet==0.21.0
     {py3.7,py3.8,py3.9}-oldest: eventlet==0.25.1
-    {py3.10}-oldest: eventlet==0.33.0
+    {py3.10}-oldest: eventlet==0.33.3
     {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9}-oldest: kombu==4.2.0
     {py3.10}-oldest: kombu==5.2.0
     oldest: mock==1.2.0
@@ -20,13 +20,13 @@ deps =
 
     # pinned library versions
     {py3.5,py3.6,py3.7}-{pinned,extra}: eventlet==0.26.0
-    {py3.10}-{pinned,extra}: eventlet==0.33.0
+    {py3.10}-{pinned,extra}: eventlet==0.33.3
     {py2.7}-pinned: kombu==4.6.11 
     {py3.5,py3.6}-pinned: kombu==5.1.0
     {py3.7,py3.9,py3.9,py3.10}-pinned: kombu==5.2.0
     pinned: mock==2.0.0
     pinned: path.py==11.0.1
-    pinned: requests==2.19.1
+    pinned: requests==2.27.1
     pinned: six==1.11.0
     pinned: werkzeug==1.0.1
     pinned: wrapt==1.10.11
@@ -37,7 +37,7 @@ setenv =
     branchcoverage: ENABLE_BRANCH_COVERAGE=1
     examples: ENABLE_BRANCH_COVERAGE=1
 
-whitelist_externals = make
+allowlist_externals = make
 
 commands =
     mastereventlet: pip install --editable .[dev]