backport changes from nameko#767 to master branch

* fix flakiness in publish retry tests not clear how this ever passed. it's unhelpfully tied to the kombu implementation which probably means it's a bad test, but just fixing it up for now * upgrade isort * rename whitelist -> allowlist * upgrade moto and pyjwt * bump requests to resolve conflict with boto * pyjwt requires you to specify algorithm now * upgrade urllib to resolve dependency tree * expected retries isn't flakey; depends on version of kombu * update import following deprecation, removes warning * pytest arg --strict renamed to --strict-markers * import sort * no cover pragmas for kombu version conditionals * pyjwt and moto require older versions on py2 * run docs on ubuntu 20.04; drop 3.6 from matrix * put 3.6 back and try all on 20.04 * drop 3.6 again * sort 5.12.0 not available on py3.7 * pin down jinja to a releaase that is still compatible with our old version of sphinx. it's not worth trying to upgrade sphinx because the new docs use mkdocs * importlib-metadata 5.0 is not compatible with latest kombu on py3.7 importlib-metadata is not on later versions of python, so we can use a conditional pin. see # celery/celery#7783 * upgrade pinned version of eventlet fixes https://stackoverflow.com/questions/75137717/eventlet-dns-python-attribute-error-module-dns-rdtypes-has-no-attribute-any * whoops; fix syntax in setup.py * oldest eventlet on py3.10 needs updating too * match "pinned" requests to the one in dev reqs also choose a version that works on all currently supported pythons * refactor conditional to avoid another pragma * copy/paste snafu * try running tests on ubuntu latest # Conflicts: # docs/examples/test/test_examples.py # nameko/cli/__init__.py # nameko/cli/utils/__init__.py # nameko/rpc.py # nameko/testing/pytest.py # test/amqp/test_publish.py
mattbennett · Feb 9, 2023 · e5004f4 · e5004f4
1 parent 17ecee2
commit e5004f4
Show file tree

Hide file tree

Showing 9 changed files with 56 additions and 27 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -82,7 +82,6 @@ jobs:
           - 3.9
           - 3.8
           - 3.7
-          - 3.6
           - 2.7
         dependency-set:
           - pinned
@@ -138,7 +137,7 @@ jobs:
         TOXENV: py${{ matrix.python-version }}-${{ matrix.dependency-set }}-${{ matrix.command }}
 
   docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - name: checkout
       uses: actions/checkout@v2

diff --git a/Makefile b/Makefile
@@ -17,14 +17,14 @@ pylint:
 	pylint --rcfile=pylintrc nameko -E
 
 imports:
-	isort -rc $(autofix) nameko test
+	isort $(autofix) nameko test
 
 test_lib:
-	BRANCH=$(ENABLE_BRANCH_COVERAGE) coverage run -m nameko test test -v --strict --timeout 30
+	BRANCH=$(ENABLE_BRANCH_COVERAGE) coverage run -m nameko test test -v --strict-markers --timeout 30
 	BRANCH=$(ENABLE_BRANCH_COVERAGE) coverage report
 
 test_examples:
-	BRANCH=$(ENABLE_BRANCH_COVERAGE) nameko test docs/examples/test --strict --timeout 30 --cov=docs/examples --cov-config=$(CURDIR)/.coveragerc
+	BRANCH=$(ENABLE_BRANCH_COVERAGE) nameko test docs/examples/test --strict-markers --timeout 30 --cov=docs/examples --cov-config=$(CURDIR)/.coveragerc
 	nameko test docs/examples/testing
 
 test_docs: docs spelling #linkcheck

diff --git a/docs/examples/auth.py b/docs/examples/auth.py
@@ -33,7 +33,7 @@ def authenticate(self, username, password):
                 'username': username,
                 'roles': self.users[username]['roles']
             }
-            token = jwt.encode(payload, key=JWT_SECRET).decode('utf-8')
+            token = jwt.encode(payload, key=JWT_SECRET, algorithm="HS256")
             self.worker_ctx.context_data['auth'] = token
             return token
 
@@ -43,7 +43,9 @@ def has_role(self, role):
                 raise Unauthenticated()
 
             try:
-                payload = jwt.decode(token, key=JWT_SECRET, verify=True)
+                payload = jwt.decode(
+                    token, key=JWT_SECRET, verify=True, algorithms=["HS256"]
+                )
                 if role in payload['roles']:
                     return True
             except Exception:

diff --git a/docs/examples/test/test_examples.py b/docs/examples/test/test_examples.py
@@ -391,7 +391,7 @@ def test_authenticate(self, db):
         worker_ctx = Mock(context_data={})
         dep = Auth.Api(db, worker_ctx)
         token = dep.authenticate("matt", "secret")
-        jwt.decode(token, key=JWT_SECRET, verify=True)
+        jwt.decode(token, key=JWT_SECRET, verify=True, algorithms=["HS256"])
         assert worker_ctx.context_data['auth'] == token
 
     def test_authenticate_bad_username(self, db):
@@ -503,7 +503,7 @@ def test_sensitive_arguments(self, container_factory, rabbit_config):
 
         with ServiceRpcProxy("service", rabbit_config) as proxy:
             token = proxy.login("matt", "secret")
-            jwt.decode(token, key=JWT_SECRET, verify=True)
+            jwt.decode(token, key=JWT_SECRET, verify=True, algorithms=["HS256"])
             with pytest.raises(RemoteError) as exc:
                 proxy.login("matt", "incorrect")
             assert exc.value.exc_type == "Unauthenticated"

diff --git a/docs/examples/testing/alternative_dependency_unit_test.py b/docs/examples/testing/alternative_dependency_unit_test.py
@@ -3,8 +3,7 @@
 
 import pytest
 from sqlalchemy import Column, Integer, String, create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import declarative_base, sessionmaker
 
 from nameko.rpc import rpc
 from nameko.testing.services import worker_factory

diff --git a/nameko/testing/pytest.py b/nameko/testing/pytest.py
@@ -122,7 +122,8 @@ def empty_config():
 @pytest.fixture
 def mock_container(request, empty_config):
     from mock import create_autospec
-    from nameko.constants import SERIALIZER_CONFIG_KEY, DEFAULT_SERIALIZER
+
+    from nameko.constants import DEFAULT_SERIALIZER, SERIALIZER_CONFIG_KEY
     from nameko.containers import ServiceContainer
 
     container = create_autospec(ServiceContainer)
@@ -148,11 +149,13 @@ def vhost_pipeline(request, rabbit_manager):
     except ImportError:  # pragma: no cover
         # py2 compatibility
         from collections import Iterable  # pylint: disable=E0611
-    from six.moves.urllib.parse import urlparse  # pylint: disable=E0401
     import random
     import socket
     import string
+
     from kombu.pools import connections
+    from six.moves.urllib.parse import urlparse  # pylint: disable=E0401
+
     from nameko.testing.utils import ResourcePipeline
     from nameko.utils.retry import retry
     from requests.exceptions import HTTPError
@@ -393,6 +396,7 @@ def make_runner(config, *service_classes):
 @pytest.fixture
 def predictable_call_ids(request):
     import itertools
+
     from mock import patch
 
     with patch('nameko.containers.new_call_id', autospec=True) as get_id:
@@ -437,6 +441,7 @@ def request(self, method, url, *args, **kwargs):
 @pytest.fixture()
 def websocket(web_config_port):
     import eventlet
+
     from nameko.testing.websocket import make_virtual_socket
 
     active_sockets = []

diff --git a/setup.py b/setup.py
@@ -28,6 +28,7 @@
         "eventlet>=0.33.0 ; python_version>='3.10'",
         "kombu>=4.2.0",
         "kombu>=5.2.0 ; python_version>='3.10'",
+        "importlib-metadata<5 ; python_version<='3.7'",  # https://github.com/celery/celery/issues/7783
         "mock>=1.2",
         "path.py>=6.2",
         "pyyaml>=5.1",
@@ -41,15 +42,16 @@
         'dev': [
             "coverage==5.5",
             "flake8==3.9.2",
-            "isort==4.2.15",
+            "isort==4.3.21 ; python_version<'3'",
+            "isort==5.11.5 ; python_version>'3'",
             "pylint==1.9.5 ; python_version<'3'",
             "pylint==2.11.1 ; python_version>'3'",
             "pytest==4.6.11 ; python_version<'3'",
             "pytest==6.2.5 ; python_version>'3'",
             "pytest-cov==2.5.1",
             "pytest-timeout==1.3.3",
-            "requests==2.19.1",
-            "urllib3==1.23",
+            "requests==2.27.1",
+            "urllib3==1.26.4",
             "websocket-client==0.48.0",
         ],
         'docs': [
@@ -58,11 +60,14 @@
             "sphinxcontrib-spelling==4.2.1",
             "sphinx-nameko-theme==0.0.3",
             "docutils<0.18",  # https://github.com/sphinx-doc/sphinx/issues/9788
+            "jinja2<3.1.0",  # https://github.com/readthedocs/readthedocs.org/issues/9037
         ],
         'examples': [
             "nameko-sqlalchemy==0.0.1",
-            "PyJWT==1.5.2",
-            "moto==1.3.6",
+            "PyJWT==1.7.1 ; python_version<'3'",
+            "PyJWT==2.6.0 ; python_version>'3'",
+            "moto==1.3.6 ; python_version<'3'",
+            "moto==4.1.2 ; python_version>'3'",
             "bcrypt==3.1.3",
             "regex==2018.2.21"
         ]

diff --git a/test/amqp/test_publish.py b/test/amqp/test_publish.py
@@ -2,6 +2,7 @@
 
 from time import time
 
+import kombu
 import pytest
 from amqp.exceptions import (
     NotFound, PreconditionFailed, RecoverableConnectionError
@@ -13,6 +14,7 @@
 from kombu.messaging import Exchange, Producer, Queue
 from kombu.serialization import registry
 from mock import ANY, MagicMock, Mock, call, patch
+from packaging import version
 from six.moves import queue
 
 from nameko.amqp.publish import (
@@ -21,8 +23,12 @@
 from nameko.constants import AMQP_SSL_CONFIG_KEY
 
 
+IS_LEGACY_KOMBU = version.parse(kombu.__version__) < version.Version("5.2.4")
+
+
 def test_get_connection(rabbit_config):
     amqp_uri = rabbit_config['AMQP_URI']
+
     connection_ids = []
 
     with get_connection(amqp_uri) as connection:
@@ -382,13 +388,19 @@ def test_retry(
         mock_publish = MagicMock(__name__="", __doc__="", __module__="")
         mock_publish.side_effect = RecoverableConnectionError("error")
 
-        expected_retries = publisher.retry_policy['max_retries'] + 1
-
         # with retry
         with patch.object(Producer, '_publish', new=mock_publish):
             with pytest.raises(OperationalError):
                 publisher.publish("payload", retry=True)
-        assert mock_publish.call_count == 1 + expected_retries
+
+        expected_publish_calls = publisher.retry_policy['max_retries'] + (
+            # plus two because the first publish doesn't count as a "retry",
+            # and older versions of kombu allowed one extra attempt. see
+            # https://github.com/celery/kombu/commit/5bed2a8f983a3bf61c12443e7704ffd89991ef9a
+            2 if IS_LEGACY_KOMBU else 1   # pragma: no cover (for branches)
+        )
+
+        assert mock_publish.call_count == expected_publish_calls
 
         mock_publish.reset_mock()
 
@@ -407,12 +419,19 @@ def test_retry_policy(
         retry_policy = {
             'max_retries': 5
         }
-        expected_retries = retry_policy['max_retries'] + 1
+
+        expected_publish_calls = retry_policy['max_retries'] + (
+            # plus two because the first publish doesn't count as a "retry",
+            # and older versions of kombu allowed one extra attempt. see
+            # https://github.com/celery/kombu/commit/5bed2a8f983a3bf61c12443e7704ffd89991ef9a
+            2 if IS_LEGACY_KOMBU else 1  # pragma: no cover (for branches)
+        )
 
         with patch.object(Producer, '_publish', new=mock_publish):
             with pytest.raises(OperationalError):
                 publisher.publish("payload", retry_policy=retry_policy)
-        assert mock_publish.call_count == 1 + expected_retries
+
+        assert mock_publish.call_count == expected_publish_calls
 
 
 class TestDefaults(object):

diff --git a/tox.ini b/tox.ini
@@ -8,7 +8,7 @@ deps =
     {py2.7,py3.5}-oldest: eventlet==0.20.1
     {py3.6}-oldest: eventlet==0.21.0
     {py3.7,py3.8,py3.9}-oldest: eventlet==0.25.1
-    {py3.10}-oldest: eventlet==0.33.0
+    {py3.10}-oldest: eventlet==0.33.3
     {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9}-oldest: kombu==4.2.0
     {py3.10}-oldest: kombu==5.2.0
     oldest: mock==1.2.0
@@ -20,13 +20,13 @@ deps =
 
     # pinned library versions
     {py3.5,py3.6,py3.7}-{pinned,extra}: eventlet==0.26.0
-    {py3.10}-{pinned,extra}: eventlet==0.33.0
+    {py3.10}-{pinned,extra}: eventlet==0.33.3
     {py2.7}-pinned: kombu==4.6.11 
     {py3.5,py3.6}-pinned: kombu==5.1.0
     {py3.7,py3.9,py3.9,py3.10}-pinned: kombu==5.2.0
     pinned: mock==2.0.0
     pinned: path.py==11.0.1
-    pinned: requests==2.19.1
+    pinned: requests==2.27.1
     pinned: six==1.11.0
     pinned: werkzeug==1.0.1
     pinned: wrapt==1.10.11
@@ -38,7 +38,7 @@ setenv =
     branchcoverage: ENABLE_BRANCH_COVERAGE=1
     examples: ENABLE_BRANCH_COVERAGE=1
 
-whitelist_externals = make
+allowlist_externals = make
 
 commands =
     mastereventlet: pip install --editable .[dev]