Migrate from Flake8 & Autopep8 to Ruff (#133)

.gitignore

@@ -1,6 +1,7 @@
 __pycache__
 .mypy_cache
 .pytest_cache
+.ruff_cache
 
 .venv
 .direnv

CHANGELOG.md

@@ -1,10 +1,12 @@
 Changelog
 =========
 
-[1.3.1](../../releases/tag/v1.3.1) - Unreleased
+[1.4.0](../../releases/tag/v1.4.0) - Unreleased
 -----------------------------------------------
 
-...
+### Internal changes
+
+- Migrate from Autopep8 and Flake8 to Ruff
 
 [1.3.0](../../releases/tag/v1.3.0) - 2023-11-15
 -----------------------------------------------

CONTRIBUTING.md

@@ -24,11 +24,11 @@ To install this package and its development dependencies, run `make install-dev`
 
 ## Formatting
 
-We use `autopep8` and `isort` to automatically format the code to a common format. To run the formatting, just run `make format`.
+We use `ruff` to automatically format the code to a common format. To run the formatting, just run `make format`.
 
 ## Linting, type-checking and unit testing
 
-We use `flake8` for linting, `mypy` for type checking and `pytest` for unit testing. To run these tools, just run `make check-code`.
+We use `ruff` for linting, `mypy` for type checking and `pytest` for unit testing. To run these tools, just run `make check-code`.
 
 ## Integration tests
 

Makefile

@@ -1,27 +1,29 @@
 .PHONY: clean install-dev build publish twine-check lint unit-tests integration-tests type-check check-code format check-version-availability check-changelog-entry build-api-reference
 
+DIRS_WITH_CODE = src tests scripts
+
 # This is default for local testing, but GitHub workflows override it to a higher value in CI
 INTEGRATION_TESTS_CONCURRENCY = 1
 
 clean:
 	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
 
 install-dev:
-	python -m pip install --upgrade pip
+	python3 -m pip install --upgrade pip
 	pip install --no-cache-dir -e ".[dev,scrapy]"
 	pre-commit install
 
 build:
-	python -m build
+	python3 -m build
 
 publish:
-	python -m twine upload dist/*
+	python3 -m twine upload dist/*
 
 twine-check:
-	python -m twine check dist/*
+	python3 -m twine check dist/*
 
 lint:
-	python3 -m flake8
+	python3 -m ruff check $(DIRS_WITH_CODE)
 
 unit-tests:
 	python3 -m pytest -n auto -ra tests/unit
@@ -30,13 +32,13 @@ integration-tests:
 	python3 -m pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration
 
 type-check:
-	python3 -m mypy
+	python3 -m mypy $(DIRS_WITH_CODE)
 
 check-code: lint type-check unit-tests
 
 format:
-	python3 -m isort src tests
-	python3 -m autopep8 --in-place --recursive src tests
+	python3 -m ruff check --fix $(DIRS_WITH_CODE)
+	python3 -m ruff format $(DIRS_WITH_CODE)
 
 check-version-availability:
 	python3 scripts/check_version_availability.py

pyproject.toml

@@ -1,12 +1,10 @@
 [project]
 name = "apify"
-version = "1.3.1"
+version = "1.4.0"
 description = "Apify SDK for Python"
 readme = "README.md"
-license = {text = "Apache Software License"}
-authors = [
-    {name = "Apify Technologies s.r.o.", email = "support@apify.com"},
-]
+license = { text = "Apache Software License" }
+authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 keywords = ["apify", "sdk", "actor", "scraping", "automation"]
 
 classifiers = [
@@ -26,8 +24,8 @@ requires-python = ">=3.8"
 dependencies = [
     "aiofiles >= 22.1.0",
     "aioshutil >= 1.0",
-    "apify-client ~= 1.5.0",
-    "apify-shared ~= 1.0.4",
+    "apify-client ~= 1.6.0",
+    "apify-shared ~= 1.1.0",
     "colorama >= 0.4.6",
     "cryptography >= 39.0.0",
     "httpx >= 0.24.1",
@@ -40,25 +38,9 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "autopep8 ~= 2.0.4",
     "build ~= 1.0.3",
     "filelock ~= 3.12.4",
-    "flake8 ~= 6.1.0",
-    "flake8-bugbear ~= 23.9.16",
-    "flake8-commas ~= 2.1.0; python_version < '3.12'",
-    "flake8-comprehensions ~= 3.14.0",
-    "flake8-datetimez ~= 20.10.0",
-    "flake8-docstrings ~= 1.7.0",
-    "flake8-encodings ~= 0.5.0",
-    "flake8-isort ~= 6.1.0",
-    "flake8-noqa ~= 1.3.1; python_version < '3.12'",
-    "flake8-pytest-style ~= 1.7.2",
-    "flake8-quotes ~= 3.3.2; python_version < '3.12'",
-    "flake8-simplify ~= 0.21.0",
-    "flake8-unused-arguments ~= 0.0.13",
-    "isort ~= 5.12.0",
-    "mypy ~= 1.5.1",
-    "pep8-naming ~= 0.13.3",
+    "mypy ~= 1.7.1",
     "pre-commit ~= 3.4.0",
     "pydoc-markdown ~= 4.8.2",
     "pytest ~= 7.4.2",
@@ -67,6 +49,7 @@ dev = [
     "pytest-timeout ~= 2.2.0",
     "pytest-xdist ~= 3.3.1",
     "respx ~= 0.20.1",
+    "ruff ~= 0.1.6",
     "twine ~= 4.0.2",
     "types-aiofiles ~= 23.2.0.0",
     "types-colorama ~= 0.4.15.11",
@@ -94,3 +77,72 @@ include = ["apify*"]
 
 [tool.setuptools.package-data]
 apify = ["py.typed"]
+
+[tool.ruff]
+line-length = 150
+select = ["ALL"]
+ignore = [
+    "ANN401",  # Dynamically typed expressions (typing.Any) are disallowed in {filename}
+    "BLE001",  # Do not catch blind exception
+    "C901",    # `{name}` is too complex
+    "COM812",  # This rule may cause conflicts when used with the formatter
+    "D100",    # Missing docstring in public module
+    "D104",    # Missing docstring in public package
+    "EM",      # flake8-errmsg
+    "G004",    # Logging statement uses f-string
+    "ISC001",  # This rule may cause conflicts when used with the formatter
+    "FIX",     # flake8-fixme
+    "PGH003",  # Use specific rule codes when ignoring type issues
+    "PLR0911", # Too many return statements
+    "PLR0913", # Too many arguments in function definition
+    "PLR0915", # Too many statements
+    "PTH",     # flake8-use-pathlib
+    "PYI034",  # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
+    "PYI036",  # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
+    "S102",    # Use of `exec` detected
+    "S105",    # Possible hardcoded password assigned to
+    "S106",    # Possible hardcoded password assigned to argument: "{name}"
+    "S301",    # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
+    "S303",    # Use of insecure MD2, MD4, MD5, or SHA1 hash function
+    "S311",    # Standard pseudo-random generators are not suitable for cryptographic purposes
+    "TD002",   # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
+    "TID252",  # Relative imports from parent modules are bannedRuff
+    "TRY003",  # Avoid specifying long messages outside the exception class
+
+    # TODO: Remove this once the following issue is fixed
+    # https://github.com/apify/apify-sdk-python/issues/150
+    "SLF001", # Private member accessed: `{name}`
+]
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+
+[tool.ruff.lint.per-file-ignores]
+"**/__init__.py" = [
+    "F401", # Unused imports
+]
+"**/{scripts}/*" = [
+    "D",       # Everything from the pydocstyle
+    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
+    "T20",     # flake8-print
+]
+"**/{tests}/*" = [
+    "D",       # Everything from the pydocstyle
+    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
+    "S101",    # Use of assert detected
+    "T20",     # flake8-print
+    "TRY301",  # Abstract `raise` to an inner function
+]
+
+[tool.ruff.lint.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "single"
+
+[tool.ruff.lint.isort]
+known-first-party = ["apify", "apify_client", "apify_shared"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"

scripts/check_version_availability.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+
+from __future__ import annotations
+
 from utils import get_current_package_version, get_published_package_versions
 
 # Checks whether the current package version number was not already used in a published release.

scripts/check_version_in_changelog.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+from __future__ import annotations
+
 import re
 
 from utils import REPO_ROOT, get_current_package_version
@@ -16,7 +18,7 @@
     with open(CHANGELOG_PATH, encoding='utf-8') as changelog_file:
         for line in changelog_file:
             # The heading for the changelog entry for the given version can start with either the version number, or the version number in a link
-            if re.match(fr'\[?{current_package_version}([\] ]|$)', line):
+            if re.match(rf'\[?{current_package_version}([\] ]|$)', line):
                 break
         else:
             raise RuntimeError(f'There is no entry in the changelog for the current package version ({current_package_version})')

scripts/print_current_package_version.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+from __future__ import annotations
+
 from utils import get_current_package_version
 
 # Print the current package version from the pyproject.toml file to stdout

scripts/update_version_for_prerelease.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+from __future__ import annotations
+
 import re
 import sys
 

scripts/utils.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 import json
 import pathlib
-import urllib.request
+from urllib.error import HTTPError
+from urllib.request import urlopen
 
 PACKAGE_NAME = 'apify'
 REPO_ROOT = pathlib.Path(__file__).parent.resolve() / '..'
@@ -10,13 +13,12 @@
 # Load the current version number from pyproject.toml
 # It is on a line in the format `version = "1.2.3"`
 def get_current_package_version() -> str:
-    with open(PYPROJECT_TOML_FILE_PATH, 'r', encoding='utf-8') as pyproject_toml_file:
+    with open(PYPROJECT_TOML_FILE_PATH, encoding='utf-8') as pyproject_toml_file:
         for line in pyproject_toml_file:
             if line.startswith('version = '):
                 delim = '"' if '"' in line else "'"
-                version = line.split(delim)[1]
-                return version
-        else:
+                return line.split(delim)[1]
+        else:  # noqa: PLW0120
             raise RuntimeError('Unable to find version string.')
 
 
@@ -29,7 +31,7 @@ def set_current_package_version(version: str) -> None:
         for line in pyproject_toml_file:
             if line.startswith('version = '):
                 version_string_found = True
-                line = f'version = "{version}"\n'
+                line = f'version = "{version}"\n'  # noqa: PLW2901
             updated_pyproject_toml_file_lines.append(line)
 
         if not version_string_found:
@@ -44,11 +46,11 @@ def set_current_package_version(version: str) -> None:
 def get_published_package_versions() -> list:
     package_info_url = f'https://pypi.org/pypi/{PACKAGE_NAME}/json'
     try:
-        package_data = json.load(urllib.request.urlopen(package_info_url))
+        package_data = json.load(urlopen(package_info_url))  # noqa: S310
         published_versions = list(package_data['releases'].keys())
     # If the URL returns 404, it means the package has no releases yet (which is okay in our case)
-    except urllib.error.HTTPError as e:
-        if e.code != 404:
-            raise e
+    except HTTPError as exc:
+        if exc.code != 404:
+            raise
         published_versions = []
     return published_versions