diff --git a/etc/docker/dev/configure_qbittorrent.py b/etc/docker/dev/configure_qbittorrent.py index 477a35e6c7..5d16e07f32 100644 --- a/etc/docker/dev/configure_qbittorrent.py +++ b/etc/docker/dev/configure_qbittorrent.py @@ -14,12 +14,12 @@ # limitations under the License. import json -import re import os +import re import sys import time -import urllib.request import urllib.parse +import urllib.request def wait_for_server(host, port, max_wait): diff --git a/lib/rucio/api/did.py b/lib/rucio/api/did.py index c25d27af82..2a6cae7c34 100644 --- a/lib/rucio/api/did.py +++ b/lib/rucio/api/did.py @@ -23,13 +23,14 @@ from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import api_update_return_dict from rucio.core import did, naming_convention -from rucio.core.rse import get_rse_id from rucio.core import meta_conventions as meta_convention_core +from rucio.core.rse import get_rse_id from rucio.db.sqla.constants import DIDType from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: - from typing import Any, Optional, Iterator + from typing import Any, Iterator, Optional + from sqlalchemy.orm import Session diff --git a/lib/rucio/api/identity.py b/lib/rucio/api/identity.py index fb73680582..0271fb0537 100644 --- a/lib/rucio/api/identity.py +++ b/lib/rucio/api/identity.py @@ -28,6 +28,7 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session diff --git a/lib/rucio/api/meta_conventions.py b/lib/rucio/api/meta_conventions.py index 138c262272..18d85810ca 100644 --- a/lib/rucio/api/meta_conventions.py +++ b/lib/rucio/api/meta_conventions.py @@ -13,16 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING, Union, Optional +from typing import TYPE_CHECKING, Optional, Union from rucio.api.permission import has_permission from rucio.common.exception import AccessDenied from rucio.core import meta_conventions -from rucio.db.sqla.session import read_session, transactional_session from rucio.db.sqla.constants import KeyType +from rucio.db.sqla.session import read_session, transactional_session if TYPE_CHECKING: from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount diff --git a/lib/rucio/api/rse.py b/lib/rucio/api/rse.py index 2b45126e65..900664d7c0 100644 --- a/lib/rucio/api/rse.py +++ b/lib/rucio/api/rse.py @@ -26,6 +26,7 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session diff --git a/lib/rucio/api/rule.py b/lib/rucio/api/rule.py index 710afb2225..1b396bce2e 100644 --- a/lib/rucio/api/rule.py +++ b/lib/rucio/api/rule.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from rucio.api.permission import has_permission from rucio.common.config import config_get_bool diff --git a/lib/rucio/api/subscription.py b/lib/rucio/api/subscription.py index e4c7bb87b9..d76074ff94 100644 --- a/lib/rucio/api/subscription.py +++ b/lib/rucio/api/subscription.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING from rucio.api.permission import has_permission -from rucio.common.exception import InvalidObject, AccessDenied +from rucio.common.exception import AccessDenied, InvalidObject from rucio.common.schema import validate_schema from rucio.common.types import InternalAccount, InternalScope from rucio.core import subscription diff --git a/lib/rucio/client/accountclient.py b/lib/rucio/client/accountclient.py index 58cbe51da0..3d00792cb9 100644 --- a/lib/rucio/client/accountclient.py +++ b/lib/rucio/client/accountclient.py @@ -18,8 +18,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/accountlimitclient.py b/lib/rucio/client/accountlimitclient.py index e2186fe7a5..a8d965e362 100644 --- a/lib/rucio/client/accountlimitclient.py +++ b/lib/rucio/client/accountlimitclient.py @@ -18,8 +18,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py index 66faea8e77..8b5657bc78 100644 --- a/lib/rucio/client/baseclient.py +++ b/lib/rucio/client/baseclient.py @@ -24,24 +24,22 @@ import sys import time from configparser import NoOptionError, NoSectionError -from os import environ, fdopen, path, makedirs, geteuid +from os import environ, fdopen, geteuid, makedirs, path from shutil import move from tempfile import mkstemp from urllib.parse import urlparse from dogpile.cache import make_region -from requests import Session, Response +from requests import Response, Session from requests.exceptions import ConnectionError from requests.status_codes import codes from rucio import version from rucio.common import exception from rucio.common.config import config_get, config_get_bool, config_get_int -from rucio.common.exception import (CannotAuthenticate, ClientProtocolNotSupported, - NoAuthInformation, MissingClientParameter, - MissingModuleException, ServerConnectionException) +from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, MissingClientParameter, MissingModuleException, NoAuthInformation, ServerConnectionException from rucio.common.extra import import_extras -from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, ssh_sign, setup_logger +from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign EXTRA_MODULES = import_extras(['requests_kerberos']) diff --git a/lib/rucio/client/configclient.py b/lib/rucio/client/configclient.py index ce60a541dd..c66505c53d 100644 --- a/lib/rucio/client/configclient.py +++ b/lib/rucio/client/configclient.py @@ -17,8 +17,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/credentialclient.py b/lib/rucio/client/credentialclient.py index 2b0672e115..6275bcfdbe 100644 --- a/lib/rucio/client/credentialclient.py +++ b/lib/rucio/client/credentialclient.py @@ -15,8 +15,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/didclient.py b/lib/rucio/client/didclient.py index 5045676ffe..9cb8fea493 100644 --- a/lib/rucio/client/didclient.py +++ b/lib/rucio/client/didclient.py @@ -20,10 +20,9 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.exception import DeprecationError -from rucio.common.utils import build_url, render_json, render_json_list, date_to_str +from rucio.common.utils import build_url, date_to_str, render_json, render_json_list class DIDClient(BaseClient): diff --git a/lib/rucio/client/diracclient.py b/lib/rucio/client/diracclient.py index 516cc5e001..2d42559d7e 100644 --- a/lib/rucio/client/diracclient.py +++ b/lib/rucio/client/diracclient.py @@ -17,8 +17,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py index 6835a5d0ce..ecec0e4b27 100644 --- a/lib/rucio/client/downloadclient.py +++ b/lib/rucio/client/downloadclient.py @@ -23,18 +23,16 @@ import signal import subprocess import time -from queue import Queue, Empty, deque +from queue import Empty, Queue, deque from threading import Thread from rucio import version from rucio.client.client import Client from rucio.common.config import config_get from rucio.common.didtype import DID -from rucio.common.exception import (InputValidationError, NoFilesDownloaded, NotAllFilesDownloaded, RucioException) +from rucio.common.exception import InputValidationError, NoFilesDownloaded, NotAllFilesDownloaded, RucioException from rucio.common.pcache import Pcache -from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, CHECKSUM_ALGO_DICT, PREFERRED_CHECKSUM -from rucio.common.utils import adler32, detect_client_location, generate_uuid, parse_replicas_from_string, \ - send_trace, sizefmt, execute, parse_replicas_from_file, extract_scope +from rucio.common.utils import CHECKSUM_ALGO_DICT, GLOBALLY_SUPPORTED_CHECKSUMS, PREFERRED_CHECKSUM, adler32, detect_client_location, execute, extract_scope, generate_uuid, parse_replicas_from_file, parse_replicas_from_string, send_trace, sizefmt from rucio.rse import rsemanager as rsemgr diff --git a/lib/rucio/client/exportclient.py b/lib/rucio/client/exportclient.py index e6b4c2f901..eb7b328315 100644 --- a/lib/rucio/client/exportclient.py +++ b/lib/rucio/client/exportclient.py @@ -15,8 +15,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url, parse_response diff --git a/lib/rucio/client/fileclient.py b/lib/rucio/client/fileclient.py index cbb91fefd6..cdf07ab136 100644 --- a/lib/rucio/client/fileclient.py +++ b/lib/rucio/client/fileclient.py @@ -18,8 +18,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/importclient.py b/lib/rucio/client/importclient.py index d331ef2ba8..990e7c251d 100644 --- a/lib/rucio/client/importclient.py +++ b/lib/rucio/client/importclient.py @@ -15,8 +15,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url, render_json diff --git a/lib/rucio/client/lifetimeclient.py b/lib/rucio/client/lifetimeclient.py index ebec40727a..cc6098d0e3 100644 --- a/lib/rucio/client/lifetimeclient.py +++ b/lib/rucio/client/lifetimeclient.py @@ -17,8 +17,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url, render_json diff --git a/lib/rucio/client/lockclient.py b/lib/rucio/client/lockclient.py index c17960da9f..a15b388203 100644 --- a/lib/rucio/client/lockclient.py +++ b/lib/rucio/client/lockclient.py @@ -17,8 +17,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url, render_json diff --git a/lib/rucio/client/metaconventionsclient.py b/lib/rucio/client/metaconventionsclient.py index 3fd244ea62..7b7e213f62 100644 --- a/lib/rucio/client/metaconventionsclient.py +++ b/lib/rucio/client/metaconventionsclient.py @@ -14,15 +14,13 @@ # limitations under the License. from json import dumps, loads +from typing import TYPE_CHECKING, Optional, Union from urllib.parse import quote_plus from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url -from typing import Union, Optional, TYPE_CHECKING - if TYPE_CHECKING: from rucio.db.sqla.constants import KeyType diff --git a/lib/rucio/client/replicaclient.py b/lib/rucio/client/replicaclient.py index 95b1704835..41754a593d 100644 --- a/lib/rucio/client/replicaclient.py +++ b/lib/rucio/client/replicaclient.py @@ -19,9 +19,8 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice -from rucio.common.utils import build_url, render_json, chunks +from rucio.client.baseclient import BaseClient, choice +from rucio.common.utils import build_url, chunks, render_json class ReplicaClient(BaseClient): diff --git a/lib/rucio/client/requestclient.py b/lib/rucio/client/requestclient.py index 7fbc43cee6..7f5a880794 100644 --- a/lib/rucio/client/requestclient.py +++ b/lib/rucio/client/requestclient.py @@ -17,8 +17,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/rseclient.py b/lib/rucio/client/rseclient.py index 3ae19cee38..efeefd729a 100644 --- a/lib/rucio/client/rseclient.py +++ b/lib/rucio/client/rseclient.py @@ -18,8 +18,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/ruleclient.py b/lib/rucio/client/ruleclient.py index 4aec50db6e..7ad4d19617 100644 --- a/lib/rucio/client/ruleclient.py +++ b/lib/rucio/client/ruleclient.py @@ -19,8 +19,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/scopeclient.py b/lib/rucio/client/scopeclient.py index be6d8eaee2..9d497ad5cd 100644 --- a/lib/rucio/client/scopeclient.py +++ b/lib/rucio/client/scopeclient.py @@ -18,8 +18,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/subscriptionclient.py b/lib/rucio/client/subscriptionclient.py index 4f314a3d45..dc281c55a8 100644 --- a/lib/rucio/client/subscriptionclient.py +++ b/lib/rucio/client/subscriptionclient.py @@ -17,8 +17,7 @@ from requests.status_codes import codes -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.utils import build_url diff --git a/lib/rucio/client/touchclient.py b/lib/rucio/client/touchclient.py index f99131511b..7eb6f78550 100644 --- a/lib/rucio/client/touchclient.py +++ b/lib/rucio/client/touchclient.py @@ -17,8 +17,7 @@ from requests import post -from rucio.client.baseclient import BaseClient -from rucio.client.baseclient import choice +from rucio.client.baseclient import BaseClient, choice from rucio.common.exception import RucioException, UnsupportedDIDType diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py index 8cc24759ba..8b83fa3858 100644 --- a/lib/rucio/client/uploadclient.py +++ b/lib/rucio/client/uploadclient.py @@ -25,13 +25,23 @@ from rucio import version from rucio.client.client import Client -from rucio.common.config import config_get_int, config_get, config_get_bool -from rucio.common.exception import (RucioException, RSEWriteBlocked, DataIdentifierAlreadyExists, RSEOperationNotSupported, - DataIdentifierNotFound, NoFilesUploaded, NotAllFilesUploaded, FileReplicaAlreadyExists, - ResourceTemporaryUnavailable, ServiceUnavailable, InputValidationError, RSEChecksumUnavailable, - ScopeNotFound) -from rucio.common.utils import (adler32, detect_client_location, execute, generate_uuid, make_valid_did, md5, send_trace, - retry, bittorrent_v2_merkle_sha256, GLOBALLY_SUPPORTED_CHECKSUMS) +from rucio.common.config import config_get, config_get_bool, config_get_int +from rucio.common.exception import ( + DataIdentifierAlreadyExists, + DataIdentifierNotFound, + FileReplicaAlreadyExists, + InputValidationError, + NoFilesUploaded, + NotAllFilesUploaded, + ResourceTemporaryUnavailable, + RSEChecksumUnavailable, + RSEOperationNotSupported, + RSEWriteBlocked, + RucioException, + ScopeNotFound, + ServiceUnavailable, +) +from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, adler32, bittorrent_v2_merkle_sha256, detect_client_location, execute, generate_uuid, make_valid_did, md5, retry, send_trace from rucio.rse import rsemanager as rsemgr diff --git a/lib/rucio/common/config.py b/lib/rucio/common/config.py index 1732d5e103..bf163d921a 100644 --- a/lib/rucio/common/config.py +++ b/lib/rucio/common/config.py @@ -19,7 +19,7 @@ import json import os from collections.abc import Callable -from typing import TYPE_CHECKING, overload, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Optional, TypeVar, Union, overload from rucio.common import exception from rucio.common.exception import ConfigNotFound, DatabaseException diff --git a/lib/rucio/common/dumper/consistency.py b/lib/rucio/common/dumper/consistency.py index 4ea25cf91c..54b1f01404 100644 --- a/lib/rucio/common/dumper/consistency.py +++ b/lib/rucio/common/dumper/consistency.py @@ -21,7 +21,7 @@ import tempfile from rucio.common import dumper -from rucio.common.dumper import error, DUMPS_CACHE_DIR, data_models, path_parsing +from rucio.common.dumper import DUMPS_CACHE_DIR, data_models, error, path_parsing subcommands = ['consistency', 'consistency-manual'] diff --git a/lib/rucio/common/dumper/data_models.py b/lib/rucio/common/dumper/data_models.py index 5599b3f1c3..69320406a8 100644 --- a/lib/rucio/common/dumper/data_models.py +++ b/lib/rucio/common/dumper/data_models.py @@ -27,13 +27,7 @@ from tabulate import tabulate -from rucio.common.dumper import DUMPS_CACHE_DIR -from rucio.common.dumper import HTTPDownloadFailed -from rucio.common.dumper import get_requests_session -from rucio.common.dumper import http_download_to_file -from rucio.common.dumper import smart_open -from rucio.common.dumper import temp_file -from rucio.common.dumper import to_datetime +from rucio.common.dumper import DUMPS_CACHE_DIR, HTTPDownloadFailed, get_requests_session, http_download_to_file, smart_open, temp_file, to_datetime class DataModel(object): diff --git a/lib/rucio/common/pcache.py b/lib/rucio/common/pcache.py index 6647bb20bf..67fb147605 100644 --- a/lib/rucio/common/pcache.py +++ b/lib/rucio/common/pcache.py @@ -15,6 +15,7 @@ # limitations under the License. import errno +import fcntl import getopt import os import re @@ -26,8 +27,6 @@ from urllib.parse import urlencode from urllib.request import urlopen -import fcntl - # The pCache Version pcacheversion = "4.2.3" diff --git a/lib/rucio/common/plugins.py b/lib/rucio/common/plugins.py index 00e44e0252..d513002814 100644 --- a/lib/rucio/common/plugins.py +++ b/lib/rucio/common/plugins.py @@ -16,12 +16,11 @@ import importlib import os from configparser import NoOptionError, NoSectionError -from typing import Any, Callable, Dict, TypeVar, Type +from typing import Any, Callable, Dict, Type, TypeVar from rucio.common import config from rucio.common.exception import InvalidAlgorithmName - PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms') diff --git a/lib/rucio/common/schema/atlas.py b/lib/rucio/common/schema/atlas.py index 389e61a41e..5f5a6fb0f2 100644 --- a/lib/rucio/common/schema/atlas.py +++ b/lib/rucio/common/schema/atlas.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/belleii.py b/lib/rucio/common/schema/belleii.py index 9c46c610eb..ca86240c06 100644 --- a/lib/rucio/common/schema/belleii.py +++ b/lib/rucio/common/schema/belleii.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/cms.py b/lib/rucio/common/schema/cms.py index e871db664c..15ddade0a3 100644 --- a/lib/rucio/common/schema/cms.py +++ b/lib/rucio/common/schema/cms.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/domatpc.py b/lib/rucio/common/schema/domatpc.py index 5547a8cf31..02fade7827 100644 --- a/lib/rucio/common/schema/domatpc.py +++ b/lib/rucio/common/schema/domatpc.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/escape.py b/lib/rucio/common/schema/escape.py index aa4fed38ee..d267dc3817 100644 --- a/lib/rucio/common/schema/escape.py +++ b/lib/rucio/common/schema/escape.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/generic.py b/lib/rucio/common/schema/generic.py index 95a73b45eb..6afa5d6172 100644 --- a/lib/rucio/common/schema/generic.py +++ b/lib/rucio/common/schema/generic.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/generic_multi_vo.py b/lib/rucio/common/schema/generic_multi_vo.py index 3d240f0507..3560611267 100644 --- a/lib/rucio/common/schema/generic_multi_vo.py +++ b/lib/rucio/common/schema/generic_multi_vo.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/icecube.py b/lib/rucio/common/schema/icecube.py index 3103a89582..be3f289cba 100644 --- a/lib/rucio/common/schema/icecube.py +++ b/lib/rucio/common/schema/icecube.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/schema/lsst.py b/lib/rucio/common/schema/lsst.py index 91a726c8f7..1a133a132b 100644 --- a/lib/rucio/common/schema/lsst.py +++ b/lib/rucio/common/schema/lsst.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from jsonschema import validate, ValidationError +from jsonschema import ValidationError, validate from rucio.common.exception import InvalidObject diff --git a/lib/rucio/common/test_rucio_server.py b/lib/rucio/common/test_rucio_server.py index 2a3128f93a..8fbcd1d987 100644 --- a/lib/rucio/common/test_rucio_server.py +++ b/lib/rucio/common/test_rucio_server.py @@ -17,7 +17,8 @@ from os import remove from os.path import basename -from rucio.common.utils import generate_uuid as uuid, execute +from rucio.common.utils import execute +from rucio.common.utils import generate_uuid as uuid def file_generator(size=2048, namelen=10): diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py index 9e3771e07a..4b91fd522c 100644 --- a/lib/rucio/common/utils.py +++ b/lib/rucio/common/utils.py @@ -25,6 +25,8 @@ import itertools import json import logging +import math +import mmap import os import os.path import re @@ -34,6 +36,7 @@ import tempfile import threading import time +import zlib from collections import OrderedDict from collections.abc import Callable, Sequence from enum import Enum @@ -41,18 +44,14 @@ from io import StringIO from itertools import zip_longest from typing import TYPE_CHECKING, Optional, Type, TypeVar -from urllib.parse import urlparse, urlencode, quote, parse_qsl, urlunparse +from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse from uuid import uuid4 as uuid from xml.etree import ElementTree -import math -import mmap import requests -import zlib from rucio.common.config import config_get, config_has_section -from rucio.common.exception import MissingModuleException, InvalidType, InputValidationError, MetalinkJsonParsingError, RucioException, \ - DuplicateCriteriaInDIDFilter, DIDFilterSyntaxError, PolicyPackageVersionError +from rucio.common.exception import DIDFilterSyntaxError, DuplicateCriteriaInDIDFilter, InputValidationError, InvalidType, MetalinkJsonParsingError, MissingModuleException, PolicyPackageVersionError, RucioException from rucio.common.extra import import_extras from rucio.common.plugins import PolicyPackageAlgorithms from rucio.common.types import InternalAccount, InternalScope @@ -2063,6 +2062,7 @@ def _priority_increased(self, item): def check_policy_package_version(package): import importlib + from rucio.version import version_string ''' Checks that the Rucio version supported by the policy package is compatible diff --git a/lib/rucio/core/account.py b/lib/rucio/core/account.py index b65d702e1b..7688dffe52 100644 --- a/lib/rucio/core/account.py +++ b/lib/rucio/core/account.py @@ -19,7 +19,7 @@ from traceback import format_exc from typing import TYPE_CHECKING -from sqlalchemy import select, and_ +from sqlalchemy import and_, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import exc @@ -30,7 +30,7 @@ from rucio.core.vo import vo_exists from rucio.db.sqla import models from rucio.db.sqla.constants import AccountStatus, AccountType -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: from sqlalchemy.orm import Session diff --git a/lib/rucio/core/account_counter.py b/lib/rucio/core/account_counter.py index c0626bd9f8..1f103278d4 100644 --- a/lib/rucio/core/account_counter.py +++ b/lib/rucio/core/account_counter.py @@ -15,10 +15,10 @@ import datetime from typing import TYPE_CHECKING -from sqlalchemy import literal, insert, select +from sqlalchemy import insert, literal, select from sqlalchemy.orm.exc import NoResultFound -from rucio.db.sqla import models, filter_thread_work +from rucio.db.sqla import filter_thread_work, models from rucio.db.sqla.session import read_session, transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/core/account_limit.py b/lib/rucio/core/account_limit.py index fd7b84722d..b420589625 100644 --- a/lib/rucio/core/account_limit.py +++ b/lib/rucio/core/account_limit.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING from sqlalchemy.orm.exc import NoResultFound -from sqlalchemy.sql import func, select, literal +from sqlalchemy.sql import func, literal, select from sqlalchemy.sql.expression import and_, or_ from rucio.core.account import get_all_rse_usages_per_account diff --git a/lib/rucio/core/authentication.py b/lib/rucio/core/authentication.py index cd5775ea39..9e97faa5d1 100644 --- a/lib/rucio/core/authentication.py +++ b/lib/rucio/core/authentication.py @@ -30,18 +30,18 @@ from rucio.common.cache import make_region_memcached from rucio.common.config import config_get_bool from rucio.common.exception import CannotAuthenticate, RucioException -from rucio.common.utils import chunks, generate_uuid, date_to_str +from rucio.common.utils import chunks, date_to_str, generate_uuid from rucio.core.account import account_exists from rucio.core.oidc import validate_jwt -from rucio.db.sqla import filter_thread_work -from rucio.db.sqla import models +from rucio.db.sqla import filter_thread_work, models from rucio.db.sqla.constants import IdentityType from rucio.db.sqla.session import read_session, transactional_session if TYPE_CHECKING: - from sqlalchemy.orm import Session from typing import Any, Union + from sqlalchemy.orm import Session + def strip_x509_proxy_attributes(dn: str) -> str: """Strip X509 proxy attributes from a DN. diff --git a/lib/rucio/core/credential.py b/lib/rucio/core/credential.py index 33136c7dc1..047fce0cf8 100644 --- a/lib/rucio/core/credential.py +++ b/lib/rucio/core/credential.py @@ -18,7 +18,7 @@ import hmac import time from hashlib import sha1 -from urllib.parse import urlparse, urlencode +from urllib.parse import urlencode, urlparse import boto3 from botocore.client import Config diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py index 0011360a1a..031cbc8108 100644 --- a/lib/rucio/core/did.py +++ b/lib/rucio/core/did.py @@ -21,31 +21,33 @@ from re import match from typing import TYPE_CHECKING -from sqlalchemy import and_, or_, exists, update, delete, insert +from sqlalchemy import and_, delete, exists, insert, or_, update from sqlalchemy.exc import DatabaseError, IntegrityError from sqlalchemy.orm.exc import NoResultFound -from sqlalchemy.sql import not_, func -from sqlalchemy.sql.expression import bindparam, case, select, true, false, null +from sqlalchemy.sql import func, not_ +from sqlalchemy.sql.expression import bindparam, case, false, null, select, true import rucio.core.replica # import add_replicas import rucio.core.rule from rucio.common import exception from rucio.common.config import config_get_bool, config_get_int -from rucio.common.utils import is_archive, chunks +from rucio.common.utils import chunks, is_archive from rucio.core import did_meta_plugins from rucio.core.message import add_message from rucio.core.monitor import MetricManager from rucio.core.naming_convention import validate_name -from rucio.db.sqla import models, filter_thread_work -from rucio.db.sqla.constants import DIDType, DIDReEvaluation, DIDAvailability, RuleState, BadFilesStatus -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.db.sqla import filter_thread_work, models +from rucio.db.sqla.constants import BadFilesStatus, DIDAvailability, DIDReEvaluation, DIDType, RuleState +from rucio.db.sqla.session import read_session, stream_session, transactional_session from rucio.db.sqla.util import temp_table_mngr if TYPE_CHECKING: from collections.abc import Iterator, Sequence from typing import Any, Optional, Union + from sqlalchemy.orm import Session from sqlalchemy.schema import Table + from rucio.common.types import InternalAccount, InternalScope, LoggerFunction diff --git a/lib/rucio/core/did_meta_plugins/did_column_meta.py b/lib/rucio/core/did_meta_plugins/did_column_meta.py index 76f42be4b7..02c9a80643 100644 --- a/lib/rucio/core/did_meta_plugins/did_column_meta.py +++ b/lib/rucio/core/did_meta_plugins/did_column_meta.py @@ -17,7 +17,7 @@ from datetime import datetime, timedelta from typing import TYPE_CHECKING -from sqlalchemy import update, inspect +from sqlalchemy import inspect, update from sqlalchemy.exc import CompileError, InvalidRequestError from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.sql import func @@ -29,10 +29,11 @@ from rucio.core.did_meta_plugins.filter_engine import FilterEngine from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType -from rucio.db.sqla.session import stream_session, read_session, transactional_session +from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session diff --git a/lib/rucio/core/did_meta_plugins/did_meta_plugin_interface.py b/lib/rucio/core/did_meta_plugins/did_meta_plugin_interface.py index 7bdc679af1..88b9043290 100644 --- a/lib/rucio/core/did_meta_plugins/did_meta_plugin_interface.py +++ b/lib/rucio/core/did_meta_plugins/did_meta_plugin_interface.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session diff --git a/lib/rucio/core/did_meta_plugins/filter_engine.py b/lib/rucio/core/did_meta_plugins/filter_engine.py index c817ecc444..db7d534949 100644 --- a/lib/rucio/core/did_meta_plugins/filter_engine.py +++ b/lib/rucio/core/did_meta_plugins/filter_engine.py @@ -16,12 +16,12 @@ import ast import fnmatch import operator -from datetime import datetime, timedelta, date +from datetime import date, datetime, timedelta from importlib import import_module from typing import TYPE_CHECKING import sqlalchemy -from sqlalchemy import cast, or_, and_ +from sqlalchemy import and_, cast, or_ from sqlalchemy.sql.expression import text from rucio.common import exception diff --git a/lib/rucio/core/did_meta_plugins/json_meta.py b/lib/rucio/core/did_meta_plugins/json_meta.py index 06cbb0ee62..3930c91bf8 100644 --- a/lib/rucio/core/did_meta_plugins/json_meta.py +++ b/lib/rucio/core/did_meta_plugins/json_meta.py @@ -15,7 +15,7 @@ import json as json_lib import operator -from typing import TYPE_CHECKING, cast, Any +from typing import TYPE_CHECKING, Any, cast from sqlalchemy.exc import DataError from sqlalchemy.orm.exc import NoResultFound @@ -25,7 +25,7 @@ from rucio.core.did_meta_plugins.filter_engine import FilterEngine from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.db.sqla.session import read_session, stream_session, transactional_session from rucio.db.sqla.util import json_implemented if TYPE_CHECKING: diff --git a/lib/rucio/core/did_meta_plugins/mongo_meta.py b/lib/rucio/core/did_meta_plugins/mongo_meta.py index 87a3162a9c..fe21464b5b 100644 --- a/lib/rucio/core/did_meta_plugins/mongo_meta.py +++ b/lib/rucio/core/did_meta_plugins/mongo_meta.py @@ -18,14 +18,14 @@ import pymongo -from rucio.common import config -from rucio.common import exception +from rucio.common import config, exception from rucio.common.types import InternalScope from rucio.core.did_meta_plugins.did_meta_plugin_interface import DidMetaPlugin from rucio.core.did_meta_plugins.filter_engine import FilterEngine if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session IMMUTABLE_KEYS = [ diff --git a/lib/rucio/core/did_meta_plugins/postgres_meta.py b/lib/rucio/core/did_meta_plugins/postgres_meta.py index 13184e90d9..a071332d3b 100644 --- a/lib/rucio/core/did_meta_plugins/postgres_meta.py +++ b/lib/rucio/core/did_meta_plugins/postgres_meta.py @@ -20,14 +20,14 @@ import psycopg2 import psycopg2.extras -from rucio.common import config -from rucio.common import exception +from rucio.common import config, exception from rucio.common.types import InternalScope from rucio.core.did_meta_plugins.did_meta_plugin_interface import DidMetaPlugin from rucio.core.did_meta_plugins.filter_engine import FilterEngine if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session diff --git a/lib/rucio/core/dirac.py b/lib/rucio/core/dirac.py index 05d5b4bf18..77bc43877c 100644 --- a/lib/rucio/core/dirac.py +++ b/lib/rucio/core/dirac.py @@ -21,8 +21,8 @@ from sqlalchemy.orm.exc import NoResultFound from rucio.common.config import config_get -from rucio.common.exception import InvalidType, UnsupportedOperation, ConfigNotFound, RucioException -from rucio.common.types import InternalScope, InternalAccount +from rucio.common.exception import ConfigNotFound, InvalidType, RucioException, UnsupportedOperation +from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import extract_scope from rucio.core.did import add_did, attach_dids_to_dids from rucio.core.replica import add_replicas @@ -30,7 +30,7 @@ from rucio.core.scope import list_scopes from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType -from rucio.db.sqla.session import transactional_session, read_session +from rucio.db.sqla.session import read_session, transactional_session if TYPE_CHECKING: from sqlalchemy.orm import Session diff --git a/lib/rucio/core/distance.py b/lib/rucio/core/distance.py index 54da9d5d05..a46a87ce2d 100644 --- a/lib/rucio/core/distance.py +++ b/lib/rucio/core/distance.py @@ -19,8 +19,8 @@ from sqlalchemy.orm import aliased from rucio.common import exception -from rucio.db.sqla.models import Distance, RSE -from rucio.db.sqla.session import transactional_session, read_session +from rucio.db.sqla.models import RSE, Distance +from rucio.db.sqla.session import read_session, transactional_session if TYPE_CHECKING: from sqlalchemy.orm import Session diff --git a/lib/rucio/core/exporter.py b/lib/rucio/core/exporter.py index a4863ab811..8e93d8cb80 100644 --- a/lib/rucio/core/exporter.py +++ b/lib/rucio/core/exporter.py @@ -15,7 +15,8 @@ from typing import TYPE_CHECKING -from rucio.core import rse as rse_module, distance as distance_module +from rucio.core import distance as distance_module +from rucio.core import rse as rse_module from rucio.db.sqla.session import transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/core/identity.py b/lib/rucio/core/identity.py index 895dac216f..e3da92a0eb 100644 --- a/lib/rucio/core/identity.py +++ b/lib/rucio/core/identity.py @@ -16,8 +16,7 @@ import hashlib import os from re import match -from typing import Optional -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import select, true from sqlalchemy.exc import IntegrityError diff --git a/lib/rucio/core/importer.py b/lib/rucio/core/importer.py index ced3ed7a66..63e741420f 100644 --- a/lib/rucio/core/importer.py +++ b/lib/rucio/core/importer.py @@ -18,9 +18,12 @@ from rucio.common.config import config_get from rucio.common.exception import RSEOperationNotSupported from rucio.common.types import InternalAccount -from rucio.core import rse as rse_module, distance as distance_module, account as account_module, identity as identity_module +from rucio.core import account as account_module +from rucio.core import distance as distance_module +from rucio.core import identity as identity_module +from rucio.core import rse as rse_module from rucio.db.sqla import models -from rucio.db.sqla.constants import RSEType, AccountType, IdentityType +from rucio.db.sqla.constants import AccountType, IdentityType, RSEType from rucio.db.sqla.session import transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/core/lifetime_exception.py b/lib/rucio/core/lifetime_exception.py index c09e3c4f30..4fd7064da6 100644 --- a/lib/rucio/core/lifetime_exception.py +++ b/lib/rucio/core/lifetime_exception.py @@ -24,20 +24,22 @@ import rucio.common.policy from rucio.common.config import config_get, config_get_int, config_get_list -from rucio.common.exception import RucioException, LifetimeExceptionDuplicate, LifetimeExceptionNotFound, UnsupportedOperation, ConfigNotFound +from rucio.common.exception import ConfigNotFound, LifetimeExceptionDuplicate, LifetimeExceptionNotFound, RucioException, UnsupportedOperation from rucio.common.utils import generate_uuid, str_to_date from rucio.core.message import add_message from rucio.core.rse import list_rse_attributes from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType, LifetimeExceptionsState -from rucio.db.sqla.session import transactional_session, stream_session, read_session +from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: from collections.abc import Iterator from typing import Any, Optional, Union - from rucio.common.types import InternalAccount, InternalScope + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount, InternalScope + @stream_session def list_exceptions( diff --git a/lib/rucio/core/lock.py b/lib/rucio/core/lock.py index 10b97b327c..cc1c27287e 100644 --- a/lib/rucio/core/lock.py +++ b/lib/rucio/core/lock.py @@ -26,9 +26,9 @@ from rucio.common.types import InternalScope from rucio.core.lifetime_exception import define_eol from rucio.core.rse import get_rse_attribute, get_rse_name -from rucio.db.sqla import models, filter_thread_work -from rucio.db.sqla.constants import LockState, RuleState, RuleGrouping, DIDType, RuleNotification -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.db.sqla import filter_thread_work, models +from rucio.db.sqla.constants import DIDType, LockState, RuleGrouping, RuleNotification, RuleState +from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: from sqlalchemy.orm import Session diff --git a/lib/rucio/core/message.py b/lib/rucio/core/message.py index 760c5a8dc5..fb5a00b84b 100644 --- a/lib/rucio/core/message.py +++ b/lib/rucio/core/message.py @@ -16,11 +16,11 @@ import json from typing import TYPE_CHECKING -from sqlalchemy import or_, delete, update, insert +from sqlalchemy import delete, insert, or_, update from sqlalchemy.exc import IntegrityError from rucio.common.config import config_get_list -from rucio.common.constants import HermesService, MAX_MESSAGE_LENGTH +from rucio.common.constants import MAX_MESSAGE_LENGTH, HermesService from rucio.common.exception import InvalidObject, RucioException from rucio.common.utils import APIEncoder, chunks from rucio.db.sqla import filter_thread_work @@ -29,6 +29,7 @@ if TYPE_CHECKING: from typing import Any, Optional + from sqlalchemy.orm import Session MessageType = dict[str, Any] diff --git a/lib/rucio/core/meta_conventions.py b/lib/rucio/core/meta_conventions.py index 1301de7644..14beb65a18 100644 --- a/lib/rucio/core/meta_conventions.py +++ b/lib/rucio/core/meta_conventions.py @@ -16,13 +16,11 @@ from re import match from typing import TYPE_CHECKING, Optional, Union -from sqlalchemy import select, and_ +from sqlalchemy import and_, select from sqlalchemy.exc import IntegrityError, NoResultFound from rucio.common.constraints import AUTHORIZED_VALUE_TYPES -from rucio.common.exception import (Duplicate, RucioException, - KeyNotFound, InvalidValueForKey, UnsupportedValueType, - InvalidObject, UnsupportedKeyType) +from rucio.common.exception import Duplicate, InvalidObject, InvalidValueForKey, KeyNotFound, RucioException, UnsupportedKeyType, UnsupportedValueType from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType, KeyType from rucio.db.sqla.session import read_session, transactional_session diff --git a/lib/rucio/core/monitor.py b/lib/rucio/core/monitor.py index c6a6a91f29..447f2b4920 100644 --- a/lib/rucio/core/monitor.py +++ b/lib/rucio/core/monitor.py @@ -17,7 +17,6 @@ Graphite and prometheus metrics """ -import __main__ as main import atexit import logging import os @@ -30,10 +29,10 @@ from threading import Lock from typing import Any, Optional, TypeVar, Union -from prometheus_client import (Counter, Gauge, Histogram, REGISTRY, CollectorRegistry, generate_latest, multiprocess, - push_to_gateway, start_http_server, values) +from prometheus_client import REGISTRY, CollectorRegistry, Counter, Gauge, Histogram, generate_latest, multiprocess, push_to_gateway, start_http_server, values from statsd import StatsClient +import __main__ as main from rucio.common.config import config_get, config_get_bool, config_get_int from rucio.common.stopwatch import Stopwatch from rucio.common.utils import retrying diff --git a/lib/rucio/core/naming_convention.py b/lib/rucio/core/naming_convention.py index d39c0efc6b..d0a82e51f9 100644 --- a/lib/rucio/core/naming_convention.py +++ b/lib/rucio/core/naming_convention.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from re import match, compile, error +from re import compile, error, match from traceback import format_exc from typing import TYPE_CHECKING @@ -21,7 +21,7 @@ from sqlalchemy.exc import IntegrityError from rucio.common.cache import make_region_memcached -from rucio.common.exception import Duplicate, RucioException, InvalidObject +from rucio.common.exception import Duplicate, InvalidObject, RucioException from rucio.db.sqla import models from rucio.db.sqla.constants import KeyType from rucio.db.sqla.session import read_session, transactional_session diff --git a/lib/rucio/core/oidc.py b/lib/rucio/core/oidc.py index 54c42a87b5..d85a0122e8 100644 --- a/lib/rucio/core/oidc.py +++ b/lib/rucio/core/oidc.py @@ -20,19 +20,18 @@ import subprocess import traceback from datetime import datetime, timedelta -from typing import Any, Final, TYPE_CHECKING, Optional -from urllib.parse import urljoin, urlparse, parse_qs +from math import floor +from typing import TYPE_CHECKING, Any, Final, Optional +from urllib.parse import parse_qs, urljoin, urlparse import requests from dogpile.cache.api import NoValue from jwkest.jws import JWS from jwkest.jwt import JWT -from math import floor from oic import rndstr from oic.oauth2.message import CCAccessTokenRequest -from oic.oic import Client, Grant, Token, REQUEST2ENDPOINT -from oic.oic.message import (AccessTokenResponse, AuthorizationResponse, - Message, RegistrationResponse) +from oic.oic import REQUEST2ENDPOINT, Client, Grant, Token +from oic.oic.message import AccessTokenResponse, AuthorizationResponse, Message, RegistrationResponse from oic.utils import time_util from oic.utils.authn.client import CLIENT_AUTHN_METHOD from sqlalchemy import delete, select, update @@ -41,15 +40,13 @@ from rucio.common import types from rucio.common.cache import make_region_memcached from rucio.common.config import config_get, config_get_int -from rucio.common.exception import (CannotAuthenticate, CannotAuthorize, - RucioException) +from rucio.common.exception import CannotAuthenticate, CannotAuthorize, RucioException from rucio.common.stopwatch import Stopwatch from rucio.common.utils import all_oidc_req_claims_present, build_url, val_to_space_sep_str from rucio.core.account import account_exists from rucio.core.identity import exist_identity_account, get_default_account from rucio.core.monitor import MetricManager -from rucio.db.sqla import filter_thread_work -from rucio.db.sqla import models +from rucio.db.sqla import filter_thread_work, models from rucio.db.sqla.constants import IdentityType from rucio.db.sqla.session import read_session, transactional_session diff --git a/lib/rucio/core/permission/__init__.py b/lib/rucio/core/permission/__init__.py index 69c94225a5..09c80195c9 100644 --- a/lib/rucio/core/permission/__init__.py +++ b/lib/rucio/core/permission/__init__.py @@ -23,6 +23,7 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session # dictionary of permission modules for each VO diff --git a/lib/rucio/core/permission/atlas.py b/lib/rucio/core/permission/atlas.py index 4acf2bcae6..165b5fb7ad 100644 --- a/lib/rucio/core/permission/atlas.py +++ b/lib/rucio/core/permission/atlas.py @@ -17,16 +17,18 @@ import rucio.core.did import rucio.core.scope -from rucio.core.account import list_account_attributes, has_account_attribute +from rucio.core.account import has_account_attribute, list_account_attributes from rucio.core.identity import exist_identity_account from rucio.core.rse import list_rse_attributes from rucio.core.rse_expression_parser import parse_expression from rucio.core.rule import get_rule -from rucio.db.sqla.constants import IdentityType, BadPFNStatus +from rucio.db.sqla.constants import BadPFNStatus, IdentityType if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount diff --git a/lib/rucio/core/permission/belleii.py b/lib/rucio/core/permission/belleii.py index 21e892a06f..89c8b1ad59 100644 --- a/lib/rucio/core/permission/belleii.py +++ b/lib/rucio/core/permission/belleii.py @@ -17,7 +17,7 @@ import rucio.core.scope from rucio.common.config import config_get -from rucio.common.types import InternalScope, InternalAccount +from rucio.common.types import InternalAccount, InternalScope from rucio.core.account import has_account_attribute, list_account_attributes from rucio.core.did import get_metadata from rucio.core.identity import exist_identity_account @@ -29,6 +29,7 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session diff --git a/lib/rucio/core/permission/cms.py b/lib/rucio/core/permission/cms.py index bb5667f559..f9799309ea 100644 --- a/lib/rucio/core/permission/cms.py +++ b/lib/rucio/core/permission/cms.py @@ -26,7 +26,9 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount diff --git a/lib/rucio/core/permission/escape.py b/lib/rucio/core/permission/escape.py index 068edee885..1f4c63fba2 100644 --- a/lib/rucio/core/permission/escape.py +++ b/lib/rucio/core/permission/escape.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING import rucio.core.scope -from rucio.core.account import list_account_attributes, has_account_attribute +from rucio.core.account import has_account_attribute, list_account_attributes from rucio.core.identity import exist_identity_account from rucio.core.lifetime_exception import list_exceptions from rucio.core.rse import list_rse_attributes @@ -25,7 +25,9 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount diff --git a/lib/rucio/core/permission/generic.py b/lib/rucio/core/permission/generic.py index 82730e1566..7ef601fce5 100644 --- a/lib/rucio/core/permission/generic.py +++ b/lib/rucio/core/permission/generic.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING import rucio.core.scope -from rucio.core.account import list_account_attributes, has_account_attribute +from rucio.core.account import has_account_attribute, list_account_attributes from rucio.core.identity import exist_identity_account from rucio.core.lifetime_exception import list_exceptions from rucio.core.rse import list_rse_attributes @@ -25,7 +25,9 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount diff --git a/lib/rucio/core/permission/generic_multi_vo.py b/lib/rucio/core/permission/generic_multi_vo.py index 4c05c538a6..80b7e3f81f 100644 --- a/lib/rucio/core/permission/generic_multi_vo.py +++ b/lib/rucio/core/permission/generic_multi_vo.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING import rucio.core.scope -from rucio.core.account import list_account_attributes, has_account_attribute +from rucio.core.account import has_account_attribute, list_account_attributes from rucio.core.identity import exist_identity_account from rucio.core.lifetime_exception import list_exceptions from rucio.core.rse import list_rse_attributes @@ -26,7 +26,9 @@ if TYPE_CHECKING: from typing import Optional + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount diff --git a/lib/rucio/core/quarantined_replica.py b/lib/rucio/core/quarantined_replica.py index edd9e8c13a..342f314958 100644 --- a/lib/rucio/core/quarantined_replica.py +++ b/lib/rucio/core/quarantined_replica.py @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections.abc import Iterable import datetime +from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Optional from sqlalchemy import and_, or_ from sqlalchemy.sql.expression import false, insert from rucio.common.utils import chunks -from rucio.db.sqla import models, filter_thread_work +from rucio.db.sqla import filter_thread_work, models from rucio.db.sqla.session import read_session, transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py index 61467e1dcf..8b55cbe444 100644 --- a/lib/rucio/core/replica.py +++ b/lib/rucio/core/replica.py @@ -15,6 +15,7 @@ import copy import heapq import logging +import math import random from collections import defaultdict, namedtuple from curses.ascii import isprint @@ -27,15 +28,14 @@ from traceback import format_exc from typing import TYPE_CHECKING -import math import requests from dogpile.cache.api import NO_VALUE -from sqlalchemy import func, and_, or_, exists, not_, update, delete, insert, union +from sqlalchemy import and_, delete, exists, func, insert, not_, or_, union, update from sqlalchemy.exc import DatabaseError, IntegrityError from sqlalchemy.orm import aliased from sqlalchemy.orm.exc import FlushError, NoResultFound from sqlalchemy.sql import label -from sqlalchemy.sql.expression import case, select, text, false, true, null, literal, literal_column +from sqlalchemy.sql.expression import case, false, literal, literal_column, null, select, text, true import rucio.core.did import rucio.core.lock @@ -44,27 +44,27 @@ from rucio.common.config import config_get, config_get_bool from rucio.common.constants import SuspiciousAvailability from rucio.common.types import InternalScope -from rucio.common.utils import chunks, clean_surls, str_to_date, add_url_query +from rucio.common.utils import add_url_query, chunks, clean_surls, str_to_date from rucio.core.credential import get_signed_url from rucio.core.message import add_messages from rucio.core.monitor import MetricManager -from rucio.core.rse import get_rse, get_rse_name, get_rse_attribute, get_rse_vo, list_rses +from rucio.core.rse import get_rse, get_rse_attribute, get_rse_name, get_rse_vo, list_rses from rucio.core.rse_counter import decrease, increase from rucio.core.rse_expression_parser import parse_expression -from rucio.db.sqla import models, filter_thread_work -from rucio.db.sqla.constants import (DIDType, ReplicaState, OBSOLETE, DIDAvailability, - BadFilesStatus, RuleState, BadPFNStatus) -from rucio.db.sqla.session import (read_session, stream_session, transactional_session, - DEFAULT_SCHEMA_NAME, BASE) +from rucio.db.sqla import filter_thread_work, models +from rucio.db.sqla.constants import OBSOLETE, BadFilesStatus, BadPFNStatus, DIDAvailability, DIDType, ReplicaState, RuleState +from rucio.db.sqla.session import BASE, DEFAULT_SCHEMA_NAME, read_session, stream_session, transactional_session from rucio.db.sqla.util import temp_table_mngr from rucio.rse import rsemanager as rsemgr if TYPE_CHECKING: from collections.abc import Iterator, Sequence - from rucio.rse.protocols.protocol import RSEProtocol from typing import Any, Optional + from sqlalchemy.orm import Session + from rucio.rse.protocols.protocol import RSEProtocol + REGION = make_region_memcached(expiration_time=60) METRICS = MetricManager(module=__name__) diff --git a/lib/rucio/core/replica_sorter.py b/lib/rucio/core/replica_sorter.py index c66f9a5c21..1309e31c0b 100644 --- a/lib/rucio/core/replica_sorter.py +++ b/lib/rucio/core/replica_sorter.py @@ -19,6 +19,7 @@ import tarfile from collections import OrderedDict from datetime import datetime, timedelta +from math import asin, cos, radians, sin, sqrt from pathlib import Path from tempfile import TemporaryDirectory, TemporaryFile from typing import TYPE_CHECKING, Union @@ -27,7 +28,6 @@ import geoip2.database import requests from dogpile.cache.api import NO_VALUE -from math import asin, cos, radians, sin, sqrt from rucio.common import utils from rucio.common.cache import make_region_memcached diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py index 4921783944..c59bdc1d2e 100644 --- a/lib/rucio/core/request.py +++ b/lib/rucio/core/request.py @@ -23,28 +23,28 @@ import traceback import uuid from abc import ABCMeta, abstractmethod -from collections import namedtuple, defaultdict -from collections.abc import Sequence, Mapping, Iterator +from collections import defaultdict, namedtuple +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional, Union -from sqlalchemy import and_, or_, update, select, delete, exists, insert +from sqlalchemy import and_, delete, exists, insert, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import aliased -from sqlalchemy.sql.expression import asc, true, false, null, func +from sqlalchemy.sql.expression import asc, false, func, null, true from rucio.common.config import config_get_bool, config_get_int -from rucio.common.exception import RequestNotFound, RucioException, UnsupportedOperation, InvalidRSEExpression +from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RucioException, UnsupportedOperation from rucio.common.types import InternalAccount, InternalScope -from rucio.common.utils import generate_uuid, chunks +from rucio.common.utils import chunks, generate_uuid from rucio.core.distance import get_distances from rucio.core.message import add_message, add_messages from rucio.core.monitor import MetricManager -from rucio.core.rse import get_rse_attribute, get_rse_name, get_rse_vo, RseData, RseCollection +from rucio.core.rse import RseCollection, RseData, get_rse_attribute, get_rse_name, get_rse_vo from rucio.core.rse_expression_parser import parse_expression -from rucio.db.sqla import models, filter_thread_work -from rucio.db.sqla.constants import RequestState, RequestType, LockState, RequestErrMsg, ReplicaState, TransferLimitDirection -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.db.sqla import filter_thread_work, models +from rucio.db.sqla.constants import LockState, ReplicaState, RequestErrMsg, RequestState, RequestType, TransferLimitDirection +from rucio.db.sqla.session import read_session, stream_session, transactional_session from rucio.db.sqla.util import temp_table_mngr RequestAndState = namedtuple('RequestAndState', ['request_id', 'request_state']) @@ -52,6 +52,7 @@ if TYPE_CHECKING: from sqlalchemy.orm import Session + from rucio.rse.protocols.protocol import RSEProtocol """ diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py index d508c93960..dd4c6415b9 100644 --- a/lib/rucio/core/rse.py +++ b/lib/rucio/core/rse.py @@ -14,28 +14,27 @@ # limitations under the License. import json -from collections.abc import Iterator, Iterable +from collections.abc import Iterable, Iterator from datetime import datetime from io import StringIO from re import match -from typing import Any, Generic, Optional, TypeVar, Union, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union import sqlalchemy from dogpile.cache.api import NO_VALUE from sqlalchemy.exc import DatabaseError, IntegrityError, OperationalError from sqlalchemy.orm import aliased from sqlalchemy.orm.exc import FlushError -from sqlalchemy.sql.expression import or_, and_, desc, true, false, func, select, delete +from sqlalchemy.sql.expression import and_, delete, desc, false, func, or_, select, true -from rucio.common import exception, utils +from rucio.common import exception, types, utils from rucio.common.cache import make_region_memcached from rucio.common.config import get_lfn2pfn_algorithm_default -from rucio.common import types from rucio.common.utils import CHECKSUM_KEY, GLOBALLY_SUPPORTED_CHECKSUMS, Availability from rucio.core.rse_counter import add_counter, get_counter from rucio.db.sqla import models -from rucio.db.sqla.constants import RSEType, ReplicaState -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.db.sqla.constants import ReplicaState, RSEType +from rucio.db.sqla.session import read_session, stream_session, transactional_session from rucio.db.sqla.util import temp_table_mngr if TYPE_CHECKING: diff --git a/lib/rucio/core/rse_counter.py b/lib/rucio/core/rse_counter.py index 9c3d01d320..cbedecca25 100644 --- a/lib/rucio/core/rse_counter.py +++ b/lib/rucio/core/rse_counter.py @@ -17,7 +17,7 @@ from sqlalchemy.orm.exc import NoResultFound from rucio.common.exception import CounterNotFound -from rucio.db.sqla import models, filter_thread_work +from rucio.db.sqla import filter_thread_work, models from rucio.db.sqla.session import read_session, transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/core/rse_expression_parser.py b/lib/rucio/core/rse_expression_parser.py index 5a87d937fd..b255dbd17d 100644 --- a/lib/rucio/core/rse_expression_parser.py +++ b/lib/rucio/core/rse_expression_parser.py @@ -22,7 +22,7 @@ from rucio.common.cache import make_region_memcached from rucio.common.exception import InvalidRSEExpression, RSEWriteBlocked -from rucio.core.rse import list_rses, get_rses_with_attribute, get_rse_attribute +from rucio.core.rse import get_rse_attribute, get_rses_with_attribute, list_rses from rucio.db.sqla.session import transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/core/rse_selector.py b/lib/rucio/core/rse_selector.py index 0c016408c9..48b123abc8 100644 --- a/lib/rucio/core/rse_selector.py +++ b/lib/rucio/core/rse_selector.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from random import uniform, shuffle +from random import shuffle, uniform from typing import TYPE_CHECKING from rucio.common.exception import InsufficientAccountLimit, InsufficientTargetRSEs, InvalidRuleWeight, RSEOverQuota -from rucio.core.account import has_account_attribute, get_usage, get_all_rse_usages_per_account -from rucio.core.account_limit import get_local_account_limit, get_global_account_limits -from rucio.core.rse import list_rse_attributes, has_rse_attribute, get_rse_limits +from rucio.core.account import get_all_rse_usages_per_account, get_usage, has_account_attribute +from rucio.core.account_limit import get_global_account_limits, get_local_account_limit +from rucio.core.rse import get_rse_limits, has_rse_attribute, list_rse_attributes from rucio.core.rse_counter import get_counter as get_rse_counter from rucio.core.rse_expression_parser import parse_expression from rucio.db.sqla.session import read_session diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py index 220e724928..e16d5c84f9 100644 --- a/lib/rucio/core/rule.py +++ b/lib/rucio/core/rule.py @@ -26,43 +26,62 @@ from dogpile.cache.api import NoValue from sqlalchemy import delete, desc, select, update -from sqlalchemy.exc import IntegrityError, StatementError -from sqlalchemy.exc import NoResultFound # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.exc.NoResultFound.html +from sqlalchemy.exc import ( + IntegrityError, + NoResultFound, # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.exc.NoResultFound.html + StatementError, +) from sqlalchemy.sql import func -from sqlalchemy.sql.expression import and_, or_, true, null, tuple_, false +from sqlalchemy.sql.expression import and_, false, null, or_, true, tuple_ import rucio.core.did import rucio.core.lock # import get_replica_locks, get_files_and_replica_locks_of_dataset import rucio.core.replica # import get_and_lock_file_replicas, get_and_lock_file_replicas_for_dataset from rucio.common.cache import make_region_memcached from rucio.common.config import config_get -from rucio.common.exception import (InvalidRSEExpression, InvalidReplicationRule, InsufficientAccountLimit, - DataIdentifierNotFound, RuleNotFound, InputValidationError, RSEOverQuota, - ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, RucioException, - InvalidRuleWeight, StagingAreaRuleRequiresLifetime, DuplicateRule, - InvalidObject, RSEWriteBlocked, RuleReplaceFailed, RequestNotFound, - ManualRuleApprovalBlocked, UnsupportedOperation, UndefinedPolicy, InvalidValueForKey, - InvalidSourceReplicaExpression) +from rucio.common.exception import ( + DataIdentifierNotFound, + DuplicateRule, + InputValidationError, + InsufficientAccountLimit, + InsufficientTargetRSEs, + InvalidObject, + InvalidReplicationRule, + InvalidRSEExpression, + InvalidRuleWeight, + InvalidSourceReplicaExpression, + InvalidValueForKey, + ManualRuleApprovalBlocked, + ReplicationRuleCreationTemporaryFailed, + RequestNotFound, + RSEOverQuota, + RSEWriteBlocked, + RucioException, + RuleNotFound, + RuleReplaceFailed, + StagingAreaRuleRequiresLifetime, + UndefinedPolicy, + UnsupportedOperation, +) from rucio.common.plugins import PolicyPackageAlgorithms -from rucio.common.policy import policy_filter, get_scratchdisk_lifetime +from rucio.common.policy import get_scratchdisk_lifetime, policy_filter from rucio.common.schema import validate_schema -from rucio.common.types import InternalScope, InternalAccount, LoggerFunction, DIDDict, RuleDict -from rucio.common.utils import str_to_date, sizefmt, chunks -from rucio.core import account_counter, rse_counter, request as request_core, transfer as transfer_core -from rucio.core.account import get_account -from rucio.core.account import has_account_attribute +from rucio.common.types import DIDDict, InternalAccount, InternalScope, LoggerFunction, RuleDict +from rucio.common.utils import chunks, sizefmt, str_to_date +from rucio.core import account_counter, rse_counter +from rucio.core import request as request_core +from rucio.core import transfer as transfer_core +from rucio.core.account import get_account, has_account_attribute from rucio.core.lifetime_exception import define_eol from rucio.core.message import add_message from rucio.core.monitor import MetricManager -from rucio.core.rse import get_rse_name, list_rse_attributes, get_rse, get_rse_usage +from rucio.core.rse import get_rse, get_rse_name, get_rse_usage, list_rse_attributes from rucio.core.rse_expression_parser import parse_expression from rucio.core.rse_selector import RSESelector -from rucio.core.rule_grouping import apply_rule_grouping, repair_stuck_locks_and_apply_rule_grouping, create_transfer_dict, apply_rule -from rucio.db.sqla import models, filter_thread_work -from rucio.db.sqla.constants import (LockState, ReplicaState, RuleState, RuleGrouping, - DIDAvailability, DIDReEvaluation, DIDType, BadFilesStatus, - RequestType, RuleNotification, OBSOLETE, RSEType) -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.core.rule_grouping import apply_rule, apply_rule_grouping, create_transfer_dict, repair_stuck_locks_and_apply_rule_grouping +from rucio.db.sqla import filter_thread_work, models +from rucio.db.sqla.constants import OBSOLETE, BadFilesStatus, DIDAvailability, DIDReEvaluation, DIDType, LockState, ReplicaState, RequestType, RSEType, RuleGrouping, RuleNotification, RuleState +from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: from sqlalchemy.orm import Session diff --git a/lib/rucio/core/rule_grouping.py b/lib/rucio/core/rule_grouping.py index b295020ee3..2e7b5615ad 100644 --- a/lib/rucio/core/rule_grouping.py +++ b/lib/rucio/core/rule_grouping.py @@ -27,11 +27,12 @@ from rucio.common.config import config_get_int from rucio.common.exception import InsufficientTargetRSEs from rucio.common.types import InternalScope -from rucio.core import account_counter, rse_counter, request as request_core -from rucio.core.rse_selector import RSESelector +from rucio.core import account_counter, rse_counter +from rucio.core import request as request_core from rucio.core.rse import get_rse, get_rse_attribute, get_rse_name +from rucio.core.rse_selector import RSESelector from rucio.db.sqla import models -from rucio.db.sqla.constants import LockState, RuleGrouping, ReplicaState, RequestType, DIDType, OBSOLETE +from rucio.db.sqla.constants import OBSOLETE, DIDType, LockState, ReplicaState, RequestType, RuleGrouping from rucio.db.sqla.session import transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/core/subscription.py b/lib/rucio/core/subscription.py index 3fcfe9ee0a..9df8d954c8 100644 --- a/lib/rucio/core/subscription.py +++ b/lib/rucio/core/subscription.py @@ -26,15 +26,17 @@ from sqlalchemy.orm.exc import NoResultFound from rucio.common.config import config_get -from rucio.common.exception import SubscriptionNotFound, SubscriptionDuplicate, RucioException +from rucio.common.exception import RucioException, SubscriptionDuplicate, SubscriptionNotFound from rucio.db.sqla import models from rucio.db.sqla.constants import SubscriptionState -from rucio.db.sqla.session import transactional_session, stream_session, read_session +from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: from collections.abc import Callable, Iterator from typing import Any, Optional + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount LoggerFunction = Callable[..., Any] SubscriptionType = dict diff --git a/lib/rucio/core/topology.py b/lib/rucio/core/topology.py index 160225e208..6192fe7864 100644 --- a/lib/rucio/core/topology.py +++ b/lib/rucio/core/topology.py @@ -20,12 +20,12 @@ import weakref from collections.abc import Callable, Iterable, Iterator from decimal import Decimal -from typing import TYPE_CHECKING, cast, Any, Generic, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast from sqlalchemy import and_, select -from rucio.common.config import config_get_int, config_get -from rucio.common.exception import NoDistance, RSEProtocolNotSupported, InvalidRSEExpression +from rucio.common.config import config_get, config_get_int +from rucio.common.exception import InvalidRSEExpression, NoDistance, RSEProtocolNotSupported from rucio.common.utils import PriorityQueue from rucio.core.rse import RseCollection, RseData from rucio.core.rse_expression_parser import parse_expression @@ -39,10 +39,10 @@ TE = TypeVar("TE", bound="Edge") if TYPE_CHECKING: - from sqlalchemy.orm import Session - from typing import Protocol + from sqlalchemy.orm import Session + class _StateProvider(Protocol): @property def cost(self) -> _Number: diff --git a/lib/rucio/core/trace.py b/lib/rucio/core/trace.py index ca92f597ed..0cd0bc0fd5 100644 --- a/lib/rucio/core/trace.py +++ b/lib/rucio/core/trace.py @@ -24,12 +24,12 @@ import socket import stomp -from jsonschema import validate, ValidationError, Draft7Validator +from jsonschema import Draft7Validator, ValidationError, validate from rucio.common.config import config_get, config_get_int from rucio.common.exception import InvalidObject from rucio.common.logging import rucio_log_formatter -from rucio.common.schema.generic import UUID, TIME_ENTRY, IPv4orIPv6 +from rucio.common.schema.generic import TIME_ENTRY, UUID, IPv4orIPv6 from rucio.core.monitor import MetricManager METRICS = MetricManager(module=__name__) diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py index 905b12ebe9..927ef7e95f 100644 --- a/lib/rucio/core/transfer.py +++ b/lib/rucio/core/transfer.py @@ -31,30 +31,32 @@ from rucio.common import constants from rucio.common.config import config_get, config_get_list from rucio.common.constants import SUPPORTED_PROTOCOLS -from rucio.common.exception import (InvalidRSEExpression, - RequestNotFound, RSEProtocolNotSupported, - RucioException, UnsupportedOperation) +from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RSEProtocolNotSupported, RucioException, UnsupportedOperation from rucio.common.utils import construct_surl -from rucio.core import did, message as message_core, request as request_core +from rucio.core import did +from rucio.core import message as message_core +from rucio.core import request as request_core from rucio.core.account import list_accounts from rucio.core.monitor import MetricManager -from rucio.core.request import transition_request_state, RequestWithSources, RequestSource, TransferDestination, DirectTransfer +from rucio.core.request import DirectTransfer, RequestSource, RequestWithSources, TransferDestination, transition_request_state from rucio.core.rse import RseData from rucio.core.rse_expression_parser import parse_expression from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType, RequestState, RequestType, TransferLimitDirection -from rucio.db.sqla.session import read_session, transactional_session, stream_session +from rucio.db.sqla.session import read_session, stream_session, transactional_session from rucio.rse import rsemanager as rsemgr -from rucio.transfertool.transfertool import TransferStatusReport, Transfertool from rucio.transfertool.bittorrent import BittorrentTransfertool from rucio.transfertool.fts3 import FTS3Transfertool from rucio.transfertool.globus import GlobusTransferTool from rucio.transfertool.mock import MockTransfertool +from rucio.transfertool.transfertool import TransferStatusReport, Transfertool if TYPE_CHECKING: - from collections.abc import Callable, Iterator, Iterable, Mapping, Sequence + from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from typing import Any, Optional, Type + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount from rucio.core.topology import Topology from rucio.rse.protocols.protocol import RSEProtocol diff --git a/lib/rucio/core/vo.py b/lib/rucio/core/vo.py index 46c3f5ac0e..6240421090 100644 --- a/lib/rucio/core/vo.py +++ b/lib/rucio/core/vo.py @@ -20,7 +20,7 @@ from sqlalchemy.orm.exc import NoResultFound from rucio.common import exception -from rucio.common.config import config_get_bool, config_get +from rucio.common.config import config_get, config_get_bool from rucio.common.types import InternalAccount from rucio.db.sqla import models from rucio.db.sqla.constants import AccountType, IdentityType diff --git a/lib/rucio/core/volatile_replica.py b/lib/rucio/core/volatile_replica.py index 3316250721..b5c78dd331 100644 --- a/lib/rucio/core/volatile_replica.py +++ b/lib/rucio/core/volatile_replica.py @@ -17,7 +17,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any -from sqlalchemy import and_, or_, exists, update, insert +from sqlalchemy import and_, exists, insert, or_, update from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.sql.expression import select diff --git a/lib/rucio/daemons/abacus/account.py b/lib/rucio/daemons/abacus/account.py index 78de5d45c4..6055188d78 100644 --- a/lib/rucio/daemons/abacus/account.py +++ b/lib/rucio/daemons/abacus/account.py @@ -26,7 +26,7 @@ from rucio.common import exception from rucio.common.logging import setup_logging from rucio.common.utils import get_thread_with_periodic_running_function -from rucio.core.account_counter import get_updated_account_counters, update_account_counter, fill_account_counter_history_table +from rucio.core.account_counter import fill_account_counter_history_table, get_updated_account_counters, update_account_counter from rucio.daemons.common import run_daemon if TYPE_CHECKING: diff --git a/lib/rucio/daemons/abacus/rse.py b/lib/rucio/daemons/abacus/rse.py index 76da776ebd..f4fd4f0eeb 100644 --- a/lib/rucio/daemons/abacus/rse.py +++ b/lib/rucio/daemons/abacus/rse.py @@ -26,7 +26,7 @@ from rucio.common import exception from rucio.common.logging import setup_logging from rucio.common.utils import get_thread_with_periodic_running_function -from rucio.core.rse_counter import get_updated_rse_counters, update_rse_counter, fill_rse_counter_history_table +from rucio.core.rse_counter import fill_rse_counter_history_table, get_updated_rse_counters, update_rse_counter from rucio.daemons.common import run_daemon if TYPE_CHECKING: diff --git a/lib/rucio/daemons/auditor/__init__.py b/lib/rucio/daemons/auditor/__init__.py index d60be16492..f08d7bb918 100644 --- a/lib/rucio/daemons/auditor/__init__.py +++ b/lib/rucio/daemons/auditor/__init__.py @@ -18,21 +18,17 @@ import logging import os import queue as Queue -from datetime import datetime -from datetime import timedelta - import select +from datetime import datetime, timedelta from rucio.common import config -from rucio.common.dumper import LogPipeHandler -from rucio.common.dumper import mkdir -from rucio.common.dumper import temp_file +from rucio.common.dumper import LogPipeHandler, mkdir, temp_file from rucio.common.dumper.consistency import Consistency from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import chunks from rucio.core.quarantined_replica import add_quarantined_replicas from rucio.core.replica import declare_bad_file_replicas, list_replicas -from rucio.core.rse import get_rse_usage, get_rse_id +from rucio.core.rse import get_rse_id, get_rse_usage from rucio.daemons.auditor import srmdumps from rucio.daemons.auditor.hdfs import ReplicaFromHDFS from rucio.db.sqla.constants import BadFilesStatus diff --git a/lib/rucio/daemons/auditor/hdfs.py b/lib/rucio/daemons/auditor/hdfs.py index 4c44efe866..44e2075f8a 100644 --- a/lib/rucio/daemons/auditor/hdfs.py +++ b/lib/rucio/daemons/auditor/hdfs.py @@ -21,8 +21,7 @@ import subprocess import tempfile -from rucio.common.dumper import DUMPS_CACHE_DIR -from rucio.common.dumper import temp_file +from rucio.common.dumper import DUMPS_CACHE_DIR, temp_file from rucio.common.dumper.data_models import Replica diff --git a/lib/rucio/daemons/auditor/srmdumps.py b/lib/rucio/daemons/auditor/srmdumps.py index 5b63894718..42d72a97c0 100644 --- a/lib/rucio/daemons/auditor/srmdumps.py +++ b/lib/rucio/daemons/auditor/srmdumps.py @@ -27,8 +27,7 @@ import requests from rucio.common.config import get_config_dirs -from rucio.common.dumper import DUMPS_CACHE_DIR -from rucio.common.dumper import http_download_to_file, gfal_download_to_file, ddmendpoint_url, temp_file +from rucio.common.dumper import DUMPS_CACHE_DIR, ddmendpoint_url, gfal_download_to_file, http_download_to_file, temp_file CHUNK_SIZE = 10485760 diff --git a/lib/rucio/daemons/automatix/automatix.py b/lib/rucio/daemons/automatix/automatix.py index 99b4d9a2ed..7cff2e5432 100644 --- a/lib/rucio/daemons/automatix/automatix.py +++ b/lib/rucio/daemons/automatix/automatix.py @@ -28,7 +28,7 @@ from rucio.client import Client from rucio.client.uploadclient import UploadClient from rucio.common import exception -from rucio.common.config import config_get, config_get_int, config_get_bool +from rucio.common.config import config_get, config_get_bool, config_get_int from rucio.common.logging import setup_logging from rucio.common.stopwatch import Stopwatch from rucio.common.types import InternalScope diff --git a/lib/rucio/daemons/badreplicas/minos.py b/lib/rucio/daemons/badreplicas/minos.py index 4c7640fa8f..0549b43b2c 100644 --- a/lib/rucio/daemons/badreplicas/minos.py +++ b/lib/rucio/daemons/badreplicas/minos.py @@ -15,24 +15,22 @@ import functools import logging +import math import re import threading from collections.abc import Callable from datetime import datetime from typing import TYPE_CHECKING -import math from sqlalchemy.exc import DatabaseError import rucio.db.sqla.util from rucio.common.config import config_get_int -from rucio.common.exception import UnsupportedOperation, DataIdentifierNotFound, ReplicaNotFound, DatabaseException +from rucio.common.exception import DatabaseException, DataIdentifierNotFound, ReplicaNotFound, UnsupportedOperation from rucio.common.logging import setup_logging from rucio.common.utils import chunks from rucio.core.did import get_metadata -from rucio.core.replica import (get_bad_pfns, get_pfn_to_rse, declare_bad_file_replicas, - get_did_from_pfns, update_replicas_states, bulk_add_bad_replicas, - bulk_delete_bad_pfns, get_replicas_state) +from rucio.core.replica import bulk_add_bad_replicas, bulk_delete_bad_pfns, declare_bad_file_replicas, get_bad_pfns, get_did_from_pfns, get_pfn_to_rse, get_replicas_state, update_replicas_states from rucio.core.rse import get_rse_name from rucio.daemons.common import run_daemon from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, BadPFNStatus, ReplicaState @@ -42,8 +40,8 @@ from types import FrameType from typing import Optional - from rucio.daemons.common import HeartbeatHandler from rucio.common.types import InternalAccount + from rucio.daemons.common import HeartbeatHandler graceful_stop = threading.Event() DAEMON_NAME = 'minos' diff --git a/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py b/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py index 1deecd9e5d..4239b93aa5 100644 --- a/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py +++ b/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py @@ -15,22 +15,21 @@ import functools import logging +import math import re import threading import traceback from typing import TYPE_CHECKING -import math from sqlalchemy.exc import DatabaseError import rucio.db.sqla.util from rucio.common import exception -from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound, DatabaseException +from rucio.common.exception import DatabaseException, DataIdentifierNotFound, ReplicaNotFound from rucio.common.logging import setup_logging from rucio.common.utils import chunks from rucio.core.did import get_metadata -from rucio.core.replica import (update_replicas_states, get_replicas_state, - bulk_delete_bad_replicas, list_expired_temporary_unavailable_replicas) +from rucio.core.replica import bulk_delete_bad_replicas, get_replicas_state, list_expired_temporary_unavailable_replicas, update_replicas_states from rucio.daemons.common import run_daemon from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, ReplicaState from rucio.db.sqla.session import get_session diff --git a/lib/rucio/daemons/badreplicas/necromancer.py b/lib/rucio/daemons/badreplicas/necromancer.py index 8c44ef14cd..a085dfee8a 100644 --- a/lib/rucio/daemons/badreplicas/necromancer.py +++ b/lib/rucio/daemons/badreplicas/necromancer.py @@ -31,9 +31,8 @@ from rucio.common.exception import DatabaseException from rucio.common.logging import setup_logging from rucio.core.monitor import MetricManager -from rucio.core.replica import list_bad_replicas, get_replicas_state, get_bad_replicas_backlog -from rucio.core.rule import (update_rules_for_lost_replica, update_rules_for_bad_replica, - get_evaluation_backlog) +from rucio.core.replica import get_bad_replicas_backlog, get_replicas_state, list_bad_replicas +from rucio.core.rule import get_evaluation_backlog, update_rules_for_bad_replica, update_rules_for_lost_replica from rucio.daemons.common import run_daemon from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ReplicaState diff --git a/lib/rucio/daemons/bb8/bb8.py b/lib/rucio/daemons/bb8/bb8.py index 708cc38e93..b37dd99611 100644 --- a/lib/rucio/daemons/bb8/bb8.py +++ b/lib/rucio/daemons/bb8/bb8.py @@ -26,10 +26,10 @@ from rucio.common.config import config_get_float from rucio.common.exception import InvalidRSEExpression from rucio.common.logging import setup_logging -from rucio.core.heartbeat import sanity_check, list_payload_counts +from rucio.core.heartbeat import list_payload_counts, sanity_check from rucio.core.rse import get_rse_usage from rucio.core.rse_expression_parser import parse_expression -from rucio.daemons.bb8.common import rebalance_rse, get_active_locks +from rucio.daemons.bb8.common import get_active_locks, rebalance_rse from rucio.daemons.common import run_daemon if TYPE_CHECKING: diff --git a/lib/rucio/daemons/bb8/common.py b/lib/rucio/daemons/bb8/common.py index f15544f005..913a2a27ce 100644 --- a/lib/rucio/daemons/bb8/common.py +++ b/lib/rucio/daemons/bb8/common.py @@ -14,30 +14,30 @@ # limitations under the License. import logging -from datetime import datetime, date, timedelta +from datetime import date, datetime, timedelta from string import Template from requests import get -from sqlalchemy import func, and_, or_, cast, BigInteger +from sqlalchemy import BigInteger, and_, cast, func, or_ from sqlalchemy.orm import aliased from sqlalchemy.sql.expression import case, select -from rucio.common.config import config_get, config_get_int, config_get_bool +from rucio.common.config import config_get, config_get_bool, config_get_int from rucio.common.exception import ( - InsufficientTargetRSEs, - RuleNotFound, DuplicateRule, InsufficientAccountLimit, + InsufficientTargetRSEs, + RuleNotFound, ) from rucio.common.types import InternalAccount, InternalScope from rucio.core.lock import get_dataset_locks -from rucio.core.rse import list_rse_attributes, get_rse_name, get_rse_vo +from rucio.core.rse import get_rse_name, get_rse_vo, list_rse_attributes from rucio.core.rse_expression_parser import parse_expression from rucio.core.rse_selector import RSESelector -from rucio.core.rule import get_rule, add_rule, update_rule +from rucio.core.rule import add_rule, get_rule, update_rule from rucio.db.sqla import models -from rucio.db.sqla.constants import DIDType, RuleState, RuleGrouping, LockState -from rucio.db.sqla.session import transactional_session, read_session +from rucio.db.sqla.constants import DIDType, LockState, RuleGrouping, RuleState +from rucio.db.sqla.session import read_session, transactional_session @transactional_session diff --git a/lib/rucio/daemons/bb8/nuclei_background_rebalance.py b/lib/rucio/daemons/bb8/nuclei_background_rebalance.py index 1c621ab174..22bef087f0 100644 --- a/lib/rucio/daemons/bb8/nuclei_background_rebalance.py +++ b/lib/rucio/daemons/bb8/nuclei_background_rebalance.py @@ -19,7 +19,7 @@ from sqlalchemy import or_ -from rucio.core.rse import get_rse_usage, get_rse_attribute +from rucio.core.rse import get_rse_attribute, get_rse_usage from rucio.core.rse_expression_parser import parse_expression from rucio.daemons.bb8.common import rebalance_rse from rucio.db.sqla import models diff --git a/lib/rucio/daemons/bb8/t2_background_rebalance.py b/lib/rucio/daemons/bb8/t2_background_rebalance.py index ced702cea1..efeeffd7ee 100644 --- a/lib/rucio/daemons/bb8/t2_background_rebalance.py +++ b/lib/rucio/daemons/bb8/t2_background_rebalance.py @@ -19,7 +19,7 @@ from sqlalchemy import or_ -from rucio.core.rse import get_rse_usage, get_rse_attribute +from rucio.core.rse import get_rse_attribute, get_rse_usage from rucio.core.rse_expression_parser import parse_expression from rucio.daemons.bb8.common import rebalance_rse from rucio.db.sqla import models diff --git a/lib/rucio/daemons/c3po/algorithms/t2_free_space.py b/lib/rucio/daemons/c3po/algorithms/t2_free_space.py index 286794d031..f23e64dec0 100644 --- a/lib/rucio/daemons/c3po/algorithms/t2_free_space.py +++ b/lib/rucio/daemons/c3po/algorithms/t2_free_space.py @@ -20,7 +20,7 @@ from rucio.common.exception import DataIdentifierNotFound from rucio.core.did import get_did from rucio.core.replica import list_dataset_replicas -from rucio.core.rse import list_rse_attributes, get_rse_name +from rucio.core.rse import get_rse_name, list_rse_attributes from rucio.core.rse_expression_parser import parse_expression from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector from rucio.daemons.c3po.utils.dataset_cache import DatasetCache diff --git a/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py b/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py index 5679d7ba09..c4d6db810a 100644 --- a/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +++ b/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py @@ -20,7 +20,7 @@ from rucio.common.exception import DataIdentifierNotFound from rucio.core.did import get_did from rucio.core.replica import list_dataset_replicas -from rucio.core.rse import list_rse_attributes, get_rse_name +from rucio.core.rse import get_rse_name, list_rse_attributes from rucio.core.rse_expression_parser import parse_expression from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector from rucio.daemons.c3po.utils.expiring_dataset_cache import ExpiringDatasetCache diff --git a/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py b/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py index 7948c7b553..212e20fb70 100644 --- a/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +++ b/lib/rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py @@ -20,7 +20,7 @@ from rucio.common.exception import DataIdentifierNotFound from rucio.core.did import get_did from rucio.core.replica import list_dataset_replicas -from rucio.core.rse import list_rse_attributes, get_rse, get_rse_name +from rucio.core.rse import get_rse, get_rse_name, list_rse_attributes from rucio.core.rse_expression_parser import parse_expression from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector from rucio.daemons.c3po.collectors.network_metrics import NetworkMetricsCollector diff --git a/lib/rucio/daemons/c3po/collectors/free_space.py b/lib/rucio/daemons/c3po/collectors/free_space.py index d7696ba422..b2ff1c5dcb 100644 --- a/lib/rucio/daemons/c3po/collectors/free_space.py +++ b/lib/rucio/daemons/c3po/collectors/free_space.py @@ -17,7 +17,7 @@ Collector to get the SRM free and used information for DATADISK RSEs. """ -from rucio.db.sqla.models import RSEUsage, RSEAttrAssociation +from rucio.db.sqla.models import RSEAttrAssociation, RSEUsage from rucio.db.sqla.session import read_session diff --git a/lib/rucio/daemons/conveyor/common.py b/lib/rucio/daemons/conveyor/common.py index 26b1723ec0..bd23218ee5 100644 --- a/lib/rucio/daemons/conveyor/common.py +++ b/lib/rucio/daemons/conveyor/common.py @@ -25,31 +25,33 @@ from typing import TYPE_CHECKING from rucio.common.config import config_get_bool -from rucio.common.exception import (InvalidRSEExpression, TransferToolTimeout, TransferToolWrongAnswer, RequestNotFound, - DuplicateFileTransferSubmission, VONotFound, DatabaseException) +from rucio.common.exception import DatabaseException, DuplicateFileTransferSubmission, InvalidRSEExpression, RequestNotFound, TransferToolTimeout, TransferToolWrongAnswer, VONotFound from rucio.common.stopwatch import Stopwatch -from rucio.core import request as request_core, transfer as transfer_core +from rucio.core import request as request_core +from rucio.core import transfer as transfer_core from rucio.core.monitor import MetricManager from rucio.core.replica import add_replicas, tombstone_from_delay, update_replica_state -from rucio.core.request import transition_request_state, queue_requests +from rucio.core.request import queue_requests, transition_request_state from rucio.core.rse import list_rses from rucio.core.rse_expression_parser import parse_expression from rucio.core.transfer import build_transfer_paths from rucio.core.vo import list_vos from rucio.db.sqla import models -from rucio.db.sqla.constants import RequestState, ReplicaState +from rucio.db.sqla.constants import ReplicaState, RequestState from rucio.db.sqla.session import transactional_session from rucio.rse import rsemanager as rsemgr if TYPE_CHECKING: from collections.abc import Callable, Sequence - from typing import Optional, Mapping + from typing import Mapping, Optional + + from sqlalchemy.orm import Session + from rucio.common.types import InternalAccount from rucio.core.request import DirectTransfer, RequestWithSources from rucio.core.topology import Topology from rucio.core.transfer import ProtocolFactory from rucio.transfertool.transfertool import TransferToolBuilder - from sqlalchemy.orm import Session METRICS = MetricManager(module=__name__) diff --git a/lib/rucio/daemons/conveyor/finisher.py b/lib/rucio/daemons/conveyor/finisher.py index e4708eb7ef..1dc3b09195 100644 --- a/lib/rucio/daemons/conveyor/finisher.py +++ b/lib/rucio/daemons/conveyor/finisher.py @@ -32,19 +32,20 @@ import rucio.db.sqla.util from rucio.common.cache import make_region_memcached -from rucio.common.config import config_get_list, config_get_bool -from rucio.common.exception import DatabaseException, UnsupportedOperation, ReplicaNotFound, RequestNotFound, RSEProtocolNotSupported +from rucio.common.config import config_get_bool, config_get_list +from rucio.common.exception import DatabaseException, ReplicaNotFound, RequestNotFound, RSEProtocolNotSupported, UnsupportedOperation from rucio.common.logging import setup_logging from rucio.common.stopwatch import Stopwatch from rucio.common.types import InternalAccount from rucio.common.utils import chunks -from rucio.core import request as request_core, replica as replica_core +from rucio.core import replica as replica_core +from rucio.core import request as request_core from rucio.core.monitor import MetricManager from rucio.core.rse import list_rses +from rucio.core.topology import ExpiringObjectCache, Topology from rucio.core.transfer import ProtocolFactory -from rucio.core.topology import Topology, ExpiringObjectCache -from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon -from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, RequestState, RequestType, ReplicaState, BadFilesStatus +from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue +from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, ReplicaState, RequestState, RequestType from rucio.db.sqla.session import transactional_session if TYPE_CHECKING: diff --git a/lib/rucio/daemons/conveyor/poller.py b/lib/rucio/daemons/conveyor/poller.py index 7ceaab6687..fba63b0775 100644 --- a/lib/rucio/daemons/conveyor/poller.py +++ b/lib/rucio/daemons/conveyor/poller.py @@ -39,13 +39,14 @@ from rucio.common.stopwatch import Stopwatch from rucio.common.types import InternalAccount from rucio.common.utils import dict_chunks -from rucio.core import transfer as transfer_core, request as request_core +from rucio.core import request as request_core +from rucio.core import transfer as transfer_core from rucio.core.monitor import MetricManager -from rucio.core.topology import Topology, ExpiringObjectCache -from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon +from rucio.core.topology import ExpiringObjectCache, Topology +from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, RequestState, RequestType -from rucio.transfertool.transfertool import Transfertool from rucio.transfertool.fts3 import FTS3Transfertool +from rucio.transfertool.transfertool import Transfertool if TYPE_CHECKING: from rucio.daemons.common import HeartbeatHandler diff --git a/lib/rucio/daemons/conveyor/preparer.py b/lib/rucio/daemons/conveyor/preparer.py index cc0e6e1d67..82eeb01109 100644 --- a/lib/rucio/daemons/conveyor/preparer.py +++ b/lib/rucio/daemons/conveyor/preparer.py @@ -25,14 +25,15 @@ from rucio.common.exception import RucioException from rucio.common.logging import setup_logging from rucio.core import transfer as transfer_core -from rucio.core.request import transition_requests_state_if_possible, list_and_mark_transfer_requests_and_source_replicas -from rucio.core.topology import Topology, ExpiringObjectCache -from rucio.core.transfer import prepare_transfers, list_transfer_admin_accounts, build_transfer_paths, ProtocolFactory -from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon +from rucio.core.request import list_and_mark_transfer_requests_and_source_replicas, transition_requests_state_if_possible +from rucio.core.topology import ExpiringObjectCache, Topology +from rucio.core.transfer import ProtocolFactory, build_transfer_paths, list_transfer_admin_accounts, prepare_transfers +from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue from rucio.db.sqla.constants import RequestState, RequestType if TYPE_CHECKING: from sqlalchemy.orm import Session + from rucio.daemons.common import HeartbeatHandler GRACEFUL_STOP = threading.Event() diff --git a/lib/rucio/daemons/conveyor/receiver.py b/lib/rucio/daemons/conveyor/receiver.py index 5cc95f6de3..356bbe8e9c 100644 --- a/lib/rucio/daemons/conveyor/receiver.py +++ b/lib/rucio/daemons/conveyor/receiver.py @@ -33,8 +33,8 @@ from rucio.common.config import config_get, config_get_bool, config_get_int from rucio.common.logging import setup_logging from rucio.common.policy import get_policy -from rucio.core import transfer as transfer_core from rucio.core import request as request_core +from rucio.core import transfer as transfer_core from rucio.core.monitor import MetricManager from rucio.daemons.common import HeartbeatHandler from rucio.db.sqla.session import transactional_session diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py index 48e84109b3..3d701fbd45 100644 --- a/lib/rucio/daemons/conveyor/submitter.py +++ b/lib/rucio/daemons/conveyor/submitter.py @@ -24,18 +24,17 @@ import rucio.db.sqla.util from rucio.common import exception -from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list, config_get_float +from rucio.common.config import config_get, config_get_bool, config_get_float, config_get_int, config_get_list from rucio.common.logging import setup_logging from rucio.common.schema import get_schema_value from rucio.common.stopwatch import Stopwatch from rucio.core.monitor import MetricManager from rucio.core.request import list_and_mark_transfer_requests_and_source_replicas -from rucio.core.topology import Topology, ExpiringObjectCache -from rucio.core.transfer import DEFAULT_MULTIHOP_TOMBSTONE_DELAY, list_transfer_admin_accounts, transfer_path_str, \ - TRANSFERTOOL_CLASSES_BY_NAME, ProtocolFactory -from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon -from rucio.daemons.conveyor.common import submit_transfer, get_conveyor_rses, pick_and_prepare_submission_path -from rucio.db.sqla.constants import RequestType, RequestState +from rucio.core.topology import ExpiringObjectCache, Topology +from rucio.core.transfer import DEFAULT_MULTIHOP_TOMBSTONE_DELAY, TRANSFERTOOL_CLASSES_BY_NAME, ProtocolFactory, list_transfer_admin_accounts, transfer_path_str +from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue +from rucio.daemons.conveyor.common import get_conveyor_rses, pick_and_prepare_submission_path, submit_transfer +from rucio.db.sqla.constants import RequestState, RequestType from rucio.transfertool.fts3 import FTS3Transfertool from rucio.transfertool.globus import GlobusTransferTool diff --git a/lib/rucio/daemons/conveyor/throttler.py b/lib/rucio/daemons/conveyor/throttler.py index d090ee7946..f93794202e 100644 --- a/lib/rucio/daemons/conveyor/throttler.py +++ b/lib/rucio/daemons/conveyor/throttler.py @@ -17,25 +17,23 @@ Conveyor throttler is a daemon to manage rucio internal queue. """ import logging +import math import threading import traceback from collections import defaultdict from types import FrameType from typing import TYPE_CHECKING, Optional -import math from sqlalchemy import null import rucio.db.sqla.util from rucio.common import exception from rucio.common.logging import setup_logging from rucio.core.monitor import MetricManager -from rucio.core.request import (get_request_stats, release_all_waiting_requests, release_waiting_requests_fifo, - release_waiting_requests_grouped_fifo, set_transfer_limit_stats, re_sync_all_transfer_limits, - reset_stale_waiting_requests) +from rucio.core.request import get_request_stats, re_sync_all_transfer_limits, release_all_waiting_requests, release_waiting_requests_fifo, release_waiting_requests_grouped_fifo, reset_stale_waiting_requests, set_transfer_limit_stats from rucio.core.rse import RseCollection from rucio.core.transfer import applicable_rse_transfer_limits -from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon +from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue from rucio.db.sqla.constants import RequestState, TransferLimitDirection if TYPE_CHECKING: diff --git a/lib/rucio/daemons/follower/follower.py b/lib/rucio/daemons/follower/follower.py index 7e9c422e73..b86dd8dc72 100644 --- a/lib/rucio/daemons/follower/follower.py +++ b/lib/rucio/daemons/follower/follower.py @@ -25,7 +25,7 @@ from rucio.common.logging import setup_logging from rucio.common.utils import get_thread_with_periodic_running_function from rucio.core.did import create_reports -from rucio.core.heartbeat import live, die, sanity_check +from rucio.core.heartbeat import die, live, sanity_check if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/daemons/hermes/hermes.py b/lib/rucio/daemons/hermes/hermes.py index 5d1445159f..df279681a0 100644 --- a/lib/rucio/daemons/hermes/hermes.py +++ b/lib/rucio/daemons/hermes/hermes.py @@ -35,6 +35,7 @@ import requests import stomp +from requests.auth import HTTPBasicAuth import rucio.db.sqla.util from rucio.common.config import ( @@ -48,7 +49,6 @@ from rucio.core.message import delete_messages, retrieve_messages from rucio.core.monitor import MetricManager from rucio.daemons.common import run_daemon -from requests.auth import HTTPBasicAuth if TYPE_CHECKING: from collections.abc import Callable diff --git a/lib/rucio/daemons/judge/cleaner.py b/lib/rucio/daemons/judge/cleaner.py index 99b2b2a966..43fd24a1b1 100644 --- a/lib/rucio/daemons/judge/cleaner.py +++ b/lib/rucio/daemons/judge/cleaner.py @@ -25,17 +25,17 @@ from random import randint from re import match from typing import TYPE_CHECKING -from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX from sqlalchemy.exc import DatabaseError import rucio.db.sqla.util from rucio.common import exception -from rucio.common.exception import DatabaseException, UnsupportedOperation, RuleNotFound +from rucio.common.exception import DatabaseException, RuleNotFound, UnsupportedOperation from rucio.common.logging import setup_logging from rucio.core.monitor import MetricManager from rucio.core.rule import delete_rule, get_expired_rules from rucio.daemons.common import run_daemon +from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX from rucio.db.sqla.util import get_db_time if TYPE_CHECKING: diff --git a/lib/rucio/daemons/judge/evaluator.py b/lib/rucio/daemons/judge/evaluator.py index 808b749fcf..f26a5fa6e0 100644 --- a/lib/rucio/daemons/judge/evaluator.py +++ b/lib/rucio/daemons/judge/evaluator.py @@ -26,7 +26,6 @@ from random import randint from re import match from typing import TYPE_CHECKING -from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX from sqlalchemy.exc import DatabaseError from sqlalchemy.orm.exc import FlushError @@ -36,8 +35,9 @@ from rucio.common.logging import setup_logging from rucio.common.types import InternalScope from rucio.core.monitor import MetricManager -from rucio.core.rule import re_evaluate_did, get_updated_dids, delete_updated_did +from rucio.core.rule import delete_updated_did, get_updated_dids, re_evaluate_did from rucio.daemons.common import run_daemon +from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/daemons/judge/injector.py b/lib/rucio/daemons/judge/injector.py index 91432a2822..a214064597 100644 --- a/lib/rucio/daemons/judge/injector.py +++ b/lib/rucio/daemons/judge/injector.py @@ -25,17 +25,16 @@ from random import randint from re import match from typing import TYPE_CHECKING -from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX from sqlalchemy.exc import DatabaseError import rucio.db.sqla.util -from rucio.common.exception import (DatabaseException, RuleNotFound, RSEWriteBlocked, - ReplicationRuleCreationTemporaryFailed, InsufficientAccountLimit) +from rucio.common.exception import DatabaseException, InsufficientAccountLimit, ReplicationRuleCreationTemporaryFailed, RSEWriteBlocked, RuleNotFound from rucio.common.logging import setup_logging from rucio.core.monitor import MetricManager -from rucio.core.rule import inject_rule, get_injected_rules, update_rule +from rucio.core.rule import get_injected_rules, inject_rule, update_rule from rucio.daemons.common import run_daemon +from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/daemons/judge/repairer.py b/lib/rucio/daemons/judge/repairer.py index 26d3250ed2..b58f24969a 100644 --- a/lib/rucio/daemons/judge/repairer.py +++ b/lib/rucio/daemons/judge/repairer.py @@ -26,7 +26,6 @@ from random import randint from re import match from typing import TYPE_CHECKING -from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX from sqlalchemy.exc import DatabaseError @@ -35,8 +34,9 @@ from rucio.common.exception import DatabaseException from rucio.common.logging import setup_logging from rucio.core.monitor import MetricManager -from rucio.core.rule import repair_rule, get_stuck_rules +from rucio.core.rule import get_stuck_rules, repair_rule from rucio.daemons.common import run_daemon +from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/daemons/oauthmanager/oauthmanager.py b/lib/rucio/daemons/oauthmanager/oauthmanager.py index 10bc4f52ed..32e15b4834 100644 --- a/lib/rucio/daemons/oauthmanager/oauthmanager.py +++ b/lib/rucio/daemons/oauthmanager/oauthmanager.py @@ -32,7 +32,6 @@ import traceback from re import match from typing import TYPE_CHECKING -from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX from sqlalchemy.exc import DatabaseError @@ -43,8 +42,8 @@ from rucio.core.authentication import delete_expired_tokens from rucio.core.monitor import MetricManager from rucio.core.oidc import delete_expired_oauthrequests, refresh_jwt_tokens -from rucio.daemons.common import HeartbeatHandler -from rucio.daemons.common import run_daemon +from rucio.daemons.common import HeartbeatHandler, run_daemon +from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/daemons/reaper/dark_reaper.py b/lib/rucio/daemons/reaper/dark_reaper.py index f0f85aeef0..40c322802f 100644 --- a/lib/rucio/daemons/reaper/dark_reaper.py +++ b/lib/rucio/daemons/reaper/dark_reaper.py @@ -30,15 +30,11 @@ import rucio.db.sqla.util from rucio.common import exception from rucio.common.config import config_get_bool -from rucio.common.exception import (SourceNotFound, ServiceUnavailable, - RSEAccessDenied, ResourceTemporaryUnavailable, - RSENotFound, VONotFound) +from rucio.common.exception import ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, ServiceUnavailable, SourceNotFound, VONotFound from rucio.common.logging import setup_logging from rucio.core.message import add_message from rucio.core.monitor import MetricManager -from rucio.core.quarantined_replica import (list_quarantined_replicas, - delete_quarantined_replicas, - list_rses_with_quarantined_replicas) +from rucio.core.quarantined_replica import delete_quarantined_replicas, list_quarantined_replicas, list_rses_with_quarantined_replicas from rucio.core.rse_expression_parser import parse_expression from rucio.core.vo import list_vos from rucio.daemons.common import run_daemon diff --git a/lib/rucio/daemons/reaper/reaper.py b/lib/rucio/daemons/reaper/reaper.py index 5528354724..069d3da6d1 100644 --- a/lib/rucio/daemons/reaper/reaper.py +++ b/lib/rucio/daemons/reaper/reaper.py @@ -25,19 +25,16 @@ import traceback from configparser import NoOptionError, NoSectionError from datetime import datetime, timedelta +from math import log2 from typing import TYPE_CHECKING from dogpile.cache.api import NoValue -from math import log2 from sqlalchemy.exc import DatabaseError, IntegrityError import rucio.db.sqla.util from rucio.common.cache import make_region_memcached from rucio.common.config import config_get_bool, config_get_int -from rucio.common.exception import (DatabaseException, RSENotFound, - ReplicaUnAvailable, ReplicaNotFound, ServiceUnavailable, - RSEAccessDenied, ResourceTemporaryUnavailable, SourceNotFound, - VONotFound, RSEProtocolNotSupported) +from rucio.common.exception import DatabaseException, ReplicaNotFound, ReplicaUnAvailable, ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, RSEProtocolNotSupported, ServiceUnavailable, SourceNotFound, VONotFound from rucio.common.logging import setup_logging from rucio.common.stopwatch import Stopwatch from rucio.common.utils import chunks @@ -46,9 +43,8 @@ from rucio.core.message import add_message from rucio.core.monitor import MetricManager from rucio.core.oidc import request_token -from rucio.core.replica import list_and_mark_unlocked_replicas, delete_replicas -from rucio.core.rse import (determine_audience_for_rse, determine_scope_for_rse, - list_rses, RseData) +from rucio.core.replica import delete_replicas, list_and_mark_unlocked_replicas +from rucio.core.rse import RseData, determine_audience_for_rse, determine_scope_for_rse, list_rses from rucio.core.rse_expression_parser import parse_expression from rucio.core.rule import get_evaluation_backlog from rucio.core.vo import list_vos diff --git a/lib/rucio/daemons/rsedecommissioner/profiles/generic.py b/lib/rucio/daemons/rsedecommissioner/profiles/generic.py index d6cbc8f9db..819b6f6681 100644 --- a/lib/rucio/daemons/rsedecommissioner/profiles/generic.py +++ b/lib/rucio/daemons/rsedecommissioner/profiles/generic.py @@ -21,8 +21,7 @@ from sqlalchemy.exc import NoResultFound -from rucio.common.exception import (Duplicate, RequestNotFound, ReplicaNotFound, RucioException, - RuleNotFound, RuleReplaceFailed, UnsupportedOperation) +from rucio.common.exception import Duplicate, ReplicaNotFound, RequestNotFound, RucioException, RuleNotFound, RuleReplaceFailed, UnsupportedOperation from rucio.core.lock import get_replica_locks, get_replica_locks_for_rule_id from rucio.core.replica import list_replicas_per_rse, set_tombstone, update_replica_state from rucio.core.request import get_request_by_did diff --git a/lib/rucio/daemons/rsedecommissioner/rse_decommissioner.py b/lib/rucio/daemons/rsedecommissioner/rse_decommissioner.py index 0b5782677a..73e00a0ee3 100644 --- a/lib/rucio/daemons/rsedecommissioner/rse_decommissioner.py +++ b/lib/rucio/daemons/rsedecommissioner/rse_decommissioner.py @@ -20,19 +20,19 @@ profile", which must be specified for each RSE upon triggering decommissioning. """ -from collections.abc import Callable import logging import random import socket import threading +from collections.abc import Callable from types import FrameType from typing import TYPE_CHECKING, Any, Optional, Union from rucio.common.config import config_get_int -from rucio.common.logging import setup_logging from rucio.common.exception import RucioException +from rucio.common.logging import setup_logging from rucio.core.heartbeat import sanity_check -from rucio.core.rse import get_rses_with_attribute, get_rse_attribute +from rucio.core.rse import get_rse_attribute, get_rses_with_attribute from rucio.daemons.common import run_daemon from rucio.db.sqla.constants import RuleState diff --git a/lib/rucio/daemons/storage/consistency/actions.py b/lib/rucio/daemons/storage/consistency/actions.py index c606473eba..6ae8762bf3 100644 --- a/lib/rucio/daemons/storage/consistency/actions.py +++ b/lib/rucio/daemons/storage/consistency/actions.py @@ -37,17 +37,18 @@ from rucio.common.logging import formatted_logger, setup_logging from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import daemon_sleep -from rucio.core.heartbeat import live, die, sanity_check +from rucio.core.heartbeat import die, live, sanity_check from rucio.core.monitor import MetricManager from rucio.core.quarantined_replica import add_quarantined_replicas from rucio.core.replica import __exist_replicas, update_replicas_states -from rucio.core.rse import list_rses, get_rse_id +from rucio.core.rse import get_rse_id, list_rses + # FIXME: these are needed by local version of declare_bad_file_replicas() # TODO: remove after move of this code to core/replica.py - see https://github.com/rucio/rucio/pull/5068 from rucio.db.sqla import models -from rucio.db.sqla.constants import (ReplicaState, BadFilesStatus) +from rucio.db.sqla.constants import BadFilesStatus, ReplicaState from rucio.db.sqla.session import transactional_session -from rucio.rse.rsemanager import lfns2pfns, get_rse_info, parse_pfns +from rucio.rse.rsemanager import get_rse_info, lfns2pfns, parse_pfns if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/daemons/tracer/kronos.py b/lib/rucio/daemons/tracer/kronos.py index 5556250cf7..5118d900ef 100644 --- a/lib/rucio/daemons/tracer/kronos.py +++ b/lib/rucio/daemons/tracer/kronos.py @@ -22,7 +22,8 @@ import re from configparser import NoOptionError, NoSectionError from datetime import datetime -from json import loads as jloads, dumps as jdumps +from json import dumps as jdumps +from json import loads as jloads from queue import Queue from threading import Event, Thread from time import time @@ -30,18 +31,18 @@ import rucio.db.sqla.util from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list -from rucio.common.exception import RSENotFound, DatabaseException +from rucio.common.exception import DatabaseException, RSENotFound from rucio.common.logging import setup_logging from rucio.common.stomp_utils import StompConnectionManager from rucio.common.stopwatch import Stopwatch from rucio.common.types import InternalAccount, InternalScope -from rucio.core.did import touch_dids, list_parent_dids +from rucio.core.did import list_parent_dids, touch_dids from rucio.core.lock import touch_dataset_locks from rucio.core.monitor import MetricManager -from rucio.core.replica import touch_replica, touch_collection_replicas, declare_bad_file_replicas +from rucio.core.replica import declare_bad_file_replicas, touch_collection_replicas, touch_replica from rucio.core.rse import get_rse_id from rucio.daemons.common import HeartbeatHandler, run_daemon -from rucio.db.sqla.constants import DIDType, BadFilesStatus +from rucio.db.sqla.constants import BadFilesStatus, DIDType if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/daemons/transmogrifier/transmogrifier.py b/lib/rucio/daemons/transmogrifier/transmogrifier.py index 474bb909f1..5a68b3839a 100644 --- a/lib/rucio/daemons/transmogrifier/transmogrifier.py +++ b/lib/rucio/daemons/transmogrifier/transmogrifier.py @@ -20,34 +20,34 @@ import time from collections.abc import Callable from datetime import datetime -from json import loads, dumps +from json import dumps, loads from typing import TYPE_CHECKING import rucio.db.sqla.util from rucio.common.config import config_get from rucio.common.exception import ( DatabaseException, - InvalidReplicationRule, DuplicateRule, - InvalidRSEExpression, - InsufficientTargetRSEs, InsufficientAccountLimit, - RSEOverQuota, + InsufficientTargetRSEs, + InvalidReplicationRule, + InvalidRSEExpression, InvalidRuleWeight, + RSEOverQuota, StagingAreaRuleRequiresLifetime, - SubscriptionWrongParameter, SubscriptionNotFound, + SubscriptionWrongParameter, ) from rucio.common.logging import setup_logging from rucio.common.stopwatch import Stopwatch from rucio.common.types import InternalAccount from rucio.common.utils import chunks -from rucio.core.did import list_new_dids, set_new_dids, get_metadata +from rucio.core.did import get_metadata, list_new_dids, set_new_dids from rucio.core.monitor import MetricManager -from rucio.core.rse import list_rses, rse_exists, get_rse_id, list_rse_attributes +from rucio.core.rse import get_rse_id, list_rse_attributes, list_rses, rse_exists from rucio.core.rse_expression_parser import parse_expression from rucio.core.rse_selector import resolve_rse_expression -from rucio.core.rule import add_rule, list_rules, get_rule +from rucio.core.rule import add_rule, get_rule, list_rules from rucio.core.subscription import list_subscriptions, update_subscription from rucio.daemons.common import run_daemon from rucio.db.sqla.constants import DIDType, SubscriptionState @@ -56,8 +56,8 @@ from types import FrameType from typing import Optional - from rucio.daemons.common import HeartbeatHandler from rucio.common.types import InternalScope + from rucio.daemons.common import HeartbeatHandler METRICS = MetricManager(module=__name__) graceful_stop = threading.Event() diff --git a/lib/rucio/daemons/undertaker/undertaker.py b/lib/rucio/daemons/undertaker/undertaker.py index 1db57f3201..8673ed9230 100644 --- a/lib/rucio/daemons/undertaker/undertaker.py +++ b/lib/rucio/daemons/undertaker/undertaker.py @@ -28,16 +28,16 @@ from typing import TYPE_CHECKING from sqlalchemy.exc import DatabaseError -from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_LOCK_NOT_AVAILABLE_REGEX import rucio.db.sqla.util -from rucio.common.exception import DatabaseException, UnsupportedOperation, RuleNotFound +from rucio.common.exception import DatabaseException, RuleNotFound, UnsupportedOperation from rucio.common.logging import setup_logging from rucio.common.types import InternalAccount from rucio.common.utils import chunks -from rucio.core.did import list_expired_dids, delete_dids +from rucio.core.did import delete_dids, list_expired_dids from rucio.core.monitor import MetricManager -from rucio.daemons.common import run_daemon, HeartbeatHandler +from rucio.daemons.common import HeartbeatHandler, run_daemon +from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_LOCK_NOT_AVAILABLE_REGEX if TYPE_CHECKING: from types import FrameType diff --git a/lib/rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py b/lib/rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py index d311b84f1f..bfbe66c89a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py @@ -17,7 +17,6 @@ from alembic import context, op from alembic.op import create_check_constraint - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py index 44b455873f..5640376ad4 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py @@ -16,11 +16,9 @@ ''' add eol_at in rules ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '0437a40dbfd1' down_revision = 'a5f6f6e928a7' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py b/lib/rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py index 9cddac3bc4..579f8a2ffd 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, create_index, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py index 37c25dbfb8..dc99aefa11 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py @@ -16,11 +16,9 @@ ''' added stuck_at column to rules ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '102efcf145f4' down_revision = '70587619328' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py b/lib/rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py index 91091faeae..a634162fcb 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py @@ -18,9 +18,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_index, create_check_constraint, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table from rucio.db.sqla.constants import TransferLimitDirection from rucio.db.sqla.types import GUID diff --git a/lib/rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py b/lib/rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py index a448e5c9fa..53da643a3d 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py @@ -16,9 +16,8 @@ ''' cleanup distances table ''' import sqlalchemy as sa - from alembic import context -from alembic.op import add_column, drop_column, alter_column +from alembic.op import add_column, alter_column, drop_column # Alembic revision identifiers revision = '140fef722e91' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py b/lib/rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py index ba7aab7020..5697f45f6b 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py @@ -16,11 +16,9 @@ ''' add request external_host ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '14ec5aeb64cf' down_revision = '52fd9f4916fa' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py b/lib/rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py index 77da3accaa..a477786f82 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py @@ -16,9 +16,7 @@ ''' add request_type to requests idx ''' from alembic import context -from alembic.op import (create_foreign_key, create_unique_constraint, create_index, - drop_constraint, drop_index) - +from alembic.op import create_foreign_key, create_index, create_unique_constraint, drop_constraint, drop_index # Alembic revision identifiers revision = '156fb5b5a14' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py b/lib/rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py index d2ce43383e..a46db5b34a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py @@ -18,9 +18,8 @@ import sqlalchemy as sa from alembic import context from alembic.op import add_column, drop_column, get_bind -from sqlalchemy.sql.expression import true - from rucio.db.sqla.types import GUID +from sqlalchemy.sql.expression import true # Alembic revision identifiers revision = "1677d4d803c8" diff --git a/lib/rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py b/lib/rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py index 8cc875a7b4..2d52b4e85e 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '16a0aca82e12' down_revision = None diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py index 5970c138c4..1734e73165 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context, op from alembic.op import add_column, drop_column - from rucio.db.sqla.constants import DIDType # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py b/lib/rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py index 3599b08163..07399c1811 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '1a80adff031a' down_revision = '3ad36e2268b0' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py b/lib/rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py index c13da38d39..e8a6848ff9 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context, op from alembic.op import alter_column, create_check_constraint, create_foreign_key, drop_constraint, execute - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py b/lib/rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py index 862f132fae..54d8551a19 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_table from rucio.common.schema import get_schema_value from rucio.db.sqla.types import GUID diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py b/lib/rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py index 7dcb195502..cbd6e41c61 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context, op from alembic.op import add_column, create_check_constraint, drop_column - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py b/lib/rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py index 643d8718d9..4749536f17 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py @@ -16,11 +16,9 @@ ''' add bytes column to bad_replicas ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '1f46c5f240ac' down_revision = '688ef1840840' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py b/lib/rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py index 67ca35081a..dc59357c0b 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py @@ -20,7 +20,6 @@ import sqlalchemy as sa from alembic import context from alembic.op import create_table, drop_table - from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py b/lib/rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py index bd2987425f..8bb17323be 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context from alembic.op import get_bind - from rucio.db.sqla.types import GUID, BooleanString # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py index ffe532c2b8..a4194647f6 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py @@ -17,7 +17,6 @@ from alembic import context, op from alembic.op import create_check_constraint - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py b/lib/rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py index ce070d6e2d..e586d7569b 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py @@ -16,11 +16,9 @@ ''' add availability column to table RSEs ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '22cf51430c78' down_revision = '49a21b4d4357' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py b/lib/rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py index a9ca103165..c4eee238d1 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, create_index, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py index b65b6a434d..43e637a379 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py @@ -16,9 +16,7 @@ ''' remove unique constraint on requests ''' from alembic import context -from alembic.op import (create_foreign_key, create_index, drop_index, - drop_constraint, create_unique_constraint) - +from alembic.op import create_foreign_key, create_index, create_unique_constraint, drop_constraint, drop_index # Alembic revision identifiers revision = '25821a8a45a3' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py index 657d358acd..e94015b255 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '25fc855625cf' down_revision = '4a7182d9578b' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py b/lib/rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py index fb22db4edb..b8f359afa5 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py @@ -16,11 +16,9 @@ ''' add repair_cnt to locks ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '269fee20dee9' down_revision = '1d96f484df21' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py index 99fdb72d60..fb5f9e183d 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py @@ -16,11 +16,9 @@ ''' add ignore_availability column to rules ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '271a46ea6244' down_revision = 'd6dceb1de2d' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py b/lib/rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py index 01b5c0aaca..710910efe7 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py @@ -17,8 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import create_primary_key, add_column, drop_constraint, drop_column - +from alembic.op import add_column, create_primary_key, drop_column, drop_constraint from rucio.db.sqla.models import String # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py b/lib/rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py index b28410f5bd..28250cdca6 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context from alembic.op import add_column, drop_column - from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py index ceaf4193dd..9d1bf4ce0a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py @@ -16,7 +16,6 @@ """ processed_by and _at in requests """ import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py b/lib/rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py index 143eadfb1e..d781743dd9 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py @@ -16,11 +16,9 @@ ''' add access_cnt column in the DID table ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '2962ece31cf4' down_revision = '94a5961ddbf2' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py b/lib/rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py index c918e14594..a6d8425ba4 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py @@ -17,9 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, drop_constraint, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_constraint, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py b/lib/rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py index efe68ff776..3c0abf3e42 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py @@ -16,7 +16,6 @@ ''' Add columns for third_party_copy_read and third_party_copy_write ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py b/lib/rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py index de41165b90..59ac7234f1 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, - create_check_constraint, - drop_constraint, drop_table) +from alembic.op import create_check_constraint, create_primary_key, create_table, drop_constraint, drop_table # Alembic revision identifiers revision = '2b8e7bcb4783' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py b/lib/rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py index c2ad98cf4a..e50841d55a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py @@ -16,11 +16,9 @@ ''' add_submitted_at_to_requests_table ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '2ba5229cb54c' down_revision = '22d887e4ec0a' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py b/lib/rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py index 599cba6ddd..f9cb2b6e6a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py @@ -16,10 +16,8 @@ ''' added column volume to rse_transfer_limits ''' import sqlalchemy as sa - from alembic import context -from alembic.op import (drop_column, add_column) - +from alembic.op import add_column, drop_column # Alembic revision identifiers revision = '2cbee484dcf9' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py b/lib/rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py index 292b6ad0ed..e9e6a82a12 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context from alembic.op import add_column, drop_column - from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py b/lib/rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py index 84557cfa09..b4300edb2f 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py @@ -16,8 +16,7 @@ ''' change tokens pk ''' from alembic import context -from alembic.op import create_primary_key, create_foreign_key, drop_constraint - +from alembic.op import create_foreign_key, create_primary_key, drop_constraint # Alembic revision identifiers revision = '2eef46be23d4' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py b/lib/rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py index e2fc7e840a..d2c94e7fbb 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '2f648fc909f3' down_revision = 'bb695f45c04' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py b/lib/rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py index 2cb60f7ea6..6b736f7433 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, add_column, - create_check_constraint, drop_column, drop_table) - +from alembic.op import add_column, create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_column, drop_table from rucio.common.schema import get_schema_value from rucio.db.sqla.constants import KeyType diff --git a/lib/rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py b/lib/rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py index e4b3cdbfd0..74f436110a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py @@ -15,9 +15,8 @@ ''' Add index on service column in the message table ''' import sqlalchemy as sa - from alembic import context -from alembic.op import create_index, drop_index, alter_column +from alembic.op import alter_column, create_index, drop_index # Alembic revision identifiers revision = '30fa38b6434e' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py b/lib/rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py index 32fc09b5a0..54390098de 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py @@ -17,8 +17,7 @@ import sqlalchemy as sa from alembic import context, op -from alembic.op import add_column, create_check_constraint, drop_constraint, drop_column - +from alembic.op import add_column, create_check_constraint, drop_column, drop_constraint from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py b/lib/rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py index 3c264ae450..7dcc23c96d 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, create_index, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py b/lib/rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py index f1a41a2a25..ba38bdb6a2 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py @@ -16,8 +16,7 @@ ''' replicas table PK definition is in wrong order ''' from alembic import context -from alembic.op import create_primary_key, drop_constraint, create_foreign_key, drop_index - +from alembic.op import create_foreign_key, create_primary_key, drop_constraint, drop_index # revision identifiers used by alembic revision = '3345511706b8' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py index 2f0af8eb9e..7851a9cea4 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '35ef10d1e11b' down_revision = '3152492b110b' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py b/lib/rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py index 3e6e569132..f60d19c6bc 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py b/lib/rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py index 63bfc3b98d..eb193e2a73 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py @@ -19,9 +19,8 @@ import sqlalchemy as sa from alembic import context -from alembic.op import create_table, create_index, drop_table, drop_index, create_primary_key - -from rucio.db.sqla.constants import (DIDType, RuleGrouping, RuleState, RuleNotification) +from alembic.op import create_index, create_primary_key, create_table, drop_index, drop_table +from rucio.db.sqla.constants import DIDType, RuleGrouping, RuleNotification, RuleState from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py b/lib/rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py index 377dfdc165..dc6b683ebf 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py @@ -16,11 +16,9 @@ ''' extend distance table ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '3ac1660a1a72' down_revision = '5673b4b6e843' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py b/lib/rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py index 3dfb3c8fbc..5752de3bd9 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py @@ -19,10 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, add_column, - create_check_constraint, create_index, - drop_constraint, drop_column, drop_table, drop_index) - +from alembic.op import add_column, create_check_constraint, create_index, create_primary_key, create_table, drop_column, drop_constraint, drop_index, drop_table from rucio.db.sqla.constants import DIDType from rucio.db.sqla.types import GUID diff --git a/lib/rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py b/lib/rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py index 40d20d7708..227a55e649 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py @@ -17,7 +17,6 @@ from alembic import context, op from alembic.op import create_check_constraint - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py b/lib/rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py index a850bef06b..0ceb047232 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py @@ -17,7 +17,6 @@ from alembic import context from alembic.op import create_check_constraint - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py b/lib/rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py index cad879fc5f..cbe0ba7ea5 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py @@ -16,11 +16,9 @@ ''' add_transferred_at_to_requests_table ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '40ad39ce3160' down_revision = '2ba5229cb54c' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py index 97663aeb68..f581eeae77 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context, op from alembic.op import add_column, drop_column - from rucio.db.sqla.constants import RuleNotification from rucio.db.sqla.util import try_drop_constraint diff --git a/lib/rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py b/lib/rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py index c2096182ea..1b4fd245ba 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '42db2617c364' down_revision = '4bab9edd01fc' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py index 9b5a4133bb..f1e071a572 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py @@ -16,11 +16,9 @@ ''' added column activity to table requests ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '436827b13f82' down_revision = '102efcf145f4' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py b/lib/rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py index 08e2319953..1239c29dd2 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '44278720f774' down_revision = '40ad39ce3160' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py b/lib/rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py index 76617c436d..c7bafc9285 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py @@ -19,10 +19,8 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, create_index, drop_table) - -from rucio.db.sqla.constants import ReplicaState, DIDType +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table +from rucio.db.sqla.constants import DIDType, ReplicaState from rucio.db.sqla.types import GUID from rucio.db.sqla.util import try_drop_constraint diff --git a/lib/rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py b/lib/rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py index d33dfd4615..9d1f7c802d 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - revision = '469d262be19' down_revision = '16a0aca82e12' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py b/lib/rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py index d69f486d18..b3d4625606 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, create_index, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py b/lib/rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py index ea757a9611..08ec19d485 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py @@ -16,9 +16,7 @@ ''' add tokens index ''' from alembic import context -from alembic.op import (create_foreign_key, create_index, - drop_constraint, drop_index) - +from alembic.op import create_foreign_key, create_index, drop_constraint, drop_index # Alembic revision identifiers revision = '49a21b4d4357' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py b/lib/rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py index 37a7de7776..06b6d48c2e 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py @@ -16,11 +16,9 @@ ''' add_source_replica_expression_column_to_rules ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '4a2cbedda8b9' down_revision = 'a616581ee47' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py b/lib/rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py index 7a53133d04..e28fb2fd86 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py @@ -16,11 +16,9 @@ ''' added bytes, length, accessed_at columns ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '4a7182d9578b' down_revision = 'c129ccdb2d5' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py b/lib/rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py index ccf92ba8f0..f1343aa0d7 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py @@ -18,7 +18,6 @@ from alembic import context from alembic.op import create_index, drop_index - # Alembic revision identifiers revision = '4bab9edd01fc' down_revision = 'ae2a56fcc89' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py b/lib/rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py index 5dc937d2cb..215ebe2fb9 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py @@ -19,8 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_check_constraint, create_index, drop_table) +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table # Alembic revision identifiers revision = '4c3a4acfe006' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py b/lib/rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py index 24a7702f7d..eb088e8bd4 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py @@ -16,11 +16,9 @@ ''' adding transient metadata ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '4cf0a2e127d4' down_revision = '271a46ea6244' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py b/lib/rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py index 1b73e3c2b8..12c3e695ee 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py @@ -18,9 +18,8 @@ import sqlalchemy as sa from alembic import context from alembic.op import create_index, create_primary_key, create_table, drop_table - from rucio.common.schema import get_schema_value -from rucio.db.sqla.types import InternalScopeString, GUID +from rucio.db.sqla.types import GUID, InternalScopeString # Alembic revision identifiers revision = '4df2c5ddabc0' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py b/lib/rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py index a156a9a21b..96e297136f 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py @@ -16,7 +16,6 @@ ''' add qos class to rse ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column diff --git a/lib/rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py b/lib/rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py index 7eea083593..fb860e9904 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py @@ -19,7 +19,6 @@ from alembic import context from alembic.op import create_foreign_key, create_index, drop_constraint, drop_index - # Alembic revision identifiers revision = '52153819589c' down_revision = '30fa38b6434e' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py index 0071978d82..d2c76f4e88 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py @@ -16,11 +16,9 @@ ''' added share to rules ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '52fd9f4916fa' down_revision = '4a2cbedda8b9' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py b/lib/rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py index 6da9edd94f..7465c6b282 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py @@ -16,11 +16,9 @@ ''' fix did_meta table missing updated_at, created_at columns ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '53b479c3cb0f' down_revision = '2cbee484dcf9' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py b/lib/rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py index afc00fd2ae..d354fa6d9d 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py @@ -16,11 +16,9 @@ ''' add metadata to rule tables ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '5673b4b6e843' down_revision = 'e59300c8b179' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py b/lib/rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py index 95adcd4d0e..d3e9388ffe 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py @@ -17,8 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import create_table, add_column, drop_column, drop_table - +from alembic.op import add_column, create_table, drop_column, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py index 956a12ec62..9dd35ccced 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py @@ -16,11 +16,9 @@ ''' add started_at to requests ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '58bff7008037' down_revision = '3c9df354071b' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py b/lib/rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py index fe8558f224..b68b774cb0 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py @@ -16,9 +16,7 @@ ''' rename callback to message ''' from alembic import context, op -from alembic.op import (create_primary_key, create_check_constraint, - drop_constraint, rename_table) - +from alembic.op import create_check_constraint, create_primary_key, drop_constraint, rename_table from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py b/lib/rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py index f3a1712bec..bca8333c00 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py @@ -17,9 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_foreign_key, add_column, create_index, - drop_constraint, drop_column, drop_index) - +from alembic.op import add_column, create_foreign_key, create_index, drop_column, drop_constraint, drop_index from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py b/lib/rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py index 16702fd83c..debd41d4b4 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py @@ -17,8 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import create_primary_key, create_table, create_foreign_key, drop_table - +from alembic.op import create_foreign_key, create_primary_key, create_table, drop_table from rucio.db.sqla.types import JSON # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py index bc72c69b8a..d3298ff141 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py @@ -16,11 +16,9 @@ ''' add new split_container column to rules ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '6e572a9bfbf3' down_revision = '914b8f02df38' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py b/lib/rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py index 5bba86003b..352b242759 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py @@ -16,11 +16,9 @@ ''' add comment column for subscriptions ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '70587619328' down_revision = '4207be2fd914' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py b/lib/rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py index 4f62bd14f6..461f44cce0 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py @@ -16,7 +16,7 @@ ''' remove history table pks ''' from alembic import context -from alembic.op import drop_constraint, create_primary_key +from alembic.op import create_primary_key, drop_constraint # Alembic revision identifiers revision = '739064d31565' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py b/lib/rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py index 3e42c485f0..b35d811fbb 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_check_constraint, - create_foreign_key, create_index, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table from rucio.db.sqla.constants import DIDType # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py b/lib/rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py index 928648b429..f6de46949f 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py @@ -17,7 +17,6 @@ from alembic import context, op from alembic.op import create_check_constraint - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py b/lib/rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py index 533407ba9f..4fc20d8bc8 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py @@ -16,10 +16,8 @@ ''' added_columns_rse_transfer_limits ''' import sqlalchemy as sa - from alembic import context -from alembic.op import (add_column, drop_column) - +from alembic.op import add_column, drop_column # Alembic revision identifiers revision = '810a41685bc1' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py b/lib/rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py index 096625c10d..d57d0418e0 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py @@ -16,7 +16,6 @@ ''' correct rse_expression length ''' import sqlalchemy as sa - from alembic import context from alembic.op import alter_column diff --git a/lib/rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py b/lib/rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py index ad69eb7d08..24f2649444 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py @@ -16,11 +16,9 @@ ''' increase size of extended_attributes column ''' import sqlalchemy as sa - from alembic import context from alembic.op import alter_column - # Alembic revision identifiers revision = '8523998e2e76' down_revision = '7ec22226cdbf' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py b/lib/rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py index 171a487ab0..4466c6f398 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py @@ -16,7 +16,7 @@ ''' Adding missing function based indices ''' from alembic import context -from alembic.op import create_index, drop_index, drop_constraint, create_foreign_key +from alembic.op import create_foreign_key, create_index, drop_constraint, drop_index # Alembic revision identifiers revision = '8ea9122275b1' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py b/lib/rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py index be5a788d49..886c084efd 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py @@ -16,11 +16,9 @@ ''' add clob payload to messages ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '90f47792bb76' down_revision = 'bf3baa1c1474' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py b/lib/rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py index 89d73b79ac..2b4da66539 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, - create_check_constraint, drop_table) - +from alembic.op import create_check_constraint, create_primary_key, create_table, drop_table from rucio.db.sqla.constants import DIDType, LifetimeExceptionsState from rucio.db.sqla.types import GUID diff --git a/lib/rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py b/lib/rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py index f6ce645d45..0d635271dc 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py @@ -16,11 +16,9 @@ ''' add estimator columns to request table ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = '94a5961ddbf2' down_revision = '1c45d9730ca6' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py b/lib/rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py index 5add827aed..a42ddab859 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py @@ -17,7 +17,6 @@ from alembic import context from alembic.op import create_check_constraint, drop_constraint, execute - from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py b/lib/rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py index 74351e2405..bfe8bef2d9 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py @@ -17,8 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import create_primary_key, create_table, create_foreign_key, drop_table - +from alembic.op import create_foreign_key, create_primary_key, create_table, drop_table from rucio.common.schema import get_schema_value from rucio.db.sqla.types import JSON diff --git a/lib/rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py b/lib/rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py index 3b9b521286..fd201c5881 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py @@ -17,7 +17,6 @@ from alembic import context, op - # Alembic revision identifiers revision = '9eb936a81eb1' down_revision = 'b96a1c7e1cc4' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py b/lib/rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py index 1a5aabaaee..bba7a60c18 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py @@ -19,10 +19,9 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, - create_index, create_check_constraint, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID + # Alembic revision identifiers revision = 'a08fa8de1545' down_revision = '4df2c5ddabc0' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py b/lib/rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py index f43457ff27..eac0befefb 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py @@ -19,8 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (add_column, create_primary_key, create_table, create_unique_constraint, - drop_column, drop_constraint, drop_table, bulk_insert) +from alembic.op import add_column, bulk_insert, create_primary_key, create_table, create_unique_constraint, drop_column, drop_constraint, drop_table from sqlalchemy import String # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py b/lib/rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py index d46dad8586..2266dee212 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py @@ -16,7 +16,6 @@ ''' Add status column in messages ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column diff --git a/lib/rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py b/lib/rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py index eb69f3ecf6..b01be0577e 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py @@ -16,10 +16,8 @@ ''' add columns for 1.7.0 release ''' import sqlalchemy as sa - from alembic import context -from alembic.op import create_check_constraint, create_foreign_key, add_column, drop_column, drop_constraint - +from alembic.op import add_column, create_check_constraint, create_foreign_key, drop_column, drop_constraint # Alembic revision identifiers revision = 'a5f6f6e928a7' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py b/lib/rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py index b1c52b79b9..a6509ace49 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context from alembic.op import add_column, drop_column - from rucio.db.sqla.models import String # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py b/lib/rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py index c004f4bcdd..d32ec86061 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py @@ -19,8 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_primary_key, create_check_constraint, - create_table, drop_table, create_foreign_key) +from alembic.op import create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_table # Alembic revision identifiers revision = 'a74275a1ad30' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py b/lib/rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py index 84db384eda..61b8231115 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, - create_check_constraint, create_index, - drop_constraint, drop_table) +from alembic.op import create_check_constraint, create_index, create_primary_key, create_table, drop_constraint, drop_table # Alembic revision identifiers revision = 'a93e4e47bda' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py index 669e3fc0fc..04339e672b 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py @@ -18,7 +18,6 @@ import sqlalchemy as sa from alembic import context from alembic.op import add_column, drop_column - from rucio.db.sqla.models import String # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py b/lib/rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py index f22b422bd3..5a81ff946a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py @@ -16,11 +16,9 @@ ''' added column identity to table tokens ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = 'b4293a99f344' down_revision = '3ac1660a1a72' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py b/lib/rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py index 83fc0c52ca..bf9f65c8cc 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py @@ -16,8 +16,7 @@ ''' Removal of ReplicaState.SOURCE ''' from alembic import context, op -from alembic.op import (create_check_constraint) - +from alembic.op import create_check_constraint from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py b/lib/rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py index 262b448be2..44ad092001 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py @@ -16,8 +16,7 @@ ''' add index to quarantined replicas ''' from alembic import context -from alembic.op import (create_index, drop_index) - +from alembic.op import create_index, drop_index # revision identifiers, used by Alembic. revision = 'b818052fa670' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py b/lib/rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py index d4e375bb1f..6b3f0faccc 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py @@ -16,11 +16,9 @@ ''' add comments column for subscriptions_history ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = 'b8caac94d7f0' down_revision = '8523998e2e76' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py b/lib/rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py index 304b68732c..165863ad12 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py @@ -19,11 +19,7 @@ import sqlalchemy as sa from alembic import context, op -from alembic.op import (create_primary_key, create_check_constraint, - drop_constraint, create_table, create_foreign_key, - add_column, create_index, drop_table, drop_column, - drop_index) - +from alembic.op import add_column, create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_column, drop_constraint, drop_index, drop_table from rucio.db.sqla.constants import BadPFNStatus from rucio.db.sqla.util import try_drop_constraint diff --git a/lib/rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py b/lib/rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py index 65b3947fd7..a3327f3196 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py @@ -17,9 +17,7 @@ import sqlalchemy as sa from alembic import context, op -from alembic.op import (add_column, create_check_constraint, - drop_constraint, drop_column) - +from alembic.op import add_column, create_check_constraint, drop_column, drop_constraint from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py b/lib/rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py index 713be7b54d..23b377aef5 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py @@ -17,11 +17,9 @@ ''' add staging timestamps to request ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = 'bc68e9946deb' down_revision = '9a1b149a2044' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py b/lib/rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py index 4dfaf58a1a..6498690016 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py @@ -17,9 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_primary_key, drop_constraint, - drop_index, drop_column, add_column) - +from alembic.op import add_column, create_primary_key, drop_column, drop_constraint, drop_index from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py b/lib/rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py index ed631bb127..13eb3bc441 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py @@ -19,8 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import create_table, create_primary_key, create_foreign_key, drop_table, create_check_constraint - +from alembic.op import create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py b/lib/rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py index c4e3c1f0d4..17bf5a7d3a 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py @@ -16,11 +16,9 @@ ''' add lumiblocknr to dids ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = 'c129ccdb2d5' down_revision = '156fb5b5a14' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py b/lib/rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py index 46b440aa61..a40c82dcb2 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py @@ -17,10 +17,7 @@ import sqlalchemy as sa from alembic.context import get_context -from alembic.op import (add_column, drop_column, - create_index, drop_index, - execute) - +from alembic.op import add_column, create_index, drop_column, drop_index, execute from rucio.db.sqla.constants import DIDType from rucio.db.sqla.util import try_drop_constraint diff --git a/lib/rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py b/lib/rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py index 422d65a1bc..38d118ef23 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py @@ -17,8 +17,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import add_column, drop_column, create_index, drop_index - +from alembic.op import add_column, create_index, drop_column, drop_index from rucio.db.sqla.models import String # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py b/lib/rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py index c4a79e6e91..dced31faf6 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py @@ -19,10 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (add_column, alter_column, drop_column, - create_table, create_primary_key, create_index, - create_check_constraint, drop_table, execute) - +from alembic.op import add_column, alter_column, create_check_constraint, create_index, create_primary_key, create_table, drop_column, drop_table, execute from rucio.db.sqla.types import InternalAccountString from rucio.db.sqla.util import try_drop_constraint diff --git a/lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py b/lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py index 73a3216421..a099b1f18f 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py @@ -17,9 +17,7 @@ Add PREPARING state to Request model. """ -from alembic import context -from alembic import op - +from alembic import context, op from rucio.db.sqla.util import try_drop_constraint # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py b/lib/rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py index 2aa220dea6..5393ff5277 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py @@ -16,10 +16,8 @@ ''' added purge column to rules ''' import sqlalchemy as sa - from alembic import context -from alembic.op import add_column, drop_column, create_check_constraint - +from alembic.op import add_column, create_check_constraint, drop_column # Alembic revision identifiers revision = 'd6dceb1de2d' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py b/lib/rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py index ae48fa7e59..07251bec9d 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py @@ -16,7 +16,6 @@ ''' remove third_party_copy column from rse ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column diff --git a/lib/rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py b/lib/rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py index 7771b49731..9d5d23e6ca 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_check_constraint, create_table, create_primary_key, - create_foreign_key, drop_table) - +from alembic.op import create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_table from rucio.db.sqla.types import GUID # Alembic revision identifiers diff --git a/lib/rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py b/lib/rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py index a9026286cb..1bf28e255e 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py @@ -16,11 +16,9 @@ ''' Extending columns for filter and replication_rules in subscriptions ''' import sqlalchemy as sa - from alembic import context from alembic.op import alter_column - # Alembic revision identifiers revision = 'e138c364ebd0' down_revision = 'f85a2962b021' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py b/lib/rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py index 858e671652..1b355073af 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py @@ -19,9 +19,7 @@ import sqlalchemy as sa from alembic import context -from alembic.op import (create_table, create_primary_key, create_foreign_key, add_column, - create_index, drop_column, drop_table) - +from alembic.op import add_column, create_foreign_key, create_index, create_primary_key, create_table, drop_column, drop_table from rucio.db.sqla.models import String from rucio.db.sqla.types import GUID diff --git a/lib/rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py b/lib/rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py index 79a07a21ce..7703b35fb4 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py @@ -18,9 +18,8 @@ import sqlalchemy as sa from alembic import context from alembic.op import create_table, drop_table - from rucio.common.schema import get_schema_value -from rucio.db.sqla.types import InternalScopeString, String, GUID +from rucio.db.sqla.types import GUID, InternalScopeString, String # Alembic revision identifiers revision = 'f41ffe206f37' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py b/lib/rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py index 545cff0e93..7cbf3a0ec6 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py @@ -16,9 +16,8 @@ ''' adding transfertool column and index to requests table''' import sqlalchemy as sa - from alembic import context -from alembic.op import add_column, drop_column, create_index, drop_index +from alembic.op import add_column, create_index, drop_column, drop_index # Alembic revision identifiers revision = 'f85a2962b021' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py b/lib/rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py index 1d4e007b40..4ed80eb50b 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py @@ -16,7 +16,6 @@ ''' increase refresh token size ''' import sqlalchemy as sa - from alembic import context from alembic.op import alter_column diff --git a/lib/rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py b/lib/rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py index 053118e529..47aba2989b 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py @@ -16,7 +16,7 @@ ''' set third_party_copy_read and write fields ''' from alembic import context -from alembic.op import execute, alter_column # pylint: disable=no-member +from alembic.op import alter_column, execute # pylint: disable=no-member # Alembic revision identifiers revision = 'fe1a65b176c9' diff --git a/lib/rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py b/lib/rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py index c5ea779d95..a3f868643f 100644 --- a/lib/rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +++ b/lib/rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py @@ -16,11 +16,9 @@ ''' add third_party_copy column to rse_protocols ''' import sqlalchemy as sa - from alembic import context from alembic.op import add_column, drop_column - # Alembic revision identifiers revision = 'fe8ea2fa9788' down_revision = '0437a40dbfd1' diff --git a/lib/rucio/db/sqla/models.py b/lib/rucio/db/sqla/models.py index a11a0570ee..a8ac0ac039 100644 --- a/lib/rucio/db/sqla/models.py +++ b/lib/rucio/db/sqla/models.py @@ -17,28 +17,43 @@ from datetime import datetime, timedelta from typing import Any, Optional, Union -from sqlalchemy import BigInteger, Boolean, DateTime, Enum, Float, Integer, SmallInteger, String, Text, event, UniqueConstraint +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, Float, Integer, SmallInteger, String, Text, UniqueConstraint, event from sqlalchemy.engine import Engine from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import mapped_column, object_mapper, relationship, Mapped -from sqlalchemy.schema import Index, ForeignKeyConstraint, PrimaryKeyConstraint, CheckConstraint, Table +from sqlalchemy.orm import Mapped, mapped_column, object_mapper, relationship +from sqlalchemy.schema import CheckConstraint, ForeignKeyConstraint, Index, PrimaryKeyConstraint, Table from sqlalchemy.sql import Delete from sqlalchemy.types import LargeBinary from rucio.common import utils from rucio.common.schema import get_schema_value from rucio.common.types import InternalAccount, InternalScope -from rucio.db.sqla.constants import (AccountStatus, AccountType, DIDAvailability, DIDType, DIDReEvaluation, - KeyType, IdentityType, LockState, RuleGrouping, BadFilesStatus, - RuleState, ReplicaState, RequestState, RequestType, RSEType, - ScopeStatus, SubscriptionState, RuleNotification, LifetimeExceptionsState, - BadPFNStatus, TransferLimitDirection) +from rucio.db.sqla.constants import ( + AccountStatus, + AccountType, + BadFilesStatus, + BadPFNStatus, + DIDAvailability, + DIDReEvaluation, + DIDType, + IdentityType, + KeyType, + LifetimeExceptionsState, + LockState, + ReplicaState, + RequestState, + RequestType, + RSEType, + RuleGrouping, + RuleNotification, + RuleState, + ScopeStatus, + SubscriptionState, + TransferLimitDirection, +) from rucio.db.sqla.session import BASE -from rucio.db.sqla.types import GUID, BooleanString, JSON -from rucio.db.sqla.types import InternalAccountString -from rucio.db.sqla.types import InternalScopeString - +from rucio.db.sqla.types import GUID, JSON, BooleanString, InternalAccountString, InternalScopeString # SQLAlchemy defines the corresponding code behind TYPE_CHECKING # https://github.com/sqlalchemy/sqlalchemy/blob/d9acd6223299c118464d30abfa483e26a536239d/lib/sqlalchemy/orm/base.py#L814 diff --git a/lib/rucio/db/sqla/sautils.py b/lib/rucio/db/sqla/sautils.py index 1c736ff53a..8af213e074 100644 --- a/lib/rucio/db/sqla/sautils.py +++ b/lib/rucio/db/sqla/sautils.py @@ -19,7 +19,7 @@ ''' from sqlalchemy.ext.compiler import compiles -from sqlalchemy.sql.expression import Executable, ClauseElement +from sqlalchemy.sql.expression import ClauseElement, Executable class InsertFromSelect(Executable, ClauseElement): diff --git a/lib/rucio/db/sqla/session.py b/lib/rucio/db/sqla/session.py index 3591d18eaf..1aefd30c59 100644 --- a/lib/rucio/db/sqla/session.py +++ b/lib/rucio/db/sqla/session.py @@ -19,19 +19,19 @@ import sys from datetime import datetime, timedelta from functools import update_wrapper -from inspect import isgeneratorfunction, getfullargspec +from inspect import getfullargspec, isgeneratorfunction from os.path import basename -from time import sleep from threading import Lock +from time import sleep from typing import TYPE_CHECKING -from sqlalchemy import create_engine, event, MetaData, text -from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, TimeoutError, SQLAlchemyError -from sqlalchemy.orm import DeclarativeBase, sessionmaker, scoped_session, Session -from sqlalchemy.pool import QueuePool, SingletonThreadPool, NullPool +from sqlalchemy import MetaData, create_engine, event, text +from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, SQLAlchemyError, TimeoutError +from sqlalchemy.orm import DeclarativeBase, Session, scoped_session, sessionmaker +from sqlalchemy.pool import NullPool, QueuePool, SingletonThreadPool from rucio.common.config import config_get -from rucio.common.exception import RucioException, DatabaseException, InputValidationError +from rucio.common.exception import DatabaseException, InputValidationError, RucioException from rucio.common.extra import import_extras from rucio.common.utils import retrying diff --git a/lib/rucio/db/sqla/types.py b/lib/rucio/db/sqla/types.py index 44a8786061..99113f4497 100644 --- a/lib/rucio/db/sqla/types.py +++ b/lib/rucio/db/sqla/types.py @@ -17,10 +17,10 @@ import sqlalchemy.types as types from sqlalchemy.dialects.mysql import BINARY -from sqlalchemy.dialects.oracle import RAW, CLOB -from sqlalchemy.dialects.postgresql import UUID, JSONB +from sqlalchemy.dialects.oracle import CLOB, RAW +from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.sql import operators -from sqlalchemy.types import TypeDecorator, CHAR, String +from sqlalchemy.types import CHAR, String, TypeDecorator from rucio.common.exception import InvalidType from rucio.common.types import InternalAccount, InternalScope diff --git a/lib/rucio/db/sqla/util.py b/lib/rucio/db/sqla/util.py index 636b02b3df..d111dbf451 100644 --- a/lib/rucio/db/sqla/util.py +++ b/lib/rucio/db/sqla/util.py @@ -23,11 +23,11 @@ from alembic import command, op from alembic.config import Config from dogpile.cache.api import NoValue -from sqlalchemy import func, inspect, Column, PrimaryKeyConstraint +from sqlalchemy import Column, PrimaryKeyConstraint, func, inspect from sqlalchemy.dialects.postgresql.base import PGInspector -from sqlalchemy.exc import IntegrityError, DatabaseError +from sqlalchemy.exc import DatabaseError, IntegrityError from sqlalchemy.orm import declarative_base -from sqlalchemy.schema import CreateSchema, MetaData, Table, CreateTable, DropTable, ForeignKeyConstraint, DropConstraint +from sqlalchemy.schema import CreateSchema, CreateTable, DropConstraint, DropTable, ForeignKeyConstraint, MetaData, Table from sqlalchemy.sql.ddl import DropSchema from sqlalchemy.sql.expression import select, text @@ -39,13 +39,14 @@ from rucio.common.utils import generate_uuid from rucio.db.sqla import models from rucio.db.sqla.constants import AccountStatus, AccountType, IdentityType -from rucio.db.sqla.session import get_engine, get_session, get_dump_engine +from rucio.db.sqla.session import get_dump_engine, get_engine, get_session from rucio.db.sqla.types import InternalScopeString, String if TYPE_CHECKING: from typing import Optional, Union # noqa: F401 - from sqlalchemy.orm import Session # noqa: F401 + from sqlalchemy.engine import Inspector # noqa: F401 + from sqlalchemy.orm import Session # noqa: F401 REGION = make_region_memcached(expiration_time=600, memcached_expire_time=3660) diff --git a/lib/rucio/rse/__init__.py b/lib/rucio/rse/__init__.py index 5fc50100b2..4de8a6f76c 100644 --- a/lib/rucio/rse/__init__.py +++ b/lib/rucio/rse/__init__.py @@ -47,8 +47,8 @@ def get_signed_url_server(rse, service, op, url, vo='def'): ''' get_signed_url_server ''' - from rucio.core.rse import get_rse_id from rucio.core.credential import get_signed_url + from rucio.core.rse import get_rse_id rse_id = get_rse_id(rse=rse, vo=vo) return get_signed_url(rse_id, service, op, url) @@ -78,9 +78,9 @@ def generate_key(rse, vo='def', session=None): if rsemanager.SERVER_MODE: # pylint:disable=no-member - from rucio.core.rse import get_rse_protocols, get_rse_id - from rucio.core.vo import map_vo from rucio.common.cache import make_region_memcached + from rucio.core.rse import get_rse_id, get_rse_protocols + from rucio.core.vo import map_vo def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None): if rse_id is None: diff --git a/lib/rucio/rse/protocols/bittorrent.py b/lib/rucio/rse/protocols/bittorrent.py index ef30255865..c4ab54a442 100644 --- a/lib/rucio/rse/protocols/bittorrent.py +++ b/lib/rucio/rse/protocols/bittorrent.py @@ -17,14 +17,13 @@ import logging import os.path import time -from urllib.parse import urlparse, urlencode, parse_qs +from urllib.parse import parse_qs, urlencode, urlparse from rucio.common import exception +from rucio.common.extra import import_extras from rucio.common.utils import construct_torrent, resolve_ip -from rucio.rse.protocols.protocol import RSEProtocol from rucio.rse import rsemanager - -from rucio.common.extra import import_extras +from rucio.rse.protocols.protocol import RSEProtocol EXTRA_MODULES = import_extras(['libtorrent']) diff --git a/lib/rucio/rse/protocols/gfal.py b/lib/rucio/rse/protocols/gfal.py index 127f11864d..af4e88b423 100644 --- a/lib/rucio/rse/protocols/gfal.py +++ b/lib/rucio/rse/protocols/gfal.py @@ -22,7 +22,7 @@ import urllib.parse as urlparse from threading import Timer -from rucio.common import exception, config +from rucio.common import config, exception from rucio.common.constraints import STRING_TYPES from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, PREFERRED_CHECKSUM from rucio.rse.protocols import protocol diff --git a/lib/rucio/rse/protocols/globus.py b/lib/rucio/rse/protocols/globus.py index 2640013496..3305f3bf70 100644 --- a/lib/rucio/rse/protocols/globus.py +++ b/lib/rucio/rse/protocols/globus.py @@ -20,7 +20,7 @@ from rucio.common.extra import import_extras from rucio.core.rse import get_rse_attribute from rucio.rse.protocols.protocol import RSEProtocol -from rucio.transfertool.globus_library import get_transfer_client, send_delete_task, send_bulk_delete_task +from rucio.transfertool.globus_library import get_transfer_client, send_bulk_delete_task, send_delete_task EXTRA_MODULES = import_extras(['globus_sdk']) diff --git a/lib/rucio/rse/protocols/protocol.py b/lib/rucio/rse/protocols/protocol.py index 9cf15879f5..0ae446b4a1 100644 --- a/lib/rucio/rse/protocols/protocol.py +++ b/lib/rucio/rse/protocols/protocol.py @@ -21,8 +21,8 @@ import hashlib import logging from configparser import NoOptionError, NoSectionError -from urllib.parse import urlparse from typing import TypeVar +from urllib.parse import urlparse from rucio.common import config, exception from rucio.common.plugins import PolicyPackageAlgorithms diff --git a/lib/rucio/rse/protocols/rclone.py b/lib/rucio/rse/protocols/rclone.py index 3af8d2a8a1..1763651906 100644 --- a/lib/rucio/rse/protocols/rclone.py +++ b/lib/rucio/rse/protocols/rclone.py @@ -19,7 +19,7 @@ from rucio.common import exception from rucio.common.config import get_config_dirs -from rucio.common.utils import execute, PREFERRED_CHECKSUM +from rucio.common.utils import PREFERRED_CHECKSUM, execute from rucio.rse.protocols import protocol diff --git a/lib/rucio/rse/protocols/ssh.py b/lib/rucio/rse/protocols/ssh.py index 5a9fac3af4..6e9cc00a54 100644 --- a/lib/rucio/rse/protocols/ssh.py +++ b/lib/rucio/rse/protocols/ssh.py @@ -18,7 +18,7 @@ import re from rucio.common import exception -from rucio.common.utils import execute, PREFERRED_CHECKSUM +from rucio.common.utils import PREFERRED_CHECKSUM, execute from rucio.rse.protocols import protocol diff --git a/lib/rucio/rse/protocols/webdav.py b/lib/rucio/rse/protocols/webdav.py index a05001ad96..c3ec871372 100644 --- a/lib/rucio/rse/protocols/webdav.py +++ b/lib/rucio/rse/protocols/webdav.py @@ -16,11 +16,11 @@ import os import sys import xml.etree.ElementTree as ET +from dataclasses import dataclass from typing import Optional from urllib.parse import urlparse import requests -from dataclasses import dataclass from requests.adapters import HTTPAdapter from urllib3.poolmanager import PoolManager diff --git a/lib/rucio/rse/protocols/xrootd.py b/lib/rucio/rse/protocols/xrootd.py index e88f3aac16..702b9faa1b 100644 --- a/lib/rucio/rse/protocols/xrootd.py +++ b/lib/rucio/rse/protocols/xrootd.py @@ -17,7 +17,7 @@ import os from rucio.common import exception -from rucio.common.utils import execute, PREFERRED_CHECKSUM +from rucio.common.utils import PREFERRED_CHECKSUM, execute from rucio.rse.protocols import protocol diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py index 6bd2562fab..715812096d 100644 --- a/lib/rucio/rse/rsemanager.py +++ b/lib/rucio/rse/rsemanager.py @@ -19,12 +19,11 @@ from time import sleep from urllib.parse import urlparse -from rucio.common import exception, utils, constants -from rucio.common import types +from rucio.common import constants, exception, types, utils from rucio.common.config import config_get_int from rucio.common.constraints import STRING_TYPES from rucio.common.logging import formatted_logger -from rucio.common.utils import make_valid_did, GLOBALLY_SUPPORTED_CHECKSUMS +from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, make_valid_did def get_rse_info(rse=None, vo='def', rse_id=None, session=None) -> types.RSESettingsDict: diff --git a/lib/rucio/tests/common.py b/lib/rucio/tests/common.py index 43b3d080b7..cbd558d91d 100644 --- a/lib/rucio/tests/common.py +++ b/lib/rucio/tests/common.py @@ -20,14 +20,15 @@ import tempfile from collections import namedtuple from random import choice, choices -from string import ascii_uppercase, ascii_letters, digits +from string import ascii_letters, ascii_uppercase, digits from typing import Optional import pytest import requests from rucio.common.config import config_get, config_get_bool, get_config_dirs -from rucio.common.utils import generate_uuid as uuid, execute +from rucio.common.utils import execute +from rucio.common.utils import generate_uuid as uuid skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg')) for d in get_config_dirs()), reason='fails if no rse-accounts.cfg found') diff --git a/lib/rucio/tests/common_server.py b/lib/rucio/tests/common_server.py index 38eafd2e6e..8a9d281dfb 100644 --- a/lib/rucio/tests/common_server.py +++ b/lib/rucio/tests/common_server.py @@ -20,7 +20,8 @@ from rucio.core import config as core_config from rucio.core.vo import map_vo from rucio.db.sqla import models -from rucio.db.sqla.session import transactional_session, get_session +from rucio.db.sqla.session import get_session, transactional_session + from .common import get_long_vo # Functions containing server-only includes that can't be included in client tests diff --git a/lib/rucio/transfertool/bittorrent.py b/lib/rucio/transfertool/bittorrent.py index 03744722d6..70d62ce685 100644 --- a/lib/rucio/transfertool/bittorrent.py +++ b/lib/rucio/transfertool/bittorrent.py @@ -24,7 +24,8 @@ from rucio.common.extra import import_extras from rucio.common.utils import construct_torrent from rucio.core.did_meta_plugins import get_metadata -from rucio.transfertool.transfertool import Transfertool, TransferToolBuilder, TransferStatusReport +from rucio.transfertool.transfertool import TransferStatusReport, Transfertool, TransferToolBuilder + from .bittorrent_driver import BittorrentDriver if TYPE_CHECKING: diff --git a/lib/rucio/transfertool/bittorrent_driver_qbittorrent.py b/lib/rucio/transfertool/bittorrent_driver_qbittorrent.py index a5cdef6d8d..b7281e3745 100644 --- a/lib/rucio/transfertool/bittorrent_driver_qbittorrent.py +++ b/lib/rucio/transfertool/bittorrent_driver_qbittorrent.py @@ -15,7 +15,7 @@ import logging from collections.abc import Sequence -from typing import TYPE_CHECKING, cast, Optional +from typing import TYPE_CHECKING, Optional, cast from urllib.parse import urlparse import qbittorrentapi @@ -26,11 +26,14 @@ from rucio.core.oidc import request_token from rucio.db.sqla.constants import RequestState from rucio.transfertool.transfertool import TransferStatusReport + from .bittorrent_driver import BittorrentDriver if TYPE_CHECKING: from typing import Type + from sqlalchemy.orm import Session + from rucio.core.rse import RseData diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py index df6ebd4cb4..e0ea29f5a1 100644 --- a/lib/rucio/transfertool/fts3.py +++ b/lib/rucio/transfertool/fts3.py @@ -32,23 +32,23 @@ from rucio.common.cache import make_region_memcached from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list -from rucio.common.constants import FTS_JOB_TYPE, FTS_STATE, FTS_COMPLETE_STATE -from rucio.common.exception import TransferToolTimeout, TransferToolWrongAnswer, DuplicateFileTransferSubmission +from rucio.common.constants import FTS_COMPLETE_STATE, FTS_JOB_TYPE, FTS_STATE +from rucio.common.exception import DuplicateFileTransferSubmission, TransferToolTimeout, TransferToolWrongAnswer from rucio.common.stopwatch import Stopwatch -from rucio.common.utils import APIEncoder, chunks, PREFERRED_CHECKSUM, deep_merge_dict +from rucio.common.utils import PREFERRED_CHECKSUM, APIEncoder, chunks, deep_merge_dict from rucio.core.monitor import MetricManager from rucio.core.oidc import request_token from rucio.core.request import get_source_rse, get_transfer_error -from rucio.core.rse import (determine_audience_for_rse, determine_scope_for_rse, - get_rse_supported_checksums_from_attributes) +from rucio.core.rse import determine_audience_for_rse, determine_scope_for_rse, get_rse_supported_checksums_from_attributes from rucio.db.sqla.constants import RequestState -from rucio.transfertool.transfertool import Transfertool, TransferToolBuilder, TransferStatusReport from rucio.transfertool.fts3_plugins import FTS3TapeMetadataPlugin +from rucio.transfertool.transfertool import TransferStatusReport, Transfertool, TransferToolBuilder if TYPE_CHECKING: + from sqlalchemy.orm import Session + from rucio.core.request import DirectTransfer from rucio.core.rse import RseData - from sqlalchemy.orm import Session logging.getLogger("requests").setLevel(logging.CRITICAL) disable_warnings() diff --git a/lib/rucio/transfertool/fts3_plugins.py b/lib/rucio/transfertool/fts3_plugins.py index 974047f6f7..da6bf56e47 100644 --- a/lib/rucio/transfertool/fts3_plugins.py +++ b/lib/rucio/transfertool/fts3_plugins.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import json +import logging import sys from configparser import NoSectionError from typing import Any, Callable, Optional, Type, TypeVar diff --git a/lib/rucio/transfertool/globus.py b/lib/rucio/transfertool/globus.py index 4bfbde9138..f32a6c4d45 100644 --- a/lib/rucio/transfertool/globus.py +++ b/lib/rucio/transfertool/globus.py @@ -17,8 +17,9 @@ from rucio.common.utils import chunks from rucio.db.sqla.constants import RequestState -from rucio.transfertool.transfertool import Transfertool, TransferToolBuilder, TransferStatusReport -from .globus_library import bulk_submit_xfer, submit_xfer, bulk_check_xfers +from rucio.transfertool.transfertool import TransferStatusReport, Transfertool, TransferToolBuilder + +from .globus_library import bulk_check_xfers, bulk_submit_xfer, submit_xfer def bulk_group_transfers(transfer_paths, policy='single', group_bulk=200): diff --git a/lib/rucio/transfertool/globus_library.py b/lib/rucio/transfertool/globus_library.py index f469b948de..4e452a696d 100644 --- a/lib/rucio/transfertool/globus_library.py +++ b/lib/rucio/transfertool/globus_library.py @@ -24,8 +24,8 @@ EXTRA_MODULES = import_extras(['globus_sdk']) if EXTRA_MODULES['globus_sdk']: - from globus_sdk import NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData, DeleteData # pylint: disable=import-error import yaml # pylint: disable=import-error + from globus_sdk import DeleteData, NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData # pylint: disable=import-error METRICS = MetricManager(module=__name__) diff --git a/lib/rucio/transfertool/mock.py b/lib/rucio/transfertool/mock.py index b086287af3..a8997e4de1 100644 --- a/lib/rucio/transfertool/mock.py +++ b/lib/rucio/transfertool/mock.py @@ -20,7 +20,7 @@ from typing import TYPE_CHECKING, Any, Optional from rucio.db.sqla.constants import RequestState -from rucio.transfertool.transfertool import Transfertool, TransferToolBuilder, TransferStatusReport +from rucio.transfertool.transfertool import TransferStatusReport, Transfertool, TransferToolBuilder if TYPE_CHECKING: from rucio.db.sqla.session import Session diff --git a/lib/rucio/transfertool/transfertool.py b/lib/rucio/transfertool/transfertool.py index 5f0e96af6e..4ad8e17365 100644 --- a/lib/rucio/transfertool/transfertool.py +++ b/lib/rucio/transfertool/transfertool.py @@ -23,8 +23,8 @@ if TYPE_CHECKING: from typing import Optional - from rucio.core.rse import RseData from rucio.core.request import DirectTransfer + from rucio.core.rse import RseData class TransferToolBuilder(object): diff --git a/lib/rucio/web/rest/flaskapi/v1/accountlimits.py b/lib/rucio/web/rest/flaskapi/v1/accountlimits.py index 40fc2d7274..fe7dfda179 100644 --- a/lib/rucio/web/rest/flaskapi/v1/accountlimits.py +++ b/lib/rucio/web/rest/flaskapi/v1/accountlimits.py @@ -15,12 +15,10 @@ from flask import Flask, request -from rucio.api.account_limit import set_local_account_limit, delete_local_account_limit, set_global_account_limit, \ - delete_global_account_limit -from rucio.common.exception import RSENotFound, AccessDenied, AccountNotFound +from rucio.api.account_limit import delete_global_account_limit, delete_local_account_limit, set_global_account_limit, set_local_account_limit +from rucio.common.exception import AccessDenied, AccountNotFound, RSENotFound from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, ErrorHandlingMethodView, \ - generate_http_error_flask, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, generate_http_error_flask, json_parameters, param_get, response_headers class LocalAccountLimit(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/accounts.py b/lib/rucio/web/rest/flaskapi/v1/accounts.py index 84545d71e2..aa7012e3de 100644 --- a/lib/rucio/web/rest/flaskapi/v1/accounts.py +++ b/lib/rucio/web/rest/flaskapi/v1/accounts.py @@ -16,21 +16,17 @@ from datetime import datetime from json import dumps -from flask import Flask, Response, request, redirect, jsonify +from flask import Flask, Response, jsonify, redirect, request -from rucio.api.account import add_account, del_account, get_account_info, list_accounts, list_identities, \ - list_account_attributes, add_account_attribute, del_account_attribute, update_account, get_usage_history -from rucio.api.account_limit import get_local_account_limits, get_local_account_limit, get_local_account_usage, \ - get_global_account_limit, get_global_account_limits, get_global_account_usage +from rucio.api.account import add_account, add_account_attribute, del_account, del_account_attribute, get_account_info, get_usage_history, list_account_attributes, list_accounts, list_identities, update_account +from rucio.api.account_limit import get_global_account_limit, get_global_account_limits, get_global_account_usage, get_local_account_limit, get_local_account_limits, get_local_account_usage from rucio.api.identity import add_account_identity, del_account_identity from rucio.api.rule import list_replication_rules from rucio.api.scope import add_scope, get_scopes -from rucio.common.exception import AccountNotFound, Duplicate, AccessDenied, RuleNotFound, RSENotFound, \ - IdentityError, CounterNotFound, ScopeNotFound, InvalidObject +from rucio.common.exception import AccessDenied, AccountNotFound, CounterNotFound, Duplicate, IdentityError, InvalidObject, RSENotFound, RuleNotFound, ScopeNotFound from rucio.common.utils import APIEncoder, render_json from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - try_stream, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers, try_stream class Attributes(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/archives.py b/lib/rucio/web/rest/flaskapi/v1/archives.py index 28cd291fbd..39a80a0100 100644 --- a/lib/rucio/web/rest/flaskapi/v1/archives.py +++ b/lib/rucio/web/rest/flaskapi/v1/archives.py @@ -19,8 +19,7 @@ from rucio.api.did import list_archive_content from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - parse_scope_name, try_stream, generate_http_error_flask, ErrorHandlingMethodView +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, parse_scope_name, response_headers, try_stream class Archive(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/auth.py b/lib/rucio/web/rest/flaskapi/v1/auth.py index b8ca7fbe7b..753757e743 100644 --- a/lib/rucio/web/rest/flaskapi/v1/auth.py +++ b/lib/rucio/web/rest/flaskapi/v1/auth.py @@ -20,28 +20,39 @@ from typing import TYPE_CHECKING from urllib.parse import urlparse -from flask import Flask, Blueprint, request, Response, redirect, render_template +from flask import Blueprint, Flask, Response, redirect, render_template, request from werkzeug.datastructures import Headers -from rucio.api.authentication import get_auth_token_user_pass, get_auth_token_gss, get_auth_token_x509, \ - get_auth_token_ssh, get_ssh_challenge_token, validate_auth_token, get_auth_oidc, redirect_auth_oidc, \ - get_token_oidc, refresh_cli_auth_token, get_auth_token_saml +from rucio.api.authentication import ( + get_auth_oidc, + get_auth_token_gss, + get_auth_token_saml, + get_auth_token_ssh, + get_auth_token_user_pass, + get_auth_token_x509, + get_ssh_challenge_token, + get_token_oidc, + redirect_auth_oidc, + refresh_cli_auth_token, + validate_auth_token, +) from rucio.common.config import config_get -from rucio.common.exception import AccessDenied, IdentityError, IdentityNotFound, CannotAuthenticate, CannotAuthorize +from rucio.common.exception import AccessDenied, CannotAuthenticate, CannotAuthorize, IdentityError, IdentityNotFound from rucio.common.extra import import_extras from rucio.common.utils import date_to_str from rucio.core.authentication import strip_x509_proxy_attributes -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, error_headers, \ - extract_vo, generate_http_error_flask, ErrorHandlingMethodView, get_account_from_verified_identity +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, error_headers, extract_vo, generate_http_error_flask, get_account_from_verified_identity if TYPE_CHECKING: from typing import Optional, Union + from rucio.web.rest.flaskapi.v1.common import HeadersType EXTRA_MODULES = import_extras(['onelogin']) if EXTRA_MODULES['onelogin']: from onelogin.saml2.auth import OneLogin_Saml2_Auth # pylint: disable=import-error + from rucio.web.ui.flask.common.utils import prepare_saml_request diff --git a/lib/rucio/web/rest/flaskapi/v1/common.py b/lib/rucio/web/rest/flaskapi/v1/common.py index d629d8bb8a..bbf47cbb84 100644 --- a/lib/rucio/web/rest/flaskapi/v1/common.py +++ b/lib/rucio/web/rest/flaskapi/v1/common.py @@ -29,16 +29,16 @@ from werkzeug.wrappers import Request, Response from rucio.api.authentication import validate_auth_token -from rucio.api.identity import list_accounts_for_identity, get_default_account, verify_identity +from rucio.api.identity import get_default_account, list_accounts_for_identity, verify_identity from rucio.common import config -from rucio.common.exception import DatabaseException, IdentityError, RucioException, CannotAuthenticate, UnsupportedRequestedContentType +from rucio.common.exception import CannotAuthenticate, DatabaseException, IdentityError, RucioException, UnsupportedRequestedContentType from rucio.common.schema import get_schema_value from rucio.common.utils import generate_uuid, render_json from rucio.core.vo import map_vo if TYPE_CHECKING: from collections.abc import Callable, Iterable, Sequence - from typing import Optional, Union, Literal, Any + from typing import Any, Literal, Optional, Union HeadersType = Union[Headers, dict[str, str], Sequence[tuple[str, str]]] diff --git a/lib/rucio/web/rest/flaskapi/v1/config.py b/lib/rucio/web/rest/flaskapi/v1/config.py index 140657e14c..33fd55f9e3 100644 --- a/lib/rucio/web/rest/flaskapi/v1/config.py +++ b/lib/rucio/web/rest/flaskapi/v1/config.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Flask, request as request, jsonify +from flask import Flask, jsonify +from flask import request as request from rucio.api import config -from rucio.common.exception import ConfigurationError, AccessDenied, ConfigNotFound +from rucio.common.exception import AccessDenied, ConfigNotFound, ConfigurationError from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - generate_http_error_flask, ErrorHandlingMethodView, json_parameters +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, response_headers class Config(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/credentials.py b/lib/rucio/web/rest/flaskapi/v1/credentials.py index b346b44a8d..51d02c00b4 100644 --- a/lib/rucio/web/rest/flaskapi/v1/credentials.py +++ b/lib/rucio/web/rest/flaskapi/v1/credentials.py @@ -21,11 +21,11 @@ from rucio.api.credential import get_signed_url from rucio.common.exception import CannotAuthenticate from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, extract_vo, \ - generate_http_error_flask, ErrorHandlingMethodView, response_headers +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, extract_vo, generate_http_error_flask, response_headers if TYPE_CHECKING: from typing import Optional + from rucio.web.rest.flaskapi.v1.common import HeadersType diff --git a/lib/rucio/web/rest/flaskapi/v1/dids.py b/lib/rucio/web/rest/flaskapi/v1/dids.py index f7cbd17c08..52ea7dd9cf 100644 --- a/lib/rucio/web/rest/flaskapi/v1/dids.py +++ b/lib/rucio/web/rest/flaskapi/v1/dids.py @@ -18,20 +18,60 @@ from flask import Flask, Response, request -from rucio.api.did import add_did, add_dids, list_content, list_content_history, list_dids, \ - list_files, scope_list, get_did, set_metadata, get_metadata, get_metadata_bulk, set_status, attach_dids, \ - detach_dids, attach_dids_to_dids, get_dataset_by_guid, list_parent_dids, create_did_sample, list_new_dids, \ - resurrect, get_users_following_did, remove_did_from_followed, add_did_to_followed, delete_metadata, \ - set_metadata_bulk, set_dids_metadata_bulk, bulk_list_files -from rucio.api.rule import list_replication_rules, list_associated_replication_rules_for_file -from rucio.common.exception import ScopeNotFound, DatabaseException, DataIdentifierNotFound, DataIdentifierAlreadyExists, \ - DuplicateContent, AccessDenied, KeyNotFound, Duplicate, InvalidValueForKey, UnsupportedStatus, \ - UnsupportedOperation, RSENotFound, RuleNotFound, InvalidMetadata, InvalidPath, FileAlreadyExists, InvalidObject, FileConsistencyMismatch -from rucio.common.utils import render_json, APIEncoder, parse_response +from rucio.api.did import ( + add_did, + add_did_to_followed, + add_dids, + attach_dids, + attach_dids_to_dids, + bulk_list_files, + create_did_sample, + delete_metadata, + detach_dids, + get_dataset_by_guid, + get_did, + get_metadata, + get_metadata_bulk, + get_users_following_did, + list_content, + list_content_history, + list_dids, + list_files, + list_new_dids, + list_parent_dids, + remove_did_from_followed, + resurrect, + scope_list, + set_dids_metadata_bulk, + set_metadata, + set_metadata_bulk, + set_status, +) +from rucio.api.rule import list_associated_replication_rules_for_file, list_replication_rules +from rucio.common.exception import ( + AccessDenied, + DatabaseException, + DataIdentifierAlreadyExists, + DataIdentifierNotFound, + Duplicate, + DuplicateContent, + FileAlreadyExists, + FileConsistencyMismatch, + InvalidMetadata, + InvalidObject, + InvalidPath, + InvalidValueForKey, + KeyNotFound, + RSENotFound, + RuleNotFound, + ScopeNotFound, + UnsupportedOperation, + UnsupportedStatus, +) +from rucio.common.utils import APIEncoder, parse_response, render_json from rucio.db.sqla.constants import DIDType from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - parse_scope_name, try_stream, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, json_list, param_get, json_parse +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_list, json_parameters, json_parse, param_get, parse_scope_name, response_headers, try_stream class Scope(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/dirac.py b/lib/rucio/web/rest/flaskapi/v1/dirac.py index 9083cef873..bde880ffde 100644 --- a/lib/rucio/web/rest/flaskapi/v1/dirac.py +++ b/lib/rucio/web/rest/flaskapi/v1/dirac.py @@ -16,12 +16,10 @@ from flask import Flask, request from rucio.api.dirac import add_files -from rucio.common.exception import AccessDenied, DataIdentifierAlreadyExists, DatabaseException, \ - Duplicate, InvalidPath, ResourceTemporaryUnavailable, RSENotFound, UnsupportedOperation +from rucio.common.exception import AccessDenied, DatabaseException, DataIdentifierAlreadyExists, Duplicate, InvalidPath, ResourceTemporaryUnavailable, RSENotFound, UnsupportedOperation from rucio.common.utils import parse_response from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, generate_http_error_flask, \ - ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, generate_http_error_flask, json_parameters, param_get, response_headers class AddFiles(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/export.py b/lib/rucio/web/rest/flaskapi/v1/export.py index a9b9989b24..d94de60217 100644 --- a/lib/rucio/web/rest/flaskapi/v1/export.py +++ b/lib/rucio/web/rest/flaskapi/v1/export.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Flask, request, Response +from flask import Flask, Response, request from rucio.api.exporter import export_data from rucio.common.utils import render_json from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - ErrorHandlingMethodView +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, response_headers class Export(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/heartbeats.py b/lib/rucio/web/rest/flaskapi/v1/heartbeats.py index e5c0b0a49c..3313f81b4e 100644 --- a/lib/rucio/web/rest/flaskapi/v1/heartbeats.py +++ b/lib/rucio/web/rest/flaskapi/v1/heartbeats.py @@ -17,12 +17,11 @@ from flask import Flask, Response, request -from rucio.api.heartbeat import list_heartbeats, create_heartbeat -from rucio.common.exception import UnsupportedValueType, UnsupportedKeyType, KeyNotFound, AccessDenied +from rucio.api.heartbeat import create_heartbeat, list_heartbeats +from rucio.common.exception import AccessDenied, KeyNotFound, UnsupportedKeyType, UnsupportedValueType from rucio.common.utils import APIEncoder from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - ErrorHandlingMethodView, json_parameters, param_get, generate_http_error_flask +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers class Heartbeat(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/identities.py b/lib/rucio/web/rest/flaskapi/v1/identities.py index 8c330d4600..d2fb0a7d5f 100644 --- a/lib/rucio/web/rest/flaskapi/v1/identities.py +++ b/lib/rucio/web/rest/flaskapi/v1/identities.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Flask, request, jsonify +from flask import Flask, jsonify, request -from rucio.api.identity import add_identity, add_account_identity, list_accounts_for_identity +from rucio.api.identity import add_account_identity, add_identity, list_accounts_for_identity from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - ErrorHandlingMethodView +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, response_headers class UserPass(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/import.py b/lib/rucio/web/rest/flaskapi/v1/import.py index 44d6ad63ac..eb372a4fe7 100644 --- a/lib/rucio/web/rest/flaskapi/v1/import.py +++ b/lib/rucio/web/rest/flaskapi/v1/import.py @@ -18,7 +18,7 @@ from rucio.api.importer import import_data from rucio.common.utils import parse_response from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, ErrorHandlingMethodView, json_parameters +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, json_parameters, response_headers class Import(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/lifetime_exceptions.py b/lib/rucio/web/rest/flaskapi/v1/lifetime_exceptions.py index 60b1cde49e..57c33a4e28 100644 --- a/lib/rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +++ b/lib/rucio/web/rest/flaskapi/v1/lifetime_exceptions.py @@ -17,13 +17,11 @@ from flask import Flask, Response, request -from rucio.api.lifetime_exception import list_exceptions, add_exception, update_exception -from rucio.common.exception import LifetimeExceptionNotFound, UnsupportedOperation, InvalidObject, AccessDenied, \ - LifetimeExceptionDuplicate +from rucio.api.lifetime_exception import add_exception, list_exceptions, update_exception +from rucio.common.exception import AccessDenied, InvalidObject, LifetimeExceptionDuplicate, LifetimeExceptionNotFound, UnsupportedOperation from rucio.common.utils import APIEncoder from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - try_stream, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers, try_stream class LifetimeException(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/locks.py b/lib/rucio/web/rest/flaskapi/v1/locks.py index 8027085ae6..b69d65d100 100644 --- a/lib/rucio/web/rest/flaskapi/v1/locks.py +++ b/lib/rucio/web/rest/flaskapi/v1/locks.py @@ -15,12 +15,11 @@ from flask import Flask, request -from rucio.api.lock import get_dataset_locks_by_rse, get_dataset_locks, get_dataset_locks_bulk +from rucio.api.lock import get_dataset_locks, get_dataset_locks_bulk, get_dataset_locks_by_rse from rucio.common.exception import RSENotFound from rucio.common.utils import render_json from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \ - response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parse +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parse, parse_scope_name, response_headers, try_stream class LockByRSE(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/main.py b/lib/rucio/web/rest/flaskapi/v1/main.py index b968010fee..6caaa85002 100644 --- a/lib/rucio/web/rest/flaskapi/v1/main.py +++ b/lib/rucio/web/rest/flaskapi/v1/main.py @@ -15,13 +15,13 @@ # limitations under the License. import importlib +import logging from flask import Flask from rucio.common.config import config_get from rucio.common.exception import ConfigurationError from rucio.common.logging import setup_logging -import logging from rucio.web.rest.flaskapi.v1.common import CORSMiddleware DEFAULT_ENDPOINTS = [ diff --git a/lib/rucio/web/rest/flaskapi/v1/meta_conventions.py b/lib/rucio/web/rest/flaskapi/v1/meta_conventions.py index 9e1c740db8..b54867faf9 100644 --- a/lib/rucio/web/rest/flaskapi/v1/meta_conventions.py +++ b/lib/rucio/web/rest/flaskapi/v1/meta_conventions.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Flask, request, jsonify +from flask import Flask, jsonify, request from rucio.api.meta_conventions import add_key, add_value, list_keys, list_values -from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedValueType, UnsupportedKeyType +from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedKeyType, UnsupportedValueType from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, response_headers, \ - generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers class MetaConventions(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/metrics.py b/lib/rucio/web/rest/flaskapi/v1/metrics.py index dea3d6594e..c359814dba 100644 --- a/lib/rucio/web/rest/flaskapi/v1/metrics.py +++ b/lib/rucio/web/rest/flaskapi/v1/metrics.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Flask, Blueprint +from flask import Blueprint, Flask from rucio.core.monitor import generate_prometheus_metrics from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView diff --git a/lib/rucio/web/rest/flaskapi/v1/nongrid_traces.py b/lib/rucio/web/rest/flaskapi/v1/nongrid_traces.py index d6a913979a..c58ba67280 100644 --- a/lib/rucio/web/rest/flaskapi/v1/nongrid_traces.py +++ b/lib/rucio/web/rest/flaskapi/v1/nongrid_traces.py @@ -16,14 +16,15 @@ import time from typing import TYPE_CHECKING -from flask import Flask, Blueprint, request +from flask import Blueprint, Flask, request from werkzeug.datastructures import Headers from rucio.core.nongrid_trace import trace -from rucio.web.rest.flaskapi.v1.common import response_headers, ErrorHandlingMethodView, json_parameters +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, json_parameters, response_headers if TYPE_CHECKING: from typing import Optional + from rucio.web.rest.flaskapi.v1.common import HeadersType diff --git a/lib/rucio/web/rest/flaskapi/v1/ping.py b/lib/rucio/web/rest/flaskapi/v1/ping.py index c84dfa48aa..3d84f07930 100644 --- a/lib/rucio/web/rest/flaskapi/v1/ping.py +++ b/lib/rucio/web/rest/flaskapi/v1/ping.py @@ -15,15 +15,15 @@ from typing import TYPE_CHECKING -from flask import Flask, Blueprint, jsonify, request +from flask import Blueprint, Flask, jsonify, request from werkzeug.datastructures import Headers from rucio import version -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - ErrorHandlingMethodView +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, response_headers if TYPE_CHECKING: from typing import Optional + from rucio.web.rest.flaskapi.v1.common import HeadersType diff --git a/lib/rucio/web/rest/flaskapi/v1/redirect.py b/lib/rucio/web/rest/flaskapi/v1/redirect.py index e25a3f24d4..f0ccdbe974 100644 --- a/lib/rucio/web/rest/flaskapi/v1/redirect.py +++ b/lib/rucio/web/rest/flaskapi/v1/redirect.py @@ -16,17 +16,17 @@ import itertools from typing import TYPE_CHECKING -from flask import Flask, Blueprint, request, redirect +from flask import Blueprint, Flask, redirect, request from werkzeug.datastructures import Headers from rucio.api.replica import list_replicas from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound from rucio.core.replica_sorter import site_selector, sort_replicas -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \ - extract_vo, generate_http_error_flask, ErrorHandlingMethodView +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, extract_vo, generate_http_error_flask, parse_scope_name, try_stream if TYPE_CHECKING: from typing import Optional + from rucio.web.rest.flaskapi.v1.common import HeadersType diff --git a/lib/rucio/web/rest/flaskapi/v1/replicas.py b/lib/rucio/web/rest/flaskapi/v1/replicas.py index 60f4b2a4b8..0325877291 100644 --- a/lib/rucio/web/rest/flaskapi/v1/replicas.py +++ b/lib/rucio/web/rest/flaskapi/v1/replicas.py @@ -22,20 +22,33 @@ from flask import Flask, Response, request from rucio.api.quarantined_replica import quarantine_file_replicas -from rucio.api.replica import add_replicas, list_replicas, list_dataset_replicas, list_dataset_replicas_bulk, \ - delete_replicas, get_did_from_pfns, update_replicas_states, declare_bad_file_replicas, add_bad_dids, add_bad_pfns, \ - get_suspicious_files, declare_suspicious_file_replicas, list_bad_replicas_status, get_bad_replicas_summary, \ - list_datasets_per_rse, set_tombstone, list_dataset_replicas_vp +from rucio.api.replica import ( + add_bad_dids, + add_bad_pfns, + add_replicas, + declare_bad_file_replicas, + declare_suspicious_file_replicas, + delete_replicas, + get_bad_replicas_summary, + get_did_from_pfns, + get_suspicious_files, + list_bad_replicas_status, + list_dataset_replicas, + list_dataset_replicas_bulk, + list_dataset_replicas_vp, + list_datasets_per_rse, + list_replicas, + set_tombstone, + update_replicas_states, +) from rucio.common.config import config_get, config_get_int from rucio.common.constants import SUPPORTED_PROTOCOLS -from rucio.common.exception import AccessDenied, DataIdentifierAlreadyExists, InvalidType, DataIdentifierNotFound, \ - Duplicate, InvalidPath, ResourceTemporaryUnavailable, RSENotFound, ReplicaNotFound, InvalidObject, ScopeNotFound, ReplicaIsLocked -from rucio.common.utils import parse_response, APIEncoder, render_json_list +from rucio.common.exception import AccessDenied, DataIdentifierAlreadyExists, DataIdentifierNotFound, Duplicate, InvalidObject, InvalidPath, InvalidType, ReplicaIsLocked, ReplicaNotFound, ResourceTemporaryUnavailable, RSENotFound, ScopeNotFound +from rucio.common.utils import APIEncoder, parse_response, render_json_list from rucio.core.replica_sorter import sort_replicas from rucio.db.sqla.constants import BadFilesStatus from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, try_stream, parse_scope_name, \ - response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream def _sorted_with_priorities(replicas, sorted_pfns, limit=None): diff --git a/lib/rucio/web/rest/flaskapi/v1/requests.py b/lib/rucio/web/rest/flaskapi/v1/requests.py index 26b22bd4b1..7ee74837a2 100644 --- a/lib/rucio/web/rest/flaskapi/v1/requests.py +++ b/lib/rucio/web/rest/flaskapi/v1/requests.py @@ -14,10 +14,10 @@ # limitations under the License. import json +from typing import TYPE_CHECKING import flask from flask import Flask, Response -from typing import TYPE_CHECKING from rucio.api import request from rucio.common.exception import RequestNotFound @@ -25,8 +25,7 @@ from rucio.core.rse import get_rses_with_attribute_value from rucio.db.sqla.constants import RequestState from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \ - response_headers, generate_http_error_flask, ErrorHandlingMethodView +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, parse_scope_name, response_headers, try_stream if TYPE_CHECKING: from typing import Iterator diff --git a/lib/rucio/web/rest/flaskapi/v1/rses.py b/lib/rucio/web/rest/flaskapi/v1/rses.py index f40dd4ec5a..2da77e33ab 100644 --- a/lib/rucio/web/rest/flaskapi/v1/rses.py +++ b/lib/rucio/web/rest/flaskapi/v1/rses.py @@ -15,21 +15,57 @@ from json import dumps -from flask import Flask, Response, request, jsonify +from flask import Flask, Response, jsonify, request from rucio.api.account_limit import get_rse_account_usage -from rucio.api.rse import add_rse, update_rse, list_rses, del_rse, add_rse_attribute, list_rse_attributes, \ - del_rse_attribute, add_protocol, get_rse_protocols, del_protocols, update_protocols, get_rse, set_rse_usage, \ - get_rse_usage, list_rse_usage_history, set_rse_limits, get_rse_limits, delete_rse_limits, parse_rse_expression, \ - add_distance, get_distance, update_distance, delete_distance, list_qos_policies, add_qos_policy, delete_qos_policy -from rucio.common.exception import Duplicate, AccessDenied, RSENotFound, RSEOperationNotSupported, \ - RSEProtocolNotSupported, InvalidObject, RSEProtocolDomainNotSupported, RSEProtocolPriorityError, \ - InvalidRSEExpression, RSEAttributeNotFound, CounterNotFound, InvalidPath, ReplicaNotFound, InputValidationError -from rucio.common.utils import Availability, render_json, APIEncoder +from rucio.api.rse import ( + add_distance, + add_protocol, + add_qos_policy, + add_rse, + add_rse_attribute, + del_protocols, + del_rse, + del_rse_attribute, + delete_distance, + delete_qos_policy, + delete_rse_limits, + get_distance, + get_rse, + get_rse_limits, + get_rse_protocols, + get_rse_usage, + list_qos_policies, + list_rse_attributes, + list_rse_usage_history, + list_rses, + parse_rse_expression, + set_rse_limits, + set_rse_usage, + update_distance, + update_protocols, + update_rse, +) +from rucio.common.exception import ( + AccessDenied, + CounterNotFound, + Duplicate, + InputValidationError, + InvalidObject, + InvalidPath, + InvalidRSEExpression, + ReplicaNotFound, + RSEAttributeNotFound, + RSENotFound, + RSEOperationNotSupported, + RSEProtocolDomainNotSupported, + RSEProtocolNotSupported, + RSEProtocolPriorityError, +) +from rucio.common.utils import APIEncoder, Availability, render_json from rucio.rse import rsemanager from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - try_stream, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers, try_stream class RSEs(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/rules.py b/lib/rucio/web/rest/flaskapi/v1/rules.py index 41705ae387..92a1eb4e61 100644 --- a/lib/rucio/web/rest/flaskapi/v1/rules.py +++ b/lib/rucio/web/rest/flaskapi/v1/rules.py @@ -16,20 +16,44 @@ from json import dumps from typing import Any -from flask import Flask, request, Response +from flask import Flask, Response, request from rucio.api.lock import get_replica_locks_for_rule_id -from rucio.api.rule import add_replication_rule, delete_replication_rule, get_replication_rule, \ - update_replication_rule, reduce_replication_rule, list_replication_rule_history, \ - list_replication_rule_full_history, list_replication_rules, examine_replication_rule, move_replication_rule -from rucio.common.exception import InputValidationError, InsufficientAccountLimit, RuleNotFound, AccessDenied, InvalidRSEExpression, \ - InvalidReplicationRule, DataIdentifierNotFound, InsufficientTargetRSEs, ReplicationRuleCreationTemporaryFailed, \ - InvalidRuleWeight, StagingAreaRuleRequiresLifetime, DuplicateRule, InvalidObject, AccountNotFound, \ - RuleReplaceFailed, ScratchDiskLifetimeConflict, ManualRuleApprovalBlocked, UnsupportedOperation -from rucio.common.utils import render_json, APIEncoder +from rucio.api.rule import ( + add_replication_rule, + delete_replication_rule, + examine_replication_rule, + get_replication_rule, + list_replication_rule_full_history, + list_replication_rule_history, + list_replication_rules, + move_replication_rule, + reduce_replication_rule, + update_replication_rule, +) +from rucio.common.exception import ( + AccessDenied, + AccountNotFound, + DataIdentifierNotFound, + DuplicateRule, + InputValidationError, + InsufficientAccountLimit, + InsufficientTargetRSEs, + InvalidObject, + InvalidReplicationRule, + InvalidRSEExpression, + InvalidRuleWeight, + ManualRuleApprovalBlocked, + ReplicationRuleCreationTemporaryFailed, + RuleNotFound, + RuleReplaceFailed, + ScratchDiskLifetimeConflict, + StagingAreaRuleRequiresLifetime, + UnsupportedOperation, +) +from rucio.common.utils import APIEncoder, render_json from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \ - response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream class Rule(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/scopes.py b/lib/rucio/web/rest/flaskapi/v1/scopes.py index 09701bc4fc..f696c967db 100644 --- a/lib/rucio/web/rest/flaskapi/v1/scopes.py +++ b/lib/rucio/web/rest/flaskapi/v1/scopes.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Flask, request, jsonify +from flask import Flask, jsonify, request -from rucio.api.scope import add_scope, list_scopes, get_scopes +from rucio.api.scope import add_scope, get_scopes, list_scopes from rucio.common.exception import AccountNotFound, Duplicate, ScopeNotFound from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, response_headers, \ - generate_http_error_flask, ErrorHandlingMethodView +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, response_headers class Scope(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/subscriptions.py b/lib/rucio/web/rest/flaskapi/v1/subscriptions.py index b395531f33..45fb254ee6 100644 --- a/lib/rucio/web/rest/flaskapi/v1/subscriptions.py +++ b/lib/rucio/web/rest/flaskapi/v1/subscriptions.py @@ -18,14 +18,11 @@ from flask import Flask, Response, request from rucio.api.rule import list_replication_rules -from rucio.api.subscription import list_subscriptions, add_subscription, update_subscription, \ - list_subscription_rule_states, get_subscription_by_id -from rucio.common.exception import InvalidObject, SubscriptionDuplicate, SubscriptionNotFound, RuleNotFound, \ - AccessDenied -from rucio.common.utils import render_json, APIEncoder +from rucio.api.subscription import add_subscription, get_subscription_by_id, list_subscription_rule_states, list_subscriptions, update_subscription +from rucio.common.exception import AccessDenied, InvalidObject, RuleNotFound, SubscriptionDuplicate, SubscriptionNotFound +from rucio.common.utils import APIEncoder, render_json from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, try_stream, \ - response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers, try_stream class Subscription(ErrorHandlingMethodView): diff --git a/lib/rucio/web/rest/flaskapi/v1/traces.py b/lib/rucio/web/rest/flaskapi/v1/traces.py index 9d83d2b31a..e4512eea72 100644 --- a/lib/rucio/web/rest/flaskapi/v1/traces.py +++ b/lib/rucio/web/rest/flaskapi/v1/traces.py @@ -19,14 +19,15 @@ import uuid from typing import TYPE_CHECKING -from flask import Flask, Blueprint, request +from flask import Blueprint, Flask, request from werkzeug.datastructures import Headers from rucio.core.trace import trace -from rucio.web.rest.flaskapi.v1.common import response_headers, ErrorHandlingMethodView, json_parameters +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, json_parameters, response_headers if TYPE_CHECKING: from typing import Optional + from rucio.web.rest.flaskapi.v1.common import HeadersType diff --git a/lib/rucio/web/rest/flaskapi/v1/vos.py b/lib/rucio/web/rest/flaskapi/v1/vos.py index 5a4fcb468e..717a096abb 100644 --- a/lib/rucio/web/rest/flaskapi/v1/vos.py +++ b/lib/rucio/web/rest/flaskapi/v1/vos.py @@ -16,11 +16,10 @@ from flask import Flask, request from rucio.api.vo import add_vo, list_vos, recover_vo_root_identity, update_vo -from rucio.common.exception import AccessDenied, AccountNotFound, Duplicate, VONotFound, UnsupportedOperation +from rucio.common.exception import AccessDenied, AccountNotFound, Duplicate, UnsupportedOperation, VONotFound from rucio.common.utils import render_json from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint -from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \ - try_stream, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get +from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers, try_stream class VOs(ErrorHandlingMethodView): diff --git a/lib/rucio/web/ui/flask/bp.py b/lib/rucio/web/ui/flask/bp.py index 678b036d28..2851df56d8 100644 --- a/lib/rucio/web/ui/flask/bp.py +++ b/lib/rucio/web/ui/flask/bp.py @@ -14,12 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Blueprint, request, render_template, make_response +from flask import Blueprint, make_response, render_template, request from rucio.api.authentication import get_auth_token_x509 from rucio.common.config import config_get, config_get_bool from rucio.web.rest.flaskapi.v1.common import generate_http_error_flask -from rucio.web.ui.flask.common.utils import get_token, authenticate, userpass_auth, x509token_auth, saml_auth, oidc_auth, finalize_auth, AUTH_ISSUERS, SAML_SUPPORT +from rucio.web.ui.flask.common.utils import AUTH_ISSUERS, SAML_SUPPORT, authenticate, finalize_auth, get_token, oidc_auth, saml_auth, userpass_auth, x509token_auth MULTI_VO = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) POLICY = config_get('policy', 'permission') diff --git a/lib/rucio/web/ui/flask/common/utils.py b/lib/rucio/web/ui/flask/common/utils.py index eccf97d70c..f4c58ee256 100644 --- a/lib/rucio/web/ui/flask/common/utils.py +++ b/lib/rucio/web/ui/flask/common/utils.py @@ -21,14 +21,16 @@ from time import time from urllib.parse import quote, unquote -from flask import request, render_template, redirect, make_response +from flask import make_response, redirect, render_template, request -from rucio.api import authentication as auth, identity +from rucio.api import authentication as auth +from rucio.api import identity from rucio.api.account import account_exists, get_account_info, list_account_attributes from rucio.common.config import config_get, config_get_bool from rucio.common.exception import CannotAuthenticate from rucio.common.extra import import_extras -from rucio.core import identity as identity_core, vo as vo_core +from rucio.core import identity as identity_core +from rucio.core import vo as vo_core from rucio.db.sqla.constants import AccountType, IdentityType EXTRA_MODULES = import_extras(['onelogin']) diff --git a/setup_rucio.py b/setup_rucio.py index 0268ae9ce8..03a9ba1cb5 100644 --- a/setup_rucio.py +++ b/setup_rucio.py @@ -18,17 +18,17 @@ import shutil import sys -from setuptools import setup, find_packages +from setuptools import find_packages, setup if sys.version_info < (3, 9): print('ERROR: Rucio Server requires at least Python 3.9 to run.') sys.exit(1) try: - from setuputil import server_requirements_table, match_define_requirements, get_rucio_version + from setuputil import get_rucio_version, match_define_requirements, server_requirements_table except ImportError: sys.path.append(os.path.abspath(os.path.dirname(__file__))) - from setuputil import server_requirements_table, match_define_requirements, get_rucio_version + from setuputil import get_rucio_version, match_define_requirements, server_requirements_table install_requires, extras_require = match_define_requirements(server_requirements_table) diff --git a/setup_webui.py b/setup_webui.py index ae07c139f8..ad707102d0 100644 --- a/setup_webui.py +++ b/setup_webui.py @@ -18,7 +18,6 @@ from setuptools import setup - if sys.version_info < (3, 9): print('ERROR: Rucio WebUI requires at least Python 3.9 to run.') sys.exit(1) diff --git a/setuputil.py b/setuputil.py index 600519ced3..2647c173fa 100644 --- a/setuputil.py +++ b/setuputil.py @@ -16,8 +16,7 @@ import subprocess import sys -from pkg_resources import safe_name, parse_requirements - +from pkg_resources import parse_requirements, safe_name clients_requirements_table = { 'install_requires': [ diff --git a/tests/conftest.py b/tests/conftest.py index 851357e565..82722923ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,16 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import traceback -import re import functools +import re +import traceback from os import environ from random import choice from string import ascii_uppercase import pytest - _del_test_prefix = functools.partial(re.compile(r'^[Tt][Ee][Ss][Tt]_?').sub, '') # local imports in the fixtures to make this file loadable in e.g. client tests @@ -98,7 +97,7 @@ def vo(): @pytest.fixture(scope='session') def second_vo(): from rucio.common.config import config_get_bool - from rucio.core.vo import vo_exists, add_vo + from rucio.core.vo import add_vo, vo_exists multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) if not multi_vo: pytest.skip('multi_vo mode is not enabled. Running multi_vo tests in single_vo mode would result in failures.') @@ -172,9 +171,8 @@ def download_client(): @pytest.fixture def rest_client(): - from rucio.tests.common import print_response - from flask.testing import FlaskClient + from rucio.tests.common import print_response from rucio.web.rest.flaskapi.v1.main import application class WrappedFlaskClient(FlaskClient): @@ -201,7 +199,7 @@ def open(self, path='/', *args, **kwargs): @pytest.fixture def auth_token(rest_client, long_vo): - from rucio.tests.common import vohdr, headers, loginhdr + from rucio.tests.common import headers, loginhdr, vohdr auth_response = rest_client.get('/auth/userpass', headers=headers(loginhdr('root', 'ddmlab', 'secret'), vohdr(long_vo))) assert auth_response.status_code == 200 @@ -330,9 +328,9 @@ def file_factory(tmp_path_factory): @pytest.fixture def scope_factory(): + from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid from rucio.core.scope import add_scope - from rucio.common.types import InternalAccount, InternalScope def create_scopes(vos, account_name=None): scope_uuid = str(generate_uuid()).lower()[:16] @@ -395,12 +393,13 @@ def __create_in_memory_db_table(name, *columns, **kwargs): declarative base. """ import datetime - from sqlalchemy import Column, DateTime, CheckConstraint + + from rucio.db.sqla.models import ModelBase + from rucio.db.sqla.session import create_engine, get_maker + from sqlalchemy import CheckConstraint, Column, DateTime + from sqlalchemy.orm import registry from sqlalchemy.pool import StaticPool from sqlalchemy.schema import Table - from sqlalchemy.orm import registry - from rucio.db.sqla.models import ModelBase - from rucio.db.sqla.session import get_maker, create_engine engine = create_engine('sqlite://', connect_args={'check_same_thread': False}, poolclass=StaticPool) @@ -439,9 +438,10 @@ def message_mock(): Fixture which overrides the Message table with a private instance """ from unittest import mock - from sqlalchemy import Column + from rucio.common.utils import generate_uuid - from rucio.db.sqla.models import String, PrimaryKeyConstraint, CheckConstraint, Text, Index, GUID + from rucio.db.sqla.models import GUID, CheckConstraint, Index, PrimaryKeyConstraint, String, Text + from sqlalchemy import Column InMemoryMessage = __create_in_memory_db_table( 'message_' + generate_uuid(), @@ -473,10 +473,11 @@ def core_config_mock(request): Accesses to the "models.Config" table are then redirected to this temporary table via mock.patch(). """ from unittest import mock - from sqlalchemy import Column + from rucio.common.utils import generate_uuid - from rucio.db.sqla.models import String, PrimaryKeyConstraint + from rucio.db.sqla.models import PrimaryKeyConstraint, String from rucio.db.sqla.session import get_session + from sqlalchemy import Column # Get the fixture parameters table_content = [] @@ -512,7 +513,8 @@ def file_config_mock(request): via the API, as the server config is not changed. """ from unittest import mock - from rucio.common.config import Config, config_set, config_has_section, config_add_section + + from rucio.common.config import Config, config_add_section, config_has_section, config_set # Get the fixture parameters overrides = [] @@ -541,8 +543,9 @@ def caches_mock(request): The fixture acts by by mock.patch the REGION object in the provided list of modules to mock. """ - from unittest import mock from contextlib import ExitStack + from unittest import mock + from dogpile.cache import make_region caches_to_mock = [] @@ -571,6 +574,7 @@ def metrics_mock(): """ from unittest import mock + from prometheus_client import CollectorRegistry, values with mock.patch('rucio.core.monitor.REGISTRY', new=CollectorRegistry()) as registry, \ diff --git a/tests/mocks/mock_http_server.py b/tests/mocks/mock_http_server.py index 91d0f19b45..e55dcc7770 100644 --- a/tests/mocks/mock_http_server.py +++ b/tests/mocks/mock_http_server.py @@ -14,8 +14,7 @@ # limitations under the License. -from http.server import SimpleHTTPRequestHandler -from http.server import HTTPServer +from http.server import HTTPServer, SimpleHTTPRequestHandler from threading import Thread diff --git a/tests/rsemgr_api_test.py b/tests/rsemgr_api_test.py index 926f9d5066..0d39e3d1a3 100644 --- a/tests/rsemgr_api_test.py +++ b/tests/rsemgr_api_test.py @@ -15,17 +15,16 @@ import itertools import os +import os.path import shutil import tempfile -import os.path from uuid import uuid4 as uuid import pytest - from rucio.common import exception from rucio.common.utils import adler32, md5 from rucio.rse import rsemanager as mgr -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts @skip_rse_tests_with_accounts diff --git a/tests/ruciopytest/__init__.py b/tests/ruciopytest/__init__.py index 16b1dcc82f..a617c9918b 100644 --- a/tests/ruciopytest/__init__.py +++ b/tests/ruciopytest/__init__.py @@ -15,6 +15,7 @@ import enum + class NoParallelGroups(enum.Enum): # Special group. Tests with this marker will never run in parallel with any other test EXCLUSIVE = 'exclusive' @@ -29,4 +30,3 @@ class NoParallelGroups(enum.Enum): # Accessing predefined RSEs XRD = 'xrd' WEB = 'web' - diff --git a/tests/ruciopytest/xdist_noparallel_remote.py b/tests/ruciopytest/xdist_noparallel_remote.py index 4f9ad022bc..a389ca311b 100644 --- a/tests/ruciopytest/xdist_noparallel_remote.py +++ b/tests/ruciopytest/xdist_noparallel_remote.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os -import pytest +import sys -from xdist.remote import WorkerInteractor as OriginalWorkerInteractor, remote_initconfig, setup_config, _prepareconfig, Config # noqa +import pytest +from xdist.remote import Config, _prepareconfig, remote_initconfig, setup_config +from xdist.remote import WorkerInteractor as OriginalWorkerInteractor # noqa from tests.ruciopytest import NoParallelGroups diff --git a/tests/ruciopytest/xdist_noparallel_scheduler.py b/tests/ruciopytest/xdist_noparallel_scheduler.py index 71c22970cb..457dffcf08 100644 --- a/tests/ruciopytest/xdist_noparallel_scheduler.py +++ b/tests/ruciopytest/xdist_noparallel_scheduler.py @@ -16,11 +16,9 @@ import math import pytest - from xdist.scheduler import LoadScheduling -from . import NoParallelGroups -from . import xdist_noparallel_remote +from . import NoParallelGroups, xdist_noparallel_remote @pytest.hookimpl diff --git a/tests/temp_factories.py b/tests/temp_factories.py index 914d4acaaa..dcb21c4053 100644 --- a/tests/temp_factories.py +++ b/tests/temp_factories.py @@ -24,14 +24,14 @@ from rucio.common.schema import get_schema_value from rucio.common.types import InternalScope from rucio.common.utils import execute, generate_uuid -from rucio.core import rse as rse_core from rucio.core import did as did_core +from rucio.core import rse as rse_core from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType from rucio.db.sqla.session import transactional_session from rucio.tests.common import did_name_generator from rucio.tests.common_server import cleanup_db_deps -from sqlalchemy import and_, or_, delete +from sqlalchemy import and_, delete, or_ def _to_external(did): diff --git a/tests/test_abacus_account.py b/tests/test_abacus_account.py index f9ab722e8f..4e325981a4 100644 --- a/tests/test_abacus_account.py +++ b/tests/test_abacus_account.py @@ -15,7 +15,6 @@ import pytest - from rucio.common.schema import get_schema_value from rucio.core.account import get_usage_history from rucio.core.account_counter import update_account_counter_history diff --git a/tests/test_abacus_collection_replica.py b/tests/test_abacus_collection_replica.py index b9f57c9fe8..c48229bf47 100644 --- a/tests/test_abacus_collection_replica.py +++ b/tests/test_abacus_collection_replica.py @@ -15,7 +15,6 @@ import pytest - from rucio.common.exception import DataIdentifierNotFound from rucio.common.schema import get_schema_value from rucio.core.did import add_did, get_did diff --git a/tests/test_abacus_rse.py b/tests/test_abacus_rse.py index b1e276fccb..1698ed37c2 100644 --- a/tests/test_abacus_rse.py +++ b/tests/test_abacus_rse.py @@ -14,7 +14,6 @@ # limitations under the License. import pytest - from rucio.common.schema import get_schema_value from rucio.core.rse import get_rse_usage from rucio.daemons.abacus.rse import rse_update diff --git a/tests/test_account.py b/tests/test_account.py index 3de654f9ca..dfc96f838a 100644 --- a/tests/test_account.py +++ b/tests/test_account.py @@ -16,16 +16,15 @@ from json import loads import pytest - -from rucio.api.account import add_account, account_exists, del_account, update_account, get_account_info +from rucio.api.account import account_exists, add_account, del_account, get_account_info, update_account from rucio.common.config import config_get from rucio.common.exception import AccountNotFound, Duplicate, InvalidObject from rucio.common.types import InternalAccount from rucio.common.utils import generate_uuid as uuid -from rucio.core.account import list_identities, add_account_attribute, list_account_attributes, del_account_attribute +from rucio.core.account import add_account_attribute, del_account_attribute, list_account_attributes, list_identities from rucio.core.identity import add_account_identity, add_identity from rucio.db.sqla.constants import AccountStatus, IdentityType -from rucio.tests.common import account_name_generator, headers, auth, vohdr, loginhdr, skip_non_belleii +from rucio.tests.common import account_name_generator, auth, headers, loginhdr, skip_non_belleii, vohdr class TestAccountCoreApi: diff --git a/tests/test_api_external_representation.py b/tests/test_api_external_representation.py index 4fa6e23e3f..b61e1b1b4b 100644 --- a/tests/test_api_external_representation.py +++ b/tests/test_api_external_representation.py @@ -19,7 +19,6 @@ from json import loads import pytest - import rucio.api.account_limit as api_acc_lim import rucio.api.rse as api_rse import rucio.core.account_counter as account_counter @@ -30,9 +29,8 @@ from rucio.api.replica import add_replicas, get_did_from_pfns, list_replicas from rucio.api.request import get_request_by_did, list_requests, queue_requests from rucio.api.rule import add_replication_rule -from rucio.api.scope import add_scope, list_scopes, get_scopes -from rucio.api.subscription import add_subscription, list_subscriptions, list_subscription_rule_states, \ - get_subscription_by_id +from rucio.api.scope import add_scope, get_scopes, list_scopes +from rucio.api.subscription import add_subscription, get_subscription_by_id, list_subscription_rule_states, list_subscriptions from rucio.common.config import config_get_bool from rucio.common.types import InternalScope from rucio.common.utils import api_update_return_dict, generate_uuid @@ -41,7 +39,7 @@ from rucio.daemons.judge import cleaner from rucio.daemons.reaper import reaper from rucio.db.sqla import constants -from rucio.tests.common import rse_name_generator, did_name_generator +from rucio.tests.common import did_name_generator, rse_name_generator @pytest.fixture(scope='class') diff --git a/tests/test_archive.py b/tests/test_archive.py index 545c42f923..9afc2b794b 100644 --- a/tests/test_archive.py +++ b/tests/test_archive.py @@ -14,9 +14,9 @@ # limitations under the License. from rucio.common.utils import generate_uuid +from rucio.core.did import attach_dids, get_metadata from rucio.core.replica import add_replicas, delete_replicas from rucio.core.rse import add_protocol, update_rse -from rucio.core.did import attach_dids, get_metadata def test_add_and_list_archive(rse_factory, replica_client, did_client, mock_scope): diff --git a/tests/test_auditor.py b/tests/test_auditor.py index b17a9c38bd..efe59a6c81 100644 --- a/tests/test_auditor.py +++ b/tests/test_auditor.py @@ -18,12 +18,10 @@ import multiprocessing import os import tempfile -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta from unittest import mock import pytest - from rucio.daemons import auditor diff --git a/tests/test_auditor_hdfs.py b/tests/test_auditor_hdfs.py index d12c1b60c5..3d19237c58 100644 --- a/tests/test_auditor_hdfs.py +++ b/tests/test_auditor_hdfs.py @@ -19,7 +19,6 @@ from unittest import mock import pytest - from rucio.daemons.auditor import hdfs diff --git a/tests/test_auditor_srmdumps.py b/tests/test_auditor_srmdumps.py index da45f75df8..d703532723 100644 --- a/tests/test_auditor_srmdumps.py +++ b/tests/test_auditor_srmdumps.py @@ -18,7 +18,6 @@ from unittest import mock import pytest - from rucio.daemons.auditor import srmdumps diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 4ba72310dd..bf3c181479 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -14,21 +14,19 @@ # limitations under the License. import base64 - import datetime -import pytest -from requests import session import time -from rucio.api.authentication import get_auth_token_user_pass, get_auth_token_ssh, get_ssh_challenge_token, \ - get_auth_token_saml -from rucio.common.exception import Duplicate, AccessDenied, CannotAuthenticate +import pytest +from requests import session +from rucio.api.authentication import get_auth_token_saml, get_auth_token_ssh, get_auth_token_user_pass, get_ssh_challenge_token +from rucio.common.exception import AccessDenied, CannotAuthenticate, Duplicate from rucio.common.utils import ssh_sign -from rucio.core.identity import add_account_identity, del_account_identity from rucio.core.authentication import strip_x509_proxy_attributes +from rucio.core.identity import add_account_identity, del_account_identity from rucio.db.sqla import models from rucio.db.sqla.constants import IdentityType -from rucio.tests.common import headers, hdrdict, loginhdr, vohdr +from rucio.tests.common import hdrdict, headers, loginhdr, vohdr PUBLIC_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL"\ "614sulXQ7wnIr1aGhGtl8b+HCB/0FhMSMTHwSjX78UbfqEorZ"\ diff --git a/tests/test_automatix.py b/tests/test_automatix.py index 227f7d942a..12f6a8d30f 100644 --- a/tests/test_automatix.py +++ b/tests/test_automatix.py @@ -15,15 +15,14 @@ import json import os +import tempfile from random import choice from string import ascii_uppercase -import tempfile import pytest - -from rucio.common.config import config_add_section, config_has_section, config_set, config_remove_option +from rucio.common.config import config_add_section, config_has_section, config_remove_option, config_set from rucio.common.types import InternalScope -from rucio.core.did import list_dids, list_files, get_metadata +from rucio.core.did import get_metadata, list_dids, list_files from rucio.core.scope import add_scope from rucio.daemons.automatix.automatix import automatix from rucio.rse import rsemanager as rsemgr diff --git a/tests/test_bad_replica.py b/tests/test_bad_replica.py index 16446f80e1..cb4044d9fd 100644 --- a/tests/test_bad_replica.py +++ b/tests/test_bad_replica.py @@ -17,20 +17,17 @@ from json import dumps, loads import pytest - -from rucio.common.exception import RucioException, UnsupportedOperation, InvalidType -from rucio.common.utils import generate_uuid, clean_surls -from rucio.core.did import delete_dids -from rucio.core.replica import (add_replicas, get_replicas_state, list_replicas, - declare_bad_file_replicas, list_bad_replicas, get_bad_pfns, - get_bad_replicas_backlog, list_bad_replicas_status, get_pfn_to_rse) from rucio.client.rseclient import RSEClient +from rucio.common.exception import InvalidType, RucioException, UnsupportedOperation +from rucio.common.utils import clean_surls, generate_uuid +from rucio.core.did import delete_dids +from rucio.core.replica import add_replicas, declare_bad_file_replicas, get_bad_pfns, get_bad_replicas_backlog, get_pfn_to_rse, get_replicas_state, list_bad_replicas, list_bad_replicas_status, list_replicas from rucio.daemons.badreplicas.minos import run as minos_run from rucio.daemons.badreplicas.minos_temporary_expiration import run as minos_temp_run -from rucio.daemons.badreplicas.necromancer import run as necromancer_run from rucio.daemons.badreplicas.necromancer import REGION -from rucio.db.sqla.constants import DIDType, ReplicaState, BadPFNStatus, BadFilesStatus -from rucio.tests.common import headers, auth +from rucio.daemons.badreplicas.necromancer import run as necromancer_run +from rucio.db.sqla.constants import BadFilesStatus, BadPFNStatus, DIDType, ReplicaState +from rucio.tests.common import auth, headers @pytest.fixture @@ -250,7 +247,7 @@ def test_client_add_list_bad_replicas(rse_factory, replica_client, did_client): assert len(replicas) == nbbadrep list_rep.extend(['srm://%s.cern.ch/test_%s/%s/%s' % (rse2_id, rse2_id, tmp_scope, generate_uuid()), ]) - with pytest.raises(InvalidType): + with pytest.raises(InvalidType): # this should fail becase the replica list will now contain a mix of PFNs and dictionaries replica_client.declare_bad_file_replicas(list_rep, 'This is a good reason') diff --git a/tests/test_bb8.py b/tests/test_bb8.py index fd2ee62823..acc8557c7d 100644 --- a/tests/test_bb8.py +++ b/tests/test_bb8.py @@ -16,22 +16,22 @@ from datetime import datetime, timedelta import pytest - from rucio.common.exception import RuleNotFound, UnsupportedOperation from rucio.core.account_limit import set_local_account_limit -from rucio.core.did import attach_dids, set_status, set_metadata +from rucio.core.did import attach_dids, set_metadata, set_status from rucio.core.lock import successful_transfer from rucio.core.replica import add_replicas from rucio.core.rse import add_rse_attribute, fill_rse_expired, get_rse_usage, set_rse_usage -from rucio.core.rule import add_rule, get_rule, delete_rule, update_rule from rucio.core.rse_expression_parser import REGION +from rucio.core.rule import add_rule, delete_rule, get_rule, update_rule from rucio.daemons.abacus.rse import run as run_abacus -from rucio.daemons.bb8.common import rebalance_rule from rucio.daemons.bb8.bb8 import run as bb8_run +from rucio.daemons.bb8.common import rebalance_rule from rucio.daemons.judge.cleaner import rule_cleaner from rucio.daemons.judge.evaluator import re_evaluator from rucio.daemons.undertaker import undertaker from rucio.db.sqla.constants import RuleState + from .test_rule import create_files, tag_generator diff --git a/tests/test_belleii.py b/tests/test_belleii.py index 39aeaab981..df8984ed05 100644 --- a/tests/test_belleii.py +++ b/tests/test_belleii.py @@ -16,10 +16,9 @@ from datetime import datetime import pytest - from rucio.common.exception import InvalidObject from rucio.common.schema.belleii import validate_schema -from rucio.common.utils import generate_uuid, extract_scope +from rucio.common.utils import extract_scope, generate_uuid from rucio.core.config import set as config_set from rucio.tests.common import did_name_generator, skip_non_belleii @@ -106,7 +105,7 @@ def test_dirac_addfile_with_parents_meta(rse_factory, did_factory, root_account, replicas = [rep for rep in replica_client.list_replicas(dids=files)] for replica in replicas: assert {'scope': replica['scope'], 'name': replica['name'], 'rse': list(replica['rses'].keys())[0]} in reps - + # check if metadata if properly created for file and parents for lfn in lfns: scope, name = extract_scope(lfn['lfn'], []) diff --git a/tests/test_bin_rucio.py b/tests/test_bin_rucio.py index 10c02b1dbf..350b1fe13b 100755 --- a/tests/test_bin_rucio.py +++ b/tests/test_bin_rucio.py @@ -18,22 +18,21 @@ import re import tempfile from datetime import datetime, timedelta -from os import remove, unlink, listdir, rmdir, stat, path, environ +from os import environ, listdir, path, remove, rmdir, stat, unlink import pytest - from rucio.client.accountlimitclient import AccountLimitClient +from rucio.client.configclient import ConfigClient from rucio.client.didclient import DIDClient +from rucio.client.lifetimeclient import LifetimeClient from rucio.client.replicaclient import ReplicaClient from rucio.client.rseclient import RSEClient from rucio.client.ruleclient import RuleClient -from rucio.client.configclient import ConfigClient -from rucio.client.lifetimeclient import LifetimeClient from rucio.common.config import config_get, config_get_bool -from rucio.common.types import InternalScope, InternalAccount +from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid, get_tmp_dir, md5, render_json from rucio.rse import rsemanager as rsemgr -from rucio.tests.common import execute, account_name_generator, rse_name_generator, file_generator, scope_name_generator, get_long_vo +from rucio.tests.common import account_name_generator, execute, file_generator, get_long_vo, rse_name_generator, scope_name_generator class TestBinRucio: @@ -362,7 +361,7 @@ def test_upload_file_register_after_upload(self): # removing replica -> file on RSE should be overwritten # (simulating an upload error, where a part of the file is uploaded but the replica is not registered) if 'SUITE' not in environ or environ['SUITE'] != 'client': - from rucio.db.sqla import session, models + from rucio.db.sqla import models, session db_session = session.get_session() internal_scope = InternalScope(self.user, **self.vo) db_session.query(models.RSEFileAssociation).filter_by(name=tmp_file1_name, scope=internal_scope).delete() @@ -1836,9 +1835,9 @@ def test_list_account_limits(self): @pytest.mark.skipif('SUITE' in os.environ and os.environ['SUITE'] == 'client', reason='uses abacus daemon and core functions') def test_list_account_usage(self): """ CLIENT (USER): list account usage. """ - from rucio.db.sqla import session, models from rucio.core.account_counter import increase from rucio.daemons.abacus import account as abacus_account + from rucio.db.sqla import models, session db_session = session.get_session() db_session.query(models.AccountUsage).delete() diff --git a/tests/test_clients.py b/tests/test_clients.py index 8aad23db46..e0a7f3df7d 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -14,16 +14,15 @@ # limitations under the License. from datetime import datetime, timedelta - from os import rename import pytest - from rucio.client.baseclient import BaseClient from rucio.client.client import Client -from rucio.common.config import config_get, config_set, Config +from rucio.common.config import Config, config_get, config_set from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException from rucio.common.utils import execute + from tests.mocks.mock_http_server import MockServer diff --git a/tests/test_common_types.py b/tests/test_common_types.py index 7509aa0ae2..2afaae7869 100644 --- a/tests/test_common_types.py +++ b/tests/test_common_types.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from rucio.common.types import InternalScope, InternalAccount, InternalType +from rucio.common.types import InternalAccount, InternalScope, InternalType class TestInternalType: diff --git a/tests/test_config.py b/tests/test_config.py index 9ab43f3085..e4d0a65d67 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -14,7 +14,6 @@ # limitations under the License. import pytest - import rucio.core.config as core_config from rucio.client.configclient import ConfigClient from rucio.common import exception diff --git a/tests/test_conveyor.py b/tests/test_conveyor.py index 7cb1f396ba..4429b6369b 100644 --- a/tests/test_conveyor.py +++ b/tests/test_conveyor.py @@ -18,15 +18,13 @@ from datetime import datetime, timedelta from tempfile import TemporaryDirectory from unittest.mock import patch -from urllib.parse import urlencode, urlparse, parse_qsl, urlunparse -from sqlalchemy import update +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse import pytest - import rucio.daemons.reaper.reaper -from rucio.common.types import InternalAccount -from rucio.common.utils import generate_uuid, adler32 from rucio.common.exception import ReplicaNotFound, RequestNotFound +from rucio.common.types import InternalAccount +from rucio.common.utils import adler32, generate_uuid from rucio.core import config as core_config from rucio.core import did as did_core from rucio.core import distance as distance_core @@ -40,18 +38,21 @@ from rucio.daemons.conveyor.finisher import finisher from rucio.daemons.conveyor.poller import poller from rucio.daemons.conveyor.preparer import preparer -from rucio.daemons.conveyor.submitter import submitter +from rucio.daemons.conveyor.receiver import GRACEFUL_STOP as receiver_graceful_stop +from rucio.daemons.conveyor.receiver import Receiver, receiver from rucio.daemons.conveyor.stager import stager +from rucio.daemons.conveyor.submitter import submitter from rucio.daemons.conveyor.throttler import throttler -from rucio.daemons.conveyor.receiver import receiver, GRACEFUL_STOP as receiver_graceful_stop, Receiver from rucio.daemons.reaper.reaper import reaper from rucio.db.sqla import models -from rucio.db.sqla.constants import LockState, RequestState, RequestType, ReplicaState, RSEType, RuleState +from rucio.db.sqla.constants import LockState, ReplicaState, RequestState, RequestType, RSEType, RuleState from rucio.db.sqla.session import read_session, transactional_session from rucio.tests.common import skip_rse_tests_with_accounts from rucio.transfertool.fts3 import FTS3Transfertool -from tests.ruciopytest import NoParallelGroups +from sqlalchemy import update + from tests.mocks.mock_http_server import MockServer +from tests.ruciopytest import NoParallelGroups MAX_POLL_WAIT_SECONDS = 100 TEST_FTS_HOST = 'https://fts:8446' @@ -129,9 +130,10 @@ def __get_source(request_id, src_rse_id, scope, name, *, session=None): @pytest.fixture def scitags_mock(core_config_mock): """Run a mock http server which always returns the content of scitags.json from test/inputs""" - from tests.inputs import SCITAGS_JSON from pathlib import Path + from tests.inputs import SCITAGS_JSON + class _SendScitagsJson(MockServer.Handler): def do_GET(self): file_content = Path(SCITAGS_JSON).read_text() diff --git a/tests/test_conveyor_submitter.py b/tests/test_conveyor_submitter.py index 03187eccdf..0b883bfd29 100644 --- a/tests/test_conveyor_submitter.py +++ b/tests/test_conveyor_submitter.py @@ -13,26 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - import itertools from datetime import datetime, timedelta from random import randint from unittest.mock import patch -from sqlalchemy import delete +import pytest from rucio.common.exception import RequestNotFound +from rucio.core import config as core_config from rucio.core import distance as distance_core +from rucio.core import replica as replica_core from rucio.core import request as request_core from rucio.core import rse as rse_core -from rucio.core import replica as replica_core from rucio.core import rule as rule_core -from rucio.core import config as core_config from rucio.daemons.conveyor.submitter import submitter from rucio.daemons.reaper.reaper import reaper -from rucio.db.sqla.models import Request, Source from rucio.db.sqla.constants import RequestState +from rucio.db.sqla.models import Request, Source from rucio.db.sqla.session import read_session, transactional_session +from sqlalchemy import delete + from tests.ruciopytest import NoParallelGroups diff --git a/tests/test_counter.py b/tests/test_counter.py index 0501247d17..6388a703a3 100644 --- a/tests/test_counter.py +++ b/tests/test_counter.py @@ -16,7 +16,6 @@ from time import sleep import pytest - from rucio.core import account_counter, rse_counter from rucio.core.account import get_usage from rucio.daemons.abacus.account import account_update diff --git a/tests/test_credential.py b/tests/test_credential.py index da00230672..108c423369 100644 --- a/tests/test_credential.py +++ b/tests/test_credential.py @@ -16,7 +16,6 @@ import os import pytest - from rucio.common.exception import UnsupportedOperation from rucio.core.credential import get_signed_url from rucio.core.replica import add_replicas diff --git a/tests/test_curl.py b/tests/test_curl.py index c1fea1297d..15b728960a 100644 --- a/tests/test_curl.py +++ b/tests/test_curl.py @@ -17,9 +17,8 @@ import os import pytest - from rucio.common.config import config_get, config_get_bool -from rucio.tests.common import account_name_generator, rse_name_generator, execute, get_long_vo +from rucio.tests.common import account_name_generator, execute, get_long_vo, rse_name_generator class TestCurlRucio: diff --git a/tests/test_daemons.py b/tests/test_daemons.py index 51dfd4fe3a..fbea2766ad 100644 --- a/tests/test_daemons.py +++ b/tests/test_daemons.py @@ -16,7 +16,6 @@ from unittest import mock import pytest - import rucio.db.sqla.util from rucio.common import exception from rucio.daemons.abacus import account, collection_replica, rse @@ -25,7 +24,7 @@ from rucio.daemons.badreplicas import minos, minos_temporary_expiration, necromancer from rucio.daemons.c3po import c3po from rucio.daemons.cache import consumer -from rucio.daemons.conveyor import finisher, poller, receiver, stager, submitter, throttler, preparer +from rucio.daemons.conveyor import finisher, poller, preparer, receiver, stager, submitter, throttler from rucio.daemons.follower import follower from rucio.daemons.hermes import hermes from rucio.daemons.judge import cleaner, evaluator, injector, repairer diff --git a/tests/test_dataset_replicas.py b/tests/test_dataset_replicas.py index 6716df8741..6b975f4b6f 100644 --- a/tests/test_dataset_replicas.py +++ b/tests/test_dataset_replicas.py @@ -14,20 +14,18 @@ # limitations under the License. import pytest -from sqlalchemy.orm.exc import NoResultFound - from rucio.client.didclient import DIDClient from rucio.client.replicaclient import ReplicaClient from rucio.client.ruleclient import RuleClient from rucio.common.exception import InvalidObject from rucio.common.schema import get_schema_value -from rucio.core.did import attach_dids, add_dids -from rucio.core.replica import list_datasets_per_rse, update_collection_replica, \ - get_cleaned_updated_collection_replicas, delete_replicas, add_replicas -from rucio.core.rse import add_rse, del_rse, add_protocol, get_rse_id -from rucio.db.sqla import models, constants +from rucio.core.did import add_dids, attach_dids +from rucio.core.replica import add_replicas, delete_replicas, get_cleaned_updated_collection_replicas, list_datasets_per_rse, update_collection_replica +from rucio.core.rse import add_protocol, add_rse, del_rse, get_rse_id +from rucio.db.sqla import constants, models from rucio.db.sqla.constants import ReplicaState -from rucio.tests.common import rse_name_generator, did_name_generator +from rucio.tests.common import did_name_generator, rse_name_generator +from sqlalchemy.orm.exc import NoResultFound class TestDatasetReplicaClient: diff --git a/tests/test_db.py b/tests/test_db.py index 80f70f1cf7..c465b410d8 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest from unittest.mock import patch -from sqlalchemy import text -from rucio.db.sqla.session import get_session, _get_engine_poolclass, NullPool, QueuePool, SingletonThreadPool +import pytest from rucio.common.exception import InputValidationError +from rucio.db.sqla.session import NullPool, QueuePool, SingletonThreadPool, _get_engine_poolclass, get_session +from sqlalchemy import text def test_db_connection(): @@ -43,8 +43,8 @@ def test_config_poolclass(): @pytest.mark.noparallel(reason='Changes an internal method of MethodView.') def test_pooloverload(): """ DB (WEB): Test response to a DatabaseException due to Pool Overflow """ - from rucio.web.rest.flaskapi.v1.ping import Ping from rucio.common.exception import DatabaseException + from rucio.web.rest.flaskapi.v1.ping import Ping # Create a new ErrorHandlingMethodView as_view ping_view = Ping.as_view('ping') diff --git a/tests/test_did.py b/tests/test_did.py index 85e68ff12c..d547cbae5f 100644 --- a/tests/test_did.py +++ b/tests/test_did.py @@ -16,23 +16,35 @@ from datetime import datetime, timedelta import pytest - -from rucio.api import did -from rucio.api import scope -from rucio.db.sqla.util import json_implemented +from rucio.api import did, scope from rucio.common import exception -from rucio.common.exception import (DataIdentifierNotFound, DataIdentifierAlreadyExists, - InvalidPath, UnsupportedOperation, - UnsupportedStatus, ScopeNotFound, FileAlreadyExists, FileConsistencyMismatch) +from rucio.common.exception import DataIdentifierAlreadyExists, DataIdentifierNotFound, FileAlreadyExists, FileConsistencyMismatch, InvalidPath, ScopeNotFound, UnsupportedOperation, UnsupportedStatus from rucio.common.types import InternalScope from rucio.common.utils import generate_uuid -from rucio.core.did import (list_dids, add_did, delete_dids, get_did_atime, touch_dids, attach_dids, detach_dids, - get_metadata, set_metadata, get_did, get_did_access_cnt, add_did_to_followed, - get_users_following_did, remove_did_from_followed, set_status, list_new_dids, - set_new_dids, bulk_list_files) +from rucio.core.did import ( + add_did, + add_did_to_followed, + attach_dids, + bulk_list_files, + delete_dids, + detach_dids, + get_did, + get_did_access_cnt, + get_did_atime, + get_metadata, + get_users_following_did, + list_dids, + list_new_dids, + remove_did_from_followed, + set_metadata, + set_new_dids, + set_status, + touch_dids, +) from rucio.core.replica import add_replica, get_replica from rucio.db.sqla.constants import DIDType -from rucio.tests.common import rse_name_generator, scope_name_generator, did_name_generator +from rucio.db.sqla.util import json_implemented +from rucio.tests.common import did_name_generator, rse_name_generator, scope_name_generator def skip_without_json(): diff --git a/tests/test_did_meta_plugins.py b/tests/test_did_meta_plugins.py index 7a41ad4a76..a848a0fd32 100644 --- a/tests/test_did_meta_plugins.py +++ b/tests/test_did_meta_plugins.py @@ -16,16 +16,15 @@ from copy import deepcopy import pytest - from rucio.client.didclient import DIDClient from rucio.common.exception import KeyNotFound from rucio.common.utils import generate_uuid -from rucio.core.did import add_did, delete_dids, set_metadata_bulk, set_dids_metadata_bulk -from rucio.core.did_meta_plugins import list_dids, get_metadata, set_metadata +from rucio.core.did import add_did, delete_dids, set_dids_metadata_bulk, set_metadata_bulk +from rucio.core.did_meta_plugins import get_metadata, list_dids, set_metadata from rucio.core.did_meta_plugins.mongo_meta import MongoDidMeta from rucio.core.did_meta_plugins.postgres_meta import ExternalPostgresJSONDidMeta from rucio.db.sqla.util import json_implemented -from rucio.tests.common import skip_rse_tests_with_accounts, did_name_generator +from rucio.tests.common import did_name_generator, skip_rse_tests_with_accounts def skip_without_json(): diff --git a/tests/test_download.py b/tests/test_download.py index 95206d4b6b..c2ce80363b 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -22,8 +22,7 @@ from zipfile import ZipFile import pytest - -from rucio.client.downloadclient import DownloadClient +from rucio.client.downloadclient import DownloadClient, FileDownloadState from rucio.common.config import config_add_section, config_set from rucio.common.exception import InputValidationError, NoFilesDownloaded, RucioException from rucio.common.types import InternalScope @@ -31,10 +30,9 @@ from rucio.core import did as did_core from rucio.core import scope as scope_core from rucio.core.rse import add_protocol -from rucio.client.downloadclient import FileDownloadState from rucio.rse import rsemanager as rsemgr from rucio.rse.protocols.posix import Default as PosixProtocol -from rucio.tests.common import skip_rse_tests_with_accounts, scope_name_generator, file_generator +from rucio.tests.common import file_generator, scope_name_generator, skip_rse_tests_with_accounts @pytest.fixture diff --git a/tests/test_dumper.py b/tests/test_dumper.py index fab6741d59..8c00813bb7 100644 --- a/tests/test_dumper.py +++ b/tests/test_dumper.py @@ -23,11 +23,9 @@ import pytest import requests +from rucio.common import config, dumper +from rucio.tests.common import make_temp_file, mock_open -from rucio.common import config -from rucio.common import dumper -from rucio.tests.common import make_temp_file -from rucio.tests.common import mock_open from .mocks import gfal2 DATE_SECONDS = "2015-03-10 14:00:35" diff --git a/tests/test_dumper_consistency.py b/tests/test_dumper_consistency.py index 9c61f73265..c12f22913e 100644 --- a/tests/test_dumper_consistency.py +++ b/tests/test_dumper_consistency.py @@ -18,12 +18,7 @@ from datetime import datetime from unittest import mock -from rucio.common.dumper.consistency import Consistency -from rucio.common.dumper.consistency import _try_to_advance -from rucio.common.dumper.consistency import compare3 -from rucio.common.dumper.consistency import gnu_sort -from rucio.common.dumper.consistency import min3 -from rucio.common.dumper.consistency import parse_and_filter_file +from rucio.common.dumper.consistency import Consistency, _try_to_advance, compare3, gnu_sort, min3, parse_and_filter_file from rucio.tests.common import make_temp_file RSEPROTOCOL = { diff --git a/tests/test_dumper_data_model.py b/tests/test_dumper_data_model.py index 3447ce3857..1a029f6022 100644 --- a/tests/test_dumper_data_model.py +++ b/tests/test_dumper_data_model.py @@ -20,7 +20,6 @@ import pytest import requests - from rucio.common import dumper from rucio.common.dumper import data_models diff --git a/tests/test_dumper_path_parsing.py b/tests/test_dumper_path_parsing.py index fcba2633fe..9f8c645111 100644 --- a/tests/test_dumper_path_parsing.py +++ b/tests/test_dumper_path_parsing.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from rucio.common.dumper.path_parsing import components -from rucio.common.dumper.path_parsing import remove_prefix +from rucio.common.dumper.path_parsing import components, remove_prefix class TestPathParsing(object): diff --git a/tests/test_filter_engine.py b/tests/test_filter_engine.py index 8f7c093948..05d76e0226 100644 --- a/tests/test_filter_engine.py +++ b/tests/test_filter_engine.py @@ -14,18 +14,17 @@ # limitations under the License. import operator -from datetime import datetime, timedelta import unittest +from datetime import datetime, timedelta import pytest - from rucio.common.exception import DuplicateCriteriaInDIDFilter from rucio.common.utils import generate_uuid from rucio.core.did import add_did from rucio.core.did_meta_plugins import set_metadata +from rucio.core.did_meta_plugins.filter_engine import FilterEngine from rucio.db.sqla import models from rucio.db.sqla.util import json_implemented -from rucio.core.did_meta_plugins.filter_engine import FilterEngine class TestFilterEngineDummy: diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 86a1ecbbfe..46e47a33c0 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -18,8 +18,7 @@ from datetime import datetime, timedelta import pytest - -from rucio.core.heartbeat import live, die, cardiac_arrest, list_payload_counts, list_heartbeats, sanity_check +from rucio.core.heartbeat import cardiac_arrest, die, list_heartbeats, list_payload_counts, live, sanity_check from rucio.db.sqla.models import Heartbeats from rucio.db.sqla.session import transactional_session diff --git a/tests/test_hermes.py b/tests/test_hermes.py index 0f298fb78f..343b5fad04 100644 --- a/tests/test_hermes.py +++ b/tests/test_hermes.py @@ -17,14 +17,13 @@ Hermes Test """ +import time from datetime import datetime from json import loads -import requests -import pytest +import pytest +import requests import stomp -import time - from rucio.common.config import config_get, config_get_int from rucio.core.message import add_message, retrieve_messages, truncate_messages from rucio.daemons.hermes import hermes diff --git a/tests/test_identity.py b/tests/test_identity.py index f9e57751bb..56ea327680 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -17,16 +17,15 @@ import string import pytest - from rucio.common.config import config_get_bool +from rucio.common.exception import IdentityError, IdentityNotFound from rucio.common.types import InternalAccount from rucio.common.utils import generate_uuid as uuid from rucio.core.account import add_account, del_account -from rucio.core.identity import add_identity, del_identity, add_account_identity, del_account_identity, list_identities, verify_identity +from rucio.core.identity import add_account_identity, add_identity, del_account_identity, del_identity, list_identities, verify_identity from rucio.db.sqla.constants import AccountType, IdentityType -from rucio.tests.common import account_name_generator, headers, hdrdict, auth, rfc2253_dn_generator +from rucio.tests.common import account_name_generator, auth, hdrdict, headers, rfc2253_dn_generator from rucio.tests.common_server import get_vo -from rucio.common.exception import IdentityNotFound, IdentityError @pytest.mark.noparallel(reason='adds/removes entities with non-unique names') diff --git a/tests/test_impl_upload_download.py b/tests/test_impl_upload_download.py index bbed155069..7044676fb5 100644 --- a/tests/test_impl_upload_download.py +++ b/tests/test_impl_upload_download.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import path import re +from os import path import pytest - -from rucio.common.utils import generate_uuid as uuid, execute +from rucio.common.utils import execute +from rucio.common.utils import generate_uuid as uuid from rucio.tests.common import skip_rse_tests_with_accounts diff --git a/tests/test_import_export.py b/tests/test_import_export.py index 10cb01a122..1aac3c8a25 100644 --- a/tests/test_import_export.py +++ b/tests/test_import_export.py @@ -16,24 +16,21 @@ from copy import deepcopy import pytest - from rucio.client.exportclient import ExportClient from rucio.client.importclient import ImportClient -from rucio.common.config import config_set, config_add_section, config_has_section +from rucio.common.config import config_add_section, config_has_section, config_set from rucio.common.exception import RSENotFound from rucio.common.types import InternalAccount -from rucio.common.utils import render_json, parse_response +from rucio.common.utils import parse_response, render_json from rucio.core.account import add_account, get_account from rucio.core.distance import add_distance, get_distances from rucio.core.exporter import export_data, export_rses -from rucio.core.identity import add_identity, list_identities, add_account_identity, list_accounts_for_identity +from rucio.core.identity import add_account_identity, add_identity, list_accounts_for_identity, list_identities from rucio.core.importer import import_data, import_rses -from rucio.core.rse import get_rse_id, get_rse_name, add_rse, get_rse, add_protocol, get_rse_protocols, \ - list_rse_attributes, get_rse_limits, set_rse_limits, add_rse_attribute, list_rses, export_rse, del_rse, \ - get_rse_attribute -from rucio.db.sqla import session, models -from rucio.db.sqla.constants import RSEType, AccountType, IdentityType, AccountStatus -from rucio.tests.common import rse_name_generator, headers, auth, hdrdict +from rucio.core.rse import add_protocol, add_rse, add_rse_attribute, del_rse, export_rse, get_rse, get_rse_attribute, get_rse_id, get_rse_limits, get_rse_name, get_rse_protocols, list_rse_attributes, list_rses, set_rse_limits +from rucio.db.sqla import models, session +from rucio.db.sqla.constants import AccountStatus, AccountType, IdentityType, RSEType +from rucio.tests.common import auth, hdrdict, headers, rse_name_generator def check_rse(rse_name, test_data, vo='def'): diff --git a/tests/test_judge_cleaner.py b/tests/test_judge_cleaner.py index dafccb93be..4fe4cb9dbe 100644 --- a/tests/test_judge_cleaner.py +++ b/tests/test_judge_cleaner.py @@ -14,7 +14,6 @@ # limitations under the License. import pytest - from rucio.common.config import config_get_bool from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid as uuid @@ -26,6 +25,7 @@ from rucio.daemons.judge.cleaner import rule_cleaner from rucio.db.sqla.constants import DIDType from rucio.tests.common_server import get_vo + from .test_rule import create_files, tag_generator diff --git a/tests/test_judge_evaluator.py b/tests/test_judge_evaluator.py index e40352c829..63480689a3 100644 --- a/tests/test_judge_evaluator.py +++ b/tests/test_judge_evaluator.py @@ -13,8 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +from typing import TYPE_CHECKING +import pytest from rucio.common.config import config_get_bool from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid as uuid @@ -32,9 +33,9 @@ from rucio.db.sqla.session import transactional_session from rucio.tests.common import RSE_namedtuple from rucio.tests.common_server import get_vo + from .test_rule import create_files, tag_generator -from typing import TYPE_CHECKING if TYPE_CHECKING: from .temp_factories import TemporaryDidFactory, TemporaryRSEFactory diff --git a/tests/test_judge_injector.py b/tests/test_judge_injector.py index 02918697bf..93b52390f2 100644 --- a/tests/test_judge_injector.py +++ b/tests/test_judge_injector.py @@ -16,7 +16,6 @@ from datetime import datetime, timedelta import pytest - from rucio.common.config import config_get_bool from rucio.common.exception import RuleNotFound from rucio.common.types import InternalAccount, InternalScope @@ -25,12 +24,13 @@ from rucio.core.did import add_did, attach_dids from rucio.core.lock import get_replica_locks from rucio.core.rse import add_rse_attribute -from rucio.core.rule import add_rule, get_rule, approve_rule, deny_rule, list_rules +from rucio.core.rule import add_rule, approve_rule, deny_rule, get_rule, list_rules from rucio.daemons.judge.injector import rule_injector from rucio.db.sqla.constants import DIDType, RuleState from rucio.db.sqla.models import ReplicationRule from rucio.db.sqla.session import transactional_session from rucio.tests.common_server import get_vo + from .test_rule import create_files, tag_generator diff --git a/tests/test_judge_repairer.py b/tests/test_judge_repairer.py index 446072993e..0cc57079bd 100644 --- a/tests/test_judge_repairer.py +++ b/tests/test_judge_repairer.py @@ -18,25 +18,24 @@ import pytest from dogpile.cache import make_region - -from rucio.common.config import config_get -from rucio.common.config import config_get_bool +from rucio.common.config import config_get, config_get_bool from rucio.common.types import InternalAccount, InternalScope from rucio.core.account_limit import set_local_account_limit from rucio.core.did import add_did, attach_dids -from rucio.core.lock import successful_transfer, failed_transfer, get_replica_locks +from rucio.core.lock import failed_transfer, get_replica_locks, successful_transfer from rucio.core.replica import get_replica from rucio.core.request import cancel_request_did +from rucio.core.rse import add_rse, add_rse_attribute, update_rse +from rucio.core.rule import add_rule, get_rule from rucio.core.transfer import cancel_transfers -from rucio.core.rse import add_rse_attribute, add_rse, update_rse -from rucio.core.rule import get_rule, add_rule from rucio.daemons.judge.evaluator import re_evaluator from rucio.daemons.judge.repairer import rule_repairer from rucio.db.sqla import models -from rucio.db.sqla.constants import DIDType, RuleState, ReplicaState +from rucio.db.sqla.constants import DIDType, ReplicaState, RuleState from rucio.db.sqla.session import get_session -from rucio.tests.common import rse_name_generator, did_name_generator +from rucio.tests.common import did_name_generator, rse_name_generator from rucio.tests.common_server import get_vo + from .test_rule import create_files, tag_generator diff --git a/tests/test_lifetime.py b/tests/test_lifetime.py index a4d5c7c274..d2cbf4f2cc 100644 --- a/tests/test_lifetime.py +++ b/tests/test_lifetime.py @@ -13,24 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import json - +import os +from configparser import NoSectionError from datetime import datetime, timedelta import pytest -from configparser import NoSectionError - -from rucio.common.exception import UnsupportedOperation, ConfigNotFound +from rucio.common.exception import ConfigNotFound, UnsupportedOperation from rucio.common.policy import REGION from rucio.common.utils import generate_uuid as uuid -from rucio.tests.common import skip_multivo from rucio.core import config as core_config -from rucio.core.rule import add_rule, get_rule -from rucio.core.did import set_metadata, get_metadata +from rucio.core.did import get_metadata, set_metadata from rucio.core.lifetime_exception import add_exception +from rucio.core.rule import add_rule, get_rule from rucio.daemons.atropos.atropos import atropos from rucio.db.sqla.constants import DIDType +from rucio.tests.common import skip_multivo @skip_multivo(reason='only valid for ATLAS') diff --git a/tests/test_message.py b/tests/test_message.py index 2910face4d..32ebd32e4b 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -18,13 +18,12 @@ import string import pytest - -from rucio.db.sqla.models import Message -from rucio.db.sqla.session import get_session from rucio.common.constants import MAX_MESSAGE_LENGTH from rucio.common.exception import InvalidObject, RucioException from rucio.common.utils import generate_uuid -from rucio.core.message import add_message, add_messages, retrieve_messages, delete_messages, truncate_messages +from rucio.core.message import add_message, add_messages, delete_messages, retrieve_messages, truncate_messages +from rucio.db.sqla.models import Message +from rucio.db.sqla.session import get_session @pytest.mark.noparallel(reason='fails when run in parallel') diff --git a/tests/test_meta_conventions.py b/tests/test_meta_conventions.py index 25964ac0a3..9225f2b7ea 100644 --- a/tests/test_meta_conventions.py +++ b/tests/test_meta_conventions.py @@ -14,11 +14,10 @@ # limitations under the License. import pytest - -from rucio.common.exception import InvalidValueForKey, RucioException, UnsupportedValueType, UnsupportedKeyType +from rucio.common.exception import InvalidValueForKey, RucioException, UnsupportedKeyType, UnsupportedValueType from rucio.common.utils import generate_uuid as uuid from rucio.core.meta_conventions import add_key -from rucio.db.sqla import session, models +from rucio.db.sqla import models, session from rucio.db.sqla.constants import DIDType, KeyType diff --git a/tests/test_meta_did.py b/tests/test_meta_did.py index dc8c27759e..9c0d89a8d7 100644 --- a/tests/test_meta_did.py +++ b/tests/test_meta_did.py @@ -14,7 +14,6 @@ # limitations under the License. import pytest - from rucio.common.utils import generate_uuid as uuid diff --git a/tests/test_multi_vo.py b/tests/test_multi_vo.py index e744f45fba..c29c76ea9e 100644 --- a/tests/test_multi_vo.py +++ b/tests/test_multi_vo.py @@ -18,13 +18,12 @@ from os import remove from random import choice from re import search -from string import ascii_uppercase, ascii_lowercase, ascii_letters, digits +from string import ascii_letters, ascii_lowercase, ascii_uppercase, digits from unittest.mock import patch -from urllib.parse import urlparse, parse_qs +from urllib.parse import parse_qs, urlparse import pytest from oic import rndstr - from rucio.api import vo as vo_api from rucio.api.account import add_account, list_accounts from rucio.api.account_limit import set_local_account_limit @@ -42,23 +41,24 @@ from rucio.client.replicaclient import ReplicaClient from rucio.client.subscriptionclient import SubscriptionClient from rucio.client.uploadclient import UploadClient -from rucio.common.config import config_remove_option, config_set, config_has_section, config_add_section -from rucio.common.exception import AccessDenied, Duplicate, InvalidRSEExpression, UnsupportedAccountName, \ - UnsupportedOperation, RucioException +from rucio.common.config import config_add_section, config_has_section, config_remove_option, config_set +from rucio.common.exception import AccessDenied, Duplicate, InvalidRSEExpression, RucioException, UnsupportedAccountName, UnsupportedOperation from rucio.common.types import InternalAccount from rucio.common.utils import generate_uuid, get_tmp_dir, parse_response, ssh_sign from rucio.core import config as core_config from rucio.core.account_counter import add_counter from rucio.core.replica import add_replica -from rucio.core.rse import get_rses_with_attribute_value, get_rse_id, get_rse_vo +from rucio.core.rse import get_rse_id, get_rse_vo, get_rses_with_attribute_value from rucio.core.rse_expression_parser import parse_expression from rucio.core.rule import add_rule from rucio.core.vo import map_vo from rucio.daemons.automatix.automatix import automatix -from rucio.db.sqla import models, session as db_session -from rucio.tests.common import execute, headers, hdrdict, vohdr, auth, loginhdr +from rucio.db.sqla import models +from rucio.db.sqla import session as db_session +from rucio.tests.common import auth, execute, hdrdict, headers, loginhdr, vohdr + from .test_authentication import PRIVATE_KEY, PUBLIC_KEY -from .test_oidc import get_mock_oidc_client, NEW_TOKEN_DICT +from .test_oidc import NEW_TOKEN_DICT, get_mock_oidc_client LOG = getLogger(__name__) diff --git a/tests/test_naming_convention.py b/tests/test_naming_convention.py index 929c204362..42bd3ab3f7 100644 --- a/tests/test_naming_convention.py +++ b/tests/test_naming_convention.py @@ -14,14 +14,10 @@ # limitations under the License. import pytest - from rucio.common.exception import InvalidObject from rucio.common.types import InternalScope from rucio.common.utils import generate_uuid -from rucio.core.naming_convention import (add_naming_convention, - validate_name, - list_naming_conventions, - delete_naming_convention) +from rucio.core.naming_convention import add_naming_convention, delete_naming_convention, list_naming_conventions, validate_name from rucio.db.sqla.constants import KeyType diff --git a/tests/test_oauthmanager.py b/tests/test_oauthmanager.py index b59ff0a393..908c547609 100644 --- a/tests/test_oauthmanager.py +++ b/tests/test_oauthmanager.py @@ -19,12 +19,11 @@ import pytest from oic import rndstr -from sqlalchemy import and_, or_ -from sqlalchemy.sql.expression import true - from rucio.daemons.oauthmanager.oauthmanager import run, stop from rucio.db.sqla import models from rucio.db.sqla.session import get_session +from sqlalchemy import and_, or_ +from sqlalchemy.sql.expression import true new_token_dict = {'access_token': '', 'expires_in': 3599, diff --git a/tests/test_oidc.py b/tests/test_oidc.py index 01d7d79ac3..66ad747877 100644 --- a/tests/test_oidc.py +++ b/tests/test_oidc.py @@ -18,25 +18,20 @@ import uuid from datetime import datetime, timedelta from unittest.mock import MagicMock, patch -from urllib.parse import urlparse, parse_qs +from urllib.parse import parse_qs, urlparse import pytest from jwkest.jwt import JWT from oic import rndstr - from rucio.common.config import config_get_bool -from rucio.common.exception import (CannotAuthenticate, DatabaseException) -from rucio.common.exception import Duplicate +from rucio.common.exception import CannotAuthenticate, DatabaseException, Duplicate from rucio.common.types import InternalAccount from rucio.core.account import add_account from rucio.core.authentication import redirect_auth_oidc, validate_auth_token from rucio.core.identity import add_account_identity -from rucio.core.oidc import (get_auth_oidc, get_token_oidc, get_token_for_account_operation, - EXPECTED_OIDC_AUDIENCE, EXPECTED_OIDC_SCOPE, oidc_identity_string, - _token_cache_get, _token_cache_set) +from rucio.core.oidc import EXPECTED_OIDC_AUDIENCE, EXPECTED_OIDC_SCOPE, _token_cache_get, _token_cache_set, get_auth_oidc, get_token_for_account_operation, get_token_oidc, oidc_identity_string from rucio.db.sqla import models -from rucio.db.sqla.constants import AccountType -from rucio.db.sqla.constants import IdentityType +from rucio.db.sqla.constants import AccountType, IdentityType from rucio.db.sqla.session import get_session from rucio.tests.common_server import get_vo diff --git a/tests/test_permission.py b/tests/test_permission.py index b0102489b0..b6330a6d58 100644 --- a/tests/test_permission.py +++ b/tests/test_permission.py @@ -16,8 +16,8 @@ from rucio.api.permission import has_permission from rucio.common.config import config_get from rucio.common.types import InternalScope -from rucio.core.scope import add_scope from rucio.core.account import add_account_attribute +from rucio.core.scope import add_scope from rucio.tests.common import scope_name_generator, skip_non_belleii diff --git a/tests/test_preparer.py b/tests/test_preparer.py index fb7212fc2b..206cb9ed6e 100644 --- a/tests/test_preparer.py +++ b/tests/test_preparer.py @@ -15,16 +15,15 @@ import pytest - -from rucio.core.distance import get_distances, add_distance +from rucio.core.distance import add_distance, get_distances from rucio.core.replica import add_replicas -from rucio.core.request import list_and_mark_transfer_requests_and_source_replicas, set_transfer_limit, list_transfer_limits, get_request +from rucio.core.request import get_request, list_and_mark_transfer_requests_and_source_replicas, list_transfer_limits, set_transfer_limit +from rucio.core.rse import RseCollection, RseData, add_rse_attribute from rucio.core.transfer import get_supported_transfertools -from rucio.core.rse import add_rse_attribute, RseData, RseCollection from rucio.daemons.conveyor.preparer import preparer -from rucio.db.sqla.session import get_session from rucio.db.sqla import models from rucio.db.sqla.constants import RequestState +from rucio.db.sqla.session import get_session @pytest.fixture diff --git a/tests/test_qos.py b/tests/test_qos.py index 7af95cb30b..7f7394c58f 100644 --- a/tests/test_qos.py +++ b/tests/test_qos.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from rucio.core.rse import update_rse, get_rse +from rucio.core.rse import get_rse, update_rse class TestQoS: diff --git a/tests/test_quarantined_replica.py b/tests/test_quarantined_replica.py index 8a677e3035..1c778d63f6 100644 --- a/tests/test_quarantined_replica.py +++ b/tests/test_quarantined_replica.py @@ -14,11 +14,9 @@ # limitations under the License. import pytest - from rucio.common.config import config_get_bool from rucio.common.utils import generate_uuid -from rucio.core.quarantined_replica import add_quarantined_replicas, list_quarantined_replicas, \ - delete_quarantined_replicas +from rucio.core.quarantined_replica import add_quarantined_replicas, delete_quarantined_replicas, list_quarantined_replicas from rucio.core.rse import get_rse_id from rucio.tests.common_server import get_vo diff --git a/tests/test_reaper.py b/tests/test_reaper.py index a3674eeb79..71d9596243 100644 --- a/tests/test_reaper.py +++ b/tests/test_reaper.py @@ -16,14 +16,9 @@ from datetime import datetime, timedelta import pytest -from sqlalchemy import and_, or_ - from rucio.api import replica as replica_api from rucio.api import rse as rse_api -from rucio.db.sqla import models -from rucio.db.sqla.constants import OBSOLETE -from rucio.db.sqla.session import get_session -from rucio.common.exception import ReplicaNotFound, DataIdentifierNotFound +from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid from rucio.core import did as did_core @@ -31,12 +26,16 @@ from rucio.core import replica as replica_core from rucio.core import rse as rse_core from rucio.core import rule as rule_core -from rucio.daemons.reaper.reaper import reaper from rucio.daemons.reaper.dark_reaper import reaper as dark_reaper +from rucio.daemons.reaper.reaper import reaper from rucio.daemons.reaper.reaper import run as run_reaper +from rucio.db.sqla import models +from rucio.db.sqla.constants import OBSOLETE from rucio.db.sqla.models import ConstituentAssociationHistory -from rucio.db.sqla.session import read_session +from rucio.db.sqla.session import get_session, read_session from rucio.tests.common import rse_name_generator, skip_rse_tests_with_accounts +from sqlalchemy import and_, or_ + from tests.ruciopytest import NoParallelGroups __mock_protocol = {'scheme': 'MOCK', diff --git a/tests/test_replica.py b/tests/test_replica.py index 20485eee1f..b24caa0edf 100644 --- a/tests/test_replica.py +++ b/tests/test_replica.py @@ -18,36 +18,29 @@ import time from datetime import datetime, timedelta from json import dumps +from typing import TYPE_CHECKING from unittest import mock from xml.etree import ElementTree import pytest import xmltodict -from werkzeug.datastructures import MultiDict -from werkzeug.datastructures import Headers - from rucio.client.ruleclient import RuleClient -from rucio.common.exception import (DataIdentifierNotFound, AccessDenied, RucioException, - ReplicaIsLocked, ReplicaNotFound, ScopeNotFound, - DatabaseException, InputValidationError) +from rucio.common.exception import AccessDenied, DatabaseException, DataIdentifierNotFound, InputValidationError, ReplicaIsLocked, ReplicaNotFound, RucioException, ScopeNotFound from rucio.common.schema import get_schema_value -from rucio.common.utils import generate_uuid, clean_surls, parse_response +from rucio.common.utils import clean_surls, generate_uuid, parse_response from rucio.core.config import set as cconfig_set -from rucio.core.did import add_did, attach_dids, get_did, set_status, list_files, get_did_atime -from rucio.core.replica import (add_replica, add_replicas, delete_replicas, get_replicas_state, - get_replica, list_replicas, update_replica_state, - get_RSEcoverage_of_dataset, get_replica_atime, - touch_replica, get_bad_pfns, set_tombstone, add_bad_dids) +from rucio.core.did import add_did, attach_dids, get_did, get_did_atime, list_files, set_status +from rucio.core.replica import add_bad_dids, add_replica, add_replicas, delete_replicas, get_bad_pfns, get_replica, get_replica_atime, get_replicas_state, get_RSEcoverage_of_dataset, list_replicas, set_tombstone, touch_replica, update_replica_state from rucio.core.rse import add_protocol, add_rse_attribute, del_rse_attribute from rucio.daemons.badreplicas.minos import minos from rucio.daemons.badreplicas.minos_temporary_expiration import minos_tu_expiration from rucio.db.sqla import models -from rucio.db.sqla.constants import DIDType, ReplicaState, BadPFNStatus, OBSOLETE +from rucio.db.sqla.constants import OBSOLETE, BadPFNStatus, DIDType, ReplicaState from rucio.db.sqla.session import transactional_session from rucio.rse import rsemanager as rsemgr -from rucio.tests.common import execute, headers, auth, Mime, accept, did_name_generator +from rucio.tests.common import Mime, accept, auth, did_name_generator, execute, headers +from werkzeug.datastructures import Headers, MultiDict -from typing import TYPE_CHECKING if TYPE_CHECKING: from .temp_factories import TemporaryRSEFactory diff --git a/tests/test_replica_recoverer.py b/tests/test_replica_recoverer.py index 528c0d542f..dd22bb29b0 100644 --- a/tests/test_replica_recoverer.py +++ b/tests/test_replica_recoverer.py @@ -14,19 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from datetime import datetime, timedelta from os import remove from time import sleep import pytest -import json - -from rucio.core.replica import (update_replica_state, list_replicas, list_bad_replicas_status) -from rucio.core.rse import add_rse_attribute -from rucio.core.did import set_metadata from rucio.core import rse_expression_parser +from rucio.core.did import set_metadata +from rucio.core.replica import list_bad_replicas_status, list_replicas, update_replica_state +from rucio.core.rse import add_rse_attribute from rucio.daemons.replicarecoverer.suspicious_replica_recoverer import run, stop -from rucio.db.sqla.constants import DIDType, BadFilesStatus, ReplicaState +from rucio.db.sqla.constants import BadFilesStatus, DIDType, ReplicaState from rucio.tests.common import execute diff --git a/tests/test_replica_sorting.py b/tests/test_replica_sorting.py index 4f6c5134c7..a83f0391e7 100644 --- a/tests/test_replica_sorting.py +++ b/tests/test_replica_sorting.py @@ -16,19 +16,19 @@ import copy import json import os +from tempfile import mkstemp from unittest import mock from urllib.parse import urlparse -from tempfile import mkstemp import geoip2.database import pytest - -from rucio.common.utils import parse_replicas_from_string from rucio.common.config import config_get -from rucio.core import rse_expression_parser, replica_sorter +from rucio.common.utils import parse_replicas_from_string +from rucio.core import replica_sorter, rse_expression_parser from rucio.core.replica import add_replicas, delete_replicas -from rucio.core.rse import add_rse, del_rse, add_rse_attribute, add_protocol, del_rse_attribute -from rucio.tests.common import rse_name_generator, headers, auth, vohdr, Mime, accept +from rucio.core.rse import add_protocol, add_rse, add_rse_attribute, del_rse, del_rse_attribute +from rucio.tests.common import Mime, accept, auth, headers, rse_name_generator, vohdr + from .inputs import GEOIP_LITE2_CITY_TEST_DB LOCATION_TO_IP = { diff --git a/tests/test_request.py b/tests/test_request.py index 03e0f1e15e..f79864675a 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -18,16 +18,15 @@ from typing import Union import pytest - from rucio.common.config import config_get_bool from rucio.common.utils import generate_uuid, parse_response from rucio.core.distance import add_distance from rucio.core.replica import add_replica -from rucio.core.request import queue_requests, get_request_by_did, list_requests, list_requests_history, set_transfer_limit, TransferStatsManager +from rucio.core.request import TransferStatsManager, get_request_by_did, list_requests, list_requests_history, queue_requests, set_transfer_limit from rucio.core.rse import add_rse_attribute -from rucio.db.sqla import models, constants -from rucio.db.sqla.constants import RequestType, RequestState -from rucio.tests.common import vohdr, hdrdict, headers, auth +from rucio.db.sqla import constants, models +from rucio.db.sqla.constants import RequestState, RequestType +from rucio.tests.common import auth, hdrdict, headers, vohdr @pytest.mark.parametrize("file_config_mock", [ diff --git a/tests/test_root_proxy.py b/tests/test_root_proxy.py index 8498eedcb4..286f417150 100644 --- a/tests/test_root_proxy.py +++ b/tests/test_root_proxy.py @@ -16,11 +16,10 @@ from urllib.parse import urlencode import pytest - from rucio.core.config import set as config_set from rucio.core.replica import add_replicas, delete_replicas -from rucio.core.rse import add_rse, add_rse_attribute, del_rse, add_protocol -from rucio.tests.common import rse_name_generator, vohdr, headers +from rucio.core.rse import add_protocol, add_rse, add_rse_attribute, del_rse +from rucio.tests.common import headers, rse_name_generator, vohdr client_location_without_proxy = {'ip': '192.168.0.1', 'fqdn': 'anomalous-materials.blackmesa.com', diff --git a/tests/test_rse.py b/tests/test_rse.py index e4d45943b6..06ef631054 100644 --- a/tests/test_rse.py +++ b/tests/test_rse.py @@ -14,32 +14,40 @@ # limitations under the License. import pytest - from rucio.client.replicaclient import ReplicaClient from rucio.common import exception -from rucio.common.exception import (Duplicate, RSENotFound, RSEProtocolNotSupported, - InvalidObject, ResourceTemporaryUnavailable, - RSEAttributeNotFound, RSEOperationNotSupported, - InputValidationError) +from rucio.common.exception import Duplicate, InputValidationError, InvalidObject, ResourceTemporaryUnavailable, RSEAttributeNotFound, RSENotFound, RSEOperationNotSupported, RSEProtocolNotSupported from rucio.common.schema import get_schema_value -from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, CHECKSUM_KEY -from rucio.core.account_limit import set_local_account_limit, get_rse_account_usage +from rucio.common.utils import CHECKSUM_KEY, GLOBALLY_SUPPORTED_CHECKSUMS +from rucio.core.account_limit import get_rse_account_usage, set_local_account_limit from rucio.core.did import add_did, attach_dids +from rucio.core.request import delete_transfer_limit, set_transfer_limit +from rucio.core.rse import ( + add_rse, + add_rse_attribute, + del_rse, + del_rse_attribute, + get_rse, + get_rse_attribute, + get_rse_id, + get_rse_protocols, + get_rse_supported_checksums_from_attributes, + get_rse_transfer_limits, + list_rse_attributes, + list_rses, + parse_checksum_support_attribute, + restore_rse, + rse_exists, + rse_is_empty, + update_rse, +) from rucio.core.rule import add_rule -from rucio.core.request import set_transfer_limit, delete_transfer_limit -from rucio.core.rse import (add_rse, get_rse_id, del_rse, restore_rse, list_rses, - rse_exists, add_rse_attribute, list_rse_attributes, - get_rse_transfer_limits, - get_rse_protocols, - del_rse_attribute, get_rse_attribute, get_rse, rse_is_empty, - parse_checksum_support_attribute, - get_rse_supported_checksums_from_attributes, - update_rse) from rucio.daemons.abacus.account import account_update -from rucio.db.sqla import session, models -from rucio.db.sqla.constants import RSEType, DIDType +from rucio.db.sqla import models, session +from rucio.db.sqla.constants import DIDType, RSEType from rucio.rse import rsemanager as mgr -from rucio.tests.common import rse_name_generator, hdrdict, auth, headers, did_name_generator +from rucio.tests.common import auth, did_name_generator, hdrdict, headers, rse_name_generator + from .test_rule import create_files diff --git a/tests/test_rse_expression_parser.py b/tests/test_rse_expression_parser.py index 9d3aab5e32..4a9950daed 100644 --- a/tests/test_rse_expression_parser.py +++ b/tests/test_rse_expression_parser.py @@ -14,13 +14,11 @@ # limitations under the License. from random import choice -from string import ascii_uppercase, ascii_lowercase +from string import ascii_lowercase, ascii_uppercase import pytest - from rucio.common.exception import InvalidRSEExpression, RSEWriteBlocked -from rucio.core import rse -from rucio.core import rse_expression_parser +from rucio.core import rse, rse_expression_parser def attribute_name_generator(size=10): diff --git a/tests/test_rse_lfn2path.py b/tests/test_rse_lfn2path.py index 3084985cda..468b5a9866 100644 --- a/tests/test_rse_lfn2path.py +++ b/tests/test_rse_lfn2path.py @@ -15,14 +15,12 @@ import copy import os +from configparser import NoOptionError, NoSectionError import pytest - from rucio.common import config from rucio.rse.protocols.protocol import RSEDeterministicTranslation -from configparser import NoOptionError, NoSectionError - @pytest.mark.noparallel(reason='uses pre-defined RSE, changes global configuration value') class TestDeterministicTranslation: diff --git a/tests/test_rse_protocol_gfal2.py b/tests/test_rse_protocol_gfal2.py index b1839d8252..b0e2ff70ad 100644 --- a/tests/test_rse_protocol_gfal2.py +++ b/tests/test_rse_protocol_gfal2.py @@ -16,10 +16,10 @@ import os import pytest - from rucio.common.utils import execute from rucio.rse import rsemanager as mgr -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_gfal2_impl.py b/tests/test_rse_protocol_gfal2_impl.py index 67c916728c..6312060468 100644 --- a/tests/test_rse_protocol_gfal2_impl.py +++ b/tests/test_rse_protocol_gfal2_impl.py @@ -16,10 +16,10 @@ import os import pytest - from rucio.common.utils import execute from rucio.rse import rsemanager -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_posix.py b/tests/test_rse_protocol_posix.py index 47130a459a..ad0e2b4c5b 100644 --- a/tests/test_rse_protocol_posix.py +++ b/tests/test_rse_protocol_posix.py @@ -17,9 +17,9 @@ import shutil import pytest - from rucio.rse import rsemanager as mgr -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_rclone.py b/tests/test_rse_protocol_rclone.py index 230e0c8700..c327a896d4 100644 --- a/tests/test_rse_protocol_rclone.py +++ b/tests/test_rse_protocol_rclone.py @@ -16,10 +16,10 @@ import os import pytest - -from rucio.common.utils import execute, PREFERRED_CHECKSUM, set_preferred_checksum +from rucio.common.utils import PREFERRED_CHECKSUM, execute, set_preferred_checksum from rucio.rse import rsemanager -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_rsync.py b/tests/test_rse_protocol_rsync.py index 458bee4235..20c8cecf04 100644 --- a/tests/test_rse_protocol_rsync.py +++ b/tests/test_rse_protocol_rsync.py @@ -16,10 +16,10 @@ import os import pytest - -from rucio.common.utils import execute, PREFERRED_CHECKSUM, set_preferred_checksum +from rucio.common.utils import PREFERRED_CHECKSUM, execute, set_preferred_checksum from rucio.rse import rsemanager -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_srm.py b/tests/test_rse_protocol_srm.py index 0b081beed9..5c2555545c 100644 --- a/tests/test_rse_protocol_srm.py +++ b/tests/test_rse_protocol_srm.py @@ -16,10 +16,10 @@ import os import pytest - from rucio.common.utils import execute from rucio.rse import rsemanager as mgr -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_ssh.py b/tests/test_rse_protocol_ssh.py index 03dd084d97..fb57844708 100644 --- a/tests/test_rse_protocol_ssh.py +++ b/tests/test_rse_protocol_ssh.py @@ -16,10 +16,10 @@ import os import pytest - -from rucio.common.utils import execute, PREFERRED_CHECKSUM, set_preferred_checksum +from rucio.common.utils import PREFERRED_CHECKSUM, execute, set_preferred_checksum from rucio.rse import rsemanager -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_webdav.py b/tests/test_rse_protocol_webdav.py index 8a06632210..1c1f077125 100644 --- a/tests/test_rse_protocol_webdav.py +++ b/tests/test_rse_protocol_webdav.py @@ -17,10 +17,10 @@ import pytest import requests - from rucio.common.exception import FileReplicaAlreadyExists from rucio.rse import rsemanager -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_protocol_xrootd.py b/tests/test_rse_protocol_xrootd.py index 235f2557e3..505907037f 100644 --- a/tests/test_rse_protocol_xrootd.py +++ b/tests/test_rse_protocol_xrootd.py @@ -16,10 +16,10 @@ import os import pytest - from rucio.common.utils import execute from rucio.rse import rsemanager -from rucio.tests.common import skip_rse_tests_with_accounts, load_test_conf_file +from rucio.tests.common import load_test_conf_file, skip_rse_tests_with_accounts + from .rsemgr_api_test import MgrTestCases diff --git a/tests/test_rse_selector.py b/tests/test_rse_selector.py index 8d47257964..5b5b7a3e8d 100644 --- a/tests/test_rse_selector.py +++ b/tests/test_rse_selector.py @@ -15,10 +15,9 @@ import pytest - from rucio.common.exception import InsufficientAccountLimit, InsufficientTargetRSEs -from rucio.core.account_counter import update_account_counter, increase -from rucio.core.account_limit import set_local_account_limit, set_global_account_limit +from rucio.core.account_counter import increase, update_account_counter +from rucio.core.account_limit import set_global_account_limit, set_local_account_limit from rucio.core.rse_selector import RSESelector diff --git a/tests/test_rucio_server.py b/tests/test_rucio_server.py index d1a491d310..d4466427fb 100644 --- a/tests/test_rucio_server.py +++ b/tests/test_rucio_server.py @@ -14,7 +14,6 @@ # limitations under the License. import pytest - import rucio.common.test_rucio_server as server_test diff --git a/tests/test_rule.py b/tests/test_rule.py index 2c779f3542..19a7a85224 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -13,45 +13,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple import json import random import string +from collections import namedtuple from logging import getLogger +from typing import TYPE_CHECKING import pytest - import rucio.api.rule from rucio.api.account import add_account from rucio.client.ruleclient import RuleClient from rucio.common.config import config_get_bool -from rucio.common.exception import (RucioException, RuleNotFound, AccessDenied, InsufficientAccountLimit, DuplicateRule, RSEWriteBlocked, - RSEOverQuota, RuleReplaceFailed, ManualRuleApprovalBlocked, InputValidationError, - UnsupportedOperation, InvalidValueForKey, InvalidSourceReplicaExpression) +from rucio.common.exception import ( + AccessDenied, + DuplicateRule, + InputValidationError, + InsufficientAccountLimit, + InvalidSourceReplicaExpression, + InvalidValueForKey, + ManualRuleApprovalBlocked, + RSEOverQuota, + RSEWriteBlocked, + RucioException, + RuleNotFound, + RuleReplaceFailed, + UnsupportedOperation, +) from rucio.common.policy import get_policy from rucio.common.schema import get_schema_value from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid as uuid -from rucio.core.account import add_account_attribute, get_usage, get_account -from rucio.core.account_limit import set_local_account_limit, set_global_account_limit +from rucio.core.account import add_account_attribute, get_account, get_usage +from rucio.core.account_limit import set_global_account_limit, set_local_account_limit from rucio.core.did import add_did, attach_dids, set_status -from rucio.core.lock import get_replica_locks, get_dataset_locks, successful_transfer +from rucio.core.lock import get_dataset_locks, get_replica_locks, successful_transfer from rucio.core.replica import add_replica, get_replica from rucio.core.request import get_request_by_did -from rucio.core.rse import add_rse_attribute, add_rse, update_rse, get_rse_id, del_rse, del_rse_attribute, set_rse_limits +from rucio.core.rse import add_rse, add_rse_attribute, del_rse, del_rse_attribute, get_rse_id, set_rse_limits, update_rse from rucio.core.rse_counter import get_counter as get_rse_counter -from rucio.core.rule import add_rule, get_rule, delete_rule, add_rules, update_rule, reduce_rule, move_rule, list_rules +from rucio.core.rule import add_rule, add_rules, delete_rule, get_rule, list_rules, move_rule, reduce_rule, update_rule from rucio.core.scope import add_scope from rucio.daemons.abacus.account import account_update from rucio.daemons.abacus.rse import rse_update from rucio.daemons.judge.evaluator import re_evaluator from rucio.db.sqla import models -from rucio.db.sqla.constants import DIDType, OBSOLETE, RuleState, LockState +from rucio.db.sqla.constants import OBSOLETE, DIDType, LockState, RuleState from rucio.db.sqla.session import transactional_session -from rucio.tests.common import rse_name_generator, account_name_generator, did_name_generator +from rucio.tests.common import account_name_generator, did_name_generator, rse_name_generator from rucio.tests.common_server import get_vo -from typing import TYPE_CHECKING if TYPE_CHECKING: from .temp_factories import TemporaryDidFactory, TemporaryRSEFactory @@ -181,9 +192,10 @@ def test_move_rule_invalid_source_replica_expression( evaluates to an empty set. """ - from dogpile.cache.api import NO_VALUE from hashlib import sha256 + from dogpile.cache.api import NO_VALUE + # create RSEs A, B, C (structured into namedtuples) rseA, rseB, rseC = ( RSE_namedtuple(*rse_factory.make_mock_rse()) for _ in range(3) diff --git a/tests/test_schema_cms.py b/tests/test_schema_cms.py index b3538c8ded..21e363c3a5 100644 --- a/tests/test_schema_cms.py +++ b/tests/test_schema_cms.py @@ -14,11 +14,9 @@ # limitations under the License. import pytest - from rucio.common.exception import InvalidObject from rucio.common.schema.cms import validate_schema - # Some tests adapted from https://github.com/dmwm/WMCore/blob/master/test/python/WMCore_t/Lexicon_t.py class TestSchemaCMS: diff --git a/tests/test_scope.py b/tests/test_scope.py index bc5f672be9..1a41c77415 100644 --- a/tests/test_scope.py +++ b/tests/test_scope.py @@ -16,12 +16,11 @@ from json import loads import pytest - -from rucio.common.exception import AccountNotFound, Duplicate, ScopeNotFound, InvalidObject +from rucio.common.exception import AccountNotFound, Duplicate, InvalidObject, ScopeNotFound from rucio.common.types import InternalScope from rucio.common.utils import generate_uuid as uuid -from rucio.core.scope import get_scopes, add_scope, is_scope_owner -from rucio.tests.common import account_name_generator, scope_name_generator, headers, auth, hdrdict +from rucio.core.scope import add_scope, get_scopes, is_scope_owner +from rucio.tests.common import account_name_generator, auth, hdrdict, headers, scope_name_generator class TestScopeCoreApi: diff --git a/tests/test_subscription.py b/tests/test_subscription.py index a84ce98810..a93b907e90 100644 --- a/tests/test_subscription.py +++ b/tests/test_subscription.py @@ -18,23 +18,20 @@ from json.decoder import JSONDecodeError import pytest - -from rucio.api.subscription import list_subscriptions, add_subscription, update_subscription, \ - list_subscription_rule_states, get_subscription_by_id -from rucio.db.sqla.constants import RuleState -from rucio.common.exception import InvalidObject, SubscriptionNotFound, SubscriptionDuplicate +from rucio.api.subscription import add_subscription, get_subscription_by_id, list_subscription_rule_states, list_subscriptions, update_subscription +from rucio.common.exception import InvalidObject, SubscriptionDuplicate, SubscriptionNotFound from rucio.common.schema import get_schema_value from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid as uuid +from rucio.core import subscription as subscription_core from rucio.core.account import add_account -from rucio.core.did import add_did, set_new_dids, list_new_dids, attach_dids, set_status -from rucio.core.rule import add_rule +from rucio.core.did import add_did, attach_dids, list_new_dids, set_new_dids, set_status from rucio.core.rse import add_rse_attribute +from rucio.core.rule import add_rule from rucio.core.scope import add_scope -from rucio.core import subscription as subscription_core -from rucio.daemons.transmogrifier.transmogrifier import run, get_subscriptions -from rucio.db.sqla.constants import AccountType, DIDType -from rucio.tests.common import headers, auth, did_name_generator, rse_name_generator +from rucio.daemons.transmogrifier.transmogrifier import get_subscriptions, run +from rucio.db.sqla.constants import AccountType, DIDType, RuleState +from rucio.tests.common import auth, did_name_generator, headers, rse_name_generator class TestSubscriptionCoreApi: diff --git a/tests/test_throttler.py b/tests/test_throttler.py index 42f97c3c2d..f159c9183e 100644 --- a/tests/test_throttler.py +++ b/tests/test_throttler.py @@ -16,21 +16,27 @@ from datetime import datetime, timedelta import pytest -from sqlalchemy import delete - from rucio.common.utils import generate_uuid -from rucio.core.did import attach_dids, add_did +from rucio.core.did import add_did, attach_dids from rucio.core.distance import add_distance from rucio.core.replica import add_replica -from rucio.core.request import (queue_requests, get_request_by_did, release_waiting_requests_per_deadline, - release_all_waiting_requests, release_waiting_requests_fifo, release_waiting_requests_grouped_fifo, - release_waiting_requests_per_free_volume, delete_transfer_limit) -from rucio.daemons.conveyor.throttler import throttler +from rucio.core.request import ( + delete_transfer_limit, + get_request_by_did, + queue_requests, + release_all_waiting_requests, + release_waiting_requests_fifo, + release_waiting_requests_grouped_fifo, + release_waiting_requests_per_deadline, + release_waiting_requests_per_free_volume, +) from rucio.daemons.conveyor.preparer import preparer +from rucio.daemons.conveyor.throttler import throttler from rucio.db.sqla import models -from rucio.db.sqla.session import transactional_session, get_session -from rucio.db.sqla.constants import DIDType, RequestType, RequestState, TransferLimitDirection +from rucio.db.sqla.constants import DIDType, RequestState, RequestType, TransferLimitDirection +from rucio.db.sqla.session import get_session, transactional_session from rucio.tests.common import skiplimitedsql +from sqlalchemy import delete @pytest.fixture diff --git a/tests/test_tpc.py b/tests/test_tpc.py index 7c96f0abca..a7f5829346 100644 --- a/tests/test_tpc.py +++ b/tests/test_tpc.py @@ -13,23 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time import datetime -import pytest import hashlib import re +import time +import pytest +from rucio.client.rseclient import RSEClient +from rucio.client.ruleclient import RuleClient +from rucio.common.utils import generate_uuid, run_cmd_process from rucio.core.request import get_request_by_did, list_and_mark_transfer_requests_and_source_replicas +from rucio.core.rule import add_rule from rucio.core.topology import Topology from rucio.core.transfer import ProtocolFactory -from rucio.core.rule import add_rule -from rucio.common.utils import generate_uuid -from rucio.daemons.judge.evaluator import re_evaluator -from rucio.daemons.conveyor import submitter, poller, finisher +from rucio.daemons.conveyor import finisher, poller, submitter from rucio.daemons.conveyor.common import build_transfer_paths -from rucio.client.rseclient import RSEClient -from rucio.client.ruleclient import RuleClient -from rucio.common.utils import run_cmd_process +from rucio.daemons.judge.evaluator import re_evaluator MAX_POLL_WAIT_SECONDS = 60 diff --git a/tests/test_trace.py b/tests/test_trace.py index c462c9a7f2..1630481b7f 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -14,15 +14,15 @@ # limitations under the License. import datetime +import json +import logging import time import uuid -import logging + import pytest -import json +from rucio.common.exception import InvalidObject from rucio.common.schema.generic import IPv4orIPv6 from rucio.core.trace import SCHEMAS, validate_schema -from rucio.common.exception import InvalidObject - LOGGER = logging.getLogger(__name__) diff --git a/tests/test_transfer.py b/tests/test_transfer.py index ad8cf64f8b..815c331db4 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -13,24 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest -from concurrent.futures import ThreadPoolExecutor import datetime +from concurrent.futures import ThreadPoolExecutor +import pytest from rucio.common.exception import NoDistance +from rucio.common.utils import generate_uuid +from rucio.core import request as request_core +from rucio.core import rse as rse_core +from rucio.core import rule as rule_core from rucio.core.distance import add_distance from rucio.core.replica import add_replicas from rucio.core.request import list_and_mark_transfer_requests_and_source_replicas -from rucio.core.transfer import build_transfer_paths, ProtocolFactory -from rucio.core.topology import get_hops, Topology -from rucio.core import rule as rule_core -from rucio.core import request as request_core -from rucio.core import rse as rse_core +from rucio.core.topology import Topology, get_hops +from rucio.core.transfer import ProtocolFactory, build_transfer_paths +from rucio.daemons.conveyor.common import assign_paths_to_transfertool_and_create_hops, pick_and_prepare_submission_path from rucio.db.sqla import models -from rucio.db.sqla.constants import RSEType, RequestState +from rucio.db.sqla.constants import RequestState, RSEType from rucio.db.sqla.session import get_session -from rucio.common.utils import generate_uuid -from rucio.daemons.conveyor.common import assign_paths_to_transfertool_and_create_hops, pick_and_prepare_submission_path def _prepare_submission(rses): diff --git a/tests/test_transfer_plugins.py b/tests/test_transfer_plugins.py index 6eeecd1ee1..391ad1f410 100644 --- a/tests/test_transfer_plugins.py +++ b/tests/test_transfer_plugins.py @@ -13,22 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import logging -from rucio.transfertool.fts3 import FTS3Transfertool, build_job_params +import pytest +from rucio.core import distance as distance_core +from rucio.core import replica as replica_core +from rucio.core import rule as rule_core +from rucio.core.request import list_and_mark_transfer_requests_and_source_replicas from rucio.core.topology import Topology from rucio.core.transfer import ProtocolFactory, build_transfer_paths -from rucio.core.request import list_and_mark_transfer_requests_and_source_replicas - from rucio.db.sqla.session import get_session +from rucio.transfertool.fts3 import FTS3Transfertool, build_job_params from rucio.transfertool.fts3_plugins import FTS3TapeMetadataPlugin -from rucio.core import distance as distance_core -from rucio.core import replica as replica_core -from rucio.core import rule as rule_core - - mock_session = get_session() MAX_POLL_WAIT_SECONDS = 60 diff --git a/tests/test_undertaker.py b/tests/test_undertaker.py index 58f39fe7bf..a14b935853 100644 --- a/tests/test_undertaker.py +++ b/tests/test_undertaker.py @@ -17,18 +17,17 @@ from logging import getLogger import pytest - from rucio.common.policy import get_policy from rucio.common.types import InternalScope from rucio.core.account_limit import set_local_account_limit -from rucio.core.did import add_dids, attach_dids, list_expired_dids, get_did, set_metadata +from rucio.core.did import add_dids, attach_dids, get_did, list_expired_dids, set_metadata from rucio.core.replica import add_replicas, get_replica from rucio.core.rse import add_rse from rucio.core.rule import add_rules, list_rules from rucio.daemons.judge.cleaner import rule_cleaner from rucio.daemons.undertaker.undertaker import undertaker from rucio.db.sqla.util import json_implemented -from rucio.tests.common import rse_name_generator, did_name_generator +from rucio.tests.common import did_name_generator, rse_name_generator LOG = getLogger(__name__) diff --git a/tests/test_upload.py b/tests/test_upload.py index 7c62f8bf99..e21482b821 100644 --- a/tests/test_upload.py +++ b/tests/test_upload.py @@ -16,11 +16,11 @@ import json import logging import os -import pytest import shutil from tempfile import TemporaryDirectory from unittest.mock import patch +import pytest from rucio.client.uploadclient import UploadClient from rucio.common.config import config_add_section, config_set from rucio.common.exception import InputValidationError, NoFilesUploaded, NotAllFilesUploaded diff --git a/tests/test_utils.py b/tests/test_utils.py index 2c61d44b9b..43fe64e00b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,10 +19,9 @@ from re import match import pytest - from rucio.common.exception import InvalidType -from rucio.common.utils import md5, adler32, parse_did_filter_from_string, Availability, retrying, bittorrent_v2_merkle_sha256 from rucio.common.logging import formatted_logger +from rucio.common.utils import Availability, adler32, bittorrent_v2_merkle_sha256, md5, parse_did_filter_from_string, retrying class TestUtils: diff --git a/tools/bootstrap.py b/tools/bootstrap.py index b407bef0e7..8dc3943193 100755 --- a/tools/bootstrap.py +++ b/tools/bootstrap.py @@ -20,13 +20,14 @@ # Verify for default SQLite: # for i in `sqlite3 /tmp/rucio.db ".tables"`; do echo $i:; sqlite3 /tmp/rucio.db "select * from $i"; echo; done -import sys import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) -from rucio.db.sqla.util import (build_database, create_base_vo, create_root_account) # noqa: E402 +from rucio.db.sqla.util import build_database, create_base_vo, create_root_account # noqa: E402 if __name__ == '__main__': build_database() diff --git a/tools/bootstrap_tests.py b/tools/bootstrap_tests.py index 4fd1a0898b..66e3787b8d 100755 --- a/tools/bootstrap_tests.py +++ b/tools/bootstrap_tests.py @@ -14,13 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os.path import os +import os.path import sys -import requests import time from json import dumps +import requests + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) @@ -29,11 +30,11 @@ from rucio.api.vo import add_vo # noqa: E402 from rucio.client import Client # noqa: E402 from rucio.common.config import config_get, config_get_bool # noqa: E402 -from rucio.common.exception import Duplicate, RucioException, DuplicateContent # noqa: E402 +from rucio.common.exception import Duplicate, DuplicateContent, RucioException # noqa: E402 +from rucio.common.types import InternalAccount # noqa: E402 from rucio.common.utils import extract_scope # noqa: E402 from rucio.core.account import add_account_attribute # noqa: E402 from rucio.core.vo import map_vo # noqa: E402 -from rucio.common.types import InternalAccount # noqa: E402 from rucio.tests.common_server import reset_config_table # noqa: E402 diff --git a/tools/convert_database_vo.py b/tools/convert_database_vo.py index 938ff69422..ecefe71a2f 100755 --- a/tools/convert_database_vo.py +++ b/tools/convert_database_vo.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) @@ -24,13 +25,6 @@ from datetime import datetime # noqa: E402 from traceback import format_exc # noqa: E402 -from sqlalchemy import func # noqa: E402 -from sqlalchemy.engine import reflection # noqa: E402 -from sqlalchemy.schema import AddConstraint, DropConstraint, ForeignKeyConstraint, MetaData, Table # noqa: E402 -from sqlalchemy.sql import bindparam # noqa: E402 -from sqlalchemy.sql.expression import cast # noqa: E402 -from sqlalchemy.types import CHAR # noqa: E402 - from rucio.common.config import config_get_bool # noqa: E402 from rucio.common.types import InternalAccount # noqa: E402 from rucio.common.utils import StoreTrueAndDeprecateWarningAction # noqa: E402 @@ -38,6 +32,12 @@ from rucio.core.vo import list_vos # noqa: E402 from rucio.db.sqla import session # noqa: E402 from rucio.db.sqla.util import create_root_account # noqa: E402 +from sqlalchemy import func # noqa: E402 +from sqlalchemy.engine import reflection # noqa: E402 +from sqlalchemy.schema import AddConstraint, DropConstraint, ForeignKeyConstraint, MetaData, Table # noqa: E402 +from sqlalchemy.sql import bindparam # noqa: E402 +from sqlalchemy.sql.expression import cast # noqa: E402 +from sqlalchemy.types import CHAR # noqa: E402 def split_vo(dialect, column, return_vo=False): diff --git a/tools/destroy_database.py b/tools/destroy_database.py index 90cc2a144d..0ca9342b91 100755 --- a/tools/destroy_database.py +++ b/tools/destroy_database.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) diff --git a/tools/generate_version.py b/tools/generate_version.py index 328bd7cd6b..6808a84967 100644 --- a/tools/generate_version.py +++ b/tools/generate_version.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) diff --git a/tools/merge_rucio_configs.py b/tools/merge_rucio_configs.py index d6dfbe8aa2..8a98d83333 100644 --- a/tools/merge_rucio_configs.py +++ b/tools/merge_rucio_configs.py @@ -14,12 +14,11 @@ # limitations under the License. import argparse +import json import logging -from pathlib import Path - import os -import json import sys +from pathlib import Path try: import yaml @@ -27,7 +26,6 @@ yaml = None import configparser - # Multi-word sections used in kubernetes are slightly different from what rucio expects. # Usually, it's just a .replace('-', '_'), but not for hermes2, which doesn't follow any convention. multi_word_sections = { diff --git a/tools/monitoring/extract.py b/tools/monitoring/extract.py index 155e827fb6..0f5009b2db 100644 --- a/tools/monitoring/extract.py +++ b/tools/monitoring/extract.py @@ -22,11 +22,13 @@ es_username = "" es_password = "" +from json import loads as jloads from time import sleep + import elasticsearch as es -from json import loads as jloads import stomp + class ElasticConn(): def __init__(self, host_port, auth): self.__es = es.Elasticsearch([host_port[0]],http_auth=auth,consumer_port=host_port[1]) diff --git a/tools/reset_database.py b/tools/reset_database.py index 9cec39e918..2e03a963dd 100755 --- a/tools/reset_database.py +++ b/tools/reset_database.py @@ -14,15 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) from argparse import ArgumentParser # noqa: E402 -from rucio.db.sqla.util import build_database, destroy_database, drop_everything, create_root_account, create_base_vo # noqa: E402 +from rucio.db.sqla.util import build_database, create_base_vo, create_root_account, destroy_database, drop_everything # noqa: E402 if __name__ == '__main__': diff --git a/tools/run_pyright/__main__.py b/tools/run_pyright/__main__.py index e3a4bbb20e..24735f32e0 100755 --- a/tools/run_pyright/__main__.py +++ b/tools/run_pyright/__main__.py @@ -17,8 +17,7 @@ import sys from argparse import ArgumentParser -from . import generate -from . import compare +from . import compare, generate def parse_arguments(): diff --git a/tools/run_pyright/generate.py b/tools/run_pyright/generate.py index 8e686ecf97..b3b9099bbb 100644 --- a/tools/run_pyright/generate.py +++ b/tools/run_pyright/generate.py @@ -13,16 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from argparse import ArgumentParser, Namespace -from pathlib import Path import json import subprocess import sys +from argparse import ArgumentParser, Namespace +from pathlib import Path -from .models import ReportDict, Report +from .models import Report, ReportDict from .utils import save_json - PATHS = ( 'lib/', ) diff --git a/tools/run_pyright/models.py b/tools/run_pyright/models.py index 8438720232..0058ce8367 100644 --- a/tools/run_pyright/models.py +++ b/tools/run_pyright/models.py @@ -13,10 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from dataclasses import dataclass from enum import Enum from typing import Any -from dataclasses import dataclass - ReportDict = dict[str, Any] diff --git a/tools/run_pyright/utils.py b/tools/run_pyright/utils.py index 8be22fa4a2..06a36ec883 100644 --- a/tools/run_pyright/utils.py +++ b/tools/run_pyright/utils.py @@ -20,7 +20,6 @@ from .models import ReportDict - _T = TypeVar('_T') _K = TypeVar('_K') diff --git a/tools/sync_meta.py b/tools/sync_meta.py index 6caacca220..87051176fe 100755 --- a/tools/sync_meta.py +++ b/tools/sync_meta.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) diff --git a/tools/sync_rses.py b/tools/sync_rses.py index a599685797..24697788d5 100755 --- a/tools/sync_rses.py +++ b/tools/sync_rses.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) diff --git a/tools/test/ignoretool.py b/tools/test/ignoretool.py index 32eab0d60a..12f808dfdc 100755 --- a/tools/test/ignoretool.py +++ b/tools/test/ignoretool.py @@ -17,7 +17,6 @@ import argparse import os - # from setup_rucio_clients PACKAGES = [ 'rucio', diff --git a/tools/test/run_tests.py b/tools/test/run_tests.py index 9c2abf578b..f57cf7870c 100755 --- a/tools/test/run_tests.py +++ b/tools/test/run_tests.py @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import io import itertools +import json import multiprocessing import os import pathlib @@ -28,7 +28,7 @@ import uuid from datetime import datetime from tempfile import NamedTemporaryFile -from typing import Optional, Union, NoReturn +from typing import NoReturn, Optional, Union import yaml diff --git a/tools/test/votest_helper.py b/tools/test/votest_helper.py index 032ebdede5..d79768712c 100755 --- a/tools/test/votest_helper.py +++ b/tools/test/votest_helper.py @@ -14,16 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -import traceback -import json import argparse -import logging -import functools import configparser -from pathlib import Path +import functools import glob import itertools +import json +import logging +import sys +import traceback +from pathlib import Path + import yaml logger = logging.getLogger(__name__) diff --git a/tools/update_ui_version.py b/tools/update_ui_version.py index 5612e351cb..cf26fe7661 100755 --- a/tools/update_ui_version.py +++ b/tools/update_ui_version.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import os.path +import sys + base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_path) os.chdir(base_path) @@ -24,7 +25,6 @@ from re import sub # noqa: E402 from sys import argv # noqa: E402 - files = ['/static/rucio.js', '/static/base.js', '/static/rucio.css']