diff --git a/tests/test_abacus_collection_replica.py b/tests/test_abacus_collection_replica.py index 4987c2724b..b9f57c9fe8 100644 --- a/tests/test_abacus_collection_replica.py +++ b/tests/test_abacus_collection_replica.py @@ -18,7 +18,6 @@ from rucio.common.exception import DataIdentifierNotFound from rucio.common.schema import get_schema_value -from rucio.common.types import InternalAccount from rucio.core.did import add_did, get_did from rucio.core.replica import delete_replicas, get_cleaned_updated_collection_replicas from rucio.daemons.abacus import collection_replica @@ -33,7 +32,7 @@ @pytest.mark.noparallel(reason='uses daemons, fails when run in parallel') class TestAbacusCollectionReplica(): - def test_abacus_collection_replica_cleanup(self, vo, mock_scope, rse_factory, did_client): + def test_abacus_collection_replica_cleanup(self, vo, mock_scope, rse_factory, did_client, jdoe_account): """ ABACUS (COLLECTION REPLICA): Test if the cleanup procedure works correctly. """ collection_replica.run(once=True) db_session = session.get_session() @@ -41,8 +40,7 @@ def test_abacus_collection_replica_cleanup(self, vo, mock_scope, rse_factory, di rse2, rse_id2 = rse_factory.make_rse() dataset = did_name_generator('dataset') - jdoe = InternalAccount('jdoe', vo) - add_did(mock_scope, dataset, DIDType.DATASET, jdoe) + add_did(mock_scope, dataset, DIDType.DATASET, jdoe_account) models.CollectionReplica(scope=mock_scope, name=dataset, rse_id=rse_id1, did_type=DIDType.DATASET, state=ReplicaState.AVAILABLE, bytes=1, length=1).save(session=db_session, flush=False) diff --git a/tests/test_account.py b/tests/test_account.py index 83f13ee33f..3de654f9ca 100644 --- a/tests/test_account.py +++ b/tests/test_account.py @@ -53,25 +53,23 @@ def test_update_account(self, vo): assert email == 'test' del_account(usr, 'root', vo=vo) - def test_list_account_identities(self, vo): + def test_list_account_identities(self, root_account): """ ACCOUNT (CORE): Test listing of account identities """ email = 'email' identity = uuid() identity_type = IdentityType.USERPASS - account = InternalAccount('root', vo=vo) - add_account_identity(identity, identity_type, account, email, password='secret') - identities = list_identities(account) + add_account_identity(identity, identity_type, root_account, email, password='secret') + identities = list_identities(root_account) assert {'type': identity_type, 'identity': identity, 'email': email} in identities - def test_add_account_attribute(self, vo): + def test_add_account_attribute(self, root_account): """ ACCOUNT (CORE): Test adding attribute to account """ - account = InternalAccount('root', vo=vo) key = account_name_generator() value = True - add_account_attribute(account, key, value) - assert {'key': key, 'value': True} in list_account_attributes(account) + add_account_attribute(root_account, key, value) + assert {'key': key, 'value': True} in list_account_attributes(root_account) with pytest.raises(Duplicate): - add_account_attribute(account, key, value) + add_account_attribute(root_account, key, value) def test_create_user_success(rest_client, auth_token): diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 00a6065794..4ba72310dd 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -23,7 +23,6 @@ from rucio.api.authentication import get_auth_token_user_pass, get_auth_token_ssh, get_ssh_challenge_token, \ get_auth_token_saml from rucio.common.exception import Duplicate, AccessDenied, CannotAuthenticate -from rucio.common.types import InternalAccount from rucio.common.utils import ssh_sign from rucio.core.identity import add_account_identity, del_account_identity from rucio.core.authentication import strip_x509_proxy_attributes @@ -118,12 +117,11 @@ def test_get_auth_token_user_pass_fail(self, vo): result = get_auth_token_user_pass(account='root', username='ddmlab', password='not_secret', appid='test', ip='127.0.0.1', vo=vo) assert result is None - def test_get_auth_token_ssh_success(self, vo): + def test_get_auth_token_ssh_success(self, vo, root_account): """AUTHENTICATION (CORE): SSH RSA public key exchange (good signature).""" - root = InternalAccount('root', vo=vo) try: - add_account_identity(PUBLIC_KEY, IdentityType.SSH, root, email='ph-adp-ddm-lab@cern.ch') + add_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account, email='ph-adp-ddm-lab@cern.ch') except Duplicate: pass # might already exist, can skip @@ -135,14 +133,13 @@ def test_get_auth_token_ssh_success(self, vo): assert result is not None - del_account_identity(PUBLIC_KEY, IdentityType.SSH, root) + del_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account) - def test_get_auth_token_ssh_fail(self, vo): + def test_get_auth_token_ssh_fail(self, vo, root_account): """AUTHENTICATION (CORE): SSH RSA public key exchange (wrong signature).""" - root = InternalAccount('root', vo=vo) try: - add_account_identity(PUBLIC_KEY, IdentityType.SSH, root, email='ph-adp-ddm-lab@cern.ch') + add_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account, email='ph-adp-ddm-lab@cern.ch') except Duplicate: pass # might already exist, can skip @@ -152,14 +149,13 @@ def test_get_auth_token_ssh_fail(self, vo): assert result is None - del_account_identity(PUBLIC_KEY, IdentityType.SSH, root) + del_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account) - def test_invalid_padding(self, vo): + def test_invalid_padding(self, vo, root_account): """AUTHENTICATION (CORE): SSH RSA public key exchange (public key with invalid padding).""" - root = InternalAccount('root', vo=vo) try: - add_account_identity(INVALID_PADDED_PUBLIC_KEY, IdentityType.SSH, root, email='ph-adp-ddm-lab@cern.ch') + add_account_identity(INVALID_PADDED_PUBLIC_KEY, IdentityType.SSH, root_account, email='ph-adp-ddm-lab@cern.ch') except Duplicate: pass # might already exist, can skip @@ -170,33 +166,31 @@ def test_invalid_padding(self, vo): result = get_auth_token_ssh(account='root', signature=signature, appid='test', ip='127.0.0.1', vo=vo) assert result is not None - del_account_identity(INVALID_PADDED_PUBLIC_KEY, IdentityType.SSH, root) + del_account_identity(INVALID_PADDED_PUBLIC_KEY, IdentityType.SSH, root_account) - def test_get_auth_token_saml_success(self, vo): + def test_get_auth_token_saml_success(self, vo, root_account): """AUTHENTICATION (CORE): SAML NameID (correct credentials).""" - root = InternalAccount('root', vo=vo) try: - add_account_identity('ddmlab', IdentityType.SAML, root, email='ph-adp-ddm-lab@cern.ch') + add_account_identity('ddmlab', IdentityType.SAML, root_account, email='ph-adp-ddm-lab@cern.ch') except Duplicate: pass # might already exist, can skip result = get_auth_token_saml(account='root', saml_nameid='ddmlab', appid='test', ip='127.0.0.1', vo=vo) assert result is not None - del_account_identity('ddmlab', IdentityType.SAML, root) + del_account_identity('ddmlab', IdentityType.SAML, root_account) - def test_get_auth_token_saml_fail(self, vo): + def test_get_auth_token_saml_fail(self, vo, root_account): """AUTHENTICATION (CORE): SAML NameID (wrong credentials).""" - root = InternalAccount('root', vo=vo) try: - add_account_identity('ddmlab', IdentityType.SAML, root, email='ph-adp-ddm-lab@cern.ch') + add_account_identity('ddmlab', IdentityType.SAML, root_account, email='ph-adp-ddm-lab@cern.ch') except Duplicate: pass # might already exist, can skip with pytest.raises(AccessDenied): get_auth_token_saml(account='root', saml_nameid='not_ddmlab', appid='test', ip='127.0.0.1', vo=vo) - del_account_identity('ddmlab', IdentityType.SAML, root) + del_account_identity('ddmlab', IdentityType.SAML, root_account) def test_userpass_fail(vo, rest_client): @@ -213,12 +207,11 @@ def test_userpass_success(vo, rest_client): @pytest.mark.noparallel(reason='changes identities of the same account') -def test_ssh_success(vo, rest_client): +def test_ssh_success(vo, rest_client, root_account): """AUTHENTICATION (REST): SSH RSA public key exchange (correct credentials).""" - root = InternalAccount('root', vo=vo) try: - add_account_identity(PUBLIC_KEY, IdentityType.SSH, root, email='ph-adp-ddm-lab@cern.ch') + add_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account, email='ph-adp-ddm-lab@cern.ch') except Duplicate: pass # might already exist, can skip @@ -234,16 +227,15 @@ def test_ssh_success(vo, rest_client): assert response.status_code == 200 assert len(response.headers.get('X-Rucio-Auth-Token')) > 32 - del_account_identity(PUBLIC_KEY, IdentityType.SSH, root) + del_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account) @pytest.mark.noparallel(reason='changes identities of the same account') -def test_ssh_fail(vo, rest_client): +def test_ssh_fail(vo, rest_client, root_account): """AUTHENTICATION (REST): SSH RSA public key exchange (wrong credentials).""" - root = InternalAccount('root', vo=vo) try: - add_account_identity(PUBLIC_KEY, IdentityType.SSH, root, email='ph-adp-ddm-lab@cern.ch') + add_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account, email='ph-adp-ddm-lab@cern.ch') except Duplicate: pass # might already exist, can skip @@ -253,7 +245,7 @@ def test_ssh_fail(vo, rest_client): response = rest_client.get('/auth/ssh', headers=headers(hdrdict(headers_dict), vohdr(vo))) assert response.status_code == 401 - del_account_identity(PUBLIC_KEY, IdentityType.SSH, root) + del_account_identity(PUBLIC_KEY, IdentityType.SSH, root_account) @pytest.mark.xfail(reason='The WebUI isn\'t linked to CERN SSO yet so this needs to be fixed once it is linked') diff --git a/tests/test_conveyor.py b/tests/test_conveyor.py index ef3ea9e801..9da04fc403 100644 --- a/tests/test_conveyor.py +++ b/tests/test_conveyor.py @@ -1349,7 +1349,7 @@ def on_receive(job_params): ('conveyor', 'usercert', 'DEFAULT_DUMMY_CERT'), ('vo_certs', 'new', 'NEW_VO_DUMMY_CERT'), ]}], indirect=True) -def test_multi_vo_certificates(file_config_mock, rse_factory, did_factory, scope_factory, vo, second_vo): +def test_multi_vo_certificates(file_config_mock, rse_factory, did_factory, scope_factory, vo, second_vo, root_account): """ Test that submitter and poller call fts with correct certificates in multi-vo env """ diff --git a/tests/test_dataset_replicas.py b/tests/test_dataset_replicas.py index 29cfee7ec8..6716df8741 100644 --- a/tests/test_dataset_replicas.py +++ b/tests/test_dataset_replicas.py @@ -94,7 +94,7 @@ def test_list_datasets_per_rse(self, vo, mock_scope): filters={'scope': mock_scope, 'name': pattern})] assert replicas != [] - def test_list_dataset_replicas_archive(self, vo): + def test_list_dataset_replicas_archive(self, vo, did_client): """ REPLICA (CLIENT): List dataset replicas with archives. """ activity = get_schema_value('ACTIVITY')['enum'][0] diff --git a/tests/test_did.py b/tests/test_did.py index 0b5f9af645..0ed082b3de 100644 --- a/tests/test_did.py +++ b/tests/test_did.py @@ -24,7 +24,7 @@ from rucio.common.exception import (DataIdentifierNotFound, DataIdentifierAlreadyExists, InvalidPath, UnsupportedOperation, UnsupportedStatus, ScopeNotFound, FileAlreadyExists, FileConsistencyMismatch) -from rucio.common.types import InternalAccount, InternalScope +from rucio.common.types import InternalScope from rucio.common.utils import generate_uuid from rucio.core.did import (list_dids, add_did, delete_dids, get_did_atime, touch_dids, attach_dids, detach_dids, get_metadata, set_metadata, get_did, get_did_access_cnt, add_did_to_followed, @@ -434,7 +434,6 @@ def test_add_did(self, vo, did_client, rse_factory): tmp_rse, rse_id = rse_factory.make_mock_rse() rse2, rse2_id = rse_factory.make_mock_rse() tmp_dsn = did_name_generator('dataset') - root = InternalAccount('root', vo=vo) # PFN example: rfio://castoratlas.cern.ch/castor/cern.ch/grid/atlas/tzero/xx/xx/xx/filename dataset_meta = {'project': 'data13_hip', diff --git a/tests/test_import_export.py b/tests/test_import_export.py index 6bd7128c9e..10cb01a122 100644 --- a/tests/test_import_export.py +++ b/tests/test_import_export.py @@ -82,7 +82,7 @@ def reset_rses(): @pytest.fixture -def importer_example_data(vo): +def importer_example_data(vo, jdoe_account): if not config_has_section('importer'): config_add_section('importer') config_set('importer', 'rse_sync_method', 'hard') @@ -291,7 +291,7 @@ def check_accounts(self): } ] }, { - 'account': InternalAccount('jdoe', vo=vo), + 'account': jdoe_account, 'email': 'email' }] } diff --git a/tests/test_multi_vo.py b/tests/test_multi_vo.py index 1ad0571880..e744f45fba 100644 --- a/tests/test_multi_vo.py +++ b/tests/test_multi_vo.py @@ -45,7 +45,7 @@ from rucio.common.config import config_remove_option, config_set, config_has_section, config_add_section from rucio.common.exception import AccessDenied, Duplicate, InvalidRSEExpression, UnsupportedAccountName, \ UnsupportedOperation, RucioException -from rucio.common.types import InternalAccount, InternalScope +from rucio.common.types import InternalAccount from rucio.common.utils import generate_uuid, get_tmp_dir, parse_response, ssh_sign from rucio.core import config as core_config from rucio.core.account_counter import add_counter @@ -56,7 +56,7 @@ from rucio.core.vo import map_vo from rucio.daemons.automatix.automatix import automatix from rucio.db.sqla import models, session as db_session -from rucio.tests.common import execute, headers, hdrdict, vohdr, auth, loginhdr, get_long_vo +from rucio.tests.common import execute, headers, hdrdict, vohdr, auth, loginhdr from .test_authentication import PRIVATE_KEY, PUBLIC_KEY from .test_oidc import get_mock_oidc_client, NEW_TOKEN_DICT @@ -83,17 +83,16 @@ def test_multi_vo_flag(self, vo, second_vo): config_set('common', 'multi_vo', 'True') @pytest.mark.noparallel(reason='uses global RSE (MOCK) and fails when run in parallel') - def test_access_rule_vo(self, vo, second_vo): + def test_access_rule_vo(self, vo, second_vo, root_account, mock_scope): """ MULTI VO (CORE): Test accessing rules from a different VO """ - scope = InternalScope('mock', vo=vo) + scope = mock_scope dataset = 'dataset_' + str(generate_uuid()) - account = InternalAccount('root', vo=vo) rse_str = ''.join(choice(ascii_uppercase) for x in range(10)) rse_name = 'MOCK_%s' % rse_str rse_id = add_rse(rse_name, 'root', vo=vo) - add_replica(rse_id=rse_id, scope=scope, name=dataset, bytes_=10, account=account) - rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=account, copies=1, rse_expression='MOCK', grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0] + add_replica(rse_id=rse_id, scope=scope, name=dataset, bytes_=10, account=root_account) + rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=root_account, copies=1, rse_expression='MOCK', grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None)[0] with pytest.raises(AccessDenied): delete_replication_rule(rule_id=rule_id, purge_replicas=False, issuer='root', vo=second_vo) @@ -641,14 +640,13 @@ def test_rest_vomap_bad(self, rest_client): class TestMultiVoClients: - def test_get_vo_from_config(self, vo): + def test_get_vo_from_config(self, long_vo): """ MULTI VO (CLIENT): Get vo from config file when starting clients """ # Start clients with vo explicitly set to None replica_client = ReplicaClient(vo=None) client = Client(vo=None) upload_client = UploadClient(_client=client) # Check the vo has been got from the config file - long_vo = get_long_vo() assert replica_client.vo == long_vo assert upload_client.client.vo == long_vo diff --git a/tests/test_permission.py b/tests/test_permission.py index b771846730..b0102489b0 100644 --- a/tests/test_permission.py +++ b/tests/test_permission.py @@ -15,7 +15,7 @@ from rucio.api.permission import has_permission from rucio.common.config import config_get -from rucio.common.types import InternalAccount, InternalScope +from rucio.common.types import InternalScope from rucio.core.scope import add_scope from rucio.core.account import add_account_attribute from rucio.tests.common import scope_name_generator, skip_non_belleii @@ -28,10 +28,10 @@ class TestPermissionCoreApi: usr = 'jdoe' - def test_permission_add_did(self, vo): + def test_permission_add_did(self, vo, root_account): """ PERMISSION(CORE): Check permission to add a did""" scope = scope_name_generator() - add_scope(scope=InternalScope(scope, vo=vo), account=InternalAccount('root', vo=vo)) + add_scope(scope=InternalScope(scope, vo=vo), account=root_account) assert has_permission(issuer='panda', action='add_did', kwargs={'scope': scope}, vo=vo) assert not has_permission(issuer='spock', action='add_did', kwargs={'scope': scope}, vo=vo) diff --git a/tests/test_replica_sorting.py b/tests/test_replica_sorting.py index 19680edfe8..4f6c5134c7 100644 --- a/tests/test_replica_sorting.py +++ b/tests/test_replica_sorting.py @@ -23,7 +23,6 @@ import geoip2.database import pytest -from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import parse_replicas_from_string from rucio.common.config import config_get from rucio.core import rse_expression_parser, replica_sorter @@ -101,17 +100,16 @@ def _get_lat_long_mock(se, gi): @pytest.fixture -def protocols_setup(vo): +def protocols_setup(vo, root_account, mock_scope): rse_info = copy.deepcopy(base_rse_info) - files = [{'scope': InternalScope('mock', vo=vo), 'name': 'element_0', 'bytes': 1234, 'adler32': 'deadbeef'}] - root = InternalAccount('root', vo=vo) + files = [{'scope': mock_scope, 'name': 'element_0', 'bytes': 1234, 'adler32': 'deadbeef'}] for idx in range(len(rse_info)): rse_info[idx]['name'] = '%s_%s' % (rse_info[idx]['site'], rse_name_generator()) rse_info[idx]['id'] = add_rse(rse_info[idx]['name'], vo=vo) add_rse_attribute(rse_id=rse_info[idx]['id'], key='site', value=base_rse_info[idx]['site']) - add_replicas(rse_id=rse_info[idx]['id'], files=files, account=root) + add_replicas(rse_id=rse_info[idx]['id'], files=files, account=root_account) # invalidate cache for parse_expression('site=…') rse_expression_parser.REGION.invalidate() diff --git a/tests/test_root_proxy.py b/tests/test_root_proxy.py index cd1941a520..8498eedcb4 100644 --- a/tests/test_root_proxy.py +++ b/tests/test_root_proxy.py @@ -17,7 +17,6 @@ import pytest -from rucio.common.types import InternalAccount, InternalScope from rucio.core.config import set as config_set from rucio.core.replica import add_replicas, delete_replicas from rucio.core.rse import add_rse, add_rse_attribute, del_rse, add_protocol @@ -33,7 +32,7 @@ @pytest.fixture(scope='module', autouse=True) -def root_proxy_example_data(vo): +def root_proxy_example_data(vo, root_account, mock_scope): rse_without_proxy = rse_name_generator() rse_without_proxy_id = add_rse(rse_without_proxy, vo=vo) add_rse_attribute(rse_id=rse_without_proxy_id, @@ -49,7 +48,7 @@ def root_proxy_example_data(vo): # APERTURE1 site has an internal proxy config_set('root-proxy-internal', 'APERTURE1', 'proxy.aperture.com:1094') - files = [{'scope': InternalScope('mock', vo=vo), + files = [{'scope': mock_scope, 'name': 'half-life_%s' % i, 'bytes': 1234, 'adler32': 'deadbeef', @@ -57,7 +56,7 @@ def root_proxy_example_data(vo): for rse_id in [rse_with_proxy_id, rse_without_proxy_id]: add_replicas(rse_id=rse_id, files=files, - account=InternalAccount('root', vo=vo), + account=root_account, ignore_availability=True) add_protocol(rse_without_proxy_id, {'scheme': 'root', diff --git a/tests/test_rse.py b/tests/test_rse.py index d8a7fe52df..e4d45943b6 100644 --- a/tests/test_rse.py +++ b/tests/test_rse.py @@ -22,7 +22,6 @@ RSEAttributeNotFound, RSEOperationNotSupported, InputValidationError) from rucio.common.schema import get_schema_value -from rucio.common.types import InternalScope from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, CHECKSUM_KEY from rucio.core.account_limit import set_local_account_limit, get_rse_account_usage from rucio.core.did import add_did, attach_dids @@ -1358,19 +1357,18 @@ def test_update_protocol_wrong_priority(self, rucio_client): finally: rucio_client.delete_rse(protocol_rse) - def test_get_rse_usage(self, vo, rucio_client, rse_factory, jdoe_account, root_account): + def test_get_rse_usage(self, vo, rucio_client, rse_factory, jdoe_account, root_account, mock_scope): """ RSE (CLIENTS): Test getting the RSE usage. """ file_sizes = 100 nfiles = 3 rse, rse_id = rse_factory.make_posix_rse() set_local_account_limit(account=jdoe_account, rse_id=rse_id, bytes_=10000) - tmp_scope = InternalScope('mock', vo=vo) activity = get_schema_value('ACTIVITY')['enum'][0] - files = create_files(nfiles, tmp_scope, rse_id, bytes_=file_sizes) + files = create_files(nfiles, mock_scope, rse_id, bytes_=file_sizes) dataset = did_name_generator('dataset') - add_did(tmp_scope, dataset, DIDType.DATASET, jdoe_account) - attach_dids(tmp_scope, dataset, files, jdoe_account) - rules = add_rule(dids=[{'scope': tmp_scope, 'name': dataset}], account=jdoe_account, copies=1, rse_expression=rse, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, activity=activity) + add_did(mock_scope, dataset, DIDType.DATASET, jdoe_account) + attach_dids(mock_scope, dataset, files, jdoe_account) + rules = add_rule(dids=[{'scope': mock_scope, 'name': dataset}], account=jdoe_account, copies=1, rse_expression=rse, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, activity=activity) assert rules account_update(once=True) usages = rucio_client.get_rse_usage(rse=rse, filters={'per_account': True}) diff --git a/tests/test_rule.py b/tests/test_rule.py index e62093d43e..9271e2bc1d 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -1303,19 +1303,18 @@ def mktree(scope, account): assert (len(dsl3) == 0) -def test_rule_boost(vo, mock_scope, rse_factory, file_factory): +def test_rule_boost(vo, mock_scope, rse_factory, jdoe_account): """ REPLICATION RULE (CORE): Update a replication rule to quicken the translation from stuck to replicating """ - jdoe = InternalAccount('jdoe', vo) _, tmp_rse_id = rse_factory.make_mock_rse() rse, rse_id = rse_factory.make_mock_rse() update_rse(rse_id, {'availability_write': False}) - set_local_account_limit(jdoe, rse_id, -1) + set_local_account_limit(jdoe_account, rse_id, -1) files = create_files(3, mock_scope, tmp_rse_id) dataset1 = 'dataset_' + str(uuid()) - add_did(mock_scope, dataset1, DIDType.DATASET, jdoe) - attach_dids(mock_scope, dataset1, files, jdoe) + add_did(mock_scope, dataset1, DIDType.DATASET, jdoe_account) + attach_dids(mock_scope, dataset1, files, jdoe_account) - rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dataset1}], account=jdoe, copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True)[0] + rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dataset1}], account=jdoe_account, copies=1, rse_expression=rse, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True)[0] before_update_rule = {} for file in files: for filtered_lock in [lock for lock in get_replica_locks(scope=file['scope'], name=file['name'])]: diff --git a/tests/test_subscription.py b/tests/test_subscription.py index 80a13eb5e2..a84ce98810 100644 --- a/tests/test_subscription.py +++ b/tests/test_subscription.py @@ -142,11 +142,10 @@ def test_update_nonexisting_subscription(self, vo): with pytest.raises(SubscriptionNotFound): update_subscription(name=subscription_name, account='root', metadata={'filter': {'project': ['toto', ]}}, issuer='root', vo=vo) - def test_list_rules_states(self, vo, rse_factory): + def test_list_rules_states(self, vo, rse_factory, root_account): """ SUBSCRIPTION (API): Test listing of rule states for subscription """ tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo) - root = InternalAccount('root', vo=vo) - add_scope(tmp_scope, root) + add_scope(tmp_scope, root_account) rse1, _ = rse_factory.make_mock_rse() rse2, _ = rse_factory.make_mock_rse() rse_expression = '%s|%s' % (rse1, rse2) @@ -157,7 +156,7 @@ def test_list_rules_states(self, vo, rse_factory): # add a new dataset dsn = did_name_generator('dataset') add_did(scope=tmp_scope, name=dsn, - did_type=DIDType.DATASET, account=root) + did_type=DIDType.DATASET, account=root_account) subscription_name = uuid() subid = add_subscription(name=subscription_name, @@ -172,8 +171,8 @@ def test_list_rules_states(self, vo, rse_factory): vo=vo) # Add two rules - add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=rse3, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) - add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) + add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root_account, copies=1, rse_expression=rse3, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) + add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root_account, copies=1, rse_expression=rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) for rule in list_subscription_rule_states(account='root', name=subscription_name, vo=vo): assert rule[3] == 2 @@ -257,11 +256,10 @@ def test_update_nonexisting_subscription(rest_client, auth_token): @pytest.mark.noparallel(reason='uses pre-defined RSE') -def test_list_rules_states(vo, rse_factory, rest_client, auth_token): +def test_list_rules_states(vo, rse_factory, rest_client, auth_token, root_account): """ SUBSCRIPTION (REST): Test listing of rule states for subscription """ tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo) - root = InternalAccount('root', vo=vo) - add_scope(tmp_scope, root) + add_scope(tmp_scope, root_account) activity = get_schema_value('ACTIVITY')['enum'][0] rse1, _ = rse_factory.make_mock_rse() rse2, _ = rse_factory.make_mock_rse() @@ -273,7 +271,7 @@ def test_list_rules_states(vo, rse_factory, rest_client, auth_token): # add a new dataset dsn = 'dataset-%s' % uuid() add_did(scope=tmp_scope, name=dsn, - did_type=DIDType.DATASET, account=root) + did_type=DIDType.DATASET, account=root_account) subscription_name = uuid() subid = add_subscription(name=subscription_name, @@ -288,8 +286,8 @@ def test_list_rules_states(vo, rse_factory, rest_client, auth_token): vo=vo) # Add two rules - add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=rse3, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) - add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root, copies=1, rse_expression=rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) + add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root_account, copies=1, rse_expression=rse3, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) + add_rule(dids=[{'scope': tmp_scope, 'name': dsn}], account=root_account, copies=1, rse_expression=rse4, grouping='NONE', weight=None, lifetime=None, locked=False, subscription_id=subid) response = rest_client.get('/subscriptions/%s/%s/Rules/States' % ('root', subscription_name), headers=headers(auth(auth_token))) assert response.status_code == 200 @@ -380,7 +378,7 @@ def test_create_and_list_subscription_by_name(self, rse_factory, rucio_client): assert subid == result[0] @pytest.mark.noparallel(reason='runs transmogrifier. Cannot be run at the same time with other tests running it') - def test_run_transmogrifier(self, vo, rse_factory, rucio_client): + def test_run_transmogrifier(self, vo, rse_factory, rucio_client, root_account): """ SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """ new_dids = [did for did in list_new_dids(did_type=None, thread=None, total_threads=None, chunk_size=100000, session=None)] set_new_dids(new_dids, None) @@ -390,12 +388,11 @@ def test_run_transmogrifier(self, vo, rse_factory, rucio_client): rse3, _ = rse_factory.make_mock_rse() rse_expression = '%s|%s|%s' % (rse1, rse2, rse3) tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo) - root = InternalAccount('root', vo=vo) - add_scope(tmp_scope, root) + add_scope(tmp_scope, root_account) subscription_name = uuid() dsn_prefix = did_name_generator('dataset') dsn = '%sdataset-%s' % (dsn_prefix, uuid()) - add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root) + add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root_account) subid = rucio_client.add_subscription(name=subscription_name, account='root', filter_={'scope': [tmp_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': True}, replication_rules=[{'lifetime': 86400, 'rse_expression': rse_expression, 'copies': 2, 'activity': self.activity}], @@ -409,7 +406,7 @@ def test_run_transmogrifier(self, vo, rse_factory, rucio_client): assert len(rules) == 2 @pytest.mark.noparallel(reason='runs transmogrifier. Cannot be run at the same time with other tests running it') - def test_run_transmogrifier_did_type(self, vo, rse_factory, rucio_client): + def test_run_transmogrifier_did_type(self, vo, rse_factory, rucio_client, root_account): """ SUBSCRIPTION (DAEMON): Test the transmogrifier with did_type subscriptions """ new_dids = [did for did in list_new_dids(did_type=None, thread=None, total_threads=None, chunk_size=100000, session=None)] set_new_dids(new_dids, None) @@ -418,12 +415,11 @@ def test_run_transmogrifier_did_type(self, vo, rse_factory, rucio_client): rse3, _ = rse_factory.make_mock_rse() rse_expression = '%s|%s|%s' % (rse1, rse2, rse3) tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo) - root = InternalAccount('root', vo=vo) - add_scope(tmp_scope, root) + add_scope(tmp_scope, root_account) subscription_name = uuid() dsn_prefix = did_name_generator('dataset') dsn = '%sdataset-%s' % (dsn_prefix, uuid()) - add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root) + add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root_account) subid = rucio_client.add_subscription(name=subscription_name, account='root', filter_={'scope': [tmp_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': True, 'did_type': ['DATASET', ]}, replication_rules=[{'lifetime': 86400, 'rse_expression': rse_expression, 'copies': 2, 'activity': self.activity}], diff --git a/tests/test_undertaker.py b/tests/test_undertaker.py index 05fb0bc904..58f39fe7bf 100644 --- a/tests/test_undertaker.py +++ b/tests/test_undertaker.py @@ -19,7 +19,7 @@ import pytest from rucio.common.policy import get_policy -from rucio.common.types import InternalAccount, InternalScope +from rucio.common.types import InternalScope from rucio.core.account_limit import set_local_account_limit from rucio.core.did import add_dids, attach_dids, list_expired_dids, get_did, set_metadata from rucio.core.replica import add_replicas, get_replica @@ -37,15 +37,13 @@ @pytest.mark.noparallel(reason='uses pre-defined rses; runs undertaker, which impacts other tests') class TestUndertaker: - def test_undertaker(self, vo, rse_factory, mock_scope, root_account): + def test_undertaker(self, vo, rse_factory, mock_scope, root_account, jdoe_account): """ UNDERTAKER (CORE): Test the undertaker. """ - jdoe = InternalAccount('jdoe', vo=vo) - nbdatasets = 5 nbfiles = 5 rse, rse_id = rse_factory.make_rse() - set_local_account_limit(jdoe, rse_id, -1) + set_local_account_limit(jdoe_account, rse_id, -1) dsns1 = [{'name': did_name_generator('dataset'), 'scope': mock_scope, @@ -56,7 +54,7 @@ def test_undertaker(self, vo, rse_factory, mock_scope, root_account): 'scope': mock_scope, 'type': 'DATASET', 'lifetime': -1, - 'rules': [{'account': jdoe, 'copies': 1, + 'rules': [{'account': jdoe_account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for _ in range(nbdatasets)] @@ -75,7 +73,7 @@ def test_undertaker(self, vo, rse_factory, mock_scope, root_account): attach_dids(scope=mock_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root_account) replicas += files - add_rules(dids=dsns1, rules=[{'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]) + add_rules(dids=dsns1, rules=[{'account': jdoe_account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]) undertaker(once=True) undertaker(once=True) @@ -83,19 +81,18 @@ def test_undertaker(self, vo, rse_factory, mock_scope, root_account): for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is not None - def test_list_expired_dids_with_locked_rules(self, rse_factory, vo, mock_scope, root_account): + def test_list_expired_dids_with_locked_rules(self, rse_factory, vo, mock_scope, root_account, jdoe_account): """ UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules""" - jdoe = InternalAccount('jdoe', vo=vo) # Add quota rse, rse_id = rse_factory.make_rse() - set_local_account_limit(jdoe, rse_id, -1) + set_local_account_limit(jdoe_account, rse_id, -1) dsn = {'name': did_name_generator('dataset'), 'scope': mock_scope, 'type': 'DATASET', 'lifetime': -1, - 'rules': [{'account': jdoe, 'copies': 1, + 'rules': [{'account': jdoe_account, 'copies': 1, 'rse_expression': rse, 'locked': True, 'grouping': 'DATASET'}]} @@ -104,27 +101,25 @@ def test_list_expired_dids_with_locked_rules(self, rse_factory, vo, mock_scope, for did in list_expired_dids(limit=1000): assert (did['scope'], did['name']) != (dsn['scope'], dsn['name']) - def test_atlas_archival_policy(self, vo, mock_scope, root_account): + def test_atlas_archival_policy(self, vo, mock_scope, root_account, jdoe_account): """ UNDERTAKER (CORE): Test the atlas archival policy. """ if get_policy() != 'atlas': LOG.info("Skipping atlas-specific test") return - jdoe = InternalAccount('jdoe', vo=vo) - nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() rse_id = add_rse(rse, vo=vo) - set_local_account_limit(jdoe, rse_id, -1) + set_local_account_limit(jdoe_account, rse_id, -1) dsns2 = [{'name': did_name_generator('dataset'), 'scope': mock_scope, 'type': 'DATASET', 'lifetime': -1, - 'rules': [{'account': jdoe, 'copies': 1, + 'rules': [{'account': jdoe_account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET'}]} for _ in range(nbdatasets)]