Client: Allow client to initialize without a set config file rucio#6410

lib/rucio/common/config.py

@@ -19,17 +19,23 @@
 import json
 import os
 from collections.abc import Callable
-from typing import TYPE_CHECKING, overload, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, overload, Optional, TypeVar, Union
 
 from rucio.common import exception
 from rucio.common.exception import ConfigNotFound, DatabaseException
+from rucio.common.utils import is_client
+
+import logging
+import tempfile
 
 _T = TypeVar('_T')
 _U = TypeVar('_U')
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
+    LoggerFunction = Callable[..., Any]
+
 
 def convert_to_any_type(value) -> Union[bool, int, float, str]:
     if value.lower() in ['true', 'yes', 'on']:
@@ -197,7 +203,6 @@ def config_get(
         except RuntimeError:
             pass
 
-        from rucio.common.utils import is_client
         client_mode = is_client()
 
         if not client_mode and check_config_table:
@@ -777,7 +782,7 @@ class Config:
     The configuration class reading the config file on init, located by using
     get_config_dirs or the use of the RUCIO_CONFIG environment variable.
     """
-    def __init__(self):
+    def __init__(self, logger: "LoggerFunction" = logging.log):
         self.parser = configparser.ConfigParser()
 
         if 'RUCIO_CONFIG' in os.environ:
@@ -786,9 +791,27 @@ def __init__(self):
             configs = [os.path.join(confdir, 'rucio.cfg') for confdir in get_config_dirs()]
             self.configfile = next(iter(filter(os.path.exists, configs)), None)
             if self.configfile is None:
-                raise RuntimeError('Could not load Rucio configuration file. '
-                                   'Rucio looked in the following paths for a configuration file, in order:'
-                                   '\n\t' + '\n\t'.join(configs))
+                client_mode = is_client()
+
+                if not client_mode:
+                    raise RuntimeError(
+                        'Could not load configuration file. A configuration file is required to run in server mode. '
+                        'Rucio looked in the following paths for a configuration file, in order: '
+                        '\n\t' + '\n\t'.join(configs)
+                    )
+
+                logger.WARNING(
+                    'Could not load Rucio configuration file. '
+                    'Rucio looked in the following paths for a configuration file, in order: '
+                    '\n\t' + '\n\t'.join(configs) + ''
+                    '\n\t Using empty configuration.'
+                )
+
+                # Make a temp cfg file
+                self.configfile = tempfile.NamedTemporaryFile(suffix=".cfg").name
+                with open(self.configfile, 'w') as f:
+                    f.write("")  # File must have some content
+                f.close()
 
         if not self.parser.read(self.configfile) == [self.configfile]:
             raise RuntimeError('Could not load Rucio configuration file. '

tests/test_clients.py

@@ -15,15 +15,18 @@
 
 from datetime import datetime, timedelta
 
-from os import rename
+import os
+from functools import wraps
+from random import choice
+from string import ascii_lowercase
 
 import pytest
 
-from rucio.client.baseclient import BaseClient
-from rucio.client.client import Client
 from rucio.common.config import config_get, config_set, Config
-from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException
+from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException, RequestNotFound
 from rucio.common.utils import execute
+from rucio.common.utils import setup_logger
+from rucio.tests.common import skip_non_atlas
 from tests.mocks.mock_http_server import MockServer
 
 
@@ -35,6 +38,19 @@ def client_token_path_override(file_config_mock, function_scope_prefix, tmp_path
     config_set('client', 'auth_token_file_path', str(tmp_path / f'{function_scope_prefix}token'))
 
 
+def run_without_config(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        configfile = Config().configfile
+        os.rename(configfile, f"{configfile}.tmp")
+
+        try:
+            func(*args, **kwargs)
+        finally:
+            os.rename(f"{configfile}.tmp", configfile)
+    return wrapper
+
+
 @pytest.mark.usefixtures("client_token_path_override")
 class TestBaseClient:
     """ To test Clients"""
@@ -44,42 +60,49 @@ class TestBaseClient:
     userkey = config_get('test', 'userkey')
 
     def testUserpass(self, vo):
+        from rucio.client.baseclient import BaseClient
         """ CLIENTS (BASECLIENT): authenticate with userpass."""
         creds = {'username': 'ddmlab', 'password': 'secret'}
-        client = BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo)
-        print(client)
+        BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo)
 
     def testUserpassWrongCreds(self, vo):
+        from rucio.client.baseclient import BaseClient
         """ CLIENTS (BASECLIENT): try to authenticate with wrong username."""
         creds = {'username': 'wrong', 'password': 'secret'}
         with pytest.raises(CannotAuthenticate):
             BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo)
 
     def testUserpassNoCACert(self, vo):
+        from rucio.client.baseclient import BaseClient
         """ CLIENTS (BASECLIENT): authenticate with userpass without ca cert."""
         creds = {'username': 'wrong', 'password': 'secret'}
         with pytest.raises(CannotAuthenticate):
             BaseClient(account='root', auth_type='userpass', creds=creds, vo=vo)
 
     def testx509(self, vo):
+        from rucio.client.baseclient import BaseClient
         """ CLIENTS (BASECLIENT): authenticate with x509."""
         creds = {'client_cert': self.usercert,
                  'client_key': self.userkey}
-        BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo)
+        logger = setup_logger(verbose=True)
+        BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo, logger=logger)
 
     def testx509NonExistingCert(self, vo):
+        from rucio.client.baseclient import BaseClient
         """ CLIENTS (BASECLIENT): authenticate with x509 with missing certificate."""
         creds = {'client_cert': '/opt/rucio/etc/web/notthere.crt'}
         with pytest.raises(CannotAuthenticate):
             BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo)
 
     def testClientProtocolNotSupported(self, vo):
+        from rucio.client.baseclient import BaseClient
         """ CLIENTS (BASECLIENT): try to pass an host with a not supported protocol."""
         creds = {'username': 'ddmlab', 'password': 'secret'}
         with pytest.raises(ClientProtocolNotSupported):
             BaseClient(rucio_host='localhost', auth_host='junk://localhost', account='root', auth_type='userpass', creds=creds, vo=vo)
 
     def testRetryOn502AlwaysFail(self, vo):
+        from rucio.client.baseclient import BaseClient
         """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes, but fails on repeated errors"""
 
         class AlwaysFailWith502(MockServer.Handler):
@@ -97,6 +120,8 @@ def do_GET(self):
 
     def testRetryOn502SucceedsEventually(self, vo):
         """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes"""
+
+        from rucio.client.baseclient import BaseClient
         invocations = []
 
         class FailTwiceWith502(MockServer.Handler):
@@ -118,37 +143,186 @@ def do_GET(self, invocations=invocations):
         assert datetime.utcnow() - start_time > timedelta(seconds=0.9)
 
 
-class TestRucioClients:
-    """ To test Clients"""
+# Run the whole suite without the config file in place
+@pytest.mark.noparallel()
+@run_without_config
+def test_import_no_config_file():
 
-    cacert = config_get('test', 'cacert')
-    marker = '$> '
+    exitcode, _, err = execute("python -c 'from rucio.client import Client'")
 
-    def test_ping(self, vo):
-        """ PING (CLIENT): Ping Rucio """
-        creds = {'username': 'ddmlab', 'password': 'secret'}
+    assert Config().configfile.split('/') != "rucio.cfg"  # Cannot find the config file
 
-        client = Client(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo)
+    assert exitcode == 0
+    assert "RuntimeError: Could not load Rucio configuration file." not in err
 
-        print(client.ping())
 
-    @pytest.mark.noparallel(reason='We temporarily remove the config file.')
-    def test_import_without_config_file(self, vo):
-        """
-        The Client should be importable without a config file, since it is
-        configurable afterwards.
+@pytest.mark.noparallel()
+@run_without_config
+def test_ping_no_config():
+    from rucio.client.client import Client
+    creds = {'username': 'ddmlab', 'password': 'secret'}
+    ca_cert = '/etc/grid-security/certificates/5fca1cb1.0'
+    log = setup_logger(verbose=True)
+    client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log)
+    client.ping()
 
-        We are in a fully configured environment with a default config file. We
-        thus have to disable the access to it (move it) and make sure to run the
-        code in a different environment.
-        """
-        configfile = Config().configfile
-        rename(configfile, f"{configfile}.tmp")
-        try:
-            exitcode, _, err = execute("python -c 'from rucio.client import Client'")
-            print(exitcode, err)
-            assert exitcode == 0
-            assert "RuntimeError: Could not load Rucio configuration file." not in err
-        finally:
-            # This is utterly important to not mess up the environment.
-            rename(f"{configfile}.tmp", configfile)
+
+@pytest.mark.noparallel()
+@run_without_config
+def test_account_no_config():
+    from rucio.client.client import Client
+    creds = {'username': 'ddmlab', 'password': 'secret'}
+    ca_cert = '/etc/grid-security/certificates/5fca1cb1.0'
+    log = setup_logger(verbose=True)
+    client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log)
+    client.list_accounts()
+
+    # List who am i
+    client.whoami()
+    name = ''.join(choice(ascii_lowercase) for _ in range(6))
+    # Make an account
+    account_name, type_, email = f"mock_name_{name}", "user", "mock_email"
+    assert client.add_account(account_name, type_, email)
+
+    # Delete that same account
+    assert client.delete_account(account_name)
+
+
+@pytest.mark.noparallel()
+@run_without_config
+def test_did_no_config(did_factory):
+    from rucio.client.client import Client
+
+    did1, did2 = did_factory.random_dataset_did(), did_factory.random_dataset_did()
+
+    creds = {'username': 'ddmlab', 'password': 'secret'}
+    ca_cert = '/etc/grid-security/certificates/5fca1cb1.0'
+    log = setup_logger(verbose=True)
+    client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log)
+
+    assert client.add_did(scope='mock', name=did1['name'], did_type='dataset')
+    assert client.add_did(scope='mock', name=did2['name'], did_type='container')
+
+    # Make an attachment
+    assert client.attach_dids(scope="mock", name=did2['name'], dids=[did1])
+    # List did
+    client.list_dids(scope='mock', filters=[])
+
+
+@pytest.mark.noparallel()
+@run_without_config
+@skip_non_atlas  # Avoid failure from did format
+def test_upload_download_no_config(file_factory, rse_factory):
+    import os
+    # Make item
+    from rucio.client.client import Client
+    from rucio.client.uploadclient import UploadClient
+    from rucio.client.downloadclient import DownloadClient
+
+    creds = {'username': 'ddmlab', 'password': 'secret'}
+    ca_cert = '/etc/grid-security/certificates/5fca1cb1.0'
+    log = setup_logger(verbose=True)
+    client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log)
+
+    scope = 'mock'
+    rse, _ = rse_factory.make_posix_rse()
+    local_file = file_factory.file_generator()
+    download_dir = file_factory.base_dir
+    fn = os.path.basename(local_file)
+
+    # upload a file
+    status = UploadClient(client).upload([{
+        'path': local_file,
+        'rse': rse,
+        'did_scope': scope,
+        'did_name': fn,
+    }])
+    assert status == 0
+
+    # download the file
+    did = f"{scope}:{fn}"
+    DownloadClient(client).download_dids([{'did': did, 'base_dir': download_dir}])
+
+    downloaded_file = f"{download_dir}/{scope}/{fn}"
+    assert os.path.exists(downloaded_file)
+
+
+@pytest.mark.noparallel()
+@run_without_config
+@skip_non_atlas
+def test_replica_no_config(rse_factory, did_factory, file_factory):
+    from rucio.client.client import Client
+    from rucio.client.uploadclient import UploadClient
+
+    creds = {'username': 'ddmlab', 'password': 'secret'}
+    ca_cert = '/etc/grid-security/certificates/5fca1cb1.0'
+    log = setup_logger(verbose=True)
+    client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log)
+
+    scope = 'mock'
+    rse, _ = rse_factory.make_posix_rse()
+
+    def make_replica():
+        local_file = file_factory.file_generator()
+        fn = os.path.basename(local_file)
+
+        # upload a file
+        UploadClient(client).upload([{
+            'path': local_file,
+            'rse': rse,
+            'did_scope': scope,
+            'did_name': fn,
+        }])
+
+        mock_scope = 'mock'
+        replicas = [{'scope': mock_scope, 'name': fn}]
+
+        return replicas
+    replica = make_replica()
+
+    assert client.add_replicas(rse, replica)
+    assert client.list_dataset_replicas("mock", replica[0]['name'])
+    new_replica = make_replica()
+    assert client.add_replicas(rse, new_replica)
+    new_replica[0]['state'] = 'D'
+    assert client.update_replicas_states(rse, new_replica)
+
+    # TODO Check for an account with the auth to delete files
+
+
+@pytest.mark.noparallel()
+@run_without_config
+@skip_non_atlas
+def test_request_no_config(rse_factory, did_factory, file_factory):
+    from rucio.client.client import Client
+    from rucio.client.uploadclient import UploadClient
+    creds = {'username': 'ddmlab', 'password': 'secret'}
+    ca_cert = '/etc/grid-security/certificates/5fca1cb1.0'
+    log = setup_logger(verbose=True)
+    client = Client(account='root', auth_type='userpass', ca_cert=ca_cert, creds=creds, logger=log)
+
+    mock_scope = 'mock'
+    rse, _ = rse_factory.make_posix_rse()
+    rse2, _ = rse_factory.make_posix_rse()
+
+    local_file = file_factory.file_generator()
+    fn = os.path.basename(local_file)
+
+    # upload a file
+    UploadClient(client).upload([{
+        'path': local_file,
+        'rse': rse,
+        'did_scope': mock_scope,
+        'did_name': fn,
+    }])
+
+    did = {"name": fn, "scope": mock_scope}
+
+    client.list_requests(rse, rse2, request_states='Q')
+    client.list_requests_history(rse, rse2, 'Q')
+
+    with pytest.raises(RequestNotFound):
+        client.list_request_by_did(did['name'], rse, mock_scope)
+
+    with pytest.raises(RequestNotFound):
+        client.list_request_history_by_did(did['name'], rse, mock_scope)