From 6ccb8412ac49da8f7eff79ab82d6a7f92058fafc Mon Sep 17 00:00:00 2001 From: voetberg Date: Fri, 22 Mar 2024 15:29:10 -0500 Subject: [PATCH] Client: Run client without configuration file; Fix #6410 Changes: * Change `RuntimeError(Config Not Found)` to `ConfigNotFound` Error, config_get correctly handles those and returns a default client config * Added typing for baseclient, initialization broken into functions to avoid typing conflicts for optionals. * Aded test to run multiple client commands without configuration file in place, include func dectorator for extended testing --- lib/rucio/client/baseclient.py | 276 ++++++++++++++++++--------------- lib/rucio/common/config.py | 20 +-- lib/rucio/common/utils.py | 4 +- lib/rucio/tests/common.py | 41 ++++- tests/test_clients.py | 84 ++++++++-- 5 files changed, 263 insertions(+), 162 deletions(-) diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py index 77a5df050d..e1f58eb88b 100644 --- a/lib/rucio/client/baseclient.py +++ b/lib/rucio/client/baseclient.py @@ -23,11 +23,14 @@ import sys import time from configparser import NoOptionError, NoSectionError +from logging import Logger from os import environ, fdopen, geteuid, makedirs, path from shutil import move from tempfile import mkstemp +from typing import Any, Generator, Optional from urllib.parse import urlparse +import requests from dogpile.cache import make_region from requests import Response, Session from requests.exceptions import ConnectionError @@ -36,7 +39,7 @@ from rucio import version from rucio.common import exception from rucio.common.config import config_get, config_get_bool, config_get_int -from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, MissingClientParameter, MissingModuleException, NoAuthInformation, ServerConnectionException +from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, ConfigNotFound, MissingClientParameter, MissingModuleException, NoAuthInformation, ServerConnectionException from rucio.common.extra import import_extras from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign @@ -76,7 +79,17 @@ class BaseClient: TOKEN_PREFIX = 'auth_token_' TOKEN_EXP_PREFIX = 'auth_token_exp_' - def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, auth_type=None, creds=None, timeout=600, user_agent='rucio-clients', vo=None, logger=None): + def __init__(self, + rucio_host: Optional[str] = None, + auth_host: Optional[str] = None, + account: Optional[str] = None, + ca_cert: Optional[str] = None, + auth_type: Optional[str] = None, + creds: Optional[dict[str, Any]] = None, + timeout: Optional[int] = 600, + user_agent: Optional[str] = 'rucio-clients', + vo: Optional[str] = None, + logger: Logger = LOG) -> None: """ Constructor of the BaseClient. :param rucio_host: The address of the rucio server, if None it is read from the config file. @@ -94,9 +107,8 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, """ self.host = rucio_host - self.list_hosts = [] self.auth_host = auth_host - self.logger = logger or LOG + self.logger = logger self.session = Session() self.user_agent = "%s/%s" % (user_agent, version.version_string()) # e.g. "rucio-clients/0.2.13" sys.argv[0] = sys.argv[0].split('/')[-1] @@ -113,110 +125,26 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, try: self.trace_host = config_get('trace', 'trace_host') - except (NoOptionError, NoSectionError): + except (NoOptionError, NoSectionError, ConfigNotFound): self.trace_host = self.host self.logger.debug('No trace_host passed. Using rucio_host instead') + self.list_hosts = [self.host] self.account = account self.vo = vo self.ca_cert = ca_cert - self.auth_type = auth_type - self.creds = creds - self.auth_token = None - self.auth_token_file_path = config_get('client', 'auth_token_file_path', False, None) + self.auth_token = "" self.headers = {} self.timeout = timeout self.request_retries = self.REQUEST_RETRIES self.token_exp_epoch = None - self.token_exp_epoch_file = None self.auth_oidc_refresh_active = config_get_bool('client', 'auth_oidc_refresh_active', False, False) + # defining how many minutes before token expires, oidc refresh (if active) should start self.auth_oidc_refresh_before_exp = config_get_int('client', 'auth_oidc_refresh_before_exp', False, 20) - if auth_type is None: - self.logger.debug('No auth_type passed. Trying to get it from the environment variable RUCIO_AUTH_TYPE and config file.') - if 'RUCIO_AUTH_TYPE' in environ: - if environ['RUCIO_AUTH_TYPE'] not in ['userpass', 'x509', 'x509_proxy', 'gss', 'ssh', 'saml', 'oidc']: - raise MissingClientParameter('Possible RUCIO_AUTH_TYPE values: userpass, x509, x509_proxy, gss, ssh, saml, oidc, vs. ' + environ['RUCIO_AUTH_TYPE']) - self.auth_type = environ['RUCIO_AUTH_TYPE'] - else: - try: - self.auth_type = config_get('client', 'auth_type') - except (NoOptionError, NoSectionError) as error: - raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0]) - - if self.auth_type == 'oidc': - if not self.creds: - self.creds = {} - # if there are defautl values, check if rucio.cfg does not specify them, otherwise put default - if 'oidc_refresh_lifetime' not in self.creds or self.creds['oidc_refresh_lifetime'] is None: - self.creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None) - if 'oidc_issuer' not in self.creds or self.creds['oidc_issuer'] is None: - self.creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None) - if 'oidc_audience' not in self.creds or self.creds['oidc_audience'] is None: - self.creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None) - if 'oidc_auto' not in self.creds or self.creds['oidc_auto'] is False: - self.creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False) - if self.creds['oidc_auto']: - if 'oidc_username' not in self.creds or self.creds['oidc_username'] is None: - self.creds['oidc_username'] = config_get('client', 'oidc_username', False, None) - if 'oidc_password' not in self.creds or self.creds['oidc_password'] is None: - self.creds['oidc_password'] = config_get('client', 'oidc_password', False, None) - if 'oidc_scope' not in self.creds or self.creds['oidc_scope'] == 'openid profile': - self.creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile') - if 'oidc_polling' not in self.creds or self.creds['oidc_polling'] is False: - self.creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False) - - if not self.creds: - self.logger.debug('No creds passed. Trying to get it from the config file.') - self.creds = {} - try: - if self.auth_type in ['userpass', 'saml']: - self.creds['username'] = config_get('client', 'username') - self.creds['password'] = config_get('client', 'password') - elif self.auth_type == 'x509': - if "RUCIO_CLIENT_CERT" in environ: - client_cert = environ["RUCIO_CLIENT_CERT"] - else: - client_cert = config_get('client', 'client_cert') - self.creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(client_cert))) - if not path.exists(self.creds['client_cert']): - raise MissingClientParameter('X.509 client certificate not found: %s' % self.creds['client_cert']) - - if "RUCIO_CLIENT_KEY" in environ: - client_key = environ["RUCIO_CLIENT_KEY"] - else: - client_key = config_get('client', 'client_key') - self.creds['client_key'] = path.abspath(path.expanduser(path.expandvars(client_key))) - if not path.exists(self.creds['client_key']): - raise MissingClientParameter('X.509 client key not found: %s' % self.creds['client_key']) - else: - perms = oct(os.stat(self.creds['client_key']).st_mode)[-3:] - if perms not in ['400', '600']: - raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (self.creds['client_key'], perms)) - - elif self.auth_type == 'x509_proxy': - try: - self.creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy')))) - except NoOptionError: - # Recreate the classic GSI logic for locating the proxy: - # - $X509_USER_PROXY, if it is set. - # - /tmp/x509up_u`id -u` otherwise. - # If neither exists (at this point, we don't care if it exists but is invalid), then rethrow - if 'X509_USER_PROXY' in environ: - self.creds['client_proxy'] = environ['X509_USER_PROXY'] - else: - fname = '/tmp/x509up_u%d' % geteuid() - if path.exists(fname): - self.creds['client_proxy'] = fname - else: - raise MissingClientParameter('Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and ' - '\'x509_proxy\' not set in the configuration file.' % fname) - elif self.auth_type == 'ssh': - self.creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key')))) - except (NoOptionError, NoSectionError) as error: - if error.args[0] != 'client_key': - raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0]) + self.auth_type = self._get_auth_type(auth_type) + self.creds = self._get_creds(creds) rucio_scheme = urlparse(self.host).scheme auth_scheme = urlparse(self.auth_host).scheme @@ -238,8 +166,6 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).') self.ca_cert = True - self.list_hosts = [self.host] - if account is None: self.logger.debug('No account passed. Trying to get it from the RUCIO_ACCOUNT environment variable or the config file.') try: @@ -262,30 +188,126 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, self.logger.debug('No VO found. Using default VO.') self.vo = 'def' - token_filename_suffix = "for_default_account" if self.account is None else "for_account_" + self.account + self.auth_token_file_path, self.token_exp_epoch_file, self.token_file, self.token_path = self._get_auth_tokens() + self.__authenticate() + try: + self.request_retries = config_get_int('client', 'request_retries') + except (NoOptionError, ConfigNotFound): + self.logger.debug('request_retries not specified in config file. Taking default.') + except ValueError: + self.logger.debug('request_retries must be an integer. Taking default.') + + def _get_auth_tokens(self) -> tuple[Optional[str], str, str, str]: # if token file path is defined in the rucio.cfg file, use that file. Currently this prevents authenticating as another user or VO. - if self.auth_token_file_path: - self.token_file = self.auth_token_file_path - self.token_path = '/'.join(self.token_file.split('/')[:-1]) + auth_token_file_path = config_get('client', 'auth_token_file_path', False, None) + token_filename_suffix = "for_default_account" if self.account is None else "for_account_" + self.account + + if auth_token_file_path: + token_file = auth_token_file_path + token_path = '/'.join(auth_token_file_path.split('/')[:-1]) + else: - self.token_path = self.TOKEN_PATH_PREFIX + getpass.getuser() + token_path = self.TOKEN_PATH_PREFIX + getpass.getuser() if self.vo != 'def': - self.token_path += '@%s' % self.vo - self.token_file = self.token_path + '/' + self.TOKEN_PREFIX + token_filename_suffix + token_path += '@%s' % self.vo - self.token_exp_epoch_file = self.token_path + '/' + self.TOKEN_EXP_PREFIX + token_filename_suffix + token_file = token_path + '/' + self.TOKEN_PREFIX + token_filename_suffix - self.__authenticate() + token_exp_epoch_file = token_path + '/' + self.TOKEN_EXP_PREFIX + token_filename_suffix + return auth_token_file_path, token_exp_epoch_file, token_file, token_path - try: - self.request_retries = config_get_int('client', 'request_retries') - except (NoOptionError, RuntimeError): - LOG.debug('request_retries not specified in config file. Taking default.') - except ValueError: - self.logger.debug('request_retries must be an integer. Taking default.') + def _get_auth_type(self, auth_type: Optional[str]) -> str: + if auth_type is None: + self.logger.debug('No auth_type passed. Trying to get it from the environment variable RUCIO_AUTH_TYPE and config file.') + if 'RUCIO_AUTH_TYPE' in environ: + if environ['RUCIO_AUTH_TYPE'] not in ['userpass', 'x509', 'x509_proxy', 'gss', 'ssh', 'saml', 'oidc']: + raise MissingClientParameter('Possible RUCIO_AUTH_TYPE values: userpass, x509, x509_proxy, gss, ssh, saml, oidc, vs. ' + environ['RUCIO_AUTH_TYPE']) + auth_type = environ['RUCIO_AUTH_TYPE'] + else: + try: + auth_type = config_get('client', 'auth_type') + except (NoOptionError, NoSectionError) as error: + raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0]) + return auth_type + + def _get_creds(self, creds: Optional[dict[str, Any]]) -> dict[str, Any]: + if self.auth_type == 'oidc': + if not creds: + creds = {} + # if there are defautl values, check if rucio.cfg does not specify them, otherwise put default + if 'oidc_refresh_lifetime' not in creds or creds['oidc_refresh_lifetime'] is None: + creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None) + if 'oidc_issuer' not in creds or creds['oidc_issuer'] is None: + creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None) + if 'oidc_audience' not in creds or creds['oidc_audience'] is None: + creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None) + if 'oidc_auto' not in creds or creds['oidc_auto'] is False: + creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False) + if creds['oidc_auto']: + if 'oidc_username' not in creds or creds['oidc_username'] is None: + creds['oidc_username'] = config_get('client', 'oidc_username', False, None) + if 'oidc_password' not in creds or creds['oidc_password'] is None: + creds['oidc_password'] = config_get('client', 'oidc_password', False, None) + if 'oidc_scope' not in creds or creds['oidc_scope'] == 'openid profile': + creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile') + if 'oidc_polling' not in creds or creds['oidc_polling'] is False: + creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False) + + if creds is None: + self.logger.debug('No creds passed. Trying to get it from the config file.') + creds = {} + try: + if self.auth_type in ['userpass', 'saml']: + creds['username'] = config_get('client', 'username') + creds['password'] = config_get('client', 'password') + elif self.auth_type == 'x509': + if "RUCIO_CLIENT_CERT" in environ: + client_cert = environ["RUCIO_CLIENT_CERT"] + else: + client_cert = config_get('client', 'client_cert') + creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(client_cert))) + if not path.exists(creds['client_cert']): + raise MissingClientParameter('X.509 client certificate not found: %s' % creds['client_cert']) + + if "RUCIO_CLIENT_KEY" in environ: + client_key = environ["RUCIO_CLIENT_KEY"] + else: + client_key = config_get('client', 'client_key') + creds['client_key'] = path.abspath(path.expanduser(path.expandvars(client_key))) + if not path.exists(creds['client_key']): + raise MissingClientParameter('X.509 client key not found: %s' % creds['client_key']) + else: + perms = oct(os.stat(creds['client_key']).st_mode)[-3:] + if perms not in ['400', '600']: + raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms)) + + elif self.auth_type == 'x509_proxy': + try: + creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy')))) + except NoOptionError: + # Recreate the classic GSI logic for locating the proxy: + # - $X509_USER_PROXY, if it is set. + # - /tmp/x509up_u`id -u` otherwise. + # If neither exists (at this point, we don't care if it exists but is invalid), then rethrow + if 'X509_USER_PROXY' in environ: + creds['client_proxy'] = environ['X509_USER_PROXY'] + else: + fname = '/tmp/x509up_u%d' % geteuid() + if path.exists(fname): + creds['client_proxy'] = fname + else: + raise MissingClientParameter( + 'Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and ' + '\'x509_proxy\' not set in the configuration file.' % fname) + elif self.auth_type == 'ssh': + creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key')))) + except (NoOptionError, NoSectionError) as error: + if error.args[0] != 'client_key': + raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0]) + return creds - def _get_exception(self, headers, status_code=None, data=None): + def _get_exception(self, headers: dict[str, str], status_code: Optional[int] = None, data=None) -> tuple[type[exception.RucioException], str]: """ Helper method to parse an error string send by the server and transform it into the corresponding rucio exception. @@ -316,7 +338,7 @@ def _get_exception(self, headers, status_code=None, data=None): else: return exception.RucioException, "%s: %s" % (exc_cls, exc_msg) - def _load_json_data(self, response): + def _load_json_data(self, response: requests.Response) -> Generator[Any, Any, Any]: """ Helper method to correctly load json data based on the content type of the http response. @@ -332,13 +354,13 @@ def _load_json_data(self, response): if response.text: yield response.text - def _reduce_data(self, data, maxlen=132): + def _reduce_data(self, data, maxlen: int = 132) -> str: text = data if isinstance(data, str) else data.decode("utf-8") if len(text) > maxlen: text = "%s ... %s" % (text[:maxlen - 15], text[-10:]) return text - def _back_off(self, retry_number, reason): + def _back_off(self, retry_number: int, reason: str) -> None: """ Sleep a certain amount of time which increases with the retry count :param retry_number: the retry iteration @@ -433,7 +455,7 @@ def _send_request(self, url, headers=None, type_='GET', data=None, params=None, raise ServerConnectionException return result - def __get_token_userpass(self): + def __get_token_userpass(self) -> bool: """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses username/password. @@ -469,7 +491,7 @@ def __get_token_userpass(self): self.auth_token = result.headers['x-rucio-auth-token'] return True - def __refresh_token_OIDC(self): + def __refresh_token_OIDC(self) -> bool: """ Checks if there is active refresh token and if so returns either active token with expiration timestamp or requests a new @@ -523,7 +545,7 @@ def __refresh_token_OIDC(self): \nRucio Auth Server when attempting token refresh.") return False - def __get_token_OIDC(self): + def __get_token_OIDC(self) -> bool: """ First authenticates the user via a Identity Provider server (with user's username & password), by specifying oidc_scope, @@ -647,7 +669,7 @@ def __get_token_OIDC(self): self.__refresh_token_OIDC() return True - def __get_token_x509(self): + def __get_token_x509(self) -> bool: """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses x509 authentication. @@ -664,7 +686,7 @@ def __get_token_x509(self): url = build_url(self.auth_host, path='auth/x509_proxy') client_cert = self.creds['client_proxy'] - if not path.exists(client_cert): + if (client_cert is not None) and not (path.exists(client_cert)): self.logger.error('given client cert (%s) doesn\'t exist' % client_cert) return False if client_key is not None and not path.exists(client_key): @@ -692,7 +714,7 @@ def __get_token_x509(self): self.auth_token = result.headers['x-rucio-auth-token'] return True - def __get_token_ssh(self): + def __get_token_ssh(self) -> bool: """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses SSH key exchange authentication. @@ -748,7 +770,7 @@ def __get_token_ssh(self): self.auth_token = result.headers['x-rucio-auth-token'] return True - def __get_token_gss(self): + def __get_token_gss(self) -> bool: """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses Kerberos authentication. @@ -774,7 +796,7 @@ def __get_token_gss(self): self.auth_token = result.headers['x-rucio-auth-token'] return True - def __get_token_saml(self): + def __get_token_saml(self) -> bool: """ Sends a request to get an auth token from the server and stores it as a class attribute. Uses saml authentication. @@ -804,7 +826,7 @@ def __get_token_saml(self): self.auth_token = result.headers['X-Rucio-Auth-Token'] return True - def __get_token(self): + def __get_token(self) -> None: """ Calls the corresponding method to receive an auth token depending on the auth type. To be used if a 401 - Unauthorized error is received. """ @@ -846,7 +868,7 @@ def __get_token(self): if self.auth_token is None: raise CannotAuthenticate('cannot get an auth token from server') - def __read_token(self): + def __read_token(self) -> bool: """ Checks if a local token file exists and reads the token from it. @@ -868,7 +890,7 @@ def __read_token(self): self.logger.debug('got token from file') return True - def __write_token(self): + def __write_token(self) -> None: """ Write the current auth_token to the local token file. """ @@ -895,7 +917,7 @@ def __write_token(self): except Exception: raise - def __authenticate(self): + def __authenticate(self) -> None: """ Main method for authentication. It first tries to read a locally saved token. If not available it requests a new one. """ diff --git a/lib/rucio/common/config.py b/lib/rucio/common/config.py index df44fc9f85..2e64cc732b 100644 --- a/lib/rucio/common/config.py +++ b/lib/rucio/common/config.py @@ -188,12 +188,12 @@ def config_get( """ try: return convert_type_fnc(get_config().get(section, option)) - except (configparser.NoOptionError, configparser.NoSectionError, RuntimeError) as err: + except (configparser.NoOptionError, configparser.NoSectionError, ConfigNotFound) as err: try: legacy_config = get_legacy_config(section, option) if legacy_config is not None: return convert_type_fnc(legacy_config) - except RuntimeError: + except ConfigNotFound: pass from rucio.common.utils import is_client @@ -729,7 +729,7 @@ def get_lfn2pfn_algorithm_default(): default_lfn2pfn = "hash" try: default_lfn2pfn = config_get('policy', 'lfn2pfn_algorithm_default') - except (configparser.NoOptionError, configparser.NoSectionError, RuntimeError): + except (configparser.NoOptionError, configparser.NoSectionError, ConfigNotFound, RuntimeError): pass return default_lfn2pfn @@ -785,11 +785,13 @@ def __init__(self): configs = [os.path.join(confdir, 'rucio.cfg') for confdir in get_config_dirs()] self.configfile = next(iter(filter(os.path.exists, configs)), None) if self.configfile is None: - raise RuntimeError('Could not load Rucio configuration file. ' - 'Rucio looked in the following paths for a configuration file, in order:' - '\n\t' + '\n\t'.join(configs)) + raise ConfigNotFound( + 'Could not load Rucio configuration file. ' + 'Rucio looked in the following paths for a configuration file, in order:' + '\n\t' + '\n\t'.join(configs)) if not self.parser.read(self.configfile) == [self.configfile]: - raise RuntimeError('Could not load Rucio configuration file. ' - 'Rucio tried loading the following configuration file:' - '\n\t' + self.configfile) + raise ConfigNotFound( + 'Could not load Rucio configuration file. ' + 'Rucio tried loading the following configuration file:' + '\n\t' + self.configfile) diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py index 3f16b28561..8744cfd7d6 100644 --- a/lib/rucio/common/utils.py +++ b/lib/rucio/common/utils.py @@ -50,7 +50,7 @@ import requests from rucio.common.config import config_get, config_has_section -from rucio.common.exception import DIDFilterSyntaxError, DuplicateCriteriaInDIDFilter, InputValidationError, InvalidType, MetalinkJsonParsingError, MissingModuleException, PolicyPackageVersionError, RucioException +from rucio.common.exception import ConfigNotFound, DIDFilterSyntaxError, DuplicateCriteriaInDIDFilter, InputValidationError, InvalidType, MetalinkJsonParsingError, MissingModuleException, PolicyPackageVersionError, RucioException from rucio.common.extra import import_extras from rucio.common.plugins import PolicyPackageAlgorithms from rucio.common.types import InternalAccount, InternalScope @@ -1844,7 +1844,7 @@ def is_client(): client_mode = True else: client_mode = False - except RuntimeError: + except (RuntimeError, ConfigNotFound): # If no configuration file is found the default value should be True client_mode = True else: diff --git a/lib/rucio/tests/common.py b/lib/rucio/tests/common.py index e0b6f93d5a..b530572b54 100644 --- a/lib/rucio/tests/common.py +++ b/lib/rucio/tests/common.py @@ -18,9 +18,11 @@ import os import tempfile from collections import namedtuple +from functools import wraps +from os import rename from random import choice, choices from string import ascii_letters, ascii_uppercase, digits -from typing import Optional +from typing import Any, Callable, Iterable, Optional import pytest import requests @@ -129,7 +131,7 @@ def rse_name_generator(size: int = 10) -> str: return 'MOCK-' + ''.join(choice(ascii_uppercase) for x in range(size)) -def rfc2253_dn_generator(): +def rfc2253_dn_generator() -> str: """ Generate a random DN in RFC 2253 format. :returns: A random DN @@ -141,7 +143,7 @@ def rfc2253_dn_generator(): return random_dn -def file_generator(size: int = 2, namelen: int = 10): +def file_generator(size: int = 2, namelen: int = 10) -> str: """ Create a bogus file and returns it's name. :param size: size in bytes :returns: The name of the generated file. @@ -199,11 +201,11 @@ def print_response(rest_response): print(text if text else '') -def headers(*iterables): +def headers(*iterables: Iterable): return list(itertools.chain(*iterables)) -def loginhdr(account, username, password): +def loginhdr(account: str, username: str, password: str): yield 'X-Rucio-Account', str(account) yield 'X-Rucio-Username', str(username) yield 'X-Rucio-Password', str(password) @@ -213,12 +215,12 @@ def auth(token): yield 'X-Rucio-Auth-Token', str(token) -def vohdr(vo): +def vohdr(vo: str): if vo: yield 'X-Rucio-VO', str(vo) -def hdrdict(dictionary): +def hdrdict(dictionary: dict): for key in dictionary: yield str(key), str(dictionary[key]) @@ -235,10 +237,33 @@ class Mime: BINARY = 'application/octet-stream' -def load_test_conf_file(file_name): +def load_test_conf_file(file_name: str) -> dict[str, Any]: config_dir = next(filter(lambda d: os.path.exists(os.path.join(d, file_name)), get_config_dirs())) with open(os.path.join(config_dir, file_name)) as f: return json.load(f) +def remove_config(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs): + for configfile in get_config_dirs(): + # Rename the config to .tmp + try: + rename(f"{configfile}rucio.cfg", f"{configfile}rucio.cfg.tmp") + except FileNotFoundError: + pass # When a test uses a os.env assigned conf, there's nothing stating the default location has something + try: + # Execute the test + func(*args, **kwargs) + finally: + # And put the config back + for configfile in get_config_dirs(): + try: + rename(f"{configfile}rucio.cfg.tmp", f"{configfile}rucio.cfg") + except FileNotFoundError: + pass + + return wrapper + + RSE_namedtuple = namedtuple('RSE_namedtuple', ['name', 'id']) diff --git a/tests/test_clients.py b/tests/test_clients.py index a7ff353a42..03e67e2d23 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -13,15 +13,12 @@ # limitations under the License. from datetime import datetime, timedelta -from os import rename import pytest -from rucio.client.baseclient import BaseClient -from rucio.client.client import Client -from rucio.common.config import Config, config_get, config_set from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException from rucio.common.utils import execute +from rucio.tests.common import remove_config from tests.mocks.mock_http_server import MockServer @@ -30,6 +27,7 @@ def client_token_path_override(file_config_mock, function_scope_prefix, tmp_path """ Ensure each running client has a different path for the token, otherwise tests cannot run in parallel """ + from rucio.common.config import config_set config_set('client', 'auth_token_file_path', str(tmp_path / f'{function_scope_prefix}token')) @@ -37,6 +35,8 @@ def client_token_path_override(file_config_mock, function_scope_prefix, tmp_path class TestBaseClient: """ To test Clients""" + from rucio.common.config import config_get + cacert = config_get('test', 'cacert') usercert = config_get('test', 'usercert') userkey = config_get('test', 'userkey') @@ -44,23 +44,30 @@ class TestBaseClient: def testUserpass(self, vo): """ CLIENTS (BASECLIENT): authenticate with userpass.""" creds = {'username': 'ddmlab', 'password': 'secret'} + from rucio.client.baseclient import BaseClient client = BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo) print(client) def testUserpassWrongCreds(self, vo): """ CLIENTS (BASECLIENT): try to authenticate with wrong username.""" creds = {'username': 'wrong', 'password': 'secret'} + from rucio.client.baseclient import BaseClient + with pytest.raises(CannotAuthenticate): BaseClient(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo) def testUserpassNoCACert(self, vo): """ CLIENTS (BASECLIENT): authenticate with userpass without ca cert.""" creds = {'username': 'wrong', 'password': 'secret'} + from rucio.client.baseclient import BaseClient + with pytest.raises(CannotAuthenticate): BaseClient(account='root', auth_type='userpass', creds=creds, vo=vo) def testx509(self, vo): """ CLIENTS (BASECLIENT): authenticate with x509.""" + from rucio.client.baseclient import BaseClient + creds = {'client_cert': self.usercert, 'client_key': self.userkey} BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo) @@ -68,17 +75,22 @@ def testx509(self, vo): def testx509NonExistingCert(self, vo): """ CLIENTS (BASECLIENT): authenticate with x509 with missing certificate.""" creds = {'client_cert': '/opt/rucio/etc/web/notthere.crt'} + from rucio.client.baseclient import BaseClient + with pytest.raises(CannotAuthenticate): BaseClient(account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, vo=vo) def testClientProtocolNotSupported(self, vo): """ CLIENTS (BASECLIENT): try to pass an host with a not supported protocol.""" creds = {'username': 'ddmlab', 'password': 'secret'} + from rucio.client.baseclient import BaseClient + with pytest.raises(ClientProtocolNotSupported): BaseClient(rucio_host='localhost', auth_host='junk://localhost', account='root', auth_type='userpass', creds=creds, vo=vo) def testRetryOn502AlwaysFail(self, vo): """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes, but fails on repeated errors""" + from rucio.client.baseclient import BaseClient class AlwaysFailWith502(MockServer.Handler): def do_GET(self): @@ -88,6 +100,7 @@ def do_GET(self): with pytest.raises(CannotAuthenticate): creds = {'username': 'ddmlab', 'password': 'secret'} BaseClient(rucio_host=server.base_url, auth_host=server.base_url, account='root', auth_type='userpass', creds=creds, vo=vo) + with pytest.raises(RucioException): creds = {'client_cert': self.usercert, 'client_key': self.userkey} @@ -96,6 +109,7 @@ def do_GET(self): def testRetryOn502SucceedsEventually(self, vo): """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes""" invocations = [] + from rucio.client.baseclient import BaseClient class FailTwiceWith502(MockServer.Handler): def do_GET(self, invocations=invocations): @@ -119,18 +133,22 @@ def do_GET(self, invocations=invocations): class TestRucioClients: """ To test Clients""" - cacert = config_get('test', 'cacert') marker = '$> ' def test_ping(self, vo): """ PING (CLIENT): Ping Rucio """ + creds = {'username': 'ddmlab', 'password': 'secret'} - client = Client(account='root', ca_cert=self.cacert, auth_type='userpass', creds=creds, vo=vo) + from rucio.client.client import Client + from rucio.common.config import config_get + cacert = config_get('test', 'cacert') + client = Client(account='root', ca_cert=cacert, auth_type='userpass', creds=creds, vo=vo) print(client.ping()) @pytest.mark.noparallel(reason='We temporarily remove the config file.') + @remove_config def test_import_without_config_file(self, vo): """ The Client should be importable without a config file, since it is @@ -140,13 +158,47 @@ def test_import_without_config_file(self, vo): thus have to disable the access to it (move it) and make sure to run the code in a different environment. """ - configfile = Config().configfile - rename(configfile, f"{configfile}.tmp") - try: - exitcode, _, err = execute("python -c 'from rucio.client import Client'") - print(exitcode, err) - assert exitcode == 0 - assert "RuntimeError: Could not load Rucio configuration file." not in err - finally: - # This is utterly important to not mess up the environment. - rename(f"{configfile}.tmp", configfile) + + exitcode, _, err = execute("python -c 'from rucio.client import Client'") + print(exitcode, err) + assert exitcode == 0 + assert "Could not load Rucio configuration file." not in err + + @pytest.mark.noparallel(reason='We temporarily remove the config file.') + @remove_config + def test_run_client_no_config(self, vo): + rucio_host = "https://rucio:443" + auth_host = "https://rucio:443" + ca_cert = "/etc/grid-security/certificates/5fca1cb1.0" + creds = {'username': 'ddmlab', 'password': 'secret'} + + from rucio.client.client import Client + + client = Client( + rucio_host=rucio_host, + auth_host=auth_host, + ca_cert=ca_cert, + account='root', + auth_type='userpass', + creds=creds, + vo=vo) + + # Couple of basic calls + # account + assert client.whoami()['account'] == 'root' + assert client.get_account('jdoe')['account'] == "jdoe" + + # config + assert client.set_config_option("mock_section", "mock_option", value=0) + + # did + assert list(client.list_dids(scope='mock', filters={})) is not None + + # replica + assert list(client.list_replicas(dids=[])) is not None + + # rse + assert list(client.list_rses()) is not None + + # rule + assert list(client.list_replication_rules()) is not None