Skip to content

Commit

Permalink
Client: Run client without configuration file; Fix #6410
Browse files Browse the repository at this point in the history 
Changes:
            * Change `RuntimeError(Config Not Found)` to `ConfigNotFound`
              Error, config_get correctly handles those and returns a
default client config
            * Added typing for baseclient, initialization broken into
              functions to avoid typing conflicts for optionals.
            * Aded test to run multiple client commands without
              configuration file in place, include func dectorator for extended testing
  • Loading branch information
@voetberg
voetberg committed Feb 29, 2024
1 parent 788a0d4 commit 2482b12
Show file tree
Hide file tree
Showing 5 changed files with 240 additions and 141 deletions.
245 changes: 133 additions & 112 deletions lib/rucio/client/baseclient.py
View file
Expand Up @@ -27,6 +27,7 @@
from os import environ, fdopen, path, makedirs, geteuid
from shutil import move
from tempfile import mkstemp
from typing import Any, Callable, Optional
from urllib.parse import urlparse

from dogpile.cache import make_region
Expand All @@ -39,7 +40,8 @@
from rucio.common.config import config_get, config_get_bool, config_get_int
from rucio.common.exception import (CannotAuthenticate, ClientProtocolNotSupported,
NoAuthInformation, MissingClientParameter,
MissingModuleException, ServerConnectionException)
MissingModuleException, ServerConnectionException,
ConfigNotFound)
from rucio.common.extra import import_extras
from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, ssh_sign, setup_logger

Expand Down Expand Up @@ -79,7 +81,17 @@ class BaseClient(object):
TOKEN_PREFIX = 'auth_token_'
TOKEN_EXP_PREFIX = 'auth_token_exp_'

def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, auth_type=None, creds=None, timeout=600, user_agent='rucio-clients', vo=None, logger=None):
def __init__(self,
rucio_host: Optional[str] = None,
auth_host: Optional[str] = None,
account: Optional[str] = None,
ca_cert: Optional[str] = None,
auth_type: Optional[str] = None,
creds: Optional[dict[str, Any]] = None,
timeout: Optional[int] = 600,
user_agent: Optional[str] = 'rucio-clients',
vo: Optional[str] = None,
logger: Callable = LOG): # type: ignore
"""
Constructor of the BaseClient.
:param rucio_host: The address of the rucio server, if None it is read from the config file.
Expand All @@ -97,9 +109,8 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None,
"""

self.host = rucio_host
self.list_hosts = []
self.auth_host = auth_host
self.logger = logger or LOG
self.logger = logger
self.session = Session()
self.user_agent = "%s/%s" % (user_agent, version.version_string()) # e.g. "rucio-clients/0.2.13"
sys.argv[0] = sys.argv[0].split('/')[-1]
Expand All @@ -116,110 +127,26 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None,

try:
self.trace_host = config_get('trace', 'trace_host')
except (NoOptionError, NoSectionError):
except (NoOptionError, NoSectionError, ConfigNotFound):
self.trace_host = self.host
self.logger.debug('No trace_host passed. Using rucio_host instead')

self.list_hosts = [self.host]
self.account = account
self.vo = vo
self.ca_cert = ca_cert
self.auth_type = auth_type
self.creds = creds
self.auth_token = None
self.auth_token_file_path = config_get('client', 'auth_token_file_path', False, None)
self.auth_token = ""
self.headers = {}
self.timeout = timeout
self.request_retries = self.REQUEST_RETRIES
self.token_exp_epoch = None
self.token_exp_epoch_file = None
self.auth_oidc_refresh_active = config_get_bool('client', 'auth_oidc_refresh_active', False, False)

# defining how many minutes before token expires, oidc refresh (if active) should start
self.auth_oidc_refresh_before_exp = config_get_int('client', 'auth_oidc_refresh_before_exp', False, 20)

if auth_type is None:
self.logger.debug('No auth_type passed. Trying to get it from the environment variable RUCIO_AUTH_TYPE and config file.')
if 'RUCIO_AUTH_TYPE' in environ:
if environ['RUCIO_AUTH_TYPE'] not in ['userpass', 'x509', 'x509_proxy', 'gss', 'ssh', 'saml', 'oidc']:
raise MissingClientParameter('Possible RUCIO_AUTH_TYPE values: userpass, x509, x509_proxy, gss, ssh, saml, oidc, vs. ' + environ['RUCIO_AUTH_TYPE'])
self.auth_type = environ['RUCIO_AUTH_TYPE']
else:
try:
self.auth_type = config_get('client', 'auth_type')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])

if self.auth_type == 'oidc':
if not self.creds:
self.creds = {}
# if there are defautl values, check if rucio.cfg does not specify them, otherwise put default
if 'oidc_refresh_lifetime' not in self.creds or self.creds['oidc_refresh_lifetime'] is None:
self.creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None)
if 'oidc_issuer' not in self.creds or self.creds['oidc_issuer'] is None:
self.creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None)
if 'oidc_audience' not in self.creds or self.creds['oidc_audience'] is None:
self.creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None)
if 'oidc_auto' not in self.creds or self.creds['oidc_auto'] is False:
self.creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False)
if self.creds['oidc_auto']:
if 'oidc_username' not in self.creds or self.creds['oidc_username'] is None:
self.creds['oidc_username'] = config_get('client', 'oidc_username', False, None)
if 'oidc_password' not in self.creds or self.creds['oidc_password'] is None:
self.creds['oidc_password'] = config_get('client', 'oidc_password', False, None)
if 'oidc_scope' not in self.creds or self.creds['oidc_scope'] == 'openid profile':
self.creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile')
if 'oidc_polling' not in self.creds or self.creds['oidc_polling'] is False:
self.creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False)

if not self.creds:
self.logger.debug('No creds passed. Trying to get it from the config file.')
self.creds = {}
try:
if self.auth_type in ['userpass', 'saml']:
self.creds['username'] = config_get('client', 'username')
self.creds['password'] = config_get('client', 'password')
elif self.auth_type == 'x509':
if "RUCIO_CLIENT_CERT" in environ:
client_cert = environ["RUCIO_CLIENT_CERT"]
else:
client_cert = config_get('client', 'client_cert')
self.creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(client_cert)))
if not path.exists(self.creds['client_cert']):
raise MissingClientParameter('X.509 client certificate not found: %s' % self.creds['client_cert'])

if "RUCIO_CLIENT_KEY" in environ:
client_key = environ["RUCIO_CLIENT_KEY"]
else:
client_key = config_get('client', 'client_key')
self.creds['client_key'] = path.abspath(path.expanduser(path.expandvars(client_key)))
if not path.exists(self.creds['client_key']):
raise MissingClientParameter('X.509 client key not found: %s' % self.creds['client_key'])
else:
perms = oct(os.stat(self.creds['client_key']).st_mode)[-3:]
if perms not in ['400', '600']:
raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (self.creds['client_key'], perms))

elif self.auth_type == 'x509_proxy':
try:
self.creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
except NoOptionError:
# Recreate the classic GSI logic for locating the proxy:
# - $X509_USER_PROXY, if it is set.
# - /tmp/x509up_u`id -u` otherwise.
# If neither exists (at this point, we don't care if it exists but is invalid), then rethrow
if 'X509_USER_PROXY' in environ:
self.creds['client_proxy'] = environ['X509_USER_PROXY']
else:
fname = '/tmp/x509up_u%d' % geteuid()
if path.exists(fname):
self.creds['client_proxy'] = fname
else:
raise MissingClientParameter('Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
'\'x509_proxy\' not set in the configuration file.' % fname)
elif self.auth_type == 'ssh':
self.creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key'))))
except (NoOptionError, NoSectionError) as error:
if error.args[0] != 'client_key':
raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
self.auth_type = self._get_auth_type(auth_type)
self.creds = self._get_creds(creds)

rucio_scheme = urlparse(self.host).scheme
auth_scheme = urlparse(self.auth_host).scheme
Expand All @@ -241,8 +168,6 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None,
self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).')
self.ca_cert = True

self.list_hosts = [self.host]

if account is None:
self.logger.debug('No account passed. Trying to get it from the RUCIO_ACCOUNT environment variable or the config file.')
try:
Expand All @@ -265,28 +190,124 @@ def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None,
self.logger.debug('No VO found. Using default VO.')
self.vo = 'def'

token_filename_suffix = "for_default_account" if self.account is None else "for_account_" + self.account
self.auth_token_file_path, self.token_exp_epoch_file, self.token_file, self.token_path = self._get_auth_tokens()
self.__authenticate()

try:
self.request_retries = config_get_int('client', 'request_retries')
except (NoOptionError, ConfigNotFound):
self.logger.debug('request_retries not specified in config file. Taking default.')
except ValueError:
self.logger.debug('request_retries must be an integer. Taking default.')

def _get_auth_tokens(self) -> tuple[Optional[str], str, str, str]:
# if token file path is defined in the rucio.cfg file, use that file. Currently this prevents authenticating as another user or VO.
if self.auth_token_file_path:
self.token_file = self.auth_token_file_path
self.token_path = '/'.join(self.token_file.split('/')[:-1])
auth_token_file_path = config_get('client', 'auth_token_file_path', False, None)
token_filename_suffix = "for_default_account" if self.account is None else "for_account_" + self.account

if auth_token_file_path:
token_file = auth_token_file_path
token_path = '/'.join(auth_token_file_path.split('/')[:-1])

else:
self.token_path = self.TOKEN_PATH_PREFIX + getpass.getuser()
token_path = self.TOKEN_PATH_PREFIX + getpass.getuser()
if self.vo != 'def':
self.token_path += '@%s' % self.vo
self.token_file = self.token_path + '/' + self.TOKEN_PREFIX + token_filename_suffix
token_path += '@%s' % self.vo

self.token_exp_epoch_file = self.token_path + '/' + self.TOKEN_EXP_PREFIX + token_filename_suffix
token_file = token_path + '/' + self.TOKEN_PREFIX + token_filename_suffix

self.__authenticate()
token_exp_epoch_file = token_path + '/' + self.TOKEN_EXP_PREFIX + token_filename_suffix
return auth_token_file_path, token_exp_epoch_file, token_file, token_path

try:
self.request_retries = config_get_int('client', 'request_retries')
except (NoOptionError, RuntimeError):
LOG.debug('request_retries not specified in config file. Taking default.')
except ValueError:
self.logger.debug('request_retries must be an integer. Taking default.')
def _get_auth_type(self, auth_type: Optional[str]) -> str:
if auth_type is None:
self.logger.debug('No auth_type passed. Trying to get it from the environment variable RUCIO_AUTH_TYPE and config file.')
if 'RUCIO_AUTH_TYPE' in environ:
if environ['RUCIO_AUTH_TYPE'] not in ['userpass', 'x509', 'x509_proxy', 'gss', 'ssh', 'saml', 'oidc']:
raise MissingClientParameter('Possible RUCIO_AUTH_TYPE values: userpass, x509, x509_proxy, gss, ssh, saml, oidc, vs. ' + environ['RUCIO_AUTH_TYPE'])
auth_type = environ['RUCIO_AUTH_TYPE']
else:
try:
auth_type = config_get('client', 'auth_type')
except (NoOptionError, NoSectionError) as error:
raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
return auth_type

def _get_creds(self, creds: Optional[dict[str, Any]]) -> dict[str, Any]:
if self.auth_type == 'oidc':
if not creds:
creds = {}
# if there are defautl values, check if rucio.cfg does not specify them, otherwise put default
if 'oidc_refresh_lifetime' not in creds or creds['oidc_refresh_lifetime'] is None:
creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None)
if 'oidc_issuer' not in creds or creds['oidc_issuer'] is None:
creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None)
if 'oidc_audience' not in creds or creds['oidc_audience'] is None:
creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None)
if 'oidc_auto' not in creds or creds['oidc_auto'] is False:
creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False)
if creds['oidc_auto']:
if 'oidc_username' not in creds or creds['oidc_username'] is None:
creds['oidc_username'] = config_get('client', 'oidc_username', False, None)
if 'oidc_password' not in creds or creds['oidc_password'] is None:
creds['oidc_password'] = config_get('client', 'oidc_password', False, None)
if 'oidc_scope' not in creds or creds['oidc_scope'] == 'openid profile':
creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile')
if 'oidc_polling' not in creds or creds['oidc_polling'] is False:
creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False)

if creds is None:
self.logger.debug('No creds passed. Trying to get it from the config file.')
creds = {}
try:
if self.auth_type in ['userpass', 'saml']:
creds['username'] = config_get('client', 'username')
creds['password'] = config_get('client', 'password')
elif self.auth_type == 'x509':
if "RUCIO_CLIENT_CERT" in environ:
client_cert = environ["RUCIO_CLIENT_CERT"]
else:
client_cert = config_get('client', 'client_cert')
creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(client_cert)))
if not path.exists(creds['client_cert']):
raise MissingClientParameter('X.509 client certificate not found: %s' % creds['client_cert'])

if "RUCIO_CLIENT_KEY" in environ:
client_key = environ["RUCIO_CLIENT_KEY"]
else:
client_key = config_get('client', 'client_key')
creds['client_key'] = path.abspath(path.expanduser(path.expandvars(client_key)))
if not path.exists(creds['client_key']):
raise MissingClientParameter('X.509 client key not found: %s' % creds['client_key'])
else:
perms = oct(os.stat(creds['client_key']).st_mode)[-3:]
if perms not in ['400', '600']:
raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms))

elif self.auth_type == 'x509_proxy':
try:
creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
except NoOptionError:
# Recreate the classic GSI logic for locating the proxy:
# - $X509_USER_PROXY, if it is set.
# - /tmp/x509up_u`id -u` otherwise.
# If neither exists (at this point, we don't care if it exists but is invalid), then rethrow
if 'X509_USER_PROXY' in environ:
creds['client_proxy'] = environ['X509_USER_PROXY']
else:
fname = '/tmp/x509up_u%d' % geteuid()
if path.exists(fname):
creds['client_proxy'] = fname
else:
raise MissingClientParameter(
'Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
'\'x509_proxy\' not set in the configuration file.' % fname)
elif self.auth_type == 'ssh':
creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key'))))
except (NoOptionError, NoSectionError) as error:
if error.args[0] != 'client_key':
raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
return creds

def _get_exception(self, headers, status_code=None, data=None):
"""
Expand Down Expand Up @@ -667,7 +688,7 @@ def __get_token_x509(self):
url = build_url(self.auth_host, path='auth/x509_proxy')
client_cert = self.creds['client_proxy']

if not path.exists(client_cert):
if (client_cert is not None) and not (path.exists(client_cert)):
self.logger.error('given client cert (%s) doesn\'t exist' % client_cert)
return False
if client_key is not None and not path.exists(client_key):
Expand Down

0 comments on commit 2482b12

Please sign in to comment.