Metadata: Rename core.meta to core.meta_conventions Fix #5224

Changes:
	* Rename core.meta to core.meta_conventions
	* Rename DIDKey -> DIDMetaConventionsKey
	* Rename DIDKeyValue -> DIDMetaConventionsConstraints
	* Update docstrings to reflect naming, more descriptive
	* Typing on *.meta_conventions

lib/rucio/api/did.py

@@ -22,8 +22,9 @@
 from rucio.common.schema import validate_schema
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import api_update_return_dict
-from rucio.core import did, naming_convention, meta as meta_core
+from rucio.core import did, naming_convention
 from rucio.core.rse import get_rse_id
+from rucio.core import meta_conventions as meta_convention_core
 from rucio.db.sqla.constants import DIDType
 from rucio.db.sqla.session import read_session, stream_session, transactional_session
 
@@ -117,7 +118,7 @@ def add_did(scope, name, did_type, issuer, account=None, statuses={}, meta={}, r
                 raise rucio.common.exception.InvalidObject("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k]))
 
         # Validate metadata
-        meta_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session)
+        meta_convention_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session)
 
     return did.add_did(scope=scope, name=name, did_type=DIDType[did_type.upper()], account=account or issuer,
                        statuses=statuses, meta=meta, rules=rules, lifetime=lifetime,

lib/rucio/api/meta.py → lib/rucio/api/meta_conventions.py

@@ -13,47 +13,49 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Union, Optional
 
 from rucio.api.permission import has_permission
 from rucio.common.exception import AccessDenied
-from rucio.core import meta
+from rucio.core import meta_conventions
 from rucio.db.sqla.session import read_session, transactional_session
+from rucio.db.sqla.constants import KeyType
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
+    from rucio.common.types import InternalAccount
 
 
 @read_session
 def list_keys(*, session: "Session"):
     """
-    Lists all keys.
+    Lists all keys for DID Metadata Conventions.
 
     :param session: The database session in use.
 
     :returns: A list containing all keys.
     """
-    return meta.list_keys(session=session)
+    return meta_conventions.list_keys(session=session)
 
 
 @read_session
-def list_values(key, *, session: "Session"):
+def list_values(key: str, *, session: "Session"):
     """
-    Lists all values for a key.
+    Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
 
     :param key: the name for the key.
     :param session: The database session in use.
 
 
     :returns: A list containing all values.
     """
-    return meta.list_values(key=key, session=session)
+    return meta_conventions.list_values(key=key, session=session)
 
 
 @transactional_session
-def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def', *, session: "Session"):
+def add_key(key: str, key_type: Union[KeyType, str], issuer: "InternalAccount", value_type: Optional[str] = None, value_regexp: Optional[str] = None, vo: str = 'def', *, session: "Session"):
     """
-    Add a new allowed key.
+    Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
 
     :param key: the name for the new key.
     :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
@@ -66,13 +68,13 @@ def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def',
     kwargs = {'key': key, 'key_type': key_type, 'value_type': value_type, 'value_regexp': value_regexp}
     if not has_permission(issuer=issuer, vo=vo, action='add_key', kwargs=kwargs, session=session):
         raise AccessDenied('Account %s can not add key' % (issuer))
-    return meta.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session)
+    return meta_conventions.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session)
 
 
 @transactional_session
-def add_value(key, value, issuer, vo='def', *, session: "Session"):
+def add_value(key: str, value: str, issuer: "InternalAccount", vo: str = 'def', *, session: "Session"):
     """
-    Add a new value to a key.
+    Add an allowed value for DID metadata (update a key in DID Metadata Conventions table).
 
     :param key: the name for the key.
     :param value: the value.
@@ -82,4 +84,4 @@ def add_value(key, value, issuer, vo='def', *, session: "Session"):
     kwargs = {'key': key, 'value': value}
     if not has_permission(issuer=issuer, vo=vo, action='add_value', kwargs=kwargs, session=session):
         raise AccessDenied('Account %s can not add value %s to key %s' % (issuer, value, key))
-    return meta.add_value(key=key, value=value, session=session)
+    return meta_conventions.add_value(key=key, value=value, session=session)

lib/rucio/client/client.py

@@ -27,7 +27,7 @@
 from rucio.client.importclient import ImportClient
 from rucio.client.lifetimeclient import LifetimeClient
 from rucio.client.lockclient import LockClient
-from rucio.client.metaclient import MetaClient
+from rucio.client.metaconventionsclient import MetaConventionClient
 from rucio.client.pingclient import PingClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.client.requestclient import RequestClient
@@ -40,7 +40,7 @@
 
 class Client(AccountClient,
              AccountLimitClient,
-             MetaClient,
+             MetaConventionClient,
              PingClient,
              ReplicaClient,
              RequestClient,

lib/rucio/client/metaclient.py → lib/rucio/client/metaconventionsclient.py

@@ -21,17 +21,19 @@
 from rucio.client.baseclient import BaseClient
 from rucio.client.baseclient import choice
 from rucio.common.utils import build_url
+from rucio.db.sqla.constants import KeyType
+from typing import Union, Optional
 
 
-class MetaClient(BaseClient):
+class MetaConventionClient(BaseClient):
 
-    """Meta client class for working with data identifier attributes"""
+    """Metadata client class for working with data identifier attributes"""
 
-    META_BASEURL = 'meta'
+    META_BASEURL = 'meta_conventions'
 
-    def add_key(self, key, key_type, value_type=None, value_regexp=None):
+    def add_key(self, key: str, key_type: Union[KeyType, str], value_type: Optional[str] = None, value_regexp: Optional[str] = None) -> Optional[bool]:
         """
-        Sends the request to add a new key.
+        Sends the request to add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
 
         :param key: the name for the new key.
         :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
@@ -56,9 +58,9 @@ def add_key(self, key, key_type, value_type=None, value_regexp=None):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def list_keys(self):
+    def list_keys(self) -> Optional[list[str]]:
         """
-        Sends the request to list all keys.
+        Sends the request to list all keys for DID Metadata Conventions.
 
         :return: a list containing the names of all keys.
         """
@@ -72,9 +74,10 @@ def list_keys(self):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def list_values(self, key):
+    def list_values(self, key: str) -> Optional[list[str]]:
         """
-        Sends the request to list all values for a key.
+        Sends the request to lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
+.
 
         :return: a list containing the names of all values for a key.
         """
@@ -88,9 +91,9 @@ def list_values(self, key):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def add_value(self, key, value):
+    def add_value(self, key: str, value: str) -> Optional[bool]:
         """
-        Sends the request to add a value to a key.
+        Sends the request to add a value for a key in DID Metadata Convention.
 
         :param key: the name for key.
         :param value: the value.
@@ -111,7 +114,7 @@ def add_value(self, key, value):
 
     def del_value(self, key, value):
         """
-        Delete a value for a key.
+        Delete a key in the DID Metadata Conventions table.
 
         :param key: the name for key.
         :param value: the value.

lib/rucio/core/meta.py → lib/rucio/core/meta_conventions.py

@@ -14,10 +14,9 @@
 # limitations under the License.
 
 from re import match
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional, Union
 
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.exc import IntegrityError, NoResultFound
 
 from rucio.common.constraints import AUTHORIZED_VALUE_TYPES
 from rucio.common.exception import (Duplicate, RucioException,
@@ -32,9 +31,9 @@
 
 
 @transactional_session
-def add_key(key, key_type, value_type=None, value_regexp=None, *, session: "Session"):
+def add_key(key: str, key_type: Union[KeyType, str], value_type: Optional[str] = None, value_regexp: Optional[str] = None, *, session: "Session") -> None:
     """
-    Adds a new allowed key.
+    Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
 
     :param key: the name for the new key.
     :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
@@ -65,7 +64,7 @@ def add_key(key, key_type, value_type=None, value_regexp=None, *, session: "Sess
     except ValueError:
         raise UnsupportedKeyType('The type \'%s\' is not supported for keys!' % str(key_type))
 
-    new_key = models.DIDKey(key=key, value_type=value_type and str(value_type), value_regexp=value_regexp, key_type=key_type)
+    new_key = models.DIDMetaConventionsKey(key=key, value_type=value_type and str(value_type), value_regexp=value_regexp, key_type=key_type)
     try:
         new_key.save(session=session)
     except IntegrityError as error:
@@ -77,46 +76,50 @@ def add_key(key, key_type, value_type=None, value_regexp=None, *, session: "Sess
            or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
            or match('.*IntegrityError.*columns? key.*not unique.*', error.args[0]):
             raise Duplicate(f"key '{key}' already exists!")
-        raise
+        raise RucioException(error.args)
 
 
 @transactional_session
-def del_key(key, *, session: "Session"):
+def del_key(key: str, *, session: "Session") -> None:
     """
-    Deletes a key.
+    Delete a key in the DID Metadata Conventions table.
 
     :param key: the name for the key.
     :param session: The database session in use.
     """
-    session.query(models.DIDKey).filter(key == key).delete()
+    session.query(models.DIDMetaConventionsKey).filter(key == key).delete()
 
 
 @read_session
-def list_keys(*, session: "Session"):
+def list_keys(*, session: "Session") -> list[str]:
     """
-    Lists all keys.
+    Lists all keys for DID Metadata Conventions.
 
     :param session: The database session in use.
 
     :returns: A list containing all keys.
     """
     key_list = []
-    query = session.query(models.DIDKey)
+    query = session.query(models.DIDMetaConventionsKey)
     for row in query:
         key_list.append(row.key)
     return key_list
 
 
 @transactional_session
-def add_value(key, value, *, session: "Session"):
+def add_value(key: str, value: str, *, session: "Session") -> None:
     """
-    Adds a new value to a key.
+    Adds a new value for a key in DID Metadata Convention.
 
     :param key: the name for the key.
     :param value: the value.
     :param session: The database session in use.
+
+    :raises Duplicate: Key-Value pair exists
+    :raises KeyNotFound: Key not in metadata conventions table
+    :raises InvalidValueForKey: Value conflicts with rse expression for key values or does not have the correct type
     """
-    new_value = models.DIDKeyValueAssociation(key=key, value=value)
+    new_value = models.DIDMetaConventionsConstraints(key=key, value=value)
     try:
         new_value.save(session=session)
     except IntegrityError as error:
@@ -137,51 +140,51 @@ def add_value(key, value, *, session: "Session"):
 
         raise RucioException(error.args)
 
-    k = session.query(models.DIDKey).filter_by(key=key).one()
+    k = session.query(models.DIDMetaConventionsKey).filter_by(key=key).one()
 
     # Check value against regexp, if defined
     if k.value_regexp and not match(k.value_regexp, value):
         raise InvalidValueForKey("The value '%s' for the key '%s' does not match the regular expression '%s'" % (value, key, k.value_regexp))
 
     # Check value type, if defined
     type_map = dict([(str(t), t) for t in AUTHORIZED_VALUE_TYPES])
-    if k.value_type and not isinstance(value, type_map.get(k.value_type)):
+    if k.value_type and not isinstance(value, type_map.get(k.value_type)):  # type: ignore ; Typing error caused by 'isinstaince' not thinking types count as classes
         raise InvalidValueForKey("The value '%s' for the key '%s' does not match the required type '%s'" % (value, key, k.value_type))
 
 
 @read_session
-def list_values(key, *, session: "Session"):
+def list_values(key: str, *, session: "Session") -> list[str]:
     """
-    Lists all values for a key.
+    Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
 
     :param key: the name for the key.
     :param session: The database session in use.
 
     :returns: A list containing all values.
     """
     value_list = []
-    query = session.query(models.DIDKeyValueAssociation).filter_by(key=key)
+    query = session.query(models.DIDMetaConventionsConstraints).filter_by(key=key)
     for row in query:
         value_list.append(row.value)
     return value_list
 
 
 @read_session
-def validate_meta(meta, did_type, *, session: "Session"):
+def validate_meta(meta: dict, did_type: DIDType, *, session: "Session") -> None:
     """
     Validates metadata for a did.
 
     :param meta: the dictionary of metadata.
     :param meta: the type of the did, e.g, DATASET, CONTAINER, FILE.
     :param session: The database session in use.
 
-    :returns: True
+    :raises InvalidObject:
     """
     # For now only validate the datatype for datasets
     key = 'datatype'
     if did_type == DIDType.DATASET and key in meta:
         try:
-            session.query(models.DIDKeyValueAssociation.value).\
+            session.query(models.DIDMetaConventionsConstraints.value).\
                 filter_by(key=key).\
                 filter_by(value=meta[key]).\
                 one()

lib/rucio/db/sqla/models.py

@@ -605,8 +605,8 @@ class QuarantinedReplicaHistory(BASE, ModelBase):
     _table_args = ()
 
 
-class DIDKey(BASE, ModelBase):
-    """Represents Data IDentifier property keys"""
+class DIDMetaConventionsKey(BASE, ModelBase):
+    """Represents allowed keys of DID Metadata"""
     __tablename__ = 'did_keys'
     key: Mapped[str] = mapped_column(String(255))
     is_enum: Mapped[bool] = mapped_column(Boolean(name='DID_KEYS_IS_ENUM_CHK', create_constraint=True),
@@ -621,8 +621,8 @@ class DIDKey(BASE, ModelBase):
                    CheckConstraint('is_enum IS NOT NULL', name='DID_KEYS_IS_ENUM_NN'))
 
 
-class DIDKeyValueAssociation(BASE, ModelBase):
-    """Represents Data IDentifier property key/values"""
+class DIDMetaConventionsConstraints(BASE, ModelBase):
+    """Represents a map for constraint values a DID metadata key must follow """
     __tablename__ = 'did_key_map'
     key: Mapped[str] = mapped_column(String(255))
     value: Mapped[str] = mapped_column(String(255))
@@ -1696,8 +1696,8 @@ def register_models(engine):
               ConstituentAssociationHistory,
               DataIdentifierAssociation,
               DataIdentifierAssociationHistory,
-              DIDKey,
-              DIDKeyValueAssociation,
+              DIDMetaConventionsKey,
+              DIDMetaConventionsConstraints,
               DataIdentifier,
               DidMeta,
               VirtualPlacements,
@@ -1766,8 +1766,8 @@ def unregister_models(engine):
               ConstituentAssociationHistory,
               DataIdentifierAssociation,
               DataIdentifierAssociationHistory,
-              DIDKey,
-              DIDKeyValueAssociation,
+              DIDMetaConventionsKey,
+              DIDMetaConventionsConstraints,
               DidMeta,
               DataIdentifier,
               DeletedDataIdentifier,