From b2e5150f191c04acb47ad98cef88512451aff81d Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Fri, 15 Apr 2022 10:42:12 -0700 Subject: [PATCH] feat: add AbortIncompleteMultipartUpload lifecycle rule (#765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #753 🦕 --- google/cloud/storage/bucket.py | 58 +++++++++++++++++++++++++++--- tests/system/_helpers.py | 2 +- tests/system/test_bucket.py | 17 +++++++++ tests/unit/test_bucket.py | 65 +++++++++++++++++++++++++++++++++- 4 files changed, 136 insertions(+), 6 deletions(-) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 85c9302f7..be99ad141 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -323,7 +323,7 @@ class LifecycleRuleDelete(dict): def __init__(self, **kw): conditions = LifecycleRuleConditions(**kw) rule = {"action": {"type": "Delete"}, "condition": dict(conditions)} - super(LifecycleRuleDelete, self).__init__(rule) + super().__init__(rule) @classmethod def from_api_repr(cls, resource): @@ -356,7 +356,7 @@ def __init__(self, storage_class, **kw): "action": {"type": "SetStorageClass", "storageClass": storage_class}, "condition": dict(conditions), } - super(LifecycleRuleSetStorageClass, self).__init__(rule) + super().__init__(rule) @classmethod def from_api_repr(cls, resource): @@ -365,7 +365,7 @@ def from_api_repr(cls, resource): :type resource: dict :param resource: mapping as returned from API call. - :rtype: :class:`LifecycleRuleDelete` + :rtype: :class:`LifecycleRuleSetStorageClass` :returns: Instance created from resource. """ action = resource["action"] @@ -374,6 +374,38 @@ def from_api_repr(cls, resource): return instance +class LifecycleRuleAbortIncompleteMultipartUpload(dict): + """Map a rule aborting incomplete multipart uploads of matching items. + + The "age" lifecycle condition is the only supported condition for this rule. + + :type kw: dict + :params kw: arguments passed to :class:`LifecycleRuleConditions`. + """ + + def __init__(self, **kw): + conditions = LifecycleRuleConditions(**kw) + rule = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": dict(conditions), + } + super().__init__(rule) + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct instance from resource. + + :type resource: dict + :param resource: mapping as returned from API call. + + :rtype: :class:`LifecycleRuleAbortIncompleteMultipartUpload` + :returns: Instance created from resource. + """ + instance = cls(_factory=True) + instance.update(resource) + return instance + + _default = object() @@ -2240,6 +2272,8 @@ def lifecycle_rules(self): yield LifecycleRuleDelete.from_api_repr(rule) elif action_type == "SetStorageClass": yield LifecycleRuleSetStorageClass.from_api_repr(rule) + elif action_type == "AbortIncompleteMultipartUpload": + yield LifecycleRuleAbortIncompleteMultipartUpload.from_api_repr(rule) else: warnings.warn( "Unknown lifecycle rule type received: {}. Please upgrade to the latest version of google-cloud-storage.".format( @@ -2289,7 +2323,7 @@ def add_lifecycle_delete_rule(self, **kw): self.lifecycle_rules = rules def add_lifecycle_set_storage_class_rule(self, storage_class, **kw): - """Add a "delete" rule to lifestyle rules configured for this bucket. + """Add a "set storage class" rule to lifestyle rules. See https://cloud.google.com/storage/docs/lifecycle and https://cloud.google.com/storage/docs/json_api/v1/buckets @@ -2309,6 +2343,22 @@ def add_lifecycle_set_storage_class_rule(self, storage_class, **kw): rules.append(LifecycleRuleSetStorageClass(storage_class, **kw)) self.lifecycle_rules = rules + def add_lifecycle_abort_incomplete_multipart_upload_rule(self, **kw): + """Add a "abort incomplete multipart upload" rule to lifestyle rules. + + Note that the "age" lifecycle condition is the only supported condition + for this rule. + + See https://cloud.google.com/storage/docs/lifecycle and + https://cloud.google.com/storage/docs/json_api/v1/buckets + + :type kw: dict + :params kw: arguments passed to :class:`LifecycleRuleConditions`. + """ + rules = list(self.lifecycle_rules) + rules.append(LifecycleRuleAbortIncompleteMultipartUpload(**kw)) + self.lifecycle_rules = rules + _location = _scalar_property("location") @property diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py index c172129d6..70c1f2a5d 100644 --- a/tests/system/_helpers.py +++ b/tests/system/_helpers.py @@ -23,7 +23,7 @@ retry_429 = RetryErrors(exceptions.TooManyRequests) retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10) retry_429_503 = RetryErrors( - [exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=10 + (exceptions.TooManyRequests, exceptions.ServiceUnavailable), max_tries=10 ) retry_failures = RetryErrors(AssertionError) diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 4826ce8a6..de1a04aa9 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -42,6 +42,7 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): from google.cloud.storage import constants from google.cloud.storage.bucket import LifecycleRuleDelete from google.cloud.storage.bucket import LifecycleRuleSetStorageClass + from google.cloud.storage.bucket import LifecycleRuleAbortIncompleteMultipartUpload bucket_name = _helpers.unique_name("w-lifcycle-rules") custom_time_before = datetime.date(2018, 8, 1) @@ -64,6 +65,9 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): is_live=False, matches_storage_class=[constants.NEARLINE_STORAGE_CLASS], ) + bucket.add_lifecycle_abort_incomplete_multipart_upload_rule( + age=42, + ) expected_rules = [ LifecycleRuleDelete( @@ -79,6 +83,9 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): is_live=False, matches_storage_class=[constants.NEARLINE_STORAGE_CLASS], ), + LifecycleRuleAbortIncompleteMultipartUpload( + age=42, + ), ] _helpers.retry_429_503(bucket.create)(location="us") @@ -87,6 +94,16 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): assert bucket.name == bucket_name assert list(bucket.lifecycle_rules) == expected_rules + # Test modifying lifecycle rules + expected_rules[0] = LifecycleRuleDelete(age=30) + rules = list(bucket.lifecycle_rules) + rules[0]["condition"] = {"age": 30} + bucket.lifecycle_rules = rules + bucket.patch() + + assert list(bucket.lifecycle_rules) == expected_rules + + # Test clearing lifecycle rules bucket.clear_lifecyle_rules() bucket.patch() diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index c5f1df5d2..eb402de9e 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -337,6 +337,43 @@ def test_from_api_repr(self): self.assertEqual(dict(rule), resource) +class Test_LifecycleRuleAbortIncompleteMultipartUpload(unittest.TestCase): + @staticmethod + def _get_target_class(): + from google.cloud.storage.bucket import ( + LifecycleRuleAbortIncompleteMultipartUpload, + ) + + return LifecycleRuleAbortIncompleteMultipartUpload + + def _make_one(self, **kw): + return self._get_target_class()(**kw) + + def test_ctor_wo_conditions(self): + with self.assertRaises(ValueError): + self._make_one() + + def test_ctor_w_condition(self): + rule = self._make_one(age=10) + expected = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": {"age": 10}, + } + self.assertEqual(dict(rule), expected) + + def test_from_api_repr(self): + klass = self._get_target_class() + conditions = { + "age": 10, + } + resource = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": conditions, + } + rule = klass.from_api_repr(resource) + self.assertEqual(dict(rule), resource) + + class Test_IAMConfiguration(unittest.TestCase): @staticmethod def _get_target_class(): @@ -2242,6 +2279,7 @@ def test_lifecycle_rules_getter(self): from google.cloud.storage.bucket import ( LifecycleRuleDelete, LifecycleRuleSetStorageClass, + LifecycleRuleAbortIncompleteMultipartUpload, ) NAME = "name" @@ -2250,7 +2288,11 @@ def test_lifecycle_rules_getter(self): "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"}, "condition": {"isLive": False}, } - rules = [DELETE_RULE, SSC_RULE] + MULTIPART_RULE = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": {"age": 42}, + } + rules = [DELETE_RULE, SSC_RULE, MULTIPART_RULE] properties = {"lifecycle": {"rule": rules}} bucket = self._make_one(name=NAME, properties=properties) @@ -2264,6 +2306,12 @@ def test_lifecycle_rules_getter(self): self.assertIsInstance(ssc_rule, LifecycleRuleSetStorageClass) self.assertEqual(dict(ssc_rule), SSC_RULE) + multipart_rule = found[2] + self.assertIsInstance( + multipart_rule, LifecycleRuleAbortIncompleteMultipartUpload + ) + self.assertEqual(dict(multipart_rule), MULTIPART_RULE) + def test_lifecycle_rules_setter_w_dicts(self): NAME = "name" DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}} @@ -2348,6 +2396,21 @@ def test_add_lifecycle_set_storage_class_rule(self): self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) self.assertTrue("lifecycle" in bucket._changes) + def test_add_lifecycle_abort_incomplete_multipart_upload_rule(self): + NAME = "name" + AIMPU_RULE = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": {"age": 42}, + } + rules = [AIMPU_RULE] + bucket = self._make_one(name=NAME) + self.assertEqual(list(bucket.lifecycle_rules), []) + + bucket.add_lifecycle_abort_incomplete_multipart_upload_rule(age=42) + + self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) + self.assertTrue("lifecycle" in bucket._changes) + def test_cors_getter(self): NAME = "name" CORS_ENTRY = {