Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add AbortIncompleteMultipartUpload lifecycle rule #765

Merged
merged 6 commits into from
Apr 15, 2022
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
58 changes: 54 additions & 4 deletions google/cloud/storage/bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ class LifecycleRuleDelete(dict):
def __init__(self, **kw):
conditions = LifecycleRuleConditions(**kw)
rule = {"action": {"type": "Delete"}, "condition": dict(conditions)}
super(LifecycleRuleDelete, self).__init__(rule)
super().__init__(rule)

@classmethod
def from_api_repr(cls, resource):
Expand Down Expand Up @@ -356,7 +356,7 @@ def __init__(self, storage_class, **kw):
"action": {"type": "SetStorageClass", "storageClass": storage_class},
"condition": dict(conditions),
}
super(LifecycleRuleSetStorageClass, self).__init__(rule)
super().__init__(rule)

@classmethod
def from_api_repr(cls, resource):
Expand All @@ -365,7 +365,7 @@ def from_api_repr(cls, resource):
:type resource: dict
:param resource: mapping as returned from API call.

:rtype: :class:`LifecycleRuleDelete`
:rtype: :class:`LifecycleRuleSetStorageClass`
:returns: Instance created from resource.
"""
action = resource["action"]
Expand All @@ -374,6 +374,38 @@ def from_api_repr(cls, resource):
return instance


class LifecycleRuleAbortIncompleteMultipartUpload(dict):
"""Map a rule aborting incomplete multipart uploads of matching items.

The "age" lifecycle condition is the only supported condition for this rule.

:type kw: dict
:params kw: arguments passed to :class:`LifecycleRuleConditions`.
andrewsg marked this conversation as resolved.
Show resolved Hide resolved
"""

def __init__(self, **kw):
conditions = LifecycleRuleConditions(**kw)
rule = {
"action": {"type": "AbortIncompleteMultipartUpload"},
"condition": dict(conditions),
}
super().__init__(rule)

@classmethod
def from_api_repr(cls, resource):
"""Factory: construct instance from resource.

:type resource: dict
:param resource: mapping as returned from API call.

:rtype: :class:`LifecycleRuleAbortIncompleteMultipartUpload`
:returns: Instance created from resource.
"""
instance = cls(_factory=True)
instance.update(resource)
return instance


_default = object()


Expand Down Expand Up @@ -2240,6 +2272,8 @@ def lifecycle_rules(self):
yield LifecycleRuleDelete.from_api_repr(rule)
elif action_type == "SetStorageClass":
yield LifecycleRuleSetStorageClass.from_api_repr(rule)
elif action_type == "AbortIncompleteMultipartUpload":
yield LifecycleRuleAbortIncompleteMultipartUpload.from_api_repr(rule)
else:
warnings.warn(
"Unknown lifecycle rule type received: {}. Please upgrade to the latest version of google-cloud-storage.".format(
Expand Down Expand Up @@ -2289,7 +2323,7 @@ def add_lifecycle_delete_rule(self, **kw):
self.lifecycle_rules = rules

def add_lifecycle_set_storage_class_rule(self, storage_class, **kw):
"""Add a "delete" rule to lifestyle rules configured for this bucket.
"""Add a "set storage class" rule to lifestyle rules.

See https://cloud.google.com/storage/docs/lifecycle and
https://cloud.google.com/storage/docs/json_api/v1/buckets
Expand All @@ -2309,6 +2343,22 @@ def add_lifecycle_set_storage_class_rule(self, storage_class, **kw):
rules.append(LifecycleRuleSetStorageClass(storage_class, **kw))
self.lifecycle_rules = rules

def add_lifecycle_abort_incomplete_multipart_upload_rule(self, **kw):
"""Add a "abort incomplete multipart upload" rule to lifestyle rules.

Note that the "age" lifecycle condition is the only supported condition
for this rule.

See https://cloud.google.com/storage/docs/lifecycle and
https://cloud.google.com/storage/docs/json_api/v1/buckets

:type kw: dict
:params kw: arguments passed to :class:`LifecycleRuleConditions`.
"""
rules = list(self.lifecycle_rules)
rules.append(LifecycleRuleAbortIncompleteMultipartUpload(**kw))
self.lifecycle_rules = rules

_location = _scalar_property("location")

@property
Expand Down
2 changes: 1 addition & 1 deletion tests/system/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
retry_429 = RetryErrors(exceptions.TooManyRequests)
retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10)
retry_429_503 = RetryErrors(
[exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=10
(exceptions.TooManyRequests, exceptions.ServiceUnavailable), max_tries=10
)
retry_failures = RetryErrors(AssertionError)

Expand Down
19 changes: 19 additions & 0 deletions tests/system/test_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete):
from google.cloud.storage import constants
from google.cloud.storage.bucket import LifecycleRuleDelete
from google.cloud.storage.bucket import LifecycleRuleSetStorageClass
from google.cloud.storage.bucket import LifecycleRuleAbortIncompleteMultipartUpload

bucket_name = _helpers.unique_name("w-lifcycle-rules")
custom_time_before = datetime.date(2018, 8, 1)
Expand All @@ -64,6 +65,9 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete):
is_live=False,
matches_storage_class=[constants.NEARLINE_STORAGE_CLASS],
)
bucket.add_lifecycle_abort_incomplete_multipart_upload_rule(
age=42,
)

expected_rules = [
LifecycleRuleDelete(
Expand All @@ -79,6 +83,9 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete):
is_live=False,
matches_storage_class=[constants.NEARLINE_STORAGE_CLASS],
),
LifecycleRuleAbortIncompleteMultipartUpload(
age=42,
),
andrewsg marked this conversation as resolved.
Show resolved Hide resolved
]

_helpers.retry_429_503(bucket.create)(location="us")
Expand All @@ -87,9 +94,21 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete):
assert bucket.name == bucket_name
assert list(bucket.lifecycle_rules) == expected_rules

# Test modifying lifecycle rules
expected_rules[0] = LifecycleRuleDelete(age=30)
rules = list(bucket.lifecycle_rules)
rules[0]["condition"] = {"age": 30}
bucket.lifecycle_rules = rules
bucket.patch()

bucket.reload()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Curious do we need to reload the bucket here, same on L111? The patch call should return the bucket metadata? Otherwise, LGTM, thanks!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, you're right, we should be able to skip that. I'll remove them and confirm it still works.

assert list(bucket.lifecycle_rules) == expected_rules

# Test clearing lifecycle rules
bucket.clear_lifecyle_rules()
bucket.patch()

bucket.reload()
assert list(bucket.lifecycle_rules) == []


Expand Down
65 changes: 64 additions & 1 deletion tests/unit/test_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,43 @@ def test_from_api_repr(self):
self.assertEqual(dict(rule), resource)


class Test_LifecycleRuleAbortIncompleteMultipartUpload(unittest.TestCase):
@staticmethod
def _get_target_class():
from google.cloud.storage.bucket import (
LifecycleRuleAbortIncompleteMultipartUpload,
)

return LifecycleRuleAbortIncompleteMultipartUpload

def _make_one(self, **kw):
return self._get_target_class()(**kw)

def test_ctor_wo_conditions(self):
with self.assertRaises(ValueError):
self._make_one()

def test_ctor_w_condition(self):
rule = self._make_one(age=10)
expected = {
"action": {"type": "AbortIncompleteMultipartUpload"},
"condition": {"age": 10},
}
self.assertEqual(dict(rule), expected)

def test_from_api_repr(self):
klass = self._get_target_class()
conditions = {
"age": 10,
}
resource = {
"action": {"type": "AbortIncompleteMultipartUpload"},
"condition": conditions,
}
rule = klass.from_api_repr(resource)
self.assertEqual(dict(rule), resource)


class Test_IAMConfiguration(unittest.TestCase):
@staticmethod
def _get_target_class():
Expand Down Expand Up @@ -2242,6 +2279,7 @@ def test_lifecycle_rules_getter(self):
from google.cloud.storage.bucket import (
LifecycleRuleDelete,
LifecycleRuleSetStorageClass,
LifecycleRuleAbortIncompleteMultipartUpload,
)

NAME = "name"
Expand All @@ -2250,7 +2288,11 @@ def test_lifecycle_rules_getter(self):
"action": {"type": "SetStorageClass", "storageClass": "NEARLINE"},
"condition": {"isLive": False},
}
rules = [DELETE_RULE, SSC_RULE]
MULTIPART_RULE = {
"action": {"type": "AbortIncompleteMultipartUpload"},
"condition": {"age": 42},
}
rules = [DELETE_RULE, SSC_RULE, MULTIPART_RULE]
properties = {"lifecycle": {"rule": rules}}
bucket = self._make_one(name=NAME, properties=properties)

Expand All @@ -2264,6 +2306,12 @@ def test_lifecycle_rules_getter(self):
self.assertIsInstance(ssc_rule, LifecycleRuleSetStorageClass)
self.assertEqual(dict(ssc_rule), SSC_RULE)

multipart_rule = found[2]
self.assertIsInstance(
multipart_rule, LifecycleRuleAbortIncompleteMultipartUpload
)
self.assertEqual(dict(multipart_rule), MULTIPART_RULE)

def test_lifecycle_rules_setter_w_dicts(self):
NAME = "name"
DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}}
Expand Down Expand Up @@ -2348,6 +2396,21 @@ def test_add_lifecycle_set_storage_class_rule(self):
self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules)
self.assertTrue("lifecycle" in bucket._changes)

def test_add_lifecycle_abort_incomplete_multipart_upload_rule(self):
NAME = "name"
AIMPU_RULE = {
"action": {"type": "AbortIncompleteMultipartUpload"},
"condition": {"age": 42},
}
rules = [AIMPU_RULE]
bucket = self._make_one(name=NAME)
self.assertEqual(list(bucket.lifecycle_rules), [])

bucket.add_lifecycle_abort_incomplete_multipart_upload_rule(age=42)

self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules)
self.assertTrue("lifecycle" in bucket._changes)

def test_cors_getter(self):
NAME = "name"
CORS_ENTRY = {
Expand Down