feat: make tests run against environments other than prod (#883)

Made changes that override api endpoint information while triggering builds from kokoro .
All tests against prod are still successful.
Some system tests are currently incompatible with nonprod environments and hence are marked to be skipped if endpoint is being overriden.

.kokoro/build.sh

@@ -33,6 +33,16 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
 # Setup project id.
 export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
 
+# Export variable to override api endpoint
+export API_ENDPOINT_OVERRIDE
+
+# Export variable to override api endpoint version
+export API_VERSION_OVERRIDE
+
+# Export dual region locations
+export DUAL_REGION_LOC_1
+export DUAL_REGION_LOC_2
+
 # Remove old nox
 python3 -m pip uninstall --yes --quiet nox-automation
 

google/cloud/storage/_helpers.py

@@ -33,9 +33,14 @@
 STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
 """Environment variable defining host for Storage emulator."""
 
-_DEFAULT_STORAGE_HOST = "https://storage.googleapis.com"
+_DEFAULT_STORAGE_HOST = os.getenv(
+    "API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com"
+)
 """Default storage host for JSON API."""
 
+_API_VERSION = os.getenv("API_VERSION_OVERRIDE", "v1")
+"""API version of the default storage host"""
+
 _BASE_STORAGE_URI = "storage.googleapis.com"
 """Base request endpoint URI for JSON API."""
 

google/cloud/storage/_http.py

@@ -15,7 +15,6 @@
 """Create / interact with Google Cloud Storage connections."""
 
 import functools
-
 from google.cloud import _http
 from google.cloud.storage import __version__
 from google.cloud.storage import _helpers
@@ -35,7 +34,7 @@ class Connection(_http.JSONConnection):
     :param api_endpoint: (Optional) api endpoint to use.
     """
 
-    DEFAULT_API_ENDPOINT = "https://storage.googleapis.com"
+    DEFAULT_API_ENDPOINT = _helpers._DEFAULT_STORAGE_HOST
     DEFAULT_API_MTLS_ENDPOINT = "https://storage.mtls.googleapis.com"
 
     def __init__(self, client, client_info=None, api_endpoint=None):
@@ -52,7 +51,7 @@ def __init__(self, client, client_info=None, api_endpoint=None):
         if agent_version not in self._client_info.user_agent:
             self._client_info.user_agent += f" {agent_version} "
 
-    API_VERSION = "v1"
+    API_VERSION = _helpers._API_VERSION
     """The version of the API, used in building the API call's URL."""
 
     API_URL_TEMPLATE = "{api_base_url}/storage/{api_version}{path}"

google/cloud/storage/blob.py

@@ -60,6 +60,8 @@
 from google.cloud.storage._signing import generate_signed_url_v2
 from google.cloud.storage._signing import generate_signed_url_v4
 from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE
+from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
+from google.cloud.storage._helpers import _API_VERSION
 from google.cloud.storage.acl import ACL
 from google.cloud.storage.acl import ObjectACL
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
@@ -78,10 +80,12 @@
 from google.cloud.storage.fileio import BlobWriter
 
 
-_API_ACCESS_ENDPOINT = "https://storage.googleapis.com"
+_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST
 _DEFAULT_CONTENT_TYPE = "application/octet-stream"
-_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/v1{path}?alt=media"
-_BASE_UPLOAD_TEMPLATE = "{hostname}/upload/storage/v1{bucket_path}/o?uploadType="
+_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media"
+_BASE_UPLOAD_TEMPLATE = (
+    "{hostname}/upload/storage/{api_version}{bucket_path}/o?uploadType="
+)
 _MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "multipart"
 _RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "resumable"
 # NOTE: "acl" is also writeable but we defer ACL management to
@@ -823,7 +827,9 @@ def _get_download_url(
         name_value_pairs = []
         if self.media_link is None:
             hostname = _get_host_name(client._connection)
-            base_url = _DOWNLOAD_URL_TEMPLATE.format(hostname=hostname, path=self.path)
+            base_url = _DOWNLOAD_URL_TEMPLATE.format(
+                hostname=hostname, path=self.path, api_version=_API_VERSION
+            )
             if self.generation is not None:
                 name_value_pairs.append(("generation", f"{self.generation:d}"))
         else:
@@ -1838,7 +1844,7 @@ def _do_multipart_upload(
 
         hostname = _get_host_name(client._connection)
         base_url = _MULTIPART_URL_TEMPLATE.format(
-            hostname=hostname, bucket_path=self.bucket.path
+            hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
         )
         name_value_pairs = []
 
@@ -2025,7 +2031,7 @@ def _initiate_resumable_upload(
 
         hostname = _get_host_name(client._connection)
         base_url = _RESUMABLE_URL_TEMPLATE.format(
-            hostname=hostname, bucket_path=self.bucket.path
+            hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
         )
         name_value_pairs = []
 

noxfile.py

@@ -111,6 +111,7 @@ def system(session):
     """Run the system test suite."""
     system_test_path = os.path.join("tests", "system.py")
     system_test_folder_path = os.path.join("tests", "system")
+    rerun_count = 0
 
     # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
     if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false":
@@ -121,6 +122,12 @@ def system(session):
     # mTLS tests requires pyopenssl.
     if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true":
         session.install("pyopenssl")
+    # Check if endpoint is being overriden for rerun_count
+    if (
+        os.getenv("API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com")
+        != "https://storage.googleapis.com"
+    ):
+        rerun_count = 3
 
     system_test_exists = os.path.exists(system_test_path)
     system_test_folder_exists = os.path.exists(system_test_folder_path)
@@ -138,7 +145,7 @@ def system(session):
     # 2021-05-06: defer installing 'google-cloud-*' to after this package,
     #             in order to work around Python 2.7 googolapis-common-protos
     #             issue.
-    session.install("mock", "pytest", "-c", constraints_path)
+    session.install("mock", "pytest", "pytest-rerunfailures", "-c", constraints_path)
     session.install("-e", ".", "-c", constraints_path)
     session.install(
         "google-cloud-testutils",
@@ -151,9 +158,21 @@ def system(session):
 
     # Run py.test against the system tests.
     if system_test_exists:
-        session.run("py.test", "--quiet", system_test_path, *session.posargs)
+        session.run(
+            "py.test",
+            "--quiet",
+            "--reruns={}".format(rerun_count),
+            system_test_path,
+            *session.posargs,
+        )
     if system_test_folder_exists:
-        session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)
+        session.run(
+            "py.test",
+            "--quiet",
+            "--reruns={}".format(rerun_count),
+            system_test_folder_path,
+            *session.posargs,
+        )
 
 
 @nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS)

tests/system/_helpers.py

@@ -20,6 +20,7 @@
 from test_utils.retry import RetryErrors
 from test_utils.retry import RetryInstanceState
 from test_utils.system import unique_resource_id
+from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
 
 retry_429 = RetryErrors(exceptions.TooManyRequests)
 retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10)
@@ -31,6 +32,7 @@
 user_project = os.environ.get("GOOGLE_CLOUD_TESTS_USER_PROJECT")
 testing_mtls = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true"
 signing_blob_content = b"This time for sure, Rocky!"
+is_api_endpoint_override = _DEFAULT_STORAGE_HOST != "https://storage.googleapis.com"
 
 
 def _bad_copy(bad_request):

tests/system/test__signing.py

@@ -17,7 +17,7 @@
 import hashlib
 import os
 import time
-
+import pytest
 import requests
 
 from google.api_core import path_template
@@ -41,7 +41,11 @@ def _create_signed_list_blobs_url_helper(
     expiration = _morph_expiration(version, expiration)
 
     signed_url = bucket.generate_signed_url(
-        expiration=expiration, method=method, client=client, version=version
+        expiration=expiration,
+        method=method,
+        client=client,
+        version=version,
+        api_access_endpoint=_helpers._DEFAULT_STORAGE_HOST,
     )
 
     response = requests.get(signed_url)
@@ -371,6 +375,10 @@ def test_create_signed_resumable_upload_url_v4(storage_client, signing_bucket, n
     )
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
 def test_generate_signed_post_policy_v4(
     storage_client, buckets_to_delete, blobs_to_delete, service_account, no_mtls
 ):

tests/system/test_blob.py

@@ -97,6 +97,10 @@ def test_large_file_write_from_stream_w_failed_checksum(
     assert not blob.exists()
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
 def test_large_file_write_from_stream_w_encryption_key(
     storage_client,
     shared_bucket,

tests/system/test_bucket.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import datetime
-
 import pytest
 
 from google.api_core import exceptions
@@ -124,6 +123,10 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete):
     assert list(bucket.lifecycle_rules) == []
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
 def test_bucket_update_labels(storage_client, buckets_to_delete):
     bucket_name = _helpers.unique_name("update-labels")
     bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name)
@@ -797,6 +800,10 @@ def test_bucket_lock_retention_policy(
         bucket.patch()
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
 def test_new_bucket_w_ubla(
     storage_client,
     buckets_to_delete,
@@ -966,6 +973,10 @@ def test_new_bucket_created_w_enforced_pap(
     assert not bucket.iam_configuration.uniform_bucket_level_access_enabled
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
 def test_new_bucket_with_rpo(
     storage_client,
     buckets_to_delete,

tests/system/test_client.py

@@ -14,6 +14,7 @@
 
 import io
 import re
+import os
 import tempfile
 
 import pytest
@@ -23,9 +24,15 @@
 from . import _helpers
 
 
+dual_data_loc_1 = os.getenv("DUAL_REGION_LOC_1", "US-EAST1")
+dual_data_loc_2 = os.getenv("DUAL_REGION_LOC_2", "US-WEST1")
 public_bucket = "gcp-public-data-landsat"
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
 @vpcsc_config.skip_if_inside_vpcsc
 def test_anonymous_client_access_to_public_bucket():
     from google.cloud.storage.client import Client
@@ -40,6 +47,10 @@ def test_anonymous_client_access_to_public_bucket():
         _helpers.retry_429_503(blob.download_to_file)(stream)
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
 def test_get_service_account_email(storage_client, service_account):
     domain = "gs-project-accounts.iam.gserviceaccount.com"
     email = storage_client.get_service_account_email()
@@ -69,7 +80,8 @@ def test_create_bucket_dual_region(storage_client, buckets_to_delete):
 
     new_bucket_name = _helpers.unique_name("dual-region-bucket")
     location = "US"
-    data_locations = ["US-EAST1", "US-WEST1"]
+
+    data_locations = [dual_data_loc_1, dual_data_loc_2]
 
     with pytest.raises(exceptions.NotFound):
         storage_client.get_bucket(new_bucket_name)