test: re-enable system tests (#670)

.gitignore

@@ -30,6 +30,7 @@ scripts/local_test_setup
 tests/data/key.json
 tests/data/key.p12
 tests/data/user-key.json
+system_tests/data/
 
 # PyCharm configuration:
 .idea

.kokoro/build.sh

@@ -31,7 +31,14 @@ env | grep KOKORO
 export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
 
 # Setup project id.
-export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
+export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.txt")
+
+# Activate gcloud with service account credentials	
+gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS	
+gcloud config set project ${PROJECT_ID}	
+
+# Decrypt system test secrets	
+./scripts/decrypt-secrets.sh
 
 # Remove old nox
 python3 -m pip uninstall --yes --quiet nox-automation
@@ -47,3 +54,10 @@ if [[ -n "${NOX_SESSION:-}" ]]; then
 else
     python3 -m nox
 fi
+
+
+# Decrypt system test secrets
+./scripts/decrypt-secrets.sh
+
+# Run system tests which use a different noxfile
+python3 -m nox -f system_tests/noxfile.py

.kokoro/continuous/common.cfg

@@ -11,7 +11,7 @@ action {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Download resources for system tests (service account key, etc.)
-gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python"
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-auth-library-python"
 
 # Use the trampoline script to run in docker.
 build_file: "google-auth-library-python/.kokoro/trampoline.sh"

.kokoro/presubmit/common.cfg

@@ -11,7 +11,7 @@ action {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Download resources for system tests (service account key, etc.)
-gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python"
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-auth-library-python"
 
 # Use the trampoline script to run in docker.
 build_file: "google-auth-library-python/.kokoro/trampoline.sh"

synth.py

@@ -10,8 +10,8 @@
 s.move(
     templated_files / ".kokoro",
     excludes=[
-        ".kokoro/continuous/common.cfg",
-        ".kokoro/presubmit/common.cfg",
-        ".kokoro/build.sh",
+        "continuous/common.cfg",
+        "presubmit/common.cfg",
+        "build.sh",
     ],
 )  # just move kokoro configs

system_tests/noxfile.py

@@ -30,7 +30,7 @@
 import py.path
 
 HERE = os.path.abspath(os.path.dirname(__file__))
-LIBRARY_DIR = os.path.join(HERE, "..")
+LIBRARY_DIR = os.path.abspath(os.path.dirname(HERE))
 DATA_DIR = os.path.join(HERE, "data")
 SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, "service_account.json")
 AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, "authorized_user.json")
@@ -169,7 +169,7 @@ def configure_cloud_sdk(session, application_default_credentials, project=False)
 # Test sesssions
 
 TEST_DEPENDENCIES_ASYNC = ["aiohttp", "pytest-asyncio", "nest-asyncio"]
-TEST_DEPENDENCIES_SYNC = ["pytest", "requests"]
+TEST_DEPENDENCIES_SYNC = ["pytest", "requests", "mock"]
 PYTHON_VERSIONS_ASYNC = ["3.7"]
 PYTHON_VERSIONS_SYNC = ["2.7", "3.7"]
 
@@ -249,6 +249,7 @@ def app_engine(session):
         session.log("Skipping App Engine tests.")
         return
 
+    session.install(LIBRARY_DIR)
     # Unlike the default tests above, the App Engine system test require a
     # 'real' gcloud sdk installation that is configured to deploy to an
     # app engine project.
@@ -269,9 +270,8 @@ def app_engine(session):
     application_url = GAE_APP_URL_TMPL.format(GAE_TEST_APP_SERVICE, project_id)
 
     # Vendor in the test application's dependencies
-    session.chdir(os.path.join(HERE, "../app_engine_test_app"))
+    session.chdir(os.path.join(HERE, "system_tests_sync/app_engine_test_app"))
     session.install(*TEST_DEPENDENCIES_SYNC)
-    session.install(LIBRARY_DIR)
     session.run(
         "pip", "install", "--target", "lib", "-r", "requirements.txt", silent=True
     )
@@ -288,7 +288,7 @@ def app_engine(session):
 @nox.session(python=PYTHON_VERSIONS_SYNC)
 def grpc(session):
     session.install(LIBRARY_DIR)
-    session.install(*TEST_DEPENDENCIES_SYNC, "google-cloud-pubsub==1.0.0")
+    session.install(*TEST_DEPENDENCIES_SYNC, "google-cloud-pubsub==1.7.0")
     session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE
     session.run("pytest", "system_tests_sync/test_grpc.py")
 

system_tests/system_tests_async/conftest.py

@@ -71,7 +71,8 @@ async def _token_info(access_token=None, id_token=None):
         url = _helpers.update_query(sync_conftest.TOKEN_INFO_URL, query_params)
 
         response = await http_request(url=url, method="GET")
-        data = await response.data.read()
+
+        data = await response.content()
 
         return json.loads(data.decode("utf-8"))
 

system_tests/system_tests_async/test_default.py

@@ -15,14 +15,13 @@
 import os
 import pytest
 
-import google.auth
+from google.auth import _default_async
 
 EXPECT_PROJECT_ID = os.environ.get("EXPECT_PROJECT_ID")
 
 @pytest.mark.asyncio
 async def test_application_default_credentials(verify_refresh):
-    credentials, project_id = google.auth.default_async()
-    #breakpoint()
+    credentials, project_id = _default_async.default_async()
 
     if EXPECT_PROJECT_ID is not None:
         assert project_id is not None

system_tests/system_tests_sync/app_engine_test_app/requirements.txt

@@ -1,3 +1,3 @@
 urllib3
 # Relative path to google-auth-python's source.
-../..
+../../..

system_tests/system_tests_sync/test_grpc.py

@@ -17,8 +17,6 @@
 import google.auth.jwt
 import google.auth.transport.grpc
 from google.cloud import pubsub_v1
-from google.cloud.pubsub_v1.gapic import publisher_client
-from google.cloud.pubsub_v1.gapic.transports import publisher_grpc_transport
 
 
 def test_grpc_request_with_regular_credentials(http_request):
@@ -27,13 +25,8 @@ def test_grpc_request_with_regular_credentials(http_request):
         credentials, ["https://www.googleapis.com/auth/pubsub"]
     )
 
-    transport = publisher_grpc_transport.PublisherGrpcTransport(
-        address=publisher_client.PublisherClient.SERVICE_ADDRESS,
-        credentials=credentials,
-    )
-
     # Create a pub/sub client.
-    client = pubsub_v1.PublisherClient(transport=transport)
+    client = pubsub_v1.PublisherClient(credentials=credentials)
 
     # list the topics and drain the iterator to test that an authorized API
     # call works.
@@ -48,13 +41,8 @@ def test_grpc_request_with_jwt_credentials():
         credentials, audience=audience
     )
 
-    transport = publisher_grpc_transport.PublisherGrpcTransport(
-        address=publisher_client.PublisherClient.SERVICE_ADDRESS,
-        credentials=credentials,
-    )
-
     # Create a pub/sub client.
-    client = pubsub_v1.PublisherClient(transport=transport)
+    client = pubsub_v1.PublisherClient(credentials=credentials)
 
     # list the topics and drain the iterator to test that an authorized API
     # call works.
@@ -68,13 +56,8 @@ def test_grpc_request_with_on_demand_jwt_credentials():
         credentials
     )
 
-    transport = publisher_grpc_transport.PublisherGrpcTransport(
-        address=publisher_client.PublisherClient.SERVICE_ADDRESS,
-        credentials=credentials,
-    )
-
     # Create a pub/sub client.
-    client = pubsub_v1.PublisherClient(transport=transport)
+    client = pubsub_v1.PublisherClient(credentials=credentials)
 
     # list the topics and drain the iterator to test that an authorized API
     # call works.

system_tests/system_tests_sync/test_mtls_http.py

@@ -13,8 +13,11 @@
 # limitations under the License.
 
 import json
-from os import path
+import mock
+import os
 import time
+from os import path
+
 
 import google.auth
 import google.auth.credentials