.github/.OwlBot.lock.yaml

@@ -1,3 +1,3 @@
 docker:
   image: gcr.io/repo-automation-bots/owlbot-python:latest
-  digest: sha256:99d90d097e4a4710cc8658ee0b5b963f4426d0e424819787c3ac1405c9a26719
+  digest: sha256:aea14a583128771ae8aefa364e1652f3c56070168ef31beb203534222d842b8b

.kokoro/docker/docs/Dockerfile

@@ -40,6 +40,7 @@ RUN apt-get update \
     libssl-dev \
     libsqlite3-dev \
     portaudio19-dev \
+    python3-distutils \
     redis-server \
     software-properties-common \
     ssh \
@@ -59,40 +60,8 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/* \
   && rm -f /var/cache/apt/archives/*.deb
 
-
-COPY fetch_gpg_keys.sh /tmp
-# Install the desired versions of Python.
-RUN set -ex \
-    && export GNUPGHOME="$(mktemp -d)" \
-    && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
-    && /tmp/fetch_gpg_keys.sh \
-    && for PYTHON_VERSION in 3.7.8 3.8.5; do \
-        wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
-        && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
-        && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
-        && rm -r python-${PYTHON_VERSION}.tar.xz.asc \
-        && mkdir -p /usr/src/python-${PYTHON_VERSION} \
-        && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
-        && rm python-${PYTHON_VERSION}.tar.xz \
-        && cd /usr/src/python-${PYTHON_VERSION} \
-        && ./configure \
-            --enable-shared \
-            # This works only on Python 2.7 and throws a warning on every other
-            # version, but seems otherwise harmless.
-            --enable-unicode=ucs4 \
-            --with-system-ffi \
-            --without-ensurepip \
-        && make -j$(nproc) \
-        && make install \
-        && ldconfig \
-  ; done \
-  && rm -rf "${GNUPGHOME}" \
-  && rm -rf /usr/src/python* \
-  && rm -rf ~/.cache/
-
 RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
-  && python3.7 /tmp/get-pip.py \
   && python3.8 /tmp/get-pip.py \
   && rm /tmp/get-pip.py
 
-CMD ["python3.7"]
+CMD ["python3.8"]

.kokoro/samples/python3.6/periodic-head.cfg

@@ -7,5 +7,5 @@ env_vars: {
 
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
+    value: "github/python-bigquery-sqlalchemy/.kokoro/test-samples-against-head.sh"
 }

.kokoro/samples/python3.7/periodic-head.cfg

@@ -7,5 +7,5 @@ env_vars: {
 
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
+    value: "github/python-bigquery-sqlalchemy/.kokoro/test-samples-against-head.sh"
 }

.kokoro/samples/python3.8/periodic-head.cfg

@@ -7,5 +7,5 @@ env_vars: {
 
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
+    value: "github/python-bigquery-sqlalchemy/.kokoro/test-samples-against-head.sh"
 }

.kokoro/samples/python3.9/periodic-head.cfg

@@ -7,5 +7,5 @@ env_vars: {
 
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
+    value: "github/python-bigquery-sqlalchemy/.kokoro/test-samples-against-head.sh"
 }

.kokoro/test-samples-impl.sh

@@ -20,9 +20,9 @@ set -eo pipefail
 # Enables `**` to include files nested inside sub-folders
 shopt -s globstar
 
-# Exit early if samples directory doesn't exist
-if [ ! -d "./samples" ]; then
-  echo "No tests run. `./samples` not found"
+# Exit early if samples don't exist
+if ! find samples -name 'requirements.txt' | grep -q .; then
+  echo "No tests run. './samples/**/requirements.txt' not found"
   exit 0
 fi
 

.repo-metadata.json

@@ -6,5 +6,6 @@
   "language": "python",
   "library_type": "INTEGRATION",
   "repo": "googleapis/python-bigquery-sqlalchemy",
-  "distribution_name": "pybigquery"
-}
+  "distribution_name": "pybigquery",
+  "api_id": "bigquery.googleapis.com"
+}

CHANGELOG.md

@@ -4,6 +4,13 @@
 
 [1]: https://pypi.org/project/pybigquery/#history
 
+### [0.10.1](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v0.10.0...v0.10.1) (2021-07-30)
+
+
+### Bug Fixes
+
+* **deps:** pin 'google-{api,cloud}-core', 'google-auth' to allow 2.x versions ([#220](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/220)) ([bf1f47c](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/bf1f47c794e747a2ea878347322c040636e8c2d4))
+
 ## [0.10.0](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v0.9.1...v0.10.0) (2021-07-06)
 
 

CONTRIBUTING.rst

@@ -177,6 +177,30 @@ Build the docs via:
 
    $ nox -s docs
 
+*************************
+Samples and code snippets
+*************************
+
+Code samples and snippets live in the `samples/` catalogue. Feel free to
+provide more examples, but make sure to write tests for those examples.
+Each folder containing example code requires its own `noxfile.py` script
+which automates testing. If you decide to create a new folder, you can
+base it on the `samples/snippets` folder (providing `noxfile.py` and
+the requirements files).
+
+The tests will run against a real Google Cloud Project, so you should
+configure them just like the System Tests.
+
+- To run sample tests, you can execute::
+
+   # Run all tests in a folder
+   $ cd samples/snippets
+   $ nox -s py-3.8
+
+   # Run a single sample test
+   $ cd samples/snippets
+   $ nox -s py-3.8 -- -k <name of test>
+
 ********************************************
 Note About ``README`` as it pertains to PyPI
 ********************************************

setup.py

@@ -24,7 +24,7 @@
 # Package metadata.
 
 name = "pybigquery"
-version = "0.10.0"
+version = "0.10.1"
 description = "SQLAlchemy dialect for BigQuery"
 
 # Should be one of:
@@ -66,7 +66,10 @@ def readme():
     platforms="Posix; MacOS X; Windows",
     install_requires=[
         "google-api-core>=1.30.0",  # Work-around bug in cloud core deps.
-        "google-auth>=1.25.0,<2.0dev",  # Work around pip wack.
+        # NOTE: Maintainers, please do not require google-auth>=2.x.x
+        # Until this issue is closed
+        # https://github.com/googleapis/google-cloud-python/issues/10566
+        "google-auth>=1.25.0,<3.0.0dev",  # Work around pip wack.
         "google-cloud-bigquery>=2.19.0",
         "sqlalchemy>=1.2.0,<1.5.0dev",
         "future",

tests/sqlalchemy_dialect_compliance/conftest.py

@@ -18,7 +18,6 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import contextlib
-import random
 import re
 import traceback
 
@@ -30,8 +29,14 @@
 )
 
 import google.cloud.bigquery.dbapi.connection
+import test_utils.prefixer
+
 import pybigquery.sqlalchemy_bigquery
 
+prefixer = test_utils.prefixer.Prefixer(
+    "python-bigquery-sqlalchemy", "tests/compliance"
+)
+
 pybigquery.sqlalchemy_bigquery.BigQueryDialect.preexecute_autoincrement_sequences = True
 google.cloud.bigquery.dbapi.connection.Connection.rollback = lambda self: None
 
@@ -62,7 +67,7 @@ def visit_delete(self, delete_stmt, *args, **kw):
 
 
 def pytest_sessionstart(session):
-    dataset_id = f"test_pybigquery_sqla{random.randint(0, 1<<63)}"
+    dataset_id = prefixer.create_prefix()
     session.config.option.dburi = [f"bigquery:///{dataset_id}"]
     with contextlib.closing(google.cloud.bigquery.Client()) as client:
         client.create_dataset(dataset_id)
@@ -73,4 +78,7 @@ def pytest_sessionfinish(session):
     dataset_id = config.db.dialect.dataset_id
     _pytest_sessionfinish(session)
     with contextlib.closing(google.cloud.bigquery.Client()) as client:
-        client.delete_dataset(dataset_id, delete_contents=True)
+        client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True)
+        for dataset in client.list_datasets():
+            if prefixer.should_cleanup(dataset.dataset_id):
+                client.delete_dataset(dataset, delete_contents=True, not_found_ok=True)

tests/system/conftest.py

@@ -17,25 +17,19 @@
 # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import datetime
 import pathlib
-import random
 from typing import List
 
 import pytest
 
 from google.cloud import bigquery
+import test_utils.prefixer
 
+prefixer = test_utils.prefixer.Prefixer("python-bigquery-sqlalchemy", "tests/system")
 
 DATA_DIR = pathlib.Path(__file__).parent / "data"
 
 
-def temp_suffix():
-    timestamp = datetime.datetime.utcnow().strftime("%y%m%d_%H%M%S")
-    random_string = hex(random.randrange(1000000))[2:]
-    return f"{timestamp}_{random_string}"
-
-
 def load_sample_data(
     full_table_id: str,
     bigquery_client: bigquery.Client,
@@ -73,7 +67,7 @@ def bigquery_dataset(
     bigquery_client: bigquery.Client, bigquery_schema: List[bigquery.SchemaField]
 ):
     project_id = bigquery_client.project
-    dataset_id = f"test_pybigquery_{temp_suffix()}"
+    dataset_id = prefixer.create_prefix()
     dataset = bigquery.Dataset(f"{project_id}.{dataset_id}")
     dataset = bigquery_client.create_dataset(dataset)
     sample_table_id = f"{project_id}.{dataset_id}.sample"
@@ -112,7 +106,7 @@ def bigquery_empty_table(
 @pytest.fixture(scope="session")
 def bigquery_regional_dataset(bigquery_client, bigquery_schema):
     project_id = bigquery_client.project
-    dataset_id = f"test_pybigquery_location_{temp_suffix()}"
+    dataset_id = prefixer.create_prefix()
     dataset = bigquery.Dataset(f"{project_id}.{dataset_id}")
     dataset.location = "asia-northeast1"
     dataset = bigquery_client.create_dataset(dataset)
@@ -126,3 +120,12 @@ def bigquery_regional_dataset(bigquery_client, bigquery_schema):
     job.result()
     yield dataset_id
     bigquery_client.delete_dataset(dataset_id, delete_contents=True)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cleanup_datasets(bigquery_client: bigquery.Client):
+    for dataset in bigquery_client.list_datasets():
+        if prefixer.should_cleanup(dataset.dataset_id):
+            bigquery_client.delete_dataset(
+                dataset, delete_contents=True, not_found_ok=True
+            )

tests/system/test_alembic.py

@@ -82,9 +82,9 @@ def test_alembic_scenario(alembic_table):
         Column("description", String(200)),
     )
     assert alembic_table("account", "schema") == [
-        "SchemaField('id', 'INTEGER', 'REQUIRED')",
-        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name')",
-        "SchemaField('description', 'STRING(200)', 'NULLABLE')",
+        "SchemaField('id', 'INTEGER', 'REQUIRED', None, (), ())",
+        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name', (), ())",
+        "SchemaField('description', 'STRING(200)', 'NULLABLE', None, (), ())",
     ]
 
     op.bulk_insert(
@@ -107,10 +107,11 @@ def test_alembic_scenario(alembic_table):
     )
 
     assert alembic_table("account", "schema") == [
-        "SchemaField('id', 'INTEGER', 'REQUIRED')",
-        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name')",
-        "SchemaField('description', 'STRING(200)', 'NULLABLE')",
-        "SchemaField('last_transaction_date', 'DATETIME', 'NULLABLE', 'when updated')",
+        "SchemaField('id', 'INTEGER', 'REQUIRED', None, (), ())",
+        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name', (), ())",
+        "SchemaField('description', 'STRING(200)', 'NULLABLE', None, (), ())",
+        "SchemaField('last_transaction_date', 'DATETIME', 'NULLABLE', 'when updated'"
+        ", (), ())",
     ]
 
     op.create_table(
@@ -126,8 +127,8 @@ def test_alembic_scenario(alembic_table):
 
     op.drop_column("account_w_comment", "description")
     assert alembic_table("account_w_comment", "schema") == [
-        "SchemaField('id', 'INTEGER', 'REQUIRED')",
-        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name')",
+        "SchemaField('id', 'INTEGER', 'REQUIRED', None, (), ())",
+        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name', (), ())",
     ]
 
     op.drop_table("account_w_comment")
@@ -136,10 +137,11 @@ def test_alembic_scenario(alembic_table):
     op.rename_table("account", "accounts")
     assert alembic_table("account") is None
     assert alembic_table("accounts", "schema") == [
-        "SchemaField('id', 'INTEGER', 'REQUIRED')",
-        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name')",
-        "SchemaField('description', 'STRING(200)', 'NULLABLE')",
-        "SchemaField('last_transaction_date', 'DATETIME', 'NULLABLE', 'when updated')",
+        "SchemaField('id', 'INTEGER', 'REQUIRED', None, (), ())",
+        "SchemaField('name', 'STRING(50)', 'REQUIRED', 'The name', (), ())",
+        "SchemaField('description', 'STRING(200)', 'NULLABLE', None, (), ())",
+        "SchemaField('last_transaction_date', 'DATETIME', 'NULLABLE', 'when updated'"
+        ", (), ())",
     ]
     op.drop_table("accounts")
     assert alembic_table("accounts") is None
@@ -159,9 +161,9 @@ def test_alembic_scenario(alembic_table):
     # nullable:
     op.alter_column("transactions", "amount", True)
     assert alembic_table("transactions", "schema") == [
-        "SchemaField('account', 'INTEGER', 'REQUIRED')",
-        "SchemaField('transaction_time', 'DATETIME', 'REQUIRED')",
-        "SchemaField('amount', 'NUMERIC(11, 2)', 'NULLABLE')",
+        "SchemaField('account', 'INTEGER', 'REQUIRED', None, (), ())",
+        "SchemaField('transaction_time', 'DATETIME', 'REQUIRED', None, (), ())",
+        "SchemaField('amount', 'NUMERIC(11, 2)', 'NULLABLE', None, (), ())",
     ]
 
     op.create_table_comment("transactions", "Transaction log")