.pre-commit-config.yaml

@@ -26,6 +26,6 @@ repos:
     hooks:
     - id: black
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.0
+    rev: 3.9.1
     hooks:
     - id: flake8

CHANGELOG.md

@@ -4,6 +4,14 @@
 
 [1]: https://pypi.org/project/pybigquery/#history
 
+## [0.9.0](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v0.8.0...v0.9.0) (2021-05-25)
+
+
+### Features
+
+* Alembic support ([#183](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/183)) ([4d5a17c](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/4d5a17c8f63328d4484ea7b2ccc58334a421ba81))
+* Support parameterized NUMERIC, BIGNUMERIC, STRING, and BYTES types ([#180](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/180)) ([d118238](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/d1182385b9f1551e605acdc7e2dd45dff22c8064))
+
 ## [0.8.0](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v0.7.0...v0.8.0) (2021-05-21)
 
 

CONTRIBUTING.rst

@@ -160,21 +160,7 @@ Running System Tests
   auth settings and change some configuration in your project to
   run all the tests.
 
-- System tests will be run against an actual project and
-  so you'll need to provide some environment variables to facilitate
-  authentication to your project:
-
-  - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file;
-    Such a file can be downloaded directly from the developer's console by clicking
-    "Generate new JSON key". See private key
-    `docs <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`__
-    for more details.
-
-- Once you have downloaded your json keys, set the environment variable 
-  ``GOOGLE_APPLICATION_CREDENTIALS`` to the absolute path of the json file::
-
-   $ export GOOGLE_APPLICATION_CREDENTIALS="/Users/<your_username>/path/to/app_credentials.json"
-
+- System tests will be run against an actual project. You should use local credentials from gcloud when possible. See `Best practices for application authentication <https://cloud.google.com/docs/authentication/best-practices-applications#local_development_and_testing_with_the>`__. Some tests require a service account. For those tests see `Authenticating as a service account <https://cloud.google.com/docs/authentication/production>`__.
 
 *************
 Test Coverage

docs/alembic.rst

@@ -0,0 +1,51 @@
+Alembic support
+---------------
+
+`Alembic <https://alembic.sqlalchemy.org>`_ is a lightweight database
+migration tool for usage with the SQLAlchemy Database Toolkit for
+Python.  It can use this BigQuery SQLAlchemy support to manage
+BigQuery shemas.
+
+Some features, like management of constrains and indexes, aren't
+supported because `BigQuery doesn't support them
+<https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language>`_.
+
+Supported operations:
+
+`add_column(table_name, column, schema=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.add_column>`_
+
+`alter_column(table_name, column_name, nullable=None, schema=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.alter_column>`_
+
+`bulk_insert(table, rows, multiinsert=True)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.bulk_insert>`_
+
+`create_table(table_name, *columns, **kw)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.create_table>`_
+
+`create_table_comment(table_name, comment, schema=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.create_table_comment>`_
+
+`drop_column(table_name, column_name, schema=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.drop_column>`_
+
+`drop_table(table_name, schema=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.drop_table>`_
+
+`drop_table_comment(table_name, schema=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.drop_table_comment>`_
+
+`execute(sqltext, execution_options=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.execute>`_
+
+`rename_table(old_table_name, new_table_name, schema=None)
+<https://alembic.sqlalchemy.org/en/latest/ops.html#alembic.operations.Operations.rename_table>`_
+
+Note that some of the operations above have limited capability, again
+do to `BigQuery limitations
+<https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language>`_.
+
+The `execute` operation allows access to BigQuery-specific
+`data-definition-language
+<https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language>`_.

docs/index.rst

@@ -2,6 +2,8 @@
 
 .. include:: multiprocessing.rst
 
+.. include:: alembic.rst
+
 API Reference
 -------------
 .. toctree::

noxfile.py

@@ -66,16 +66,9 @@ def lint(session):
     session.run("flake8", "pybigquery", "tests")
 
 
-@nox.session(python="3.6")
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def blacken(session):
-    """Run black.
-
-    Format code to uniform standard.
-
-    This currently uses Python 3.6 due to the automated Kokoro run of synthtool.
-    That run uses an image that doesn't have 3.6 installed. Before updating this
-    check the state of the `gcp_ubuntu_config` we use for that Kokoro run.
-    """
+    """Run black. Format code to uniform standard."""
     session.install(BLACK_VERSION)
     session.run(
         "black", *BLACK_PATHS,
@@ -89,14 +82,30 @@ def lint_setup_py(session):
     session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
 
 
+def install_alembic_for_python_38(session, constraints_path):
+    """
+    install alembic for Python 3.8 unit and system tests
+
+    We don't require alembic and most tests should run without it, however
+
+    - We run some unit tests (Python 3.8) to cover the alembic
+      registration that happens when alembic is installed.
+
+    - We have a system test that demonstrates working with alembic and
+      proves that the things we think should work do work. :)
+    """
+    if session.python == "3.8":
+        session.install("alembic", "-c", constraints_path)
+
+
 def default(session):
     # Install all test dependencies, then install this package in-place.
 
     constraints_path = str(
         CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
     )
     session.install("mock", "pytest", "pytest-cov", "-c", constraints_path)
-
+    install_alembic_for_python_38(session, constraints_path)
     session.install("-e", ".", "-c", constraints_path)
 
     # Run py.test against the unit tests.
@@ -133,9 +142,6 @@ def system(session):
     # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
     if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false":
         session.skip("RUN_SYSTEM_TESTS is set to false, skipping")
-    # Sanity check: Only run tests if the environment variable is set.
-    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
-        session.skip("Credentials must be set via environment variable")
     # Install pyopenssl for mTLS testing.
     if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true":
         session.install("pyopenssl")
@@ -152,6 +158,7 @@ def system(session):
     # Install all test dependencies, then install this package into the
     # virtualenv's dist-packages.
     session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path)
+    install_alembic_for_python_38(session, constraints_path)
     session.install("-e", ".", "-c", constraints_path)
 
     # Run py.test against the system tests.

pybigquery/_helpers.py

@@ -69,9 +69,6 @@ def substitute_re_method(r, flags=0, repl=None):
 
     r = re.compile(r, flags)
 
-    if isinstance(repl, str):
-        return lambda self, s: r.sub(repl, s)
-
     @functools.wraps(repl)
     def sub(self, s, *args, **kw):
         def repl_(m):
@@ -80,3 +77,8 @@ def repl_(m):
         return r.sub(repl_, s)
 
     return sub
+
+
+def substitute_string_re_method(r, *, repl, flags=0):
+    r = re.compile(r, flags)
+    return lambda self, s: r.sub(repl, s)