feat: instrument metadata ops with OTel tracing (#2) (#1267)

* feat: instrument metadata ops with Otel tracing * update README plus test * update decorator name per review session * fix typo in readme
googleapis · May 9, 2024 · 6f70198 · 6f70198
1 parent e6ed8e8
commit 6f70198
Show file tree

Hide file tree

Showing 10 changed files with 130 additions and 15 deletions.
diff --git a/README.rst b/README.rst
@@ -108,6 +108,59 @@ Windows
     .\<your-env>\Scripts\activate
     pip install google-cloud-storage
 
+
+Tracing With OpenTelemetry
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This library uses `OpenTelemetry`_ to generate traces on calls to Google Cloud Storage.
+For information on the benefits and utility of tracing, read the `Cloud Trace Overview <https://cloud.google.com/trace/docs/overview>`_.
+
+To enable OpenTelemetry tracing in the Cloud Storage client, first install OpenTelemetry:
+
+.. code-block:: console
+
+    pip install google-cloud-storage[tracing]
+
+Set the ``ENABLE_GCS_PYTHON_CLIENT_OTEL_TRACES`` environment variable to selectively opt-in tracing for the Cloud Storage client:
+
+.. code-block:: console
+
+    export ENABLE_GCS_PYTHON_CLIENT_OTEL_TRACES=True
+
+You will also need to tell OpenTelemetry which exporter to use. An example to export traces to Google Cloud Trace can be found below.
+
+.. code-block:: console
+
+    # Install the Google Cloud Trace exporter and propagator, however you can use any exporter of your choice.
+    pip install opentelemetry-exporter-gcp-trace opentelemetry-propagator-gcp
+
+    # [Optional] Install the OpenTelemetry Requests Instrumentation to trace the underlying HTTP requests.
+    pip install opentelemetry-instrumentation-requests
+
+.. code-block:: python
+
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
+
+    tracer_provider = TracerProvider()
+    tracer_provider.add_span_processor(BatchSpanProcessor(CloudTraceSpanExporter()))
+    trace.set_tracer_provider(tracer_provider)
+
+    # Optional yet recommended to instrument the requests HTTP library
+    from opentelemetry.instrumentation.requests import RequestsInstrumentor
+    RequestsInstrumentor().instrument(tracer_provider=tracer_provider)
+
+In this example, tracing data will be published to the `Google Cloud Trace`_ console.
+Tracing is most effective when many libraries are instrumented to provide insight over the entire lifespan of a request.
+For a list of libraries that can be instrumented, refer to the `OpenTelemetry Registry`_.
+
+.. _OpenTelemetry: https://opentelemetry.io
+.. _OpenTelemetry Registry: https://opentelemetry.io/ecosystem/registry
+.. _Google Cloud Trace: https://cloud.google.com/trace
+
+
 Next Steps
 ~~~~~~~~~~
 

diff --git a/google/cloud/storage/_http.py b/google/cloud/storage/_http.py
@@ -18,7 +18,7 @@
 from google.cloud import _http
 from google.cloud.storage import __version__
 from google.cloud.storage import _helpers
-from google.cloud.storage._opentelemetry_tracing import create_span
+from google.cloud.storage._opentelemetry_tracing import create_trace_span
 
 
 class Connection(_http.JSONConnection):
@@ -72,7 +72,7 @@ def api_request(self, *args, **kwargs):
             "gccl-invocation-id": invocation_id,
         }
         call = functools.partial(super(Connection, self).api_request, *args, **kwargs)
-        with create_span(
+        with create_trace_span(
             name="Storage.Connection.api_request",
             attributes=span_attributes,
             client=self._client,

diff --git a/google/cloud/storage/_opentelemetry_tracing.py b/google/cloud/storage/_opentelemetry_tracing.py
@@ -42,7 +42,7 @@
 
 
 @contextmanager
-def create_span(
+def create_trace_span(
     name, attributes=None, client=None, api_request=None, retry=None, **kwargs
 ):
     """Creates a context manager for a new span and set it as the current span

diff --git a/google/cloud/storage/acl.py b/google/cloud/storage/acl.py
@@ -15,6 +15,7 @@
 """Manage access to objects and buckets."""
 
 from google.cloud.storage._helpers import _add_generation_match_parameters
+from google.cloud.storage._opentelemetry_tracing import create_trace_span
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
 from google.cloud.storage.retry import DEFAULT_RETRY
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
@@ -359,6 +360,7 @@ def _require_client(self, client):
             client = self.client
         return client
 
+    @create_trace_span(name="Storage.ACL.reload")
     def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY):
         """Reload the ACL data from Cloud Storage.
 
@@ -484,6 +486,7 @@ def _save(
 
         self.loaded = True
 
+    @create_trace_span(name="Storage.ACL.save")
     def save(
         self,
         acl=None,
@@ -552,6 +555,7 @@ def save(
                 retry=retry,
             )
 
+    @create_trace_span(name="Storage.ACL.savePredefined")
     def save_predefined(
         self,
         predefined,
@@ -617,6 +621,7 @@ def save_predefined(
             retry=retry,
         )
 
+    @create_trace_span(name="Storage.ACL.clear")
     def clear(
         self,
         client=None,

diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py
@@ -63,6 +63,7 @@
 from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE
 from google.cloud.storage._helpers import _API_VERSION
 from google.cloud.storage._helpers import _virtual_hosted_style_base_url
+from google.cloud.storage._opentelemetry_tracing import create_trace_span
 from google.cloud.storage.acl import ACL
 from google.cloud.storage.acl import ObjectACL
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
@@ -639,6 +640,7 @@ def generate_signed_url(
             access_token=access_token,
         )
 
+    @create_trace_span(name="Storage.Blob.exists")
     def exists(
         self,
         client=None,
@@ -744,6 +746,7 @@ def exists(
             return False
         return True
 
+    @create_trace_span(name="Storage.Blob.delete")
     def delete(
         self,
         client=None,
@@ -3281,6 +3284,7 @@ def create_resumable_upload_session(
         except resumable_media.InvalidResponse as exc:
             _raise_from_invalid_response(exc)
 
+    @create_trace_span(name="Storage.Blob.getIamPolicy")
     def get_iam_policy(
         self,
         client=None,
@@ -3349,6 +3353,7 @@ def get_iam_policy(
         )
         return Policy.from_api_repr(info)
 
+    @create_trace_span(name="Storage.Blob.setIamPolicy")
     def set_iam_policy(
         self,
         policy,
@@ -3410,6 +3415,7 @@ def set_iam_policy(
         )
         return Policy.from_api_repr(info)
 
+    @create_trace_span(name="Storage.Blob.testIamPermissions")
     def test_iam_permissions(
         self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
     ):
@@ -3464,6 +3470,7 @@ def test_iam_permissions(
 
         return resp.get("permissions", [])
 
+    @create_trace_span(name="Storage.Blob.makePublic")
     def make_public(
         self,
         client=None,
@@ -3517,6 +3524,7 @@ def make_public(
             retry=retry,
         )
 
+    @create_trace_span(name="Storage.Blob.makePrivate")
     def make_private(
         self,
         client=None,

diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py
@@ -38,6 +38,7 @@
 from google.cloud.storage._signing import generate_signed_url_v4
 from google.cloud.storage._helpers import _bucket_bound_hostname_url
 from google.cloud.storage._helpers import _virtual_hosted_style_base_url
+from google.cloud.storage._opentelemetry_tracing import create_trace_span
 from google.cloud.storage.acl import BucketACL
 from google.cloud.storage.acl import DefaultObjectACL
 from google.cloud.storage.blob import Blob
@@ -827,6 +828,7 @@ def notification(
             notification_id=notification_id,
         )
 
+    @create_trace_span(name="Storage.Bucket.exists")
     def exists(
         self,
         client=None,
@@ -911,6 +913,7 @@ def exists(
             return False
         return True
 
+    @create_trace_span(name="Storage.Bucket.create")
     def create(
         self,
         client=None,
@@ -986,6 +989,7 @@ def create(
             retry=retry,
         )
 
+    @create_trace_span(name="Storage.Bucket.update")
     def update(
         self,
         client=None,
@@ -1030,6 +1034,7 @@ def update(
             retry=retry,
         )
 
+    @create_trace_span(name="Storage.Bucket.reload")
     def reload(
         self,
         client=None,
@@ -1091,6 +1096,7 @@ def reload(
             retry=retry,
         )
 
+    @create_trace_span(name="Storage.Bucket.patch")
     def patch(
         self,
         client=None,
@@ -1174,6 +1180,7 @@ def path(self):
 
         return self.path_helper(self.name)
 
+    @create_trace_span(name="Storage.Bucket.getBlob")
     def get_blob(
         self,
         blob_name,
@@ -1290,6 +1297,7 @@ def get_blob(
         else:
             return blob
 
+    @create_trace_span(name="Storage.Bucket.listBlobs")
     def list_blobs(
         self,
         max_results=None,
@@ -1425,6 +1433,7 @@ def list_blobs(
             soft_deleted=soft_deleted,
         )
 
+    @create_trace_span(name="Storage.Bucket.listNotifications")
     def list_notifications(
         self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
     ):
@@ -1462,6 +1471,7 @@ def list_notifications(
         iterator.bucket = self
         return iterator
 
+    @create_trace_span(name="Storage.Bucket.getNotification")
     def get_notification(
         self,
         notification_id,
@@ -1499,6 +1509,7 @@ def get_notification(
         notification.reload(client=client, timeout=timeout, retry=retry)
         return notification
 
+    @create_trace_span(name="Storage.Bucket.delete")
     def delete(
         self,
         force=False,
@@ -1605,6 +1616,7 @@ def delete(
             _target_object=None,
         )
 
+    @create_trace_span(name="Storage.Bucket.deleteBlob")
     def delete_blob(
         self,
         blob_name,
@@ -1685,6 +1697,7 @@ def delete_blob(
             _target_object=None,
         )
 
+    @create_trace_span(name="Storage.Bucket.deleteBlobs")
     def delete_blobs(
         self,
         blobs,
@@ -2085,6 +2098,7 @@ def rename_blob(
             )
         return new_blob
 
+    @create_trace_span(name="Storage.Bucket.restore_blob")
     def restore_blob(
         self,
         blob_name,
@@ -2957,6 +2971,7 @@ def disable_website(self):
         """
         return self.configure_website(None, None)
 
+    @create_trace_span(name="Storage.Bucket.getIamPolicy")
     def get_iam_policy(
         self,
         client=None,
@@ -3019,6 +3034,7 @@ def get_iam_policy(
         )
         return Policy.from_api_repr(info)
 
+    @create_trace_span(name="Storage.Bucket.setIamPolicy")
     def set_iam_policy(
         self,
         policy,
@@ -3075,6 +3091,7 @@ def set_iam_policy(
 
         return Policy.from_api_repr(info)
 
+    @create_trace_span(name="Storage.Bucket.testIamPermissions")
     def test_iam_permissions(
         self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
     ):
@@ -3122,6 +3139,7 @@ def test_iam_permissions(
         )
         return resp.get("permissions", [])
 
+    @create_trace_span(name="Storage.Bucket.makePublic")
     def make_public(
         self,
         recursive=False,
@@ -3219,6 +3237,7 @@ def make_public(
                     timeout=timeout,
                 )
 
+    @create_trace_span(name="Storage.Bucket.makePrivate")
     def make_private(
         self,
         recursive=False,
@@ -3366,6 +3385,7 @@ def generate_upload_policy(self, conditions, expiration=None, client=None):
 
         return fields
 
+    @create_trace_span(name="Storage.Bucket.lockRetentionPolicy")
     def lock_retention_policy(
         self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
     ):