Skip to content

Commit

Permalink
Improve the usability when operating on a larger number of records (g…
Browse files Browse the repository at this point in the history
…oogle#2209)

This prevents a messier Datastore API failure (the IN operator only
supports up to 30 values) when querying for more than 30 Bugs and
signposts a solution.
  • Loading branch information
andrewpollock committed May 15, 2024
1 parent 29e15fe commit 8ff91b9
Showing 1 changed file with 9 additions and 1 deletion.
10 changes: 9 additions & 1 deletion tools/datafix/reimport_gcs_record.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import functools

MAX_BATCH_SIZE = 500
MAX_QUERY_SIZE = 30


class UnexpectedSituation(Exception):
Expand Down Expand Up @@ -121,7 +122,10 @@ def main() -> None:
parser = argparse.ArgumentParser(
description="Trigger the reimport of individual GCS-sourced records")
parser.add_argument(
"bugs", action="append", nargs="+", help="The bug IDs to operate on")
"bugs",
action="append",
nargs="+",
help=f"The bug IDs to operate on ({MAX_QUERY_SIZE} at most)")
parser.add_argument(
"--dry-run",
action=argparse.BooleanOptionalAction,
Expand All @@ -148,6 +152,10 @@ def main() -> None:
help="Local directory to copy to from GCS")
args = parser.parse_args()

if len(args.bugs[0]) > MAX_QUERY_SIZE:
parser.error(f"Only {MAX_QUERY_SIZE} bugs can be supplied. "
f"Try running with xargs -n {MAX_QUERY_SIZE}")

ds_client = datastore.Client(project=args.project)
url_base = url_for_project(args.project)

Expand Down

0 comments on commit 8ff91b9

Please sign in to comment.