From 47e722c64498aa35b64756d83a19a0e6ac22f0e7 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Tue, 16 Feb 2021 13:44:19 -0500 Subject: [PATCH 1/4] kv: allow manual GC that does not advance threshold Relates to #60585. This commit updates the GC queue's `process` method to allow GC even if it determines that doing so would not advance the GC threshold. This check is retained in the queue's `shouldQueue` method, so this change does not impact automated queueing decisions. However, it does ensure that if someone manually enqueues a range in the GC queue with the `ShouldSkipQueue` option, then the range is actually processed. This is important because manually the GC threshold is the first thing that the GC bumps when processing, meaning that the GC threshold alone is not a definitive indication that there is no more work to perform on a range. A processing pass that will not bump the threshold can still be useful, especially with the long context timeout associated with manual enqueuing. Release note (ui change): Manually enqueue range in a replica GC queue now properly respects the SkipShouldQueue option. This can be useful to force a GC of a specific Range. --- pkg/kv/kvserver/gc_queue.go | 9 ++++-- .../kvserver/replica_protected_timestamp.go | 32 ++++++++----------- .../replica_protected_timestamp_test.go | 22 +++++++------ 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/pkg/kv/kvserver/gc_queue.go b/pkg/kv/kvserver/gc_queue.go index 06049f63781..380713d6f2a 100644 --- a/pkg/kv/kvserver/gc_queue.go +++ b/pkg/kv/kvserver/gc_queue.go @@ -158,10 +158,15 @@ func (gcq *gcQueue) shouldQueue( // Consult the protected timestamp state to determine whether we can GC and // the timestamp which can be used to calculate the score. _, zone := repl.DescAndZone() - canGC, _, gcTimestamp, _ := repl.checkProtectedTimestampsForGC(ctx, *zone.GC) + canGC, _, gcTimestamp, oldThreshold, newThreshold := repl.checkProtectedTimestampsForGC(ctx, *zone.GC) if !canGC { return false, 0 } + // If performing a GC will not advance the GC threshold, there's no reason + // to GC again. + if newThreshold.Equal(oldThreshold) { + return false, 0 + } r := makeGCQueueScore(ctx, repl, gcTimestamp, *zone.GC) return r.ShouldQueue, r.FinalScore } @@ -442,7 +447,7 @@ func (gcq *gcQueue) process( // Consult the protected timestamp state to determine whether we can GC and // the timestamp which can be used to calculate the score and updated GC // threshold. - canGC, cacheTimestamp, gcTimestamp, newThreshold := repl.checkProtectedTimestampsForGC(ctx, *zone.GC) + canGC, cacheTimestamp, gcTimestamp, _, newThreshold := repl.checkProtectedTimestampsForGC(ctx, *zone.GC) if !canGC { return false, nil } diff --git a/pkg/kv/kvserver/replica_protected_timestamp.go b/pkg/kv/kvserver/replica_protected_timestamp.go index 62502240d68..d6e04fe0bee 100644 --- a/pkg/kv/kvserver/replica_protected_timestamp.go +++ b/pkg/kv/kvserver/replica_protected_timestamp.go @@ -220,20 +220,20 @@ func (r *Replica) protectedTimestampRecordCurrentlyApplies( return false, read.readAt.Less(args.RecordAliveAt), nil } -// checkProtectedTimestampsForGC determines whether the Replica can run GC. -// If the Replica can run GC, this method returns the latest timestamp which -// can be used to determine a valid new GCThreshold. The policy is passed in -// rather than read from the replica state to ensure that the same value used -// for this calculation is used later. +// checkProtectedTimestampsForGC determines whether the Replica can run GC. If +// the Replica can run GC, this method returns the latest timestamp which can be +// used to determine a valid new GCThreshold. The policy is passed in rather +// than read from the replica state to ensure that the same value used for this +// calculation is used later. // -// In the case that GC can proceed, three timestamps are returned: The timestamp +// In the case that GC can proceed, four timestamps are returned: The timestamp // corresponding to the state of the cache used to make the determination (used // for markPendingGC when actually performing GC), the timestamp used as the -// basis to calculate the new gc threshold (used for scoring and reporting), and -// the new gc threshold itself. +// basis to calculate the new gc threshold (used for scoring and reporting), the +// old gc threshold, and the new gc threshold. func (r *Replica) checkProtectedTimestampsForGC( ctx context.Context, policy zonepb.GCPolicy, -) (canGC bool, cacheTimestamp, gcTimestamp, newThreshold hlc.Timestamp) { +) (canGC bool, cacheTimestamp, gcTimestamp, oldThreshold, newThreshold hlc.Timestamp) { // We may be reading the protected timestamp cache while we're holding // the Replica.mu for reading. If we do so and find newer state in the cache @@ -247,10 +247,10 @@ func (r *Replica) checkProtectedTimestampsForGC( defer r.mu.RUnlock() defer read.clearIfNotNewer(r.mu.cachedProtectedTS) - gcThreshold := *r.mu.state.GCThreshold + oldThreshold = *r.mu.state.GCThreshold lease := *r.mu.state.Lease - // earliestValidRecord is the record with the earliest timestamp which is + // read.earliestRecord is the record with the earliest timestamp which is // greater than the existing gcThreshold. read = r.readProtectedTimestampsRLocked(ctx, nil) gcTimestamp = read.readAt @@ -266,18 +266,12 @@ func (r *Replica) checkProtectedTimestampsForGC( if gcTimestamp.Less(lease.Start.ToTimestamp()) { log.VEventf(ctx, 1, "not gc'ing replica %v due to new lease %v started after %v", r, lease, gcTimestamp) - return false, hlc.Timestamp{}, hlc.Timestamp{}, hlc.Timestamp{} + return false, hlc.Timestamp{}, hlc.Timestamp{}, hlc.Timestamp{}, hlc.Timestamp{} } newThreshold = gc.CalculateThreshold(gcTimestamp, policy) - // If we've already GC'd right up to this record, there's no reason to - // gc again. - if newThreshold.Equal(gcThreshold) { - return false, hlc.Timestamp{}, hlc.Timestamp{}, hlc.Timestamp{} - } - - return true, read.readAt, gcTimestamp, newThreshold + return true, read.readAt, gcTimestamp, oldThreshold, newThreshold } // markPendingGC is called just prior to sending the GC request to increase the diff --git a/pkg/kv/kvserver/replica_protected_timestamp_test.go b/pkg/kv/kvserver/replica_protected_timestamp_test.go index ac18c93e7fc..e2560b4a98c 100644 --- a/pkg/kv/kvserver/replica_protected_timestamp_test.go +++ b/pkg/kv/kvserver/replica_protected_timestamp_test.go @@ -315,7 +315,7 @@ func TestCheckProtectedTimestampsForGC(t *testing.T) { name: "lease is too new", test: func(t *testing.T, r *Replica, mt *manualCache) { r.mu.state.Lease.Start = r.store.Clock().NowAsClockTimestamp() - canGC, _, gcTimestamp, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) + canGC, _, gcTimestamp, _, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) require.False(t, canGC) require.Zero(t, gcTimestamp) }, @@ -337,7 +337,7 @@ func TestCheckProtectedTimestampsForGC(t *testing.T) { }) // We should allow gc to proceed with the normal new threshold if that // threshold is earlier than all of the records. - canGC, _, gcTimestamp, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) + canGC, _, gcTimestamp, _, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) require.True(t, canGC) require.Equal(t, mt.asOf, gcTimestamp) }, @@ -362,8 +362,9 @@ func TestCheckProtectedTimestampsForGC(t *testing.T) { // We should allow gc to proceed up to the timestamp which precedes the // protected timestamp. This means we expect a GC timestamp 10 seconds // after ts.Prev() given the policy. - canGC, _, gcTimestamp, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) + canGC, _, gcTimestamp, oldThreshold, newThreshold := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) require.True(t, canGC) + require.False(t, newThreshold.Equal(oldThreshold)) require.Equal(t, ts.Prev().Add(10*time.Second.Nanoseconds(), 0), gcTimestamp) }, }, @@ -386,11 +387,14 @@ func TestCheckProtectedTimestampsForGC(t *testing.T) { }, }, }) - // We should not allow GC if the threshold is already the predecessor - // of the earliest valid record. - canGC, _, gcTimestamp, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) - require.False(t, canGC) - require.Zero(t, gcTimestamp) + // We should allow GC even if the threshold is already the + // predecessor of the earliest valid record. However, the GC + // queue does not enqueue ranges in such cases, so this is only + // applicable to manually enqueued ranges. + canGC, _, gcTimestamp, oldThreshold, newThreshold := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) + require.True(t, canGC) + require.True(t, newThreshold.Equal(oldThreshold)) + require.Equal(t, th.Add(10*time.Second.Nanoseconds(), 0), gcTimestamp) }, }, { @@ -411,7 +415,7 @@ func TestCheckProtectedTimestampsForGC(t *testing.T) { }, }, }) - canGC, _, gcTimestamp, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) + canGC, _, gcTimestamp, _, _ := r.checkProtectedTimestampsForGC(ctx, makePolicy(10)) require.True(t, canGC) require.Equal(t, mt.asOf, gcTimestamp) }, From b01fc598bea0762a48a172f972a978dcb77e3625 Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Wed, 17 Feb 2021 14:02:25 +1100 Subject: [PATCH 2/4] sql: fix RENAME COLUMN for REGIONAL BY ROW We need to correct the column reference on the table descriptor if we are renaming the REGIONAL BY ROW column. Release note: None --- .../testdata/logic_test/regional_by_row | 150 ++++++++++++++++++ pkg/sql/catalog/descriptor.go | 1 + pkg/sql/rename_column.go | 10 ++ 3 files changed, 161 insertions(+) diff --git a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row index f3cefcbb99c..18922668716 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row +++ b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row @@ -716,6 +716,102 @@ PARTITION "ap-southeast-2" OF INDEX regional_by_row_table@unique_b_a ALTER PART voter_constraints = '{+region=ap-southeast-2: 2}', lease_preferences = '[[+region=ap-southeast-2]]' +# Tests renaming a referenced implicit column in REGIONAL BY ROW succeeds. +statement ok +ALTER TABLE regional_by_row_table RENAME COLUMN crdb_region TO crdb_region2 + +query T +SELECT create_statement FROM [SHOW CREATE TABLE regional_by_row_table] +---- +CREATE TABLE public.regional_by_row_table ( + pk INT8 NOT NULL, + pk2 INT8 NOT NULL, + a INT8 NOT NULL, + b INT8 NOT NULL, + j JSONB NULL, + crdb_region2 public.crdb_internal_region NOT VISIBLE NOT NULL DEFAULT default_to_database_primary_region(gateway_region())::public.crdb_internal_region, + CONSTRAINT "primary" PRIMARY KEY (pk2 ASC), + UNIQUE INDEX regional_by_row_table_pk_key (pk ASC), + INDEX regional_by_row_table_a_idx (a ASC), + UNIQUE INDEX regional_by_row_table_b_key (b ASC), + INVERTED INDEX regional_by_row_table_j_idx (j), + INDEX new_idx (a ASC, b ASC), + UNIQUE INDEX unique_b_a (b ASC, a ASC), + FAMILY fam_0_pk_pk2_a_b_j_crdb_region (pk, pk2, a, b, j, crdb_region2) +) LOCALITY REGIONAL BY ROW AS crdb_region2; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table@new_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table@primary CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_a_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_b_key CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_j_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table@unique_b_a CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table@new_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table@primary CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_a_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_b_key CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_j_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table@unique_b_a CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table@new_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table@primary CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_a_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_b_key CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table@regional_by_row_table_j_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table@unique_b_a CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]' + # Tests for REGIONAL BY TABLE AS statement error cannot use column crdb_region_col which has type INT8 in REGIONAL BY ROW AS\nDETAIL:\s+REGIONAL BY ROW AS must reference a column of type crdb_internal_region. CREATE TABLE regional_by_row_table_as ( @@ -814,6 +910,60 @@ pk a b crdb_region_col statement error cannot drop column crdb_region_col as it is used to store the region in a REGIONAL BY ROW table\nHINT: You must change the table locality before dropping this table ALTER TABLE regional_by_row_table_as DROP COLUMN crdb_region_col +# Tests renaming a referenced explicit column in REGIONAL BY ROW succeeds. +statement ok +ALTER TABLE regional_by_row_table_as RENAME COLUMN crdb_region_col TO cr + +query T +SELECT create_statement FROM [SHOW CREATE TABLE regional_by_row_table_as] +---- +CREATE TABLE public.regional_by_row_table_as ( + pk INT8 NOT NULL, + a INT8 NULL, + b INT8 NULL, + cr public.crdb_internal_region NOT NULL AS (CASE WHEN pk <= 10:::INT8 THEN 'us-east-1':::public.crdb_internal_region ELSE 'ap-southeast-2':::public.crdb_internal_region END) STORED, + CONSTRAINT "primary" PRIMARY KEY (pk ASC), + INDEX regional_by_row_table_as_a_idx (a ASC), + UNIQUE INDEX regional_by_row_table_as_b_key (b ASC), + FAMILY fam_0_pk_a_b_crdb_region_col (pk, a, b, cr) +) LOCALITY REGIONAL BY ROW AS cr; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table_as@primary CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table_as@regional_by_row_table_as_a_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ap-southeast-2" OF INDEX multi_region_test_db.public.regional_by_row_table_as@regional_by_row_table_as_b_key CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ap-southeast-2: 2}', + lease_preferences = '[[+region=ap-southeast-2]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table_as@primary CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table_as@regional_by_row_table_as_a_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "ca-central-1" OF INDEX multi_region_test_db.public.regional_by_row_table_as@regional_by_row_table_as_b_key CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=ca-central-1: 2}', + lease_preferences = '[[+region=ca-central-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table_as@primary CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table_as@regional_by_row_table_as_a_idx CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]'; +ALTER PARTITION "us-east-1" OF INDEX multi_region_test_db.public.regional_by_row_table_as@regional_by_row_table_as_b_key CONFIGURE ZONE USING + num_voters = 5, + voter_constraints = '{+region=us-east-1: 2}', + lease_preferences = '[[+region=us-east-1]]' + # Tests for altering the survivability of a REGIONAL BY ROW table. statement ok CREATE DATABASE alter_survive_db PRIMARY REGION "us-east-1" REGIONS "ca-central-1", "ap-southeast-2" SURVIVE REGION FAILURE diff --git a/pkg/sql/catalog/descriptor.go b/pkg/sql/catalog/descriptor.go index 56342a2b7fe..1172dd99070 100644 --- a/pkg/sql/catalog/descriptor.go +++ b/pkg/sql/catalog/descriptor.go @@ -267,6 +267,7 @@ type TableDescriptor interface { IsLocalityRegionalByTable() bool IsLocalityGlobal() bool GetRegionalByTableRegion() (descpb.RegionName, error) + GetRegionalByRowTableRegionColumnName() (tree.Name, error) } // Index is an interface around the index descriptor types. diff --git a/pkg/sql/rename_column.go b/pkg/sql/rename_column.go index 836e1b47602..fe56fce09d7 100644 --- a/pkg/sql/rename_column.go +++ b/pkg/sql/rename_column.go @@ -250,6 +250,16 @@ func (p *planner) renameColumn( } } + // Rename the REGIONAL BY ROW column reference. + if tableDesc.IsLocalityRegionalByRow() { + rbrColName, err := tableDesc.GetRegionalByRowTableRegionColumnName() + if err != nil { + return false, err + } + if rbrColName == *oldName { + tableDesc.SetTableLocalityRegionalByRow(*newName) + } + } return true, nil } From 5886177a3da25b8155f199c23fb283f921c0dff4 Mon Sep 17 00:00:00 2001 From: Angela Xu Date: Fri, 12 Feb 2021 12:41:48 -0800 Subject: [PATCH 3/4] opt: inverted-index accelerate filters of the form j->'a' = '{"b": "c"}' Previously, the optimizer did not plan inverted index scans for queries with the JSON fetch value operator `->` when an object or array was on the right side of the equality operator `=`, only when it was a boolean, string, number, or null. This change allows the inverted index to be used in these types of queries. It supports objects with multiple key/value pairs, nested objects, arrays, arrays nested within objects, and objects nested within arrays. Fixes: #59605 Release note: None --- .../testdata/logic_test/inverted_index | 93 +++++- pkg/sql/opt/invertedidx/json_array.go | 21 +- pkg/sql/opt/invertedidx/json_array_test.go | 22 +- pkg/sql/opt/memo/testdata/stats/inverted-json | 283 ++++++++++++++++++ pkg/sql/opt/xform/testdata/rules/select | 26 +- 5 files changed, 420 insertions(+), 25 deletions(-) diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_index b/pkg/sql/logictest/testdata/logic_test/inverted_index index 4ec455b113d..a18cd46d351 100644 --- a/pkg/sql/logictest/testdata/logic_test/inverted_index +++ b/pkg/sql/logictest/testdata/logic_test/inverted_index @@ -752,7 +752,22 @@ INSERT INTO f VALUES (10, '{"a": {"b": {"c": 1}}}'), (11, '{"a": {"b": {"c": 1, "d": 2}}}}'), (12, '{"a": {"b": {"d": 2}}}}'), - (13, '{"a": {"b": {"c": [1, 2]}}}') + (13, '{"a": {"b": {"c": [1, 2]}}}'), + (14, '{"a": {"b": {"c": [1, 2, 3]}}}'), + (15, '{"a": []}'), + (16, '{"a": {}}}'), + (17, '{"a": {"b": "c"}}'), + (18, '{"a": {"b": ["c", "d", "e"]}}'), + (19, '{"a": ["b", "c", "d", "e"]}'), + (20, '{"a": ["b", "e", "c", "d"]}'), + (21, '{"z": {"a": "b", "c": "d"}}'), + (22, '{"z": {"a": "b", "c": "d", "e": "f"}}'), + (23, '{"a": "b", "x": ["c", "d", "e"]}}'), + (24, '{"a": "b", "c": [{"d": 1}, {"e": 2}]}}'), + (25, '{"a": {"b": "c", "d": "e"}}'), + (26, '{"a": {"b": "c"}, "d": "e"}'), + (27, '[1, 2, {"b": "c"}]'), + (28, '[{"a": {"b": "c"}}, "d", "e"]') query T SELECT j FROM f@i WHERE j->'a' = '1' ORDER BY k @@ -789,6 +804,82 @@ SELECT j FROM f@i WHERE j->'a'->'b'->'c' = '1' ORDER BY k {"a": {"b": {"c": 1}}} {"a": {"b": {"c": 1, "d": 2}}} +query T +SELECT j FROM f@i WHERE j->'a' = '[]' ORDER BY k +---- +{"a": []} + +query T +SELECT j FROM f@i WHERE j->'a' = '{}' ORDER BY k +---- +{"a": {}} + +query T +SELECT j FROM f@i WHERE j->'a' = '["b"]' ORDER BY k +---- + +query T +SELECT j FROM f@i WHERE j->'a' = '"b"' ORDER BY k +---- +{"a": "b", "x": ["c", "d", "e"]} +{"a": "b", "c": [{"d": 1}, {"e": 2}]} + +query T +SELECT j FROM f@i WHERE j->'a' = '{"b": "c"}' ORDER BY k +---- +{"a": {"b": "c"}} +{"a": {"b": "c"}, "d": "e"} + +query T +SELECT j FROM f@i WHERE j->'a'->'b'->'c' = '[1, 2]' ORDER BY k +---- +{"a": {"b": {"c": [1, 2]}}} + +query T +SELECT j FROM f@i WHERE j->'z' = '{"a": "b", "c": "d"}' ORDER BY k +---- +{"z": {"a": "b", "c": "d"}} + +query T +SELECT j FROM f@i WHERE j->'a' = '["b", "c", "d", "e"]' ORDER BY k +---- +{"a": ["b", "c", "d", "e"]} + +query T +SELECT j FROM f@i WHERE j->'a' = '["b", "c", "d", "e"]' OR j->'a' = '["b", "e", "c", "d"]' ORDER BY k +---- +{"a": ["b", "c", "d", "e"]} +{"a": ["b", "e", "c", "d"]} + +query T +SELECT j FROM f@i WHERE j->'a' = '{"b": ["c", "d", "e"]}' ORDER BY k +---- +{"a": {"b": ["c", "d", "e"]}} + +query T +SELECT j FROM f@i WHERE j->'a'->'b' = '["c", "d", "e"]' ORDER BY k +---- +{"a": {"b": ["c", "d", "e"]}} + +query T +SELECT j FROM f@i WHERE j->'z'->'c' = '"d"' ORDER BY k +---- +{"z": {"a": "b", "c": "d"}} +{"z": {"a": "b", "c": "d", "e": "f"}} + +query T +SELECT j FROM f@i WHERE j->'z' = '{"c": "d"}' ORDER BY k +---- + +query T +SELECT j FROM f@i WHERE j->'a' = '"b"' AND j->'c' = '[{"d": 1}]' ORDER BY k +---- + +query T +SELECT j FROM f@i WHERE j->'a' = '"b"' AND j->'c' = '[{"d": 1}, {"e": 2}]' ORDER BY k +---- +{"a": "b", "c": [{"d": 1}, {"e": 2}]} + subtest arrays statement ok diff --git a/pkg/sql/opt/invertedidx/json_array.go b/pkg/sql/opt/invertedidx/json_array.go index 680da639cf3..a1331338858 100644 --- a/pkg/sql/opt/invertedidx/json_array.go +++ b/pkg/sql/opt/invertedidx/json_array.go @@ -342,12 +342,11 @@ func (j *jsonOrArrayFilterPlanner) extractJSONOrArrayContainsCondition( // expression in the form [col]->[index0]->[index1]->...->[indexN] where col is // a variable or expression referencing the inverted column in the inverted // index and each index is a constant string. The right expression must be a -// constant JSON value that is not an object or an array. +// constant JSON value. func (j *jsonOrArrayFilterPlanner) extractJSONFetchValEqCondition( evalCtx *tree.EvalContext, left *memo.FetchValExpr, right opt.ScalarExpr, ) inverted.Expression { - // The right side of the equals expression should be a constant JSON value - // that is not an object or array. + // The right side of the equals expression should be a constant JSON value. if !memo.CanExtractConstDatum(right) { return inverted.NonInvertedColExpression{} } @@ -355,10 +354,6 @@ func (j *jsonOrArrayFilterPlanner) extractJSONFetchValEqCondition( if !ok { return inverted.NonInvertedColExpression{} } - typ := val.JSON.Type() - if typ == json.ObjectJSONType || typ == json.ArrayJSONType { - return inverted.NonInvertedColExpression{} - } // Recursively traverse fetch val expressions and collect keys with which to // build the InvertedExpression. If it is not possible to build an inverted @@ -431,5 +426,15 @@ func (j *jsonOrArrayFilterPlanner) extractJSONFetchValEqCondition( obj = b.Build() } - return getInvertedExprForJSONOrArrayIndex(evalCtx, tree.NewDJSON(obj)) + invertedExpr := getInvertedExprForJSONOrArrayIndex(evalCtx, tree.NewDJSON(obj)) + + // When the right side is an array or object, the InvertedExpression + // generated is not tight. We must indicate it is non-tight so an additional + // filter is added. + typ := val.JSON.Type() + if typ == json.ArrayJSONType || typ == json.ObjectJSONType { + invertedExpr.SetNotTight() + } + + return invertedExpr } diff --git a/pkg/sql/opt/invertedidx/json_array_test.go b/pkg/sql/opt/invertedidx/json_array_test.go index 353fd43dd20..a797077e801 100644 --- a/pkg/sql/opt/invertedidx/json_array_test.go +++ b/pkg/sql/opt/invertedidx/json_array_test.go @@ -407,16 +407,22 @@ func TestTryFilterJsonOrArrayIndex(t *testing.T) { ok: false, }, { - // Arrays on the right side of the equality are not yet supported. - filters: "j->'a' = '[1]'", - indexOrd: jsonOrd, - ok: false, + // Arrays on the right side of the equality are supported. + filters: "j->'a' = '[1]'", + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: true, + remainingFilters: "j->'a' = '[1]'", }, { - // Objects on the right side of the equality are not yet supported. - filters: `j->'a' = '{"b": "c"}'`, - indexOrd: jsonOrd, - ok: false, + // Objects on the right side of the equality are supported. + filters: `j->'a' = '{"b": "c"}'`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: true, + remainingFilters: `j->'a' = '{"b": "c"}'`, }, { // Wrong index ordinal. diff --git a/pkg/sql/opt/memo/testdata/stats/inverted-json b/pkg/sql/opt/memo/testdata/stats/inverted-json index c1c6e08d5f5..1cf26f1dc0d 100644 --- a/pkg/sql/opt/memo/testdata/stats/inverted-json +++ b/pkg/sql/opt/memo/testdata/stats/inverted-json @@ -383,3 +383,286 @@ index-join t │ <--- '\x37000138' --- '\x37000139' ├── key: (1) └── fd: (1)-->(4) + +# A query with the fetch val operator with a single key/val pair object on the +# right side uses the inverted index, and the inverted expression is not tight. +opt +SELECT * FROM t WHERE j->'a' = '{"b": "c"}' +---- +select + ├── columns: k:1(int!null) j:2(jsonb) + ├── immutable + ├── stats: [rows=222.222222] + ├── key: (1) + ├── fd: (1)-->(2) + ├── index-join t + │ ├── columns: k:1(int!null) j:2(jsonb) + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ └── scan t@j_idx + │ ├── columns: k:1(int!null) + │ ├── inverted constraint: /4/1 + │ │ └── spans: ["7a\x00\x02b\x00\x01\x12c\x00\x01", "7a\x00\x02b\x00\x01\x12c\x00\x01"] + │ ├── stats: [rows=2e-07, distinct(4)=2e-07, null(4)=0] + │ │ histogram(4)= + │ └── key: (1) + └── filters + └── (j:2->'a') = '{"b": "c"}' [type=bool, outer=(2), immutable] + +# A query with the fetch val operator with a nested object on the right side +# uses the inverted index, and the inverted expression is not tight. +opt +SELECT * FROM t WHERE j->'a' = '{"b": {"c": "d"}}' +---- +select + ├── columns: k:1(int!null) j:2(jsonb) + ├── immutable + ├── stats: [rows=222.222222] + ├── key: (1) + ├── fd: (1)-->(2) + ├── index-join t + │ ├── columns: k:1(int!null) j:2(jsonb) + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ └── scan t@j_idx + │ ├── columns: k:1(int!null) + │ ├── inverted constraint: /4/1 + │ │ └── spans: ["7a\x00\x02b\x00\x02c\x00\x01\x12d\x00\x01", "7a\x00\x02b\x00\x02c\x00\x01\x12d\x00\x01"] + │ ├── stats: [rows=2e-07, distinct(4)=2e-07, null(4)=0] + │ │ histogram(4)= + │ └── key: (1) + └── filters + └── (j:2->'a') = '{"b": {"c": "d"}}' [type=bool, outer=(2), immutable] + +# A query with the fetch val operator with an object on the right side +# with multiple key/val pairs uses the inverted index, and the inverted +# expression is not tight. +opt +SELECT * FROM t WHERE j->'a' = '{"b": "c", "d": "e"}' +---- +select + ├── columns: k:1(int!null) j:2(jsonb) + ├── immutable + ├── stats: [rows=222.222222] + ├── key: (1) + ├── fd: (1)-->(2) + ├── index-join t + │ ├── columns: k:1(int!null) j:2(jsonb) + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ └── inverted-filter + │ ├── columns: k:1(int!null) + │ ├── inverted expression: /4 + │ │ ├── tight: false, unique: true + │ │ ├── union spans: empty + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7a\x00\x02b\x00\x01\x12c\x00\x01", "7a\x00\x02b\x00\x01\x12c\x00\x01"] + │ │ └── span expression + │ │ ├── tight: true, unique: true + │ │ └── union spans: ["7a\x00\x02d\x00\x01\x12e\x00\x01", "7a\x00\x02d\x00\x01\x12e\x00\x01"] + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ └── scan t@j_idx + │ ├── columns: k:1(int!null) j_inverted_key:4(jsonb!null) + │ ├── inverted constraint: /4/1 + │ │ └── spans + │ │ ├── ["7a\x00\x02b\x00\x01\x12c\x00\x01", "7a\x00\x02b\x00\x01\x12c\x00\x01"] + │ │ └── ["7a\x00\x02d\x00\x01\x12e\x00\x01", "7a\x00\x02d\x00\x01\x12e\x00\x01"] + │ ├── stats: [rows=2e-07, distinct(1)=2e-07, null(1)=0, distinct(4)=2e-07, null(4)=0] + │ │ histogram(4)= + │ ├── key: (1) + │ └── fd: (1)-->(4) + └── filters + └── (j:2->'a') = '{"b": "c", "d": "e"}' [type=bool, outer=(2), immutable] + +# A query with the fetch val operator with an array on the right side +# uses the inverted index, and the inverted expression is not tight. +opt +SELECT * FROM t WHERE j->'a' = '["b", "c", "d", "e"]' +---- +select + ├── columns: k:1(int!null) j:2(jsonb) + ├── immutable + ├── stats: [rows=222.222222] + ├── key: (1) + ├── fd: (1)-->(2) + ├── index-join t + │ ├── columns: k:1(int!null) j:2(jsonb) + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ └── inverted-filter + │ ├── columns: k:1(int!null) + │ ├── inverted expression: /4 + │ │ ├── tight: false, unique: true + │ │ ├── union spans: empty + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ ├── union spans: empty + │ │ │ │ └── INTERSECTION + │ │ │ │ ├── span expression + │ │ │ │ │ ├── tight: true, unique: true + │ │ │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01\x12b\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12b\x00\x01"] + │ │ │ │ └── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01\x12c\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12c\x00\x01"] + │ │ │ └── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01\x12d\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12d\x00\x01"] + │ │ └── span expression + │ │ ├── tight: true, unique: true + │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01\x12e\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12e\x00\x01"] + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ └── scan t@j_idx + │ ├── columns: k:1(int!null) j_inverted_key:4(jsonb!null) + │ ├── inverted constraint: /4/1 + │ │ └── spans + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01\x12b\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12b\x00\x01"] + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01\x12c\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12c\x00\x01"] + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01\x12d\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12d\x00\x01"] + │ │ └── ["7a\x00\x02\x00\x03\x00\x01\x12e\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12e\x00\x01"] + │ ├── stats: [rows=2e-07, distinct(1)=2e-07, null(1)=0, distinct(4)=2e-07, null(4)=0] + │ │ histogram(4)= + │ ├── key: (1) + │ └── fd: (1)-->(4) + └── filters + └── (j:2->'a') = '["b", "c", "d", "e"]' [type=bool, outer=(2), immutable] + +# A query with the fetch val operator with an object on the right side +# that contains an array uses the inverted index, and the inverted expression +# is not tight. +opt +SELECT * FROM t WHERE j->'a' = '{"b": ["c", "d", "e"]}' +---- +select + ├── columns: k:1(int!null) j:2(jsonb) + ├── immutable + ├── stats: [rows=222.222222] + ├── key: (1) + ├── fd: (1)-->(2) + ├── index-join t + │ ├── columns: k:1(int!null) j:2(jsonb) + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ └── inverted-filter + │ ├── columns: k:1(int!null) + │ ├── inverted expression: /4 + │ │ ├── tight: false, unique: true + │ │ ├── union spans: empty + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ └── union spans: ["7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12c\x00\x01", "7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12c\x00\x01"] + │ │ │ └── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12d\x00\x01", "7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12d\x00\x01"] + │ │ └── span expression + │ │ ├── tight: true, unique: true + │ │ └── union spans: ["7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12e\x00\x01", "7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12e\x00\x01"] + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ └── scan t@j_idx + │ ├── columns: k:1(int!null) j_inverted_key:4(jsonb!null) + │ ├── inverted constraint: /4/1 + │ │ └── spans + │ │ ├── ["7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12c\x00\x01", "7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12c\x00\x01"] + │ │ ├── ["7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12d\x00\x01", "7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12d\x00\x01"] + │ │ └── ["7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12e\x00\x01", "7a\x00\x02b\x00\x02\x00\x03\x00\x01\x12e\x00\x01"] + │ ├── stats: [rows=2e-07, distinct(1)=2e-07, null(1)=0, distinct(4)=2e-07, null(4)=0] + │ │ histogram(4)= + │ ├── key: (1) + │ └── fd: (1)-->(4) + └── filters + └── (j:2->'a') = '{"b": ["c", "d", "e"]}' [type=bool, outer=(2), immutable] + +# A query with the fetch val operator with empty array on the right side +# uses the inverted index, and the inverted expression is not tight. +opt +SELECT * FROM t WHERE j->'a' = '[]' +---- +select + ├── columns: k:1(int!null) j:2(jsonb) + ├── immutable + ├── stats: [rows=222.222222] + ├── key: (1) + ├── fd: (1)-->(2) + ├── index-join t + │ ├── columns: k:1(int!null) j:2(jsonb) + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ └── inverted-filter + │ ├── columns: k:1(int!null) + │ ├── inverted expression: /4 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7a\x00\x018", "7a\x00\x018"] + │ │ └── ["7a\x00\x02\x00\x03", "7a\x00\x02\x00\x03"] + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ └── scan t@j_idx + │ ├── columns: k:1(int!null) j_inverted_key:4(jsonb!null) + │ ├── inverted constraint: /4/1 + │ │ └── spans + │ │ ├── ["7a\x00\x018", "7a\x00\x018"] + │ │ └── ["7a\x00\x02\x00\x03", "7a\x00\x02\x00\x03"] + │ ├── stats: [rows=2e-07, distinct(1)=2e-07, null(1)=0, distinct(4)=2e-07, null(4)=0] + │ │ histogram(4)= + │ ├── key: (1) + │ └── fd: (1)-->(4) + └── filters + └── (j:2->'a') = '[]' [type=bool, outer=(2), immutable] + +# A query with the fetch val operator with an empty object on the right side +# uses the inverted index, and the inverted expression is not tight. +opt +SELECT * FROM t WHERE j->'a' = '{}' +---- +select + ├── columns: k:1(int!null) j:2(jsonb) + ├── immutable + ├── stats: [rows=222.222222] + ├── key: (1) + ├── fd: (1)-->(2) + ├── index-join t + │ ├── columns: k:1(int!null) j:2(jsonb) + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ └── inverted-filter + │ ├── columns: k:1(int!null) + │ ├── inverted expression: /4 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7a\x00\x019", "7a\x00\x019"] + │ │ └── ["7a\x00\x02\x00\xff", "7a\x00\x03") + │ ├── stats: [rows=2e-07] + │ ├── key: (1) + │ └── scan t@j_idx + │ ├── columns: k:1(int!null) j_inverted_key:4(jsonb!null) + │ ├── inverted constraint: /4/1 + │ │ └── spans + │ │ ├── ["7a\x00\x019", "7a\x00\x019"] + │ │ └── ["7a\x00\x02\x00\xff", "7a\x00\x03") + │ ├── stats: [rows=2e-07, distinct(1)=2e-07, null(1)=0, distinct(4)=2e-07, null(4)=0] + │ │ histogram(4)= + │ ├── key: (1) + │ └── fd: (1)-->(4) + └── filters + └── (j:2->'a') = '{}' [type=bool, outer=(2), immutable] diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index 3c5fd198ad1..2faca1cd58f 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -2165,8 +2165,8 @@ project └── filters └── (j:4->0) = '"b"' [outer=(4), immutable] -# Do not generate an inverted scan when right side of the equality is an array. -opt expect-not=GenerateInvertedIndexScans +# Generate an inverted scan when right side of the equality is an array. +opt expect=GenerateInvertedIndexScans SELECT k FROM b WHERE j->'a' = '["b"]' ---- project @@ -2178,15 +2178,20 @@ project ├── immutable ├── key: (1) ├── fd: (1)-->(4) - ├── scan b + ├── index-join b │ ├── columns: k:1!null j:4 │ ├── key: (1) - │ └── fd: (1)-->(4) + │ ├── fd: (1)-->(4) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7a\x00\x02\x00\x03\x00\x01\x12b\x00\x01", "7a\x00\x02\x00\x03\x00\x01\x12b\x00\x01"] + │ └── key: (1) └── filters └── (j:4->'a') = '["b"]' [outer=(4), immutable] -# Do not generate an inverted scan when right side of the equality is an object. -opt expect-not=GenerateInvertedIndexScans +# Generate an inverted scan when right side of the equality is an object. +opt expect=GenerateInvertedIndexScans SELECT k FROM b WHERE j->'a' = '{"b": "c"}' ---- project @@ -2198,10 +2203,15 @@ project ├── immutable ├── key: (1) ├── fd: (1)-->(4) - ├── scan b + ├── index-join b │ ├── columns: k:1!null j:4 │ ├── key: (1) - │ └── fd: (1)-->(4) + │ ├── fd: (1)-->(4) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7a\x00\x02b\x00\x01\x12c\x00\x01", "7a\x00\x02b\x00\x01\x12c\x00\x01"] + │ └── key: (1) └── filters └── (j:4->'a') = '{"b": "c"}' [outer=(4), immutable] From bb69822b2134f0309f29783c29c51db594f51ccf Mon Sep 17 00:00:00 2001 From: sumeerbhola Date: Wed, 17 Feb 2021 16:30:14 -0500 Subject: [PATCH 4/4] vendor: bump pebble to 444296cf 444296cf Merge pull request #1066 from petermattis/pmattis/stability 59659388 update README with note about production readiness a516e691 *: optimize SeekPrefixGE to use Next 327d2757 sstable: hoist blockSizes from testBytesIteratedWithCompression 69b09310 sstable: add benchmarks for zstd cbe3a149 sstable: fix review comments deb29d88 sstable: include zstd into the test cases f13dea6f sstable: support zstd compression 73e8a3b0 vendor: added DataDog/zstd ec81e4c4 sstable: add zstd-related constants Release note: None --- DEPS.bzl | 15 +++++++++++---- WORKSPACE | 4 ++-- go.mod | 8 ++++---- go.sum | 16 ++++++++++------ vendor | 2 +- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 30be086edf1..cf8a8d57da4 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -428,8 +428,8 @@ def go_deps(): name = "com_github_cockroachdb_pebble", build_file_proto_mode = "disable_global", importpath = "github.com/cockroachdb/pebble", - sum = "h1:akHzBD0xt67DdAbtvSTPEgZM583K226mMhXc6i2UKoY=", - version = "v0.0.0-20210201233136-12ffb4e9273f", + sum = "h1:EyzONynthydmrlGVcEiyNmbLwDejSGb9Rzyn1NcEtNw=", + version = "v0.0.0-20210217155127-444296cfa2bb", ) go_repository( name = "com_github_cockroachdb_redact", @@ -558,6 +558,13 @@ def go_deps(): sum = "h1:2L2f5t3kKnCLxnClDD/PrDfExFFa1wjESgxHG/B1ibo=", version = "v1.3.2", ) + go_repository( + name = "com_github_datadog_zstd", + build_file_proto_mode = "disable_global", + importpath = "github.com/DataDog/zstd", + sum = "h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY=", + version = "v1.4.8", + ) go_repository( name = "com_github_dave_dst", @@ -2838,8 +2845,8 @@ def go_deps(): name = "org_golang_x_exp", build_file_proto_mode = "disable_global", importpath = "golang.org/x/exp", - sum = "h1:sZS6jzvbihmmQSPcU1iZSROTVMAjCXPEnd/ZCSyKsjA=", - version = "v0.0.0-20210201131500-d352d2db2ceb", + sum = "h1:8LMx3JFCswBZrnLWtxzpogDG5g1Hb7KWy/16Msz0hQk=", + version = "v0.0.0-20210212053707-62dc52270d37", ) go_repository( name = "org_golang_x_image", diff --git a/WORKSPACE b/WORKSPACE index 55d39e06647..adc2ff4fb89 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -48,8 +48,8 @@ go_repository( name = "org_golang_x_sys", build_file_proto_mode = "disable_global", importpath = "golang.org/x/sys", - sum = "h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=", - version = "v0.0.0-20210124154548-22da62e12c0c", + sum = "h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU=", + version = "v0.0.0-20210217105451-b926d437f341", ) go_repository( diff --git a/go.mod b/go.mod index dbd8865556b..528f9f3b274 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/Azure/go-autorest/autorest/to v0.3.0 github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect github.com/BurntSushi/toml v0.3.1 + github.com/DataDog/zstd v1.4.8 // indirect github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718 github.com/PuerkitoBio/goquery v1.5.0 github.com/Shopify/sarama v1.27.2 @@ -39,7 +40,7 @@ require ( github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 github.com/cockroachdb/gostdlib v1.13.0 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f - github.com/cockroachdb/pebble v0.0.0-20210201233136-12ffb4e9273f + github.com/cockroachdb/pebble v0.0.0-20210217155127-444296cfa2bb github.com/cockroachdb/redact v1.0.9 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 @@ -95,7 +96,6 @@ require ( github.com/kevinburke/go-bindata v3.13.0+incompatible github.com/kisielk/errcheck v1.2.0 github.com/kisielk/gotool v1.0.0 - github.com/klauspost/compress v1.11.7 // indirect github.com/knz/go-libedit v1.10.1 github.com/knz/strtime v0.0.0-20200318182718-be999391ffa9 github.com/kr/pretty v0.2.1 @@ -148,13 +148,13 @@ require ( github.com/zabawaba99/go-gitignore v0.0.0-20200117185801-39e6bddfb292 go.etcd.io/etcd/raft/v3 v3.0.0-20210215124703-719f6ce06fbc golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad - golang.org/x/exp v0.0.0-20210201131500-d352d2db2ceb + golang.org/x/exp v0.0.0-20210212053707-62dc52270d37 golang.org/x/lint v0.0.0-20200130185559-910be7a94367 golang.org/x/net v0.0.0-20210119194325-5f4716e94777 golang.org/x/oauth2 v0.0.0-20190115181402-5dab4167f31c golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c + golang.org/x/sys v0.0.0-20210217105451-b926d437f341 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 golang.org/x/text v0.3.5 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 diff --git a/go.sum b/go.sum index 7e036d80f61..0d8f130eea7 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,10 @@ github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxX github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d/go.mod h1:RVnhzAX71far8Kc3TQeA0k/dcaEKUnTDSOyet/JCmGI= github.com/DATA-DOG/go-sqlmock v1.3.2 h1:2L2f5t3kKnCLxnClDD/PrDfExFFa1wjESgxHG/B1ibo= github.com/DATA-DOG/go-sqlmock v1.3.2/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= +github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= @@ -170,8 +174,8 @@ github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249 h1:pZu github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249/go.mod h1:UJ0EZAp832vCd54Wev9N1BMKEyvcZ5+IM0AwDrnlkEc= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20210201233136-12ffb4e9273f h1:akHzBD0xt67DdAbtvSTPEgZM583K226mMhXc6i2UKoY= -github.com/cockroachdb/pebble v0.0.0-20210201233136-12ffb4e9273f/go.mod h1:9RB/z2OoNt2vP08nc73FlTVOUhwvgA2/nPSQfgSxq4g= +github.com/cockroachdb/pebble v0.0.0-20210217155127-444296cfa2bb h1:EyzONynthydmrlGVcEiyNmbLwDejSGb9Rzyn1NcEtNw= +github.com/cockroachdb/pebble v0.0.0-20210217155127-444296cfa2bb/go.mod h1:1XpB4cLQcF189RAcWi4gUc110zJgtOfT7SVNGY8sOe0= github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.0.9 h1:sjlUvGorKMIVQfo+w2RqDi5eewCHn453C/vdIXMzjzI= @@ -795,8 +799,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20210201131500-d352d2db2ceb h1:sZS6jzvbihmmQSPcU1iZSROTVMAjCXPEnd/ZCSyKsjA= -golang.org/x/exp v0.0.0-20210201131500-d352d2db2ceb/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4= +golang.org/x/exp v0.0.0-20210212053707-62dc52270d37 h1:8LMx3JFCswBZrnLWtxzpogDG5g1Hb7KWy/16Msz0hQk= +golang.org/x/exp v0.0.0-20210212053707-62dc52270d37/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -897,8 +901,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= diff --git a/vendor b/vendor index d0bf7d9b8a7..ad59c46f9a3 160000 --- a/vendor +++ b/vendor @@ -1 +1 @@ -Subproject commit d0bf7d9b8a7ae379f6e92c6ba6649c79a9d0d9ef +Subproject commit ad59c46f9a36acd3a14fda6ae5d3a9f25b3e88a6