sql: crash when collecting stats on virtual computed column with enum type #123821

michae2 · 2024-05-08T17:15:50Z

In #122313 we fixed a missing type resolver when planning stats collection on virtual computed columns with enum types. But something is still not quite right, because now the following repro crashes the node with a nil pointer dereference during type checking.

CREATE TYPE order_status AS ENUM ('pending', 'paid', 'dispatched', 'delivered');

CREATE TABLE orders (
  "id" UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  "customer_id" UUID NOT NULL,
  "total" DECIMAL NOT NULL,
  "balance" DECIMAL NOT NULL,
  "order_ts" TIMESTAMPTZ(0) NOT NULL DEFAULT now(),
  "dispatch_ts" TIMESTAMPTZ(0),
  "delivery_ts" TIMESTAMPTZ(0),
  "status" order_status AS (
    CASE
      WHEN "delivery_ts" IS NOT NULL THEN 'delivered'
      WHEN "dispatch_ts" IS NOT NULL THEN 'dispatched'
      WHEN "balance" = 0 THEN 'paid'
      ELSE 'pending'
    END) VIRTUAL,

  INDEX ("status")
) WITH (sql_stats_automatic_collection_enabled=false);

INSERT INTO orders ("customer_id", "total", "balance", "dispatch_ts", "delivery_ts") VALUES
  ('bdeb232e-12e9-4a33-9dd5-7bb9b694291a', 100, 100, NULL, NULL),
  ('0dc59725-d20b-4370-a05d-11db025a0064', 200, 0, NULL, NULL),
  ('d53d4021-9390-4b3a-9e5a-4bf1ff3e5a4c', 300, 0, now(), NULL),
  ('d53d4021-9390-4b3a-9e5a-4bf1ff3e5a4c', 400, 0, now(), now());

SET CLUSTER SETTING sql.stats.virtual_computed_columns.enabled = 'on';

ANALYZE orders;

Here's the stack trace on master (e0d587c):

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x2 addr=0x0 pc=0x103750c28]

goroutine 4550 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0x1400529bac8?, {0x10979e250, 0x14004a16d80})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:231 +0x68
panic({0x108d762e0?, 0x10ce759e0?})
	GOROOT/src/runtime/panic.go:770 +0x124
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).ReadTimestamp(0x1400529bca8?)
	github.com/cockroachdb/cockroach/pkg/kv/txn.go:401 +0x28
github.com/cockroachdb/cockroach/pkg/sql/catalog/descs.(*leasedDescriptors).getByID(0x14005667208, {0x10979e250, 0x14005760690}, {0x10976b0c0, 0x0}, 0x68)
	github.com/cockroachdb/cockroach/pkg/sql/catalog/descs/leased_descriptors.go:121 +0x64
github.com/cockroachdb/cockroach/pkg/sql/catalog/descs.(*byIDLookupContext).lookupLeased(0x1400529be00, 0x68)
	github.com/cockroachdb/cockroach/pkg/sql/catalog/descs/descriptor.go:376 +0xc8
github.com/cockroachdb/cockroach/pkg/sql/catalog/descs.getDescriptorsByID({0x10979e250, 0x14005760690}, 0x140056671e0, 0x0, {{0x8?, 0x0?}, {0x0?, 0x0?, 0x0?, 0x0?}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/catalog/descs/descriptor.go:180 +0x74c
github.com/cockroachdb/cockroach/pkg/sql/catalog/descs.ByIDGetter.Desc({{0x0?, 0x140056671e0?}, {{0x80?, 0x0?}, {0x0?, 0x0?, 0x0?, 0x0?}, {0x8?, 0x9?, ...}}}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/catalog/descs/getters.go:48 +0xc8
github.com/cockroachdb/cockroach/pkg/sql/catalog/descs.ByIDGetter.Type({{0x0?, 0x140056671e0?}, {{0xd8?, 0xc0?}, {0x29?, 0x5?, 0x40?, 0x1?}, {0x18?, 0x38?, ...}}}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/catalog/descs/getters.go:106 +0x38
github.com/cockroachdb/cockroach/pkg/sql.(*schemaResolver).GetTypeDescriptor(0x1400535c608, {0x10979e250, 0x14005760690}, 0x38f3e20?)
	github.com/cockroachdb/cockroach/pkg/sql/schema_resolver.go:379 +0xac
github.com/cockroachdb/cockroach/pkg/sql/catalog/typedesc.ResolveHydratedTByOID({0x10979e250, 0x14005760690}, 0x1668ebc8?, {0x109755aa0, 0x1400535c608})
	github.com/cockroachdb/cockroach/pkg/sql/catalog/typedesc/hydrate.go:39 +0x84
github.com/cockroachdb/cockroach/pkg/sql.(*schemaResolver).ResolveTypeByOID(...)
	github.com/cockroachdb/cockroach/pkg/sql/schema_resolver.go:371
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.ResolveType({0x10979e250?, 0x14005760690?}, {0x10974ece0?, 0x140075ae264?}, {0x10976ae40?, 0x1400535c608?})
	github.com/cockroachdb/cockroach/pkg/sql/sem/tree/type_name.go:145 +0x1a8
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*AnnotateTypeExpr).TypeCheck(0x14005761080, {0x10979e250, 0x14005760690}, 0x1400535c9e8, 0x0?)
	github.com/cockroachdb/cockroach/pkg/sql/sem/tree/type_check.go:770 +0x68
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.typeCheckSameTypedExprs({0x10979e250, 0x14005760690}, 0x1400535c9e8, 0x140039260c0, {0x14004c5dc00, 0x4, 0x4})
	github.com/cockroachdb/cockroach/pkg/sql/sem/tree/type_check.go:2832 +0x4f0
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*CaseExpr).TypeCheck(0x14004c5dbc0, {0x10979e250, 0x14005760690}, 0x1400535c9e8, 0x140039260c0)
	github.com/cockroachdb/cockroach/pkg/sql/sem/tree/type_check.go:503 +0x3cc
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.TypeCheck({0x10979e250?, 0x14005760690?}, {0x10979ecd0?, 0x14004c5dbc0?}, 0x7?, 0x0?)
	github.com/cockroachdb/cockroach/pkg/sql/sem/tree/type_check.go:350 +0x98
github.com/cockroachdb/cockroach/pkg/sql/catalog/schemaexpr.MakeComputedExprs({0x10979e250, 0x14005760690}, {0x14002074c60, 0x1, 0x140028e5ea0?}, {0x14002c58fc0, 0x7, 0x7}, {0x10988e860, 0x14003a76c08}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/catalog/schemaexpr/computed_column.go:256 +0x47c
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).createStatsPlan(_, {_, _}, _, _, {_, _}, {_, _, _}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_plan_stats.go:457 +0x4c4
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).createPlanForCreateStats(_, {_, _}, _, _, _, {{0x0, 0x0}, {{0x1400192d274, 0x6}, ...}, ...})
	github.com/cockroachdb/cockroach/pkg/sql/distsql_plan_stats.go:634 +0x394
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).planAndRunCreateStats(0x14002380640, {0x10979e250?, 0x14004a170e0?}, 0x1400535cad8, 0x14003a32dd0, 0x1400535c9e8, 0x1400644ba40, 0x140058dd380, 0x140057e87c0)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_plan_stats.go:649 +0x114
github.com/cockroachdb/cockroach/pkg/sql.(*createStatsResumer).Resume.func1({0x10979e250, 0x14004a170e0}, {0x109806480, 0x1400644bb80})
	github.com/cockroachdb/cockroach/pkg/sql/create_stats.go:672 +0x264
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn.func1({0x10979e250?, 0x14004a170e0?}, 0x1098083b8?)
	github.com/cockroachdb/cockroach/pkg/sql/internal.go:1794 +0x38
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn.func4({0x10979e250, 0x14004a170e0}, 0x1400644ba40)
	github.com/cockroachdb/cockroach/pkg/sql/internal.go:1881 +0x260
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec(0x1400644ba40, {0x10979e250, 0x14004a170e0}, 0x14004c5d700)
	github.com/cockroachdb/cockroach/pkg/kv/txn.go:1049 +0x74
github.com/cockroachdb/cockroach/pkg/kv.runTxn({0x10979e250, 0x14004a170e0}, 0x1400644ba40, 0x1?)
	github.com/cockroachdb/cockroach/pkg/kv/db.go:1089 +0x3c
github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl(0x1400529ee28?, {0x10979e250, 0x14004a170e0}, 0x9401380?, 0x1?, 0x0, 0x14004c5d700)
	github.com/cockroachdb/cockroach/pkg/kv/db.go:1052 +0xa8
github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn(...)
	github.com/cockroachdb/cockroach/pkg/kv/db.go:1027
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn(0x140045b0f30, {0x10979e250, 0x14004a170e0}, 0x140020749b0, {0x0, 0x0, 0x140058dd301?})
	github.com/cockroachdb/cockroach/pkg/sql/internal.go:1868 +0x230
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn(0x140045b0f30, {0x10979e250, 0x14004a170e0}, 0x14004c5d6c0, {0x0, 0x0, 0x0})
	github.com/cockroachdb/cockroach/pkg/sql/internal.go:1795 +0x90
github.com/cockroachdb/cockroach/pkg/sql.(*createStatsResumer).Resume(0x1400651efb0, {0x10979e250, 0x14004a170e0}, {0x1093edb80?, 0x1400354d910})
	github.com/cockroachdb/cockroach/pkg/sql/create_stats.go:655 +0x2a0
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2(0x140058dd380?, 0x0?, 0x1400529f8f8, {0x10978ca90?, 0x1400651efb0?}, {0x10979e250?, 0x14004a170e0?}, {0x1093edb80?, 0x1400354d910?})
	github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1632 +0xec
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine(0x14003a80008, {0x10979e250, 0x14004a17020}, {0x1093edb80, 0x1400354d910}, {0x10978ca90, 0x1400651efb0}, 0x140058dd380, {0x106f867ee, 0x7}, ...)
	github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1633 +0x83c
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob(0x14003a80008, {0x10979e250, 0x14004a17020}, {0x10978ca90, 0x1400651efb0}, 0x140058dd380, {0x106f867ee, 0x7}, {0x1400858c888, 0x16})
	github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:456 +0x3e4
github.com/cockroachdb/cockroach/pkg/jobs.(*StartableJob).Start.func2({0x1400416d808?, 0x14004a16750?})
	github.com/cockroachdb/cockroach/pkg/jobs/jobs.go:828 +0xdc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:485 +0x128
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 3198
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:476 +0x31c
# michae2@michae2-crl-m3 2 ~crdb %

Jira issue: CRDB-38560

The text was updated successfully, but these errors were encountered:

CREATE STATISTICS executes in multiple layers: the "outer" layer is a normal execution of the CREATE STATISTICS statement. During this execution, we create, start, and await a CreateStats job. The CreateStats job in turn starts an "inner" layer which uses a separate internal transaction to plan and execute distributed statistics collection. Within the inner layer, we were using the job's planner (JobExecContext) to plan distributed stats collection. This planner has no associated transaction. If it weren't for user-defined types, this would be fine, but user-defined types must be resolved using a transaction. We had a hack in place to set the evalCtx.Txn to the internal transaction in order to execute collection on normal columns with user-defined type. But this hack did not work for virtual computed columns with user-defined type, because type-checking their expressions uses more facilites of the planner than just the evalCtx. (Specifically the schemaResolver.) So, instead of setting the evalCtx.Txn, we create a new "inner" planner that is associated with the internal transaction of the inner layer. This works for all columns with user-defined type. Fixes: cockroachdb#123821 Release note (bug fix): Fix a crash that could occur when planning statistics collection on a table with a virtual computed column using a user-defined type.

CREATE STATISTICS executes in multiple layers: the "outer" layer is a normal execution of the CREATE STATISTICS statement. During this execution, we create, start, and await a CreateStats job. The CreateStats job in turn starts an "inner" layer which uses a separate internal transaction to plan and execute distributed statistics collection. Within the inner layer, we were using the job's planner (JobExecContext) to plan distributed stats collection. This planner has no associated transaction. If it weren't for user-defined types, this would be fine, but user-defined types must be resolved using a transaction. We had a hack in place to set the evalCtx.Txn to the internal transaction in order to execute collection on normal columns with user-defined type. But this hack did not work for virtual computed columns with user-defined type, because type-checking their expressions uses more facilites of the planner than just the evalCtx. (Specifically the schemaResolver.) So, instead of setting the evalCtx.Txn, we create a new "inner" planner that is associated with the internal transaction of the inner layer. This works for all columns with user-defined type. Fixes: cockroachdb#123821 Release note (bug fix): Fix a crash introduced in v23.2.5 and v24.1.0-beta.2 that could occur when planning statistics collection on a table with a virtual computed column using a user-defined type when the newly-introduced cluster setting `sql.stats.virtual_computed_columns.enabled` is set to true. (The setting was introduced in v23.2.4 and v24.1.0-alpha.1.)

123926: sql: fix crash when planning stats collection on virtual col with UDT r=yuzefovich a=michae2 CREATE STATISTICS executes in multiple layers: the "outer" layer is a normal execution of the CREATE STATISTICS statement. During this execution, we create, start, and await a CreateStats job. The CreateStats job in turn starts an "inner" layer which uses a separate internal transaction to plan and execute distributed statistics collection. Within the inner layer, we were using the job's planner (JobExecContext) to plan distributed stats collection. This planner has no associated transaction. If it weren't for user-defined types, this would be fine, but user-defined types must be resolved using a transaction. We had a hack in place to set the evalCtx.Txn to the internal transaction in order to execute collection on normal columns with user-defined type. But this hack did not work for virtual computed columns with user-defined type, because type-checking their expressions uses more facilites of the planner than just the evalCtx. (Specifically the schemaResolver.) So, instead of setting the evalCtx.Txn, we create a new "inner" planner that is associated with the internal transaction of the inner layer. This works for all columns with user-defined type. Fixes: #123821 Release note (bug fix): Fix a crash introduced in v23.2.5 and v24.1.0-beta.2 that could occur when planning statistics collection on a table with a virtual computed column using a user-defined type when the newly-introduced cluster setting `sql.stats.virtual_computed_columns.enabled` is set to true. (The setting was introduced in v23.2.4 and v24.1.0-alpha.1.) Co-authored-by: Michael Erickson <michae2@cockroachlabs.com>

CREATE STATISTICS executes in multiple layers: the "outer" layer is a normal execution of the CREATE STATISTICS statement. During this execution, we create, start, and await a CreateStats job. The CreateStats job in turn starts an "inner" layer which uses a separate internal transaction to plan and execute distributed statistics collection. Within the inner layer, we were using the job's planner (JobExecContext) to plan distributed stats collection. This planner has no associated transaction. If it weren't for user-defined types, this would be fine, but user-defined types must be resolved using a transaction. We had a hack in place to set the evalCtx.Txn to the internal transaction in order to execute collection on normal columns with user-defined type. But this hack did not work for virtual computed columns with user-defined type, because type-checking their expressions uses more facilites of the planner than just the evalCtx. (Specifically the schemaResolver.) So, instead of setting the evalCtx.Txn, we create a new "inner" planner that is associated with the internal transaction of the inner layer. This works for all columns with user-defined type. Fixes: cockroachdb#123821 Release note (bug fix): Fix a crash introduced in v24.1.0-beta.2 that could occur when planning statistics collection on a table with a virtual computed column using a user-defined type when the newly-introduced cluster setting `sql.stats.virtual_computed_columns.enabled` is set to true. (The setting was introduced in v24.1.0-alpha.1 default true.)

CREATE STATISTICS executes in multiple layers: the "outer" layer is a normal execution of the CREATE STATISTICS statement. During this execution, we create, start, and await a CreateStats job. The CreateStats job in turn starts an "inner" layer which uses a separate internal transaction to plan and execute distributed statistics collection. Within the inner layer, we were using the job's planner (JobExecContext) to plan distributed stats collection. This planner has no associated transaction. If it weren't for user-defined types, this would be fine, but user-defined types must be resolved using a transaction. We had a hack in place to set the evalCtx.Txn to the internal transaction in order to execute collection on normal columns with user-defined type. But this hack did not work for virtual computed columns with user-defined type, because type-checking their expressions uses more facilites of the planner than just the evalCtx. (Specifically the schemaResolver.) So, instead of setting the evalCtx.Txn, we create a new "inner" planner that is associated with the internal transaction of the inner layer. This works for all columns with user-defined type. Fixes: cockroachdb#123821 Release note (bug fix): Fix a crash introduced in v23.2.5 that could occur when planning statistics collection on a table with a virtual computed column using a user-defined type when the newly-introduced cluster setting `sql.stats.virtual_computed_columns.enabled` is set to true. (The setting was introduced in v23.2.4 default false.)

Move the definition of evalCtx down to match the backports of cockroachdb#123926. Informs: cockroachdb#123821 Release note: None

124088: sql: fix merge skew in sql.(*createStatsResumer).Resume r=yuzefovich a=michae2 Move the definition of evalCtx down to match the backports of #123926. Informs: #123821 Release note: None Co-authored-by: Michael Erickson <michae2@cockroachlabs.com>

CREATE STATISTICS executes in multiple layers: the "outer" layer is a normal execution of the CREATE STATISTICS statement. During this execution, we create, start, and await a CreateStats job. The CreateStats job in turn starts an "inner" layer which uses a separate internal transaction to plan and execute distributed statistics collection. Within the inner layer, we were using the job's planner (JobExecContext) to plan distributed stats collection. This planner has no associated transaction. If it weren't for user-defined types, this would be fine, but user-defined types must be resolved using a transaction. We had a hack in place to set the evalCtx.Txn to the internal transaction in order to execute collection on normal columns with user-defined type. But this hack did not work for virtual computed columns with user-defined type, because type-checking their expressions uses more facilites of the planner than just the evalCtx. (Specifically the schemaResolver.) So, instead of setting the evalCtx.Txn, we create a new "inner" planner that is associated with the internal transaction of the inner layer. This works for all columns with user-defined type. Fixes: cockroachdb#123821 Release note (bug fix): Fix a crash introduced in v24.1.0-beta.2 that could occur when planning statistics collection on a table with a virtual computed column using a user-defined type when the newly-introduced cluster setting `sql.stats.virtual_computed_columns.enabled` is set to true. (The setting was introduced in v24.1.0-alpha.1 default true.)

michae2 added C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. A-sql-typing SQLtype inference, typing rules, type compatibility. A-sql-table-stats Table statistics (and their automatic refresh). T-sql-queries SQL Queries Team labels May 8, 2024

michae2 self-assigned this May 8, 2024

michae2 added GA-blocker branch-release-24.1 Used to mark GA and release blockers and technical advisories for 24.1 branch-release-24.1.0-rc Used to mark GA and release blockers and technical advisories for 24.1.0-rc labels May 8, 2024

michae2 mentioned this issue May 10, 2024

sql: fix crash when planning stats collection on virtual col with UDT #123926

Merged

craig bot closed this as completed in 568c525 May 13, 2024

michae2 mentioned this issue May 13, 2024

release-24.1: sql: fix crash when planning stats collection on virtual col with UDT #124060

Merged

michae2 mentioned this issue May 13, 2024

release-24.1.0-rc: sql: fix crash when planning stats collection on virtual col with UDT #124064

Merged

michae2 mentioned this issue May 13, 2024

release-23.2: sql: fix crash when planning stats collection on virtual col with UDT #124080

Merged

michae2 added a commit to michae2/cockroach that referenced this issue May 13, 2024

sql: fix merge skew in sql.(*createStatsResumer).Resume

921731b

Move the definition of evalCtx down to match the backports of cockroachdb#123926. Informs: cockroachdb#123821 Release note: None

michae2 mentioned this issue May 13, 2024

sql: fix merge skew in sql.(*createStatsResumer).Resume #124088

Merged

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

sql: crash when collecting stats on virtual computed column with enum type #123821

sql: crash when collecting stats on virtual computed column with enum type #123821

michae2 commented May 8, 2024 •

edited by cockroach-jira-scripts

sql: crash when collecting stats on virtual computed column with enum type #123821

sql: crash when collecting stats on virtual computed column with enum type #123821

Comments

michae2 commented May 8, 2024 • edited by cockroach-jira-scripts

michae2 commented May 8, 2024 •

edited by cockroach-jira-scripts