From 50112e7650a727a96a5fed491fd0a5e339f9fc11 Mon Sep 17 00:00:00 2001 From: Viktoria Date: Tue, 24 Jan 2023 12:40:36 +0300 Subject: [PATCH] Add to clickhouse README.md database creation --- database/clickhouse/README.md | 1 + .../migrations/003_create_database.down.sql | 10 +++ .../migrations/003_create_database.up.sql | 81 +++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 database/clickhouse/examples/migrations/003_create_database.down.sql create mode 100644 database/clickhouse/examples/migrations/003_create_database.up.sql diff --git a/database/clickhouse/README.md b/database/clickhouse/README.md index 96ad79f1c..a0a6b7ca3 100644 --- a/database/clickhouse/README.md +++ b/database/clickhouse/README.md @@ -23,3 +23,4 @@ * Clickhouse cluster mode is not officially supported, since it's not tested right now, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`: * When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified. See the docs regarding [replicated table engines](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#table_engines-replication). * When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster. +* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `deafault` table, so you can't use `USE ` inside migration. In this case you can not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql)) \ No newline at end of file diff --git a/database/clickhouse/examples/migrations/003_create_database.down.sql b/database/clickhouse/examples/migrations/003_create_database.down.sql new file mode 100644 index 000000000..8e49cfbfa --- /dev/null +++ b/database/clickhouse/examples/migrations/003_create_database.down.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS driver_ratings ON CLUSTER cluster_1; +DROP TABLE IF EXISTS user_ratings ON CLUSTER cluster_1; +DROP TABLE IF EXISTS orders ON CLUSTER cluster_1; +DROP TABLE IF EXISTS driver_ratings_queue ON CLUSTER cluster_1; +DROP TABLE IF EXISTS user_ratings_queue ON CLUSTER cluster_1; +DROP TABLE IF EXISTS orders_queue ON CLUSTER cluster_1; +DROP VIEW IF EXISTS user_ratings_queue_mv ON CLUSTER cluster_1; +DROP VIEW IF EXISTS driver_ratings_queue_mv ON CLUSTER cluster_1; +DROP VIEW IF EXISTS orders_queue_mv ON CLUSTER cluster_1; +DROP DATABASE IF EXISTS analytics ON CLUSTER cluster_1; diff --git a/database/clickhouse/examples/migrations/003_create_database.up.sql b/database/clickhouse/examples/migrations/003_create_database.up.sql new file mode 100644 index 000000000..b49d66913 --- /dev/null +++ b/database/clickhouse/examples/migrations/003_create_database.up.sql @@ -0,0 +1,81 @@ +CREATE DATABASE IF NOT EXISTS analytics ON CLUSTER cluster_1; + +CREATE TABLE IF NOT EXISTS analytics.driver_ratings ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String, + inserted_time DateTime DEFAULT now() +) ENGINE = ReplicatedMergeTree +PARTITION BY driverID +ORDER BY (inserted_time); + +CREATE TABLE analytics.driver_ratings_queue ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String +) ENGINE = Kafka +SETTINGS kafka_broker_list = 'broker:9092', + kafka_topic_list = 'driver-ratings', + kafka_group_name = 'rating_readers', + kafka_format = 'Avro', + kafka_max_block_size = 1048576; + +CREATE MATERIALIZED VIEW analytics.driver_ratings_queue_mv ON CLUSTER cluster_1 TO analytics.driver_ratings AS +SELECT rate, userID, driverID, orderID +FROM analytics.driver_ratings_queue; + +CREATE TABLE IF NOT EXISTS analytics.user_ratings ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String, + inserted_time DateTime DEFAULT now() +) ENGINE = ReplicatedMergeTree + PARTITION BY userID + ORDER BY (inserted_time); + +CREATE TABLE analytics.user_ratings_queue ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String +) ENGINE = Kafka +SETTINGS kafka_broker_list = 'broker:9092', + kafka_topic_list = 'user-ratings', + kafka_group_name = 'rating_readers', + kafka_format = 'JSON', + kafka_max_block_size = 1048576; + +CREATE MATERIALIZED VIEW analytics.user_ratings_queue_mv ON CLUSTER cluster_1 TO analytics.user_ratings AS +SELECT rate, userID, driverID, orderID +FROM analytics.user_ratings_queue; + +CREATE TABLE IF NOT EXISTS analytics.orders ON CLUSTER cluster_1( + from_place String, + to_place String, + userID Int64, + driverID String, + orderID String, + inserted_time DateTime DEFAULT now() +) ENGINE = ReplicatedMergeTree + PARTITION BY driverID + ORDER BY (inserted_time); + +CREATE TABLE analytics.orders_queue ON CLUSTER cluster_1( + from_place String, + to_place String, + userID Int64, + driverID String, + orderID String +) ENGINE = Kafka +SETTINGS kafka_broker_list = 'broker:9092', + kafka_topic_list = 'orders', + kafka_group_name = 'order_readers', + kafka_format = 'Avro', + kafka_max_block_size = 1048576; + +CREATE MATERIALIZED VIEW analytics.orders_queue_mv ON CLUSTER cluster_1 TO orders AS +SELECT from_place, to_place, userID, driverID, orderID +FROM analytics.orders_queue;