Skip to content

Commit

Permalink
sql-schema-describer+crdb: parse unicode escape sequences
Browse files Browse the repository at this point in the history 
closes prisma/prisma#13842
  • Loading branch information
@tomhoule
tomhoule committed Jun 22, 2022
1 parent bd4d62f commit f049999
Show file tree
Hide file tree
Showing 8 changed files with 141 additions and 1,455 deletions.
1,417 changes: 18 additions & 1,399 deletions Cargo.lock
View file

Large diffs are not rendered by default.

24 changes: 12 additions & 12 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,18 @@ members = [
"migration-engine/json-rpc-api-build",
"migration-engine/migration-engine-tests",
"migration-engine/qe-setup",
"query-engine/connectors/query-connector",
"query-engine/connectors/sql-query-connector",
"query-engine/connectors/mongodb-query-connector",
"query-engine/core",
"query-engine/dmmf",
"query-engine/query-engine",
"query-engine/query-engine-node-api",
"query-engine/connector-test-kit-rs/query-engine-tests",
"query-engine/prisma-models",
"query-engine/request-handlers",
"query-engine/schema",
"query-engine/schema-builder",
# "query-engine/connectors/query-connector",
# "query-engine/connectors/sql-query-connector",
# "query-engine/connectors/mongodb-query-connector",
# "query-engine/core",
# "query-engine/dmmf",
# "query-engine/query-engine",
# "query-engine/query-engine-node-api",
# "query-engine/connector-test-kit-rs/query-engine-tests",
# "query-engine/prisma-models",
# "query-engine/request-handlers",
# "query-engine/schema",
# "query-engine/schema-builder",
"prisma-fmt",
"libs/datamodel/core",
"libs/datamodel/diagnostics",
Expand Down
4 changes: 2 additions & 2 deletions libs/datamodel/schema-ast/src/parser/datamodel.pest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,9 @@ numeric_literal = @{ ("-")? ~ ASCII_DIGIT+ ~("." ~ ASCII_DIGIT+)? }
// References:
// - https://www.ietf.org/rfc/rfc4627.txt
// - https://www.json.org/json-en.html
UNICODE_CONTROL_CHARACTER = _{ '\u{0000}'..'\u{001F}' }
ASCII_CONTROL_CHARACTER = _{ '\u{0000}'..'\u{001F}' }
string_escape = _{ "\\" ~ ANY }
string_content = @{ (string_escape | !("\"" | UNICODE_CONTROL_CHARACTER) ~ ANY)* }
string_content = @{ (string_escape | !("\"" | ASCII_CONTROL_CHARACTER) ~ ANY)* }
string_literal = ${ "\"" ~ string_content ~ "\"" }

constant_literal = @{ path_identifier }
3 changes: 1 addition & 2 deletions libs/datamodel/schema-ast/src/parser/parse_arguments.rs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
use diagnostics::Diagnostics;

use super::{
helpers::{parsing_catch_all, ToIdentifier, Token, TokenExtensions},
parse_expression::parse_expression,
Rule,
};
use crate::ast;
use diagnostics::Diagnostics;

pub(crate) fn parse_arguments_list(
token: &Token<'_>,
Expand Down
60 changes: 58 additions & 2 deletions libs/sql-schema-describer/src/postgres/default.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,15 +162,31 @@ fn parse_string_value(parser: &mut Parser<'_>) -> Option<String> {
Some(out)
}
Token::CStyleStringLiteral => {
// Reference for CockroachDB: https://www.cockroachlabs.com/docs/stable/sql-constants.html#string-literals-with-character-escapes
// octal and hexadecimal escape sequences seem not to be returned by crdb in defaults,
// so we do not try parsing them.
let s = parser.expect(Token::CStyleStringLiteral)?;
let mut out = String::with_capacity(s.len() - 3); // exclude the quotes and E
let mut chars = s[2..(s.len() - 1)].chars();
let mut chars = s[2..(s.len() - 1)].chars().peekable();

loop {
match chars.next() {
Some('\\') => {
let next_char = chars.next().expect("invariant");

match next_char {
'a' => {
out.push('\u{7}');
}
'b' => {
out.push('\u{8}');
}
'v' => {
out.push('\u{11}');
}
'f' => {
out.push('\u{12}');
}
'n' => {
out.push('\n');
}
Expand All @@ -180,6 +196,28 @@ fn parse_string_value(parser: &mut Parser<'_>) -> Option<String> {
't' => {
out.push('\t');
}
'u' => {
// take 4
let mut codepoint = 0u32;
for i in 0..4 {
let nibble_offset = 3 - i;
// expect crdb to return valid codepoints
let next_digit = chars.next().unwrap().to_digit(16).unwrap();
codepoint += next_digit << (nibble_offset * 4);
}
out.push(char::from_u32(codepoint).unwrap()); // assume crdb returns valid codepoints
}
'U' => {
// take 8
let mut codepoint = 0u32;
for i in 0..8 {
let nibble_offset = 7 - i;
// expect crdb to return valid codepoints
let next_digit = chars.next().unwrap().to_digit(16).unwrap();
codepoint += next_digit << (nibble_offset * 4);
}
out.push(char::from_u32(codepoint).unwrap()); // assume crdb returns valid codepoints
}
_ => out.push(next_char),
}
}
Expand Down Expand Up @@ -207,7 +245,7 @@ fn parse_identifier(parser: &mut Parser<'_>) -> Option<String> {
match parser.peek_token()? {
Token::DoubleQuotedIdentifier => {
let s = parser.expect(Token::DoubleQuotedIdentifier)?;
Some(s[1..(s.len() - 1)].replace(r#"\""#, r#"""#))
Some(parse_double_quoted_string_contents(s))
}
Token::Identifier => {
let s = parser.expect(Token::Identifier)?;
Expand All @@ -217,6 +255,24 @@ fn parse_identifier(parser: &mut Parser<'_>) -> Option<String> {
}
}

fn parse_double_quoted_string_contents(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut chars = s[1..(s.len() - 1)].chars().peekable();

while let Some(c) = chars.next() {
match c {
'\\' => {
out.push(chars.next().unwrap());
}
other => {
out.push(other);
}
}
}

out
}

fn parse_int_default(parser: &mut Parser<'_>) -> Option<DefaultValue> {
match parser.peek_token()? {
Token::Digits => {
Expand Down
11 changes: 9 additions & 2 deletions libs/sql-schema-describer/src/postgres/default/c_style_scalar_lists.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ pub(super) fn parse_array_literal(input: &str, tpe: &ColumnType) -> Option<Vec<P
}
Token::DoubleQuotedIdentifier => {
let s = parser.expect(Token::DoubleQuotedIdentifier)?;
let unquoted = s.replace(r#"\""#, r#"""#);
let unquoted = super::parse_double_quoted_string_contents(s);
values.push(parse_literal(&unquoted, tpe)?);
}
Token::StringLiteral => {
Expand Down Expand Up @@ -75,7 +75,14 @@ fn parse_literal(s: &str, tpe: &ColumnType) -> Option<PrismaValue> {
_ => None,
},
ColumnTypeFamily::Json => Some(PrismaValue::Json(s.to_owned())),
ColumnTypeFamily::Enum(_) => Some(PrismaValue::Enum(s.to_owned())),
ColumnTypeFamily::Enum(_) => {
let tokens = tokenize(s);
let mut parser = Parser::new(s, &tokens);
match super::parse_string_value(&mut parser) {
Some(string_contents) => Some(PrismaValue::Enum(string_contents)),
None => Some(PrismaValue::Enum(s.to_owned())),
}
}
ColumnTypeFamily::DateTime
| ColumnTypeFamily::Binary
| ColumnTypeFamily::Uuid
Expand Down
74 changes: 39 additions & 35 deletions migration-engine/migration-engine-tests/tests/migrations/cockroachdb.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1144,47 +1144,51 @@ fn alter_sequence(api: TestApi) {
}

// https://github.com/prisma/prisma/issues/13842
// #[test_connector(tags(CockroachDb))]
// fn mapped_enum_defaults_must_work(api: TestApi) {
// let schema = r#"
// datasource db {
// provider = "postgres"
// url = "postgres://meowmeowmeow"
// }
#[test_connector(tags(CockroachDb))]
fn mapped_enum_defaults_must_work(api: TestApi) {
let schema = r#"
datasource db {
provider = "cockroachdb"
url = "postgres://meowmeowmeow"
}
// enum Color {
// Red @map("0")
// Green @map("Grün")
// Blue @map("Blu")
// }
enum Color {
Red @map("0")
Green @map("Grün")
Blue @map("Blu")
Annoyed @map("pfuh 🙄...")
}
// model Test {
// id Int @id
// mainColor Color @default(Green)
// secondaryColor Color @default(Red)
// colorOrdering Color[] @default([Blue, Red, Green, Red, Blue, Red])
// }
// "#;
model Test {
id Int @id
mainColor Color @default(Green)
secondaryColor Color @default(Red)
colorOrdering Color[] @default([Blue, Red, Green, Red, Blue, Annoyed, Red])
}
"#;

// let expect = expect![[r#"
// -- CreateEnum
// CREATE TYPE "Color" AS ENUM ('0', 'Grün', 'Blu');
let expect = expect![[r#"
-- CreateEnum
CREATE TYPE "Color" AS ENUM ('0', 'Grün', 'Blu', 'pfuh 🙄...');
// -- CreateTable
// CREATE TABLE "Test" (
// "id" INT4 NOT NULL,
// "mainColor" "Color" NOT NULL DEFAULT 'Grün',
// "secondaryColor" "Color" NOT NULL DEFAULT '0',
// "colorOrdering" "Color"[] DEFAULT array['Blu', '0', 'Grün', '0', 'Blu', '0']::"Color"[],
-- CreateTable
CREATE TABLE "Test" (
"id" INT4 NOT NULL,
"mainColor" "Color" NOT NULL DEFAULT 'Grün',
"secondaryColor" "Color" NOT NULL DEFAULT '0',
"colorOrdering" "Color"[] DEFAULT ARRAY['Blu', '0', 'Grün', '0', 'Blu', 'pfuh 🙄...', '0']::"Color"[],
// CONSTRAINT "Test_pkey" PRIMARY KEY ("id")
// );
// "#]];
// api.expect_sql_for_schema(schema, &expect);
CONSTRAINT "Test_pkey" PRIMARY KEY ("id")
);
"#]];
api.expect_sql_for_schema(schema, &expect);

// api.schema_push(schema).send().assert_green();
// api.schema_push(schema).send().assert_green().assert_no_steps();
// }
api.schema_push(schema)
.send()
.assert_green()
.assert_has_executed_steps();
api.schema_push(schema).send().assert_green().assert_no_steps();
}

// https://github.com/prisma/prisma/issues/12095
#[test_connector(tags(CockroachDb))]
Expand Down
3 changes: 2 additions & 1 deletion migration-engine/migration-engine-tests/tests/migrations/enums.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,7 @@ fn mapped_enum_defaults_must_work(api: TestApi) {
Red @map("0")
Green @map("Grün")
Blue @map("Blu")
Annoyed @map("pfuh 🙄...")
}
model Test {
Expand All @@ -588,7 +589,7 @@ fn mapped_enum_defaults_must_work(api: TestApi) {

let expect = expect![[r#"
-- CreateEnum
CREATE TYPE "Color" AS ENUM ('0', 'Grün', 'Blu');
CREATE TYPE "Color" AS ENUM ('0', 'Grün', 'Blu', 'pfuh 🙄...');
-- CreateTable
CREATE TABLE "Test" (
Expand Down

0 comments on commit f049999

Please sign in to comment.