From a298cccf97686249c971b6815bc5651e6cb59e42 Mon Sep 17 00:00:00 2001 From: Ivan Goncharov Date: Mon, 1 Nov 2021 22:31:52 +0200 Subject: [PATCH] Lexer: fix line & column for multiline BLOCK_STRING tokens Fixes #3353 --- src/language/__tests__/lexer-test.ts | 22 ++++++++++++++++++++++ src/language/lexer.ts | 8 ++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/language/__tests__/lexer-test.ts b/src/language/__tests__/lexer-test.ts index 1e925f73cc..021e2b0b34 100644 --- a/src/language/__tests__/lexer-test.ts +++ b/src/language/__tests__/lexer-test.ts @@ -537,6 +537,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 6, + line: 1, + column: 1, value: '', }); @@ -544,6 +546,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 12, + line: 1, + column: 1, value: 'simple', }); @@ -551,6 +555,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 19, + line: 1, + column: 1, value: ' white space ', }); @@ -558,6 +564,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 22, + line: 1, + column: 1, value: 'contains " quote', }); @@ -565,6 +573,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 32, + line: 1, + column: 1, value: 'contains """ triple quote', }); @@ -572,6 +582,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 16, + line: 1, + column: 1, value: 'multi\nline', }); @@ -579,6 +591,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 28, + line: 1, + column: 1, value: 'multi\nline\nnormalized', }); @@ -586,6 +600,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 32, + line: 1, + column: 1, value: 'unescaped \\n\\r\\b\\t\\f\\u1234', }); @@ -593,6 +609,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 38, + line: 1, + column: 1, value: 'unescaped unicode outside BMP \u{1f600}', }); @@ -600,6 +618,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 19, + line: 1, + column: 1, value: 'slashes \\\\ \\/', }); @@ -615,6 +635,8 @@ describe('Lexer', () => { kind: TokenKind.BLOCK_STRING, start: 0, end: 68, + line: 1, + column: 1, value: 'spans\n multiple\n lines', }); }); diff --git a/src/language/lexer.ts b/src/language/lexer.ts index b21f1faa89..2053dd04d5 100644 --- a/src/language/lexer.ts +++ b/src/language/lexer.ts @@ -739,6 +739,9 @@ function readEscapedCharacter(lexer: Lexer, position: number): EscapeSequence { function readBlockString(lexer: Lexer, start: number): Token { const body = lexer.source.body; const bodyLength = body.length; + const startLine = lexer.line; + const startColumn = 1 + start - lexer.lineStart; + let position = start + 3; let chunkStart = position; let rawValue = ''; @@ -753,11 +756,12 @@ function readBlockString(lexer: Lexer, start: number): Token { body.charCodeAt(position + 2) === 0x0022 ) { rawValue += body.slice(chunkStart, position); - return createToken( - lexer, + return new Token( TokenKind.BLOCK_STRING, start, position + 3, + startLine, + startColumn, dedentBlockStringValue(rawValue), ); }