Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

RFC: Lexing is Greedy #2163

Merged
merged 1 commit into from Sep 12, 2019
Merged

RFC: Lexing is Greedy #2163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
128 changes: 128 additions & 0 deletions src/language/__tests__/lexer-test.js
View file
Open in desktop
Expand Up @@ -49,6 +49,65 @@ describe('Lexer', () => {
});
});

it('tracks line breaks', () => {
expect(lexOne('foo')).to.contain({
kind: TokenKind.NAME,
start: 0,
end: 3,
line: 1,
column: 1,
value: 'foo',
});
expect(lexOne('\nfoo')).to.contain({
kind: TokenKind.NAME,
start: 1,
end: 4,
line: 2,
column: 1,
value: 'foo',
});
expect(lexOne('\rfoo')).to.contain({
kind: TokenKind.NAME,
start: 1,
end: 4,
line: 2,
column: 1,
value: 'foo',
});
expect(lexOne('\r\nfoo')).to.contain({
kind: TokenKind.NAME,
start: 2,
end: 5,
line: 2,
column: 1,
value: 'foo',
});
expect(lexOne('\n\rfoo')).to.contain({
kind: TokenKind.NAME,
start: 2,
end: 5,
line: 3,
column: 1,
value: 'foo',
});
expect(lexOne('\r\r\n\nfoo')).to.contain({
kind: TokenKind.NAME,
start: 4,
end: 7,
line: 4,
column: 1,
value: 'foo',
});
expect(lexOne('\n\n\r\rfoo')).to.contain({
kind: TokenKind.NAME,
start: 4,
end: 7,
line: 5,
column: 1,
value: 'foo',
});
});

it('records line and column', () => {
expect(lexOne('\n \r\n \r foo\n')).to.contain({
kind: TokenKind.NAME,
Expand Down Expand Up @@ -164,6 +223,13 @@ describe('Lexer', () => {
});

it('lexes strings', () => {
expect(lexOne('""')).to.contain({
kind: TokenKind.STRING,
start: 0,
end: 2,
value: '',
});

expect(lexOne('"simple"')).to.contain({
kind: TokenKind.STRING,
start: 0,
Expand Down Expand Up @@ -210,6 +276,10 @@ describe('Lexer', () => {
it('lex reports useful string errors', () => {
expectSyntaxError('"', 'Unterminated string.', { line: 1, column: 2 });

expectSyntaxError('"""', 'Unterminated string.', { line: 1, column: 4 });

expectSyntaxError('""""', 'Unterminated string.', { line: 1, column: 5 });

expectSyntaxError('"no end quote', 'Unterminated string.', {
line: 1,
column: 14,
Expand Down Expand Up @@ -287,6 +357,13 @@ describe('Lexer', () => {
});

it('lexes block strings', () => {
expect(lexOne('""""""')).to.contain({
kind: TokenKind.BLOCK_STRING,
start: 0,
end: 6,
value: '',
});

expect(lexOne('"""simple"""')).to.contain({
kind: TokenKind.BLOCK_STRING,
start: 0,
Expand Down Expand Up @@ -538,6 +615,20 @@ describe('Lexer', () => {
column: 2,
});

expectSyntaxError('01', 'Invalid number, unexpected digit after 0: "1".', {
line: 1,
column: 2,
});

expectSyntaxError(
'01.23',
'Invalid number, unexpected digit after 0: "1".',
{
line: 1,
column: 2,
},
);

expectSyntaxError('+1', 'Cannot parse the unexpected character "+".', {
line: 1,
column: 1,
Expand All @@ -548,6 +639,16 @@ describe('Lexer', () => {
column: 3,
});

expectSyntaxError('1e', 'Invalid number, expected digit but got: <EOF>.', {
line: 1,
column: 3,
});

expectSyntaxError('1E', 'Invalid number, expected digit but got: <EOF>.', {
line: 1,
column: 3,
});

expectSyntaxError('1.e1', 'Invalid number, expected digit but got: "e".', {
line: 1,
column: 3,
Expand Down Expand Up @@ -578,6 +679,33 @@ describe('Lexer', () => {
line: 1,
column: 5,
});

expectSyntaxError(
'1.2e3e',
'Invalid number, expected digit but got: "e".',
{
line: 1,
column: 6,
},
);

expectSyntaxError(
'1.2e3.4',
'Invalid number, expected digit but got: ".".',
{
line: 1,
column: 6,
},
);

expectSyntaxError(
'1.23.4',
'Invalid number, expected digit but got: ".".',
{
line: 1,
column: 5,
},
);
});

it('lexes punctuation', () => {
Expand Down
10 changes: 10 additions & 0 deletions src/language/lexer.js
View file
Open in desktop
Expand Up @@ -444,6 +444,16 @@ function readNumber(source, start, firstCode, line, col, prev): Token {
code = body.charCodeAt(++position);
}
position = readDigits(source, position, code);
code = body.charCodeAt(position);
}

// Numbers cannot be followed by . or e
if (code === 46 || code === 69 || code === 101) {
throw syntaxError(
source,
position,
`Invalid number, expected digit but got: ${printCharCode(code)}.`,
);
}

return new Tok(
Expand Down