diff --git a/lib/parser/index.js b/lib/parser/index.js index bc998864..03a3e745 100644 --- a/lib/parser/index.js +++ b/lib/parser/index.js @@ -3,6 +3,7 @@ var TokenType = require('./const').TokenType; var Scanner = require('./scanner'); var List = require('../utils/list'); +var cmpChar = require('./utils').cmpChar; var cmpStr = require('./utils').cmpStr; var endsWith = require('./utils').endsWith; var isHex = require('./utils').isHex; @@ -15,7 +16,7 @@ var SPACE_NODE = { type: 'Space' }; var WHITESPACE = TokenType.Whitespace; var IDENTIFIER = TokenType.Identifier; -var DECIMALNUMBER = TokenType.Number; +var NUMBER = TokenType.Number; var STRING = TokenType.String; var COMMENT = TokenType.Comment; var EXCLAMATIONMARK = TokenType.ExclamationMark; @@ -43,6 +44,7 @@ var LEFTCURLYBRACKET = TokenType.LeftCurlyBracket; var VERTICALLINE = TokenType.VerticalLine; var RIGHTCURLYBRACKET = TokenType.RightCurlyBracket; var TILDE = TokenType.Tilde; +var N = 110; // 'n'.charCodeAt(0) var SCOPE_ATRULE_EXPRESSION = { url: getUri @@ -374,7 +376,7 @@ function getSimpleSelector(nested) { child = getNamespacedIdentifier(false); break; - case DECIMALNUMBER: + case NUMBER: child = getPercentage(getInfo(), readNumber()); break; @@ -553,13 +555,18 @@ function getValue(nested, property) { default: // check for unicode range: U+0F00, U+0F00-0FFF, u+0F00?? if (scanner.tokenType === IDENTIFIER && - scanner.lookupValue(0, 'u') && - scanner.lookupType(1) === PLUSSIGN) { - child = getUnicodeRange(); - } else { - child = getAny(SCOPE_VALUE); + scanner.lookupValue(0, 'u')) { + if ( + scanner.lookupType(1) === PLUSSIGN || ( + scanner.lookupType(1) === NUMBER && + cmpChar(scanner.source, scanner.tokenEnd, PLUSSIGN) + )) { + child = getUnicodeRange(); + break; + } } + child = getAny(SCOPE_VALUE); } if (wasSpace) { @@ -581,43 +588,36 @@ function getAny(scope) { case IDENTIFIER: break; - case FULLSTOP: - case DECIMALNUMBER: case HYPHENMINUS: + var nextType = scanner.lookupType(1); + if (nextType === IDENTIFIER || nextType === HYPHENMINUS) { + break; + } + return getOperator(); + case PLUSSIGN: + return getOperator(); + + case NUMBER: var info = getInfo(); var number = readNumber(); var type = scanner.tokenType; - if (number !== null) { - if (type === PERCENTSIGN) { - return getPercentage(info, number); - } - - if (type === IDENTIFIER) { - return getDimension(info, number); - } - - return { - type: 'Number', - info: info, - value: number - }; + if (type === PERCENTSIGN) { + return getPercentage(info, number); } - if (type === HYPHENMINUS) { - var nextType = scanner.lookupType(1); - if (nextType === IDENTIFIER || nextType === HYPHENMINUS) { - break; - } + if (type === IDENTIFIER) { + return getDimension(info, number); } - if (type === HYPHENMINUS || - type === PLUSSIGN) { - return getOperator(); - } + return { + type: 'Number', + info: info, + value: number + }; - scanner.error('Unexpected input'); + break; default: scanner.error('Unexpected input'); @@ -1160,11 +1160,12 @@ function getOldIEExpression(scope, info, name) { }; } +// https://drafts.csswg.org/css-syntax-3/#urange function scanUnicodeRange() { - var hexStart = scanner.tokenStart; + var hexStart = scanner.tokenStart + 1; // skip + var hexLength = 0; - if (scanner.tokenType === DECIMALNUMBER) { + if (scanner.tokenType === PLUSSIGN || scanner.tokenType === NUMBER) { scanner.next(); } @@ -1172,7 +1173,7 @@ function scanUnicodeRange() { scanner.next(); } - if (scanner.tokenType === DECIMALNUMBER) { + if (scanner.tokenType === NUMBER) { scanner.next(); } @@ -1180,9 +1181,7 @@ function scanUnicodeRange() { scanner.next(); } - hexLength = scanner.tokenStart - hexStart; - - if (hexLength === 0) { + if (scanner.tokenStart === hexStart) { scanner.error('Unexpected input', hexStart); } @@ -1192,16 +1191,31 @@ function scanUnicodeRange() { var code = scanner.source.charCodeAt(i); if (isHex(code) === false && (code !== HYPHENMINUS || wasHyphenMinus)) { - scanner.error('Unexpected input', hexStart + i); + scanner.error('Unexpected input', i); } if (code === HYPHENMINUS) { + // hex sequence shouldn't be an empty + if (hexLength === 0) { + scanner.error('Unexpected input', i); + } + wasHyphenMinus = true; + hexLength = 0; + } else { + hexLength++; + + // to long hex sequence + if (hexLength > 6) { + scanner.error('Unexpected input', i); + } } + } // U+abc??? if (!wasHyphenMinus) { + // consume as many U+003F QUESTION MARK (?) code points as possible for (; hexLength < 6 && !scanner.eof; scanner.next()) { if (scanner.tokenType !== QUESTIONMARK) { break; @@ -1211,6 +1225,11 @@ function scanUnicodeRange() { } } + // If there are any code points left in text, this is an invalid , + if (scanner.tokenType === IDENTIFIER) { + scanner.error('Unexpected input'); + } + return hexLength; } @@ -1218,7 +1237,7 @@ function getUnicodeRange() { var start = scanner.tokenStart; var info = getInfo(); - scanner.skip(2); // U+ or u+ + scanner.next(); // U or u scanUnicodeRange(); return { @@ -1308,6 +1327,7 @@ function getImportant() { return true; } +// https://drafts.csswg.org/css-syntax-3/#the-anb-type function getNthSelector() { var info = getInfo(); var sequence = new List(); @@ -1336,28 +1356,26 @@ function getNthSelector() { value: scanner.substrToCursor(start) }); } else { - if (scanner.tokenType === HYPHENMINUS || - scanner.tokenType === PLUSSIGN) { - sequence.appendData(getOperator()); - readSC(); - } - + var prefix = ''; var start = scanner.tokenStart; var info = getInfo(); - if (scanner.tokenType === DECIMALNUMBER) { + if (scanner.tokenType === HYPHENMINUS || + scanner.tokenType === PLUSSIGN || + scanner.tokenType === NUMBER) { + prefix = scanner.getTokenValue(); scanner.next(); } if (scanner.tokenType === IDENTIFIER) { - if (!cmpStr(scanner.source, scanner.tokenStart, scanner.tokenStart + 1, 'n')) { + if (!cmpChar(scanner.source, scanner.tokenStart, N)) { scanner.error('Unexpected input'); } sequence.appendData({ type: 'Nth', info: info, - value: scanner.source.substring(start, scanner.tokenStart + 1) + value: prefix + scanner.source.charAt(scanner.tokenStart) }); var len = scanner.tokenEnd - scanner.tokenStart; @@ -1394,43 +1412,80 @@ function getNthSelector() { }); } else { + scanner.next(); readSC(); - sequence.appendData({ - type: 'Nth', - info: getInfo(), - value: scanner.getTokenValue() - }); - scanner.eat(DECIMALNUMBER); + if (scanner.tokenType === NUMBER) { + if (cmpChar(scanner.source, scanner.tokenStart, PLUSSIGN) || + cmpChar(scanner.source, scanner.tokenStart, HYPHENMINUS)) { + scanner.error('Unexpected input'); + } + + sequence.appendData({ + type: 'Nth', + info: getInfo(), + value: scanner.getTokenValue() + }); + + scanner.next(); + } } } else { + prefix = ''; scanner.next(); readSC(); if (scanner.tokenType === HYPHENMINUS || scanner.tokenType === PLUSSIGN) { - sequence.appendData(getOperator()); - + info = getInfo(); + prefix = scanner.getTokenValue(); + scanner.next(); readSC(); + } + + if (scanner.tokenType === NUMBER) { + var sign = ''; + + if (cmpChar(scanner.source, scanner.tokenStart, PLUSSIGN) || + cmpChar(scanner.source, scanner.tokenStart, HYPHENMINUS)) { + info = getInfo(); + sign = scanner.source.charAt(scanner.tokenStart); + } + + // prefix or sign should be specified but not both + if (!(prefix === '' ^ sign === '')) { + scanner.error('Unexpected input'); + } + + if (sign) { + scanner.tokenStart++; + } + + sequence.appendData({ + type: 'Operator', + info: info, + value: prefix || sign + }); sequence.appendData({ type: 'Nth', info: getInfo(), value: scanner.getTokenValue() }); - scanner.eat(DECIMALNUMBER); + + scanner.next(); } } } else { - if (scanner.tokenStart === start) { // no number + if (prefix === '' || prefix === '-' || prefix === '+') { // no number scanner.error('Number or identifier is expected'); } sequence.appendData({ type: 'Nth', info: info, - value: scanner.substrToCursor(start) + value: prefix }); } } @@ -1442,27 +1497,11 @@ function getNthSelector() { } function readNumber() { - var start = scanner.tokenStart; - var wasDigits = false; - var offset = 0; - var tokenType = scanner.tokenType; - - if (tokenType === HYPHENMINUS) { - tokenType = scanner.lookupType(++offset); - } - - if (tokenType === DECIMALNUMBER) { - wasDigits = true; - tokenType = scanner.lookupType(++offset); - } - - if (wasDigits) { - scanner.skip(offset); + var number = scanner.getTokenValue(); - return scanner.substrToCursor(start); - } + scanner.eat(NUMBER); - return null; + return number; } // '/' | '*' | ',' | ':' | '+' | '-' @@ -1650,7 +1689,7 @@ function getHash() { scanner.eat(NUMBERSIGN); - if (scanner.tokenType !== DECIMALNUMBER && + if (scanner.tokenType !== NUMBER && scanner.tokenType !== IDENTIFIER) { scanner.error('Number or identifier is expected'); } diff --git a/lib/parser/scanner.js b/lib/parser/scanner.js index 2ee2ee6c..0f79a094 100644 --- a/lib/parser/scanner.js +++ b/lib/parser/scanner.js @@ -26,9 +26,9 @@ var STAR = 42; var SLASH = 47; var BACK_SLASH = 92; var FULLSTOP = TokenType.FullStop; -var E = 'e'.charCodeAt(0); var PLUSSIGN = TokenType.PlusSign; var HYPHENMINUS = TokenType.HyphenMinus; +var E = 101; // 'e'.charCodeAt(0) var MIN_ARRAY_SIZE = 16 * 1024; var OFFSET_MASK = 0x00FFFFFF; @@ -42,6 +42,10 @@ function firstCharOffset(source) { return source.charCodeAt(0) === 0xFEFF ? 1 : 0; } +function isNumber(code) { + return code >= 48 && code <= 57; +} + function computeLines(scanner, source) { var sourceLength = source.length; var start = firstCharOffset(source); @@ -132,29 +136,35 @@ function findDecimalNumberEnd(source, offset) { return offset; } -function findNumberEnd(source, offset, allowFullstop) { +function findNumberEnd(source, offset, allowFraction) { + var code; + offset = findDecimalNumberEnd(source, offset); - if (allowFullstop && offset + 1 < source.length && source.charCodeAt(offset) === FULLSTOP) { - var code = source.charCodeAt(offset + 1); - if (code >= 48 && code <= 57) { + // fraction: .\d+ + if (allowFraction && offset + 1 < source.length && source.charCodeAt(offset) === FULLSTOP) { + code = source.charCodeAt(offset + 1); + + if (isNumber(code)) { offset = findDecimalNumberEnd(source, offset + 1); } } + // exponent: e[+-]\d+ if (offset + 1 < source.length) { - if ((source.charCodeAt(offset) | 32) === E) { - var code = source.charCodeAt(offset + 1); + if ((source.charCodeAt(offset) | 32) === E) { // case insensitive check for `e` + code = source.charCodeAt(offset + 1); + if (code === PLUSSIGN || code === HYPHENMINUS) { if (offset + 2 >= source.length) { return offset; } code = source.charCodeAt(offset + 2); - if (code >= 48 && code <= 57) { + if (isNumber(code)) { offset = findDecimalNumberEnd(source, offset + 3); } } else { - if (code >= 48 && code <= 57) { + if (isNumber(code)) { offset = findDecimalNumberEnd(source, offset + 2); } } @@ -226,6 +236,16 @@ function tokenLayout(scanner, source, startPos) { offset = findCommentEnd(source, offset + 1); tokenCount--; // rewrite prev token } else { + // edge case for -.123 and +.123 + if (code === FULLSTOP && (prevType === PLUSSIGN || prevType === HYPHENMINUS)) { + if (offset + 1 < sourceLength && isNumber(source.charCodeAt(offset + 1))) { + type = NUMBER; + offset = findNumberEnd(source, offset + 2, false); + tokenCount--; + break; + } + } + type = code; offset = offset + 1; } @@ -234,7 +254,9 @@ function tokenLayout(scanner, source, startPos) { case NUMBER: offset = findNumberEnd(source, offset + 1, prevType !== FULLSTOP); - if (prevType === FULLSTOP) { + if (prevType === FULLSTOP || + prevType === HYPHENMINUS || + prevType === PLUSSIGN) { tokenCount--; // rewrite prev token } break; @@ -427,12 +449,14 @@ Scanner.prototype = { }, getTypes: function() { - return Array.prototype.slice.call(this.offsetAndType, 0, this.tokenCount).map(x => TokenName[x]); + return Array.prototype.slice.call(this.offsetAndType, 0, this.tokenCount).map(function(item) { + return TokenName[item >> 24]; + }); } }; // warm up tokenizer to elimitate code branches that never execute // fix soft deoptimizations (insufficient type feedback) -new Scanner('\n\r\r\n\f//""\'\'/*\r\n\f*/1a;.\\31\t\+2{url(a)}'); +new Scanner('\n\r\r\n\f//""\'\'/*\r\n\f*/1a;.\\31\t\+2{url(a);+1.2e3 -.4e-5 .6e+7}'); module.exports = Scanner; diff --git a/lib/parser/utils.js b/lib/parser/utils.js index 34fe3ec5..181c6971 100644 --- a/lib/parser/utils.js +++ b/lib/parser/utils.js @@ -4,6 +4,21 @@ function isHex(code) { (code >= 97 && code <= 102); // a .. f } +function cmpChar(testStr, offset, referenceCode) { + if (offset >= testStr.length) { + return false; + } + + var code = testStr.charCodeAt(offset); + + // code.toLowerCase() + if (code >= 65 && code <= 90) { + code = code | 32; + } + + return code === referenceCode; +} + function cmpStr(testStr, start, end, referenceStr) { if (end - start !== referenceStr.length) { return false; @@ -36,6 +51,7 @@ function endsWith(testStr, referenceStr) { module.exports = { isHex: isHex, + cmpChar: cmpChar, cmpStr: cmpStr, endsWith: endsWith }; diff --git a/test/fixture/parse-errors.json b/test/fixture/parse-errors.json index b5fd5137..4ab58aed 100644 --- a/test/fixture/parse-errors.json +++ b/test/fixture/parse-errors.json @@ -182,6 +182,110 @@ "line": 1, "column": 12 } +}, { + "css": ":nth-child(3 n)", + "error": "RightParenthesis is expected", + "position": { + "offset": 13, + "line": 1, + "column": 14 + } +}, { + "css": ":nth-child(+ 2n)", + "error": "Number or identifier is expected", + "position": { + "offset": 12, + "line": 1, + "column": 13 + } +}, { + "css": ":nth-child(+ 2)", + "error": "Number or identifier is expected", + "position": { + "offset": 12, + "line": 1, + "column": 13 + } +}, { + "css": ":nth-child(3n - +1)", + "error": "Unexpected input", + "position": { + "offset": 16, + "line": 1, + "column": 17 + } +}, { + "css": ":nth-child(3n - -1)", + "error": "Unexpected input", + "position": { + "offset": 16, + "line": 1, + "column": 17 + } +}, { + "css": ":nth-child(3n- +1)", + "error": "Unexpected input", + "position": { + "offset": 15, + "line": 1, + "column": 16 + } +}, { + "css": ":nth-child(3n- -1)", + "error": "Unexpected input", + "position": { + "offset": 15, + "line": 1, + "column": 16 + } +}, { + "css": ":nth-child(3n-2n)", + "error": "Unexpected input", + "position": { + "offset": 15, + "line": 1, + "column": 16 + } +}, { + "css": ":nth-child(3n + -6)", + "error": "Unexpected input", + "position": { + "offset": 16, + "line": 1, + "column": 17 + } +}, { + "css": ":nth-child(3n - 2n)", + "error": "RightParenthesis is expected", + "position": { + "offset": 17, + "line": 1, + "column": 18 + } +}, { + "css": ":nth-child(3n- 2n)", + "error": "RightParenthesis is expected", + "position": { + "offset": 16, + "line": 1, + "column": 17 + } +}, { + "css": ":nth-child(3n-x)", + "error": "Unexpected input", + "position": { + "offset": 14, + "line": 1, + "column": 15 + } +}, { + "css": ":nth-child(3n 1)", + "error": "Unexpected input", + "position": { + "offset": 14, + "line": 1, + "column": 15 + } }, { "css": ":not(.a{)", "error": "Unexpected input", @@ -406,4 +510,60 @@ "line": 1, "column": 10 } +}, { + "css": "a { b: U+-123 }", + "error": "Unexpected input", + "position": { + "offset": 9, + "line": 1, + "column": 10 + } +}, { + "css": "a { b: u+123???? }", + "error": "Unexpected input", + "position": { + "offset": 15, + "line": 1, + "column": 16 + } +}, { + "css": "a { b: u+1234567 }", + "error": "Unexpected input", + "position": { + "offset": 15, + "line": 1, + "column": 16 + } +}, { + "css": "a { b: u+123456z }", + "error": "Unexpected input", + "position": { + "offset": 15, + "line": 1, + "column": 16 + } +}, { + "css": "a { b: u+123456-1234567 }", + "error": "Unexpected input", + "position": { + "offset": 22, + "line": 1, + "column": 23 + } +}, { + "css": "a { b: u+123456-123456z }", + "error": "Unexpected input", + "position": { + "offset": 22, + "line": 1, + "column": 23 + } +}, { + "css": "a { b: u+123456-123??? }", + "error": "Unexpected input", + "position": { + "offset": 19, + "line": 1, + "column": 20 + } }] diff --git a/test/fixture/parse/atrule/block.json b/test/fixture/parse/atrule/block.json index bd1161f7..dd578031 100644 --- a/test/fixture/parse/atrule/block.json +++ b/test/fixture/parse/atrule/block.json @@ -107,13 +107,9 @@ "type": "Number", "value": "1" }, - { - "type": "Operator", - "value": "+" - }, { "type": "Number", - "value": "2" + "value": "+2" } ] } diff --git a/test/fixture/parse/atrule/stylesheet.json b/test/fixture/parse/atrule/stylesheet.json index 1774d631..66d0e58c 100644 --- a/test/fixture/parse/atrule/stylesheet.json +++ b/test/fixture/parse/atrule/stylesheet.json @@ -241,13 +241,9 @@ "type": "Number", "value": "1" }, - { - "type": "Operator", - "value": "+" - }, { "type": "Number", - "value": "2" + "value": "+2" } ] } diff --git a/test/fixture/parse/simpleSelector/SimpleSelector.json b/test/fixture/parse/simpleSelector/SimpleSelector.json index a5597727..ef0d27b4 100644 --- a/test/fixture/parse/simpleSelector/SimpleSelector.json +++ b/test/fixture/parse/simpleSelector/SimpleSelector.json @@ -200,13 +200,9 @@ "type": "FunctionalPseudo", "name": "nth-child", "sequence": [ - { - "type": "Operator", - "value": "+" - }, { "type": "Nth", - "value": "3n" + "value": "+3n" }, { "type": "Operator", diff --git a/test/fixture/parse/simpleSelector/nthselector.json b/test/fixture/parse/simpleSelector/nthselector.json index 419f0a02..301cda9f 100644 --- a/test/fixture/parse/simpleSelector/nthselector.json +++ b/test/fixture/parse/simpleSelector/nthselector.json @@ -26,13 +26,9 @@ "type": "FunctionalPseudo", "name": "nth-last-child", "sequence": [ - { - "type": "Operator", - "value": "+" - }, { "type": "Nth", - "value": "3n" + "value": "+3n" }, { "type": "Operator", @@ -133,19 +129,15 @@ } }, "nthselector.c.1": { - "source": ":nth-last-child(/*test*/+/*test*/3n/*test*/-/*test*/2/*test*/)", + "source": ":nth-last-child(/*test*/+3n/*test*/-/*test*/2/*test*/)", "translate": ":nth-last-child(+3n-2)", "ast": { "type": "FunctionalPseudo", "name": "nth-last-child", "sequence": [ - { - "type": "Operator", - "value": "+" - }, { "type": "Nth", - "value": "3n" + "value": "+3n" }, { "type": "Operator", @@ -181,19 +173,15 @@ } }, "nthselector.s.1": { - "source": ":nth-last-child( + 3n - 2 )", + "source": ":nth-last-child( +3n - 2 )", "translate": ":nth-last-child(+3n-2)", "ast": { "type": "FunctionalPseudo", "name": "nth-last-child", "sequence": [ - { - "type": "Operator", - "value": "+" - }, { "type": "Nth", - "value": "3n" + "value": "+3n" }, { "type": "Operator", diff --git a/test/fixture/parse/value/Function.json b/test/fixture/parse/value/Function.json index 149db167..7b513a75 100644 --- a/test/fixture/parse/value/Function.json +++ b/test/fixture/parse/value/Function.json @@ -141,13 +141,9 @@ "type": "Operator", "value": "," }, - { - "type": "Operator", - "value": "+" - }, { "type": "Percentage", - "value": "89" + "value": "+89" } ] } diff --git a/test/fixture/parse/value/Number.json b/test/fixture/parse/value/Number.json index 021a2566..3d9b9973 100644 --- a/test/fixture/parse/value/Number.json +++ b/test/fixture/parse/value/Number.json @@ -119,6 +119,13 @@ } }, "with sign #4": { + "source": "+.2", + "ast": { + "type": "Number", + "value": "+.2" + } + }, + "with sign #5": { "source": "-1.2e3", "ast": { "type": "Number", diff --git a/test/fixture/parse/value/Parentheses.json b/test/fixture/parse/value/Parentheses.json index 6b6bffba..72b9e17b 100644 --- a/test/fixture/parse/value/Parentheses.json +++ b/test/fixture/parse/value/Parentheses.json @@ -39,13 +39,9 @@ "type": "Identifier", "name": "x" }, - { - "type": "Operator", - "value": "+" - }, { "type": "Number", - "value": "1" + "value": "+1" } ] } diff --git a/test/fixture/parse/value/UnicodeRange.json b/test/fixture/parse/value/UnicodeRange.json index fef6b38d..9abd19a8 100644 --- a/test/fixture/parse/value/UnicodeRange.json +++ b/test/fixture/parse/value/UnicodeRange.json @@ -23,6 +23,18 @@ ] } }, + "unicode range hex pair start with letters": { + "source": "U+FF00-FF10", + "ast": { + "type": "Value", + "sequence": [ + { + "type": "UnicodeRange", + "name": "U+FF00-FF10" + } + ] + } + }, "unicode range hex pair #2": { "source": "u+0025-00FF", "ast": {