/*
* Lexical analysis and token construction.
*/
"use strict";
var _ = require("underscore");
var events = require("events");
var reg = require("./reg.js");
var state = require("./state.js").state;
var unicodeData = require("../data/ascii-identifier-data.js");
var asciiIdentifierStartTable = unicodeData.asciiIdentifierStartTable;
var asciiIdentifierPartTable = unicodeData.asciiIdentifierPartTable;
var nonAsciiIdentifierStartTable = require("../data/non-ascii-identifier-start.js");
var nonAsciiIdentifierPartTable = require("../data/non-ascii-identifier-part-only.js");
// Some of these token types are from JavaScript Parser API
// while others are specific to JSHint parser.
// JS Parser API: https://developer.mozilla.org/en-US/docs/SpiderMonkey/Parser_API
var Token = {
Identifier: 1,
Punctuator: 2,
NumericLiteral: 3,
StringLiteral: 4,
Comment: 5,
Keyword: 6,
NullLiteral: 7,
BooleanLiteral: 8,
RegExp: 9,
TemplateHead: 10,
TemplateMiddle: 11,
TemplateTail: 12
};
// Object that handles postponed lexing verifications that checks the parsed
// environment state.
function asyncTrigger() {
var _checks = [];
return {
push: function (fn) {
_checks.push(fn);
},
check: function () {
for (var check = 0; check < _checks.length; ++check) {
_checks[check]();
}
_checks.splice(0, _checks.length);
}
};
}
/*
* Lexer for JSHint.
*
* This object does a char-by-char scan of the provided source code
* and produces a sequence of tokens.
*
* var lex = new Lexer("var i = 0;");
* lex.start();
* lex.token(); // returns the next token
*
* You have to use the token() method to move the lexer forward
* but you don't have to use its return value to get tokens. In addition
* to token() method returning the next token, the Lexer object also
* emits events.
*
* lex.on("Identifier", function (data) {
* if (data.name.indexOf("_") >= 0) {
* // Produce a warning.
* }
* });
*
* Note that the token() method returns tokens in a JSLint-compatible
* format while the event emitter uses a slightly modified version of
* Mozilla's JavaScript Parser API. Eventually, we will move away from
* JSLint format.
*/
function Lexer(source) {
var lines = source;
if (typeof lines === "string") {
lines = lines
.replace(/\r\n/g, "\n")
.replace(/\r/g, "\n")
.split("\n");
}
// If the first line is a shebang (#!), make it a blank and move on.
// Shebangs are used by Node scripts.
if (lines[0] && lines[0].substr(0, 2) === "#!") {
if (lines[0].indexOf("node") !== -1) {
state.option.node = true;
}
lines[0] = "";
}
this.emitter = new events.EventEmitter();
this.source = source;
this.setLines(lines);
this.prereg = true;
this.line = 0;
this.char = 1;
this.from = 1;
this.input = "";
this.inComment = false;
this.inTemplate = false;
this.templateLine = null;
this.templateChar = null;
for (var i = 0; i < state.option.indent; i += 1) {
state.tab += " ";
}
}
Lexer.prototype = {
_lines: [],
getLines: function () {
this._lines = state.lines;
return this._lines;
},
setLines: function (val) {
this._lines = val;
state.lines = this._lines;
},
/*
* Return the next i character without actually moving the
* char pointer.
*/
peek: function (i) {
return this.input.charAt(i || 0);
},
/*
* Move the char pointer forward i times.
*/
skip: function (i) {
i = i || 1;
this.char += i;
this.input = this.input.slice(i);
},
/*
* Subscribe to a token event. The API for this method is similar
* Underscore.js i.e. you can subscribe to multiple events with
* one call:
*
* lex.on("Identifier Number", function (data) {
* // ...
* });
*/
on: function (names, listener) {
names.split(" ").forEach(function (name) {
this.emitter.on(name, listener);
}.bind(this));
},
/*
* Trigger a token event. All arguments will be passed to each
* listener.
*/
trigger: function () {
this.emitter.emit.apply(this.emitter, Array.prototype.slice.call(arguments));
},
/*
* Postpone a token event. the checking condition is set as
* last parameter, and the trigger function is called in a
* stored callback. To be later called using the check() function
* by the parser. This avoids parser's peek() to give the lexer
* a false context.
*/
triggerAsync: function (type, args, checks, fn) {
checks.push(function () {
if (fn()) {
this.trigger(type, args);
}
}.bind(this));
},
/*
* Extract a punctuator out of the next sequence of characters
* or return 'null' if its not possible.
*
* This method's implementation was heavily influenced by the
* scanPunctuator function in the Esprima parser's source code.
*/
scanPunctuator: function () {
var ch1 = this.peek();
var ch2, ch3, ch4;
switch (ch1) {
// Most common single-character punctuators
case ".":
if ((/^[0-9]$/).test(this.peek(1))) {
return null;
}
if (this.peek(1) === "." && this.peek(2) === ".") {
return {
type: Token.Punctuator,
value: "..."
};
}
/* falls through */
case "(":
case ")":
case ";":
case ",":
case "{":
case "}":
case "[":
case "]":
case ":":
case "~":
case "?":
return {
type: Token.Punctuator,
value: ch1
};
// A pound sign (for Node shebangs)
case "#":
return {
type: Token.Punctuator,
value: ch1
};
// We're at the end of input
case "":
return null;
}
// Peek more characters
ch2 = this.peek(1);
ch3 = this.peek(2);
ch4 = this.peek(3);
// 4-character punctuator: >>>=
if (ch1 === ">" && ch2 === ">" && ch3 === ">" && ch4 === "=") {
return {
type: Token.Punctuator,
value: ">>>="
};
}
// 3-character punctuators: === !== >>> <<= >>=
if (ch1 === "=" && ch2 === "=" && ch3 === "=") {
return {
type: Token.Punctuator,
value: "==="
};
}
if (ch1 === "!" && ch2 === "=" && ch3 === "=") {
return {
type: Token.Punctuator,
value: "!=="
};
}
if (ch1 === ">" && ch2 === ">" && ch3 === ">") {
return {
type: Token.Punctuator,
value: ">>>"
};
}
if (ch1 === "<" && ch2 === "<" && ch3 === "=") {
return {
type: Token.Punctuator,
value: "<<="
};
}
if (ch1 === ">" && ch2 === ">" && ch3 === "=") {
return {
type: Token.Punctuator,
value: ">>="
};
}
// Fat arrow punctuator
if (ch1 === "=" && ch2 === ">") {
return {
type: Token.Punctuator,
value: ch1 + ch2
};
}
// 2-character punctuators: <= >= == != ++ -- << >> && ||
// += -= *= %= &= |= ^= (but not /=, see below)
if (ch1 === ch2 && ("+-<>&|".indexOf(ch1) >= 0)) {
return {
type: Token.Punctuator,
value: ch1 + ch2
};
}
if ("<>=!+-*%&|^".indexOf(ch1) >= 0) {
if (ch2 === "=") {
return {
type: Token.Punctuator,
value: ch1 + ch2
};
}
return {
type: Token.Punctuator,
value: ch1
};
}
// Special case: /=.
if (ch1 === "/") {
if (ch2 === "=") {
return {
type: Token.Punctuator,
value: "/="
};
}
return {
type: Token.Punctuator,
value: "/"
};
}
return null;
},
/*
* Extract a comment out of the next sequence of characters and/or
* lines or return 'null' if its not possible. Since comments can
* span across multiple lines this method has to move the char
* pointer.
*
* In addition to normal JavaScript comments (// and /*) this method
* also recognizes JSHint- and JSLint-specific comments such as
* /*jshint, /*jslint, /*globals and so on.
*/
scanComments: function () {
var ch1 = this.peek();
var ch2 = this.peek(1);
var rest = this.input.substr(2);
var startLine = this.line;
var startChar = this.char;
// Create a comment token object and make sure it
// has all the data JSHint needs to work with special
// comments.
function commentToken(label, body, opt) {
var special = ["jshint", "jslint", "members", "member", "globals", "global", "exported"];
var isSpecial = false;
var value = label + body;
var commentType = "plain";
opt = opt || {};
if (opt.isMultiline) {
value += "*/";
}
body = body.replace(/\n/g, " ");
special.forEach(function (str) {
if (isSpecial) {
return;
}
// Don't recognize any special comments other than jshint for single-line
// comments. This introduced many problems with legit comments.
if (label === "//" && str !== "jshint") {
return;
}
if (body.charAt(str.length) === " " && body.substr(0, str.length) === str) {
isSpecial = true;
label = label + str;
body = body.substr(str.length);
}
if (!isSpecial && body.charAt(0) === " " && body.charAt(str.length + 1) === " " &&
body.substr(1, str.length) === str) {
isSpecial = true;
label = label + " " + str;
body = body.substr(str.length + 1);
}
if (!isSpecial) {
return;
}
switch (str) {
case "member":
commentType = "members";
break;
case "global":
commentType = "globals";
break;
default:
commentType = str;
}
});
return {
type: Token.Comment,
commentType: commentType,
value: value,
body: body,
isSpecial: isSpecial,
isMultiline: opt.isMultiline || false,
isMalformed: opt.isMalformed || false
};
}
// End of unbegun comment. Raise an error and skip that input.
if (ch1 === "*" && ch2 === "/") {
this.trigger("error", {
code: "E018",
line: startLine,
character: startChar
});
this.skip(2);
return null;
}
// Comments must start either with // or /*
if (ch1 !== "/" || (ch2 !== "*" && ch2 !== "/")) {
return null;
}
// One-line comment
if (ch2 === "/") {
this.skip(this.input.length); // Skip to the EOL.
return commentToken("//", rest);
}
var body = "";
/* Multi-line comment */
if (ch2 === "*") {
this.inComment = true;
this.skip(2);
while (this.peek() !== "*" || this.peek(1) !== "/") {
if (this.peek() === "") { // End of Line
body += "\n";
// If we hit EOF and our comment is still unclosed,
// trigger an error and end the comment implicitly.
if (!this.nextLine()) {
this.trigger("error", {
code: "E017",
line: startLine,
character: startChar
});
this.inComment = false;
return commentToken("/*", body, {
isMultiline: true,
isMalformed: true
});
}
} else {
body += this.peek();
this.skip();
}
}
this.skip(2);
this.inComment = false;
return commentToken("/*", body, { isMultiline: true });
}
},
/*
* Extract a keyword out of the next sequence of characters or
* return 'null' if its not possible.
*/
scanKeyword: function () {
var result = /^[a-zA-Z_$][a-zA-Z0-9_$]*/.exec(this.input);
var keywords = [
"if", "in", "do", "var", "for", "new",
"try", "let", "this", "else", "case",
"void", "with", "enum", "while", "break",
"catch", "throw", "const", "yield", "class",
"super", "return", "typeof", "delete",
"switch", "export", "import", "default",
"finally", "extends", "function", "continue",
"debugger", "instanceof"
];
if (result && keywords.indexOf(result[0]) >= 0) {
return {
type: Token.Keyword,
value: result[0]
};
}
return null;
},
/*
* Extract a JavaScript identifier out of the next sequence of
* characters or return 'null' if its not possible. In addition,
* to Identifier this method can also produce BooleanLiteral
* (true/false) and NullLiteral (null).
*/
scanIdentifier: function () {
var id = "";
var index = 0;
var type, char;
function isNonAsciiIdentifierStart(code) {
return nonAsciiIdentifierStartTable.indexOf(code) > -1;
}
function isNonAsciiIdentifierPart(code) {
return isNonAsciiIdentifierStart(code) || nonAsciiIdentifierPartTable.indexOf(code) > -1;
}
function isHexDigit(str) {
return (/^[0-9a-fA-F]$/).test(str);
}
var readUnicodeEscapeSequence = function() {
/*jshint validthis:true */
index += 1;
if (this.peek(index) !== "u") {
return null;
}
var ch1 = this.peek(index + 1);
var ch2 = this.peek(index + 2);
var ch3 = this.peek(index + 3);
var ch4 = this.peek(index + 4);
var code;
if (isHexDigit(ch1) && isHexDigit(ch2) && isHexDigit(ch3) && isHexDigit(ch4)) {
code = parseInt(ch1 + ch2 + ch3 + ch4, 16);
if (asciiIdentifierPartTable[code] || isNonAsciiIdentifierPart(code)) {
index += 5;
return "\\u" + ch1 + ch2 + ch3 + ch4;
}
return null;
}
return null;
}.bind(this);
var getIdentifierStart = function () {
/*jshint validthis:true */
var chr = this.peek(index);
var code = chr.charCodeAt(0);
if (code === 92) {
return readUnicodeEscapeSequence();
}
if (code < 128) {
if (asciiIdentifierStartTable[code]) {
index += 1;
return chr;
}
return null;
}
if (isNonAsciiIdentifierStart(code)) {
index += 1;
return chr;
}
return null;
}.bind(this);
var getIdentifierPart = function () {
/*jshint validthis:true */
var chr = this.peek(index);
var code = chr.charCodeAt(0);
if (code === 92) {
return readUnicodeEscapeSequence();
}
if (code < 128) {
if (asciiIdentifierPartTable[code]) {
index += 1;
return chr;
}
return null;
}
if (isNonAsciiIdentifierPart(code)) {
index += 1;
return chr;
}
return null;
}.bind(this);
function removeEscapeSequences(id) {
return id.replace(/\\u([0-9a-fA-F]{4})/g, function(m0, codepoint) {
return String.fromCharCode(parseInt(codepoint, 16));
});
}
char = getIdentifierStart();
if (char === null) {
return null;
}
id = char;
for (;;) {
char = getIdentifierPart();
if (char === null) {
break;
}
id += char;
}
switch (id) {
case "true":
case "false":
type = Token.BooleanLiteral;
break;
case "null":
type = Token.NullLiteral;
break;
default:
type = Token.Identifier;
}
return {
type: type,
value: removeEscapeSequences(id),
text: id,
tokenLength: id.length
};
},
/*
* Extract a numeric literal out of the next sequence of
* characters or return 'null' if its not possible. This method
* supports all numeric literals described in section 7.8.3
* of the EcmaScript 5 specification.
*
* This method's implementation was heavily influenced by the
* scanNumericLiteral function in the Esprima parser's source code.
*/
scanNumericLiteral: function () {
var index = 0;
var value = "";
var length = this.input.length;
var char = this.peek(index);
var bad;
var isAllowedDigit = isDecimalDigit;
var base = 10;
var isLegacy = false;
function isDecimalDigit(str) {
return (/^[0-9]$/).test(str);
}
function isOctalDigit(str) {
return (/^[0-7]$/).test(str);
}
function isBinaryDigit(str) {
return (/^[01]$/).test(str);
}
function isHexDigit(str) {
return (/^[0-9a-fA-F]$/).test(str);
}
function isIdentifierStart(ch) {
return (ch === "$") || (ch === "_") || (ch === "\\") ||
(ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z");
}
// Numbers must start either with a decimal digit or a point.
if (char !== "." && !isDecimalDigit(char)) {
return null;
}
if (char !== ".") {
value = this.peek(index);
index += 1;
char = this.peek(index);
if (value === "0") {
// Base-16 numbers.
if (char === "x" || char === "X") {
isAllowedDigit = isHexDigit;
base = 16;
index += 1;
value += char;
}
// Base-8 numbers.
if (char === "o" || char === "O") {
isAllowedDigit = isOctalDigit;
base = 8;
if (!state.option.esnext) {
this.trigger("warning", {
code: "W119",
line: this.line,
character: this.char,
data: [ "Octal integer literal" ]
});
}
index += 1;
value += char;
}
// Base-2 numbers.
if (char === "b" || char === "B") {
isAllowedDigit = isBinaryDigit;
base = 2;
if (!state.option.esnext) {
this.trigger("warning", {
code: "W119",
line: this.line,
character: this.char,
data: [ "Binary integer literal" ]
});
}
index += 1;
value += char;
}
// Legacy base-8 numbers.
if (isOctalDigit(char)) {
isAllowedDigit = isOctalDigit;
base = 8;
isLegacy = true;
bad = false;
index += 1;
value += char;
}
// Decimal numbers that start with '0' such as '09' are illegal
// but we still parse them and return as malformed.
if (!isOctalDigit(char) && isDecimalDigit(char)) {
index += 1;
value += char;
}
}
while (index < length) {
char = this.peek(index);
if (isLegacy && isDecimalDigit(char)) {
// Numbers like '019' (note the 9) are not valid octals
// but we still parse them and mark as malformed.
bad = true;
} else if (!isAllowedDigit(char)) {
break;
}
value += char;
index += 1;
}
if (isAllowedDigit !== isDecimalDigit) {
if (!isLegacy && value.length <= 2) { // 0x
return {
type: Token.NumericLiteral,
value: value,
isMalformed: true
};
}
if (index < length) {
char = this.peek(index);
if (isIdentifierStart(char)) {
return null;
}
}
return {
type: Token.NumericLiteral,
value: value,
base: base,
isLegacy: isLegacy,
isMalformed: false
};
}
}
// Decimal digits.
if (char === ".") {
value += char;
index += 1;
while (index < length) {
char = this.peek(index);
if (!isDecimalDigit(char)) {
break;
}
value += char;
index += 1;
}
}
// Exponent part.
if (char === "e" || char === "E") {
value += char;
index += 1;
char = this.peek(index);
if (char === "+" || char === "-") {
value += this.peek(index);
index += 1;
}
char = this.peek(index);
if (isDecimalDigit(char)) {
value += char;
index += 1;
while (index < length) {
char = this.peek(index);
if (!isDecimalDigit(char)) {
break;
}
value += char;
index += 1;
}
} else {
return null;
}
}
if (index < length) {
char = this.peek(index);
if (isIdentifierStart(char)) {
return null;
}
}
return {
type: Token.NumericLiteral,
value: value,
base: base,
isMalformed: !isFinite(value)
};
},
// Assumes previously parsed character was \ (=== '\\') and was not skipped.
scanEscapeSequence: function (checks) {
var allowNewLine = false;
var jump = 1;
this.skip();
var char = this.peek();
switch (char) {
case "'":
this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
data: [ "\\'" ]
}, checks, function () {return state.jsonMode; });
break;
case "b":
char = "\\b";
break;
case "f":
char = "\\f";
break;
case "n":
char = "\\n";
break;
case "r":
char = "\\r";
break;
case "t":
char = "\\t";
break;
case "0":
char = "\\0";
// Octal literals fail in strict mode.
// Check if the number is between 00 and 07.
var n = parseInt(this.peek(1), 10);
this.triggerAsync("warning", {
code: "W115",
line: this.line,
character: this.char
}, checks,
function () { return n >= 0 && n <= 7 && state.directive["use strict"]; });
break;
case "u":
char = String.fromCharCode(parseInt(this.input.substr(1, 4), 16));
jump = 5;
break;
case "v":
this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
data: [ "\\v" ]
}, checks, function () { return state.jsonMode; });
char = "\v";
break;
case "x":
var x = parseInt(this.input.substr(1, 2), 16);
this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
data: [ "\\x-" ]
}, checks, function () { return state.jsonMode; });
char = String.fromCharCode(x);
jump = 3;
break;
case "\\":
char = "\\\\";
break;
case "\"":
char = "\\\"";
break;
case "/":
break;
case "":
allowNewLine = true;
char = "";
break;
}
return {char: char, jump: jump, allowNewLine: allowNewLine};
},
/*
* Extract a template literal out of the next sequence of characters
* and/or lines or return 'null' if its not possible. Since template
* literals can span across multiple lines, this method has to move
* the char pointer.
*/
scanTemplateLiteral: function (checks) {
var tokenType;
var value = '';
var ch;
// String must start with a backtick.
if (!this.inTemplate) {
if (!state.option.esnext || this.peek() !== "`") {
return null;
}
this.templateLine = this.line;
this.templateChar = this.char;
this.skip(1);
} else if (this.peek() !== '}') {
// If we're in a template, and we don't have a '}', lex something else instead.
return null;
}
while (this.peek() !== "`") {
while ((ch = this.peek()) === "") {
if (!this.nextLine()) {
// Unclosed template literal --- point to the starting line, or the EOF?
tokenType = this.inTemplate ? Token.TemplateHead : Token.TemplateMiddle;
this.inTemplate = false;
this.trigger("error", {
code: "E052",
line: this.templateLine,
character: this.templateChar
});
return {
type: tokenType,
value: value,
isUnclosed: true
};
}
}
if (ch === '$' && this.peek(1) === '{') {
value += '${';
tokenType = value.charAt(0) === '}' ? Token.TemplateMiddle : Token.TemplateHead;
// Either TokenHead or TokenMiddle --- depending on if the initial value
// is '}' or not.
this.skip(2);
this.inTemplate = true;
return {
type: tokenType,
value: value,
isUnclosed: false
};
} else if (ch === '\\') {
var escape = this.scanEscapeSequence(checks);
value += escape.char;
this.skip(escape.jump);
} else if (ch === '`') {
break;
} else {
// Otherwise, append the value and continue.
value += ch;
this.skip(1);
}
}
// Final value is either TokenTail or NoSubstititionTemplate --- essentially a string
tokenType = this.inTemplate ? Token.TemplateTail : Token.StringLiteral;
this.inTemplate = false;
this.skip(1);
return {
type: tokenType,
value: value,
isUnclosed: false,
quote: "`"
};
},
/*
* Extract a string out of the next sequence of characters and/or
* lines or return 'null' if its not possible. Since strings can
* span across multiple lines this method has to move the char
* pointer.
*
* This method recognizes pseudo-multiline JavaScript strings:
*
* var str = "hello\
* world";
*/
scanStringLiteral: function (checks) {
/*jshint loopfunc:true */
var quote = this.peek();
// String must start with a quote.
if (quote !== "\"" && quote !== "'") {
return null;
}
// In JSON strings must always use double quotes.
this.triggerAsync("warning", {
code: "W108",
line: this.line,
character: this.char // +1?
}, checks, function () { return state.jsonMode && quote !== "\""; });
var value = "";
var startLine = this.line;
var startChar = this.char;
var allowNewLine = false;
this.skip();
while (this.peek() !== quote) {
while (this.peek() === "") { // End Of Line
// If an EOL is not preceded by a backslash, show a warning
// and proceed like it was a legit multi-line string where
// author simply forgot to escape the newline symbol.
//
// Another approach is to implicitly close a string on EOL
// but it generates too many false positives.
if (!allowNewLine) {
this.trigger("warning", {
code: "W112",
line: this.line,
character: this.char
});
} else {
allowNewLine = false;
// Otherwise show a warning if multistr option was not set.
// For JSON, show warning no matter what.
this.triggerAsync("warning", {
code: "W043",
line: this.line,
character: this.char
}, checks, function () { return !state.option.multistr; });
this.triggerAsync("warning", {
code: "W042",
line: this.line,
character: this.char
}, checks, function () { return state.jsonMode && state.option.multistr; });
}
// If we get an EOF inside of an unclosed string, show an
// error and implicitly close it at the EOF point.
if (!this.nextLine()) {
this.trigger("error", {
code: "E029",
line: startLine,
character: startChar
});
return {
type: Token.StringLiteral,
value: value,
isUnclosed: true,
quote: quote
};
}
}
allowNewLine = false;
var char = this.peek();
var jump = 1; // A length of a jump, after we're done
// parsing this character.
if (char < " ") {
// Warn about a control character in a string.
this.trigger("warning", {
code: "W113",
line: this.line,
character: this.char,
data: [ "<non-printable>" ]
});
}
// Special treatment for some escaped characters.
if (char === "\\") {
var parsed = this.scanEscapeSequence(checks);
char = parsed.char;
jump = parsed.jump;
allowNewLine = parsed.allowNewLine;
}
value += char;
this.skip(jump);
}
this.skip();
return {
type: Token.StringLiteral,
value: value,
isUnclosed: false,
quote: quote
};
},
/*
* Extract a regular expression out of the next sequence of
* characters and/or lines or return 'null' if its not possible.
*
* This method is platform dependent: it accepts almost any
* regular expression values but then tries to compile and run
* them using system's RegExp object. This means that there are
* rare edge cases where one JavaScript engine complains about
* your regular expression while others don't.
*/
scanRegExp: function () {
var index = 0;
var length = this.input.length;
var char = this.peek();
var value = char;
var body = "";
var flags = [];
var malformed = false;
var isCharSet = false;
var terminated;
var scanUnexpectedChars = function () {
// Unexpected control character
if (char < " ") {
malformed = true;
this.trigger("warning", {
code: "W048",
line: this.line,
character: this.char
});
}
// Unexpected escaped character
if (char === "<") {
malformed = true;
this.trigger("warning", {
code: "W049",
line: this.line,
character: this.char,
data: [ char ]
});
}
}.bind(this);
// Regular expressions must start with '/'
if (!this.prereg || char !== "/") {
return null;
}
index += 1;
terminated = false;
// Try to get everything in between slashes. A couple of
// cases aside (see scanUnexpectedChars) we don't really
// care whether the resulting expression is valid or not.
// We will check that later using the RegExp object.
while (index < length) {
char = this.peek(index);
value += char;
body += char;
if (isCharSet) {
if (char === "]") {
if (this.peek(index - 1) !== "\\" || this.peek(index - 2) === "\\") {
isCharSet = false;
}
}
if (char === "\\") {
index += 1;
char = this.peek(index);
body += char;
value += char;
scanUnexpectedChars();
}
index += 1;
continue;
}
if (char === "\\") {
index += 1;
char = this.peek(index);
body += char;
value += char;
scanUnexpectedChars();
if (char === "/") {
index += 1;
continue;
}
if (char === "[") {
index += 1;
continue;
}
}
if (char === "[") {
isCharSet = true;
index += 1;
continue;
}
if (char === "/") {
body = body.substr(0, body.length - 1);
terminated = true;
index += 1;
break;
}
index += 1;
}
// A regular expression that was never closed is an
// error from which we cannot recover.
if (!terminated) {
this.trigger("error", {
code: "E015",
line: this.line,
character: this.from
});
return void this.trigger("fatal", {
line: this.line,
from: this.from
});
}
// Parse flags (if any).
while (index < length) {
char = this.peek(index);
if (!/[gim]/.test(char)) {
break;
}
flags.push(char);
value += char;
index += 1;
}
// Check regular expression for correctness.
try {
new RegExp(body, flags.join(""));
} catch (err) {
malformed = true;
this.trigger("error", {
code: "E016",
line: this.line,
character: this.char,
data: [ err.message ] // Platform dependent!
});
}
return {
type: Token.RegExp,
value: value,
flags: flags,
isMalformed: malformed
};
},
/*
* Scan for any occurrence of non-breaking spaces. Non-breaking spaces
* can be mistakenly typed on OS X with option-space. Non UTF-8 web
* pages with non-breaking pages produce syntax errors.
*/
scanNonBreakingSpaces: function () {
return state.option.nonbsp ?
this.input.search(/(\u00A0)/) : -1;
},
/*
* Scan for characters that get silently deleted by one or more browsers.
*/
scanUnsafeChars: function () {
return this.input.search(reg.unsafeChars);
},
/*
* Produce the next raw token or return 'null' if no tokens can be matched.
* This method skips over all space characters.
*/
next: function (checks) {
this.from = this.char;
// Move to the next non-space character.
var start;
if (/\s/.test(this.peek())) {
start = this.char;
while (/\s/.test(this.peek())) {
this.from += 1;
this.skip();
}
}
// Methods that work with multi-line structures and move the
// character pointer.
var match = this.scanComments() ||
this.scanStringLiteral(checks) ||
this.scanTemplateLiteral(checks);
if (match) {
return match;
}
// Methods that don't move the character pointer.
match =
this.scanRegExp() ||
this.scanPunctuator() ||
this.scanKeyword() ||
this.scanIdentifier() ||
this.scanNumericLiteral();
if (match) {
this.skip(match.tokenLength || match.value.length);
return match;
}
// No token could be matched, give up.
return null;
},
/*
* Switch to the next line and reset all char pointers. Once
* switched, this method also checks for other minor warnings.
*/
nextLine: function () {
var char;
if (this.line >= this.getLines().length) {
return false;
}
this.input = this.getLines()[this.line];
this.line += 1;
this.char = 1;
this.from = 1;
var inputTrimmed = this.input.trim();
var startsWith = function () {
return _.some(arguments, function (prefix) {
return inputTrimmed.indexOf(prefix) === 0;
});
};
var endsWith = function () {
return _.some(arguments, function (suffix) {
return inputTrimmed.indexOf(suffix, inputTrimmed.length - suffix.length) !== -1;
});
};
// If we are ignoring linter errors, replace the input with empty string
// if it doesn't already at least start or end a multi-line comment
if (state.ignoreLinterErrors === true) {
if (!startsWith("/*", "//") && !(this.inComment && endsWith("*/"))) {
this.input = "";
}
}
char = this.scanNonBreakingSpaces();
if (char >= 0) {
this.trigger("warning", { code: "W125", line: this.line, character: char + 1 });
}
this.input = this.input.replace(/\t/g, state.tab);
char = this.scanUnsafeChars();
if (char >= 0) {
this.trigger("warning", { code: "W100", line: this.line, character: char });
}
// If there is a limit on line length, warn when lines get too
// long.
if (state.option.maxlen && state.option.maxlen < this.input.length) {
var inComment = this.inComment ||
startsWith.call(inputTrimmed, "//") ||
startsWith.call(inputTrimmed, "/*");
var shouldTriggerError = !inComment || !reg.maxlenException.test(inputTrimmed);
if (shouldTriggerError) {
this.trigger("warning", { code: "W101", line: this.line, character: this.input.length });
}
}
return true;
},
/*
* This is simply a synonym for nextLine() method with a friendlier
* public name.
*/
start: function () {
this.nextLine();
},
/*
* Produce the next token. This function is called by advance() to get
* the next token. It returns a token in a JSLint-compatible format.
*/
token: function () {
/*jshint loopfunc:true */
var checks = asyncTrigger();
var token;
function isReserved(token, isProperty) {
if (!token.reserved) {
return false;
}
var meta = token.meta;
if (meta && meta.isFutureReservedWord && state.option.inES5()) {
// ES3 FutureReservedWord in an ES5 environment.
if (!meta.es5) {
return false;
}
// Some ES5 FutureReservedWord identifiers are active only
// within a strict mode environment.
if (meta.strictOnly) {
if (!state.option.strict && !state.directive["use strict"]) {
return false;
}
}
if (isProperty) {
return false;
}
}
return true;
}
// Produce a token object.
var create = function (type, value, isProperty, token) {
/*jshint validthis:true */
var obj;
if (type !== "(endline)" && type !== "(end)") {
this.prereg = false;
}
if (type === "(punctuator)") {
switch (value) {
case ".":
case ")":
case "~":
case "#":
case "]":
case "++":
case "--":
this.prereg = false;
break;
default:
this.prereg = true;
}
obj = Object.create(state.syntax[value] || state.syntax["(error)"]);
}
if (type === "(identifier)") {
if (value === "return" || value === "case" || value === "typeof") {
this.prereg = true;
}
if (_.has(state.syntax, value)) {
obj = Object.create(state.syntax[value] || state.syntax["(error)"]);
// If this can't be a reserved keyword, reset the object.
if (!isReserved(obj, isProperty && type === "(identifier)")) {
obj = null;
}
}
}
if (!obj) {
obj = Object.create(state.syntax[type]);
}
obj.identifier = (type === "(identifier)");
obj.type = obj.type || type;
obj.value = value;
obj.line = this.line;
obj.character = this.char;
obj.from = this.from;
if (obj.identifier && token) obj.raw_text = token.text || token.value;
if (isProperty && obj.identifier) {
obj.isProperty = isProperty;
}
obj.check = checks.check;
return obj;
}.bind(this);
for (;;) {
if (!this.input.length) {
return create(this.nextLine() ? "(endline)" : "(end)", "");
}
token = this.next(checks);
if (!token) {
if (this.input.length) {
// Unexpected character.
this.trigger("error", {
code: "E024",
line: this.line,
character: this.char,
data: [ this.peek() ]
});
this.input = "";
}
continue;
}
switch (token.type) {
case Token.StringLiteral:
this.triggerAsync("String", {
line: this.line,
char: this.char,
from: this.from,
value: token.value,
quote: token.quote
}, checks, function () { return true; });
return create("(string)", token.value);
case Token.TemplateHead:
this.trigger("TemplateHead", {
line: this.line,
char: this.char,
from: this.from,
value: token.value
});
return create("(template)", token.value);
case Token.TemplateMiddle:
this.trigger("TemplateMiddle", {
line: this.line,
char: this.char,
from: this.from,
value: token.value
});
return create("(template middle)", token.value);
case Token.TemplateTail:
this.trigger("TemplateTail", {
line: this.line,
char: this.char,
from: this.from,
value: token.value
});
return create("(template tail)", token.value);
case Token.Identifier:
this.trigger("Identifier", {
line: this.line,
char: this.char,
from: this.form,
name: token.value,
raw_name: token.text,
isProperty: state.tokens.curr.id === "."
});
/* falls through */
case Token.Keyword:
case Token.NullLiteral:
case Token.BooleanLiteral:
return create("(identifier)", token.value, state.tokens.curr.id === ".", token);
case Token.NumericLiteral:
if (token.isMalformed) {
this.trigger("warning", {
code: "W045",
line: this.line,
character: this.char,
data: [ token.value ]
});
}
this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
data: [ "0x-" ]
}, checks, function () { return token.base === 16 && state.jsonMode; });
this.triggerAsync("warning", {
code: "W115",
line: this.line,
character: this.char
}, checks, function () {
return state.directive["use strict"] && token.base === 8 && token.isLegacy;
});
this.trigger("Number", {
line: this.line,
char: this.char,
from: this.from,
value: token.value,
base: token.base,
isMalformed: token.malformed
});
return create("(number)", token.value);
case Token.RegExp:
return create("(regexp)", token.value);
case Token.Comment:
state.tokens.curr.comment = true;
if (token.isSpecial) {
return {
id: '(comment)',
value: token.value,
body: token.body,
type: token.commentType,
isSpecial: token.isSpecial,
line: this.line,
character: this.char,
from: this.from
};
}
break;
case "":
break;
default:
return create("(punctuator)", token.value);
}
}
}
};
exports.Lexer = Lexer;