"use strict";
function jsTokensWrapperFunction() {
var Identifier, JSXIdentifier, JSXPunctuator, JSXString, JSXText, KeywordsWithExpressionAfter, KeywordsWithNoLineTerminatorAfter, LineTerminatorSequence, MultiLineComment, Newline, NumericLiteral, Punctuator, RegularExpressionLiteral, SingleLineComment, StringLiteral, Template, TokensNotPrecedingObjectLiteral, TokensPrecedingExpression, WhiteSpace, jsTokens;
RegularExpressionLiteral = /\/(?![*\/])(?:\[(?:(?![\]\\]).|\\.)*\]|(?![\/\\]).|\\.)*(\/[$_\u200C\u200D\p{ID_Continue}]*|\\)?/yu;
Punctuator = /--|\+\+|=>|\.{3}|\??\.(?!\d)|(?:&&|\|\||\?\?|[+\-%&|^]|\*{1,2}|<{1,2}|>{1,3}|!=?|={1,2}|\/(?![\/*]))=?|[?~,:;[\](){}]/y;
Identifier = /(\x23?)(?=[$_\p{ID_Start}\\])(?:[$_\u200C\u200D\p{ID_Continue}]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]+\})+/yu;
StringLiteral = /(['"])(?:(?!\1)[^\\\n\r]|\\(?:\r\n|[^]))*(\1)?/y;
NumericLiteral = /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[oO][0-7](?:_?[0-7])*|0[bB][01](?:_?[01])*)n?|0n|[1-9](?:_?\d)*n|(?:(?:0(?!\d)|0\d*[89]\d*|[1-9](?:_?\d)*)(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?|0[0-7]+/y;
Template = /[`}](?:[^`\\$]|\\[^]|\$(?!\{))*(`|\$\{)?/y;
WhiteSpace = /[\t\v\f\ufeff\p{Zs}]+/yu;
LineTerminatorSequence = /\r?\n|[\r\u2028\u2029]/y;
MultiLineComment = /\/\*(?:[^*]|\*(?!\/))*(\*\/)?/y;
SingleLineComment = /\/\/.*/y;
JSXPunctuator = /[<>.:={}]|\/(?![\/*])/y;
JSXIdentifier = /[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}-]*/yu;
JSXString = /(['"])(?:(?!\1)[^])*(\1)?/y;
JSXText = /[^<>{}]+/y;
TokensPrecedingExpression = /^(?:[\/+-]|\.{3}|\?(?:InterpolationIn(?:JSX|Template)|NoLineTerminatorHere|NonExpressionParenEnd|UnaryIncDec))?$|[{}([,;<>=*%&|^!~?:]$/;
TokensNotPrecedingObjectLiteral = /^(?:=>|[;\]){}]|else|\?(?:NoLineTerminatorHere|NonExpressionParenEnd))?$/;
KeywordsWithExpressionAfter = /^(?:await|case|default|delete|do|else|instanceof|new|return|throw|typeof|void|yield)$/;
KeywordsWithNoLineTerminatorAfter = /^(?:return|throw|yield)$/;
Newline = RegExp(LineTerminatorSequence.source);
return function*(input, {jsx = false} = {}) {
var braces, firstCodePoint, isExpression, lastIndex, lastSignificantToken, length, match, mode, nextLastIndex, nextLastSignificantToken, parenNesting, postfixIncDec, punctuator, stack;
({length} = input);
lastIndex = 0;
lastSignificantToken = "";
stack = [
{tag: "JS"}
];
braces = [];
parenNesting = 0;
postfixIncDec = false;
while (lastIndex < length) {
mode = stack[stack.length - 1];
switch (mode.tag) {
case "JS":
case "JSNonExpressionParen":
case "InterpolationInTemplate":
case "InterpolationInJSX":
if (input[lastIndex] === "/" && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) {
RegularExpressionLiteral.lastIndex = lastIndex;
if (match = RegularExpressionLiteral.exec(input)) {
lastIndex = RegularExpressionLiteral.lastIndex;
lastSignificantToken = match[0];
postfixIncDec = true;
yield ({
type: "RegularExpressionLiteral",
value: match[0],
closed: match[1] !== void 0 && match[1] !== "\\"
});
continue;
}
}
Punctuator.lastIndex = lastIndex;
if (match = Punctuator.exec(input)) {
punctuator = match[0];
nextLastIndex = Punctuator.lastIndex;
nextLastSignificantToken = punctuator;
switch (punctuator) {
case "(":
if (lastSignificantToken === "?NonExpressionParenKeyword") {
stack.push({
tag: "JSNonExpressionParen",
nesting: parenNesting
});
}
parenNesting++;
postfixIncDec = false;
break;
case ")":
parenNesting--;
postfixIncDec = true;
if (mode.tag === "JSNonExpressionParen" && parenNesting === mode.nesting) {
stack.pop();
nextLastSignificantToken = "?NonExpressionParenEnd";
postfixIncDec = false;
}
break;
case "{":
Punctuator.lastIndex = 0;
isExpression = !TokensNotPrecedingObjectLiteral.test(lastSignificantToken) && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken));
braces.push(isExpression);
postfixIncDec = false;
break;
case "}":
switch (mode.tag) {
case "InterpolationInTemplate":
if (braces.length === mode.nesting) {
Template.lastIndex = lastIndex;
match = Template.exec(input);
lastIndex = Template.lastIndex;
lastSignificantToken = match[0];
if (match[1] === "${") {
lastSignificantToken = "?InterpolationInTemplate";
postfixIncDec = false;
yield ({
type: "TemplateMiddle",
value: match[0]
});
} else {
stack.pop();
postfixIncDec = true;
yield ({
type: "TemplateTail",
value: match[0],
closed: match[1] === "`"
});
}
continue;
}
break;
case "InterpolationInJSX":
if (braces.length === mode.nesting) {
stack.pop();
lastIndex += 1;
lastSignificantToken = "}";
yield ({
type: "JSXPunctuator",
value: "}"
});
continue;
}
}
postfixIncDec = braces.pop();
nextLastSignificantToken = postfixIncDec ? "?ExpressionBraceEnd" : "}";
break;
case "]":
postfixIncDec = true;
break;
case "++":
case "--":
nextLastSignificantToken = postfixIncDec ? "?PostfixIncDec" : "?UnaryIncDec";
break;
case "<":
if (jsx && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) {
stack.push({tag: "JSXTag"});
lastIndex += 1;
lastSignificantToken = "<";
yield ({
type: "JSXPunctuator",
value: punctuator
});
continue;
}
postfixIncDec = false;
break;
default:
postfixIncDec = false;
}
lastIndex = nextLastIndex;
lastSignificantToken = nextLastSignificantToken;
yield ({
type: "Punctuator",
value: punctuator
});
continue;
}
Identifier.lastIndex = lastIndex;
if (match = Identifier.exec(input)) {
lastIndex = Identifier.lastIndex;
nextLastSignificantToken = match[0];
switch (match[0]) {
case "for":
case "if":
case "while":
case "with":
if (lastSignificantToken !== "." && lastSignificantToken !== "?.") {
nextLastSignificantToken = "?NonExpressionParenKeyword";
}
}
lastSignificantToken = nextLastSignificantToken;
postfixIncDec = !KeywordsWithExpressionAfter.test(match[0]);
yield ({
type: match[1] === "#" ? "PrivateIdentifier" : "IdentifierName",
value: match[0]
});
continue;
}
StringLiteral.lastIndex = lastIndex;
if (match = StringLiteral.exec(input)) {
lastIndex = StringLiteral.lastIndex;
lastSignificantToken = match[0];
postfixIncDec = true;
yield ({
type: "StringLiteral",
value: match[0],
closed: match[2] !== void 0
});
continue;
}
NumericLiteral.lastIndex = lastIndex;
if (match = NumericLiteral.exec(input)) {
lastIndex = NumericLiteral.lastIndex;
lastSignificantToken = match[0];
postfixIncDec = true;
yield ({
type: "NumericLiteral",
value: match[0]
});
continue;
}
Template.lastIndex = lastIndex;
if (match = Template.exec(input)) {
lastIndex = Template.lastIndex;
lastSignificantToken = match[0];
if (match[1] === "${") {
lastSignificantToken = "?InterpolationInTemplate";
stack.push({
tag: "InterpolationInTemplate",
nesting: braces.length
});
postfixIncDec = false;
yield ({
type: "TemplateHead",
value: match[0]
});
} else {
postfixIncDec = true;
yield ({
type: "NoSubstitutionTemplate",
value: match[0],
closed: match[1] === "`"
});
}
continue;
}
break;
case "JSXTag":
case "JSXTagEnd":
JSXPunctuator.lastIndex = lastIndex;
if (match = JSXPunctuator.exec(input)) {
lastIndex = JSXPunctuator.lastIndex;
nextLastSignificantToken = match[0];
switch (match[0]) {
case "<":
stack.push({tag: "JSXTag"});
break;
case ">":
stack.pop();
if (lastSignificantToken === "/" || mode.tag === "JSXTagEnd") {
nextLastSignificantToken = "?JSX";
postfixIncDec = true;
} else {
stack.push({tag: "JSXChildren"});
}
break;
case "{":
stack.push({
tag: "InterpolationInJSX",
nesting: braces.length
});
nextLastSignificantToken = "?InterpolationInJSX";
postfixIncDec = false;
break;
case "/":
if (lastSignificantToken === "<") {
stack.pop();
if (stack[stack.length - 1].tag === "JSXChildren") {
stack.pop();
}
stack.push({tag: "JSXTagEnd"});
}
}
lastSignificantToken = nextLastSignificantToken;
yield ({
type: "JSXPunctuator",
value: match[0]
});
continue;
}
JSXIdentifier.lastIndex = lastIndex;
if (match = JSXIdentifier.exec(input)) {
lastIndex = JSXIdentifier.lastIndex;
lastSignificantToken = match[0];
yield ({
type: "JSXIdentifier",
value: match[0]
});
continue;
}
JSXString.lastIndex = lastIndex;
if (match = JSXString.exec(input)) {
lastIndex = JSXString.lastIndex;
lastSignificantToken = match[0];
yield ({
type: "JSXString",
value: match[0],
closed: match[2] !== void 0
});
continue;
}
break;
case "JSXChildren":
JSXText.lastIndex = lastIndex;
if (match = JSXText.exec(input)) {
lastIndex = JSXText.lastIndex;
lastSignificantToken = match[0];
yield ({
type: "JSXText",
value: match[0]
});
continue;
}
switch (input[lastIndex]) {
case "<":
stack.push({tag: "JSXTag"});
lastIndex++;
lastSignificantToken = "<";
yield ({
type: "JSXPunctuator",
value: "<"
});
continue;
case "{":
stack.push({
tag: "InterpolationInJSX",
nesting: braces.length
});
lastIndex++;
lastSignificantToken = "?InterpolationInJSX";
postfixIncDec = false;
yield ({
type: "JSXPunctuator",
value: "{"
});
continue;
}
}
WhiteSpace.lastIndex = lastIndex;
if (match = WhiteSpace.exec(input)) {
lastIndex = WhiteSpace.lastIndex;
yield ({
type: "WhiteSpace",
value: match[0]
});
continue;
}
LineTerminatorSequence.lastIndex = lastIndex;
if (match = LineTerminatorSequence.exec(input)) {
lastIndex = LineTerminatorSequence.lastIndex;
postfixIncDec = false;
if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
lastSignificantToken = "?NoLineTerminatorHere";
}
yield ({
type: "LineTerminatorSequence",
value: match[0]
});
continue;
}
MultiLineComment.lastIndex = lastIndex;
if (match = MultiLineComment.exec(input)) {
lastIndex = MultiLineComment.lastIndex;
if (Newline.test(match[0])) {
postfixIncDec = false;
if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
lastSignificantToken = "?NoLineTerminatorHere";
}
}
yield ({
type: "MultiLineComment",
value: match[0],
closed: match[1] !== void 0
});
continue;
}
SingleLineComment.lastIndex = lastIndex;
if (match = SingleLineComment.exec(input)) {
lastIndex = SingleLineComment.lastIndex;
postfixIncDec = false;
yield ({
type: "SingleLineComment",
value: match[0]
});
continue;
}
firstCodePoint = String.fromCodePoint(input.codePointAt(lastIndex));
lastIndex += firstCodePoint.length;
lastSignificantToken = firstCodePoint;
postfixIncDec = false;
yield ({
type: mode.tag.startsWith("JSX") ? "JSXInvalid" : "Invalid",
value: firstCodePoint
});
}
};
};
const jsTokensSourceCode = jsTokensWrapperFunction.toString();
const jsxSourceCode = `
function Comment(props) {
return (
<div className="Comment">
<div className="UserInfo">
<Avatar user={props.author} />
<div className="UserInfo-name">
{props.author.name}
</div>
</div>
<div className="Comment-text">
{props.text}
</div>
<div className="Comment-date">
{formatDate(props.date)}
</div>
</div>
);
}
`;
const jsTokens = jsTokensWrapperFunction();
class Benchmark {
runIteration() {
this.tokenCount = 0;
for (var i = 0; i < 25; i++) {
for (const token of jsTokens(jsTokensSourceCode))
this.tokenCount++;
}
for (var i = 0; i < 250; i++) {
for (const token of jsTokens(jsxSourceCode, { jsx: true }))
this.tokenCount++;
}
}
validate(iterations) {
if (this.tokenCount !== 113975)
throw new Error(`this.tokenCount of ${this.tokenCount} is invalid!`);
}
}
