diff --git a/lib/ast.js b/lib/ast.js index 54111e58..c08e6222 100644 --- a/lib/ast.js +++ b/lib/ast.js @@ -538,7 +538,7 @@ var AST_PrefixedTemplateString = DEFNODE("PrefixedTemplateString", "template_str var AST_TemplateString = DEFNODE("TemplateString", "segments", { $documentation: "A template string literal", $propdoc: { - segments: "[string|AST_Expression]* One or more segments. They can be the parts that are evaluated, or the raw string parts." + segments: "[AST_TemplateSegment|AST_Expression]* One or more segments, starting with AST_TemplateSegment. AST_Expression may follow AST_TemplateSegment, but each AST_Expression must be followed by AST_TemplateSegment." }, _walk: function(visitor) { return visitor._visit(this, function(){ @@ -551,6 +551,14 @@ var AST_TemplateString = DEFNODE("TemplateString", "segments", { } }); +var AST_TemplateSegment = DEFNODE("TemplateSegment", "value raw", { + $documentation: "A segment of a template string literal", + $propdoc: { + value: "Content of the segment", + raw: "Raw content of the segment" + } +}); + /* -----[ JUMPS ]----- */ var AST_Jump = DEFNODE("Jump", null, { diff --git a/lib/compress.js b/lib/compress.js index 2a22e7f1..47fb220a 100644 --- a/lib/compress.js +++ b/lib/compress.js @@ -952,6 +952,9 @@ merge(Compressor.prototype, { (function (def){ def(AST_Node, function(){ return false }); def(AST_String, function(){ return true }); + def(AST_TemplateString, function(){ + return this.segments.length === 1; + }); def(AST_UnaryPrefix, function(){ return this.operator == "typeof"; }); @@ -1056,6 +1059,10 @@ merge(Compressor.prototype, { def(AST_Constant, function(){ return this.getValue(); }); + def(AST_TemplateString, function() { + if (this.segments.length !== 1) throw def; + return this.segments[0].value; + }); def(AST_UnaryPrefix, function(compressor){ var e = this.expression; switch (this.operator) { @@ -2988,4 +2995,37 @@ merge(Compressor.prototype, { return self; }); + OPT(AST_TemplateString, function(self, compressor){ + if (!compressor.option("evaluate") + || compressor.parent() instanceof AST_PrefixedTemplateString) + return self; + + var segments = []; + for (var i = 0; i < self.segments.length; i++) { + if (self.segments[i] instanceof AST_Node) { + var result = self.segments[i].evaluate(compressor); + // No result[1] means nothing to stringify + if (result.length === 1) { + segments.push(result[0]); + continue; + } + // Evaluate length + if (result[0].print_to_string().length + 3 /* ${} */ < (result[1]+"").length) { + segments.push(result[0]); + continue; + } + // There should always be a previous and next segment if segment is a node + segments[segments.length - 1].value = segments[segments.length - 1].value + result[1] + self.segments[++i].value; + } else { + segments.push(self.segments[i]); + } + } + self.segments = segments; + + return self; + }); + + OPT(AST_PrefixedTemplateString, function(self, compressor){ + return self; + }); })(); diff --git a/lib/output.js b/lib/output.js index e309f5bd..f75f4243 100644 --- a/lib/output.js +++ b/lib/output.js @@ -125,7 +125,22 @@ function OutputStream(options) { function quote_double() { return '"' + str.replace(/\x22/g, '\\"') + '"'; } + function quote_template() { + if (!options.ascii_only) { + str = str.replace(/\\(n|r|u2028|u2029)/g, function(s, c) { + switch(c) { + case "n": return "\n"; + case "r": return "\r"; + case "u2028": return "\u2028"; + case "u2029": return "\u2029"; + } + return s; + }); + } + return '`' + str.replace(/`/g, '\\`') + '`'; + } if (options.ascii_only) str = to_ascii(str); + if (quote === "`") return quote_template(); switch (options.quote_style) { case 1: return quote_single(); @@ -387,6 +402,10 @@ function OutputStream(options) { } print(encoded); }, + print_template_string_chars: function(str) { + var encoded = encode_string(str, '`'); + return print(encoded.substr(1, encoded.length - 2)); + }, encode_string : encode_string, next_indent : next_indent, with_indent : with_indent, @@ -889,14 +908,18 @@ function OutputStream(options) { self.template_string.print(output); }); DEFPRINT(AST_TemplateString, function(self, output) { + var is_tagged = output.parent() instanceof AST_PrefixedTemplateString; + output.print("`"); for (var i = 0; i < self.segments.length; i++) { - if (typeof self.segments[i] !== "string") { + if (!(self.segments[i] instanceof AST_TemplateSegment)) { output.print("${"); self.segments[i].print(output); output.print("}"); + } else if (is_tagged) { + output.print(self.segments[i].raw); } else { - output.print(self.segments[i]); + output.print_template_string_chars(self.segments[i].value); } } output.print("`"); diff --git a/lib/parse.js b/lib/parse.js index 1c0482f3..34e42e7f 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -120,7 +120,7 @@ var PUNC_AFTER_EXPRESSION = makePredicate(characters(";]),:")); var PUNC_BEFORE_EXPRESSION = makePredicate(characters("[{(,.;:")); -var PUNC_CHARS = makePredicate(characters("[]{}(),;:`")); +var PUNC_CHARS = makePredicate(characters("[]{}(),;:")); var REGEXP_MODIFIERS = makePredicate(characters("gmsiy")); @@ -269,6 +269,8 @@ function tokenizer($TEXT, filename, html5_comments, shebang) { tokcol : 0, newline_before : false, regex_allowed : false, + brace_counter : 0, + template_braces : [], comments_before : [], directives : {}, directive_stack : [] @@ -487,6 +489,40 @@ function tokenizer($TEXT, filename, html5_comments, shebang) { return tok; }); + var read_template_characters = with_eof_error("SyntaxError: Unterminated template", function(begin){ + if (begin) { + S.template_braces.push(S.brace_counter); + } + var content = "", raw = "", ch, tok; + next(); + while ((ch = next(true)) !== "`") { + if (ch === "$" && peek() === "{") { + next(); + S.brace_counter++; + tok = token(begin ? "template_head" : "template_substitution", content); + tok.begin = begin; + tok.raw = raw; + tok.end = false; + return tok; + } + + raw += ch; + if (ch === "\\") { + var tmp = S.pos; + ch = read_escaped_char(); + raw += S.text.substr(tmp, S.pos - tmp); + } + + content += ch; + } + S.template_braces.pop(); + tok = token(begin ? "template_head" : "template_substitution", content); + tok.begin = begin; + tok.raw = raw; + tok.end = true; + return tok; + }); + function skip_line_comment(type) { var regex_allowed = S.regex_allowed; var i = find_eol(), ret; @@ -688,6 +724,16 @@ function tokenizer($TEXT, filename, html5_comments, shebang) { return tok; } case 61: return handle_eq_sign(); + case 96: return read_template_characters(true); + case 123: + S.brace_counter++; + break; + case 125: + S.brace_counter--; + if (S.template_braces.length > 0 + && S.template_braces[S.template_braces.length - 1] === S.brace_counter) + return read_template_characters(false); + break; } if (is_digit(code)) return read_num(); if (PUNC_CHARS(ch)) return token("punc", next()); @@ -939,6 +985,7 @@ function parse($TEXT, options) { }); } return stat; + case "template_head": case "num": case "regexp": case "operator": @@ -960,7 +1007,6 @@ function parse($TEXT, options) { }); case "[": case "(": - case "`": return simple_statement(); case ";": S.in_directives = false; @@ -1600,8 +1646,6 @@ function parse($TEXT, options) { return subscripts(array_(), allow_calls); case "{": return subscripts(object_or_object_destructuring_(), allow_calls); - case "`": - return subscripts(template_string(), allow_calls); } unexpected(); } @@ -1619,6 +1663,9 @@ function parse($TEXT, options) { cls.end = prev(); return subscripts(cls, allow_calls); } + if (is("template_head")) { + return subscripts(template_string(), allow_calls); + } if (ATOMIC_START_TOKEN[S.token.type]) { return subscripts(as_atom_node(), allow_calls); } @@ -1626,28 +1673,29 @@ function parse($TEXT, options) { }; function template_string() { - var tokenizer_S = S.input, start = S.token, segments = [], segment = "", ch; + var segments = [], start = S.token; - while ((ch = tokenizer_S.next()) !== "`") { - if (ch === "$" && tokenizer_S.peek() === "{") { - segments.push(segment); segment = ""; - tokenizer_S.next(); - next(); - segments.push(expression()); - if (!is("punc", "}")) { - // force error message - expect("}"); - } - continue; - } - segment += ch; - if (ch === "\\") { - segment += tokenizer_S.next(); + segments.push(new AST_TemplateSegment({ + start: S.token, + raw: S.token.raw, + value: S.token.value, + end: S.token + })); + while (S.token.end === false) { + next(); + segments.push(expression()); + + if (!is_token("template_substitution")) { + unexpected(); } + + segments.push(new AST_TemplateSegment({ + start: S.token, + raw: S.token.raw, + value: S.token.value, + end: S.token + })); } - - segments.push(segment); - next(); return new AST_TemplateString({ @@ -2033,6 +2081,13 @@ function parse($TEXT, options) { end : prev() }), true); } + if (is("template_head")) { + return subscripts(new AST_PrefixedTemplateString({ + start: start, + prefix: expr, + template_string: template_string() + }), allow_calls); + } return expr; }; @@ -2189,13 +2244,6 @@ function parse($TEXT, options) { }); return arrow_function(expr); } - if ((expr instanceof AST_SymbolRef || expr instanceof AST_PropAccess) && is("punc", "`")) { - return new AST_PrefixedTemplateString({ - start: start, - prefix: expr, - template_string: template_string() - }) - } if (commas && is("punc", ",")) { next(); return new AST_Seq({ diff --git a/lib/transform.js b/lib/transform.js index ae839417..05e04853 100644 --- a/lib/transform.js +++ b/lib/transform.js @@ -231,4 +231,16 @@ TreeTransformer.prototype = new TreeWalker; self.expression = self.expression.transform(tw); }); + _(AST_TemplateString, function(self, tw) { + for (var i = 0; i < self.segments.length; i++) { + if (!(self.segments[i] instanceof AST_TemplateSegment)) { + self.segments[i] = self.segments[i].transform(tw); + } + } + }); + + _(AST_PrefixedTemplateString, function(self, tw) { + self.template_string = self.template_string.transform(tw); + }); + })(); diff --git a/test/compress/harmony.js b/test/compress/harmony.js index 9d17f4f4..838501ab 100644 --- a/test/compress/harmony.js +++ b/test/compress/harmony.js @@ -78,24 +78,6 @@ typeof_arrow_functions: { expect_exact: "var foo=\"function\";" } -template_strings: { - input: { - ``; - `xx\`x`; - `${ foo + 2 }`; - ` foo ${ bar + `baz ${ qux }` }`; - } - expect_exact: "``;`xx\\`x`;`${foo+2}`;` foo ${bar+`baz ${qux}`}`;"; -} - -template_string_prefixes: { - input: { - String.raw`foo`; - foo `bar`; - } - expect_exact: "String.raw`foo`;foo`bar`;"; -} - destructuring_arguments: { input: { (function ( a ) { }); diff --git a/test/compress/template-string.js b/test/compress/template-string.js new file mode 100644 index 00000000..df4ff897 --- /dev/null +++ b/test/compress/template-string.js @@ -0,0 +1,331 @@ +template_strings: { + beautify = { + quote_style: 3 + } + input: { + ``; + `xx\`x`; + `${ foo + 2 }`; + ` foo ${ bar + `baz ${ qux }` }`; + } + expect_exact: "``;`xx\\`x`;`${foo+2}`;` foo ${bar+`baz ${qux}`}`;"; +} + +template_string_prefixes: { + beautify = { + quote_style: 3 + } + input: { + String.raw`foo`; + foo `bar`; + } + expect_exact: "String.raw`foo`;foo`bar`;"; +} + +template_strings_ascii_only: { + beautify = { + ascii_only: true, + quote_style: 3 + } + input: { + var foo = `foo + bar + ↂωↂ`; + var bar = `\``; + } + expect_exact: "var foo=`foo\\n bar\\n \\u2182\\u03c9\\u2182`;var bar=`\\``;" +} + +template_strings_without_ascii_only: { + beautify = { + quote_style: 3 + } + input: { + var foo = `foo + bar + ↂωↂ` + } + expect_exact: "var foo=`foo\n bar\n ↂωↂ`;" +} + +template_string_with_constant_expression: { + options = { + evaluate: true + } + beautify = { + quote_style: 3 + } + input: { + var foo = `${4 + 4} equals 4 + 4`; + } + expect: { + var foo = `8 equals 4 + 4`; + } +} + +template_string_with_predefined_constants: { + options = { + evaluate: true + } + beautify = { + quote_style: 3 + } + input: { + var foo = `This is ${undefined}`; + var bar = `This is ${NaN}`; + var baz = `This is ${null}`; + var foofoo = `This is ${Infinity}`; + var foobar = "This is ${1/0}"; + var foobaz = 'This is ${1/0}'; + var barfoo = "This is ${NaN}"; + var bazfoo = "This is ${null}"; + var bazbaz = `This is ${1/0}`; + var barbar = `This is ${0/0}`; + var barbar = "This is ${0/0}"; + var barber = 'This is ${0/0}'; + + var a = `${4**11}`; // 8 in template vs 7 chars - 4194304 + var b = `${4**12}`; // 8 in template vs 8 chars - 16777216 + var c = `${4**14}`; // 8 in template vs 9 chars - 268435456 + } + expect: { + var foo = `This is undefined`; + var bar = `This is NaN`; + var baz = `This is null`; + var foofoo = `This is ${1/0}`; + var foobar = "This is ${1/0}"; + var foobaz = 'This is ${1/0}'; + var barfoo = "This is ${NaN}"; + var bazfoo = "This is ${null}"; + var bazbaz = `This is ${1/0}`; + var barbar = `This is NaN`; + var barbar = "This is ${0/0}"; + var barber = 'This is ${0/0}'; + + var a = `4194304`; + var b = `16777216`; // Potential for further concatentation + var c = `${4**14}`; // Not worth converting + } +} + +template_string_evaluate_with_many_segments: { + options = { + evaluate: true + } + beautify = { + quote_style: 3 + } + input: { + var foo = `Hello ${guest()}, welcome to ${location()}${"."}`; + var bar = `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`; + var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`; + var buzz = `${1}${foobar()}${2}${foobar()}${3}${foobar()}`; + } + expect: { + var foo = `Hello ${guest()}, welcome to ${location()}.`; + var bar = `1234567890`; + var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`; + var buzz = `1${foobar()}2${foobar()}3${foobar()}`; + } +} + +template_string_with_many_segments: { + beautify = { + quote_style: 3 + } + input: { + var foo = `Hello ${guest()}, welcome to ${location()}${"."}`; + var bar = `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`; + var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`; + var buzz = `${1}${foobar()}${2}${foobar()}${3}${foobar()}`; + } + expect: { + var foo = `Hello ${guest()}, welcome to ${location()}${"."}`; + var bar = `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`; + var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`; + var buzz = `${1}${foobar()}${2}${foobar()}${3}${foobar()}`; + } +} + +template_string_to_normal_string: { + options = { + evaluate: true + } + beautify = { + quote_style: 0 + } + input: { + var foo = `This is ${undefined}`; + var bar = "Decimals " + `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`; + } + expect: { + var foo = `This is undefined`; + var bar = "Decimals 1234567890"; + } +} + +template_concattenating_string: { + options = { + evaluate: true + } + beautify = { + quote_style: 3 // Yes, keep quotes + } + input: { + var foo = "Have a nice " + `day. ${`day. ` + `day.`}`; + var bar = "Have a nice " + `${day()}`; + } + expect: { + var foo = "Have a nice day. day. day."; + var bar = "Have a nice " + `${day()}`; + } +} + +evaluate_nested_templates: { + options = { + evaluate: true + } + beautify = { + quote_style: 0 + } + input: { + var baz = `${`${`${`foo`}`}`}`; + } + expect: { + var baz = `foo`; + } +} + +enforce_double_quotes: { + beautify = { + quote_style: 1 + } + input: { + var foo = `Hello world`; + var bar = `Hello ${'world'}`; + var baz = `Hello ${world()}`; + } + expect: { + var foo = `Hello world`; + var bar = `Hello ${"world"}`; + var baz = `Hello ${world()}`; + } +} + +enforce_single_quotes: { + beautify = { + quote_style: 2 + } + input: { + var foo = `Hello world`; + var bar = `Hello ${"world"}`; + var baz = `Hello ${world()}`; + } + expect: { + var foo = `Hello world`; + var bar = `Hello ${'world'}`; + var baz = `Hello ${world()}`; + } +} + +enforce_double_quotes_and_evaluate: { + beautify = { + quote_style: 1 + } + options = { + evaluate: true + } + input: { + var foo = `Hello world`; + var bar = `Hello ${'world'}`; + var baz = `Hello ${world()}`; + } + expect: { + var foo = `Hello world`; + var bar = `Hello world`; + var baz = `Hello ${world()}`; + } +} + +enforce_single_quotes_and_evaluate: { + beautify = { + quote_style: 2 + } + options = { + evaluate: true + } + input: { + var foo = `Hello world`; + var bar = `Hello ${"world"}`; + var baz = `Hello ${world()}`; + } + expect: { + var foo = `Hello world`; + var bar = `Hello world`; + var baz = `Hello ${world()}`; + } +} + +respect_inline_script: { + beautify = { + inline_script: true, + quote_style: 3 + } + input: { + var foo = `${content}`; + var bar = ``; + } + expect_exact: "var foo=`<\\/script>${content}`;var bar=`\\x3c!--`;var baz=`--\\x3e`;"; +} + +do_not_optimize_tagged_template_1: { + beautify = { + quote_style: 0 + } + options = { + evaluate: true + } + input: { + var foo = tag`Shall not be optimized. ${"But " + "this " + "is " + "fine."}`; + var bar = tag`Don't even mind changing my quotes!`; + } + expect_exact: + 'var foo=tag`Shall not be optimized. ${"But this is fine."}`;var bar=tag`Don\'t even mind changing my quotes!`;'; +} + +do_not_optimize_tagged_template_2: { + options = { + evaluate: true + } + input: { + var foo = tag`test` + " something out"; + } + expect_exact: 'var foo=tag`test`+" something out";'; +} + +keep_raw_content_in_tagged_template: { + options = { + evaluate: true + } + input: { + var foo = tag`\u0020\u{20}\u{00020}\x20\40\040 `; + } + expect_exact: "var foo=tag`\\u0020\\u{20}\\u{00020}\\x20\\40\\040 `;"; +} + +allow_chained_templates: { + input: { + var foo = tag`a``b``c``d`; + } + expect: { + var foo = tag`a``b``c``d`; + } +} + +check_escaped_chars: { + input: { + var foo = `\u0020\u{20}\u{00020}\x20\40\040 `; + } + expect_exact: "var foo=` `;"; +} diff --git a/test/mocha/template-string.js b/test/mocha/template-string.js new file mode 100644 index 00000000..cc6ba7ab --- /dev/null +++ b/test/mocha/template-string.js @@ -0,0 +1,33 @@ +var assert = require("assert"); +var uglify = require("../../"); + +describe("Template string", function() { + it("Should not accept invalid sequences", function() { + var tests = [ + // Stress invalid expression + "var foo = `Hello ${]}`", + "var foo = `Test 123 ${>}`", + "var foo = `Blah ${;}`", + + // Stress invalid template_substitution after expression + "var foo = `Blablabla ${123 456}`", + "var foo = `Blub ${123;}`", + "var foo = `Bleh ${a b}`" + ]; + + var exec = function(test) { + return function() { + uglify.parse(test); + } + }; + + var fail = function(e) { + return e instanceof uglify.JS_Parse_Error + && /^SyntaxError: Unexpected token: /.test(e.message); + }; + + for (var i = 0; i < tests.length; i++) { + assert.throws(exec(tests[i]), fail, tests[i]); + } + }); +});