diff --git a/packages/compiler/src/ml_parser/ast.ts b/packages/compiler/src/ml_parser/ast.ts index 776fbffaaff2..1d608a28861a 100644 --- a/packages/compiler/src/ml_parser/ast.ts +++ b/packages/compiler/src/ml_parser/ast.ts @@ -89,8 +89,8 @@ export class Comment implements BaseNode { export class Block implements BaseNode { constructor( public name: string, public parameters: BlockParameter[], public children: Node[], - public sourceSpan: ParseSourceSpan, public startSourceSpan: ParseSourceSpan, - public endSourceSpan: ParseSourceSpan|null = null) {} + public sourceSpan: ParseSourceSpan, public nameSpan: ParseSourceSpan, + public startSourceSpan: ParseSourceSpan, public endSourceSpan: ParseSourceSpan|null = null) {} visit(visitor: Visitor, context: any) { return visitor.visitBlock(this, context); diff --git a/packages/compiler/src/ml_parser/html_whitespaces.ts b/packages/compiler/src/ml_parser/html_whitespaces.ts index 279cf7889237..3863ad078e72 100644 --- a/packages/compiler/src/ml_parser/html_whitespaces.ts +++ b/packages/compiler/src/ml_parser/html_whitespaces.ts @@ -101,7 +101,7 @@ export class WhitespaceVisitor implements html.Visitor { visitBlock(block: html.Block, context: any): any { return new html.Block( block.name, block.parameters, visitAllWithSiblings(this, block.children), block.sourceSpan, - block.startSourceSpan, block.endSourceSpan); + block.nameSpan, block.startSourceSpan, block.endSourceSpan); } visitBlockParameter(parameter: html.BlockParameter, context: any) { diff --git a/packages/compiler/src/ml_parser/icu_ast_expander.ts b/packages/compiler/src/ml_parser/icu_ast_expander.ts index 6f56a46dd208..613fff283e64 100644 --- a/packages/compiler/src/ml_parser/icu_ast_expander.ts +++ b/packages/compiler/src/ml_parser/icu_ast_expander.ts @@ -91,7 +91,7 @@ class _Expander implements html.Visitor { visitBlock(block: html.Block, context: any) { return new html.Block( block.name, block.parameters, html.visitAll(this, block.children), block.sourceSpan, - block.startSourceSpan, block.endSourceSpan); + block.nameSpan, block.startSourceSpan, block.endSourceSpan); } visitBlockParameter(parameter: html.BlockParameter, context: any) { diff --git a/packages/compiler/src/ml_parser/lexer.ts b/packages/compiler/src/ml_parser/lexer.ts index 8ab09b013ce3..f2842341809f 100644 --- a/packages/compiler/src/ml_parser/lexer.ts +++ b/packages/compiler/src/ml_parser/lexer.ts @@ -918,7 +918,7 @@ class _Tokenizer { } if (this._tokenizeBlocks && !this._inInterpolation && !this._isInExpansion() && - (this._isBlockStart() || this._cursor.peek() === chars.$RBRACE)) { + (this._cursor.peek() === chars.$AT || this._cursor.peek() === chars.$RBRACE)) { return true; } @@ -944,19 +944,6 @@ class _Tokenizer { return false; } - private _isBlockStart(): boolean { - if (this._tokenizeBlocks && this._cursor.peek() === chars.$AT) { - const tmp = this._cursor.clone(); - - // If it is, also verify that the next character is a valid block identifier. - tmp.advance(); - if (isBlockNameChar(tmp.peek())) { - return true; - } - } - return false; - } - private _readUntil(char: number): string { const start = this._cursor.clone(); this._attemptUntilChar(char); diff --git a/packages/compiler/src/ml_parser/parser.ts b/packages/compiler/src/ml_parser/parser.ts index 45aaca43e8f4..d70550790573 100644 --- a/packages/compiler/src/ml_parser/parser.ts +++ b/packages/compiler/src/ml_parser/parser.ts @@ -474,7 +474,7 @@ class _TreeBuilder { const span = new ParseSourceSpan(token.sourceSpan.start, end, token.sourceSpan.fullStart); // Create a separate `startSpan` because `span` will be modified when there is an `end` span. const startSpan = new ParseSourceSpan(token.sourceSpan.start, end, token.sourceSpan.fullStart); - const block = new html.Block(token.parts[0], parameters, [], span, startSpan); + const block = new html.Block(token.parts[0], parameters, [], span, token.sourceSpan, startSpan); this._pushContainer(block, false); } @@ -500,7 +500,7 @@ class _TreeBuilder { const span = new ParseSourceSpan(token.sourceSpan.start, end, token.sourceSpan.fullStart); // Create a separate `startSpan` because `span` will be modified when there is an `end` span. const startSpan = new ParseSourceSpan(token.sourceSpan.start, end, token.sourceSpan.fullStart); - const block = new html.Block(token.parts[0], parameters, [], span, startSpan); + const block = new html.Block(token.parts[0], parameters, [], span, token.sourceSpan, startSpan); this._pushContainer(block, false); // Incomplete blocks don't have children so we close them immediately and report an error. diff --git a/packages/compiler/src/render3/r3_ast.ts b/packages/compiler/src/render3/r3_ast.ts index c182e9eb1d37..6fe2c2346dd5 100644 --- a/packages/compiler/src/render3/r3_ast.ts +++ b/packages/compiler/src/render3/r3_ast.ts @@ -315,7 +315,8 @@ export class IfBlockBranch implements Node { } export class UnknownBlock implements Node { - constructor(public name: string, public sourceSpan: ParseSourceSpan) {} + constructor( + public name: string, public sourceSpan: ParseSourceSpan, public nameSpan: ParseSourceSpan) {} visit(visitor: Visitor): Result { return visitor.visitUnknownBlock(this); diff --git a/packages/compiler/src/render3/r3_control_flow.ts b/packages/compiler/src/render3/r3_control_flow.ts index 9e48563ac2e0..b5960f15baf9 100644 --- a/packages/compiler/src/render3/r3_control_flow.ts +++ b/packages/compiler/src/render3/r3_control_flow.ts @@ -163,7 +163,7 @@ export function createSwitchBlock( } if ((node.name !== 'case' || node.parameters.length === 0) && node.name !== 'default') { - unknownBlocks.push(new t.UnknownBlock(node.name, node.sourceSpan)); + unknownBlocks.push(new t.UnknownBlock(node.name, node.sourceSpan, node.nameSpan)); continue; } diff --git a/packages/compiler/src/render3/r3_template_transform.ts b/packages/compiler/src/render3/r3_template_transform.ts index 5706b60a30ed..ee43a862ea8e 100644 --- a/packages/compiler/src/render3/r3_template_transform.ts +++ b/packages/compiler/src/render3/r3_template_transform.ts @@ -383,7 +383,7 @@ class HtmlAstToIvyAst implements html.Visitor { } result = { - node: new t.UnknownBlock(block.name, block.sourceSpan), + node: new t.UnknownBlock(block.name, block.sourceSpan, block.nameSpan), errors: [new ParseError(block.sourceSpan, errorMessage)], }; break; diff --git a/packages/compiler/test/ml_parser/lexer_spec.ts b/packages/compiler/test/ml_parser/lexer_spec.ts index b7a6e1ccd29d..b345adc5452e 100644 --- a/packages/compiler/test/ml_parser/lexer_spec.ts +++ b/packages/compiler/test/ml_parser/lexer_spec.ts @@ -278,6 +278,1252 @@ describe('HtmlLexer', () => { ]); }); }); + + describe('escapable raw text', () => { + it('should parse text', () => { + expect(tokenizeAndHumanizeParts(`t\ne\rs\r\nt`)).toEqual([ + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 't\ne\ns\nt'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], + ]); + }); + + it('should detect entities', () => { + expect(tokenizeAndHumanizeParts(`&`)).toEqual([ + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, ''], + [TokenType.ENCODED_ENTITY, '&', '&'], + [TokenType.ESCAPABLE_RAW_TEXT, ''], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], + ]); + }); + + it('should ignore other opening tags', () => { + expect(tokenizeAndHumanizeParts(`a<div>`)).toEqual([ + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 'a
'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], + ]); + }); + + it('should ignore other closing tags', () => { + expect(tokenizeAndHumanizeParts(`a</test>`)).toEqual([ + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 'a'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], + ]); + }); + + it('should store the locations', () => { + expect(tokenizeAndHumanizeSourceSpans(`a`)).toEqual([ + [TokenType.TAG_OPEN_START, ''], + [TokenType.ESCAPABLE_RAW_TEXT, 'a'], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], + ]); + }); + }); + + describe('parsable data', () => { + it('should parse an SVG tag', () => { + expect(tokenizeAndHumanizeParts(`<svg:title>test</svg:title>`)).toEqual([ + [TokenType.TAG_OPEN_START, 'svg', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'test'], + [TokenType.TAG_CLOSE, 'svg', 'title'], + [TokenType.EOF], + ]); + }); + + it('should parse an SVG <title> tag with children', () => { + expect(tokenizeAndHumanizeParts(`<svg:title><f>test</f></svg:title>`)).toEqual([ + [TokenType.TAG_OPEN_START, 'svg', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'f'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'test'], + [TokenType.TAG_CLOSE, '', 'f'], + [TokenType.TAG_CLOSE, 'svg', 'title'], + [TokenType.EOF], + ]); + }); + }); + + describe('expansion forms', () => { + it('should parse an expansion form', () => { + expect( + tokenizeAndHumanizeParts( + '{one.two, three, =4 {four} =5 {five} foo {bar} }', {tokenizeExpansionForms: true})) + .toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=5'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'five'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, 'foo'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'bar'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], + ]); + }); + + it('should parse an expansion form with text elements surrounding it', () => { + expect(tokenizeAndHumanizeParts( + 'before{one.two, three, =4 {four}}after', {tokenizeExpansionForms: true})) + .toEqual([ + [TokenType.TEXT, 'before'], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, 'after'], + [TokenType.EOF], + ]); + }); + + it('should parse an expansion form as a tag single child', () => { + expect(tokenizeAndHumanizeParts( + '<div><span>{a, b, =4 {c}}</span></div>', {tokenizeExpansionForms: true})) + .toEqual([ + [TokenType.TAG_OPEN_START, '', 'div'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'span'], + [TokenType.TAG_OPEN_END], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'a'], + [TokenType.RAW_TEXT, 'b'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'c'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TAG_CLOSE, '', 'span'], + [TokenType.TAG_CLOSE, '', 'div'], + [TokenType.EOF], + ]); + }); + + it('should parse an expansion form with whitespace surrounding it', () => { + expect(tokenizeAndHumanizeParts( + '<div><span> {a, b, =4 {c}} </span></div>', {tokenizeExpansionForms: true})) + .toEqual([ + [TokenType.TAG_OPEN_START, '', 'div'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'span'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, ' '], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'a'], + [TokenType.RAW_TEXT, 'b'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'c'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, ' '], + [TokenType.TAG_CLOSE, '', 'span'], + [TokenType.TAG_CLOSE, '', 'div'], + [TokenType.EOF], + ]); + }); + + it('should parse an expansion forms with elements in it', () => { + expect(tokenizeAndHumanizeParts( + '{one.two, three, =4 {four <b>a</b>}}', {tokenizeExpansionForms: true})) + .toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four '], + [TokenType.TAG_OPEN_START, '', 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'a'], + [TokenType.TAG_CLOSE, '', 'b'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], + ]); + }); + + it('should parse an expansion forms containing an interpolation', () => { + expect(tokenizeAndHumanizeParts( + '{one.two, three, =4 {four {{a}}}}', {tokenizeExpansionForms: true})) + .toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four '], + [TokenType.INTERPOLATION, '{{', 'a', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], + ]); + }); + + it('should parse nested expansion forms', () => { + expect(tokenizeAndHumanizeParts( + `{one.two, three, =4 { {xx, yy, =x {one}} }}`, {tokenizeExpansionForms: true})) + .toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'xx'], + [TokenType.RAW_TEXT, 'yy'], + [TokenType.EXPANSION_CASE_VALUE, '=x'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'one'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, ' '], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], + ]); + }); + + describe('[line ending normalization', () => { + describe('{escapedString: true}', () => { + it('should normalize line-endings in expansion forms if `i18nNormalizeLineEndingsInICUs` is true', + () => { + const result = tokenizeWithoutErrors( + `{\r\n` + + ` messages.length,\r\n` + + ` plural,\r\n` + + ` =0 {You have \r\nno\r\n messages}\r\n` + + ` =1 {One {{message}}}}\r\n`, + { + tokenizeExpansionForms: true, + escapedString: true, + i18nNormalizeLineEndingsInICUs: true + }); + + expect(humanizeParts(result.tokens)).toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], + ]); + + expect(result.nonNormalizedIcuExpressions).toEqual([]); + }); + + it('should not normalize line-endings in ICU expressions when `i18nNormalizeLineEndingsInICUs` is not defined', + () => { + const result = tokenizeWithoutErrors( + `{\r\n` + + ` messages.length,\r\n` + + ` plural,\r\n` + + ` =0 {You have \r\nno\r\n messages}\r\n` + + ` =1 {One {{message}}}}\r\n`, + {tokenizeExpansionForms: true, escapedString: true}); + + expect(humanizeParts(result.tokens)).toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], + ]); + + expect(result.nonNormalizedIcuExpressions!.length).toBe(1); + expect(result.nonNormalizedIcuExpressions![0].sourceSpan.toString()) + .toEqual('\r\n messages.length'); + }); + + it('should not normalize line endings in nested expansion forms when `i18nNormalizeLineEndingsInICUs` is not defined', + () => { + const result = tokenizeWithoutErrors( + `{\r\n` + + ` messages.length, plural,\r\n` + + ` =0 { zero \r\n` + + ` {\r\n` + + ` p.gender, select,\r\n` + + ` male {m}\r\n` + + ` }\r\n` + + ` }\r\n` + + `}`, + {tokenizeExpansionForms: true, escapedString: true}); + expect(humanizeParts(result.tokens)).toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'zero \n '], + + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n p.gender'], + [TokenType.RAW_TEXT, 'select'], + [TokenType.EXPANSION_CASE_VALUE, 'male'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'm'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + + [TokenType.TEXT, '\n '], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], + ]); + + expect(result.nonNormalizedIcuExpressions!.length).toBe(2); + expect(result.nonNormalizedIcuExpressions![0].sourceSpan.toString()) + .toEqual('\r\n messages.length'); + expect(result.nonNormalizedIcuExpressions![1].sourceSpan.toString()) + .toEqual('\r\n p.gender'); + }); + }); + + describe('{escapedString: false}', () => { + it('should normalize line-endings in expansion forms if `i18nNormalizeLineEndingsInICUs` is true', + () => { + const result = tokenizeWithoutErrors( + `{\r\n` + + ` messages.length,\r\n` + + ` plural,\r\n` + + ` =0 {You have \r\nno\r\n messages}\r\n` + + ` =1 {One {{message}}}}\r\n`, + { + tokenizeExpansionForms: true, + escapedString: false, + i18nNormalizeLineEndingsInICUs: true + }); + + expect(humanizeParts(result.tokens)).toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], + ]); + + expect(result.nonNormalizedIcuExpressions).toEqual([]); + }); + + it('should not normalize line-endings in ICU expressions when `i18nNormalizeLineEndingsInICUs` is not defined', + () => { + const result = tokenizeWithoutErrors( + `{\r\n` + + ` messages.length,\r\n` + + ` plural,\r\n` + + ` =0 {You have \r\nno\r\n messages}\r\n` + + ` =1 {One {{message}}}}\r\n`, + {tokenizeExpansionForms: true, escapedString: false}); + + expect(humanizeParts(result.tokens)).toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], + ]); + + expect(result.nonNormalizedIcuExpressions!.length).toBe(1); + expect(result.nonNormalizedIcuExpressions![0].sourceSpan.toString()) + .toEqual('\r\n messages.length'); + }); + + it('should not normalize line endings in nested expansion forms when `i18nNormalizeLineEndingsInICUs` is not defined', + () => { + const result = tokenizeWithoutErrors( + `{\r\n` + + ` messages.length, plural,\r\n` + + ` =0 { zero \r\n` + + ` {\r\n` + + ` p.gender, select,\r\n` + + ` male {m}\r\n` + + ` }\r\n` + + ` }\r\n` + + `}`, + {tokenizeExpansionForms: true}); + + expect(humanizeParts(result.tokens)).toEqual([ + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'zero \n '], + + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n p.gender'], + [TokenType.RAW_TEXT, 'select'], + [TokenType.EXPANSION_CASE_VALUE, 'male'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'm'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + + [TokenType.TEXT, '\n '], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], + ]); + + expect(result.nonNormalizedIcuExpressions!.length).toBe(2); + expect(result.nonNormalizedIcuExpressions![0].sourceSpan.toString()) + .toEqual('\r\n messages.length'); + expect(result.nonNormalizedIcuExpressions![1].sourceSpan.toString()) + .toEqual('\r\n p.gender'); + }); + }); + }); + }); + + + describe('errors', () => { + it('should report unescaped "{" on error', () => { + expect(tokenizeAndHumanizeErrors(`<p>before { after</p>`, {tokenizeExpansionForms: true})) + .toEqual([[ + TokenType.RAW_TEXT, + `Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`, + '0:21', + ]]); + }); + + it('should report unescaped "{" as an error, even after a prematurely terminated interpolation', + () => { + expect(tokenizeAndHumanizeErrors( + `<code>{{b}<!---->}</code><pre>import {a} from 'a';</pre>`, + {tokenizeExpansionForms: true})) + .toEqual([[ + TokenType.RAW_TEXT, + `Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`, + '0:56', + ]]); + }); + + it('should include 2 lines of context in message', () => { + const src = '111\n222\n333\nE\n444\n555\n666\n'; + const file = new ParseSourceFile(src, 'file://'); + const location = new ParseLocation(file, 12, 123, 456); + const span = new ParseSourceSpan(location, location); + const error = new TokenError('**ERROR**', null!, span); + expect(error.toString()) + .toEqual(`**ERROR** ("\n222\n333\n[ERROR ->]E\n444\n555\n"): file://@123:456`); + }); + }); + + describe('unicode characters', () => { + it('should support unicode characters', () => { + expect(tokenizeAndHumanizeSourceSpans(`<p>İ</p>`)).toEqual([ + [TokenType.TAG_OPEN_START, '<p'], + [TokenType.TAG_OPEN_END, '>'], + [TokenType.TEXT, 'İ'], + [TokenType.TAG_CLOSE, '</p>'], + [TokenType.EOF, ''], + ]); + }); + }); + + describe('(processing escaped strings)', () => { + it('should unescape standard escape sequences', () => { + expect(tokenizeAndHumanizeParts('\\\' \\\' \\\'', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\' \' \''], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\" \\" \\"', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\" \" \"'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\` \\` \\`', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\` \` \`'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\\\ \\\\ \\\\', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\\ \\ \\'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\n \\n \\n', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\n \n \n'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\r{{\\r}}\\r', {escapedString: true})).toEqual([ + // post processing converts `\r` to `\n` + [TokenType.TEXT, '\n'], + [TokenType.INTERPOLATION, '{{', '\n', '}}'], + [TokenType.TEXT, '\n'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\v \\v \\v', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\v \v \v'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\t \\t \\t', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\t \t \t'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\b \\b \\b', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\b \b \b'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\f \\f \\f', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\f \f \f'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts( + '\\\' \\" \\` \\\\ \\n \\r \\v \\t \\b \\f', {escapedString: true})) + .toEqual([ + [TokenType.TEXT, '\' \" \` \\ \n \n \v \t \b \f'], + [TokenType.EOF], + ]); + }); + + it('should unescape null sequences', () => { + expect(tokenizeAndHumanizeParts('\\0', {escapedString: true})).toEqual([ + [TokenType.EOF], + ]); + // \09 is not an octal number so the \0 is taken as EOF + expect(tokenizeAndHumanizeParts('\\09', {escapedString: true})).toEqual([ + [TokenType.EOF], + ]); + }); + + it('should unescape octal sequences', () => { + // \19 is read as an octal `\1` followed by a normal char `9` + // \1234 is read as an octal `\123` followed by a normal char `4` + // \999 is not an octal number so its backslash just gets removed. + expect(tokenizeAndHumanizeParts( + '\\001 \\01 \\1 \\12 \\223 \\19 \\2234 \\999', {escapedString: true})) + .toEqual([ + [TokenType.TEXT, '\x01 \x01 \x01 \x0A \x93 \x019 \x934 999'], + [TokenType.EOF], + ]); + }); + + it('should unescape hex sequences', () => { + expect(tokenizeAndHumanizeParts('\\x12 \\x4F \\xDC', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\x12 \x4F \xDC'], + [TokenType.EOF], + ]); + }); + + it('should report an error on an invalid hex sequence', () => { + expect(tokenizeAndHumanizeErrors('\\xGG', {escapedString: true})).toEqual([ + [null, 'Invalid hexadecimal escape sequence', '0:2'] + ]); + + expect(tokenizeAndHumanizeErrors('abc \\x xyz', {escapedString: true})).toEqual([ + [TokenType.TEXT, 'Invalid hexadecimal escape sequence', '0:6'] + ]); + + expect(tokenizeAndHumanizeErrors('abc\\x', {escapedString: true})).toEqual([ + [TokenType.TEXT, 'Unexpected character "EOF"', '0:5'] + ]); + }); + + it('should unescape fixed length Unicode sequences', () => { + expect(tokenizeAndHumanizeParts('\\u0123 \\uABCD', {escapedString: true})).toEqual([ + [TokenType.TEXT, '\u0123 \uABCD'], + [TokenType.EOF], + ]); + }); + + it('should error on an invalid fixed length Unicode sequence', () => { + expect(tokenizeAndHumanizeErrors('\\uGGGG', {escapedString: true})).toEqual([ + [null, 'Invalid hexadecimal escape sequence', '0:2'] + ]); + }); + + it('should unescape variable length Unicode sequences', () => { + expect(tokenizeAndHumanizeParts( + '\\u{01} \\u{ABC} \\u{1234} \\u{123AB}', {escapedString: true})) + .toEqual([ + [TokenType.TEXT, '\u{01} \u{ABC} \u{1234} \u{123AB}'], + [TokenType.EOF], + ]); + }); + + it('should error on an invalid variable length Unicode sequence', () => { + expect(tokenizeAndHumanizeErrors('\\u{GG}', {escapedString: true})).toEqual([ + [null, 'Invalid hexadecimal escape sequence', '0:3'] + ]); + }); + + it('should unescape line continuations', () => { + expect(tokenizeAndHumanizeParts('abc\\\ndef', {escapedString: true})).toEqual([ + [TokenType.TEXT, 'abcdef'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeParts('\\\nx\\\ny\\\n', {escapedString: true})).toEqual([ + [TokenType.TEXT, 'xy'], + [TokenType.EOF], + ]); + }); + + it('should remove backslash from "non-escape" sequences', () => { + expect(tokenizeAndHumanizeParts('\a \g \~', {escapedString: true})).toEqual([ + [TokenType.TEXT, 'a g ~'], + [TokenType.EOF], + ]); + }); + + it('should unescape sequences in plain text', () => { + expect(tokenizeAndHumanizeParts('abc\ndef\\nghi\\tjkl\\`\\\'\\"mno', {escapedString: true})) + .toEqual([ + [TokenType.TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [TokenType.EOF], + ]); + }); + + it('should unescape sequences in raw text', () => { + expect(tokenizeAndHumanizeParts( + '<script>abc\ndef\\nghi\\tjkl\\`\\\'\\"mno</script>', {escapedString: true})) + .toEqual([ + [TokenType.TAG_OPEN_START, '', 'script'], + [TokenType.TAG_OPEN_END], + [TokenType.RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [TokenType.TAG_CLOSE, '', 'script'], + [TokenType.EOF], + ]); + }); + + it('should unescape sequences in escapable raw text', () => { + expect(tokenizeAndHumanizeParts( + '<title>abc\ndef\\nghi\\tjkl\\`\\\'\\"mno', {escapedString: true})) + .toEqual([ + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], + ]); + }); + + it('should parse over escape sequences in tag definitions', () => { + expect(tokenizeAndHumanizeParts('', {escapedString: true})) + .toEqual([ + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_NAME, '', 'c'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.ATTR_VALUE_TEXT, 'd'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], + ]); + }); + + it('should parse over escaped new line in tag definitions', () => { + const text = ''; + expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], + ]); + }); + + it('should parse over escaped characters in tag definitions', () => { + const text = ''; + expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], + ]); + }); + + it('should unescape characters in tag names', () => { + const text = ''; + expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, '', 'td'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 'td'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeSourceSpans(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, ''], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], + ]); + }); + + it('should unescape characters in attributes', () => { + const text = ''; + expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'd'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'e'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], + ]); + }); + + it('should parse over escaped new line in attribute values', () => { + const text = ''; + expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], + ]); + }); + + it('should tokenize the correct span when there are escape sequences', () => { + const text = + 'selector: "app-root",\ntemplate: "line 1\\n\\"line 2\\"\\nline 3",\ninputs: []'; + const range = { + startPos: 33, + startLine: 1, + startCol: 10, + endPos: 59, + }; + expect(tokenizeAndHumanizeParts(text, {range, escapedString: true})).toEqual([ + [TokenType.TEXT, 'line 1\n"line 2"\nline 3'], + [TokenType.EOF], + ]); + expect(tokenizeAndHumanizeSourceSpans(text, {range, escapedString: true})).toEqual([ + [TokenType.TEXT, 'line 1\\n\\"line 2\\"\\nline 3'], + [TokenType.EOF, ''], + ]); + }); + + it('should account for escape sequences when computing source spans ', () => { + const text = 'line 1\n' + // <- unescaped line break + 'line 2\\n' + // <- escaped line break + 'line 3\\\n' + // <- line continuation + ''; + expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], [TokenType.TEXT, 'line 1'], + [TokenType.TAG_CLOSE, '', 't'], [TokenType.TEXT, '\n'], + + [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], [TokenType.TEXT, 'line 2'], + [TokenType.TAG_CLOSE, '', 't'], [TokenType.TEXT, '\n'], + + [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'line 3'], // <- line continuation does not appear in token + [TokenType.TAG_CLOSE, '', 't'], + + [TokenType.EOF] + ]); + expect(tokenizeAndHumanizeLineColumn(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, '0:0'], + [TokenType.TAG_OPEN_END, '0:2'], + [TokenType.TEXT, '0:3'], + [TokenType.TAG_CLOSE, '0:9'], + [TokenType.TEXT, '0:13'], // <- real newline increments the row + + [TokenType.TAG_OPEN_START, '1:0'], + [TokenType.TAG_OPEN_END, '1:2'], + [TokenType.TEXT, '1:3'], + [TokenType.TAG_CLOSE, '1:9'], + [TokenType.TEXT, '1:13'], // <- escaped newline does not increment the row + + [TokenType.TAG_OPEN_START, '1:15'], + [TokenType.TAG_OPEN_END, '1:17'], + [TokenType.TEXT, '1:18'], // <- the line continuation increments the row + [TokenType.TAG_CLOSE, '2:0'], + + [TokenType.EOF, '2:4'], + ]); + expect(tokenizeAndHumanizeSourceSpans(text, {escapedString: true})).toEqual([ + [TokenType.TAG_OPEN_START, ''], + [TokenType.TEXT, 'line 1'], [TokenType.TAG_CLOSE, ''], [TokenType.TEXT, '\n'], + + [TokenType.TAG_OPEN_START, ''], + [TokenType.TEXT, 'line 2'], [TokenType.TAG_CLOSE, ''], [TokenType.TEXT, '\\n'], + + [TokenType.TAG_OPEN_START, ''], + [TokenType.TEXT, 'line 3\\\n'], [TokenType.TAG_CLOSE, ''], + + [TokenType.EOF, ''] + ]); + }); + }); + + describe('blocks', () => { + it('should parse a block without parameters', () => { + const expected = [ + [TokenType.BLOCK_OPEN_START, 'foo'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]; + + expect(tokenizeAndHumanizeParts('@foo {hello}')).toEqual(expected); + expect(tokenizeAndHumanizeParts('@foo () {hello}')).toEqual(expected); + expect(tokenizeAndHumanizeParts('@foo(){hello}')).toEqual(expected); + }); + + it('should parse a block with parameters', () => { + expect(tokenizeAndHumanizeParts('@for (item of items; track item.id) {hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_PARAMETER, 'track item.id'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse a block with a trailing semicolon after the parameters', () => { + expect(tokenizeAndHumanizeParts('@for (item of items;) {hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse a block with a space in its name', () => { + expect(tokenizeAndHumanizeParts('@else if {hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'else if'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + + expect(tokenizeAndHumanizeParts('@else if (foo !== 2) {hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'else if'], + [TokenType.BLOCK_PARAMETER, 'foo !== 2'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse a block with an arbitrary amount of spaces around the parentheses', () => { + const expected = [ + [TokenType.BLOCK_OPEN_START, 'foo'], + [TokenType.BLOCK_PARAMETER, 'a'], + [TokenType.BLOCK_PARAMETER, 'b'], + [TokenType.BLOCK_PARAMETER, 'c'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]; + + expect(tokenizeAndHumanizeParts('@foo(a; b; c){hello}')).toEqual(expected); + expect(tokenizeAndHumanizeParts('@foo (a; b; c) {hello}')).toEqual(expected); + expect(tokenizeAndHumanizeParts('@foo(a; b; c) {hello}')).toEqual(expected); + expect(tokenizeAndHumanizeParts('@foo (a; b; c){hello}')).toEqual(expected); + }); + + it('should parse a block with multiple trailing semicolons', () => { + expect(tokenizeAndHumanizeParts('@for (item of items;;;;;) {hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse a block with trailing whitespace', () => { + expect(tokenizeAndHumanizeParts('@foo {hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'foo'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse a block with no trailing semicolon', () => { + expect(tokenizeAndHumanizeParts('@for (item of items){hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'for'], + [TokenType.BLOCK_PARAMETER, 'item of items'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should handle semicolons, braces and parentheses used in a block parameter', () => { + const input = `@foo (a === ";"; b === ')'; c === "("; d === '}'; e === "{") {hello}`; + expect(tokenizeAndHumanizeParts(input)).toEqual([ + [TokenType.BLOCK_OPEN_START, 'foo'], + [TokenType.BLOCK_PARAMETER, `a === ";"`], + [TokenType.BLOCK_PARAMETER, `b === ')'`], + [TokenType.BLOCK_PARAMETER, `c === "("`], + [TokenType.BLOCK_PARAMETER, `d === '}'`], + [TokenType.BLOCK_PARAMETER, `e === "{"`], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should handle object literals and function calls in block parameters', () => { + expect(tokenizeAndHumanizeParts( + `@foo (on a({a: 1, b: 2}, false, {c: 3}); when b({d: 4})) {hello}`)) + .toEqual([ + [TokenType.BLOCK_OPEN_START, 'foo'], + [TokenType.BLOCK_PARAMETER, 'on a({a: 1, b: 2}, false, {c: 3})'], + [TokenType.BLOCK_PARAMETER, 'when b({d: 4})'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse block with unclosed parameters', () => { + expect(tokenizeAndHumanizeParts(`@foo (a === b {hello}`)).toEqual([ + [TokenType.INCOMPLETE_BLOCK_OPEN, 'foo'], + [TokenType.BLOCK_PARAMETER, 'a === b {hello}'], + [TokenType.EOF], + ]); + }); + + it('should parse block with stray parentheses in the parameter position', () => { + expect(tokenizeAndHumanizeParts(`@foo a === b) {hello}`)).toEqual([ + [TokenType.INCOMPLETE_BLOCK_OPEN, 'foo a'], + [TokenType.TEXT, '=== b) {hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse @ as an incomplete block', () => { + expect(tokenizeAndHumanizeParts(`@`)).toEqual([ + [TokenType.INCOMPLETE_BLOCK_OPEN, ''], + [TokenType.EOF], + ]); + }); + + it('should parse space followed by @ as an incomplete block', () => { + expect(tokenizeAndHumanizeParts(` @`)).toEqual([ + [TokenType.TEXT, ' '], + [TokenType.INCOMPLETE_BLOCK_OPEN, ''], + [TokenType.EOF], + ]); + }); + + it('should parse @ followed by space as an incomplete block', () => { + expect(tokenizeAndHumanizeParts(`@ `)).toEqual([ + [TokenType.INCOMPLETE_BLOCK_OPEN, ''], + [TokenType.TEXT, ' '], + [TokenType.EOF], + ]); + }); + + it('should parse @ followed by newline and text as an incomplete block', () => { + expect(tokenizeAndHumanizeParts(`@\nfoo`)).toEqual([ + [TokenType.INCOMPLETE_BLOCK_OPEN, ''], + [TokenType.TEXT, '\nfoo'], + [TokenType.EOF], + ]); + }); + + it('should parse incomplete block with no name', () => { + expect(tokenizeAndHumanizeParts(`foo bar @ baz clink`)).toEqual([ + [TokenType.TEXT, 'foo bar '], + [TokenType.INCOMPLETE_BLOCK_OPEN, ''], + [TokenType.TEXT, ' baz clink'], + [TokenType.EOF], + ]); + }); + + it('should parse incomplete block with space, then name', () => { + expect(tokenizeAndHumanizeParts(`@ if`)).toEqual([ + [TokenType.INCOMPLETE_BLOCK_OPEN, ''], + [TokenType.TEXT, ' if'], + [TokenType.EOF], + ]); + }); + + it('should report invalid quotes in a parameter', () => { + expect(tokenizeAndHumanizeErrors(`@foo (a === ") {hello}`)).toEqual([ + [TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:22'] + ]); + + expect(tokenizeAndHumanizeErrors(`@foo (a === "hi') {hello}`)).toEqual([ + [TokenType.BLOCK_PARAMETER, 'Unexpected character "EOF"', '0:25'] + ]); + }); + + it('should report unclosed object literal inside a parameter', () => { + expect(tokenizeAndHumanizeParts(`@foo ({invalid: true) hello}`)).toEqual([ + [TokenType.INCOMPLETE_BLOCK_OPEN, 'foo'], + [TokenType.BLOCK_PARAMETER, '{invalid: true'], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should handle a semicolon used in a nested string inside a block parameter', () => { + expect(tokenizeAndHumanizeParts(`@if (condition === "';'") {hello}`)).toEqual([ + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, `condition === "';'"`], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should handle a semicolon next to an escaped quote used in a block parameter', () => { + expect(tokenizeAndHumanizeParts('@if (condition === "\\";") {hello}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'condition === "\\";"'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse mixed text and html content in a block', () => { + expect(tokenizeAndHumanizeParts('@if (a === 1) {foo bar baz}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'a === 1'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'foo '], + [TokenType.TAG_OPEN_START, '', 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'bar'], + [TokenType.TAG_CLOSE, '', 'b'], + [TokenType.TEXT, ' baz'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse HTML tags with attributes containing curly braces inside blocks', () => { + expect(tokenizeAndHumanizeParts('@if (a === 1) {
}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'a === 1'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'div'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, '}'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_NAME, '', 'b'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, '{'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 'div'], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse HTML tags with attribute containing block syntax', () => { + expect(tokenizeAndHumanizeParts('
')).toEqual([ + [TokenType.TAG_OPEN_START, '', 'div'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, '@if (foo) {}'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 'div'], + [TokenType.EOF], + ]); + }); + + it('should parse nested blocks', () => { + expect(tokenizeAndHumanizeParts( + '@if (a) {' + + 'hello a' + + '@if {' + + 'hello unnamed' + + '@if (b) {' + + 'hello b' + + '@if (c) {' + + 'hello c' + + '}' + + '}' + + '}' + + '}')) + .toEqual([ + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'a'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello a'], + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello unnamed'], + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'b'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello b'], + [TokenType.BLOCK_OPEN_START, 'if'], + [TokenType.BLOCK_PARAMETER, 'c'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, 'hello c'], + [TokenType.BLOCK_CLOSE], + [TokenType.BLOCK_CLOSE], + [TokenType.BLOCK_CLOSE], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse a block containing an expansion', () => { + const result = tokenizeAndHumanizeParts( + '@foo {{one.two, three, =4 {four} =5 {five} foo {bar} }}', + {tokenizeExpansionForms: true}); + + expect(result).toEqual([ + [TokenType.BLOCK_OPEN_START, 'foo'], + [TokenType.BLOCK_OPEN_END], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=5'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'five'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, 'foo'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'bar'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse a block containing an interpolation', () => { + expect(tokenizeAndHumanizeParts('@foo {{{message}}}')).toEqual([ + [TokenType.BLOCK_OPEN_START, 'foo'], + [TokenType.BLOCK_OPEN_END], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.BLOCK_CLOSE], + [TokenType.EOF], + ]); + }); + + it('should parse an incomplete block start without parameters with surrounding text', () => { + expect(tokenizeAndHumanizeParts('My email frodo@baggins.com')).toEqual([ + [TokenType.TEXT, 'My email frodo'], + [TokenType.INCOMPLETE_BLOCK_OPEN, 'baggins'], + [TokenType.TEXT, '.com'], + [TokenType.EOF], + ]); + }); + + it('should parse an incomplete block start at the end of the input', () => { + expect(tokenizeAndHumanizeParts('My username is @frodo')).toEqual([ + [TokenType.TEXT, 'My username is '], + [TokenType.INCOMPLETE_BLOCK_OPEN, 'frodo'], + [TokenType.EOF], + ]); + }); + + it('should parse an incomplete block start with parentheses but without params', () => { + expect(tokenizeAndHumanizeParts('Use the @Input() decorator')).toEqual([ + [TokenType.TEXT, 'Use the '], + [TokenType.INCOMPLETE_BLOCK_OPEN, 'Input'], + [TokenType.TEXT, 'decorator'], + [TokenType.EOF], + ]); + }); + + it('should parse an incomplete block start with parentheses and params', () => { + expect(tokenizeAndHumanizeParts('Use @Input({alias: "foo"}) to alias the input')).toEqual([ + [TokenType.TEXT, 'Use '], + [TokenType.INCOMPLETE_BLOCK_OPEN, 'Input'], + [TokenType.BLOCK_PARAMETER, '{alias: "foo"}'], + [TokenType.TEXT, 'to alias the input'], + [TokenType.EOF], + ]); + }); + }); }); describe('attributes', () => { diff --git a/packages/language-service/src/template_target.ts b/packages/language-service/src/template_target.ts index bed3337dd788..4a5eafd1bbcd 100644 --- a/packages/language-service/src/template_target.ts +++ b/packages/language-service/src/template_target.ts @@ -380,6 +380,14 @@ class TemplateTargetVisitor implements t.Visitor { // nodes. return; } + if (last instanceof t.UnknownBlock && isWithin(this.position, last.nameSpan)) { + // Autocompletions such as `@\nfoo`, where a newline follows a bare `@`, would not work + // because the language service visitor sees us inside the subsequent text node. We deal with + // this with using a special-case: if we are completing inside the name span, we don't + // continue to the subsequent text node. + return; + } + if (isTemplateNodeWithKeyAndValue(node) && !isWithinKeyValue(this.position, node)) { // If cursor is within source span but not within key span or value span, // do not return the node. diff --git a/packages/language-service/test/completions_spec.ts b/packages/language-service/test/completions_spec.ts index 038c026b9916..26fcfec220c0 100644 --- a/packages/language-service/test/completions_spec.ts +++ b/packages/language-service/test/completions_spec.ts @@ -282,12 +282,64 @@ describe('completions', () => { }); describe('for blocks', () => { - it('at top level', () => { - const {templateFile} = setup(`@`, ``); - templateFile.moveCursorToText('@¦'); - const completions = templateFile.getCompletionsAtPosition(); - expectContain( - completions, unsafeCastDisplayInfoKindToScriptElementKind(DisplayInfoKind.BLOCK), ['if']); + const completionPrefixes = ['@', '@i']; + + describe(`at top level`, () => { + for (const completionPrefix of completionPrefixes) { + it(`in empty file (with prefix ${completionPrefix})`, () => { + const {templateFile} = setup(`${completionPrefix}`, ``); + templateFile.moveCursorToText(`${completionPrefix}¦`); + const completions = templateFile.getCompletionsAtPosition(); + expectContain( + completions, unsafeCastDisplayInfoKindToScriptElementKind(DisplayInfoKind.BLOCK), + ['if']); + }); + + it(`after text (with prefix ${completionPrefix})`, () => { + const {templateFile} = setup(`foo ${completionPrefix}`, ``); + templateFile.moveCursorToText(`${completionPrefix}¦`); + const completions = templateFile.getCompletionsAtPosition(); + expectContain( + completions, unsafeCastDisplayInfoKindToScriptElementKind(DisplayInfoKind.BLOCK), + ['if']); + }); + + it(`before text (with prefix ${completionPrefix})`, () => { + const {templateFile} = setup(`${completionPrefix} foo`, ``); + templateFile.moveCursorToText(`${completionPrefix}¦`); + const completions = templateFile.getCompletionsAtPosition(); + expectContain( + completions, unsafeCastDisplayInfoKindToScriptElementKind(DisplayInfoKind.BLOCK), + ['if']); + }); + + it(`after newline with text on preceding line (with prefix ${completionPrefix})`, () => { + const {templateFile} = setup(`foo\n${completionPrefix}`, ``); + templateFile.moveCursorToText(`${completionPrefix}¦`); + const completions = templateFile.getCompletionsAtPosition(); + expectContain( + completions, unsafeCastDisplayInfoKindToScriptElementKind(DisplayInfoKind.BLOCK), + ['if']); + }); + + it(`before newline with text on newline (with prefix ${completionPrefix})`, () => { + const {templateFile} = setup(`${completionPrefix}\nfoo`, ``); + templateFile.moveCursorToText(`${completionPrefix}¦`); + const completions = templateFile.getCompletionsAtPosition(); + expectContain( + completions, unsafeCastDisplayInfoKindToScriptElementKind(DisplayInfoKind.BLOCK), + ['if']); + }); + + it(`in a practical case, on its own line (with prefix ${completionPrefix})`, () => { + const {templateFile} = setup(`
\n ${completionPrefix}\n`, ``); + templateFile.moveCursorToText(`${completionPrefix}¦`); + const completions = templateFile.getCompletionsAtPosition(); + expectContain( + completions, unsafeCastDisplayInfoKindToScriptElementKind(DisplayInfoKind.BLOCK), + ['if']); + }); + } }); it('inside if', () => {