feat(compiler): support tokenizing escaped strings (#28055)

In order to support source mapping of templates, we need
to be able to tokenize the template in its original context.
When the template is defined inline as a JavaScript string
in a TS/JS source file, the tokenizer must be able to handle
string escape sequences, such as `\n` and `\"` as they
appear in the original source file.

This commit teaches the lexer how to unescape these
sequences, but only when the `escapedString` option is
set to true.  Otherwise there is no change to the tokenizing
behaviour.

PR Close #28055

packages/compiler/src/chars.ts

@@ -7,6 +7,7 @@
  */
 
 export const $EOF = 0;
+export const $BSPACE = 8;
 export const $TAB = 9;
 export const $LF = 10;
 export const $VTAB = 11;
@@ -36,6 +37,7 @@ export const $GT = 62;
 export const $QUESTION = 63;
 
 export const $0 = 48;
+export const $7 = 55;
 export const $9 = 57;
 
 export const $A = 65;
@@ -51,6 +53,7 @@ export const $CARET = 94;
 export const $_ = 95;
 
 export const $a = 97;
+export const $b = 98;
 export const $e = 101;
 export const $f = 102;
 export const $n = 110;
@@ -87,3 +90,11 @@ export function isAsciiLetter(code: number): boolean {
 export function isAsciiHexDigit(code: number): boolean {
   return code >= $a && code <= $f || code >= $A && code <= $F || isDigit(code);
 }
+
+export function isNewLine(code: number): boolean {
+  return code === $LF || code === $CR;
+}
+
+export function isOctalDigit(code: number): boolean {
+  return $0 <= code && code <= $7;
+}