cargdev/angular
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ngcc): implement source-map flattening (#35132)

Browse Source 
The library used by ngcc to update the source files (MagicString) is able
to generate a source-map but it is not able to account for any previous
source-map that the input text is already associated with.

There have been various attempts to fix this but none have been very
successful, since it is not a trivial problem to solve.

This commit contains a novel approach that is able to load up a tree of
source-files connected by source-maps and flatten them down into a single
source-map that maps directly from the final generated file to the original
sources referenced by the intermediate source-maps.

PR Close #35132
This commit is contained in:
Pete Bacon Darwin
2020-02-16 21:07:30 +01:00
committed by Miško Hevery
parent 2a8dd4758c
commit df816c9c80
16 changed files with 1273 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
packages/compiler-cli/ngcc/src/sourcemaps/raw_source_map.ts Normal file
View File

@ -0,0 +1,21 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* This interface is the basic structure of the JSON in a raw source map that one might load from
* disk.
*/
export interface RawSourceMap {
version: number|string;
file?: string;
sourceRoot?: string;
sources: string[];
names: string[];
sourcesContent?: (string|null)[];
mappings: string;
}

88
packages/compiler-cli/ngcc/src/sourcemaps/segment_marker.ts Normal file
View File

@ -0,0 +1,88 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* A marker that indicates the start of a segment in a mapping.
*
* The end of a segment is indicated by the the first segment-marker of another mapping whose start
* is greater or equal to this one.
*/
export interface SegmentMarker {
readonly line: number;
readonly column: number;
}
/**
* Compare two segment-markers, for use in a search or sorting algorithm.
*
* @returns a positive number if `a` is after `b`, a negative number if `b` is after `a`
* and zero if they are at the same position.
*/
export function compareSegments(a: SegmentMarker, b: SegmentMarker): number {
return a.line === b.line ? a.column - b.column : a.line - b.line;
}
// The `1` is to indicate a newline character between the lines.
// Note that in the actual contents there could be more than one character that indicates a newline
// - e.g. \r\n - but that is not important here since segment-markers are in line/column pairs and
// so differences in length due to extra `\r` characters do not affect the algorithms.
const NEWLINE_MARKER_OFFSET = 1;
/**
* Compute the difference between two segment markers in a source file.
*
* @param lineLengths the lengths of each line of content of the source file where we are computing
* the difference
* @param a the start marker
* @param b the end marker
* @returns the number of characters between the two segments `a` and `b`
*/
export function segmentDiff(lineLengths: number[], a: SegmentMarker, b: SegmentMarker) {
let diff = b.column - a.column;
// Deal with `a` being before `b`
for (let lineIndex = a.line; lineIndex < b.line; lineIndex++) {
diff += lineLengths[lineIndex] + NEWLINE_MARKER_OFFSET;
}
// Deal with `a` being after `b`
for (let lineIndex = a.line - 1; lineIndex >= b.line; lineIndex--) {
// The `+ 1` is the newline character between the lines
diff -= lineLengths[lineIndex] + NEWLINE_MARKER_OFFSET;
}
return diff;
}
/**
* Return a new segment-marker that is offset by the given number of characters.
*
* @param lineLengths The length of each line in the source file whose segment-marker we are
* offsetting.
* @param marker The segment to offset.
* @param offset The number of character to offset by.
*/
export function offsetSegment(lineLengths: number[], marker: SegmentMarker, offset: number) {
if (offset === 0) {
return marker;
}
let line = marker.line;
let column = marker.column + offset;
while (line < lineLengths.length - 1 && column > lineLengths[line]) {
column -= lineLengths[line] + NEWLINE_MARKER_OFFSET;
line++;
}
while (line > 0 && column < 0) {
line--;
column += lineLengths[line] + NEWLINE_MARKER_OFFSET;
}
return {line, column};
}

313
packages/compiler-cli/ngcc/src/sourcemaps/source_file.ts Normal file
View File

@ -0,0 +1,313 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {removeComments, removeMapFileComments} from 'convert-source-map';
import {SourceMapMappings, SourceMapSegment, decode, encode} from 'sourcemap-codec';
import {AbsoluteFsPath, dirname, relative} from '../../../src/ngtsc/file_system';
import {RawSourceMap} from './raw_source_map';
import {SegmentMarker, compareSegments, offsetSegment, segmentDiff} from './segment_marker';
export function removeSourceMapComments(contents: string): string {
return removeMapFileComments(removeComments(contents)).replace(/\n\n$/, '\n');
}
export class SourceFile {
/**
* The parsed mappings that have been flattened so that any intermediate source mappings have been
* flattened.
*
* The result is that any source file mentioned in the flattened mappings have no source map (are
* pure original source files).
*/
readonly flattenedMappings: Mapping[];
readonly lineLengths: number[];
constructor(
/** The path to this source file. */
readonly sourcePath: AbsoluteFsPath,
/** The contents of this source file. */
readonly contents: string,
/** The raw source map (if any) associated with this source file. */
readonly rawMap: RawSourceMap|null,
/** Whether this source file's source map was inline or external. */
readonly inline: boolean,
/** Any source files referenced by the raw source map associated with this source file. */
readonly sources: (SourceFile|null)[]) {
this.contents = removeSourceMapComments(contents);
this.lineLengths = computeLineLengths(this.contents);
this.flattenedMappings = this.flattenMappings();
}
/**
* Render the raw source map generated from the flattened mappings.
*/
renderFlattenedSourceMap(): RawSourceMap {
const sources: SourceFile[] = [];
const names: string[] = [];
// Ensure a mapping line array for each line in the generated source.
const mappings: SourceMapMappings = this.lineLengths.map(() => []);
for (const mapping of this.flattenedMappings) {
const mappingLine = mappings[mapping.generatedSegment.line];
const sourceIndex = findIndexOrAdd(sources, mapping.originalSource);
const mappingArray: SourceMapSegment = [
mapping.generatedSegment.column,
sourceIndex,
mapping.originalSegment.line,
mapping.originalSegment.column,
];
if (mapping.name !== undefined) {
const nameIndex = findIndexOrAdd(names, mapping.name);
mappingArray.push(nameIndex);
}
mappingLine.push(mappingArray);
}
const sourcePathDir = dirname(this.sourcePath);
const sourceMap: RawSourceMap = {
version: 3,
file: relative(sourcePathDir, this.sourcePath),
sources: sources.map(sf => relative(sourcePathDir, sf.sourcePath)), names,
mappings: encode(mappings),
sourcesContent: sources.map(sf => sf.contents),
};
return sourceMap;
}
/**
* Flatten the parsed mappings for this source file, so that all the mappings are to pure original
* source files with no transitive source maps.
*/
private flattenMappings(): Mapping[] {
const mappings = parseMappings(this.rawMap, this.sources);
const originalSegments = extractOriginalSegments(mappings);
const flattenedMappings: Mapping[] = [];
for (let mappingIndex = 0; mappingIndex < mappings.length; mappingIndex++) {
const aToBmapping = mappings[mappingIndex];
const bSource = aToBmapping.originalSource;
if (bSource.flattenedMappings.length === 0) {
// The b source file has no mappings of its own (i.e. it is a pure original file)
// so just use the mapping as-is.
flattenedMappings.push(aToBmapping);
continue;
}
// The `incomingStart` and `incomingEnd` are the `SegmentMarker`s in `B` that represent the
// section of `B` source file that is being mapped to by the current `aToBmapping`.
//
// For example, consider the mappings from A to B:
//
// src A src B mapping
//
// a ----- a [0, 0]
// b b
// f - /- c [4, 2]
// g \ / d
// c -/\ e
// d \- f [2, 5]
// e
//
// For mapping [0,0] the incoming start and end are 0 and 2 (i.e. the range a, b, c)
// For mapping [4,2] the incoming start and end are 2 and 5 (i.e. the range c, d, e, f)
//
const incomingStart = aToBmapping.originalSegment;
const incomingEndIndex = originalSegments.indexOf(incomingStart) + 1;
const incomingEnd = incomingEndIndex < originalSegments.length ?
originalSegments[incomingEndIndex] :
undefined;
// The `outgoingStartIndex` and `outgoingEndIndex` are the indices of the range of mappings
// that leave `b` that we are interested in merging with the aToBmapping.
// We actually care about all the markers from the last bToCmapping directly before the
// `incomingStart` to the last bToCmaping directly before the `incomingEnd`, inclusive.
//
// For example, if we consider the range 2 to 5 from above (i.e. c, d, e, f) with the
// following mappings from B to C:
//
// src B src C mapping
// a
// b ----- b [1, 0]
// - c c
// | d d
// | e ----- 1 [4, 3]
// - f \ 2
// \ 3
// \- e [4, 6]
//
// The range with `incomingStart` at 2 and `incomingEnd` at 5 has outgoing start mapping of
// [1,0] and outgoing end mapping of [4, 6], which also includes [4, 3].
//
let outgoingStartIndex = findLastIndex(
bSource.flattenedMappings,
mapping => compareSegments(mapping.generatedSegment, incomingStart) <= 0);
if (outgoingStartIndex < 0) {
outgoingStartIndex = 0;
}
const outgoingEndIndex = incomingEnd !== undefined ?
findLastIndex(
bSource.flattenedMappings,
mapping => compareSegments(mapping.generatedSegment, incomingEnd) < 0) :
bSource.flattenedMappings.length - 1;
for (let bToCmappingIndex = outgoingStartIndex; bToCmappingIndex <= outgoingEndIndex;
bToCmappingIndex++) {
const bToCmapping: Mapping = bSource.flattenedMappings[bToCmappingIndex];
flattenedMappings.push(mergeMappings(this, aToBmapping, bToCmapping));
}
}
return flattenedMappings;
}
}
function findLastIndex<T>(items: T[], predicate: (item: T) => boolean): number {
for (let index = items.length - 1; index >= 0; index--) {
if (predicate(items[index])) {
return index;
}
}
return -1;
}
/**
* A Mapping consists of two segment markers: one in the generated source and one in the original
* source, which indicate the start of each segment. The end of a segment is indicated by the first
* segment marker of another mapping whose start is greater or equal to this one.
*
* It may also include a name associated with the segment being mapped.
*/
export interface Mapping {
readonly generatedSegment: SegmentMarker;
readonly originalSource: SourceFile;
readonly originalSegment: SegmentMarker;
readonly name?: string;
}
/**
* Find the index of `item` in the `items` array.
* If it is not found, then push `item` to the end of the array and return its new index.
*
* @param items the collection in which to look for `item`.
* @param item the item to look for.
* @returns the index of the `item` in the `items` array.
*/
function findIndexOrAdd<T>(items: T[], item: T): number {
const itemIndex = items.indexOf(item);
if (itemIndex > -1) {
return itemIndex;
} else {
items.push(item);
return items.length - 1;
}
}
/**
* Merge two mappings that go from A to B and B to C, to result in a mapping that goes from A to C.
*/
export function mergeMappings(generatedSource: SourceFile, ab: Mapping, bc: Mapping): Mapping {
const name = bc.name || ab.name;
// We need to modify the segment-markers of the new mapping to take into account the shifts that
// occur due to the combination of the two mappings.
// For example:
// * Simple map where the B->C starts at the same place the A->B ends:
//
// ```
// A: 1 2 b c d
// | A->B [2,0]
// | |
// B: b c d A->C [2,1]
// | |
// | B->C [0,1]
// C: a b c d e
// ```
// * More complicated case where diffs of segment-markers is needed:
//
// ```
// A: b 1 2 c d
// \
// | A->B [0,1*] [0,1*]
// | | |+3
// B: a b 1 2 c d A->C [0,1] [3,2]
// | / |+1 |
// | / B->C [0*,0] [4*,2]
// | /
// C: a b c d e
// ```
//
// `[0,1]` mapping from A->C:
// The difference between the "original segment-marker" of A->B (1*) and the "generated
// segment-marker of B->C (0*): `1 - 0 = +1`.
// Since it is positive we must increment the "original segment-marker" with `1` to give [0,1].
//
// `[3,2]` mapping from A->C:
// The difference between the "original segment-marker" of A->B (1*) and the "generated
// segment-marker" of B->C (4*): `1 - 4 = -3`.
// Since it is negative we must increment the "generated segment-marker" with `3` to give [3,2].
const diff = segmentDiff(ab.originalSource.lineLengths, ab.originalSegment, bc.generatedSegment);
if (diff > 0) {
return {
name,
generatedSegment: offsetSegment(generatedSource.lineLengths, ab.generatedSegment, diff),
originalSource: bc.originalSource,
originalSegment: bc.originalSegment,
};
} else {
return {
name,
generatedSegment: ab.generatedSegment,
originalSource: bc.originalSource,
originalSegment: offsetSegment(bc.originalSource.lineLengths, bc.originalSegment, -diff),
};
}
}
/**
* Parse the `rawMappings` into an array of parsed mappings, which reference source-files provided
* in the `sources` parameter.
*/
export function parseMappings(
rawMap: RawSourceMap | null, sources: (SourceFile | null)[]): Mapping[] {
if (rawMap === null) {
return [];
}
const rawMappings = decode(rawMap.mappings);
if (rawMappings === null) {
return [];
}
const mappings: Mapping[] = [];
for (let generatedLine = 0; generatedLine < rawMappings.length; generatedLine++) {
const generatedLineMappings = rawMappings[generatedLine];
for (const rawMapping of generatedLineMappings) {
if (rawMapping.length >= 4) {
const generatedColumn = rawMapping[0];
const name = rawMapping.length === 5 ? rawMap.names[rawMapping[4]] : undefined;
const mapping: Mapping = {
generatedSegment: {line: generatedLine, column: generatedColumn},
originalSource: sources[rawMapping[1] !] !,
originalSegment: {line: rawMapping[2] !, column: rawMapping[3] !}, name
};
mappings.push(mapping);
}
}
}
return mappings;
}
export function extractOriginalSegments(mappings: Mapping[]): SegmentMarker[] {
return mappings.map(mapping => mapping.originalSegment).sort(compareSegments);
}
export function computeLineLengths(str: string): number[] {
return (str.split(/\r?\n/)).map(s => s.length);
}

142
packages/compiler-cli/ngcc/src/sourcemaps/source_file_loader.ts Normal file
View File

@ -0,0 +1,142 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {commentRegex, fromComment, mapFileCommentRegex} from 'convert-source-map';
import {AbsoluteFsPath, FileSystem, absoluteFrom} from '../../../src/ngtsc/file_system';
import {RawSourceMap} from './raw_source_map';
import {SourceFile} from './source_file';
/**
* This class can be used to load a source file, its associated source map and any upstream sources.
*
* Since a source file might reference (or include) a source map, this class can load those too.
* Since a source map might reference other source files, these are also loaded as needed.
*
* This is done recursively. The result is a "tree" of `SourceFile` objects, each containing
* mappings to other `SourceFile` objects as necessary.
*/
export class SourceFileLoader {
constructor(private fs: FileSystem) {}
/**
* Load a source file, compute its source map, and recursively load any referenced source files.
*
* @param sourcePath The path to the source file to load.
* @param contents The contents of the source file to load (if known).
* The contents may be known because the source file was inlined into a source map.
* If it is not known the contents will be read from the file at the `sourcePath`.
* @param mapAndPath The raw source-map and the path to the source-map file, if known.
* @param previousPaths An internal parameter used for cyclic dependency tracking.
* @returns a SourceFile if the content for one was provided or able to be loaded from disk,
* `null` otherwise.
*/
loadSourceFile(sourcePath: AbsoluteFsPath, contents: string, mapAndPath: MapAndPath): SourceFile;
loadSourceFile(sourcePath: AbsoluteFsPath, contents: string|null): SourceFile|null;
loadSourceFile(sourcePath: AbsoluteFsPath): SourceFile|null;
loadSourceFile(
sourcePath: AbsoluteFsPath, contents: string|null, mapAndPath: null,
previousPaths: AbsoluteFsPath[]): SourceFile|null;
loadSourceFile(
sourcePath: AbsoluteFsPath, contents: string|null = null, mapAndPath: MapAndPath|null = null,
previousPaths: AbsoluteFsPath[] = []): SourceFile|null {
if (contents === null) {
if (!this.fs.exists(sourcePath)) {
return null;
}
// Track source file paths if we have loaded them from disk so that we don't get into an
// infinite recursion
if (previousPaths.includes(sourcePath)) {
throw new Error(
`Circular source file mapping dependency: ${previousPaths.join(' -> ')} -> ${sourcePath}`);
}
previousPaths = previousPaths.concat([sourcePath]);
contents = this.fs.readFile(sourcePath);
}
// If not provided try to load the source map based on the source itself
if (mapAndPath === null) {
mapAndPath = this.loadSourceMap(sourcePath, contents);
}
let map: RawSourceMap|null = null;
let inline = true;
let sources: (SourceFile | null)[] = [];
if (mapAndPath !== null) {
const basePath = mapAndPath.mapPath || sourcePath;
sources = this.processSources(basePath, mapAndPath.map, previousPaths);
map = mapAndPath.map;
inline = mapAndPath.mapPath === null;
}
return new SourceFile(sourcePath, contents, map, inline, sources);
}
/**
* Find the source map associated with the source file whose `sourcePath` and `contents` are
* provided.
*
* Source maps can be inline, as part of a base64 encoded comment, or external as a separate file
* whose path is indicated in a comment or implied from the name of the source file itself.
*/
private loadSourceMap(sourcePath: AbsoluteFsPath, contents: string): MapAndPath|null {
const inline = commentRegex.exec(contents);
if (inline !== null) {
return {map: fromComment(inline.pop() !).sourcemap, mapPath: null};
}
const external = mapFileCommentRegex.exec(contents);
if (external) {
try {
const fileName = external[1] || external[2];
const externalMapPath = this.fs.resolve(this.fs.dirname(sourcePath), fileName);
return {map: this.loadRawSourceMap(externalMapPath), mapPath: externalMapPath};
} catch {
return null;
}
}
const impliedMapPath = absoluteFrom(sourcePath + '.map');
if (this.fs.exists(impliedMapPath)) {
return {map: this.loadRawSourceMap(impliedMapPath), mapPath: impliedMapPath};
}
return null;
}
/**
* Iterate over each of the "sources" for this source file's source map, recursively loading each
* source file and its associated source map.
*/
private processSources(
basePath: AbsoluteFsPath, map: RawSourceMap,
previousPaths: AbsoluteFsPath[]): (SourceFile|null)[] {
const sourceRoot = this.fs.resolve(this.fs.dirname(basePath), map.sourceRoot || '');
return map.sources.map((source, index) => {
const path = this.fs.resolve(sourceRoot, source);
const content = map.sourcesContent && map.sourcesContent[index] || null;
return this.loadSourceFile(path, content, null, previousPaths);
});
}
/**
* Load the source map from the file at `mapPath`, parsing its JSON contents into a `RawSourceMap`
* object.
*/
private loadRawSourceMap(mapPath: AbsoluteFsPath): RawSourceMap {
return JSON.parse(this.fs.readFile(mapPath));
}
}
/** A small helper structure that is returned from `loadSourceMap()`. */
interface MapAndPath {
/** The path to the source map if it was external or `null` if it was inline. */
mapPath: AbsoluteFsPath|null;
/** The raw source map itself. */
map: RawSourceMap;
}