cargdev/angular
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(ngcc): expose the TaskDependencies mapping on BaseTaskQueue (#36083)

Browse Source 
Later when we implement the ability to continue processing when tasks have
failed to compile, we will also need to avoid processing tasks that depend
upon the failed task.

This refactoring exposes this list of dependent tasks in a way that can be
used to skip processing of tasks that depend upon a failed task.

It also changes the blocking model of the parallel mode of operation so
that non-typings tasks are now blocked on their corresponding typings task.
Previously the non-typings tasks could be triggered to run in parallel to
the typings task, since they do not have a hard dependency on each other,
but this made it difficult to skip task correctly if the typings task failed,
since it was possible that a non-typings task was already in flight when
the typings task failed. The result of this is a small potential degradation
of performance in async parallel processing mode, in the rare cases that
there were not enough unblocked tasks to make use of all the available
workers.

PR Close #36083
This commit is contained in:
Pete Bacon Darwin
2020-03-14 22:09:46 +00:00
committed by Andrew Kushnir
parent 39d4016fe9
commit 1790b63a5d
9 changed files with 388 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/compiler-cli/ngcc/src/execution/tasks/api.ts
View File

@ -51,6 +51,12 @@ export interface Task extends JsonObject {
*/
export type PartiallyOrderedTasks = PartiallyOrderedList<Task>;
/**
* A mapping from Tasks to the Tasks that depend upon them (dependents).
*/
export type TaskDependencies = Map<Task, Set<Task>>;
export const TaskDependencies = Map;
/**
* A function to create a TaskCompletedCallback function.
*/

4
packages/compiler-cli/ngcc/src/execution/tasks/queues/base_task_queue.ts
View File

@ -6,7 +6,7 @@
* found in the LICENSE file at https://angular.io/license
*/
import {PartiallyOrderedTasks, Task, TaskQueue} from '../api';
import {PartiallyOrderedTasks, Task, TaskDependencies, TaskQueue} from '../api';
import {stringifyTask} from '../utils';
@ -19,7 +19,7 @@ export abstract class BaseTaskQueue implements TaskQueue {
}
protected inProgressTasks = new Set<Task>();
constructor(protected tasks: PartiallyOrderedTasks) {}
constructor(protected tasks: PartiallyOrderedTasks, protected dependencies: TaskDependencies) {}
abstract getNextTask(): Task|null;

133
packages/compiler-cli/ngcc/src/execution/tasks/queues/parallel_task_queue.ts
View File

@ -5,39 +5,25 @@
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {DepGraph} from 'dependency-graph';
import {EntryPoint} from '../../../packages/entry_point';
import {PartiallyOrderedTasks, Task} from '../api';
import {stringifyTask} from '../utils';
import {PartiallyOrderedTasks, Task, TaskDependencies} from '../api';
import {getBlockedTasks, sortTasksByPriority, stringifyTask} from '../utils';
import {BaseTaskQueue} from './base_task_queue';
/**
* A `TaskQueue` implementation that assumes tasks are processed in parallel, thus has to ensure a
* task's dependencies have been processed before processing the task.
*/
export class ParallelTaskQueue extends BaseTaskQueue {
/**
* A mapping from each task to the list of tasks that are blocking it (if any).
* A map from Tasks to the Tasks that it depends upon.
*
* A task can block another task, if the latter's entry-point depends on the former's entry-point
* _and_ the former is also generating typings (i.e. has `processDts: true`).
*
* NOTE: If a task is not generating typings, then it cannot affect anything which depends on its
* entry-point, regardless of the dependency graph. To put this another way, only the task
* which produces the typings for a dependency needs to have been completed.
* This is the reverse mapping of `TaskDependencies`.
*/
private blockedTasks: Map<Task, Set<Task>>;
constructor(tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>) {
const blockedTasks = computeBlockedTasks(tasks, graph);
const sortedTasks = sortTasksByPriority(tasks, blockedTasks);
super(sortedTasks);
this.blockedTasks = blockedTasks;
constructor(tasks: PartiallyOrderedTasks, dependents: TaskDependencies) {
super(sortTasksByPriority(tasks, dependents), dependents);
this.blockedTasks = getBlockedTasks(dependents);
}
getNextTask(): Task|null {
@ -57,22 +43,22 @@ export class ParallelTaskQueue extends BaseTaskQueue {
markTaskCompleted(task: Task): void {
super.markTaskCompleted(task);
const unblockedTasks: Task[] = [];
if (!this.dependencies.has(task)) {
return;
}
// Remove the completed task from the lists of tasks blocking other tasks.
for (const [otherTask, blockingTasks] of Array.from(this.blockedTasks)) {
if (blockingTasks.has(task)) {
// Unblock the tasks that are dependent upon `task`
for (const dependentTask of this.dependencies.get(task) !) {
if (this.blockedTasks.has(dependentTask)) {
const blockingTasks = this.blockedTasks.get(dependentTask) !;
// Remove the completed task from the lists of tasks blocking other tasks.
blockingTasks.delete(task);
// If the other task is not blocked any more, mark it for unblocking.
if (blockingTasks.size === 0) {
unblockedTasks.push(otherTask);
// If the dependent task is not blocked any more, mark it for unblocking.
this.blockedTasks.delete(dependentTask);
}
}
}
// Unblock tasks that are no longer blocked.
unblockedTasks.forEach(task => this.blockedTasks.delete(task));
}
toString(): string {
@ -89,88 +75,3 @@ export class ParallelTaskQueue extends BaseTaskQueue {
.join('');
}
}
// Helpers
/**
* Compute a mapping of blocked tasks to the tasks that are blocking them.
*
* As a performance optimization, we take into account the fact that `tasks` are sorted in such a
* way that a task can only be blocked by earlier tasks (i.e. dependencies always come before
* dependants in the list of tasks).
*
* @param tasks A (partially ordered) list of tasks.
* @param graph The dependency graph between entry-points.
* @return The map of blocked tasks to the tasks that are blocking them.
*/
function computeBlockedTasks(
tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>): Map<Task, Set<Task>> {
const blockedTasksMap = new Map<Task, Set<Task>>();
const candidateBlockers = new Map<string, Task>();
tasks.forEach(task => {
// Find the earlier tasks (`candidateBlockers`) that are blocking this task.
const deps = graph.dependenciesOf(task.entryPoint.path);
const blockingTasks =
deps.filter(dep => candidateBlockers.has(dep)).map(dep => candidateBlockers.get(dep) !);
// If this task is blocked, add it to the map of blocked tasks.
if (blockingTasks.length > 0) {
blockedTasksMap.set(task, new Set(blockingTasks));
}
// If this task can be potentially blocking (i.e. it generates typings), add it to the list
// of candidate blockers for subsequent tasks.
if (task.processDts) {
const entryPointPath = task.entryPoint.path;
// There should only be one task per entry-point that generates typings (and thus can block
// other tasks), so the following should theoretically never happen, but check just in case.
if (candidateBlockers.has(entryPointPath)) {
const otherTask = candidateBlockers.get(entryPointPath) !;
throw new Error(
'Invariant violated: Multiple tasks are assigned generating typings for ' +
`'${entryPointPath}':\n - ${stringifyTask(otherTask)}\n - ${stringifyTask(task)}`);
}
candidateBlockers.set(entryPointPath, task);
}
});
return blockedTasksMap;
}
/**
* Sort a list of tasks by priority.
*
* Priority is determined by the number of other tasks that a task is (transitively) blocking:
* The more tasks a task is blocking the higher its priority is, because processing it will
* potentially unblock more tasks.
*
* To keep the behavior predictable, if two tasks block the same number of other tasks, their
* relative order in the original `tasks` lists is preserved.
*
* @param tasks A (partially ordered) list of tasks.
* @param blockedTasks A mapping from a task to the list of tasks that are blocking it (if any).
* @return The list of tasks sorted by priority.
*/
function sortTasksByPriority(
tasks: PartiallyOrderedTasks, blockedTasks: Map<Task, Set<Task>>): PartiallyOrderedTasks {
const priorityPerTask = new Map<Task, [number, number]>();
const allBlockingTaskSets = Array.from(blockedTasks.values());
const computePriority = (task: Task, idx: number): [number, number] =>
[allBlockingTaskSets.reduce(
(count, blockingTasks) => count + (blockingTasks.has(task) ? 1 : 0), 0),
idx,
];
tasks.forEach((task, i) => priorityPerTask.set(task, computePriority(task, i)));
return tasks.slice().sort((task1, task2) => {
const [p1, idx1] = priorityPerTask.get(task1) !;
const [p2, idx2] = priorityPerTask.get(task2) !;
return (p2 - p1) || (idx1 - idx2);
});
}

128
packages/compiler-cli/ngcc/src/execution/tasks/utils.ts
View File

@ -5,8 +5,134 @@
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {Task} from './api';
import {DepGraph} from 'dependency-graph';
import {EntryPoint} from '../../packages/entry_point';
import {PartiallyOrderedTasks, Task, TaskDependencies} from './api';
/** Stringify a task for debugging purposes. */
export const stringifyTask = (task: Task): string =>
`{entryPoint: ${task.entryPoint.name}, formatProperty: ${task.formatProperty}, processDts: ${task.processDts}}`;
/**
* Compute a mapping of tasks to the tasks that are dependent on them (if any).
*
* Task A can depend upon task B, if either:
*
* * A and B have the same entry-point _and_ B is generating the typings for that entry-point
* (i.e. has `processDts: true`).
* * A's entry-point depends on B's entry-point _and_ B is also generating typings.
*
* NOTE: If a task is not generating typings, then it cannot affect anything which depends on its
* entry-point, regardless of the dependency graph. To put this another way, only the task
* which produces the typings for a dependency needs to have been completed.
*
* As a performance optimization, we take into account the fact that `tasks` are sorted in such a
* way that a task can only depend on earlier tasks (i.e. dependencies always come before
* dependents in the list of tasks).
*
* @param tasks A (partially ordered) list of tasks.
* @param graph The dependency graph between entry-points.
* @return A map from each task to those tasks directly dependent upon it.
*/
export function computeTaskDependencies(
tasks: PartiallyOrderedTasks, graph: DepGraph<EntryPoint>): TaskDependencies {
const dependencies = new TaskDependencies();
const candidateDependencies = new Map<string, Task>();
tasks.forEach(task => {
const entryPointPath = task.entryPoint.path;
// Find the earlier tasks (`candidateDependencies`) that this task depends upon.
const deps = graph.dependenciesOf(entryPointPath);
const taskDependencies = deps.filter(dep => candidateDependencies.has(dep))
.map(dep => candidateDependencies.get(dep) !);
// If this task has dependencies, add it to the dependencies and dependents maps.
if (taskDependencies.length > 0) {
for (const dependency of taskDependencies) {
const taskDependents = getDependentsSet(dependencies, dependency);
taskDependents.add(task);
}
}
if (task.processDts) {
// SANITY CHECK:
// There should only be one task per entry-point that generates typings (and thus can be a
// dependency of other tasks), so the following should theoretically never happen, but check
// just in case.
if (candidateDependencies.has(entryPointPath)) {
const otherTask = candidateDependencies.get(entryPointPath) !;
throw new Error(
'Invariant violated: Multiple tasks are assigned generating typings for ' +
`'${entryPointPath}':\n - ${stringifyTask(otherTask)}\n - ${stringifyTask(task)}`);
}
// This task can potentially be a dependency (i.e. it generates typings), so add it to the
// list of candidate dependencies for subsequent tasks.
candidateDependencies.set(entryPointPath, task);
} else {
// This task is not generating typings so we need to add it to the dependents of the task that
// does generate typings, if that exists
if (candidateDependencies.has(entryPointPath)) {
const typingsTask = candidateDependencies.get(entryPointPath) !;
const typingsTaskDependents = getDependentsSet(dependencies, typingsTask);
typingsTaskDependents.add(task);
}
}
});
return dependencies;
}
export function getDependentsSet(map: TaskDependencies, task: Task): Set<Task> {
if (!map.has(task)) {
map.set(task, new Set());
}
return map.get(task) !;
}
/**
* Invert the given mapping of Task dependencies.
*
* @param dependencies The mapping of tasks to the tasks that depend upon them.
* @returns A mapping of tasks to the tasks that they depend upon.
*/
export function getBlockedTasks(dependencies: TaskDependencies): Map<Task, Set<Task>> {
const blockedTasks = new Map<Task, Set<Task>>();
for (const [dependency, dependents] of dependencies) {
for (const dependent of dependents) {
const dependentSet = getDependentsSet(blockedTasks, dependent);
dependentSet.add(dependency);
}
}
return blockedTasks;
}
/**
* Sort a list of tasks by priority.
*
* Priority is determined by the number of other tasks that a task is (transitively) blocking:
* The more tasks a task is blocking the higher its priority is, because processing it will
* potentially unblock more tasks.
*
* To keep the behavior predictable, if two tasks block the same number of other tasks, their
* relative order in the original `tasks` lists is preserved.
*
* @param tasks A (partially ordered) list of tasks.
* @param dependencies The mapping of tasks to the tasks that depend upon them.
* @return The list of tasks sorted by priority.
*/
export function sortTasksByPriority(
tasks: PartiallyOrderedTasks, dependencies: TaskDependencies): PartiallyOrderedTasks {
const priorityPerTask = new Map<Task, [number, number]>();
const computePriority = (task: Task, idx: number):
[number, number] => [dependencies.has(task) ? dependencies.get(task) !.size : 0, idx];
tasks.forEach((task, i) => priorityPerTask.set(task, computePriority(task, i)));
return tasks.slice().sort((task1, task2) => {
const [p1, idx1] = priorityPerTask.get(task1) !;
const [p2, idx2] = priorityPerTask.get(task2) !;
return (p2 - p1) || (idx1 - idx2);
});
}