fix(ngcc): update package.json deterministically (#34870)

Ngcc adds properties to the `package.json` files of the entry-points it
processes to mark them as processed for a format and point to the
created Ivy entry-points (in case of `--create-ivy-entry-points`). When
running ngcc in parallel mode (which is the default for the standalone
ngcc command), multiple formats can be processed simultaneously for the
same entry-point and the order of completion is not deterministic.

Previously, ngcc would append new properties at the end of the target
object in `package.json` as soon as the format processing was completed.
As a result, the order of properties in the resulting `package.json`
(when processing multiple formats for an entry-point in parallel) was
not deterministic. For tools that use file hashes for caching purposes
(such as Bazel), this lead to a high probability of cache misses.

This commit fixes the problem by ensuring that the position of
properties added to `package.json` files is deterministic and
independent of the order in which each format is processed.

Jira issue: [FW-1801](https://angular-team.atlassian.net/browse/FW-1801)

Fixes #34635

PR Close #34870

packages/compiler-cli/ngcc/src/execution/cluster/package_json_updater.ts

@@ -44,7 +44,8 @@ export class ClusterPackageJsonUpdater implements PackageJsonUpdater {
           throw new Error(`Missing property path for writing value to '${packageJsonPath}'.`);
         }
 
-        applyChange(preExistingParsedJson, propPath, value);
+        // No need to take property positioning into account for in-memory representations.
+        applyChange(preExistingParsedJson, propPath, value, 'unimportant');
       }
     }
 

packages/compiler-cli/ngcc/src/packages/build_marker.ts

@@ -60,7 +60,7 @@ export function markAsProcessed(
 
   // Update the format properties to mark them as processed.
   for (const prop of formatProperties) {
-    update.addChange(['__processed_by_ivy_ngcc__', prop], NGCC_VERSION);
+    update.addChange(['__processed_by_ivy_ngcc__', prop], NGCC_VERSION, 'alphabetic');
   }
 
   // Update the `prepublishOnly` script (keeping a backup, if necessary) to prevent `ngcc`'d

packages/compiler-cli/ngcc/src/writing/new_entry_point_file_writer.ts

@@ -93,7 +93,7 @@ export class NewEntryPointFileWriter extends InPlaceFileWriter {
             `(${formatProperties.join(', ')}) map to more than one format-path.`);
       }
 
-      update.addChange([`${formatProperty}_ivy_ngcc`], newFormatPath);
+      update.addChange([`${formatProperty}_ivy_ngcc`], newFormatPath, {before: formatProperty});
     }
 
     update.writeChanges(packageJsonPath, packageJson);

packages/compiler-cli/ngcc/src/writing/package_json_updater.ts

@@ -10,7 +10,8 @@ import {AbsoluteFsPath, FileSystem, dirname} from '../../../src/ngtsc/file_syste
 import {JsonObject, JsonValue} from '../packages/entry_point';
 
 
-export type PackageJsonChange = [string[], JsonValue];
+export type PackageJsonChange = [string[], JsonValue, PackageJsonPropertyPositioning];
+export type PackageJsonPropertyPositioning = 'unimportant' | 'alphabetic' | {before: string};
 export type WritePackageJsonChangesFn =
     (changes: PackageJsonChange[], packageJsonPath: AbsoluteFsPath, parsedJson?: JsonObject) =>
         void;
@@ -23,8 +24,9 @@ export type WritePackageJsonChangesFn =
  * const updatePackageJson = packageJsonUpdater
  *     .createUpdate()
  *     .addChange(['name'], 'package-foo')
- *     .addChange(['scripts', 'foo'], 'echo FOOOO...')
- *     .addChange(['dependencies', 'bar'], '1.0.0')
+ *     .addChange(['scripts', 'foo'], 'echo FOOOO...', 'unimportant')
+ *     .addChange(['dependencies', 'baz'], '1.0.0', 'alphabetic')
+ *     .addChange(['dependencies', 'bar'], '2.0.0', {before: 'baz'})
  *     .writeChanges('/foo/package.json');
  *     // or
  *     // .writeChanges('/foo/package.json', inMemoryParsedJson);
@@ -33,13 +35,13 @@ export type WritePackageJsonChangesFn =
 export interface PackageJsonUpdater {
   /**
    * Create a `PackageJsonUpdate` object, which provides a fluent API for batching updates to a
-   * `package.json` file. (Batching the updates is useful, because it avoid unnecessary I/O
+   * `package.json` file. (Batching the updates is useful, because it avoids unnecessary I/O
    * operations.)
    */
   createUpdate(): PackageJsonUpdate;
 
   /**
-   * Write a set of changes to the specified `package.json` file and (and optionally a pre-existing,
+   * Write a set of changes to the specified `package.json` file (and optionally a pre-existing,
    * in-memory representation of it).
    *
    * @param changes The set of changes to apply.
@@ -65,15 +67,27 @@ export class PackageJsonUpdate {
   constructor(private writeChangesImpl: WritePackageJsonChangesFn) {}
 
   /**
-   * Record a change to a `package.json` property. If the ancestor objects do not yet exist in the
-   * `package.json` file, they will be created.
+   * Record a change to a `package.json` property.
    *
-   * @param propertyPath The path of a (possibly nested) property to update.
+   * If the ancestor objects do not yet exist in the `package.json` file, they will be created. The
+   * positioning of the property can also be specified. (If the property already exists, it will be
+   * moved accordingly.)
+   *
+   * NOTE: Property positioning is only guaranteed to be respected in the serialized `package.json`
+   *       file. Positioning will not be taken into account when updating in-memory representations.
+   *
+   * NOTE 2: Property positioning only affects the last property in `propertyPath`. Ancestor
+   *         objects' positioning will not be affected.
+   *
+   * @param propertyPath The path of a (possibly nested) property to add/update.
    * @param value The new value to set the property to.
+   * @param position The desired position for the added/updated property.
    */
-  addChange(propertyPath: string[], value: JsonValue): this {
+  addChange(
+      propertyPath: string[], value: JsonValue,
+      positioning: PackageJsonPropertyPositioning = 'unimportant'): this {
     this.ensureNotApplied();
-    this.changes.push([propertyPath, value]);
+    this.changes.push([propertyPath, value, positioning]);
     return this;
   }
 
@@ -121,15 +135,16 @@ export class DirectPackageJsonUpdater implements PackageJsonUpdater {
 
     // Apply all changes to both the canonical representation (read from disk) and any pre-existing,
     // in-memory representation.
-    for (const [propPath, value] of changes) {
+    for (const [propPath, value, positioning] of changes) {
       if (propPath.length === 0) {
         throw new Error(`Missing property path for writing value to '${packageJsonPath}'.`);
       }
 
-      applyChange(parsedJson, propPath, value);
+      applyChange(parsedJson, propPath, value, positioning);
 
       if (preExistingParsedJson) {
-        applyChange(preExistingParsedJson, propPath, value);
+        // No need to take property positioning into account for in-memory representations.
+        applyChange(preExistingParsedJson, propPath, value, 'unimportant');
       }
     }
 
@@ -141,7 +156,9 @@ export class DirectPackageJsonUpdater implements PackageJsonUpdater {
 }
 
 // Helpers
-export function applyChange(ctx: JsonObject, propPath: string[], value: JsonValue): void {
+export function applyChange(
+    ctx: JsonObject, propPath: string[], value: JsonValue,
+    positioning: PackageJsonPropertyPositioning): void {
   const lastPropIdx = propPath.length - 1;
   const lastProp = propPath[lastPropIdx];
 
@@ -157,4 +174,42 @@ export function applyChange(ctx: JsonObject, propPath: string[], value: JsonValu
   }
 
   ctx[lastProp] = value;
+  positionProperty(ctx, lastProp, positioning);
+}
+
+function movePropBefore(ctx: JsonObject, prop: string, isNextProp: (p: string) => boolean): void {
+  const allProps = Object.keys(ctx);
+  const otherProps = allProps.filter(p => p !== prop);
+  const nextPropIdx = otherProps.findIndex(isNextProp);
+  const propsToShift = (nextPropIdx === -1) ? [] : otherProps.slice(nextPropIdx);
+
+  movePropToEnd(ctx, prop);
+  propsToShift.forEach(p => movePropToEnd(ctx, p));
+}
+
+function movePropToEnd(ctx: JsonObject, prop: string): void {
+  const value = ctx[prop];
+  delete ctx[prop];
+  ctx[prop] = value;
+}
+
+function positionProperty(
+    ctx: JsonObject, prop: string, positioning: PackageJsonPropertyPositioning): void {
+  switch (positioning) {
+    case 'alphabetic':
+      movePropBefore(ctx, prop, p => p > prop);
+      break;
+    case 'unimportant':
+      // Leave the property order unchanged; i.e. newly added properties will be last and existing
+      // ones will remain in their old position.
+      break;
+    default:
+      if ((typeof positioning !== 'object') || (positioning.before === undefined)) {
+        throw new Error(
+            `Unknown positioning (${JSON.stringify(positioning)}) for property '${prop}'.`);
+      }
+
+      movePropBefore(ctx, prop, p => p === positioning.before);
+      break;
+  }
 }