feat: initial commit

This commit is contained in:
Carlos
2024-08-12 23:06:35 -04:00
parent c581ce5ad2
commit de4ae5d068
776 changed files with 3868 additions and 15649 deletions

View File

@@ -1,29 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import { FarmOptions, PromiseWithCustomMessage, TaskQueue } from './types';
export default class Farm {
private _numOfWorkers;
private _callback;
private readonly _computeWorkerKey;
private readonly _workerSchedulingPolicy;
private readonly _cacheKeys;
private readonly _locks;
private _offset;
private readonly _taskQueue;
constructor(_numOfWorkers: number, _callback: Function, options?: {
computeWorkerKey?: FarmOptions['computeWorkerKey'];
workerSchedulingPolicy?: FarmOptions['workerSchedulingPolicy'];
taskQueue?: TaskQueue;
});
doWork(method: string, ...args: Array<unknown>): PromiseWithCustomMessage<unknown>;
private _process;
private _push;
private _getNextWorkerOffset;
private _lock;
private _unlock;
private _isLocked;
}

View File

@@ -4,114 +4,78 @@ Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = void 0;
var _FifoQueue = _interopRequireDefault(require('./FifoQueue'));
var _types = require('./types');
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
class Farm {
_computeWorkerKey;
_workerSchedulingPolicy;
_cacheKeys = Object.create(null);
_locks = [];
_offset = 0;
_taskQueue;
constructor(_numOfWorkers, _callback, options = {}) {
var _options$workerSchedu, _options$taskQueue;
_defineProperty(this, '_computeWorkerKey', void 0);
_defineProperty(this, '_workerSchedulingPolicy', void 0);
_defineProperty(this, '_cacheKeys', Object.create(null));
_defineProperty(this, '_locks', []);
_defineProperty(this, '_offset', 0);
_defineProperty(this, '_taskQueue', void 0);
this._numOfWorkers = _numOfWorkers;
this._callback = _callback;
this._computeWorkerKey = options.computeWorkerKey;
this._workerSchedulingPolicy =
(_options$workerSchedu = options.workerSchedulingPolicy) !== null &&
_options$workerSchedu !== void 0
? _options$workerSchedu
: 'round-robin';
this._taskQueue =
(_options$taskQueue = options.taskQueue) !== null &&
_options$taskQueue !== void 0
? _options$taskQueue
: new _FifoQueue.default();
options.workerSchedulingPolicy ?? 'round-robin';
this._taskQueue = options.taskQueue ?? new _FifoQueue.default();
}
doWork(method, ...args) {
const customMessageListeners = new Set();
const addCustomMessageListener = listener => {
customMessageListeners.add(listener);
return () => {
customMessageListeners.delete(listener);
};
};
const onCustomMessage = message => {
customMessageListeners.forEach(listener => listener(message));
};
const promise = new Promise( // Bind args to this function so it won't reference to the parent scope.
const promise = new Promise(
// Bind args to this function so it won't reference to the parent scope.
// This prevents a memory leak in v8, because otherwise the function will
// retaine args for the closure.
// retain args for the closure.
((args, resolve, reject) => {
const computeWorkerKey = this._computeWorkerKey;
const request = [_types.CHILD_MESSAGE_CALL, false, method, args];
let worker = null;
let hash = null;
if (computeWorkerKey) {
hash = computeWorkerKey.call(this, method, ...args);
worker = hash == null ? null : this._cacheKeys[hash];
}
const onStart = worker => {
if (hash != null) {
this._cacheKeys[hash] = worker;
}
};
const onEnd = (error, result) => {
customMessageListeners.clear();
if (error) {
reject(error);
} else {
resolve(result);
}
};
const task = {
onCustomMessage,
onEnd,
onStart,
request
};
if (worker) {
this._taskQueue.enqueue(task, worker.getWorkerId());
this._process(worker.getWorkerId());
} else {
this._push(task);
@@ -121,38 +85,32 @@ class Farm {
promise.UNSTABLE_onCustomMessage = addCustomMessageListener;
return promise;
}
_process(workerId) {
if (this._isLocked(workerId)) {
return this;
}
const task = this._taskQueue.dequeue(workerId);
if (!task) {
return this;
}
if (task.request[1]) {
throw new Error('Queue implementation returned processed task');
} // Reference the task object outside so it won't be retained by onEnd,
}
// Reference the task object outside so it won't be retained by onEnd,
// and other properties of the task object, such as task.request can be
// garbage collected.
const taskOnEnd = task.onEnd;
let taskOnEnd = task.onEnd;
const onEnd = (error, result) => {
taskOnEnd(error, result);
if (taskOnEnd) {
taskOnEnd(error, result);
}
taskOnEnd = null;
this._unlock(workerId);
this._process(workerId);
};
task.request[1] = true;
this._lock(workerId);
this._callback(
workerId,
task.request,
@@ -160,47 +118,35 @@ class Farm {
onEnd,
task.onCustomMessage
);
return this;
}
_push(task) {
this._taskQueue.enqueue(task);
const offset = this._getNextWorkerOffset();
for (let i = 0; i < this._numOfWorkers; i++) {
this._process((offset + i) % this._numOfWorkers);
if (task.request[1]) {
break;
}
}
return this;
}
_getNextWorkerOffset() {
switch (this._workerSchedulingPolicy) {
case 'in-order':
return 0;
case 'round-robin':
return this._offset++;
}
}
_lock(workerId) {
this._locks[workerId] = true;
}
_unlock(workerId) {
this._locks[workerId] = false;
}
_isLocked(workerId) {
return this._locks[workerId];
}
}
exports.default = Farm;

View File

@@ -1,18 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import type { QueueChildMessage, TaskQueue } from './types';
/**
* First-in, First-out task queue that manages a dedicated pool
* for each worker as well as a shared queue. The FIFO ordering is guaranteed
* across the worker specific and shared queue.
*/
export default class FifoQueue implements TaskQueue {
private _workerQueues;
private _sharedQueue;
enqueue(task: QueueChildMessage, workerId?: number): void;
dequeue(workerId: number): QueueChildMessage | null;
}

View File

@@ -4,23 +4,8 @@ Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = void 0;
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
@@ -32,140 +17,73 @@ function _defineProperty(obj, key, value) {
* across the worker specific and shared queue.
*/
class FifoQueue {
constructor() {
_defineProperty(this, '_workerQueues', []);
_defineProperty(this, '_sharedQueue', new InternalQueue());
}
_workerQueues = [];
_sharedQueue = new InternalQueue();
enqueue(task, workerId) {
if (workerId == null) {
this._sharedQueue.enqueue(task);
return;
}
let workerQueue = this._workerQueues[workerId];
if (workerQueue == null) {
workerQueue = this._workerQueues[workerId] = new InternalQueue();
}
const sharedTop = this._sharedQueue.peekLast();
const item = {
previousSharedTask: sharedTop,
task
};
workerQueue.enqueue(item);
}
dequeue(workerId) {
var _this$_workerQueues$w, _workerTop$previousSh, _workerTop$previousSh2;
const workerTop =
(_this$_workerQueues$w = this._workerQueues[workerId]) === null ||
_this$_workerQueues$w === void 0
? void 0
: _this$_workerQueues$w.peek();
const workerTop = this._workerQueues[workerId]?.peek();
const sharedTaskIsProcessed =
(_workerTop$previousSh =
workerTop === null || workerTop === void 0
? void 0
: (_workerTop$previousSh2 = workerTop.previousSharedTask) === null ||
_workerTop$previousSh2 === void 0
? void 0
: _workerTop$previousSh2.request[1]) !== null &&
_workerTop$previousSh !== void 0
? _workerTop$previousSh
: true; // Process the top task from the shared queue if
workerTop?.previousSharedTask?.request[1] ?? true;
// Process the top task from the shared queue if
// - there's no task in the worker specific queue or
// - if the non-worker-specific task after which this worker specifif task
// hasn been queued wasn't processed yet
// - if the non-worker-specific task after which this worker specific task
// has been queued wasn't processed yet
if (workerTop != null && sharedTaskIsProcessed) {
var _this$_workerQueues$w2,
_this$_workerQueues$w3,
_this$_workerQueues$w4;
return (_this$_workerQueues$w2 =
(_this$_workerQueues$w3 = this._workerQueues[workerId]) === null ||
_this$_workerQueues$w3 === void 0
? void 0
: (_this$_workerQueues$w4 = _this$_workerQueues$w3.dequeue()) ===
null || _this$_workerQueues$w4 === void 0
? void 0
: _this$_workerQueues$w4.task) !== null &&
_this$_workerQueues$w2 !== void 0
? _this$_workerQueues$w2
: null;
return this._workerQueues[workerId]?.dequeue()?.task ?? null;
}
return this._sharedQueue.dequeue();
}
}
exports.default = FifoQueue;
/**
* FIFO queue for a single worker / shared queue.
*/
class InternalQueue {
constructor() {
_defineProperty(this, '_head', null);
_defineProperty(this, '_last', null);
}
_head = null;
_last = null;
enqueue(value) {
const item = {
next: null,
value
};
if (this._last == null) {
this._head = item;
} else {
this._last.next = item;
}
this._last = item;
}
dequeue() {
if (this._head == null) {
return null;
}
const item = this._head;
this._head = item.next;
if (this._head == null) {
this._last = null;
}
return item.value;
}
peek() {
var _this$_head$value, _this$_head;
return (_this$_head$value =
(_this$_head = this._head) === null || _this$_head === void 0
? void 0
: _this$_head.value) !== null && _this$_head$value !== void 0
? _this$_head$value
: null;
return this._head?.value ?? null;
}
peekLast() {
var _this$_last$value, _this$_last;
return (_this$_last$value =
(_this$_last = this._last) === null || _this$_last === void 0
? void 0
: _this$_last.value) !== null && _this$_last$value !== void 0
? _this$_last$value
: null;
return this._last?.value ?? null;
}
}

View File

@@ -1,41 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import type { QueueChildMessage, TaskQueue } from './types';
export declare type ComputeTaskPriorityCallback = (method: string, ...args: Array<unknown>) => number;
declare type QueueItem = {
task: QueueChildMessage;
priority: number;
};
/**
* Priority queue that processes tasks in natural ordering (lower priority first)
* accoridng to the priority computed by the function passed in the constructor.
*
* FIFO ordering isn't guaranteed for tasks with the same priority.
*
* Worker specific tasks with the same priority as a non-worker specific task
* are always processed first.
*/
export default class PriorityQueue implements TaskQueue {
private _computePriority;
private _queue;
private _sharedQueue;
constructor(_computePriority: ComputeTaskPriorityCallback);
enqueue(task: QueueChildMessage, workerId?: number): void;
_enqueue(task: QueueChildMessage, queue: MinHeap<QueueItem>): void;
dequeue(workerId: number): QueueChildMessage | null;
_getWorkerQueue(workerId: number): MinHeap<QueueItem>;
}
declare type HeapItem = {
priority: number;
};
declare class MinHeap<TItem extends HeapItem> {
private _heap;
peek(): TItem | null;
add(item: TItem): void;
poll(): TItem | null;
}
export {};

View File

@@ -4,23 +4,8 @@ Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = void 0;
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
@@ -28,7 +13,7 @@ function _defineProperty(obj, key, value) {
/**
* Priority queue that processes tasks in natural ordering (lower priority first)
* accoridng to the priority computed by the function passed in the constructor.
* according to the priority computed by the function passed in the constructor.
*
* FIFO ordering isn't guaranteed for tasks with the same priority.
*
@@ -36,24 +21,19 @@ function _defineProperty(obj, key, value) {
* are always processed first.
*/
class PriorityQueue {
_queue = [];
_sharedQueue = new MinHeap();
constructor(_computePriority) {
_defineProperty(this, '_queue', []);
_defineProperty(this, '_sharedQueue', new MinHeap());
this._computePriority = _computePriority;
}
enqueue(task, workerId) {
if (workerId == null) {
this._enqueue(task, this._sharedQueue);
} else {
const queue = this._getWorkerQueue(workerId);
this._enqueue(task, queue);
}
}
_enqueue(task, queue) {
const item = {
priority: this._computePriority(task.request[2], ...task.request[3]),
@@ -61,128 +41,97 @@ class PriorityQueue {
};
queue.add(item);
}
dequeue(workerId) {
const workerQueue = this._getWorkerQueue(workerId);
const workerTop = workerQueue.peek();
const sharedTop = this._sharedQueue.peek();
const sharedTop = this._sharedQueue.peek(); // use the task from the worker queue if there's no task in the shared queue
// use the task from the worker queue if there's no task in the shared queue
// or if the priority of the worker queue is smaller or equal to the
// priority of the top task in the shared queue. The tasks of the
// worker specific queue are preferred because no other worker can pick this
// specific task up.
if (
sharedTop == null ||
(workerTop != null && workerTop.priority <= sharedTop.priority)
) {
var _workerQueue$poll$tas, _workerQueue$poll;
return (_workerQueue$poll$tas =
(_workerQueue$poll = workerQueue.poll()) === null ||
_workerQueue$poll === void 0
? void 0
: _workerQueue$poll.task) !== null && _workerQueue$poll$tas !== void 0
? _workerQueue$poll$tas
: null;
return workerQueue.poll()?.task ?? null;
}
return this._sharedQueue.poll().task;
}
_getWorkerQueue(workerId) {
let queue = this._queue[workerId];
if (queue == null) {
queue = this._queue[workerId] = new MinHeap();
}
return queue;
}
}
exports.default = PriorityQueue;
class MinHeap {
constructor() {
_defineProperty(this, '_heap', []);
}
_heap = [];
peek() {
var _this$_heap$;
return (_this$_heap$ = this._heap[0]) !== null && _this$_heap$ !== void 0
? _this$_heap$
: null;
return this._heap[0] ?? null;
}
add(item) {
const nodes = this._heap;
nodes.push(item);
if (nodes.length === 1) {
return;
}
let currentIndex = nodes.length - 1;
let currentIndex = nodes.length - 1; // Bubble up the added node as long as the parent is bigger
// Bubble up the added node as long as the parent is bigger
while (currentIndex > 0) {
const parentIndex = Math.floor((currentIndex + 1) / 2) - 1;
const parent = nodes[parentIndex];
if (parent.priority <= item.priority) {
break;
}
nodes[currentIndex] = parent;
nodes[parentIndex] = item;
currentIndex = parentIndex;
}
}
poll() {
const nodes = this._heap;
const result = nodes[0];
const lastElement = nodes.pop(); // heap was empty or removed the last element
const lastElement = nodes.pop();
// heap was empty or removed the last element
if (result == null || nodes.length === 0) {
return result !== null && result !== void 0 ? result : null;
return result ?? null;
}
let index = 0;
nodes[0] =
lastElement !== null && lastElement !== void 0 ? lastElement : null;
nodes[0] = lastElement ?? null;
const element = nodes[0];
while (true) {
let swapIndex = null;
const rightChildIndex = (index + 1) * 2;
const leftChildIndex = rightChildIndex - 1;
const rightChild = nodes[rightChildIndex];
const leftChild = nodes[leftChildIndex]; // if the left child is smaller, swap with the left
const leftChild = nodes[leftChildIndex];
// if the left child is smaller, swap with the left
if (leftChild != null && leftChild.priority < element.priority) {
swapIndex = leftChildIndex;
} // If the right child is smaller or the right child is smaller than the left
// then swap with the right child
}
// If the right child is smaller or the right child is smaller than the left
// then swap with the right child
if (
rightChild != null &&
rightChild.priority < (swapIndex == null ? element : leftChild).priority
) {
swapIndex = rightChildIndex;
}
if (swapIndex == null) {
break;
}
nodes[index] = nodes[swapIndex];
nodes[swapIndex] = element;
index = swapIndex;
}
return result;
}
}

View File

@@ -1,13 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import BaseWorkerPool from './base/BaseWorkerPool';
import type { ChildMessage, OnCustomMessage, OnEnd, OnStart, WorkerInterface, WorkerOptions, WorkerPoolInterface } from './types';
declare class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface {
send(workerId: number, request: ChildMessage, onStart: OnStart, onEnd: OnEnd, onCustomMessage: OnCustomMessage): void;
createWorker(workerOptions: WorkerOptions): WorkerInterface;
}
export default WorkerPool;

View File

@@ -4,46 +4,31 @@ Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = void 0;
var _BaseWorkerPool = _interopRequireDefault(require('./base/BaseWorkerPool'));
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
const canUseWorkerThreads = () => {
try {
require('worker_threads');
return true;
} catch {
return false;
}
};
class WorkerPool extends _BaseWorkerPool.default {
send(workerId, request, onStart, onEnd, onCustomMessage) {
this.restartWorkerIfShutDown(workerId);
this.getWorkerById(workerId).send(request, onStart, onEnd, onCustomMessage);
}
createWorker(workerOptions) {
let Worker;
if (this._options.enableWorkerThreads && canUseWorkerThreads()) {
if (this._options.enableWorkerThreads) {
Worker = require('./workers/NodeThreadsWorker').default;
} else {
Worker = require('./workers/ChildProcessWorker').default;
}
return new Worker(workerOptions);
}
}
var _default = WorkerPool;
exports.default = _default;

View File

@@ -1,21 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/// <reference types="node" />
import { PoolExitResult, WorkerInterface, WorkerOptions, WorkerPoolOptions } from '../types';
export default class BaseWorkerPool {
private readonly _stderr;
private readonly _stdout;
protected readonly _options: WorkerPoolOptions;
private readonly _workers;
constructor(workerPath: string, options: WorkerPoolOptions);
getStderr(): NodeJS.ReadableStream;
getStdout(): NodeJS.ReadableStream;
getWorkers(): Array<WorkerInterface>;
getWorkerById(workerId: number): WorkerInterface;
createWorker(_workerOptions: WorkerOptions): WorkerInterface;
end(): Promise<PoolExitResult>;
}

View File

@@ -4,120 +4,48 @@ Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = void 0;
function path() {
const data = _interopRequireWildcard(require('path'));
path = function () {
return data;
};
return data;
}
function _mergeStream() {
const data = _interopRequireDefault(require('merge-stream'));
_mergeStream = function () {
return data;
};
return data;
}
var _types = require('../types');
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
function _getRequireWildcardCache(nodeInterop) {
if (typeof WeakMap !== 'function') return null;
var cacheBabelInterop = new WeakMap();
var cacheNodeInterop = new WeakMap();
return (_getRequireWildcardCache = function (nodeInterop) {
return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
})(nodeInterop);
}
function _interopRequireWildcard(obj, nodeInterop) {
if (!nodeInterop && obj && obj.__esModule) {
return obj;
}
if (obj === null || (typeof obj !== 'object' && typeof obj !== 'function')) {
return {default: obj};
}
var cache = _getRequireWildcardCache(nodeInterop);
if (cache && cache.has(obj)) {
return cache.get(obj);
}
var newObj = {};
var hasPropertyDescriptor =
Object.defineProperty && Object.getOwnPropertyDescriptor;
for (var key in obj) {
if (key !== 'default' && Object.prototype.hasOwnProperty.call(obj, key)) {
var desc = hasPropertyDescriptor
? Object.getOwnPropertyDescriptor(obj, key)
: null;
if (desc && (desc.get || desc.set)) {
Object.defineProperty(newObj, key, desc);
} else {
newObj[key] = obj[key];
}
}
}
newObj.default = obj;
if (cache) {
cache.set(obj, newObj);
}
return newObj;
}
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
// How long to wait for the child process to terminate
// after CHILD_MESSAGE_END before sending force exiting.
const FORCE_EXIT_DELAY = 500;
/* istanbul ignore next */
// eslint-disable-next-line @typescript-eslint/no-empty-function
const emptyMethod = () => {};
class BaseWorkerPool {
_stderr;
_stdout;
_options;
_workers;
_workerPath;
constructor(workerPath, options) {
_defineProperty(this, '_stderr', void 0);
_defineProperty(this, '_stdout', void 0);
_defineProperty(this, '_options', void 0);
_defineProperty(this, '_workers', void 0);
this._options = options;
this._workerPath = workerPath;
this._workers = new Array(options.numWorkers);
if (!path().isAbsolute(workerPath)) {
workerPath = require.resolve(workerPath);
}
const stdout = (0, _mergeStream().default)();
const stderr = (0, _mergeStream().default)();
const {forkOptions, maxRetries, resourceLimits, setupArgs} = options;
for (let i = 0; i < options.numWorkers; i++) {
const workerOptions = {
forkOptions,
idleMemoryLimit: this._options.idleMemoryLimit,
maxRetries,
resourceLimits,
setupArgs,
@@ -127,42 +55,70 @@ class BaseWorkerPool {
const worker = this.createWorker(workerOptions);
const workerStdout = worker.getStdout();
const workerStderr = worker.getStderr();
if (workerStdout) {
stdout.add(workerStdout);
}
if (workerStderr) {
stderr.add(workerStderr);
}
this._workers[i] = worker;
}
this._stdout = stdout;
this._stderr = stderr;
}
getStderr() {
return this._stderr;
}
getStdout() {
return this._stdout;
}
getWorkers() {
return this._workers;
}
getWorkerById(workerId) {
return this._workers[workerId];
}
restartWorkerIfShutDown(workerId) {
if (this._workers[workerId].state === _types.WorkerStates.SHUT_DOWN) {
const {forkOptions, maxRetries, resourceLimits, setupArgs} =
this._options;
const workerOptions = {
forkOptions,
idleMemoryLimit: this._options.idleMemoryLimit,
maxRetries,
resourceLimits,
setupArgs,
workerId,
workerPath: this._workerPath
};
const worker = this.createWorker(workerOptions);
this._workers[workerId] = worker;
}
}
createWorker(_workerOptions) {
throw Error('Missing method createWorker in WorkerPool');
}
async start() {
await Promise.all(
this._workers.map(async worker => {
await worker.waitForWorkerReady();
await new Promise((resolve, reject) => {
worker.send(
[_types.CHILD_MESSAGE_CALL_SETUP],
emptyMethod,
error => {
if (error) {
reject(error);
} else {
resolve();
}
},
emptyMethod
);
});
})
);
}
async end() {
// We do not cache the request object here. If so, it would only be only
// processed by one of the workers, and we want them all to close.
@@ -172,20 +128,20 @@ class BaseWorkerPool {
emptyMethod,
emptyMethod,
emptyMethod
); // Schedule a force exit in case worker fails to exit gracefully so
// await worker.waitForExit() never takes longer than FORCE_EXIT_DELAY
);
// Schedule a force exit in case worker fails to exit gracefully so
// await worker.waitForExit() never takes longer than FORCE_EXIT_DELAY
let forceExited = false;
const forceExitTimeout = setTimeout(() => {
worker.forceExit();
forceExited = true;
}, FORCE_EXIT_DELAY);
await worker.waitForExit(); // Worker ideally exited gracefully, don't send force exit then
await worker.waitForExit();
// Worker ideally exited gracefully, don't send force exit then
clearTimeout(forceExitTimeout);
return forceExited;
});
const workerExits = await Promise.all(workerExitPromises);
return workerExits.reduce(
(result, forceExited) => ({
@@ -197,5 +153,4 @@ class BaseWorkerPool {
);
}
}
exports.default = BaseWorkerPool;

View File

@@ -1,14 +1,179 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/// <reference types="node" />
import type { FarmOptions, PoolExitResult, PromiseWithCustomMessage, TaskQueue } from './types';
export { default as PriorityQueue } from './PriorityQueue';
export { default as FifoQueue } from './FifoQueue';
export { default as messageParent } from './workers/messageParent';
import type {ForkOptions} from 'child_process';
import type {ResourceLimits} from 'worker_threads';
declare const CHILD_MESSAGE_CALL = 1;
declare const CHILD_MESSAGE_CALL_SETUP = 4;
declare const CHILD_MESSAGE_END = 2;
declare const CHILD_MESSAGE_INITIALIZE = 0;
declare const CHILD_MESSAGE_MEM_USAGE = 3;
declare type ChildMessage =
| ChildMessageInitialize
| ChildMessageCall
| ChildMessageEnd
| ChildMessageMemUsage
| ChildMessageCallSetup;
declare type ChildMessageCall = [
type: typeof CHILD_MESSAGE_CALL,
isProcessed: boolean,
methodName: string,
args: Array<unknown>,
];
declare type ChildMessageCallSetup = [type: typeof CHILD_MESSAGE_CALL_SETUP];
declare type ChildMessageEnd = [
type: typeof CHILD_MESSAGE_END,
isProcessed: boolean,
];
declare type ChildMessageInitialize = [
type: typeof CHILD_MESSAGE_INITIALIZE,
isProcessed: boolean,
fileName: string,
setupArgs: Array<unknown>,
workerId: string | undefined,
];
declare type ChildMessageMemUsage = [type: typeof CHILD_MESSAGE_MEM_USAGE];
declare type ComputeTaskPriorityCallback = (
method: string,
...args: Array<unknown>
) => number;
declare type ExcludeReservedKeys<K> = Exclude<K, ReservedKeys>;
/**
* First-in, First-out task queue that manages a dedicated pool
* for each worker as well as a shared queue. The FIFO ordering is guaranteed
* across the worker specific and shared queue.
*/
export declare class FifoQueue implements TaskQueue {
private _workerQueues;
private readonly _sharedQueue;
enqueue(task: QueueChildMessage, workerId?: number): void;
dequeue(workerId: number): QueueChildMessage | null;
}
declare type FunctionLike = (...args: any) => unknown;
declare type HeapItem = {
priority: number;
};
export declare type JestWorkerFarm<T extends Record<string, unknown>> =
Worker_2 & WorkerModule<T>;
export declare function messageParent(
message: unknown,
parentProcess?: NodeJS.Process,
): void;
declare type MethodLikeKeys<T> = {
[K in keyof T]: T[K] extends FunctionLike ? K : never;
}[keyof T];
declare class MinHeap<TItem extends HeapItem> {
private readonly _heap;
peek(): TItem | null;
add(item: TItem): void;
poll(): TItem | null;
}
declare type OnCustomMessage = (message: Array<unknown> | unknown) => void;
declare type OnEnd = (err: Error | null, result: unknown) => void;
declare type OnStart = (worker: WorkerInterface) => void;
declare type OnStateChangeHandler = (
state: WorkerStates,
oldState: WorkerStates,
) => void;
declare type PoolExitResult = {
forceExited: boolean;
};
/**
* Priority queue that processes tasks in natural ordering (lower priority first)
* according to the priority computed by the function passed in the constructor.
*
* FIFO ordering isn't guaranteed for tasks with the same priority.
*
* Worker specific tasks with the same priority as a non-worker specific task
* are always processed first.
*/
export declare class PriorityQueue implements TaskQueue {
private readonly _computePriority;
private _queue;
private readonly _sharedQueue;
constructor(_computePriority: ComputeTaskPriorityCallback);
enqueue(task: QueueChildMessage, workerId?: number): void;
_enqueue(task: QueueChildMessage, queue: MinHeap<QueueItem>): void;
dequeue(workerId: number): QueueChildMessage | null;
_getWorkerQueue(workerId: number): MinHeap<QueueItem>;
}
export declare interface PromiseWithCustomMessage<T> extends Promise<T> {
UNSTABLE_onCustomMessage?: (listener: OnCustomMessage) => () => void;
}
declare type Promisify<T extends FunctionLike> = ReturnType<T> extends Promise<
infer R
>
? (...args: Parameters<T>) => Promise<R>
: (...args: Parameters<T>) => Promise<ReturnType<T>>;
declare type QueueChildMessage = {
request: ChildMessageCall;
onStart: OnStart;
onEnd: OnEnd;
onCustomMessage: OnCustomMessage;
};
declare type QueueItem = {
task: QueueChildMessage;
priority: number;
};
declare type ReservedKeys =
| 'end'
| 'getStderr'
| 'getStdout'
| 'setup'
| 'teardown';
export declare interface TaskQueue {
/**
* Enqueues the task in the queue for the specified worker or adds it to the
* queue shared by all workers
* @param task the task to queue
* @param workerId the id of the worker that should process this task or undefined
* if there's no preference.
*/
enqueue(task: QueueChildMessage, workerId?: number): void;
/**
* Dequeues the next item from the queue for the specified worker
* @param workerId the id of the worker for which the next task should be retrieved
*/
dequeue(workerId: number): QueueChildMessage | null;
}
/**
* The Jest farm (publicly called "Worker") is a class that allows you to queue
* methods across multiple child processes, in order to parallelize work. This
@@ -34,16 +199,157 @@ export { default as messageParent } from './workers/messageParent';
* processed by the same worker. This is specially useful if your workers
* are caching results.
*/
export declare class Worker {
private _ending;
private _farm;
private _options;
private _workerPool;
constructor(workerPath: string, options?: FarmOptions);
private _bindExposedWorkerMethods;
private _callFunctionWithArgs;
getStderr(): NodeJS.ReadableStream;
getStdout(): NodeJS.ReadableStream;
end(): Promise<PoolExitResult>;
declare class Worker_2 {
private _ending;
private readonly _farm;
private readonly _options;
private readonly _workerPool;
constructor(workerPath: string | URL, options?: WorkerFarmOptions);
private _bindExposedWorkerMethods;
private _callFunctionWithArgs;
getStderr(): NodeJS.ReadableStream;
getStdout(): NodeJS.ReadableStream;
start(): Promise<void>;
end(): Promise<PoolExitResult>;
}
export type { PromiseWithCustomMessage, TaskQueue };
export {Worker_2 as Worker};
declare type WorkerCallback = (
workerId: number,
request: ChildMessage,
onStart: OnStart,
onEnd: OnEnd,
onCustomMessage: OnCustomMessage,
) => void;
declare enum WorkerEvents {
STATE_CHANGE = 'state-change',
}
export declare type WorkerFarmOptions = {
computeWorkerKey?: (method: string, ...args: Array<unknown>) => string | null;
enableWorkerThreads?: boolean;
exposedMethods?: ReadonlyArray<string>;
forkOptions?: ForkOptions;
maxRetries?: number;
numWorkers?: number;
resourceLimits?: ResourceLimits;
setupArgs?: Array<unknown>;
taskQueue?: TaskQueue;
WorkerPool?: new (
workerPath: string,
options?: WorkerPoolOptions,
) => WorkerPoolInterface;
workerSchedulingPolicy?: WorkerSchedulingPolicy;
idleMemoryLimit?: number;
};
declare interface WorkerInterface {
get state(): WorkerStates;
send(
request: ChildMessage,
onProcessStart: OnStart,
onProcessEnd: OnEnd,
onCustomMessage: OnCustomMessage,
): void;
waitForExit(): Promise<void>;
forceExit(): void;
getWorkerId(): number;
getStderr(): NodeJS.ReadableStream | null;
getStdout(): NodeJS.ReadableStream | null;
/**
* Some system level identifier for the worker. IE, process id, thread id, etc.
*/
getWorkerSystemId(): number;
getMemoryUsage(): Promise<number | null>;
/**
* Checks to see if the child worker is actually running.
*/
isWorkerRunning(): boolean;
/**
* When the worker child is started and ready to start handling requests.
*
* @remarks
* This mostly exists to help with testing so that you don't check the status
* of things like isWorkerRunning before it actually is.
*/
waitForWorkerReady(): Promise<void>;
}
declare type WorkerModule<T> = {
[K in keyof T as Extract<
ExcludeReservedKeys<K>,
MethodLikeKeys<T>
>]: T[K] extends FunctionLike ? Promisify<T[K]> : never;
};
declare type WorkerOptions_2 = {
forkOptions: ForkOptions;
resourceLimits: ResourceLimits;
setupArgs: Array<unknown>;
maxRetries: number;
workerId: number;
workerData?: unknown;
workerPath: string;
/**
* After a job has executed the memory usage it should return to.
*
* @remarks
* Note this is different from ResourceLimits in that it checks at idle, after
* a job is complete. So you could have a resource limit of 500MB but an idle
* limit of 50MB. The latter will only trigger if after a job has completed the
* memory usage hasn't returned back down under 50MB.
*/
idleMemoryLimit?: number;
/**
* This mainly exists so the path can be changed during testing.
* https://github.com/jestjs/jest/issues/9543
*/
childWorkerPath?: string;
/**
* This is useful for debugging individual tests allowing you to see
* the raw output of the worker.
*/
silent?: boolean;
/**
* Used to immediately bind event handlers.
*/
on?: {
[WorkerEvents.STATE_CHANGE]:
| OnStateChangeHandler
| ReadonlyArray<OnStateChangeHandler>;
};
};
export declare interface WorkerPoolInterface {
getStderr(): NodeJS.ReadableStream;
getStdout(): NodeJS.ReadableStream;
getWorkers(): Array<WorkerInterface>;
createWorker(options: WorkerOptions_2): WorkerInterface;
send: WorkerCallback;
start(): Promise<void>;
end(): Promise<PoolExitResult>;
}
export declare type WorkerPoolOptions = {
setupArgs: Array<unknown>;
forkOptions: ForkOptions;
resourceLimits: ResourceLimits;
maxRetries: number;
numWorkers: number;
enableWorkerThreads: boolean;
idleMemoryLimit?: number;
};
declare type WorkerSchedulingPolicy = 'round-robin' | 'in-order';
declare enum WorkerStates {
STARTING = 'starting',
OK = 'ok',
OUT_OF_MEMORY = 'oom',
RESTARTING = 'restarting',
SHUTTING_DOWN = 'shutting-down',
SHUT_DOWN = 'shut-down',
}
export {};

View File

@@ -22,63 +22,63 @@ Object.defineProperty(exports, 'messageParent', {
return _messageParent.default;
}
});
function _os() {
const data = require('os');
_os = function () {
return data;
};
return data;
}
function _path() {
const data = require('path');
_path = function () {
return data;
};
return data;
}
function _url() {
const data = require('url');
_url = function () {
return data;
};
return data;
}
var _Farm = _interopRequireDefault(require('./Farm'));
var _WorkerPool = _interopRequireDefault(require('./WorkerPool'));
var _PriorityQueue = _interopRequireDefault(require('./PriorityQueue'));
var _FifoQueue = _interopRequireDefault(require('./FifoQueue'));
var _messageParent = _interopRequireDefault(require('./workers/messageParent'));
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
function getExposedMethods(workerPath, options) {
let exposedMethods = options.exposedMethods; // If no methods list is given, try getting it by auto-requiring the module.
let exposedMethods = options.exposedMethods;
// If no methods list is given, try getting it by auto-requiring the module.
if (!exposedMethods) {
const module = require(workerPath);
exposedMethods = Object.keys(module).filter(
// @ts-expect-error: no index
name => typeof module[name] === 'function'
);
if (typeof module === 'function') {
exposedMethods = [...exposedMethods, 'default'];
}
}
return exposedMethods;
}
function getNumberOfCpus() {
return typeof _os().availableParallelism === 'function'
? (0, _os().availableParallelism)()
: (0, _os().cpus)().length;
}
/**
* The Jest farm (publicly called "Worker") is a class that allows you to queue
* methods across multiple child processes, in order to parallelize work. This
@@ -104,61 +104,35 @@ function getExposedMethods(workerPath, options) {
* processed by the same worker. This is specially useful if your workers
* are caching results.
*/
class Worker {
_ending;
_farm;
_options;
_workerPool;
constructor(workerPath, options) {
var _this$_options$enable,
_this$_options$forkOp,
_this$_options$maxRet,
_this$_options$numWor,
_this$_options$resour,
_this$_options$setupA;
_defineProperty(this, '_ending', void 0);
_defineProperty(this, '_farm', void 0);
_defineProperty(this, '_options', void 0);
_defineProperty(this, '_workerPool', void 0);
this._options = {...options};
this._ending = false;
const workerPoolOptions = {
enableWorkerThreads:
(_this$_options$enable = this._options.enableWorkerThreads) !== null &&
_this$_options$enable !== void 0
? _this$_options$enable
: false,
forkOptions:
(_this$_options$forkOp = this._options.forkOptions) !== null &&
_this$_options$forkOp !== void 0
? _this$_options$forkOp
: {},
maxRetries:
(_this$_options$maxRet = this._options.maxRetries) !== null &&
_this$_options$maxRet !== void 0
? _this$_options$maxRet
: 3,
numWorkers:
(_this$_options$numWor = this._options.numWorkers) !== null &&
_this$_options$numWor !== void 0
? _this$_options$numWor
: Math.max((0, _os().cpus)().length - 1, 1),
resourceLimits:
(_this$_options$resour = this._options.resourceLimits) !== null &&
_this$_options$resour !== void 0
? _this$_options$resour
: {},
setupArgs:
(_this$_options$setupA = this._options.setupArgs) !== null &&
_this$_options$setupA !== void 0
? _this$_options$setupA
: []
this._options = {
...options
};
this._ending = false;
if (typeof workerPath !== 'string') {
workerPath = workerPath.href;
}
if (workerPath.startsWith('file:')) {
workerPath = (0, _url().fileURLToPath)(workerPath);
} else if (!(0, _path().isAbsolute)(workerPath)) {
throw new Error(`'workerPath' must be absolute, got '${workerPath}'`);
}
const workerPoolOptions = {
enableWorkerThreads: this._options.enableWorkerThreads ?? false,
forkOptions: this._options.forkOptions ?? {},
idleMemoryLimit: this._options.idleMemoryLimit,
maxRetries: this._options.maxRetries ?? 3,
numWorkers:
this._options.numWorkers ?? Math.max(getNumberOfCpus() - 1, 1),
resourceLimits: this._options.resourceLimits ?? {},
setupArgs: this._options.setupArgs ?? []
};
if (this._options.WorkerPool) {
// @ts-expect-error: constructor target any?
this._workerPool = new this._options.WorkerPool(
workerPath,
workerPoolOptions
@@ -166,7 +140,6 @@ class Worker {
} else {
this._workerPool = new _WorkerPool.default(workerPath, workerPoolOptions);
}
this._farm = new _Farm.default(
workerPoolOptions.numWorkers,
this._workerPool.send.bind(this._workerPool),
@@ -176,48 +149,44 @@ class Worker {
workerSchedulingPolicy: this._options.workerSchedulingPolicy
}
);
this._bindExposedWorkerMethods(workerPath, this._options);
}
_bindExposedWorkerMethods(workerPath, options) {
getExposedMethods(workerPath, options).forEach(name => {
if (name.startsWith('_')) {
return;
}
// eslint-disable-next-line no-prototype-builtins
if (this.constructor.prototype.hasOwnProperty(name)) {
throw new TypeError('Cannot define a method called ' + name);
} // @ts-expect-error: dynamic extension of the class instance is expected.
throw new TypeError(`Cannot define a method called ${name}`);
}
// @ts-expect-error: dynamic extension of the class instance is expected.
this[name] = this._callFunctionWithArgs.bind(this, name);
});
}
_callFunctionWithArgs(method, ...args) {
if (this._ending) {
throw new Error('Farm is ended, no more calls can be done to it');
}
return this._farm.doWork(method, ...args);
}
getStderr() {
return this._workerPool.getStderr();
}
getStdout() {
return this._workerPool.getStdout();
}
async start() {
await this._workerPool.start();
}
async end() {
if (this._ending) {
throw new Error('Farm is ended, no more calls can be done to it');
}
this._ending = true;
return this._workerPool.end();
}
}
exports.Worker = Worker;

View File

@@ -1,143 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/// <reference types="node" />
import type { ForkOptions } from 'child_process';
import type { EventEmitter } from 'events';
export interface ResourceLimits {
maxYoungGenerationSizeMb?: number;
maxOldGenerationSizeMb?: number;
codeRangeSizeMb?: number;
stackSizeMb?: number;
}
export declare const CHILD_MESSAGE_INITIALIZE: 0;
export declare const CHILD_MESSAGE_CALL: 1;
export declare const CHILD_MESSAGE_END: 2;
export declare const PARENT_MESSAGE_OK: 0;
export declare const PARENT_MESSAGE_CLIENT_ERROR: 1;
export declare const PARENT_MESSAGE_SETUP_ERROR: 2;
export declare const PARENT_MESSAGE_CUSTOM: 3;
export declare type PARENT_MESSAGE_ERROR = typeof PARENT_MESSAGE_CLIENT_ERROR | typeof PARENT_MESSAGE_SETUP_ERROR;
export interface WorkerPoolInterface {
getStderr(): NodeJS.ReadableStream;
getStdout(): NodeJS.ReadableStream;
getWorkers(): Array<WorkerInterface>;
createWorker(options: WorkerOptions): WorkerInterface;
send(workerId: number, request: ChildMessage, onStart: OnStart, onEnd: OnEnd, onCustomMessage: OnCustomMessage): void;
end(): Promise<PoolExitResult>;
}
export interface WorkerInterface {
send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd, onCustomMessage: OnCustomMessage): void;
waitForExit(): Promise<void>;
forceExit(): void;
getWorkerId(): number;
getStderr(): NodeJS.ReadableStream | null;
getStdout(): NodeJS.ReadableStream | null;
}
export declare type PoolExitResult = {
forceExited: boolean;
};
export interface PromiseWithCustomMessage<T> extends Promise<T> {
UNSTABLE_onCustomMessage?: (listener: OnCustomMessage) => () => void;
}
export type { ForkOptions };
export interface TaskQueue {
/**
* Enqueues the task in the queue for the specified worker or adds it to the
* queue shared by all workers
* @param task the task to queue
* @param workerId the id of the worker that should process this task or undefined
* if there's no preference.
*/
enqueue(task: QueueChildMessage, workerId?: number): void;
/**
* Dequeues the next item from the queue for the speified worker
* @param workerId the id of the worker for which the next task should be retrieved
*/
dequeue(workerId: number): QueueChildMessage | null;
}
export declare type FarmOptions = {
computeWorkerKey?: (method: string, ...args: Array<unknown>) => string | null;
exposedMethods?: ReadonlyArray<string>;
forkOptions?: ForkOptions;
workerSchedulingPolicy?: 'round-robin' | 'in-order';
resourceLimits?: ResourceLimits;
setupArgs?: Array<unknown>;
maxRetries?: number;
numWorkers?: number;
taskQueue?: TaskQueue;
WorkerPool?: (workerPath: string, options?: WorkerPoolOptions) => WorkerPoolInterface;
enableWorkerThreads?: boolean;
};
export declare type WorkerPoolOptions = {
setupArgs: Array<unknown>;
forkOptions: ForkOptions;
resourceLimits: ResourceLimits;
maxRetries: number;
numWorkers: number;
enableWorkerThreads: boolean;
};
export declare type WorkerOptions = {
forkOptions: ForkOptions;
resourceLimits: ResourceLimits;
setupArgs: Array<unknown>;
maxRetries: number;
workerId: number;
workerData?: unknown;
workerPath: string;
};
export declare type MessagePort = typeof EventEmitter & {
postMessage(message: unknown): void;
};
export declare type MessageChannel = {
port1: MessagePort;
port2: MessagePort;
};
export declare type ChildMessageInitialize = [
typeof CHILD_MESSAGE_INITIALIZE,
boolean,
string,
// file
Array<unknown> | undefined,
// setupArgs
MessagePort | undefined
];
export declare type ChildMessageCall = [
typeof CHILD_MESSAGE_CALL,
boolean,
string,
Array<unknown>
];
export declare type ChildMessageEnd = [
typeof CHILD_MESSAGE_END,
boolean
];
export declare type ChildMessage = ChildMessageInitialize | ChildMessageCall | ChildMessageEnd;
export declare type ParentMessageCustom = [
typeof PARENT_MESSAGE_CUSTOM,
unknown
];
export declare type ParentMessageOk = [
typeof PARENT_MESSAGE_OK,
unknown
];
export declare type ParentMessageError = [
PARENT_MESSAGE_ERROR,
string,
string,
string,
unknown
];
export declare type ParentMessage = ParentMessageOk | ParentMessageError | ParentMessageCustom;
export declare type OnStart = (worker: WorkerInterface) => void;
export declare type OnEnd = (err: Error | null, result: unknown) => void;
export declare type OnCustomMessage = (message: Array<unknown> | unknown) => void;
export declare type QueueChildMessage = {
request: ChildMessageCall;
onStart: OnStart;
onEnd: OnEnd;
onCustomMessage: OnCustomMessage;
};

View File

@@ -3,32 +3,40 @@
Object.defineProperty(exports, '__esModule', {
value: true
});
exports.PARENT_MESSAGE_SETUP_ERROR =
exports.WorkerStates =
exports.WorkerEvents =
exports.PARENT_MESSAGE_SETUP_ERROR =
exports.PARENT_MESSAGE_OK =
exports.PARENT_MESSAGE_MEM_USAGE =
exports.PARENT_MESSAGE_CUSTOM =
exports.PARENT_MESSAGE_CLIENT_ERROR =
exports.CHILD_MESSAGE_MEM_USAGE =
exports.CHILD_MESSAGE_INITIALIZE =
exports.CHILD_MESSAGE_END =
exports.CHILD_MESSAGE_CALL_SETUP =
exports.CHILD_MESSAGE_CALL =
void 0;
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
// import type {ResourceLimits} from 'worker_threads';
// This is not present in the Node 12 typings
// Because of the dynamic nature of a worker communication process, all messages
// coming from any of the other processes cannot be typed. Thus, many types
// include "unknown" as a TS type, which is (unfortunately) correct here.
const CHILD_MESSAGE_INITIALIZE = 0;
exports.CHILD_MESSAGE_INITIALIZE = CHILD_MESSAGE_INITIALIZE;
const CHILD_MESSAGE_CALL = 1;
exports.CHILD_MESSAGE_CALL = CHILD_MESSAGE_CALL;
const CHILD_MESSAGE_END = 2;
exports.CHILD_MESSAGE_END = CHILD_MESSAGE_END;
const CHILD_MESSAGE_MEM_USAGE = 3;
exports.CHILD_MESSAGE_MEM_USAGE = CHILD_MESSAGE_MEM_USAGE;
const CHILD_MESSAGE_CALL_SETUP = 4;
exports.CHILD_MESSAGE_CALL_SETUP = CHILD_MESSAGE_CALL_SETUP;
const PARENT_MESSAGE_OK = 0;
exports.PARENT_MESSAGE_OK = PARENT_MESSAGE_OK;
const PARENT_MESSAGE_CLIENT_ERROR = 1;
@@ -37,3 +45,28 @@ const PARENT_MESSAGE_SETUP_ERROR = 2;
exports.PARENT_MESSAGE_SETUP_ERROR = PARENT_MESSAGE_SETUP_ERROR;
const PARENT_MESSAGE_CUSTOM = 3;
exports.PARENT_MESSAGE_CUSTOM = PARENT_MESSAGE_CUSTOM;
const PARENT_MESSAGE_MEM_USAGE = 4;
// Option objects.
// Messages passed from the parent to the children.
// Messages passed from the children to the parent.
// Queue types.
exports.PARENT_MESSAGE_MEM_USAGE = PARENT_MESSAGE_MEM_USAGE;
let WorkerStates = /*#__PURE__*/ (function (WorkerStates) {
WorkerStates['STARTING'] = 'starting';
WorkerStates['OK'] = 'ok';
WorkerStates['OUT_OF_MEMORY'] = 'oom';
WorkerStates['RESTARTING'] = 'restarting';
WorkerStates['SHUTTING_DOWN'] = 'shutting-down';
WorkerStates['SHUT_DOWN'] = 'shut-down';
return WorkerStates;
})({});
exports.WorkerStates = WorkerStates;
let WorkerEvents = /*#__PURE__*/ (function (WorkerEvents) {
WorkerEvents['STATE_CHANGE'] = 'state-change';
return WorkerEvents;
})({});
exports.WorkerEvents = WorkerEvents;

View File

@@ -1,51 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/// <reference types="node" />
import { ChildMessage, OnCustomMessage, OnEnd, OnStart, WorkerInterface, WorkerOptions } from '../types';
/**
* This class wraps the child process and provides a nice interface to
* communicate with. It takes care of:
*
* - Re-spawning the process if it dies.
* - Queues calls while the worker is busy.
* - Re-sends the requests if the worker blew up.
*
* The reason for queueing them here (since childProcess.send also has an
* internal queue) is because the worker could be doing asynchronous work, and
* this would lead to the child process to read its receiving buffer and start a
* second call. By queueing calls here, we don't send the next call to the
* children until we receive the result of the previous one.
*
* As soon as a request starts to be processed by a worker, its "processed"
* field is changed to "true", so that other workers which might encounter the
* same call skip it.
*/
export default class ChildProcessWorker implements WorkerInterface {
private _child;
private _options;
private _request;
private _retries;
private _onProcessEnd;
private _onCustomMessage;
private _fakeStream;
private _stdout;
private _stderr;
private _exitPromise;
private _resolveExitPromise;
constructor(options: WorkerOptions);
initialize(): void;
private _shutdown;
private _onMessage;
private _onExit;
send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd, onCustomMessage: OnCustomMessage): void;
waitForExit(): Promise<void>;
forceExit(): void;
getWorkerId(): number;
getStdout(): NodeJS.ReadableStream | null;
getStderr(): NodeJS.ReadableStream | null;
private _getFakeStream;
}

View File

@@ -3,73 +3,54 @@
Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = void 0;
exports.default = exports.SIGKILL_DELAY = void 0;
function _child_process() {
const data = require('child_process');
_child_process = function () {
return data;
};
return data;
}
function _stream() {
const data = require('stream');
_stream = function () {
function _os() {
const data = require('os');
_os = function () {
return data;
};
return data;
}
function _mergeStream() {
const data = _interopRequireDefault(require('merge-stream'));
_mergeStream = function () {
return data;
};
return data;
}
function _supportsColor() {
const data = require('supports-color');
_supportsColor = function () {
return data;
};
return data;
}
var _types = require('../types');
var _WorkerAbstract = _interopRequireDefault(require('./WorkerAbstract'));
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
const SIGNAL_BASE_EXIT_CODE = 128;
const SIGKILL_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 9;
const SIGTERM_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 15; // How long to wait after SIGTERM before sending SIGKILL
const SIGTERM_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 15;
// How long to wait after SIGTERM before sending SIGKILL
const SIGKILL_DELAY = 500;
/**
* This class wraps the child process and provides a nice interface to
* communicate with. It takes care of:
@@ -88,104 +69,118 @@ const SIGKILL_DELAY = 500;
* field is changed to "true", so that other workers which might encounter the
* same call skip it.
*/
class ChildProcessWorker {
exports.SIGKILL_DELAY = SIGKILL_DELAY;
class ChildProcessWorker extends _WorkerAbstract.default {
_child;
_options;
_request;
_retries;
_onProcessEnd;
_onCustomMessage;
_stdout;
_stderr;
_stderrBuffer = [];
_memoryUsagePromise;
_resolveMemoryUsage;
_childIdleMemoryUsage;
_childIdleMemoryUsageLimit;
_memoryUsageCheck = false;
_childWorkerPath;
constructor(options) {
_defineProperty(this, '_child', void 0);
_defineProperty(this, '_options', void 0);
_defineProperty(this, '_request', void 0);
_defineProperty(this, '_retries', void 0);
_defineProperty(this, '_onProcessEnd', void 0);
_defineProperty(this, '_onCustomMessage', void 0);
_defineProperty(this, '_fakeStream', void 0);
_defineProperty(this, '_stdout', void 0);
_defineProperty(this, '_stderr', void 0);
_defineProperty(this, '_exitPromise', void 0);
_defineProperty(this, '_resolveExitPromise', void 0);
super(options);
this._options = options;
this._request = null;
this._fakeStream = null;
this._stdout = null;
this._stderr = null;
this._exitPromise = new Promise(resolve => {
this._resolveExitPromise = resolve;
});
this._childIdleMemoryUsage = null;
this._childIdleMemoryUsageLimit = options.idleMemoryLimit || null;
this._childWorkerPath =
options.childWorkerPath || require.resolve('./processChild');
this.state = _types.WorkerStates.STARTING;
this.initialize();
}
initialize() {
if (
this.state === _types.WorkerStates.OUT_OF_MEMORY ||
this.state === _types.WorkerStates.SHUTTING_DOWN ||
this.state === _types.WorkerStates.SHUT_DOWN
) {
return;
}
if (this._child && this._child.connected) {
this._child.kill('SIGKILL');
}
this.state = _types.WorkerStates.STARTING;
const forceColor = _supportsColor().stdout
? {
FORCE_COLOR: '1'
}
: {};
const child = (0, _child_process().fork)(
require.resolve('./processChild'),
const silent = this._options.silent ?? true;
if (!silent) {
// NOTE: Detecting an out of memory crash is independent of idle memory usage monitoring. We want to
// monitor for a crash occurring so that it can be handled as required and so we can tell the difference
// between an OOM crash and another kind of crash. We need to do this because if a worker crashes due to
// an OOM event sometimes it isn't seen by the worker pool and it just sits there waiting for the worker
// to respond and it never will.
console.warn('Unable to detect out of memory event if silent === false');
}
this._stderrBuffer = [];
const options = {
cwd: process.cwd(),
env: {
...process.env,
JEST_WORKER_ID: String(this._options.workerId + 1),
// 0-indexed workerId, 1-indexed JEST_WORKER_ID
...forceColor
},
// Suppress --debug / --inspect flags while preserving others (like --harmony).
execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)),
// default to advanced serialization in order to match worker threads
serialization: 'advanced',
silent,
...this._options.forkOptions
};
this._child = (0, _child_process().fork)(
this._childWorkerPath,
[],
{
cwd: process.cwd(),
env: {
...process.env,
JEST_WORKER_ID: String(this._options.workerId + 1),
// 0-indexed workerId, 1-indexed JEST_WORKER_ID
...forceColor
},
// Suppress --debug / --inspect flags while preserving others (like --harmony).
execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)),
silent: true,
...this._options.forkOptions
}
options
);
if (child.stdout) {
if (this._child.stdout) {
if (!this._stdout) {
// We need to add a permanent stream to the merged stream to prevent it
// from ending when the subprocess stream ends
this._stdout = (0, _mergeStream().default)(this._getFakeStream());
}
this._stdout.add(child.stdout);
this._stdout.add(this._child.stdout);
}
if (child.stderr) {
if (this._child.stderr) {
if (!this._stderr) {
// We need to add a permanent stream to the merged stream to prevent it
// from ending when the subprocess stream ends
this._stderr = (0, _mergeStream().default)(this._getFakeStream());
}
this._stderr.add(child.stderr);
this._stderr.add(this._child.stderr);
this._child.stderr.on('data', this.stderrDataHandler.bind(this));
}
child.on('message', this._onMessage.bind(this));
child.on('exit', this._onExit.bind(this));
child.send([
this._child.on('message', this._onMessage.bind(this));
this._child.on('exit', this._onExit.bind(this));
this._child.on('disconnect', this._onDisconnect.bind(this));
this._child.send([
_types.CHILD_MESSAGE_INITIALIZE,
false,
this._options.workerPath,
this._options.setupArgs
]);
this._child = child;
this._retries++; // If we exceeded the amount of retries, we will emulate an error reply
this._retries++;
// If we exceeded the amount of retries, we will emulate an error reply
// coming from the child. This avoids code duplication related with cleaning
// the queue, and scheduling the next call.
if (this._retries > this._options.maxRetries) {
const error = new Error(
`Jest worker encountered ${this._retries} child process exceptions, exceeding retry limit`
);
this._onMessage([
_types.PARENT_MESSAGE_CLIENT_ERROR,
error.name,
@@ -195,139 +190,301 @@ class ChildProcessWorker {
type: 'WorkerError'
}
]);
// Clear the request so we don't keep executing it.
this._request = null;
}
this.state = _types.WorkerStates.OK;
if (this._resolveWorkerReady) {
this._resolveWorkerReady();
}
}
_shutdown() {
// End the temporary streams so the merged streams end too
if (this._fakeStream) {
this._fakeStream.end();
this._fakeStream = null;
stderrDataHandler(chunk) {
if (chunk) {
this._stderrBuffer.push(Buffer.from(chunk));
}
this._detectOutOfMemoryCrash();
if (this.state === _types.WorkerStates.OUT_OF_MEMORY) {
this._workerReadyPromise = undefined;
this._resolveWorkerReady = undefined;
this.killChild();
this._shutdown();
}
}
_detectOutOfMemoryCrash() {
try {
const bufferStr = Buffer.concat(this._stderrBuffer).toString('utf8');
if (
bufferStr.includes('heap out of memory') ||
bufferStr.includes('allocation failure;') ||
bufferStr.includes('Last few GCs')
) {
if (
this.state === _types.WorkerStates.OK ||
this.state === _types.WorkerStates.STARTING
) {
this.state = _types.WorkerStates.OUT_OF_MEMORY;
}
}
} catch (err) {
console.error('Error looking for out of memory crash', err);
}
}
_onDisconnect() {
this._workerReadyPromise = undefined;
this._resolveWorkerReady = undefined;
this._detectOutOfMemoryCrash();
if (this.state === _types.WorkerStates.OUT_OF_MEMORY) {
this.killChild();
this._shutdown();
}
this._resolveExitPromise();
}
_onMessage(response) {
// Ignore messages not intended for us
if (!Array.isArray(response)) return;
// TODO: Add appropriate type check
let error;
switch (response[0]) {
case _types.PARENT_MESSAGE_OK:
this._onProcessEnd(null, response[1]);
break;
case _types.PARENT_MESSAGE_CLIENT_ERROR:
error = response[4];
if (error != null && typeof error === 'object') {
const extra = error; // @ts-expect-error: no index
const NativeCtor = global[response[1]];
const extra = error;
// @ts-expect-error: no index
const NativeCtor = globalThis[response[1]];
const Ctor = typeof NativeCtor === 'function' ? NativeCtor : Error;
error = new Ctor(response[2]);
error.type = response[1];
error.stack = response[3];
for (const key in extra) {
error[key] = extra[key];
}
}
this._onProcessEnd(error, null);
break;
case _types.PARENT_MESSAGE_SETUP_ERROR:
error = new Error('Error when calling setup: ' + response[2]);
error = new Error(`Error when calling setup: ${response[2]}`);
error.type = response[1];
error.stack = response[3];
this._onProcessEnd(error, null);
break;
case _types.PARENT_MESSAGE_CUSTOM:
this._onCustomMessage(response[1]);
break;
case _types.PARENT_MESSAGE_MEM_USAGE:
this._childIdleMemoryUsage = response[1];
if (this._resolveMemoryUsage) {
this._resolveMemoryUsage(response[1]);
this._resolveMemoryUsage = undefined;
this._memoryUsagePromise = undefined;
}
this._performRestartIfRequired();
break;
default:
throw new TypeError('Unexpected response from worker: ' + response[0]);
// Ignore messages not intended for us
break;
}
}
_performRestartIfRequired() {
if (this._memoryUsageCheck) {
this._memoryUsageCheck = false;
let limit = this._childIdleMemoryUsageLimit;
_onExit(exitCode) {
if (
exitCode !== 0 &&
exitCode !== null &&
exitCode !== SIGTERM_EXIT_CODE &&
exitCode !== SIGKILL_EXIT_CODE
// TODO: At some point it would make sense to make use of
// stringToBytes found in jest-config, however as this
// package does not have any dependencies on an other jest
// packages that can wait until some other time.
if (limit && limit > 0 && limit <= 1) {
limit = Math.floor((0, _os().totalmem)() * limit);
} else if (limit) {
limit = Math.floor(limit);
}
if (
limit &&
this._childIdleMemoryUsage &&
this._childIdleMemoryUsage > limit
) {
this.state = _types.WorkerStates.RESTARTING;
this.killChild();
}
}
}
_onExit(exitCode, signal) {
this._workerReadyPromise = undefined;
this._resolveWorkerReady = undefined;
this._detectOutOfMemoryCrash();
if (exitCode !== 0 && this.state === _types.WorkerStates.OUT_OF_MEMORY) {
this._onProcessEnd(
new Error('Jest worker ran out of memory and crashed'),
null
);
this._shutdown();
} else if (
(exitCode !== 0 &&
exitCode !== null &&
exitCode !== SIGTERM_EXIT_CODE &&
exitCode !== SIGKILL_EXIT_CODE &&
this.state !== _types.WorkerStates.SHUTTING_DOWN) ||
this.state === _types.WorkerStates.RESTARTING
) {
this.state = _types.WorkerStates.RESTARTING;
this.initialize();
if (this._request) {
this._child.send(this._request);
}
} else {
// At this point, it's not clear why the child process exited. There could
// be several reasons:
//
// 1. The child process exited successfully after finishing its work.
// This is the most likely case.
// 2. The child process crashed in a manner that wasn't caught through
// any of the heuristic-based checks above.
// 3. The child process was killed by another process or daemon unrelated
// to Jest. For example, oom-killer on Linux may have picked the child
// process to kill because overall system memory is constrained.
//
// If there's a pending request to the child process in any of those
// situations, the request still needs to be handled in some manner before
// entering the shutdown phase. Otherwise the caller expecting a response
// from the worker will never receive indication that something unexpected
// happened and hang forever.
//
// In normal operation, the request is handled and cleared before the
// child process exits. If it's still present, it's not clear what
// happened and probably best to throw an error. In practice, this usually
// happens when the child process is killed externally.
//
// There's a reasonable argument that the child process should be retried
// with request re-sent in this scenario. However, if the problem was due
// to situations such as oom-killer attempting to free up system
// resources, retrying would exacerbate the problem.
const isRequestStillPending = !!this._request;
if (isRequestStillPending) {
// If a signal is present, we can be reasonably confident the process
// was killed externally. Log this fact so it's more clear to users that
// something went wrong externally, rather than a bug in Jest itself.
const error = new Error(
signal != null
? `A jest worker process (pid=${this._child.pid}) was terminated by another process: signal=${signal}, exitCode=${exitCode}. Operating system logs may contain more information on why this occurred.`
: `A jest worker process (pid=${this._child.pid}) crashed for an unknown reason: exitCode=${exitCode}`
);
this._onProcessEnd(error, null);
}
this._shutdown();
}
}
send(request, onProcessStart, onProcessEnd, onCustomMessage) {
this._stderrBuffer = [];
onProcessStart(this);
this._onProcessEnd = (...args) => {
const hasRequest = !!this._request;
// Clean the request to avoid sending past requests to workers that fail
// while waiting for a new request (timers, unhandled rejections...)
this._request = null;
if (
this._childIdleMemoryUsageLimit &&
this._child.connected &&
hasRequest
) {
this.checkMemoryUsage();
}
return onProcessEnd(...args);
};
this._onCustomMessage = (...arg) => onCustomMessage(...arg);
this._request = request;
this._retries = 0;
// eslint-disable-next-line @typescript-eslint/no-empty-function
this._child.send(request, () => {});
}
waitForExit() {
return this._exitPromise;
}
killChild() {
// We store a reference so that there's no way we can accidentally
// kill a new worker that has been spawned.
const childToKill = this._child;
childToKill.kill('SIGTERM');
return setTimeout(() => childToKill.kill('SIGKILL'), SIGKILL_DELAY);
}
forceExit() {
this._child.kill('SIGTERM');
const sigkillTimeout = setTimeout(
() => this._child.kill('SIGKILL'),
SIGKILL_DELAY
);
this.state = _types.WorkerStates.SHUTTING_DOWN;
const sigkillTimeout = this.killChild();
this._exitPromise.then(() => clearTimeout(sigkillTimeout));
}
getWorkerId() {
return this._options.workerId;
}
/**
* Gets the process id of the worker.
*
* @returns Process id.
*/
getWorkerSystemId() {
return this._child.pid;
}
getStdout() {
return this._stdout;
}
getStderr() {
return this._stderr;
}
_getFakeStream() {
if (!this._fakeStream) {
this._fakeStream = new (_stream().PassThrough)();
/**
* Gets the last reported memory usage.
*
* @returns Memory usage in bytes.
*/
getMemoryUsage() {
if (!this._memoryUsagePromise) {
let rejectCallback;
const promise = new Promise((resolve, reject) => {
this._resolveMemoryUsage = resolve;
rejectCallback = reject;
});
this._memoryUsagePromise = promise;
if (!this._child.connected && rejectCallback) {
rejectCallback(new Error('Child process is not running.'));
this._memoryUsagePromise = undefined;
this._resolveMemoryUsage = undefined;
return promise;
}
this._child.send([_types.CHILD_MESSAGE_MEM_USAGE], err => {
if (err && rejectCallback) {
this._memoryUsagePromise = undefined;
this._resolveMemoryUsage = undefined;
rejectCallback(err);
}
});
return promise;
}
return this._memoryUsagePromise;
}
return this._fakeStream;
/**
* Gets updated memory usage and restarts if required
*/
checkMemoryUsage() {
if (this._childIdleMemoryUsageLimit) {
this._memoryUsageCheck = true;
this._child.send([_types.CHILD_MESSAGE_MEM_USAGE], err => {
if (err) {
console.error('Unable to check memory usage', err);
}
});
} else {
console.warn(
'Memory usage of workers can only be checked if a limit is set'
);
}
}
isWorkerRunning() {
return this._child.connected && !this._child.killed;
}
}
exports.default = ChildProcessWorker;

View File

@@ -1,34 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/// <reference types="node" />
import { ChildMessage, OnCustomMessage, OnEnd, OnStart, WorkerInterface, WorkerOptions } from '../types';
export default class ExperimentalWorker implements WorkerInterface {
private _worker;
private _options;
private _request;
private _retries;
private _onProcessEnd;
private _onCustomMessage;
private _fakeStream;
private _stdout;
private _stderr;
private _exitPromise;
private _resolveExitPromise;
private _forceExited;
constructor(options: WorkerOptions);
initialize(): void;
private _shutdown;
private _onMessage;
private _onExit;
waitForExit(): Promise<void>;
forceExit(): void;
send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd | null, onCustomMessage: OnCustomMessage): void;
getWorkerId(): number;
getStdout(): NodeJS.ReadableStream | null;
getStderr(): NodeJS.ReadableStream | null;
private _getFakeStream;
}

View File

@@ -4,185 +4,114 @@ Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = void 0;
function path() {
const data = _interopRequireWildcard(require('path'));
path = function () {
function _os() {
const data = require('os');
_os = function () {
return data;
};
return data;
}
function _stream() {
const data = require('stream');
_stream = function () {
return data;
};
return data;
}
function _worker_threads() {
const data = require('worker_threads');
_worker_threads = function () {
return data;
};
return data;
}
function _mergeStream() {
const data = _interopRequireDefault(require('merge-stream'));
_mergeStream = function () {
return data;
};
return data;
}
var _types = require('../types');
var _WorkerAbstract = _interopRequireDefault(require('./WorkerAbstract'));
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
function _getRequireWildcardCache(nodeInterop) {
if (typeof WeakMap !== 'function') return null;
var cacheBabelInterop = new WeakMap();
var cacheNodeInterop = new WeakMap();
return (_getRequireWildcardCache = function (nodeInterop) {
return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
})(nodeInterop);
}
function _interopRequireWildcard(obj, nodeInterop) {
if (!nodeInterop && obj && obj.__esModule) {
return obj;
}
if (obj === null || (typeof obj !== 'object' && typeof obj !== 'function')) {
return {default: obj};
}
var cache = _getRequireWildcardCache(nodeInterop);
if (cache && cache.has(obj)) {
return cache.get(obj);
}
var newObj = {};
var hasPropertyDescriptor =
Object.defineProperty && Object.getOwnPropertyDescriptor;
for (var key in obj) {
if (key !== 'default' && Object.prototype.hasOwnProperty.call(obj, key)) {
var desc = hasPropertyDescriptor
? Object.getOwnPropertyDescriptor(obj, key)
: null;
if (desc && (desc.get || desc.set)) {
Object.defineProperty(newObj, key, desc);
} else {
newObj[key] = obj[key];
}
}
}
newObj.default = obj;
if (cache) {
cache.set(obj, newObj);
}
return newObj;
}
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
class ExperimentalWorker {
class ExperimentalWorker extends _WorkerAbstract.default {
_worker;
_options;
_request;
_retries;
_onProcessEnd;
_onCustomMessage;
_stdout;
_stderr;
_memoryUsagePromise;
_resolveMemoryUsage;
_childWorkerPath;
_childIdleMemoryUsage;
_childIdleMemoryUsageLimit;
_memoryUsageCheck = false;
constructor(options) {
_defineProperty(this, '_worker', void 0);
_defineProperty(this, '_options', void 0);
_defineProperty(this, '_request', void 0);
_defineProperty(this, '_retries', void 0);
_defineProperty(this, '_onProcessEnd', void 0);
_defineProperty(this, '_onCustomMessage', void 0);
_defineProperty(this, '_fakeStream', void 0);
_defineProperty(this, '_stdout', void 0);
_defineProperty(this, '_stderr', void 0);
_defineProperty(this, '_exitPromise', void 0);
_defineProperty(this, '_resolveExitPromise', void 0);
_defineProperty(this, '_forceExited', void 0);
super(options);
this._options = options;
this._request = null;
this._fakeStream = null;
this._stdout = null;
this._stderr = null;
this._exitPromise = new Promise(resolve => {
this._resolveExitPromise = resolve;
});
this._forceExited = false;
this._childWorkerPath =
options.childWorkerPath || require.resolve('./threadChild');
this._childIdleMemoryUsage = null;
this._childIdleMemoryUsageLimit = options.idleMemoryLimit || null;
this.initialize();
}
initialize() {
this._worker = new (_worker_threads().Worker)(
path().resolve(__dirname, './threadChild.js'),
{
eval: false,
// @ts-expect-error: added in newer versions
resourceLimits: this._options.resourceLimits,
stderr: true,
stdout: true,
workerData: this._options.workerData,
...this._options.forkOptions
}
);
if (
this.state === _types.WorkerStates.OUT_OF_MEMORY ||
this.state === _types.WorkerStates.SHUTTING_DOWN ||
this.state === _types.WorkerStates.SHUT_DOWN
) {
return;
}
if (this._worker) {
this._worker.terminate();
}
this.state = _types.WorkerStates.STARTING;
this._worker = new (_worker_threads().Worker)(this._childWorkerPath, {
eval: false,
resourceLimits: this._options.resourceLimits,
stderr: true,
stdout: true,
workerData: this._options.workerData,
...this._options.forkOptions
});
if (this._worker.stdout) {
if (!this._stdout) {
// We need to add a permanent stream to the merged stream to prevent it
// from ending when the subprocess stream ends
this._stdout = (0, _mergeStream().default)(this._getFakeStream());
}
this._stdout.add(this._worker.stdout);
}
if (this._worker.stderr) {
if (!this._stderr) {
// We need to add a permanent stream to the merged stream to prevent it
// from ending when the subprocess stream ends
this._stderr = (0, _mergeStream().default)(this._getFakeStream());
}
this._stderr.add(this._worker.stderr);
}
// This can be useful for debugging.
if (!(this._options.silent ?? true)) {
this._worker.stdout.setEncoding('utf8');
// eslint-disable-next-line no-console
this._worker.stdout.on('data', console.log);
this._worker.stderr.setEncoding('utf8');
this._worker.stderr.on('data', console.error);
}
this._worker.on('message', this._onMessage.bind(this));
this._worker.on('exit', this._onExit.bind(this));
this._worker.on('error', this._onError.bind(this));
this._worker.postMessage([
_types.CHILD_MESSAGE_INITIALIZE,
false,
@@ -191,13 +120,13 @@ class ExperimentalWorker {
String(this._options.workerId + 1) // 0-indexed workerId, 1-indexed JEST_WORKER_ID
]);
this._retries++; // If we exceeded the amount of retries, we will emulate an error reply
this._retries++;
// If we exceeded the amount of retries, we will emulate an error reply
// coming from the child. This avoids code duplication related with cleaning
// the queue, and scheduling the next call.
if (this._retries > this._options.maxRetries) {
const error = new Error('Call retries were exceeded');
this._onMessage([
_types.PARENT_MESSAGE_CLIENT_ERROR,
error.name,
@@ -208,137 +137,223 @@ class ExperimentalWorker {
}
]);
}
}
_shutdown() {
// End the permanent stream so the merged stream end too
if (this._fakeStream) {
this._fakeStream.end();
this._fakeStream = null;
this.state = _types.WorkerStates.OK;
if (this._resolveWorkerReady) {
this._resolveWorkerReady();
}
this._resolveExitPromise();
}
_onError(error) {
if (error.message.includes('heap out of memory')) {
this.state = _types.WorkerStates.OUT_OF_MEMORY;
// Threads don't behave like processes, they don't crash when they run out of
// memory. But for consistency we want them to behave like processes so we call
// terminate to simulate a crash happening that was not planned
this._worker.terminate();
}
}
_onMessage(response) {
// Ignore messages not intended for us
if (!Array.isArray(response)) return;
let error;
switch (response[0]) {
case _types.PARENT_MESSAGE_OK:
this._onProcessEnd(null, response[1]);
break;
case _types.PARENT_MESSAGE_CLIENT_ERROR:
error = response[4];
if (error != null && typeof error === 'object') {
const extra = error; // @ts-expect-error: no index
const NativeCtor = global[response[1]];
const extra = error;
// @ts-expect-error: no index
const NativeCtor = globalThis[response[1]];
const Ctor = typeof NativeCtor === 'function' ? NativeCtor : Error;
error = new Ctor(response[2]);
error.type = response[1];
error.stack = response[3];
for (const key in extra) {
// @ts-expect-error: no index
error[key] = extra[key];
}
}
this._onProcessEnd(error, null);
break;
case _types.PARENT_MESSAGE_SETUP_ERROR:
error = new Error('Error when calling setup: ' + response[2]); // @ts-expect-error: adding custom properties to errors.
error = new Error(`Error when calling setup: ${response[2]}`);
// @ts-expect-error: adding custom properties to errors.
error.type = response[1];
error.stack = response[3];
this._onProcessEnd(error, null);
break;
case _types.PARENT_MESSAGE_CUSTOM:
this._onCustomMessage(response[1]);
break;
case _types.PARENT_MESSAGE_MEM_USAGE:
this._childIdleMemoryUsage = response[1];
if (this._resolveMemoryUsage) {
this._resolveMemoryUsage(response[1]);
this._resolveMemoryUsage = undefined;
this._memoryUsagePromise = undefined;
}
this._performRestartIfRequired();
break;
default:
throw new TypeError('Unexpected response from worker: ' + response[0]);
// Ignore messages not intended for us
break;
}
}
_onExit(exitCode) {
if (exitCode !== 0 && !this._forceExited) {
this._workerReadyPromise = undefined;
this._resolveWorkerReady = undefined;
if (exitCode !== 0 && this.state === _types.WorkerStates.OUT_OF_MEMORY) {
this._onProcessEnd(
new Error('Jest worker ran out of memory and crashed'),
null
);
this._shutdown();
} else if (
(exitCode !== 0 &&
this.state !== _types.WorkerStates.SHUTTING_DOWN &&
this.state !== _types.WorkerStates.SHUT_DOWN) ||
this.state === _types.WorkerStates.RESTARTING
) {
this.initialize();
if (this._request) {
this._worker.postMessage(this._request);
}
} else {
// If the worker thread exits while a request is still pending, throw an
// error. This is unexpected and tests may not have run to completion.
const isRequestStillPending = !!this._request;
if (isRequestStillPending) {
this._onProcessEnd(
new Error(
'A Jest worker thread exited unexpectedly before finishing tests for an unknown reason. One of the ways this can happen is if process.exit() was called in testing code.'
),
null
);
}
this._shutdown();
}
}
waitForExit() {
return this._exitPromise;
}
forceExit() {
this._forceExited = true;
this.state = _types.WorkerStates.SHUTTING_DOWN;
this._worker.terminate();
}
send(request, onProcessStart, onProcessEnd, onCustomMessage) {
onProcessStart(this);
this._onProcessEnd = (...args) => {
var _onProcessEnd;
const hasRequest = !!this._request;
// Clean the request to avoid sending past requests to workers that fail
// while waiting for a new request (timers, unhandled rejections...)
this._request = null;
const res =
(_onProcessEnd = onProcessEnd) === null || _onProcessEnd === void 0
? void 0
: _onProcessEnd(...args); // Clean up the reference so related closures can be garbage collected.
if (this._childIdleMemoryUsageLimit && hasRequest) {
this.checkMemoryUsage();
}
const res = onProcessEnd?.(...args);
// Clean up the reference so related closures can be garbage collected.
onProcessEnd = null;
return res;
};
this._onCustomMessage = (...arg) => onCustomMessage(...arg);
this._request = request;
this._retries = 0;
this._worker.postMessage(request);
}
getWorkerId() {
return this._options.workerId;
}
getStdout() {
return this._stdout;
}
getStderr() {
return this._stderr;
}
_performRestartIfRequired() {
if (this._memoryUsageCheck) {
this._memoryUsageCheck = false;
let limit = this._childIdleMemoryUsageLimit;
_getFakeStream() {
if (!this._fakeStream) {
this._fakeStream = new (_stream().PassThrough)();
// TODO: At some point it would make sense to make use of
// stringToBytes found in jest-config, however as this
// package does not have any dependencies on an other jest
// packages that can wait until some other time.
if (limit && limit > 0 && limit <= 1) {
limit = Math.floor((0, _os().totalmem)() * limit);
} else if (limit) {
limit = Math.floor(limit);
}
if (
limit &&
this._childIdleMemoryUsage &&
this._childIdleMemoryUsage > limit
) {
this.state = _types.WorkerStates.RESTARTING;
this._worker.terminate();
}
}
}
return this._fakeStream;
/**
* Gets the last reported memory usage.
*
* @returns Memory usage in bytes.
*/
getMemoryUsage() {
if (!this._memoryUsagePromise) {
let rejectCallback;
const promise = new Promise((resolve, reject) => {
this._resolveMemoryUsage = resolve;
rejectCallback = reject;
});
this._memoryUsagePromise = promise;
if (!this._worker.threadId) {
rejectCallback(new Error('Child process is not running.'));
this._memoryUsagePromise = undefined;
this._resolveMemoryUsage = undefined;
return promise;
}
try {
this._worker.postMessage([_types.CHILD_MESSAGE_MEM_USAGE]);
} catch (err) {
this._memoryUsagePromise = undefined;
this._resolveMemoryUsage = undefined;
rejectCallback(err);
}
return promise;
}
return this._memoryUsagePromise;
}
/**
* Gets updated memory usage and restarts if required
*/
checkMemoryUsage() {
if (this._childIdleMemoryUsageLimit) {
this._memoryUsageCheck = true;
this._worker.postMessage([_types.CHILD_MESSAGE_MEM_USAGE]);
} else {
console.warn(
'Memory usage of workers can only be checked if a limit is set'
);
}
}
/**
* Gets the thread id of the worker.
*
* @returns Thread id.
*/
getWorkerSystemId() {
return this._worker.threadId;
}
isWorkerRunning() {
return this._worker.threadId >= 0;
}
}
exports.default = ExperimentalWorker;

View File

@@ -1,8 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/// <reference types="node" />
export default function messageParent(message: unknown, parentProcess?: NodeJS.Process): void;

View File

@@ -4,32 +4,27 @@ Object.defineProperty(exports, '__esModule', {
value: true
});
exports.default = messageParent;
function _worker_threads() {
const data = require('worker_threads');
_worker_threads = function () {
return data;
};
return data;
}
var _types = require('../types');
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
const isWorkerThread = (() => {
try {
// `Require` here to support Node v10
const {isMainThread, parentPort} = require('worker_threads');
return !isMainThread && parentPort != null;
} catch {
return false;
}
})();
function messageParent(message, parentProcess = process) {
if (isWorkerThread) {
// `Require` here to support Node v10
const {parentPort} = require('worker_threads'); // ! is safe due to `null` check in `isWorkerThread`
parentPort.postMessage([_types.PARENT_MESSAGE_CUSTOM, message]);
if (!_worker_threads().isMainThread && _worker_threads().parentPort != null) {
_worker_threads().parentPort.postMessage([
_types.PARENT_MESSAGE_CUSTOM,
message
]);
} else if (typeof parentProcess.send === 'function') {
parentProcess.send([_types.PARENT_MESSAGE_CUSTOM, message]);
} else {

View File

@@ -1,7 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
export {};

View File

@@ -1,16 +1,24 @@
'use strict';
function _jestUtil() {
const data = require('jest-util');
_jestUtil = function () {
return data;
};
return data;
}
var _types = require('../types');
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
let file = null;
let setupArgs = [];
let initialized = false;
/**
* This file is a small bootstrapper for workers. It sets up the communication
* between the worker and the parent process, interpreting parent messages and
@@ -24,123 +32,126 @@ let initialized = false;
* If an invalid message is detected, the child will exit (by throwing) with a
* non-zero exit code.
*/
const messageListener = request => {
switch (request[0]) {
case _types.CHILD_MESSAGE_INITIALIZE:
const init = request;
file = init[2];
setupArgs = request[3];
setupArgs = init[3];
break;
case _types.CHILD_MESSAGE_CALL:
const call = request;
execMethod(call[2], call[3]);
break;
case _types.CHILD_MESSAGE_END:
end();
break;
case _types.CHILD_MESSAGE_MEM_USAGE:
reportMemoryUsage();
break;
case _types.CHILD_MESSAGE_CALL_SETUP:
if (initialized) {
reportSuccess(void 0);
} else {
const main = require(file);
initialized = true;
if (main.setup) {
execFunction(
main.setup,
main,
setupArgs,
reportSuccess,
reportInitializeError
);
} else {
reportSuccess(void 0);
}
}
break;
default:
throw new TypeError(
'Unexpected request from parent process: ' + request[0]
`Unexpected request from parent process: ${request[0]}`
);
}
};
process.on('message', messageListener);
function reportSuccess(result) {
if (!process || !process.send) {
throw new Error('Child can only be used on a forked process');
}
process.send([_types.PARENT_MESSAGE_OK, result]);
}
function reportClientError(error) {
return reportError(error, _types.PARENT_MESSAGE_CLIENT_ERROR);
}
function reportInitializeError(error) {
return reportError(error, _types.PARENT_MESSAGE_SETUP_ERROR);
}
function reportMemoryUsage() {
if (!process || !process.send) {
throw new Error('Child can only be used on a forked process');
}
const msg = [_types.PARENT_MESSAGE_MEM_USAGE, process.memoryUsage().heapUsed];
process.send(msg);
}
function reportError(error, type) {
if (!process || !process.send) {
throw new Error('Child can only be used on a forked process');
}
if (error == null) {
error = new Error('"null" or "undefined" thrown');
}
process.send([
type,
error.constructor && error.constructor.name,
error.message,
error.stack,
typeof error === 'object' ? {...error} : error
typeof error === 'object'
? {
...error
}
: error
]);
}
function end() {
const main = require(file);
if (!main.teardown) {
exitProcess();
return;
}
execFunction(main.teardown, main, [], exitProcess, exitProcess);
}
function exitProcess() {
// Clean up open handles so the process ideally exits gracefully
process.removeListener('message', messageListener);
}
function execMethod(method, args) {
const main = require(file);
let fn;
if (method === 'default') {
fn = main.__esModule ? main['default'] : main;
fn = main.__esModule ? main.default : main;
} else {
fn = main[method];
}
function execHelper() {
execFunction(fn, main, args, reportSuccess, reportClientError);
}
if (initialized || !main.setup) {
execHelper();
return;
}
initialized = true;
execFunction(main.setup, main, setupArgs, execHelper, reportInitializeError);
}
const isPromise = obj =>
!!obj &&
(typeof obj === 'object' || typeof obj === 'function') &&
typeof obj.then === 'function';
function execFunction(fn, ctx, args, onResult, onError) {
let result;
try {
result = fn.apply(ctx, args);
} catch (err) {
onError(err);
return;
}
if (isPromise(result)) {
if ((0, _jestUtil().isPromise)(result)) {
result.then(onResult, onError);
} else {
onResult(result);

View File

@@ -1,7 +0,0 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
export {};

View File

@@ -2,25 +2,30 @@
function _worker_threads() {
const data = require('worker_threads');
_worker_threads = function () {
return data;
};
return data;
}
function _jestUtil() {
const data = require('jest-util');
_jestUtil = function () {
return data;
};
return data;
}
var _types = require('../types');
/**
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
let file = null;
let setupArgs = [];
let initialized = false;
/**
* This file is a small bootstrapper for workers. It sets up the communication
* between the worker and the parent process, interpreting parent messages and
@@ -34,124 +39,137 @@ let initialized = false;
* If an invalid message is detected, the child will exit (by throwing) with a
* non-zero exit code.
*/
const messageListener = request => {
switch (request[0]) {
case _types.CHILD_MESSAGE_INITIALIZE:
const init = request;
file = init[2];
setupArgs = request[3];
process.env.JEST_WORKER_ID = request[4];
setupArgs = init[3];
process.env.JEST_WORKER_ID = init[4];
break;
case _types.CHILD_MESSAGE_CALL:
const call = request;
execMethod(call[2], call[3]);
break;
case _types.CHILD_MESSAGE_END:
end();
break;
case _types.CHILD_MESSAGE_MEM_USAGE:
reportMemoryUsage();
break;
case _types.CHILD_MESSAGE_CALL_SETUP:
if (initialized) {
reportSuccess(void 0);
} else {
const main = require(file);
initialized = true;
if (main.setup) {
execFunction(
main.setup,
main,
setupArgs,
reportSuccess,
reportInitializeError
);
} else {
reportSuccess(void 0);
}
}
break;
default:
throw new TypeError(
'Unexpected request from parent process: ' + request[0]
`Unexpected request from parent process: ${request[0]}`
);
}
};
_worker_threads().parentPort.on('message', messageListener);
function reportMemoryUsage() {
if (_worker_threads().isMainThread) {
throw new Error('Child can only be used on a forked process');
}
const msg = [_types.PARENT_MESSAGE_MEM_USAGE, process.memoryUsage().heapUsed];
_worker_threads().parentPort.postMessage(msg);
}
function reportSuccess(result) {
if (_worker_threads().isMainThread) {
throw new Error('Child can only be used on a forked process');
}
_worker_threads().parentPort.postMessage([_types.PARENT_MESSAGE_OK, result]);
try {
_worker_threads().parentPort.postMessage([
_types.PARENT_MESSAGE_OK,
result
]);
} catch (err) {
// Handling it here to avoid unhandled `DataCloneError` rejection
// which is hard to distinguish on the parent side
// (such error doesn't have any message or stack trace)
reportClientError(err);
}
}
function reportClientError(error) {
return reportError(error, _types.PARENT_MESSAGE_CLIENT_ERROR);
}
function reportInitializeError(error) {
return reportError(error, _types.PARENT_MESSAGE_SETUP_ERROR);
}
function reportError(error, type) {
if (_worker_threads().isMainThread) {
throw new Error('Child can only be used on a forked process');
}
if (error == null) {
error = new Error('"null" or "undefined" thrown');
}
_worker_threads().parentPort.postMessage([
type,
error.constructor && error.constructor.name,
error.message,
error.stack,
typeof error === 'object' ? {...error} : error
typeof error === 'object'
? {
...error
}
: error
]);
}
function end() {
const main = require(file);
if (!main.teardown) {
exitProcess();
return;
}
execFunction(main.teardown, main, [], exitProcess, exitProcess);
}
function exitProcess() {
// Clean up open handles so the worker ideally exits gracefully
_worker_threads().parentPort.removeListener('message', messageListener);
}
function execMethod(method, args) {
const main = require(file);
let fn;
if (method === 'default') {
fn = main.__esModule ? main['default'] : main;
fn = main.__esModule ? main.default : main;
} else {
fn = main[method];
}
function execHelper() {
execFunction(fn, main, args, reportSuccess, reportClientError);
}
if (initialized || !main.setup) {
execHelper();
return;
}
initialized = true;
execFunction(main.setup, main, setupArgs, execHelper, reportInitializeError);
}
const isPromise = obj =>
!!obj &&
(typeof obj === 'object' || typeof obj === 'function') &&
typeof obj.then === 'function';
function execFunction(fn, ctx, args, onResult, onError) {
let result;
try {
result = fn.apply(ctx, args);
} catch (err) {
onError(err);
return;
}
if (isPromise(result)) {
if ((0, _jestUtil().isPromise)(result)) {
result.then(onResult, onError);
} else {
onResult(result);