From 3ab36b88db5297cc0ebd9ec0a55524c6bedfa61e Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 17 Feb 2023 12:04:07 +0100 Subject: [PATCH 1/6] feat(worker): add `setup` method to worker farms --- CHANGELOG.md | 2 ++ packages/jest-worker/README.md | 8 ++++++- packages/jest-worker/__benchmarks__/test.js | 5 ++-- .../jest-worker/src/base/BaseWorkerPool.ts | 23 +++++++++++++++++++ packages/jest-worker/src/index.ts | 4 ++++ packages/jest-worker/src/types.ts | 7 +++++- .../jest-worker/src/workers/processChild.ts | 23 +++++++++++++++++++ .../jest-worker/src/workers/threadChild.ts | 23 +++++++++++++++++++ 8 files changed, 90 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d2475303f20..411ae9b3ff23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Features +- `[jest-worker]` Add `setup` method to worker farms + ### Fixes ### Chore & Maintenance diff --git a/packages/jest-worker/README.md b/packages/jest-worker/README.md index 1277ed824bf2..e774312cf6a2 100644 --- a/packages/jest-worker/README.md +++ b/packages/jest-worker/README.md @@ -140,11 +140,17 @@ Returns a `ReadableStream` where the standard output of all workers is piped. No Returns a `ReadableStream` where the standard error of all workers is piped. Note that the `silent` option of the child workers must be set to `true` to make it work. This is the default set by `jest-worker`, but keep it in mind when overriding options through `forkOptions`. +#### `setup()` + +Starts up every worker and call their `setup` function, if it exists. Returns a `Promise` which resolves when all workers are running and have completed their `setup`. + +This is useful if you want to start up all your workers eagerly before they are used to call any other functions. + #### `end()` Finishes the workers by killing all workers. No further calls can be done to the `Worker` instance. -Returns a Promise that resolves with `{ forceExited: boolean }` once all workers are dead. If `forceExited` is `true`, at least one of the workers did not exit gracefully, which likely happened because it executed a leaky task that left handles open. This should be avoided, force exiting workers is a last resort to prevent creating lots of orphans. +Returns a `Promise` that resolves with `{ forceExited: boolean }` once all workers are dead. If `forceExited` is `true`, at least one of the workers did not exit gracefully, which likely happened because it executed a leaky task that left handles open. This should be avoided, force exiting workers is a last resort to prevent creating lots of orphans. **Note:** diff --git a/packages/jest-worker/__benchmarks__/test.js b/packages/jest-worker/__benchmarks__/test.js index b2aae08a35f0..c95dc01d6d03 100644 --- a/packages/jest-worker/__benchmarks__/test.js +++ b/packages/jest-worker/__benchmarks__/test.js @@ -87,11 +87,10 @@ function testJestWorker() { async function countToFinish() { if (++count === calls) { - farm.end(); const endTime = performance.now(); // Let all workers go down. - await sleep(2000); + await farm.end(); resolve({ globalTime: endTime - startTime - 2000, @@ -110,7 +109,7 @@ function testJestWorker() { farm.getStderr().pipe(process.stderr); // Let all workers come up. - await sleep(2000); + await farm.setup(); const startProcess = performance.now(); diff --git a/packages/jest-worker/src/base/BaseWorkerPool.ts b/packages/jest-worker/src/base/BaseWorkerPool.ts index 56394f4457f4..0a8823ac6376 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.ts +++ b/packages/jest-worker/src/base/BaseWorkerPool.ts @@ -7,6 +7,7 @@ import mergeStream = require('merge-stream'); import { + CHILD_MESSAGE_CALL_SETUP, CHILD_MESSAGE_END, PoolExitResult, WorkerInterface, @@ -87,6 +88,28 @@ export default class BaseWorkerPool { throw Error('Missing method createWorker in WorkerPool'); } + async setup(): Promise { + await Promise.all( + this._workers.map( + worker => + new Promise((resolve, reject) => { + worker.send( + [CHILD_MESSAGE_CALL_SETUP], + emptyMethod, + error => { + if (error) { + reject(error); + } else { + resolve(); + } + }, + emptyMethod, + ); + }), + ), + ); + } + async end(): Promise { // We do not cache the request object here. If so, it would only be only // processed by one of the workers, and we want them all to close. diff --git a/packages/jest-worker/src/index.ts b/packages/jest-worker/src/index.ts index ebc7e910834c..890325bda971 100644 --- a/packages/jest-worker/src/index.ts +++ b/packages/jest-worker/src/index.ts @@ -174,6 +174,10 @@ export class Worker { return this._workerPool.getStdout(); } + async setup(): Promise { + await this._workerPool.setup(); + } + async end(): Promise { if (this._ending) { throw new Error('Farm is ended, no more calls can be done to it'); diff --git a/packages/jest-worker/src/types.ts b/packages/jest-worker/src/types.ts index 4f6e1c1dac23..d90b5fb461eb 100644 --- a/packages/jest-worker/src/types.ts +++ b/packages/jest-worker/src/types.ts @@ -36,6 +36,7 @@ export const CHILD_MESSAGE_INITIALIZE = 0; export const CHILD_MESSAGE_CALL = 1; export const CHILD_MESSAGE_END = 2; export const CHILD_MESSAGE_MEM_USAGE = 3; +export const CHILD_MESSAGE_CALL_SETUP = 4; export const PARENT_MESSAGE_OK = 0; export const PARENT_MESSAGE_CLIENT_ERROR = 1; @@ -61,6 +62,7 @@ export interface WorkerPoolInterface { getWorkers(): Array; createWorker(options: WorkerOptions): WorkerInterface; send: WorkerCallback; + setup(): Promise; end(): Promise; } @@ -223,11 +225,14 @@ export type ChildMessageEnd = [ export type ChildMessageMemUsage = [type: typeof CHILD_MESSAGE_MEM_USAGE]; +export type ChildMessageCallSetup = [type: typeof CHILD_MESSAGE_CALL_SETUP]; + export type ChildMessage = | ChildMessageInitialize | ChildMessageCall | ChildMessageEnd - | ChildMessageMemUsage; + | ChildMessageMemUsage + | ChildMessageCallSetup; // Messages passed from the children to the parent. diff --git a/packages/jest-worker/src/workers/processChild.ts b/packages/jest-worker/src/workers/processChild.ts index 9eb1016ce715..647f5baa2a04 100644 --- a/packages/jest-worker/src/workers/processChild.ts +++ b/packages/jest-worker/src/workers/processChild.ts @@ -8,6 +8,7 @@ import {isPromise} from 'jest-util'; import { CHILD_MESSAGE_CALL, + CHILD_MESSAGE_CALL_SETUP, CHILD_MESSAGE_END, CHILD_MESSAGE_INITIALIZE, CHILD_MESSAGE_MEM_USAGE, @@ -61,6 +62,28 @@ const messageListener: NodeJS.MessageListener = (request: any) => { reportMemoryUsage(); break; + case CHILD_MESSAGE_CALL_SETUP: + if (initialized) { + reportSuccess(void 0); + } else { + const main = require(file!); + + initialized = true; + + if (main.setup) { + execFunction( + main.setup, + main, + setupArgs, + reportSuccess, + reportInitializeError, + ); + } else { + reportSuccess(void 0); + } + } + break; + default: throw new TypeError( `Unexpected request from parent process: ${request[0]}`, diff --git a/packages/jest-worker/src/workers/threadChild.ts b/packages/jest-worker/src/workers/threadChild.ts index d2703c9480a8..2380e1afd23c 100644 --- a/packages/jest-worker/src/workers/threadChild.ts +++ b/packages/jest-worker/src/workers/threadChild.ts @@ -9,6 +9,7 @@ import {isMainThread, parentPort} from 'worker_threads'; import {isPromise} from 'jest-util'; import { CHILD_MESSAGE_CALL, + CHILD_MESSAGE_CALL_SETUP, CHILD_MESSAGE_END, CHILD_MESSAGE_INITIALIZE, CHILD_MESSAGE_MEM_USAGE, @@ -63,6 +64,28 @@ const messageListener = (request: any) => { reportMemoryUsage(); break; + case CHILD_MESSAGE_CALL_SETUP: + if (initialized) { + reportSuccess(void 0); + } else { + const main = require(file!); + + initialized = true; + + if (main.setup) { + execFunction( + main.setup, + main, + setupArgs, + reportSuccess, + reportInitializeError, + ); + } else { + reportSuccess(void 0); + } + } + break; + default: throw new TypeError( `Unexpected request from parent process: ${request[0]}`, From 622c0555ff4b8cdc5cbaa80b9a44601a89c5d140 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Mon, 20 Feb 2023 08:51:12 +0100 Subject: [PATCH 2/6] link in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 411ae9b3ff23..3f945735d384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Features -- `[jest-worker]` Add `setup` method to worker farms +- `[jest-worker]` Add `setup` method to worker farms ([#13937](https://github.com/facebook/jest/pull/13937)) ### Fixes From fbd89c18212f84e51c6dfd10f0a380643e926595 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Mon, 20 Feb 2023 09:08:29 +0100 Subject: [PATCH 3/6] rename to `start` `setup` is currently reserved --- packages/jest-worker/README.md | 2 +- packages/jest-worker/__benchmarks__/test.js | 2 +- packages/jest-worker/__typetests__/jest-worker.test.ts | 2 ++ packages/jest-worker/src/base/BaseWorkerPool.ts | 2 +- packages/jest-worker/src/index.ts | 4 ++-- packages/jest-worker/src/types.ts | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/jest-worker/README.md b/packages/jest-worker/README.md index e774312cf6a2..c44588c5a0f5 100644 --- a/packages/jest-worker/README.md +++ b/packages/jest-worker/README.md @@ -140,7 +140,7 @@ Returns a `ReadableStream` where the standard output of all workers is piped. No Returns a `ReadableStream` where the standard error of all workers is piped. Note that the `silent` option of the child workers must be set to `true` to make it work. This is the default set by `jest-worker`, but keep it in mind when overriding options through `forkOptions`. -#### `setup()` +#### `start()` Starts up every worker and call their `setup` function, if it exists. Returns a `Promise` which resolves when all workers are running and have completed their `setup`. diff --git a/packages/jest-worker/__benchmarks__/test.js b/packages/jest-worker/__benchmarks__/test.js index c95dc01d6d03..86ee6dcb7efb 100644 --- a/packages/jest-worker/__benchmarks__/test.js +++ b/packages/jest-worker/__benchmarks__/test.js @@ -109,7 +109,7 @@ function testJestWorker() { farm.getStderr().pipe(process.stderr); // Let all workers come up. - await farm.setup(); + await farm.start(); const startProcess = performance.now(); diff --git a/packages/jest-worker/__typetests__/jest-worker.test.ts b/packages/jest-worker/__typetests__/jest-worker.test.ts index 14d634ea8068..e87be50c8edf 100644 --- a/packages/jest-worker/__typetests__/jest-worker.test.ts +++ b/packages/jest-worker/__typetests__/jest-worker.test.ts @@ -88,6 +88,8 @@ expectError(typedWorkerFarm.runTestAsync()); expectError(typedWorkerFarm.setup()); expectError(typedWorkerFarm.teardown()); +expectType>(typedWorkerFarm.start()); + expectError>(typedWorkerFarm.end()); expectType>(typedWorkerFarm.end()); diff --git a/packages/jest-worker/src/base/BaseWorkerPool.ts b/packages/jest-worker/src/base/BaseWorkerPool.ts index 0a8823ac6376..fddf3e1d9a40 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.ts +++ b/packages/jest-worker/src/base/BaseWorkerPool.ts @@ -88,7 +88,7 @@ export default class BaseWorkerPool { throw Error('Missing method createWorker in WorkerPool'); } - async setup(): Promise { + async start(): Promise { await Promise.all( this._workers.map( worker => diff --git a/packages/jest-worker/src/index.ts b/packages/jest-worker/src/index.ts index 890325bda971..c5f5b46bf524 100644 --- a/packages/jest-worker/src/index.ts +++ b/packages/jest-worker/src/index.ts @@ -174,8 +174,8 @@ export class Worker { return this._workerPool.getStdout(); } - async setup(): Promise { - await this._workerPool.setup(); + async start(): Promise { + await this._workerPool.start(); } async end(): Promise { diff --git a/packages/jest-worker/src/types.ts b/packages/jest-worker/src/types.ts index d90b5fb461eb..dbd1f2b430a1 100644 --- a/packages/jest-worker/src/types.ts +++ b/packages/jest-worker/src/types.ts @@ -62,7 +62,7 @@ export interface WorkerPoolInterface { getWorkers(): Array; createWorker(options: WorkerOptions): WorkerInterface; send: WorkerCallback; - setup(): Promise; + start(): Promise; end(): Promise; } From 58a035b6737b27e49521539b1a6b526288254a4e Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Mon, 20 Feb 2023 09:15:41 +0100 Subject: [PATCH 4/6] Apply suggestions from code review Co-authored-by: Tom Mrazauskas --- CHANGELOG.md | 2 +- packages/jest-worker/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f945735d384..e0777206f22d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Features -- `[jest-worker]` Add `setup` method to worker farms ([#13937](https://github.com/facebook/jest/pull/13937)) +- `[jest-worker]` Add `start` method to worker farms ([#13937](https://github.com/facebook/jest/pull/13937)) ### Fixes diff --git a/packages/jest-worker/README.md b/packages/jest-worker/README.md index c44588c5a0f5..f796d27889b5 100644 --- a/packages/jest-worker/README.md +++ b/packages/jest-worker/README.md @@ -142,7 +142,7 @@ Returns a `ReadableStream` where the standard error of all workers is piped. Not #### `start()` -Starts up every worker and call their `setup` function, if it exists. Returns a `Promise` which resolves when all workers are running and have completed their `setup`. +Starts up every worker and calls their `setup` function, if it exists. Returns a `Promise` which resolves when all workers are running and have completed their `setup`. This is useful if you want to start up all your workers eagerly before they are used to call any other functions. From d54b65776be10dfcda07f788fd159b9634a22d96 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Mon, 20 Feb 2023 09:19:22 +0100 Subject: [PATCH 5/6] also await `waitForWorkerReady` --- .../jest-worker/src/base/BaseWorkerPool.ts | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/jest-worker/src/base/BaseWorkerPool.ts b/packages/jest-worker/src/base/BaseWorkerPool.ts index fddf3e1d9a40..92896aab5b2c 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.ts +++ b/packages/jest-worker/src/base/BaseWorkerPool.ts @@ -90,23 +90,24 @@ export default class BaseWorkerPool { async start(): Promise { await Promise.all( - this._workers.map( - worker => - new Promise((resolve, reject) => { - worker.send( - [CHILD_MESSAGE_CALL_SETUP], - emptyMethod, - error => { - if (error) { - reject(error); - } else { - resolve(); - } - }, - emptyMethod, - ); - }), - ), + this._workers.map(async worker => { + await worker.waitForWorkerReady(); + + await new Promise((resolve, reject) => { + worker.send( + [CHILD_MESSAGE_CALL_SETUP], + emptyMethod, + error => { + if (error) { + reject(error); + } else { + resolve(); + } + }, + emptyMethod, + ); + }); + }), ); } From 13c9ec9ea002bd7ac750990aee4d855406578926 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Mon, 20 Feb 2023 09:36:01 +0100 Subject: [PATCH 6/6] type test --- packages/jest-worker/src/__tests__/index.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/jest-worker/src/__tests__/index.test.ts b/packages/jest-worker/src/__tests__/index.test.ts index b1d1fc755fbc..35acffc3c197 100644 --- a/packages/jest-worker/src/__tests__/index.test.ts +++ b/packages/jest-worker/src/__tests__/index.test.ts @@ -89,6 +89,7 @@ it('exposes the right API using passed worker', () => { getStdout: jest.fn(), getWorkers: jest.fn(), send: jest.fn(), + start: jest.fn(), })); const farm = new WorkerFarm('/tmp/baz.js', {