.github/workflows/e2e-parcel-workflow.yml

@@ -29,6 +29,13 @@ jobs:
         yarn init -p
         yarn add -D parcel@nightly lodash @babel/core
 
+        # See https://github.com/parcel-bundler/parcel/issues/9114
+
+        echo "packageExtensions:
+  \"@parcel/fs@*\":
+    dependencies:
+      \"@parcel/core\": '*'" > .yarnrc.yml
+
         # JavaScript
 
         echo "import _ from 'lodash';function printHello() { console.log(_.join(['Hello', 'JS'], ' '))}; printHello();" | tee index.js

CHANGELOG.md

@@ -10,6 +10,8 @@ Features in `master` can be tried out by running `yarn set version from sources`
 
 - Fixes `preferInteractive` forcing interactive mode in non-TTY environments.
 - `node-modules` linker now honors user-defined symlinks for `<workspace>/node_modules` directories
+- `node-modules` linker supports hoisting into inner workspaces that are parents of other workspaces
+- `node-modules` linker attemps to hoist tree more exhaustivel until nothing can be hoisted
 
 ## 4.1.0
 

packages/acceptance-tests/pkg-tests-specs/sources/features/plugins.test.ts

@@ -8,6 +8,7 @@ import {mockPluginServer}         from './plugins.utility';
 const COMMANDS_PLUGIN = (name: string, {async = false, printOnBoot = false, thirdParty = false} = {}) => `
 const factory = ${async ? `async` : ``} r => {
   const {Command} = r('clipanion');
+  const path = r('node:path');
 
   if (${printOnBoot})
     console.log('Booting ${name.toUpperCase()}');
@@ -22,6 +23,14 @@ const factory = ${async ? `async` : ``} r => {
             this.context.stdout.write('Executing ${name.toUpperCase()}\\n');
           }
         },
+
+        class MyCommandPath extends Command {
+          static paths = [['${name}', 'path']];
+
+          async execute() {
+            this.context.stdout.write(path.posix.join('a', 'b') + '\\n');
+          }
+        },
       ],
     },
   };
@@ -77,6 +86,19 @@ describe(`Features`, () => {
       });
     }));
 
+    test(`it should support plugins using builtin modules`, makeTemporaryEnv({
+    }, async ({path, run, source}) => {
+      await xfs.writeFilePromise(`${path}/plugin-a.js` as PortablePath, COMMANDS_PLUGIN(`a`));
+
+      await xfs.writeFilePromise(`${path}/.yarnrc.yml` as PortablePath, stringifySyml({
+        plugins: [`./plugin-a.js`],
+      }));
+
+      await expect(run(`a`, `path`)).resolves.toMatchObject({
+        stdout: `a/b\n`,
+      });
+    }));
+
     test(`it should accept asynchronous plugins`, makeTemporaryEnv({
     }, async ({path, run, source}) => {
       await xfs.writeFilePromise(`${path}/plugin-a.js` as PortablePath, COMMANDS_PLUGIN(`a`, {async: true}));

packages/plugin-compat/package.json

@@ -49,5 +49,6 @@
   ],
   "engines": {
     "node": ">=18.12.0"
-  }
+  },
+  "stableVersion": "4.0.8"
 }

packages/plugin-nm/package.json

@@ -25,7 +25,7 @@
     "@yarnpkg/cli": "workspace:^",
     "@yarnpkg/core": "workspace:^"
   },
-  "version": "4.0.3",
+  "version": "4.0.4",
   "nextVersion": {
     "semver": "2.0.0-rc.5",
     "nonce": "1989925236539513"

packages/yarnpkg-builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarnpkg/builder",
-  "version": "4.1.2",
+  "version": "4.2.0",
   "license": "BSD-2-Clause",
   "bin": "./sources/boot-cli-dev.js",
   "exports": {
@@ -47,5 +47,5 @@
   "engines": {
     "node": ">=18.12.0"
   },
-  "stableVersion": "4.1.2"
+  "stableVersion": "4.2.0"
 }

packages/yarnpkg-builder/sources/commands/build/plugin.ts

@@ -128,6 +128,15 @@ export default class BuildPluginCommand extends Command {
           minify: !this.noMinify,
           sourcemap: this.sourceMap ? `inline` : false,
           target: `node${semver.minVersion(pkg.engines.node)!.version}`,
+          supported: {
+            /*
+            Yarn plugin-runtime did not support builtin modules prefixed with "node:".
+            See https://github.com/yarnpkg/berry/pull/5997
+            As a solution, and for backwards compatibility, esbuild should strip these prefixes.
+            */
+            'node-colon-prefix-import': false,
+            'node-colon-prefix-require': false,
+          },
         });
 
         for (const warning of res.warnings) {

packages/yarnpkg-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarnpkg/cli",
-  "version": "4.4.1",
+  "version": "4.5.0",
   "license": "BSD-2-Clause",
   "main": "./sources/index.ts",
   "exports": {

packages/yarnpkg-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarnpkg/core",
-  "version": "4.1.2",
+  "version": "4.1.3",
   "license": "BSD-2-Clause",
   "main": "./sources/index.ts",
   "exports": {
@@ -82,6 +82,5 @@
   },
   "engines": {
     "node": ">=18.12.0"
-  },
-  "stableVersion": "4.1.2"
+  }
 }

packages/yarnpkg-core/sources/Configuration.ts

@@ -5,7 +5,7 @@ import camelcase
 import {isCI, isPR, GITHUB_ACTIONS}                                                                              from 'ci-info';
 import {UsageError}                                                                                              from 'clipanion';
 import {parse as parseDotEnv}                                                                                    from 'dotenv';
-import {builtinModules}                                                                                          from 'module';
+import {isBuiltin}                                                                                               from 'module';
 import pLimit, {Limit}                                                                                           from 'p-limit';
 import {PassThrough, Writable}                                                                                   from 'stream';
 import {WriteStream}                                                                                             from 'tty';
@@ -1265,8 +1265,7 @@ export class Configuration {
     const thirdPartyPlugins = new Map<string, Plugin>([]);
     if (pluginConfiguration !== null) {
       const requireEntries = new Map();
-      for (const request of builtinModules)
-        requireEntries.set(request, () => miscUtils.dynamicRequire(request));
+
       for (const [request, embedModule] of pluginConfiguration.modules)
         requireEntries.set(request, () => embedModule);
 
@@ -1284,6 +1283,9 @@ export class Configuration {
 
         const pluginRequireEntries = new Map(requireEntries);
         const pluginRequire = (request: string) => {
+          if (isBuiltin(request))
+            return miscUtils.dynamicRequire(request);
+
           if (pluginRequireEntries.has(request)) {
             return pluginRequireEntries.get(request)();
           } else {

packages/yarnpkg-core/sources/semverUtils.ts

@@ -204,7 +204,7 @@ export function stringifyComparator(comparator: Comparator) {
 }
 
 export function simplifyRanges(ranges: Array<string>) {
-  const parsedRanges = ranges.map(range => validRange(range)!.set.map(comparators => comparators.map(comparator => getComparator(comparator))));
+  const parsedRanges = ranges.map(removeSubsets).map(range => validRange(range)!.set.map(comparators => comparators.map(comparator => getComparator(comparator))));
 
   let alternatives = parsedRanges.shift()!.map(comparators => mergeComparators(comparators))
     .filter((range): range is Comparator => range !== null);
@@ -233,3 +233,22 @@ export function simplifyRanges(ranges: Array<string>) {
 
   return alternatives.map(comparator => stringifyComparator(comparator)).join(` || `);
 }
+
+function removeSubsets(rangeString: string) {
+  const parts = rangeString.split(`||`);
+  if (parts.length > 1) {
+    const newParts: Set<string> = new Set();
+    for (const potentialSubset of parts) {
+      if (!parts.some(part => part !== potentialSubset && semver.subset(potentialSubset, part))) {
+        newParts.add(potentialSubset);
+      }
+    }
+
+    if (newParts.size < parts.length) {
+      const newRange = [...newParts].join(` || `);
+      return newRange;
+    }
+  }
+
+  return rangeString;
+}

packages/yarnpkg-core/tests/semverUtils.test.ts

@@ -112,6 +112,9 @@ describe(`semverUtils`, () => {
       [[`<=1.5.3`, `1.5.3`], `1.5.3`],
       [[`1.5.3`, `1.5.3`], `1.5.3`],
       [[`1.5.0`, `1.5.3`], null],
+      [[`~1.0.1 || ~1.0.2`], `~1.0.1`],
+      [[`~1.0.1 || ~1.0.2`, `~1.0.1 || ~1.0.2`], `~1.0.1`],
+      [(new Array(1000)).fill(`~1.0.1 || ~1.0.2`), `~1.0.1`],
     ])(`should simplify %s into %s`, (ranges, expected) => {
       expect(semverUtils.simplifyRanges(ranges)).toEqual(expected);
     });

packages/yarnpkg-nm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarnpkg/nm",
-  "version": "4.0.2",
+  "version": "4.0.3",
   "license": "BSD-2-Clause",
   "main": "./sources/index.ts",
   "exports": {
@@ -36,6 +36,5 @@
   },
   "engines": {
     "node": ">=18.12.0"
-  },
-  "stableVersion": "4.0.2"
+  }
 }

packages/yarnpkg-nm/sources/hoist.ts

@@ -132,7 +132,8 @@ export const hoist = (tree: HoisterTree, opts: HoistOptions = {}): HoisterResult
   let anotherRoundNeeded = false;
   let round = 0;
   do {
-    anotherRoundNeeded = hoistTo(treeCopy, [treeCopy], new Set([treeCopy.locator]), new Map(), options).anotherRoundNeeded;
+    const result = hoistTo(treeCopy, [treeCopy], new Set([treeCopy.locator]), new Map(), options);
+    anotherRoundNeeded = result.anotherRoundNeeded || result.isGraphChanged;
     options.fastLookupPossible = false;
     round++;
   } while (anotherRoundNeeded);
@@ -480,24 +481,6 @@ const getNodeHoistInfo = (rootNode: HoisterWorkTree, rootNodePathLocators: Set<L
     }
   }
 
-  if (isHoistable) {
-    // Direct workspace dependencies must be hoisted to any common ancestor workspace of all the
-    // graph paths that include the dependency, because otherwise running app with
-    // `--preserve-symlinks` will become broken (without this flag the Node.js will pick dependency
-    // from the ancestor on the file system and with this flag it will pick ancestor from the graph
-    // and if these ancestors are different, the behavious of the application will be different).
-    // Another problem, which is prevented - is a creation of multiple hoisting layouts
-    // for the same workspace, because different dependencies of the same workspace might be hoisted
-    // differently, depending on the recepient workspace.
-    // It is difficult to find all common ancestors, but there is one easy to find common ancestor -
-    // the root workspace, so, for now, we either hoist direct dependencies into the root workspace, or we keep them
-    // unhoisted, thus we are safe from various pathological cases with `--preserve-symlinks`
-    isHoistable = parentNode.dependencyKind !== HoisterDependencyKind.WORKSPACE || parentNode.hoistedFrom.has(node.name) || rootNodePathLocators.size === 1;
-    if (outputReason && !isHoistable) {
-      reason = parentNode.reasons.get(node.name)!;
-    }
-  }
-
   if (isHoistable) {
     isHoistable = !rootNode.peerNames.has(node.name);
     if (outputReason && !isHoistable) {

packages/yarnpkg-nm/tests/hoist.test.ts

@@ -534,22 +534,52 @@ describe(`hoist`, () => {
     expect(getTreeHeight(hoist(toTree(tree), {check: true}))).toEqual(2);
   });
 
-  it(`should avoid hoisting direct workspace dependencies into non-root workspace`, () => {
+  it(`should hoist direct workspace dependencies into non-root workspace`, () => {
     // . -> W1(w) -> W2(w) -> W3(w)-> A@X
     //            -> A@Y
     //   -> W3
     //   -> A@Z
-    // The A@X must not be hoisted into W2(w)
-    // otherwise accessing A via . -> W3 with --preserve-symlinks will result in A@Z,
-    // but accessing it via W3(w) will result in A@Y
+    // The A@X must be hoisted into W2(w)
+    // Accessing A via . -> W3 with --preserve-symlinks will result in A@Z,
+    // but accessing it via W3(w) will result in A@Y, however if we don't do it,
+    // inner workspaces will have multiple unexpected copies of dependencies
     const tree = {
       '.': {dependencies: [`W1(w)`, `W3`, `A@Z`], dependencyKind: HoisterDependencyKind.WORKSPACE},
       'W1(w)': {dependencies: [`W2(w)`, `A@Y`], dependencyKind: HoisterDependencyKind.WORKSPACE},
       'W2(w)': {dependencies: [`W3(w)`], dependencyKind: HoisterDependencyKind.WORKSPACE},
       'W3(w)': {dependencies: [`A@X`], dependencyKind: HoisterDependencyKind.WORKSPACE},
     };
 
-    expect(getTreeHeight(hoist(toTree(tree), {check: true}))).toEqual(5);
+    expect(getTreeHeight(hoist(toTree(tree), {check: true}))).toEqual(4);
+  });
+
+  it(`should hoist dependencies to the top from workspaces that have no hoist borders given there is workspace with hoist borders`, () => {
+    // . -> W1(w)| -> A@X --> B
+    //             -> B@X
+    //   -> W2(w)  -> A@Y --> B
+    //             -> B@Y
+    // should be hoisted to:
+    // . -> W1(w)| -> A@X -->B
+    //             -> B@X
+    //   -> W2(w)
+    //   -> A@Y --> B
+    //   -> B@Y
+
+    const tree = {
+      '.': {dependencies: [`W1(w)`, `W2(w)`], dependencyKind: HoisterDependencyKind.WORKSPACE},
+      'W1(w)': {dependencies: [`A@X`, `B@X`], dependencyKind: HoisterDependencyKind.WORKSPACE},
+      'A@X': {dependencies: [`B@X`], peerNames: [`B`]},
+      'A@Y': {dependencies: [`B@Y`], peerNames: [`B`]},
+      'W2(w)': {dependencies: [`A@Y`, `B@Y`], dependencyKind: HoisterDependencyKind.WORKSPACE},
+    };
+
+    const hoistingLimits = new Map([
+      [`.@`, new Set([`W1(w)`])],
+    ]);
+
+    const hoistedTree = hoist(toTree(tree), {check: true, hoistingLimits});
+    const W2 = Array.from(Array.from(hoistedTree.dependencies).filter(x => x.name === `W2(w)`)[0].dependencies);
+    expect(W2).toEqual([]);
   });
 
   it(`should hoist aliased packages`, () => {

packages/yarnpkg-pnpify/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarnpkg/pnpify",
-  "version": "4.1.1",
+  "version": "4.1.2",
   "license": "BSD-2-Clause",
   "main": "./sources/index.ts",
   "bin": "./sources/boot-cli-dev.js",
@@ -49,6 +49,5 @@
   },
   "engines": {
     "node": ">=18.12.0"
-  },
-  "stableVersion": "4.1.1"
+  }
 }

packages/yarnpkg-shell/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarnpkg/shell",
-  "version": "4.0.2",
+  "version": "4.1.0",
   "license": "BSD-2-Clause",
   "main": "./sources/index.ts",
   "bin": "./sources/boot-cli-dev.js",
@@ -50,6 +50,5 @@
   },
   "engines": {
     "node": ">=18.12.0"
-  },
-  "stableVersion": "4.0.2"
+  }
 }

packages/yarnpkg-shell/sources/index.ts

@@ -166,6 +166,15 @@ const BUILTINS = new Map<string, ShellBuiltin>([
     return await setTimeout(1000 * seconds, 0);
   }],
 
+  [`unset`, async (args: Array<string>, opts: ShellOptions, state: ShellState) => {
+    for (const name of args) {
+      delete state.environment[name];
+      delete state.variables[name];
+    }
+
+    return 0;
+  }],
+
   [`__ysh_run_procedure`, async (args: Array<string>, opts: ShellOptions, state: ShellState) => {
     const procedure = state.procedures[args[0]];
 

packages/yarnpkg-shell/tests/shell.test.ts

@@ -2094,5 +2094,25 @@ describe(`Shell`, () => {
         });
       });
     });
+
+    describe(`unset`, () => {
+      it(`should unset one variable`, async () => {
+        await expectResult(bufferResult(
+          `FOO=bar; unset FOO; echo $FOO`,
+        ), {
+          exitCode: 1,
+          stderr: `Unbound variable "FOO"\n`,
+        });
+      });
+
+      it(`should unset multiple variables`, async () => {
+        await expectResult(bufferResult(
+          `A=1 B=2; unset A B; echo $A; echo $B`,
+        ), {
+          exitCode: 1,
+          stderr: `Unbound variable "A"\nUnbound variable "B"\n`,
+        });
+      });
+    });
   });
 });