Skip to content

Commit 8d6d851

Browse files
authoredJan 25, 2023
feat: added --install-strategy=linked (#6078)
Co-authored-by: Vincent Bailly <vibailly@microsoft.com> Implements the RFC https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md Packages are installed in node_modules/.store flat, and linked into the node_modules tree in depth, rather than hoisted.
1 parent 6af152b commit 8d6d851

File tree

24 files changed

+2783
-88
lines changed

24 files changed

+2783
-88
lines changed
 

‎DEPENDENCIES.md

+1
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,7 @@ graph LR;
618618
npmcli-arborist-->semver;
619619
npmcli-arborist-->ssri;
620620
npmcli-arborist-->tap;
621+
npmcli-arborist-->tar-stream;
621622
npmcli-arborist-->tcompare;
622623
npmcli-arborist-->treeverse;
623624
npmcli-arborist-->walk-up-path;

‎lib/utils/config/definitions.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1090,14 +1090,14 @@ define('install-links', {
10901090

10911091
define('install-strategy', {
10921092
default: 'hoisted',
1093-
type: ['hoisted', 'nested', 'shallow'],
1093+
type: ['hoisted', 'nested', 'shallow', 'linked'],
10941094
description: `
10951095
Sets the strategy for installing packages in node_modules.
10961096
hoisted (default): Install non-duplicated in top-level, and duplicated as
10971097
necessary within directory structure.
10981098
nested: (formerly --legacy-bundling) install in place, no hoisting.
10991099
shallow (formerly --global-style) only install direct deps at top-level.
1100-
linked: (coming soon) install in node_modules/.store, link in place,
1100+
linked: (experimental) install in node_modules/.store, link in place,
11011101
unhoisted.
11021102
`,
11031103
flatten,

‎node_modules/.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,13 @@
4343
!/are-we-there-yet
4444
!/are-we-there-yet/node_modules/
4545
/are-we-there-yet/node_modules/*
46-
!/are-we-there-yet/node_modules/buffer
4746
!/are-we-there-yet/node_modules/readable-stream
4847
!/balanced-match
4948
!/base64-js
5049
!/bin-links
5150
!/binary-extensions
5251
!/brace-expansion
52+
!/buffer
5353
!/builtins
5454
!/cacache
5555
!/chalk

‎package-lock.json

+90-23
Original file line numberDiff line numberDiff line change
@@ -2776,29 +2776,6 @@
27762776
"node": "^14.17.0 || ^16.13.0 || >=18.0.0"
27772777
}
27782778
},
2779-
"node_modules/are-we-there-yet/node_modules/buffer": {
2780-
"version": "6.0.3",
2781-
"funding": [
2782-
{
2783-
"type": "github",
2784-
"url": "https://github.com/sponsors/feross"
2785-
},
2786-
{
2787-
"type": "patreon",
2788-
"url": "https://www.patreon.com/feross"
2789-
},
2790-
{
2791-
"type": "consulting",
2792-
"url": "https://feross.org/support"
2793-
}
2794-
],
2795-
"inBundle": true,
2796-
"license": "MIT",
2797-
"dependencies": {
2798-
"base64-js": "^1.3.1",
2799-
"ieee754": "^1.2.1"
2800-
}
2801-
},
28022779
"node_modules/are-we-there-yet/node_modules/readable-stream": {
28032780
"version": "4.2.0",
28042781
"inBundle": true,
@@ -2921,6 +2898,12 @@
29212898
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
29222899
"dev": true
29232900
},
2901+
"node_modules/b4a": {
2902+
"version": "1.6.1",
2903+
"resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.1.tgz",
2904+
"integrity": "sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==",
2905+
"dev": true
2906+
},
29242907
"node_modules/bail": {
29252908
"version": "2.0.2",
29262909
"resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz",
@@ -3000,6 +2983,32 @@
30002983
"node": ">=10"
30012984
}
30022985
},
2986+
"node_modules/bl": {
2987+
"version": "6.0.0",
2988+
"resolved": "https://registry.npmjs.org/bl/-/bl-6.0.0.tgz",
2989+
"integrity": "sha512-Ik9BVIMdcWzSOCpzDv2XpQ4rJ4oZBuk3ck6MgiOv0EopdgtohN2uSCrrLlkH1Jf0KnpZZMBA3D0bUMbCdj/jgA==",
2990+
"dev": true,
2991+
"dependencies": {
2992+
"buffer": "^6.0.3",
2993+
"inherits": "^2.0.4",
2994+
"readable-stream": "^4.2.0"
2995+
}
2996+
},
2997+
"node_modules/bl/node_modules/readable-stream": {
2998+
"version": "4.3.0",
2999+
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.3.0.tgz",
3000+
"integrity": "sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==",
3001+
"dev": true,
3002+
"dependencies": {
3003+
"abort-controller": "^3.0.0",
3004+
"buffer": "^6.0.3",
3005+
"events": "^3.3.0",
3006+
"process": "^0.11.10"
3007+
},
3008+
"engines": {
3009+
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
3010+
}
3011+
},
30033012
"node_modules/boolbase": {
30043013
"version": "1.0.0",
30053014
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@@ -3054,6 +3063,30 @@
30543063
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
30553064
}
30563065
},
3066+
"node_modules/buffer": {
3067+
"version": "6.0.3",
3068+
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
3069+
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
3070+
"funding": [
3071+
{
3072+
"type": "github",
3073+
"url": "https://github.com/sponsors/feross"
3074+
},
3075+
{
3076+
"type": "patreon",
3077+
"url": "https://www.patreon.com/feross"
3078+
},
3079+
{
3080+
"type": "consulting",
3081+
"url": "https://feross.org/support"
3082+
}
3083+
],
3084+
"inBundle": true,
3085+
"dependencies": {
3086+
"base64-js": "^1.3.1",
3087+
"ieee754": "^1.2.1"
3088+
}
3089+
},
30573090
"node_modules/buffer-from": {
30583091
"version": "1.1.2",
30593092
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@@ -4964,6 +4997,12 @@
49644997
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
49654998
"dev": true
49664999
},
5000+
"node_modules/fast-fifo": {
5001+
"version": "1.1.0",
5002+
"resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.1.0.tgz",
5003+
"integrity": "sha512-Kl29QoNbNvn4nhDsLYjyIAaIqaJB6rBx5p3sL9VjaefJ+eMFBWVZiaoguaoZfzEKr5RhAti0UgM8703akGPJ6g==",
5004+
"dev": true
5005+
},
49675006
"node_modules/fast-json-stable-stringify": {
49685007
"version": "2.1.0",
49695008
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -9845,6 +9884,12 @@
98459884
],
98469885
"peer": true
98479886
},
9887+
"node_modules/queue-tick": {
9888+
"version": "1.0.1",
9889+
"resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz",
9890+
"integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==",
9891+
"dev": true
9892+
},
98489893
"node_modules/quick-lru": {
98499894
"version": "4.0.1",
98509895
"resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz",
@@ -10930,6 +10975,16 @@
1093010975
"escodegen": "^1.8.1"
1093110976
}
1093210977
},
10978+
"node_modules/streamx": {
10979+
"version": "2.13.2",
10980+
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.13.2.tgz",
10981+
"integrity": "sha512-+TWqixPhGDXEG9L/XczSbhfkmwAtGs3BJX5QNU6cvno+pOLKeszByWcnaTu6dg8efsTYqR8ZZuXWHhZfgrxMvA==",
10982+
"dev": true,
10983+
"dependencies": {
10984+
"fast-fifo": "^1.1.0",
10985+
"queue-tick": "^1.0.1"
10986+
}
10987+
},
1093310988
"node_modules/string_decoder": {
1093410989
"version": "1.3.0",
1093510990
"inBundle": true,
@@ -13193,6 +13248,17 @@
1319313248
"node": ">=10"
1319413249
}
1319513250
},
13251+
"node_modules/tar-stream": {
13252+
"version": "3.0.0",
13253+
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.0.0.tgz",
13254+
"integrity": "sha512-O6OfUKBbQOqAhh6owTWmA730J/yZCYcpmZ1DBj2YX51ZQrt7d7NgzrR+CnO9wP6nt/viWZW2XeXLavX3/ZEbEg==",
13255+
"dev": true,
13256+
"dependencies": {
13257+
"b4a": "^1.6.1",
13258+
"bl": "^6.0.0",
13259+
"streamx": "^2.12.5"
13260+
}
13261+
},
1319613262
"node_modules/tcompare": {
1319713263
"version": "5.0.7",
1319813264
"resolved": "https://registry.npmjs.org/tcompare/-/tcompare-5.0.7.tgz",
@@ -14351,6 +14417,7 @@
1435114417
"minify-registry-metadata": "^3.0.0",
1435214418
"nock": "^13.2.0",
1435314419
"tap": "^16.3.2",
14420+
"tar-stream": "^3.0.0",
1435414421
"tcompare": "^5.0.6"
1435514422
},
1435614423
"engines": {

‎smoke-tests/tap-snapshots/test/index.js.test.cjs

+3-3
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ npm ERR! npm ci
5555
npm ERR!
5656
npm ERR! Options:
5757
npm ERR! [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
58-
npm ERR! [-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
59-
npm ERR! [--legacy-bundling] [--global-style]
60-
npm ERR! [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
58+
npm ERR! [-E|--save-exact] [-g|--global]
59+
npm ERR! [--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
60+
npm ERR! [--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
6161
npm ERR! [--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
6262
npm ERR! [--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
6363
npm ERR! [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]

‎tap-snapshots/test/lib/docs.js.test.cjs

+22-20
Original file line numberDiff line numberDiff line change
@@ -1163,13 +1163,13 @@ workspaces.
11631163
#### \`install-strategy\`
11641164
11651165
* Default: "hoisted"
1166-
* Type: "hoisted", "nested", or "shallow"
1166+
* Type: "hoisted", "nested", "shallow", or "linked"
11671167
11681168
Sets the strategy for installing packages in node_modules. hoisted
11691169
(default): Install non-duplicated in top-level, and duplicated as necessary
11701170
within directory structure. nested: (formerly --legacy-bundling) install in
11711171
place, no hoisting. shallow (formerly --global-style) only install direct
1172-
deps at top-level. linked: (coming soon) install in node_modules/.store,
1172+
deps at top-level. linked: (experimental) install in node_modules/.store,
11731173
link in place, unhoisted.
11741174
11751175
#### \`json\`
@@ -2621,9 +2621,9 @@ npm ci
26212621
26222622
Options:
26232623
[-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
2624-
[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
2625-
[--legacy-bundling] [--global-style]
2626-
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
2624+
[-E|--save-exact] [-g|--global]
2625+
[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
2626+
[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
26272627
[--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
26282628
[--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
26292629
[-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -2723,7 +2723,7 @@ Usage:
27232723
npm dedupe
27242724
27252725
Options:
2726-
[--install-strategy <hoisted|nested|shallow>] [--legacy-bundling]
2726+
[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
27272727
[--global-style] [--strict-peer-deps] [--no-package-lock]
27282728
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]] [--ignore-scripts]
27292729
[--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
@@ -2994,7 +2994,7 @@ Usage:
29942994
npm find-dupes
29952995
29962996
Options:
2997-
[--install-strategy <hoisted|nested|shallow>] [--legacy-bundling]
2997+
[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
29982998
[--global-style] [--strict-peer-deps] [--no-package-lock]
29992999
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]] [--ignore-scripts]
30003000
[--no-audit] [--no-bin-links] [--no-fund]
@@ -3175,9 +3175,9 @@ npm install [<package-spec> ...]
31753175
31763176
Options:
31773177
[-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
3178-
[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
3179-
[--legacy-bundling] [--global-style]
3180-
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
3178+
[-E|--save-exact] [-g|--global]
3179+
[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
3180+
[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
31813181
[--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
31823182
[--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
31833183
[-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -3222,9 +3222,9 @@ npm install-ci-test
32223222
32233223
Options:
32243224
[-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
3225-
[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
3226-
[--legacy-bundling] [--global-style]
3227-
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
3225+
[-E|--save-exact] [-g|--global]
3226+
[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
3227+
[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
32283228
[--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
32293229
[--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
32303230
[-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -3269,9 +3269,9 @@ npm install-test [<package-spec> ...]
32693269
32703270
Options:
32713271
[-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
3272-
[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
3273-
[--legacy-bundling] [--global-style]
3274-
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
3272+
[-E|--save-exact] [-g|--global]
3273+
[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
3274+
[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
32753275
[--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
32763276
[--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
32773277
[-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -3316,8 +3316,9 @@ npm link [<package-spec>]
33163316
33173317
Options:
33183318
[-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
3319-
[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
3320-
[--legacy-bundling] [--global-style] [--strict-peer-deps] [--no-package-lock]
3319+
[-E|--save-exact] [-g|--global]
3320+
[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
3321+
[--global-style] [--strict-peer-deps] [--no-package-lock]
33213322
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]] [--ignore-scripts]
33223323
[--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
33233324
[-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -4221,8 +4222,9 @@ npm update [<pkg>...]
42214222
42224223
Options:
42234224
[-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
4224-
[-g|--global] [--install-strategy <hoisted|nested|shallow>] [--legacy-bundling]
4225-
[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
4225+
[-g|--global] [--install-strategy <hoisted|nested|shallow|linked>]
4226+
[--legacy-bundling] [--global-style]
4227+
[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
42264228
[--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
42274229
[--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
42284230
[-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]

‎workspaces/arborist/lib/arborist/build-ideal-tree.js

+1
Original file line numberDiff line numberDiff line change
@@ -1232,6 +1232,7 @@ This is a one-time fix-up, please be patient...
12321232
const isWorkspace = this.idealTree.workspaces && this.idealTree.workspaces.has(spec.name)
12331233

12341234
// spec is a directory, link it unless installLinks is set or it's a workspace
1235+
// TODO post arborist refactor, will need to check for installStrategy=linked
12351236
if (spec.type === 'directory' && (isWorkspace || !installLinks)) {
12361237
return this[_linkFromSpec](name, spec, parent, edge)
12371238
}

‎workspaces/arborist/lib/arborist/index.js

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const mixins = [
4242
require('./load-virtual.js'),
4343
require('./rebuild.js'),
4444
require('./reify.js'),
45+
require('./isolated-reifier.js'),
4546
]
4647

4748
const _workspacesEnabled = Symbol.for('workspacesEnabled')

‎workspaces/arborist/lib/arborist/isolated-reifier.js

+453
Large diffs are not rendered by default.

‎workspaces/arborist/lib/arborist/rebuild.js

+10-1
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ module.exports = cls => class Builder extends cls {
8989
const {
9090
depNodes,
9191
linkNodes,
92+
storeNodes,
9293
} = this[_retrieveNodesByType](nodes)
9394

9495
// build regular deps
@@ -99,6 +100,10 @@ module.exports = cls => class Builder extends cls {
99100
this[_resetQueues]()
100101
await this[_build](linkNodes, { type: 'links' })
101102
}
103+
if (storeNodes.size) {
104+
this[_resetQueues]()
105+
await this[_build](storeNodes, { type: 'storelinks' })
106+
}
102107

103108
process.emit('timeEnd', 'build')
104109
}
@@ -130,9 +135,12 @@ module.exports = cls => class Builder extends cls {
130135
[_retrieveNodesByType] (nodes) {
131136
const depNodes = new Set()
132137
const linkNodes = new Set()
138+
const storeNodes = new Set()
133139

134140
for (const node of nodes) {
135-
if (node.isLink) {
141+
if (node.isStoreLink) {
142+
storeNodes.add(node)
143+
} else if (node.isLink) {
136144
linkNodes.add(node)
137145
} else {
138146
depNodes.add(node)
@@ -154,6 +162,7 @@ module.exports = cls => class Builder extends cls {
154162
return {
155163
depNodes,
156164
linkNodes,
165+
storeNodes,
157166
}
158167
}
159168

‎workspaces/arborist/lib/arborist/reify.js

+61-37
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
// mixin implementing the reify method
2-
32
const onExit = require('../signal-handling.js')
43
const pacote = require('pacote')
54
const AuditReport = require('../audit-report.js')
@@ -10,8 +9,9 @@ const debug = require('../debug.js')
109
const walkUp = require('walk-up-path')
1110
const log = require('proc-log')
1211
const hgi = require('hosted-git-info')
12+
const rpj = require('read-package-json-fast')
1313

14-
const { dirname, resolve, relative } = require('path')
14+
const { dirname, resolve, relative, join } = require('path')
1515
const { depth: dfwalk } = require('treeverse')
1616
const {
1717
lstat,
@@ -106,6 +106,8 @@ const _resolvedAdd = Symbol.for('resolvedAdd')
106106
const _usePackageLock = Symbol.for('usePackageLock')
107107
const _formatPackageLock = Symbol.for('formatPackageLock')
108108

109+
const _createIsolatedTree = Symbol.for('createIsolatedTree')
110+
109111
module.exports = cls => class Reifier extends cls {
110112
constructor (options) {
111113
super(options)
@@ -138,6 +140,8 @@ module.exports = cls => class Reifier extends cls {
138140

139141
// public method
140142
async reify (options = {}) {
143+
const linked = (options.installStrategy || this.options.installStrategy) === 'linked'
144+
141145
if (this[_packageLockOnly] && this[_global]) {
142146
const er = new Error('cannot generate lockfile for global packages')
143147
er.code = 'ESHRINKWRAPGLOBAL'
@@ -154,8 +158,22 @@ module.exports = cls => class Reifier extends cls {
154158
process.emit('time', 'reify')
155159
await this[_validatePath]()
156160
await this[_loadTrees](options)
161+
162+
const oldTree = this.idealTree
163+
if (linked) {
164+
// swap out the tree with the isolated tree
165+
// this is currently technical debt which will be resolved in a refactor
166+
// of Node/Link trees
167+
log.warn('reify', 'The "linked" install strategy is EXPERIMENTAL and may contain bugs.')
168+
this.idealTree = await this[_createIsolatedTree](this.idealTree)
169+
}
157170
await this[_diffTrees]()
158171
await this[_reifyPackages]()
172+
if (linked) {
173+
// swap back in the idealTree
174+
// so that the lockfile is preserved
175+
this.idealTree = oldTree
176+
}
159177
await this[_saveIdealTree](options)
160178
await this[_copyIdealToActual]()
161179
// This is a very bad pattern and I can't wait to stop doing it
@@ -634,44 +652,40 @@ module.exports = cls => class Reifier extends cls {
634652
}
635653

636654
async [_extractOrLink] (node) {
637-
// in normal cases, node.resolved should *always* be set by now.
638-
// however, it is possible when a lockfile is damaged, or very old,
639-
// or in some other race condition bugs in npm v6, that a previously
640-
// bundled dependency will have just a version, but no resolved value,
641-
// and no 'bundled: true' setting.
642-
// Do the best with what we have, or else remove it from the tree
643-
// entirely, since we can't possibly reify it.
644-
let res = null
645-
if (node.resolved) {
646-
const registryResolved = this[_registryResolved](node.resolved)
647-
if (registryResolved) {
648-
res = `${node.name}@${registryResolved}`
649-
}
650-
} else if (node.packageName && node.version) {
651-
res = `${node.packageName}@${node.version}`
652-
}
653-
654-
// no idea what this thing is. remove it from the tree.
655-
if (!res) {
656-
const warning = 'invalid or damaged lockfile detected\n' +
657-
'please re-try this operation once it completes\n' +
658-
'so that the damage can be corrected, or perform\n' +
659-
'a fresh install with no lockfile if the problem persists.'
660-
log.warn('reify', warning)
661-
log.verbose('reify', 'unrecognized node in tree', node.path)
662-
node.parent = null
663-
node.fsParent = null
664-
this[_addNodeToTrashList](node)
665-
return
666-
}
667-
668655
const nm = resolve(node.parent.path, 'node_modules')
669656
await this[_validateNodeModules](nm)
670657

671-
if (node.isLink) {
672-
await rm(node.path, { recursive: true, force: true })
673-
await this[_symlink](node)
674-
} else {
658+
if (!node.isLink) {
659+
// in normal cases, node.resolved should *always* be set by now.
660+
// however, it is possible when a lockfile is damaged, or very old,
661+
// or in some other race condition bugs in npm v6, that a previously
662+
// bundled dependency will have just a version, but no resolved value,
663+
// and no 'bundled: true' setting.
664+
// Do the best with what we have, or else remove it from the tree
665+
// entirely, since we can't possibly reify it.
666+
let res = null
667+
if (node.resolved) {
668+
const registryResolved = this[_registryResolved](node.resolved)
669+
if (registryResolved) {
670+
res = `${node.name}@${registryResolved}`
671+
}
672+
} else if (node.package.name && node.version) {
673+
res = `${node.package.name}@${node.version}`
674+
}
675+
676+
// no idea what this thing is. remove it from the tree.
677+
if (!res) {
678+
const warning = 'invalid or damaged lockfile detected\n' +
679+
'please re-try this operation once it completes\n' +
680+
'so that the damage can be corrected, or perform\n' +
681+
'a fresh install with no lockfile if the problem persists.'
682+
log.warn('reify', warning)
683+
log.verbose('reify', 'unrecognized node in tree', node.path)
684+
node.parent = null
685+
node.fsParent = null
686+
this[_addNodeToTrashList](node)
687+
return
688+
}
675689
await debug(async () => {
676690
const st = await lstat(node.path).catch(e => null)
677691
if (st && !st.isDirectory()) {
@@ -688,7 +702,17 @@ module.exports = cls => class Reifier extends cls {
688702
resolved: node.resolved,
689703
integrity: node.integrity,
690704
})
705+
// store nodes don't use Node class so node.package doesn't get updated
706+
if (node.isInStore) {
707+
const pkg = await rpj(join(node.path, 'package.json'))
708+
node.package.scripts = pkg.scripts
709+
}
710+
return
691711
}
712+
713+
// node.isLink
714+
await rm(node.path, { recursive: true, force: true })
715+
await this[_symlink](node)
692716
}
693717

694718
async [_symlink] (node) {

‎workspaces/arborist/lib/link.js

+3-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ const _delistFromMeta = Symbol.for('_delistFromMeta')
88
const _refreshLocation = Symbol.for('_refreshLocation')
99
class Link extends Node {
1010
constructor (options) {
11-
const { root, realpath, target, parent, fsParent } = options
11+
const { root, realpath, target, parent, fsParent, isStoreLink } = options
1212

1313
if (!realpath && !(target && target.path)) {
1414
throw new TypeError('must provide realpath for Link node')
@@ -23,6 +23,8 @@ class Link extends Node {
2323
: null),
2424
})
2525

26+
this.isStoreLink = isStoreLink || false
27+
2628
if (target) {
2729
this.target = target
2830
} else if (this.realpath === this.root.path) {

‎workspaces/arborist/lib/node.js

+2
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ class Node {
9191
installLinks = false,
9292
legacyPeerDeps = false,
9393
linksIn,
94+
isInStore = false,
9495
hasShrinkwrap,
9596
overrides,
9697
loadOverrides = false,
@@ -113,6 +114,7 @@ class Node {
113114
this[_workspaces] = null
114115

115116
this.errors = error ? [error] : []
117+
this.isInStore = isInStore
116118

117119
// this will usually be null, except when modeling a
118120
// package's dependencies in a virtual root.

‎workspaces/arborist/package.json

+2
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,12 @@
4545
"minify-registry-metadata": "^3.0.0",
4646
"nock": "^13.2.0",
4747
"tap": "^16.3.2",
48+
"tar-stream": "^3.0.0",
4849
"tcompare": "^5.0.6"
4950
},
5051
"scripts": {
5152
"test": "tap",
53+
"test-only": "tap --only",
5254
"posttest": "node ../.. run lint",
5355
"snap": "tap",
5456
"postsnap": "npm run lintfix",

‎workspaces/arborist/tap-snapshots/test/link.js.test.cjs

+6
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ Link {
1919
"installLinks": false,
2020
"integrity": null,
2121
"inventory": Inventory {},
22+
"isInStore": false,
23+
"isStoreLink": false,
2224
"legacyPeerDeps": false,
2325
"linksIn": Set {},
2426
"location": "../../../../../some/other/path",
@@ -62,6 +64,7 @@ exports[`test/link.js TAP > instantiate without providing target 1`] = `
6264
"installLinks": false,
6365
"integrity": null,
6466
"inventory": Inventory {},
67+
"isInStore": false,
6568
"legacyPeerDeps": false,
6669
"linksIn": Set {
6770
<*ref_1>,
@@ -78,6 +81,8 @@ exports[`test/link.js TAP > instantiate without providing target 1`] = `
7881
"tops": Set {},
7982
},
8083
},
84+
"isInStore": false,
85+
"isStoreLink": false,
8186
"legacyPeerDeps": false,
8287
"linksIn": Set {},
8388
"location": "../../../../../some/other/path",
@@ -103,6 +108,7 @@ exports[`test/link.js TAP > instantiate without providing target 1`] = `
103108
"installLinks": false,
104109
"integrity": null,
105110
"inventory": Inventory {},
111+
"isInStore": false,
106112
"legacyPeerDeps": false,
107113
"linksIn": Set {
108114
<*ref_1>,

‎workspaces/arborist/tap-snapshots/test/node.js.test.cjs

+290
Large diffs are not rendered by default.

‎workspaces/arborist/test/arborist/reify.js

+58
Original file line numberDiff line numberDiff line change
@@ -3151,3 +3151,61 @@ t.only('should preserve exact ranges, missing actual tree', async (t) => {
31513151
await arb.reify()
31523152
})
31533153
})
3154+
3155+
t.test('install stategy linked', async (t) => {
3156+
const Arborist = require('../../lib/index.js')
3157+
const abbrev = resolve(__dirname,
3158+
'../fixtures/registry-mocks/content/abbrev/-/abbrev-1.1.1.tgz')
3159+
const abbrevTGZ = fs.readFileSync(abbrev)
3160+
3161+
const abbrevPackument = JSON.stringify({
3162+
_id: 'abbrev',
3163+
_rev: 'lkjadflkjasdf',
3164+
name: 'abbrev',
3165+
'dist-tags': { latest: '1.1.1' },
3166+
versions: {
3167+
'1.1.1': {
3168+
name: 'abbrev',
3169+
version: '1.1.1',
3170+
dist: {
3171+
tarball: 'https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz',
3172+
},
3173+
},
3174+
},
3175+
})
3176+
3177+
t.test('should install package linked', async (t) => {
3178+
const testdir = t.testdir({
3179+
project: {
3180+
'package.json': JSON.stringify({
3181+
name: 'myproject',
3182+
version: '1.0.0',
3183+
dependencies: {
3184+
abbrev: '1.1.1',
3185+
},
3186+
}),
3187+
},
3188+
})
3189+
3190+
tnock(t, 'https://registry.npmjs.org')
3191+
.get('/abbrev')
3192+
.reply(200, abbrevPackument)
3193+
3194+
tnock(t, 'https://registry.npmjs.org')
3195+
.get('/abbrev/-/abbrev-1.1.1.tgz')
3196+
.reply(200, abbrevTGZ)
3197+
3198+
const path = resolve(testdir, 'project')
3199+
const arb = new Arborist({
3200+
path,
3201+
registry: 'https://registry.npmjs.org',
3202+
cache: resolve(testdir, 'cache'),
3203+
installStrategy: 'linked',
3204+
})
3205+
await arb.reify({ installStrategy: 'linked' })
3206+
const abbrev = fs.lstatSync(resolve(path, 'node_modules', 'abbrev'))
3207+
const store = fs.lstatSync(resolve(path, 'node_modules', '.store'))
3208+
t.ok(store.isDirectory(), 'abbrev got installed')
3209+
t.ok(abbrev.isSymbolicLink(), 'abbrev got installed')
3210+
})
3211+
})
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
const nock = require('nock')
2+
const tar = require('tar-stream')
3+
const Stream = require('stream')
4+
const fs = require('fs')
5+
const path = require('path')
6+
const os = require('os')
7+
8+
/* Utility to extract a buffer out of a stream */
9+
class StreamToBuffer extends Stream.Writable {
10+
constructor (opts) {
11+
super(opts)
12+
this._bufferArray = []
13+
}
14+
15+
_write (chunk, encoding, callback) {
16+
this._bufferArray.push(chunk)
17+
callback()
18+
}
19+
20+
_final (callback) {
21+
this.buffer = Buffer.concat(this._bufferArray)
22+
callback()
23+
}
24+
}
25+
26+
/**
27+
* packPackageToStream
28+
* Uses 'tar-stream' to create the tar stream without touching the file system.
29+
*/
30+
function packPackageToStream (manifest, reg) {
31+
const {
32+
name,
33+
version,
34+
} = manifest
35+
const { shrinkwrap, bundledDeps, ...rest } = manifest
36+
37+
const pack = tar.pack()
38+
const manifestString = JSON.stringify({
39+
...rest,
40+
})
41+
const index = `console.log('Hello from ${name}@${version}!')`
42+
const unscopedName = name.replace(/^@[^/]*\//, '')
43+
pack.entry({ name: unscopedName, type: 'directory' })
44+
pack.entry({ name: `${unscopedName}/package.json` }, manifestString)
45+
pack.entry({ name: `${unscopedName}/index.js` }, index)
46+
pack.entry({ name: `${unscopedName}/bin.js` }, '#!/usr/bin/env node\nconsole.log("bin")')
47+
if (shrinkwrap) {
48+
pack.entry({ name: `${unscopedName}/npm-shrinkwrap.json` }, shrinkwrap.replace(/##REG##/g, reg))
49+
}
50+
if (bundledDeps) {
51+
pack.entry({ name: `${unscopedName}/node_modules`, type: 'directory' })
52+
bundledDeps.forEach(d => {
53+
pack.entry({ name: `${unscopedName}/node_modules/${d.name}`, type: 'directory' })
54+
pack.entry({ name: `${unscopedName}/node_modules/${d.name}/package.json` }, JSON.stringify(d))
55+
pack.entry({ name: `${unscopedName}/node_modules/${d.name}/index.js` }, `console.log('Hello from ${d.name}@${d.version}!')`
56+
)
57+
})
58+
}
59+
60+
pack.finalize()
61+
return pack
62+
}
63+
64+
/**
65+
* Pack a package for publish.
66+
* Returns a buffer containing a tarball of the package.
67+
*/
68+
async function packPackage (manifest, reg) {
69+
const packStream = packPackageToStream(manifest, reg)
70+
71+
const tarBuffer = packStream.pipe(new StreamToBuffer())
72+
73+
return new Promise((resolve, reject) => {
74+
tarBuffer.on('close', () => {
75+
resolve(tarBuffer.buffer)
76+
})
77+
tarBuffer.on('error', reject)
78+
})
79+
}
80+
81+
/**
82+
* Publish the given package to the given registry.
83+
*/
84+
async function publishPackage (registry, manifest, packuments) {
85+
const {
86+
name,
87+
version,
88+
} = manifest
89+
const { shrinkwrap, bundledDeps, ...rest } = manifest
90+
91+
if (packuments.has(name)) {
92+
packuments.get(name).versions[version] = {
93+
bin: './bin.js',
94+
...rest,
95+
dist: {
96+
tarball: `${registry}/${name.replace(/\//g, '-')}/${version}.tar`,
97+
},
98+
}
99+
} else {
100+
packuments.set(name, {
101+
name,
102+
'dist-tags': {
103+
latest: version,
104+
},
105+
versions: {
106+
[version]: {
107+
...rest,
108+
bin: './bin.js',
109+
dist: {
110+
tarball: `${registry}/${name.replace(/\//g, '-')}/${version}.tar`,
111+
},
112+
},
113+
},
114+
})
115+
}
116+
117+
const tarball = await packPackage(manifest, registry)
118+
119+
nock(registry)
120+
.persist()
121+
.get(`/${name.replace(/\//g, '-')}/${version}.tar`)
122+
.reply(200, tarball)
123+
}
124+
125+
/**
126+
* Given an object decribing a dependency graph, this funcion will materialize
127+
* this dependency graph on disk and in a registry.
128+
*
129+
* The input has the following shape:
130+
* {
131+
* registry: [ package1, package2,... ],
132+
* root: { name: <name>, version: <version>, dependencies: <dependencies> }
133+
* }
134+
* package# represent a registry package and is of the following shape:
135+
* { name: <name>, version: <version>, dependencies?: <dependencies> }
136+
*
137+
* The return value is of the form { dir, registry } where:
138+
* - dir is the location on disk of the root package
139+
* - registry is the url of the registry which has the necessary packages
140+
*
141+
* Only simple graphs are currenly supported, the following features will come in further iterations:
142+
* - lockfile
143+
* - shinkwrap
144+
* - bundled dependencies
145+
*
146+
* The API of this function is not stable and is likely to evolve as we add more features.
147+
*/
148+
async function getRepo (graph) {
149+
// Generate a new random registry every time to prevent interference between tests
150+
const registry = `https://${Math.random().toString(36).substring(2)}.test`
151+
152+
const packuments = new Map()
153+
// Publish all the registery packages
154+
await Promise.all(graph.registry.map(o =>
155+
publishPackage(registry, o, packuments)))
156+
157+
packuments.forEach((packument, name) => {
158+
nock(registry)
159+
.persist()
160+
.get(`/${name.replace(/\//g, '%2f')}`)
161+
.reply(200, packument)
162+
})
163+
164+
// Generate the root of the graph on disk
165+
const root = graph.root
166+
const workspaces = graph.workspaces || []
167+
const repo = {
168+
'package.json': JSON.stringify({
169+
workspaces: workspaces.length !== 0 ? ['packages/*'] : undefined,
170+
...root,
171+
}),
172+
packages: {},
173+
}
174+
workspaces.forEach(wp => {
175+
repo.packages[wp.name] = {
176+
'package.json': JSON.stringify(wp),
177+
'bin.js': '#!/usr/bin/env node\nconsole.log("bin")',
178+
'index.js': `console.log('Hello from workspace ${wp.name}')`,
179+
}
180+
})
181+
const dir = testdir(repo)
182+
return { dir, registry }
183+
}
184+
185+
function testdir (structure) {
186+
const dir = fs.mkdtempSync(`${fs.realpathSync(os.tmpdir())}/test-`)
187+
createDir(dir, structure)
188+
return dir
189+
}
190+
191+
function createDir (dir, structure) {
192+
Object.entries(structure).forEach(([key, value]) => {
193+
if (typeof value === 'object') {
194+
const newDir = path.join(dir, key)
195+
fs.mkdirSync(newDir)
196+
createDir(newDir, value)
197+
} else {
198+
fs.writeFileSync(path.join(dir, key), value)
199+
}
200+
})
201+
}
202+
203+
exports.getRepo = getRepo

‎workspaces/arborist/test/isolated-mode.js

+1,574
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)
Please sign in to comment.