/proc/2/root/var/lang/lib/node_modules/npm/node_modules/@npmcli/arborist/lib/arborist
This explorer reads the filesystem of the server it runs on, so /workspace/user isn't present here. Browsing and the terminal still work against this server's own disk from /.
// mixin implementing the reify methodconst PackageJson = require('@npmcli/package-json')const hgi = require('hosted-git-info')const npa = require('npm-package-arg')const packageContents = require('@npmcli/installed-package-contents')const pacote = require('pacote')const { pickRegistry } = require('npm-registry-fetch')const promiseAllRejectLate = require('promise-all-reject-late')const runScript = require('@npmcli/run-script')const { callLimit: promiseCallLimit } = require('promise-call-limit')const { depth: dfwalk } = require('treeverse')const { dirname, resolve, relative, join, sep } = require('node:path')const { log, time } = require('proc-log')const { existsSync, realpathSync } = require('node:fs')const { lstat, mkdir, readdir, readlink, rm, symlink } = require('node:fs/promises')const { moveFile } = require('@npmcli/fs')const { subset, intersects } = require('semver')const { walkUp } = require('walk-up-path') const AuditReport = require('../audit-report.js')const Diff = require('../diff.js')const calcDepFlags = require('../calc-dep-flags.js')const debug = require('../debug.js')const onExit = require('../signal-handling.js')const optionalSet = require('../optional-set.js')const relpath = require('../relpath.js')const retirePath = require('../retire-path.js')const treeCheck = require('../tree-check.js')const Shrinkwrap = require('../shrinkwrap.js')const { defaultLockfileVersion } = Shrinkwrapconst { saveTypeMap, hasSubKey } = require('../add-rm-pkg-deps.js')const { IsolatedNode, IsolatedLink } = require('../isolated-classes.js') // Part of steps (steps need refactoring before we can do anything about these)const _retireShallowNodes = Symbol.for('retireShallowNodes')const _loadBundlesAndUpdateTrees = Symbol.for('loadBundlesAndUpdateTrees')const _submitQuickAudit = Symbol('submitQuickAudit')const _unpackNewModules = Symbol.for('unpackNewModules')const _build = Symbol.for('build') // shared by rebuild mixinconst _trashList = Symbol.for('trashList')const _handleOptionalFailure = Symbol.for('handleOptionalFailure')const _loadTrees = Symbol.for('loadTrees')// defined by rebuild mixinconst _checkBins = Symbol.for('checkBins') // shared symbols for swapping out when testing// TODO tests should not be this deep into internalsconst _diffTrees = Symbol.for('diffTrees')const _createSparseTree = Symbol.for('createSparseTree')const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees')const _reifyNode = Symbol.for('reifyNode')const _updateAll = Symbol.for('updateAll')const _updateNames = Symbol.for('updateNames')const _moveContents = Symbol.for('moveContents')const _moveBackRetiredUnchanged = Symbol.for('moveBackRetiredUnchanged')const _removeTrash = Symbol.for('removeTrash')const _renamePath = Symbol.for('renamePath')const _rollbackRetireShallowNodes = Symbol.for('rollbackRetireShallowNodes')const _rollbackCreateSparseTree = Symbol.for('rollbackCreateSparseTree')const _rollbackMoveBackRetiredUnchanged = Symbol.for('rollbackMoveBackRetiredUnchanged')const _saveIdealTree = Symbol.for('saveIdealTree') // defined by build-ideal-tree mixinconst _resolvedAdd = Symbol.for('resolvedAdd')// used by build-ideal-tree mixinconst _addNodeToTrashList = Symbol.for('addNodeToTrashList') module.exports = cls => class Reifier extends cls { #bundleMissing = new Set() // child nodes we'd EXPECT to be included in a bundle, but aren't #bundleUnpacked = new Set() // the nodes we unpack to read their bundles #nmValidated = new Set() #omit #retiredPaths = {} #retiredUnchanged = {} #shrinkwrapInflated = new Set() #sparseTreeDirs = new Set() #sparseTreeRoots = new Set() #linkedActualForDiff = null // Under the linked strategy the audit runs against this non-isolated ideal tree. // The isolated tree's inventory has no queryable indexes and its edges route through symlinks, so auditing it reports no vulnerabilities. #linkedIdealForAudit = null constructor (options) { super(options) this[_trashList] = new Set() } // public method async reify (options = {}) { // Global installs are normalized to the shallow strategy in the constructor; honor that here so a per-call installStrategy:'linked' can't re-engage the unsupported linked path. const linked = !this.options.global && (options.installStrategy || this.options.installStrategy) === 'linked' if (this.options.packageLockOnly && this.options.global) { const er = new Error('cannot generate lockfile for global packages') er.code = 'ESHRINKWRAPGLOBAL' throw er } this.#omit = new Set(options.omit) // start tracker block this.addTracker('reify') const timeEnd = time.start('reify') // don't create missing dirs on dry runs if (!this.options.packageLockOnly && !this.options.dryRun) { // we do NOT want to set ownership on this folder, especially // recursively, because it can have other side effects to do that // in a project directory. We just want to make it if it's missing. await mkdir(resolve(this.path), { recursive: true }) // do not allow the top-level node_modules to be a symlink await this.#validateNodeModules(resolve(this.path, 'node_modules')) } await this[_loadTrees](options) const oldTree = this.idealTree // Kept to serialize the hidden lockfile from the on-disk .store/symlink layout. let isolatedTree = null if (linked) { // swap out the tree with the isolated tree // this is currently technical debt which will be resolved in a refactor // of Node/Link trees this.idealTree = await this.createIsolatedTree() isolatedTree = this.idealTree if (this.actualTree) { this.#linkedActualForDiff = this.#buildLinkedActualForDiff( this.idealTree, this.actualTree ) } // Keep the non-isolated tree so the quick audit can run against it. this.#linkedIdealForAudit = oldTree } try { await this[_diffTrees]() await this.#reifyPackages() if (linked) { // The sweep mutates node_modules on disk, so skip it for dry runs and lockfile-only installs (those modes also short-circuit #reifyPackages). // The sweep itself scopes to in-filter workspaces when a filter is active, so it's safe to run for filtered installs too. if (!this.options.dryRun && !this.options.packageLockOnly) { await this.#cleanOrphanedStoreEntries() } } else if (!this.options.dryRun && !this.options.packageLockOnly) { // The .store directory is exclusively a linked-strategy artifact, and load-actual ignores dot-directories, so the diff never sees it. // Remove it here when switching away from linked so it does not linger under hoisted/nested. // Only do this for a full-project install: a workspace-filtered or --workspaces=false install may leave out-of-scope workspaces with links still pointing into the store. const filtered = this.options.workspaces.length > 0 || !this.options.workspacesEnabled if (!filtered) { await this.#removeStaleStoreDir() } } } finally { // Restore the non-isolated tree so the lockfile is preserved and a reused Arborist never sees the isolated tree, even if reify throws. if (linked) { this.idealTree = oldTree } // The quick audit has captured its tree synchronously by now, so drop the stashed references even on throw. this.#linkedIdealForAudit = null this.#linkedActualForDiff = null } await this[_saveIdealTree](options) // clean inert for (const node of this.idealTree.inventory.values()) { if (node.inert) { node.parent = null } } // clean up any trash that is still in the tree for (const path of this[_trashList]) { const loc = relpath(this.idealTree.realpath, path) const node = this.idealTree.inventory.get(loc) if (node && node.root === this.idealTree) { node.parent = null } } // if we filtered to only certain nodes, then anything ELSE needs // to be untouched in the resulting actual tree, even if it differs // in the idealTree. Copy over anything that was in the actual and // was not changed, delete anything in the ideal and not actual. // Then we move the entire idealTree over to this.actualTree, and // save the hidden lockfile. if (this.diff && this.diff.filterSet.size && !linked) { const reroot = new Set() const { filterSet } = this.diff const seen = new Set() for (const [loc, ideal] of this.idealTree.inventory.entries()) { seen.add(loc) // if it's an ideal node from the filter set, then skip it // because we already made whatever changes were necessary if (filterSet.has(ideal)) { continue } // otherwise, if it's not in the actualTree, then it's not a thing // that we actually added. And if it IS in the actualTree, then // it's something that we left untouched, so we need to record // that. const actual = this.actualTree.inventory.get(loc) if (!actual) { ideal.root = null } else { if ([...actual.linksIn].some(link => filterSet.has(link))) { seen.add(actual.location) continue } const { realpath, isLink } = actual if (isLink && ideal.isLink && ideal.realpath === realpath) { continue } else { reroot.add(actual) } } } // now find any actual nodes that may not be present in the ideal // tree, but were left behind by virtue of not being in the filter for (const [loc, actual] of this.actualTree.inventory.entries()) { if (seen.has(loc)) { continue } seen.add(loc) // we know that this is something that ISN'T in the idealTree, // or else we will have addressed it in the previous loop. // If it's in the filterSet, that means we intentionally removed // it, so nothing to do here. if (filterSet.has(actual)) { continue } reroot.add(actual) } // go through the rerooted actual nodes, and move them over. for (const actual of reroot) { actual.root = this.idealTree } // prune out any tops that lack a linkIn, they are no longer relevant. for (const top of this.idealTree.tops) { if (top.linksIn.size === 0) { top.root = null } } // need to calculate dep flags, since nodes may have been marked // as extraneous or otherwise incorrect during transit. calcDepFlags(this.idealTree) } // save the ideal's meta as a hidden lockfile after we actualize it. // Under linked the logical tree is the hoisted layout, so the hidden lockfile is serialized from the isolated tree instead. if (!linked) { this.idealTree.meta.filename = this.idealTree.realpath + '/node_modules/.package-lock.json' this.idealTree.meta.hiddenLockfile = true this.idealTree.meta.lockfileVersion = defaultLockfileVersion } this.actualTree = this.idealTree this.idealTree = null if (!this.options.global && !this.options.dryRun) { if (linked) { await this.#saveLinkedHiddenLockfile(isolatedTree) } else { await this.actualTree.meta.save() } const ignoreScripts = !!this.options.ignoreScripts // if we aren't doing a dry run or ignoring scripts and we actually made changes to the dep // tree, then run the dependencies scripts if (!this.options.dryRun && !ignoreScripts && this.diff && this.diff.children.length) { const { path, package: pkg } = this.actualTree.target const stdio = this.options.foregroundScripts ? 'inherit' : 'pipe' const { scripts = {} } = pkg for (const event of ['predependencies', 'dependencies', 'postdependencies']) { if (Object.prototype.hasOwnProperty.call(scripts, event)) { log.info('run', pkg._id, event, scripts[event]) await time.start(`reify:run:${event}`, () => runScript({ event, path, pkg, stdio, scriptShell: this.options.scriptShell, })) } } } } // This is a very bad pattern and I can't wait to stop doing it this.auditReport = await this.auditReport this.finishTracker('reify') timeEnd() return treeCheck(this.actualTree) } async #reifyPackages () { // we don't submit the audit report or write to disk on dry runs if (this.options.dryRun) { return } if (this.options.packageLockOnly) { // we already have the complete tree, so just audit it now, // and that's all we have to do here. return this[_submitQuickAudit]() } // ok, we're about to start touching the fs. need to roll back // if we get an early termination. let reifyTerminated = null const removeHandler = onExit(({ signal }) => { // only call once. if signal hits twice, we just terminate removeHandler() reifyTerminated = Object.assign(new Error('process terminated'), { signal, }) return false }) // [rollbackfn, [...actions]] // after each step, if the process was terminated, execute the rollback // note that each rollback *also* calls the previous one when it's // finished, and then the first one throws the error, so we only need // a new rollback step when we have a new thing that must be done to // revert the install. const steps = [ [_rollbackRetireShallowNodes, [ _retireShallowNodes, ]], [_rollbackCreateSparseTree, [ _createSparseTree, _loadShrinkwrapsAndUpdateTrees, _loadBundlesAndUpdateTrees, _submitQuickAudit, _unpackNewModules, ]], [_rollbackMoveBackRetiredUnchanged, [ _moveBackRetiredUnchanged, _build, ]], ] for (const [rollback, actions] of steps) { for (const action of actions) { try { await this[action]() if (reifyTerminated) { throw reifyTerminated } } catch (er) { // TODO rollbacks shouldn't be relied on to throw err await this[rollback](er) /* istanbul ignore next - rollback throws, should never hit this */ throw er } } } // no rollback for this one, just exit with the error, since the // install completed and can't be safely recovered at this point. await this[_removeTrash]() if (reifyTerminated) { throw reifyTerminated } // done modifying the file system, no need to keep listening for sigs removeHandler() } // when doing a local install, we load everything and figure it all out. // when doing a global install, we *only* care about the explicit requests. [_loadTrees] (options) { const timeEnd = time.start('reify:loadTrees') const bitOpt = { ...options, complete: this.options.packageLockOnly || this.options.dryRun, } // if we're only writing a package lock, then it doesn't matter what's here if (this.options.packageLockOnly) { return this.buildIdealTree(bitOpt).then(timeEnd) } const actualOpt = this.options.global ? { ignoreMissing: true, global: true, filter: (node, kid) => { // if it's not the project root, and we have no explicit requests, // then we're already into a nested dep, so we keep it if (this.explicitRequests.size === 0 || !node.isProjectRoot) { return true } // if we added it as an edgeOut, then we want it if (this.idealTree.edgesOut.has(kid)) { return true } // if it's an explicit request, then we want it const hasExplicit = [...this.explicitRequests] .some(edge => edge.name === kid) if (hasExplicit) { return true } // ignore the rest of the global install folder return false }, } : { ignoreMissing: true } if (!this.options.global) { return Promise.all([ this.loadActual(actualOpt), this.buildIdealTree(bitOpt), ]).then(timeEnd) } // the global install space tends to have a lot of stuff in it. don't // load all of it, just what we care about. we won't be saving a // hidden lockfile in there anyway. Note that we have to load ideal // BEFORE loading actual, so that the actualOpt can use the // explicitRequests which is set during buildIdealTree return this.buildIdealTree(bitOpt) .then(() => this.loadActual(actualOpt)) .then(timeEnd) } [_diffTrees] () { if (this.options.packageLockOnly) { return } const timeEnd = time.start('reify:diffTrees') // XXX if we have an existing diff already, there should be a way // to just invalidate the parts that changed, but avoid walking the // whole tree again. const includeWorkspaces = this.options.workspacesEnabled const includeRootDeps = !includeWorkspaces || this.options.includeWorkspaceRoot && this.options.workspaces.length > 0 const filterNodes = [] if (this.options.global && this.explicitRequests.size) { const idealTree = this.idealTree.target const actualTree = this.actualTree.target // we ONLY are allowed to make changes in the global top-level // children where there's an explicit request. for (const { name } of this.explicitRequests) { const ideal = idealTree.children.get(name) if (ideal) { filterNodes.push(ideal) } const actual = actualTree.children.get(name) if (actual) { filterNodes.push(actual) } } } else { if (includeWorkspaces) { // add all ws nodes to filterNodes for (const ws of this.options.workspaces) { const ideal = this.idealTree.children.get(ws) if (ideal) { filterNodes.push(ideal) } // Skip actual-side filterNodes when using the linked diff wrapper. // Those nodes have root===actualTree, not root===linkedActualForDiff, and Diff.calculate requires filterNode.root to match actual. // The ideal filterNode alone is sufficient to scope the workspace diff. if (!this.#linkedActualForDiff) { const actual = this.actualTree.children.get(ws) if (actual) { filterNodes.push(actual) } } } } if (includeRootDeps) { // add all non-workspace nodes to filterNodes // Skip the actual tree under the linked diff wrapper: its edge targets have root===actualTree, not the wrapper, which trips Diff.calculate's filterNode guard. // The ideal-side targets alone scope the diff. const trees = this.#linkedActualForDiff ? [this.idealTree] : [this.idealTree, this.actualTree] for (const tree of trees) { for (const { type, to } of tree.edgesOut.values()) { if (type !== 'workspace' && to) { filterNodes.push(to) } } } } } // find all the nodes that need to change between the actual // and ideal trees. this.diff = Diff.calculate({ omit: this.#omit, shrinkwrapInflated: this.#shrinkwrapInflated, filterNodes, actual: this.#linkedActualForDiff || this.actualTree, ideal: this.idealTree, }) // we don't have to add 'removed' folders to the trashlist, because // they'll be moved aside to a retirement folder, and then the retired // folder will be deleted at the end. This is important when we have // a folder like FOO being "removed" in favor of a folder like "foo", // because if we remove node_modules/FOO on case-insensitive systems, // it will remove the dep that we *want* at node_modules/foo. timeEnd() } // add the node and all its bins to the list of things to be // removed later on in the process. optionally, also mark them // as a retired paths, so that we move them out of the way and // replace them when rolling back on failure. [_addNodeToTrashList] (node, retire = false) { const paths = [node.path, ...node.binPaths] const moves = this.#retiredPaths log.silly('reify', 'mark', retire ? 'retired' : 'deleted', paths) for (const path of paths) { if (retire) { const retired = retirePath(path) moves[path] = retired this[_trashList].add(retired) } else { this[_trashList].add(path) } } } // move aside the shallowest nodes in the tree that have to be // changed or removed, so that we can rollback if necessary. [_retireShallowNodes] () { const timeEnd = time.start('reify:retireShallow') const moves = this.#retiredPaths = {} for (const diff of this.diff.children) { if (diff.action === 'CHANGE' || diff.action === 'REMOVE') { // we'll have to clean these up at the end, so add them to the list this[_addNodeToTrashList](diff.actual, true) } } log.silly('reify', 'moves', moves) const movePromises = Object.entries(moves) .map(([from, to]) => this[_renamePath](from, to)) return promiseAllRejectLate(movePromises).then(timeEnd) } [_renamePath] (from, to, didMkdirp = false) { return moveFile(from, to) .catch(er => { // Occasionally an expected bin file might not exist in the package, // or a shim/symlink might have been moved aside. If we've already // handled the most common cause of ENOENT (dir doesn't exist yet), // then just ignore any ENOENT. if (er.code === 'ENOENT') { return didMkdirp ? null : mkdir(dirname(to), { recursive: true }).then(() => this[_renamePath](from, to, true)) } else if (er.code === 'EEXIST' || er.code === 'ENOTEMPTY') { return rm(to, { recursive: true, force: true }).then(() => moveFile(from, to)) } else { throw er } }) } [_rollbackRetireShallowNodes] (er) { const timeEnd = time.start('reify:rollback:retireShallow') const moves = this.#retiredPaths const movePromises = Object.entries(moves) .map(([from, to]) => this[_renamePath](to, from)) return promiseAllRejectLate(movePromises) // ignore subsequent rollback errors .catch(() => {}) .then(timeEnd) .then(() => { throw er }) } [_createSparseTree] () { const timeEnd = time.start('reify:createSparse') // if we call this fn again, we look for the previous list // so that we can avoid making the same directory multiple times const leaves = this.diff.leaves .filter(diff => { return (diff.action === 'ADD' || diff.action === 'CHANGE') && !this.#sparseTreeDirs.has(diff.ideal.path) && !diff.ideal.isLink }) .map(diff => diff.ideal) // we check this in parallel, so guard against multiple attempts to // retire the same path at the same time. const dirsChecked = new Set() return promiseAllRejectLate(leaves.map(async node => { for (const d of walkUp(node.path)) { if (d === node.top.path) { break } if (dirsChecked.has(d)) { continue } dirsChecked.add(d) const st = await lstat(d).catch(() => null) // this can happen if we have a link to a package with a name // that the filesystem treats as if it is the same thing. // would be nice to have conditional istanbul ignores here... /* istanbul ignore next - defense in depth */ if (st && !st.isDirectory()) { const retired = retirePath(d) this.#retiredPaths[d] = retired this[_trashList].add(retired) await this[_renamePath](d, retired) } } this.#sparseTreeDirs.add(node.path) const made = await mkdir(node.path, { recursive: true }) // if the directory already exists, made will be undefined. if that's the case // we don't want to remove it because we aren't the ones who created it so we // omit it from the #sparseTreeRoots /* istanbul ignore next -- pre-existing: mkdir returns undefined when dir exists, covered in reify tests but lost in aggregate coverage merge */ if (made) { this.#sparseTreeRoots.add(made) } })).then(timeEnd) } [_rollbackCreateSparseTree] (er) { const timeEnd = time.start('reify:rollback:createSparse') // cut the roots of the sparse tree that were created, not the leaves const roots = this.#sparseTreeRoots // also delete the moves that we retired, so that we can move them back const failures = [] const targets = [...roots, ...Object.keys(this.#retiredPaths)] const unlinks = targets .map(path => rm(path, { recursive: true, force: true }).catch(er => failures.push([path, er]))) return promiseAllRejectLate(unlinks).then(() => { // eslint-disable-next-line promise/always-return if (failures.length) { log.warn('cleanup', 'Failed to remove some directories', failures) } }) .then(timeEnd) .then(() => this[_rollbackRetireShallowNodes](er)) } // shrinkwrap nodes define their dependency branches with a file, so // we need to unpack them, read that shrinkwrap file, and then update // the tree by calling loadVirtual with the node as the root. [_loadShrinkwrapsAndUpdateTrees] () { const seen = this.#shrinkwrapInflated const shrinkwraps = this.diff.leaves .filter(d => (d.action === 'CHANGE' || d.action === 'ADD' || !d.action) && d.ideal.hasShrinkwrap && !seen.has(d.ideal) && !this[_trashList].has(d.ideal.path)) if (!shrinkwraps.length) { return } const timeEnd = time.start('reify:loadShrinkwraps') const Arborist = this.constructor return promiseAllRejectLate(shrinkwraps.map(diff => { const node = diff.ideal seen.add(node) return diff.action ? this[_reifyNode](node) : node })) .then(nodes => promiseAllRejectLate(nodes.map(node => new Arborist({ ...this.options, path: node.path, }).loadVirtual({ root: node, subtreeOnly: true })))) // reload the diff and sparse tree because the ideal tree changed .then(() => this[_diffTrees]()) .then(() => this[_createSparseTree]()) .then(() => this[_loadShrinkwrapsAndUpdateTrees]()) .then(timeEnd) } // create a symlink for Links, extract for Nodes // return the node object, since we usually want that // handle optional dep failures here // If reifying fails, and the node is optional, add it and its optionalSet // to the trash list // Always return the node. [_reifyNode] (node) { const timeEnd = time.start(`reifyNode:${node.location}`) this.addTracker('reify', node.name, node.location) const p = Promise.resolve().then(async () => { await this[_checkBins](node) await this.#extractOrLink(node) const { _id, deprecated } = node.package // The .catch is in _handleOptionalFailure. Not ideal, this should be cleaned up. // eslint-disable-next-line promise/always-return if (deprecated) { log.warn('deprecated', `${_id}: ${deprecated}`) } }) return this[_handleOptionalFailure](node, p) .then(() => { this.finishTracker('reify', node.name, node.location) timeEnd() return node }) } // do not allow node_modules to be a symlink async #validateNodeModules (nm) { if (this.options.force || this.#nmValidated.has(nm)) { return } const st = await lstat(nm).catch(() => null) if (!st || st.isDirectory()) { this.#nmValidated.add(nm) return } log.warn('reify', 'Removing non-directory', nm) await rm(nm, { recursive: true, force: true }) } async #extractOrLink (node) { const nm = resolve(node.parent.path, 'node_modules') await this.#validateNodeModules(nm) if (!node.isLink) { // in normal cases, node.resolved should *always* be set by now. // however, it is possible when a lockfile is damaged, or very old, // or in some other race condition bugs in npm v6, that a previously // bundled dependency will have just a version, but no resolved value, // and no 'bundled: true' setting. // Do the best with what we have, or else remove it from the tree // entirely, since we can't possibly reify it. let res = null if (node.resolved) { const registryResolved = this.#registryResolved(node.resolved) if (registryResolved) { res = `${node.name}@${registryResolved}` } } else if (node.package.name && node.version) { res = `${node.package.name}@${node.version}` } // no idea what this thing is. remove it from the tree. if (!res) { const warning = 'invalid or damaged lockfile detected\n' + 'please re-try this operation once it completes\n' + 'so that the damage can be corrected, or perform\n' + 'a fresh install with no lockfile if the problem persists.' log.warn('reify', warning) log.verbose('reify', 'unrecognized node in tree', node.path) node.parent = null node.fsParent = null this[_addNodeToTrashList](node) return } await debug(async () => { const st = await lstat(node.path).catch(() => null) if (st && !st.isDirectory()) { debug.log('unpacking into a non-directory', node) throw Object.assign(new Error('ENOTDIR: not a directory'), { code: 'ENOTDIR', path: node.path, }) } }) await pacote.extract(res, node.path, { ...this.options, resolved: node.resolved, integrity: node.integrity, // A node counts as "root" for allow-* enforcement if it satisfies at least one valid dependency edge declared by the project root or a workspace. // node.parent is unsafe here: after hoisting, transitive packages can have the project root as their tree parent. // In the linked strategy the store node has no edgesIn, so isolated-reifier precomputes isRootDependency from the source node's edges. _isRoot: node.isRootDependency || [...node.edgesIn].some(e => e.valid && (e.from?.isProjectRoot || e.from?.isWorkspace) ), // pacote's npa re-parses our `name@URL` spec as type=remote, so allowRemote would mis-fire on registry tarballs. // Override only when we can prove the URL is registry-mediated; see #isRegistryResolvedTarball. ...(this.#isRegistryResolvedTarball(node) ? { allowRemote: 'all' } : {}), }) // store nodes don't use Node class so node.package doesn't get updated if (node.isInStore) { const { content: pkg } = await PackageJson.normalize(node.path) node.package.scripts = pkg.scripts } return } // node.isLink // Tree-only Link: present in the tree for diff/filter participation, never materialized on disk. if (node.isUndeclaredWorkspaceLink) { return } await rm(node.path, { recursive: true, force: true }) // symlink const dir = dirname(node.path) const target = node.realpath let rel if (node.resolved?.startsWith('file:')) { rel = this.#calculateRelativePath(node, dir, target, nm) } else { rel = relative(dir, target) } await mkdir(dir, { recursive: true }) return symlink(rel, node.path, 'junction') } // if the node is optional, then the failure of the promise is nonfatal // just add it and its optional set to the trash list. [_handleOptionalFailure] (node, p) { return (node.optional ? p.catch(() => { const set = optionalSet(node) for (const node of set) { log.verbose('reify', 'failed optional dependency', node.path) node.inert = true this[_addNodeToTrashList](node) } }) : p).then(() => node) } #calculateRelativePath (node, dir, target) { // Check if the node is affected by a root override let hasRootOverride = [...node.edgesIn].some(edge => edge.from.isRoot && edge.overrides) // If not set via edges, see if the root package.json explicitly lists an override if (!hasRootOverride && node.root) { const rootPackage = node.root.target hasRootOverride = !!(rootPackage && rootPackage.package.overrides && rootPackage.package.overrides[node.name]) } if (!hasRootOverride) { return relative(dir, target) } // If an override is detected, attempt to retrieve the override spec from the root package.json const overrideSpec = node.root?.target?.package?.overrides?.[node.name] if (typeof overrideSpec === 'string' && overrideSpec.startsWith('file:')) { const overridePath = overrideSpec.replace(/^file:/, '') const rootDir = node.root.target.path return relative(dir, resolve(rootDir, overridePath)) } // Fallback: derive the file path from node.resolved in a platform-agnostic way const filePath = node.resolved.replace(/^file:/, '') return join(filePath) } // Serialize the hidden lockfile from the isolated tree, which mirrors the on-disk .store/symlink layout. // Its children are every materialized node_modules entry: store package dirs and all symlinks. async #saveLinkedHiddenLockfile (isolatedTree) { const path = isolatedTree.realpath const meta = new Shrinkwrap({ path, hiddenLockfile: true, lockfileVersion: defaultLockfileVersion, resolveOptions: this.options, }) meta.reset() meta.filename = resolve(path, 'node_modules/.package-lock.json') const storeRe = /^(.*\/\.store\/.+?)\/node_modules\// const containers = new Set() const nodes = new Set() for (const node of isolatedTree.children.values()) { // Tree-only undeclared workspace self-links aren't on disk. if (node.isUndeclaredWorkspaceLink) { continue } nodes.add(node) // Record the enclosing .store/<key> dir so loadVirtual can resolve a store package's sibling deps. // node.location uses the platform separator; lockfile keys are posix. const m = node.location.replace(/\\/g, '/').match(storeRe) if (m) { containers.add(m[1]) } } // Workspace dirs hold their own dep symlinks; record them so the cache can validate those subtrees. for (const ws of isolatedTree.fsChildren) { nodes.add(ws) } for (const node of nodes) { meta.add(node) } for (const loc of containers) { meta.data.packages[loc] = {} } await meta.save() } // Build a flat actual tree wrapper for linked installs so the diff can correctly match store entries that already exist on disk. // The proxy tree from createIsolatedTree() is flat (all children on root), but loadActual() produces a nested tree where store entries are deep link targets. // This wrapper surfaces them at the root level for comparison. #buildLinkedActualForDiff (idealTree, actualTree) { // Combined Map keyed by path (how allChildren() in diff.js keys) const combined = new Map() // Create synthetic actual entries for ALL ideal children that exist on disk. // The isolated ideal tree is flat (all entries as root children), but loadActual() produces a nested tree where workspace deps are under fsChildren and store entries are deep link targets. // Synthetic entries ensure the diff compares matching resolved/integrity values (e.g. workspace links have resolved=undefined in the ideal tree but resolved="file:../packages/..." in the actual tree). for (const child of idealTree.children.values()) { if (combined.has(child.path) || !existsSync(child.path)) { continue } // Skip store links whose ideal realpath doesn't exist on disk yet — the store hash changed and the symlink needs recreating via ADD. if (child.isLink && child.resolved?.startsWith('file:.store/') && !existsSync(child.realpath)) { continue } // Skip a link whose on-disk target is a valid-but-wrong store key (e.g. an interrupted update), so the diff repoints it via ADD. if (child.isLink && this.#linkTargetMismatch(child)) { continue } let entry if (child.isLink) { entry = new IsolatedLink(child) } else { entry = new IsolatedNode(child) } if (child.isLink && combined.has(child.realpath)) { entry.target = combined.get(child.realpath) } combined.set(child.path, entry) } // Proxy .get(name) to original actual tree for filterNodes compatibility // (scoped workspace installs use .get(name), allChildren uses .values()) const origGet = actualTree.children.get.bind(actualTree.children) const combinedGet = combined.get.bind(combined) /* istanbul ignore next -- only reached during scoped workspace installs */ combined.get = (key) => combinedGet(key) || origGet(key) let wrapper /* istanbul ignore next - untested! */ if (actualTree.isLink) { wrapper = new IsolatedLink(actualTree) } else { wrapper = new IsolatedNode(actualTree) } wrapper.root = wrapper wrapper.binPaths = actualTree.binPaths wrapper.children = combined wrapper.edgesOut = actualTree.edgesOut // Use empty fsChildren so that allChildren() only picks up entries from the combined map. // The actual fsChildren have real children with different resolved values (e.g. file:../../../node_modules/.store/... vs file:.store/...) that would overwrite our synthetic entries in allChildren(). wrapper.fsChildren = new Set() wrapper.integrity = actualTree.integrity wrapper.inventory = actualTree.inventory return wrapper } // True when the link's on-disk target resolves to a different path than its ideal target. // The caller only invokes this once both paths exist, so realpathSync won't throw. #linkTargetMismatch (child) { return realpathSync(child.path) !== realpathSync(child.realpath) } // When extracting a registry-resolved package, the spec we hand to pacote is name@URL. // pacote re-parses that with npa and gets spec.type === 'remote', so without an override the allow-remote gate would fire on every registry tarball (both =none and =root mis-fire). // Returns true only when we are confident this is a registry-mediated install. #isRegistryResolvedTarball (node) { if (!node.resolved || !node.isRegistryDependency) { return false } try { // Match the effective fetch URL, not the raw lockfile value. // #registryResolved applies replace-registry-host, rewriting a public-registry pin to the configured proxy/mirror so it matches. const resolvedURL = new URL(this.#registryResolved(node.resolved)) // pickRegistry only consults spec.scope, so a bare-name (tag) parse is sufficient and avoids a node.version dependency. const registry = new URL(pickRegistry(npa(node.name), this.options)) const registryPath = registry.pathname.replace(/\/?$/, '/') return resolvedURL.origin === registry.origin && (registryPath === '/' || resolvedURL.pathname.startsWith(registryPath)) } catch { return false } } #registryResolved (resolved) { // the default registry url is a magic value meaning "the currently // configured registry". // `resolved` must never be falsey. // // XXX: use a magic string that isn't also a valid value, like // ${REGISTRY} or something. This has to be threaded through the // Shrinkwrap and Node classes carefully, so for now, just treat // the default reg as the magical animal that it has been. try { const resolvedURL = hgi.parseUrl(resolved) const registryURL = new URL(this.registry) const registryPath = registryURL.pathname.replace(/\/$/, '') let matchURL = null try { matchURL = new URL(this.options.replaceRegistryHost) } catch { // keep matchURL null } const matchHost = matchURL?.hostname ?? this.options.replaceRegistryHost const matchPath = matchURL?.pathname.replace(/\/$/, '') ?? null const hasPathPrefix = (pathname, prefix) => pathname === prefix || pathname.startsWith(`${prefix}/`) const hostMatches = this.options.replaceRegistryHost === 'always' || matchHost === resolvedURL.hostname const pathMatches = !matchPath || hasPathPrefix(resolvedURL.pathname, matchPath) if (!hostMatches || !pathMatches) { return resolved } resolvedURL.protocol = registryURL.protocol resolvedURL.hostname = registryURL.hostname resolvedURL.port = registryURL.port if (matchPath) { // full-URL prefix: swap old path prefix for the registry path resolvedURL.pathname = registryPath + resolvedURL.pathname.slice(matchPath.length) } else if (registryPath && !hasPathPrefix(resolvedURL.pathname, registryPath)) { // host-only: prepend registry path if not already present resolvedURL.pathname = registryPath + resolvedURL.pathname } return resolvedURL.toString() } catch { // if we could not parse the url at all then returning nothing // here means it will get removed from the tree in the next step return undefined } } // bundles are *sort of* like shrinkwraps, in that the branch is defined // by the contents of the package. however, in their case, rather than // shipping a virtual tree that must be reified, they ship an entire // reified actual tree that must be unpacked and not modified. [_loadBundlesAndUpdateTrees] (depth = 0, bundlesByDepth) { let maxBundleDepth if (!bundlesByDepth) { bundlesByDepth = new Map() maxBundleDepth = -1 dfwalk({ tree: this.diff, visit: diff => { const node = diff.ideal if (!node) { return } if (node.isProjectRoot) { return } const { bundleDependencies } = node.package if (bundleDependencies && bundleDependencies.length) { maxBundleDepth = Math.max(maxBundleDepth, node.depth) if (!bundlesByDepth.has(node.depth)) { bundlesByDepth.set(node.depth, [node]) } else { bundlesByDepth.get(node.depth).push(node) } } }, getChildren: diff => diff.children, }) bundlesByDepth.set('maxBundleDepth', maxBundleDepth) } else { maxBundleDepth = bundlesByDepth.get('maxBundleDepth') } if (depth === 0) { time.start('reify:loadBundles') } if (depth > maxBundleDepth) { // if we did something, then prune the tree and update the diffs if (maxBundleDepth !== -1) { this.#pruneBundledMetadeps(bundlesByDepth) this[_diffTrees]() } time.end('reify:loadBundles') return } // skip any that have since been removed from the tree, eg by a // shallower bundle overwriting them with a bundled meta-dep. const set = (bundlesByDepth.get(depth) || []) .filter(node => node.root === this.idealTree && node.target !== node.root && !this[_trashList].has(node.path)) if (!set.length) { return this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth) } // extract all the nodes with bundles return promiseCallLimit(set.map(node => { return () => { this.#bundleUnpacked.add(node) return this[_reifyNode](node) } }), { rejectLate: true }) // then load their unpacked children and move into the ideal tree .then(nodes => promiseAllRejectLate(nodes.map(async node => { const arb = new this.constructor({ ...this.options, path: node.path, }) const notTransplanted = new Set(node.children.keys()) await arb.loadActual({ root: node, // don't transplant any sparse folders we created // loadActual will set node.package to {} for empty directories // if by chance there are some empty folders in the node_modules // tree for some other reason, then ok, ignore those too. transplantFilter: node => { if (node.package._id) { // it's actually in the bundle if it gets transplanted notTransplanted.delete(node.name) return true } else { return false } }, }) for (const name of notTransplanted) { this.#bundleMissing.add(node.children.get(name)) } }))) // move onto the next level of bundled items .then(() => this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth)) } // https://github.com/npm/cli/issues/1597#issuecomment-667639545 #pruneBundledMetadeps (bundlesByDepth) { const bundleShadowed = new Set() // Example dep graph: // root -> (a, c) // a -> BUNDLE(b) // b -> c // c -> b // // package tree: // root // +-- a // | +-- b(1) // | +-- c(1) // +-- b(2) // +-- c(2) // 1. mark everything that's shadowed by anything in the bundle. This // marks b(2) and c(2). // 2. anything with edgesIn from outside the set, mark not-extraneous, // remove from set. This unmarks c(2). // 3. continue until no change // 4. remove everything in the set from the tree. b(2) is pruned // create the list of nodes shadowed by children of bundlers for (const bundles of bundlesByDepth.values()) { // skip the 'maxBundleDepth' item if (!Array.isArray(bundles)) { continue } for (const node of bundles) { for (const name of node.children.keys()) { const shadow = node.parent.resolve(name) if (!shadow) { continue } bundleShadowed.add(shadow) shadow.extraneous = true } } } // lib -> (a@1.x) BUNDLE(a@1.2.3 (b@1.2.3)) // a@1.2.3 -> (b@1.2.3) // a@1.3.0 -> (b@2) // b@1.2.3 -> () // b@2 -> (c@2) // // root // +-- lib // | +-- a@1.2.3 // | +-- b@1.2.3 // +-- b@2 <-- shadowed, now extraneous // +-- c@2 <-- also shadowed, because only dependent is shadowed for (const shadow of bundleShadowed) { for (const shadDep of shadow.edgesOut.values()) { /* istanbul ignore else - pretty unusual situation, just being * defensive here. Would mean that a bundled dep has a dependency * that is unmet. which, weird, but if you bundle it, we take * whatever you put there and assume the publisher knows best. */ if (shadDep.to) { bundleShadowed.add(shadDep.to) shadDep.to.extraneous = true } } } let changed do { changed = false for (const shadow of bundleShadowed) { for (const edge of shadow.edgesIn) { if (!bundleShadowed.has(edge.from)) { shadow.extraneous = false bundleShadowed.delete(shadow) changed = true break } } } } while (changed) for (const shadow of bundleShadowed) { this[_addNodeToTrashList](shadow) shadow.root = null } } async [_submitQuickAudit] () { if (this.options.audit === false) { this.auditReport = null return } // we submit the quick audit at this point in the process, as soon as // we have all the deps resolved, so that it can overlap with the other // actions as much as possible. Stash the promise, which we resolve // before finishing the reify() and returning the tree. Thus, we do // NOT return the promise, as the intent is for this to run in parallel // with the reification, and be resolved at a later time. const timeEnd = time.start('reify:audit') const options = { ...this.options } // Under the linked strategy idealTree is the isolated tree, which the audit cannot traverse; audit the non-isolated tree instead. const tree = this.#linkedIdealForAudit || this.idealTree // if we're operating on a workspace, only audit the workspace deps if (this.options.workspaces.length) { options.filterSet = this.workspaceDependencySet( tree, this.options.workspaces, this.options.includeWorkspaceRoot ) } this.auditReport = AuditReport.load(tree, options).then(res => { timeEnd() return res }) } // ok! actually unpack stuff into their target locations! // The sparse tree has already been created, so we walk the diff // kicking off each unpack job. If any fail, we rm the sparse // tree entirely and try to put everything back where it was. [_unpackNewModules] () { const timeEnd = time.start('reify:unpack') const unpacks = [] dfwalk({ tree: this.diff, visit: diff => { // no unpacking if we don't want to change this thing if (diff.action !== 'CHANGE' && diff.action !== 'ADD') { return } const node = diff.ideal const bd = this.#bundleUnpacked.has(node) const sw = this.#shrinkwrapInflated.has(node) const bundleMissing = this.#bundleMissing.has(node) // check whether we still need to unpack this one. // test the inDepBundle last, since that's potentially a tree walk. const doUnpack = node && // can't unpack if removed! // root node already exists !node.isRoot && // already unpacked to read bundle !bd && // already unpacked to read sw !sw && // already unpacked by another dep's bundle (bundleMissing || !node.inDepBundle) if (doUnpack) { unpacks.push(this[_reifyNode](node)) } }, getChildren: diff => diff.children, }) return promiseAllRejectLate(unpacks).then(timeEnd) } // This is the part where we move back the unchanging nodes that were // the children of a node that did change. If this fails, the rollback // is a three-step process. First, we try to move the retired unchanged // nodes BACK to their retirement folders, then delete the sparse tree, // then move everything out of retirement. [_moveBackRetiredUnchanged] () { // get a list of all unchanging children of any shallow retired nodes // if they are not the ancestor of any node in the diff set, then the // directory won't already exist, so just rename it over. // This is sort of an inverse diff tree, of all the nodes where // the actualTree and idealTree _don't_ differ, starting from the // shallowest nodes that we moved aside in the first place. const timeEnd = time.start('reify:unretire') const moves = this.#retiredPaths this.#retiredUnchanged = {} return promiseAllRejectLate(this.diff.children.map(diff => { // skip if nothing was retired if (diff.action !== 'CHANGE' && diff.action !== 'REMOVE') { return } const { path: realFolder } = diff.actual const retireFolder = moves[realFolder] /* istanbul ignore next - should be impossible */ debug(() => { if (!retireFolder) { const er = new Error('trying to un-retire but not retired') throw Object.assign(er, { realFolder, retireFolder, actual: diff.actual, ideal: diff.ideal, action: diff.action, }) } }) this.#retiredUnchanged[retireFolder] = [] return promiseAllRejectLate(diff.unchanged.map(node => { // no need to roll back links, since we'll just delete them anyway if (node.isLink) { return mkdir(dirname(node.path), { recursive: true, force: true }) .then(() => this[_reifyNode](node)) } // will have been moved/unpacked along with bundler if (node.inDepBundle && !this.#bundleMissing.has(node)) { return } this.#retiredUnchanged[retireFolder].push(node) const rel = relative(realFolder, node.path) const fromPath = resolve(retireFolder, rel) // if it has bundleDependencies, then make node_modules. otherwise // skip it. const bd = node.package.bundleDependencies const dir = bd && bd.length ? node.path + '/node_modules' : node.path return mkdir(dir, { recursive: true }).then(() => this[_moveContents](node, fromPath)) })) })).then(timeEnd) } // move the contents from the fromPath to the node.path [_moveContents] (node, fromPath) { return packageContents({ path: fromPath, depth: 1, packageJsonCache: new Map([[fromPath + '/package.json', node.package]]), }).then(res => promiseAllRejectLate(res.map(path => { const rel = relative(fromPath, path) const to = resolve(node.path, rel) return this[_renamePath](path, to) }))) } [_rollbackMoveBackRetiredUnchanged] (er) { const moves = this.#retiredPaths // flip the mapping around to go back const realFolders = new Map(Object.entries(moves).map(([k, v]) => [v, k])) const promises = Object.entries(this.#retiredUnchanged) .map(([retireFolder, nodes]) => promiseAllRejectLate(nodes.map(node => { const realFolder = realFolders.get(retireFolder) const rel = relative(realFolder, node.path) const fromPath = resolve(retireFolder, rel) return this[_moveContents]({ ...node, path: fromPath }, node.path) }))) return promiseAllRejectLate(promises) .then(() => this[_rollbackCreateSparseTree](er)) } [_build] () { const timeEnd = time.start('reify:build') // for all the things being installed, run their appropriate scripts // run in tip->root order, so as to be more likely to build a node's // deps before attempting to build it itself const nodes = [] dfwalk({ tree: this.diff, leave: diff => { if (!diff.ideal.isProjectRoot) { nodes.push(diff.ideal) } }, // process adds before changes, ignore removals getChildren: diff => diff && diff.children, filter: diff => diff.action === 'ADD' || diff.action === 'CHANGE', }) // pick up link nodes from the unchanged list as we want to run their // scripts in every install despite of having a diff status change for (const node of this.diff.unchanged) { const tree = node.root.target // skip links that only live within node_modules as they are most // likely managed by packages we installed, we only want to rebuild // unchanged links we directly manage const linkedFromRoot = (node.parent === tree && !node.inert) || node.target.fsTop === tree if (node.isLink && linkedFromRoot) { nodes.push(node) } } return this.rebuild({ nodes, handleOptionalFailure: true }).then(timeEnd) } // the tree is pretty much built now, so it's cleanup time. // remove the retired folders, and any deleted nodes // If this fails, there isn't much we can do but tell the user about it. // Thankfully, it's pretty unlikely that it'll fail, since rm is a node builtin. async [_removeTrash] () { const timeEnd = time.start('reify:trash') const promises = [] const failures = [] const _rm = path => rm(path, { recursive: true, force: true }).catch(er => failures.push([path, er])) for (const path of this[_trashList]) { promises.push(_rm(path)) } await promiseAllRejectLate(promises) if (failures.length) { log.warn('cleanup', 'Failed to remove some directories', failures) } timeEnd() } // Remove the root .store left behind by a previous linked install when reifying under a non-linked strategy. async #removeStaleStoreDir () { const storeDir = resolve(this.path, 'node_modules', '.store') if (!existsSync(storeDir)) { return } log.silly('reify', 'removing stale .store from a previous linked install') await rm(storeDir, { recursive: true, force: true }) .catch(/* istanbul ignore next -- rm with force rarely fails */ er => log.warn('cleanup', 'Failed to remove stale .store directory', er)) } // After a linked install, scan node_modules/.store/ and remove any directories that are not referenced by the current ideal tree. // Store entries become orphaned when dependencies are updated or removed, because the diff never sees the old store keys. // Then sweep the top-level node_modules/ for orphaned symlinks (e.g. an uninstalled dep whose store entry was just removed) so we don't leave dangling links. async #cleanOrphanedStoreEntries () { const nmDir = resolve(this.path, 'node_modules') const storeDir = resolve(nmDir, '.store') // Enumerate on-disk store entries as full keys, descending one level into each @scope directory because scoped keys nest as .store/@scope/pkg@version-hash. let entries try { const topLevel = await readdir(storeDir, { withFileTypes: true }) entries = [] for (const ent of topLevel) { if (ent.name.startsWith('@')) { let scoped try { scoped = await readdir(resolve(storeDir, ent.name)) } catch { /* istanbul ignore next -- readdir of an entry we just listed should not fail */ continue } for (const name of scoped) { entries.push(`${ent.name}/${name}`) } } else { entries.push(ent.name) } } } catch { entries = null } // Collect valid store keys and valid top-level links per node_modules directory. // Store entries have location node_modules/.store/{key}/node_modules/{pkg}. // Top-level links have location {prefix}/node_modules/{pkg} or {prefix}/node_modules/@scope/{pkg}, where {prefix} is empty for the root project and the workspace's localLocation for workspace deps. // Locations are normalized to forward slashes here because IsolatedNode/IsolatedLink locations are built with path.join, which uses backslashes on Windows. const validKeys = new Set() const nmDirs = new Map() // Valid bin shim names per node_modules dir, collected from each top-level entry's package.bin so the .bin sweep keeps only shims a still-installed package provides. const binsByDir = new Map() const NM_PREFIX = 'node_modules/' const STORE_MARKER = '/.store/' for (const child of this.idealTree.children.values()) { const loc = child.location.replace(/\\/g, '/') if (child.isInStore) { // Store location is node_modules/.store/{key}/node_modules/{pkg}. // For a scoped package the key is @scope/pkg@version-hash, which spans two path segments, so reconstruct both instead of taking only the scope. const parts = loc.split('/') const key = parts[2].startsWith('@') ? `${parts[2]}/${parts[3]}` : parts[2] validKeys.add(key) continue } // Tree-only Links never exist on disk; skipping them lets the sweep remove any stale self-link left by an older npm version. if (child.isLink && child.isUndeclaredWorkspaceLink) { continue } // Real top-level Nodes (e.g. the root's bundled deps) fall through here too, so they are recorded as valid and never swept as stale. const nmIdx = loc.lastIndexOf(NM_PREFIX) if (nmIdx === -1 || loc.includes(STORE_MARKER)) { continue } const prefix = loc.slice(0, nmIdx) const dir = resolve(this.path, prefix, 'node_modules') const rest = loc.slice(nmIdx + NM_PREFIX.length) let entry if (rest.startsWith('@')) { const [scope, name] = rest.split('/') entry = `${scope}${sep}${name}` } else { entry = rest.split('/')[0] } let set = nmDirs.get(dir) if (!set) { set = new Set() nmDirs.set(dir, set) } set.add(entry) // package.bin is normalized to an object keyed by bin name; shim names are unscoped even for scoped packages. const bin = child.package?.bin if (bin && typeof bin === 'object') { let binSet = binsByDir.get(dir) if (!binSet) { binSet = new Set() binsByDir.set(dir, binSet) } for (const bn of Object.keys(bin)) { binSet.add(bn) } } } // Determine which node_modules directories to sweep. // For an unfiltered install, sweep the project root and every workspace's node_modules even if no top-level links remain (e.g. last dep was just uninstalled). // For a filtered install (npm install -w <ws>), restrict the sweep to the in-scope workspaces so out-of-scope workspaces are left untouched, mirroring what the diff would do. // When --include-workspace-root is set, the filter scope pulls in root deps too, so the root node_modules is included in the sweep. const filteredNames = this.options.workspaces const isFiltered = Array.isArray(filteredNames) && filteredNames.length > 0 if (isFiltered) { const allowedDirs = new Set() for (const ws of this.idealTree.fsChildren) { if (filteredNames.includes(ws.packageName) || filteredNames.includes(ws.name)) { allowedDirs.add(resolve(ws.path, 'node_modules')) } } if (this.options.includeWorkspaceRoot) { allowedDirs.add(nmDir) } for (const dir of [...nmDirs.keys()]) { if (!allowedDirs.has(dir)) { nmDirs.delete(dir) } } for (const dir of allowedDirs) { if (!nmDirs.has(dir)) { nmDirs.set(dir, new Set()) } } } else { if (!nmDirs.has(nmDir)) { nmDirs.set(nmDir, new Set()) } for (const ws of this.idealTree.fsChildren) { const wsNmDir = resolve(ws.path, 'node_modules') if (!nmDirs.has(wsNmDir)) { nmDirs.set(wsNmDir, new Set()) } } } if (entries) { const orphaned = entries.filter(e => !validKeys.has(e)) if (orphaned.length) { log.silly('reify', 'cleaning orphaned store entries', orphaned) await promiseAllRejectLate( orphaned.map(e => rm(resolve(storeDir, e), { recursive: true, force: true }) .catch(/* istanbul ignore next -- rm with force rarely fails */ er => log.warn('cleanup', `Failed to remove orphaned store entry ${e}`, er)) ) ) // Removing the last scoped orphan under a scope leaves an empty @scope directory behind, so prune any scope directory that is now empty. const scopes = new Set( orphaned.filter(e => e.startsWith('@')).map(e => e.split('/')[0]) ) await promiseAllRejectLate( [...scopes].map(async scope => { const scopeDir = resolve(storeDir, scope) try { const remaining = await readdir(scopeDir) if (!remaining.length) { await rm(scopeDir, { recursive: true, force: true }) } } catch { /* istanbul ignore next -- readdir of a scope dir we just listed should not fail */ } }) ) } } for (const [dir, valid] of nmDirs) { await this.#cleanOrphanedTopLevelLinks(dir, valid) await this.#cleanStaleBinLinks(dir, binsByDir.get(dir)) } } // Remove stale bin shims left in node_modules/.bin after an uninstall under linked, where the diff never emits an action to drop them. // A shim is stale when no still-linked package provides its name, or when it is a dangling symlink; matching by name handles both POSIX symlinks and Windows .cmd/.ps1 shims. async #cleanStaleBinLinks (nmDir, validBins = new Set()) { const binDir = resolve(nmDir, '.bin') let names try { names = await readdir(binDir) } catch { return } const stale = names.filter(name => { const base = name.replace(/\.(cmd|ps1)$/, '') return !validBins.has(base) || !existsSync(resolve(binDir, name)) }) if (!stale.length) { return } log.silly('reify', 'cleaning stale bin links', stale) await promiseAllRejectLate( stale.map(name => rm(resolve(binDir, name), { force: true }) .catch(/* istanbul ignore next -- rm with force rarely fails */ er => log.warn('cleanup', `Failed to remove stale bin link ${name}`, er)) ) ) } // Remove node_modules/ entries that aren't represented in the ideal tree. // Run for the project root and each workspace's node_modules. // The linked diff path can't see these because #buildLinkedActualForDiff derives the actual tree from the ideal, so removed deps are never compared. // Two kinds of stale entry are removed: // - symlinks whose target resolves inside the project root — store links (node_modules/.store/...) and workspace self-links (e.g. node_modules/<ws> -> ../packages/<ws>) that npm itself created. // - real package directories — hoisted-layout deps left behind when switching from the hoisted strategy to linked, where every valid top-level entry is a symlink. // Symlinks pointing outside the project (e.g. `npm link foo` without --save targeting the global prefix, or hand-made `ln -s` to an external path) and non-package real directories are preserved. async #cleanOrphanedTopLevelLinks (nmDir, validTopLevel) { const projectPrefix = resolve(this.path) + sep let dirents try { dirents = await readdir(nmDir, { withFileTypes: true }) } catch { return } const isOurOrphan = async (linkPath) => { let target try { target = await readlink(linkPath) } catch { /* istanbul ignore next -- readlink of an entry we just listed as a symlink should not fail */ return false } return resolve(dirname(linkPath), target).startsWith(projectPrefix) } // A real directory is stale only when it is an actual package (has a package.json), so unrelated user directories are never touched. const isStaleRealPkg = (dirent, entPath) => dirent.isDirectory() && existsSync(resolve(entPath, 'package.json')) const isOrphan = async (dirent, entPath) => (dirent.isSymbolicLink() && await isOurOrphan(entPath)) || isStaleRealPkg(dirent, entPath) const orphaned = [] const scopes = new Set() for (const ent of dirents) { // skip npm-managed entries (.bin, .store, .package-lock.json, etc) if (ent.name.startsWith('.')) { continue } if (ent.name.startsWith('@')) { let scoped try { scoped = await readdir(resolve(nmDir, ent.name), { withFileTypes: true }) } catch { /* istanbul ignore next -- readdir of an entry we just listed should not fail */ continue } for (const pkgEnt of scoped) { const key = `${ent.name}${sep}${pkgEnt.name}` if (!validTopLevel.has(key) && await isOrphan(pkgEnt, resolve(nmDir, key))) { orphaned.push(key) scopes.add(ent.name) } } } else if (!validTopLevel.has(ent.name) && await isOrphan(ent, resolve(nmDir, ent.name))) { orphaned.push(ent.name) } } if (!orphaned.length) { return } log.silly('reify', 'cleaning orphaned top-level entries', orphaned) await promiseAllRejectLate( orphaned.map(name => rm(resolve(nmDir, name), { recursive: true, force: true }) .catch(/* istanbul ignore next -- rm with force rarely fails */ er => log.warn('cleanup', `Failed to remove orphaned entry ${name}`, er)) ) ) // Removing the last package under a scope leaves an empty @scope directory behind, so prune any scope directory that is now empty. await promiseAllRejectLate( [...scopes].map(async scope => { const scopeDir = resolve(nmDir, scope) try { const remaining = await readdir(scopeDir) if (!remaining.length) { await rm(scopeDir, { recursive: true, force: true }) } } catch { /* istanbul ignore next -- readdir of a scope dir we just listed should not fail */ } }) ) } // last but not least, we save the ideal tree metadata to the package-lock // or shrinkwrap file, and any additions or removals to package.json async [_saveIdealTree] (options) { // the ideal tree is actualized now, hooray! // it still contains all the references to optional nodes that were removed // for install failures. Those still end up in the shrinkwrap, so we // save it first, then prune out the optional trash, and then return it. const save = !(options.save === false) // we check for updates in order to make sure we run save ideal tree // even though save=false since we want `npm update` to be able to // write to package-lock files by default const hasUpdates = this[_updateAll] || this[_updateNames].length // we're going to completely skip save ideal tree in case of a global or // dry-run install and also if the save option is set to false, EXCEPT for // update since the expected behavior for npm7+ is for update to // NOT save to package.json, we make that exception since we still want // saveIdealTree to be able to write the lockfile by default. const saveIdealTree = !( (!save && !hasUpdates) || this.options.global || this.options.dryRun ) if (!saveIdealTree) { return false } const timeEnd = time.start('reify:save') const updatedTrees = new Set() const updateNodes = nodes => { for (const { name, tree: addTree } of nodes) { // addTree either the root, or a workspace const edge = addTree.edgesOut.get(name) const pkg = addTree.package const req = npa.resolve(name, edge.spec, addTree.realpath) const { rawSpec, subSpec } = req const spec = subSpec ? subSpec.rawSpec : rawSpec const child = edge.to // if we tried to install an optional dep, but it was a version // that we couldn't resolve, this MAY be missing. if we haven't // blown up by now, it's because it was not a problem, though, so // just move on. if (!child || !addTree.isTop) { continue } let newSpec // True if the dependency is getting installed from a local file path // In this case it is not possible to do the normal version comparisons // as the new version will be a file path const isLocalDep = req.type === 'directory' || req.type === 'file' if (req.registry) { const version = child.version const prefixRange = version ? this.options.savePrefix + version : '*' // if we installed a range, then we save the range specified // if it is not a subset of the ^x.y.z. eg, installing a range // of `1.x <1.2.3` will not be saved as `^1.2.0`, because that // would allow versions outside the requested range. Tags and // specific versions save with the save-prefix. const isRange = (subSpec || req).type === 'range' let range = spec if ( !isRange || spec === '*' || subset(prefixRange, spec, { loose: true }) ) { range = prefixRange } const pname = child.packageName const alias = name !== pname newSpec = alias ? `npm:${pname}@${range}` : range } else if (req.hosted) { // save the git+https url if it has auth; otherwise, shortcut const h = req.hosted const opt = { noCommittish: false } if (h.https && h.auth) { newSpec = `git+${h.https(opt)}` } else { newSpec = h.shortcut(opt) } } else if (isLocalDep) { // when finding workspace nodes, make sure that // we save them using their version instead of // using their relative path if (edge.type === 'workspace') { const { version } = edge.to.target const prefixRange = version ? this.options.savePrefix + version : '*' newSpec = prefixRange } else { // save the relative path in package.json // Normally saveSpec is updated with the proper relative // path already, but it's possible to specify a full absolute // path initially, in which case we can end up with the wrong // thing, so just get the ultimate fetchSpec and relativize it. const p = req.fetchSpec.replace(/^file:/, '') const rel = relpath(addTree.realpath, p) newSpec = `file:${rel}` } } else { newSpec = req.saveSpec } if (options.saveType) { const depType = saveTypeMap.get(options.saveType) pkg[depType][name] = newSpec // PackageJson.normalize will have moved it here if it was in both, if it is empty it will be deleted later if (options.saveType === 'prod' && pkg.optionalDependencies) { delete pkg.optionalDependencies[name] } } else { if (hasSubKey(pkg, 'dependencies', name)) { pkg.dependencies[name] = newSpec } if (hasSubKey(pkg, 'devDependencies', name)) { pkg.devDependencies[name] = newSpec // don't update peer or optional if we don't have to if (hasSubKey(pkg, 'peerDependencies', name) && (isLocalDep || !intersects(newSpec, pkg.peerDependencies[name]))) { pkg.peerDependencies[name] = newSpec } if (hasSubKey(pkg, 'optionalDependencies', name) && (isLocalDep || !intersects(newSpec, pkg.optionalDependencies[name]))) { pkg.optionalDependencies[name] = newSpec } } else { if (hasSubKey(pkg, 'peerDependencies', name)) { pkg.peerDependencies[name] = newSpec } if (hasSubKey(pkg, 'optionalDependencies', name)) { pkg.optionalDependencies[name] = newSpec } } } updatedTrees.add(addTree) } } // Returns true if any of the edges from this node has a semver // range definition that is an exact match to the version installed // e.g: should return true if for a given and installed version 1.0.0, // range is either =1.0.0 or 1.0.0 const exactVersion = node => { for (const edge of node.edgesIn) { try { if (subset(edge.spec, node.version)) { return false } } catch { // ignore errors } } return true } // helper that retrieves an array of nodes that were // potentially updated during the reify process, in order // to limit the number of nodes to check and update, only // select nodes from the inventory that are direct deps // of a given package.json (project root or a workspace) // and in ase of using a list of `names`, restrict nodes // to only names that are found in this list const retrieveUpdatedNodes = names => { const filterDirectDependencies = node => !node.isRoot && node.resolveParent && node.resolveParent.isRoot && (!names || names.includes(node.name)) && exactVersion(node) // skip update for exact ranges const directDeps = this.idealTree.inventory .filter(filterDirectDependencies) // traverses the list of direct dependencies and collect all nodes // to be updated, since any of them might have changed during reify const nodes = [] for (const node of directDeps) { for (const edgeIn of node.edgesIn) { nodes.push({ name: node.name, tree: edgeIn.from.target, }) } } return nodes } if (save) { // when using update all alongside with save, we'll make // sure to refresh every dependency of the root idealTree if (this[_updateAll]) { const nodes = retrieveUpdatedNodes() updateNodes(nodes) } else { // resolvedAdd is the list of user add requests, but with names added // to things like git repos and tarball file/urls. However, if the // user requested 'foo@', and we have a foo@file:../foo, then we should // end up saving the spec we actually used, not whatever they gave us. if (this[_resolvedAdd].length) { updateNodes(this[_resolvedAdd]) } // if updating given dependencies by name, restrict the list of // nodes to check to only those currently in _updateNames if (this[_updateNames].length) { const nodes = retrieveUpdatedNodes(this[_updateNames]) updateNodes(nodes) } // grab any from explicitRequests that had deps removed for (const { from: tree } of this.explicitRequests) { updatedTrees.add(tree) } } } if (save) { for (const tree of updatedTrees) { // refresh the edges so they have the correct specs tree.package = tree.package const pkgJson = await PackageJson.load(tree.path, { create: true }) const { dependencies = {}, devDependencies = {}, optionalDependencies = {}, peerDependencies = {}, // bundleDependencies is not required by PackageJson like the other // fields here PackageJson also doesn't omit an empty array for this // field so defaulting this to an empty array would add that field to // every package.json file. bundleDependencies, } = tree.package pkgJson.update({ dependencies, devDependencies, optionalDependencies, peerDependencies, bundleDependencies, }) await pkgJson.save() } } // before now edge specs could be changing, affecting the `requires` field // in the package lock, so we hold off saving to the very last action if (this.options.usePackageLock) { // preserve indentation, if possible let format = this.idealTree.package[Symbol.for('indent')] if (format === undefined) { format = ' ' } // TODO this ignores options.save await this.idealTree.meta.save({ format: (this.options.formatPackageLock && format) ? format : this.options.formatPackageLock, }) } timeEnd() return true }}