File Explorer

/var/lang/lib/node_modules/npm/node_modules/@npmcli/arborist/lib/arborist

This explorer reads the filesystem of the server it runs on, so /workspace/user isn't present here. Browsing and the terminal still work against this server's own disk from /.

isolated-reifier.js20.4 KB · 514 lines
const { mkdirSync } = require('node:fs')const pacote = require('pacote')const { join } = require('node:path')const { depth } = require('treeverse')const crypto = require('node:crypto')const { IsolatedNode, IsolatedLink } = require('../isolated-classes.js')const nameFromFolder = require('@npmcli/name-from-folder') // generate short hash key based on the dependency tree starting at this nodeconst getKey = (startNode) => {  const deps = []  const branch = []  depth({    tree: startNode,    getChildren: node => node.dependencies,    visit: node => {      branch.push(`${node.packageName}@${node.version}`)      deps.push(`${branch.join('->')}::${node.resolved}`)    },    leave: () => {      branch.pop()    },  })  deps.sort()  // TODO these replaces were originally to deal with node 14 not supporting base64url and likely don't need to happen anymore  // Changing this is a pretty significant breaking change, but removing parts of the hash increases collision possibilities (even if slight).  const hash = crypto.createHash('shake256', { outputLength: 16 })    .update(deps.join(','))    .digest('base64')    .replace(/\+/g, '-')    .replace(/\//g, '_')    .replace(/=+$/m, '')  return `${startNode.packageName}@${startNode.version}-${hash}`} module.exports = cls => class IsolatedReifier extends cls {  #externalProxies = new Map()  #omit = new Set()  #rootDeclaredDeps = new Set()  #processedEdges = new Set()  #workspaceProxies = new Map()   #generateChild (node, location, pkg, isInStore, root, inBundle = false) {    const newChild = new IsolatedNode({      isInStore,      inBundle,      isRegistryDependency: node.isRegistryDependency,      isRootDependency: node.isRootDependency,      location,      name: node.packageName || node.name,      optional: node.optional,      package: pkg,      parent: root,      path: join(this.idealGraph.localPath, location),      resolved: node.resolved,      root,    })    // XXX top is from place-dep not lib/node.js    newChild.top = { path: this.idealGraph.localPath }    root.children.set(newChild.location, newChild)    root.inventory.set(newChild.location, newChild)  }   /**   * Create an ideal graph.   *   * An implementation of npm RFC-0042   * https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md   *   * This entire file should be considered technical debt that will be resolved with an Arborist refactor or rewrite.   * Embedded logic in Nodes and Links, and the incremental state of building trees and reifying contains too many assumptions to do a linked mode properly.   *   * Instead, this approach takes a tree built from build-ideal-tree, and returns a new tree-like structure without the embedded logic of Node and Link classes.   *   * Since the RFC requires leaving the package-lock in place, this approach temporarily replaces the tree state for a couple of steps of reifying.   *   **/  async makeIdealGraph () {    const idealTree = this.idealTree    this.#omit = new Set(this.options.omit)    const omit = this.#omit     // npm auto-creates 'workspace' edges from root to all workspaces.    // For isolated/linked mode, only include workspaces that root explicitly declares as dependencies.    // When omitting dep types, exclude those from the declared set so their workspaces aren't hoisted.    const rootPkg = idealTree.package    this.#rootDeclaredDeps = new Set(Object.keys(Object.assign({},      rootPkg.dependencies,      (!omit.has('dev') && rootPkg.devDependencies),      (!omit.has('optional') && rootPkg.optionalDependencies),      (!omit.has('peer') && rootPkg.peerDependencies)    )))     // XXX this sometimes acts like a node too    this.idealGraph = {      external: [],      isProjectRoot: true,      localLocation: idealTree.location,      localPath: idealTree.path,      path: idealTree.path,    }    this.counter = 0     // Skip extraneous fsChildren: workspaces removed from the root manifest can linger in fsChildren via the lockfile, and re-materializing them here would re-create a directory the user just deleted.    const fsChildren = Array.from(idealTree.fsChildren.values()).filter(w => !w.extraneous)    this.idealGraph.workspaces = await Promise.all(fsChildren.map(w => this.#workspaceProxy(w)))    const processed = new Set()    const queue = [idealTree, ...idealTree.fsChildren]    while (queue.length !== 0) {      const next = queue.pop()      if (processed.has(next.location)) {        continue      }      processed.add(next.location)      next.edgesOut.forEach(edge => {        if (edge.to && !(next.package.bundleDependencies || next.package.bundledDependencies || []).includes(edge.to.name) && !edge.to.shouldOmit?.(omit)) {          queue.push(edge.to)        }      })      // local `file:` deps are in fsChildren but are not workspaces.      // they are already handled as workspace-like proxies above and should not go through the external/store extraction path.      // Links with file: resolved paths (from `npm link`) should also be treated as local dependencies and symlinked directly instead of being extracted into the store.      const isLocalFileDep = next.isLink && next.resolved?.startsWith('file:')      if (isLocalFileDep && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {        this.idealGraph.workspaces.push(await this.#workspaceProxy(next.target))      } else if (!next.isProjectRoot && !next.isWorkspace && !next.inert && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {        this.idealGraph.external.push(await this.#externalProxy(next))      }    }     await this.#assignCommonProperties(idealTree, this.idealGraph)  }   async #workspaceProxy (node) {    if (this.#workspaceProxies.has(node)) {      return this.#workspaceProxies.get(node)    }    const result = {}    // XXX this goes recursive if we don't set here because assignCommonProperties also calls this.#workspaceProxy    this.#workspaceProxies.set(node, result)    result.localLocation = node.location    result.localPath = node.path    result.isWorkspace = true    result.resolved = node.resolved    await this.#assignCommonProperties(node, result)    return result  }   async #externalProxy (node) {    if (this.#externalProxies.has(node)) {      return this.#externalProxies.get(node)    }    const result = {}    // XXX this goes recursive if we don't set here because assignCommonProperties also calls this.#externalProxy    this.#externalProxies.set(node, result)    await this.#assignCommonProperties(node, result, !node.hasShrinkwrap)    if (node.hasShrinkwrap) {      // strip any path traversal from package.json name fields before they hit path.join below      /* istanbul ignore next - packageName is always set for real packages */      const safeName = nameFromFolder(node.packageName || node.path)      const dir = join(        node.root.path,        'node_modules',        '.store',        `${safeName}@${node.version}`      )      mkdirSync(dir, { recursive: true })      // TODO this approach feels wrong and shouldn't be necessary for shrinkwraps      await pacote.extract(node.resolved, dir, {        ...this.options,        resolved: node.resolved,        integrity: node.integrity,        // TODO _isRoot      })      const Arborist = this.constructor      const arb = new Arborist({ ...this.options, path: dir })      // Make sure that the ideal tree is build as the rest of the algorithm depends on it.      await arb.buildIdealTree({        complete: false,        dev: false,      })      await arb.makeIdealGraph()      this.idealGraph.external.push(...arb.idealGraph.external)      for (const edge of arb.idealGraph.external) {        edge.root = this.idealGraph        edge.id = `${node.id}=>${edge.id}`      }      result.localDependencies = []      result.externalDependencies = arb.idealGraph.externalDependencies      result.externalOptionalDependencies = arb.idealGraph.externalOptionalDependencies      result.dependencies = [        ...result.externalDependencies,        ...result.externalOptionalDependencies,      ]    }    result.optional = node.optional    result.resolved = node.resolved    result.version = node.version    // Carry the source node's registry-dependency flag so the store node retains it.    // IsolatedNode has no edges to recompute it from, and reify's registry-tarball allow-remote exemption depends on it.    result.isRegistryDependency = node.isRegistryDependency    // Same reasoning for allow-remote=root: the store node has no edgesIn, so capture from the source node whether it satisfies a valid edge from the project root or a workspace.    result.isRootDependency = [...node.edgesIn].some(e =>      e.valid && (e.from?.isProjectRoot || e.from?.isWorkspace)    )    return result  }   async #assignCommonProperties (node, result, populateDeps = true) {    result.root = this.idealGraph    // XXX does anything need this?    result.id = this.counter++    /* istanbul ignore next - packageName is always set for real packages */    result.name = result.isWorkspace ? (node.packageName || node.name) : node.name    // strip any path traversal from package.json name fields before they hit path.join below    result.packageName = nameFromFolder(node.packageName || node.path)    result.package = { ...node.package }    result.package.bundleDependencies = undefined     if (!populateDeps) {      return    }     let edges = [...node.edgesOut.values()].filter(edge =>      edge.to?.target &&      !(node.package.bundledDependencies || node.package.bundleDependencies)?.includes(edge.to.name)    )     // Only omit edge types for root and workspace nodes (matching shouldOmit scope)    if ((node.isProjectRoot || node.isWorkspace) && this.#omit.size) {      edges = edges.filter(edge => {        if (edge.dev && this.#omit.has('dev')) {          return false        }        if (edge.optional && this.#omit.has('optional')) {          return false        }        if (edge.peer && this.#omit.has('peer')) {          return false        }        return true      })    }     let nonOptionalDeps = edges.filter(edge => !edge.optional).map(edge => edge.to.target)     // npm auto-creates 'workspace' edges from root to all workspaces.    // For isolated/linked mode, only include workspaces that root explicitly declares as dependencies.    if (node.isProjectRoot) {      nonOptionalDeps = nonOptionalDeps.filter(n => !n.isWorkspace || this.#rootDeclaredDeps.has(n.packageName))    }     // When legacyPeerDeps is enabled, peer dep edges are not created on the node.    // Resolve them from the tree so they get symlinked in the store.    const peerDeps = node.package.peerDependencies    if (peerDeps && node.legacyPeerDeps) {      const edgeNames = new Set(edges.map(edge => edge.name))      for (const peerName in peerDeps) {        if (!edgeNames.has(peerName)) {          const resolved = node.resolve(peerName)          if (resolved && resolved !== node && !resolved.inert) {            nonOptionalDeps.push(resolved.target)          }        }      }    }     // local `file:` deps (non-workspace fsChildren) should be treated as local dependencies, not external, so they get symlinked directly instead of being extracted into the store.    // A file: dep surfaces as a Link edge whose resolved spec starts with file:; detect it from the edge so the target is treated as local even when it is absent from idealTree.fsChildren (a workspace consumer, or a target outside the repo root via npm link).    const fileLinkTargets = new Set(edges.filter(e => e.to?.isLink && e.to.resolved?.startsWith('file:')).map(e => e.to.target))    const isLocal = (n) => n.isWorkspace || node.fsChildren?.has(n) || fileLinkTargets.has(n)    const optionalDeps = edges.filter(edge => edge.optional).map(edge => edge.to.target)     // Optional peers declared only in peerDependenciesMeta (e.g. `@types/react`) have no edge, so the materialization above misses them.    // Resolve each from the tree and link it; if nobody provides it, node.resolve finds nothing and it stays omitted.    const peerMeta = node.package.peerDependenciesMeta    if (peerMeta) {      const resolvedNames = new Set([...nonOptionalDeps, ...optionalDeps].map(n => n.name))      for (const peerName in peerMeta) {        if (!peerMeta[peerName]?.optional || resolvedNames.has(peerName)) {          continue        }        const resolved = node.resolve(peerName)?.target        if (resolved && resolved !== node && !resolved.inert && !isLocal(resolved)) {          optionalDeps.push(resolved)        }      }    }    result.localDependencies = await Promise.all(nonOptionalDeps.filter(isLocal).map(n => this.#workspaceProxy(n)))    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !isLocal(n) && !n.inert).map(n => this.#externalProxy(n)))    result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.inert).map(n => this.#externalProxy(n)))    result.dependencies = [      ...result.externalDependencies,      ...result.localDependencies,      ...result.externalOptionalDependencies,    ]  }   async #createBundledTree () {    // TODO: make sure that idealTree object exists    const idealTree = this.idealTree    // TODO: test workspaces having bundled deps    const queue = []     for (const [, edge] of idealTree.edgesOut) {      if (edge.to && (idealTree.package.bundleDependencies || idealTree.package.bundledDependencies || []).includes(edge.to.name)) {        queue.push({ from: idealTree, to: edge.to })      }    }    for (const child of idealTree.fsChildren) {      for (const [, edge] of child.edgesOut) {        if (edge.to && (child.package.bundleDependencies || child.package.bundledDependencies || []).includes(edge.to.name)) {          queue.push({ from: child, to: edge.to })        }      }    }     const processed = new Set()    const nodes = new Map()    const edges = []    while (queue.length !== 0) {      const nextEdge = queue.pop()      const key = `${nextEdge.from.location}=>${nextEdge.to.location}`      // should be impossible, unless bundled is duped      /* istanbul ignore next */      if (processed.has(key)) {        continue      }      processed.add(key)      const from = nextEdge.from      if (!from.isRoot && !from.isWorkspace) {        nodes.set(from.location, { location: from.location, resolved: from.resolved, name: from.name, optional: from.optional, pkg: { ...from.package, bundleDependencies: undefined } })      }      const to = nextEdge.to      nodes.set(to.location, { location: to.location, resolved: to.resolved, name: to.name, optional: to.optional, pkg: { ...to.package, bundleDependencies: undefined } })      edges.push({ from: from.isRoot ? 'root' : from.location, to: to.location })       to.edgesOut.forEach(edge => {        // an edge out should always have a to        /* istanbul ignore else */        if (edge.to) {          queue.push({ from: edge.from, to: edge.to })        }      })    }    return { edges, nodes }  }   async createIsolatedTree () {    await this.makeIdealGraph()    const bundledTree = await this.#createBundledTree()     const root = new IsolatedNode(this.idealGraph)    root.root = root    root.inventory.set('', root)    const processed = new Set()    for (const c of this.idealGraph.workspaces) {      const wsName = c.packageName      // XXX parent? root?      const workspace = new IsolatedNode({        location: c.localLocation,        name: wsName,        package: c.package,        path: c.localPath,        resolved: c.resolved,      })      root.fsChildren.add(workspace)      root.inventory.set(workspace.location, workspace)      root.workspaces.set(wsName, workspace.path)       // Declared workspaces are symlinked at root node_modules/.      // Undeclared workspaces get a tree-only Link kept for diff/filter participation but not materialized on disk.      const isDeclared = this.#rootDeclaredDeps.has(wsName)      const wsLink = new IsolatedLink({        location: isDeclared ? join('node_modules', wsName) : join(c.localLocation, 'node_modules', wsName),        name: wsName,        package: workspace.package,        parent: root,        path: isDeclared ? join(root.path, 'node_modules', wsName) : join(root.path, c.localLocation, 'node_modules', wsName),        realpath: workspace.path,        root,        target: workspace,      })      if (!isDeclared) {        wsLink.isUndeclaredWorkspaceLink = true      }      root.children.set(wsName, wsLink)      root.inventory.set(wsLink.location, wsLink)      workspace.linksIn.add(wsLink)    }     this.idealGraph.external.forEach(c => {      const key = getKey(c)      if (processed.has(key)) {        return      }      processed.add(key)      const location = join('node_modules', '.store', key, 'node_modules', c.packageName)      this.#generateChild(c, location, c.package, true, root)    })     bundledTree.nodes.forEach(node => {      this.#generateChild(node, node.location, node.pkg, false, root, true)    })     bundledTree.edges.forEach(edge => {      const from = edge.from === 'root' ? root : root.inventory.get(edge.from)      const to = root.inventory.get(edge.to)      // Maybe optional should be propagated from the original edge      const newEdge = { optional: false, from, to }      from.edgesOut.set(to.name, newEdge)      to.edgesIn.add(newEdge)    })     this.#processEdges(this.idealGraph, false, root)    for (const node of this.idealGraph.workspaces) {      this.#processEdges(node, false, root)    }    return root  }   #processEdges (node, externalEdge, root) {    const key = getKey(node)    if (this.#processedEdges.has(key)) {      return    }    this.#processedEdges.add(key)     let from, nmFolder    if (externalEdge) {      const fromLocation = join('node_modules', '.store', key, 'node_modules', node.packageName)      from = root.children.get(fromLocation)      nmFolder = join('node_modules', '.store', key, 'node_modules')    } else {      from = node.isProjectRoot ? root : root.inventory.get(node.localLocation)      nmFolder = join(node.localLocation, 'node_modules')    }    /* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */    if (!from) {      return    }     for (const dep of node.localDependencies) {      this.#processEdges(dep, false, root)      // nonOptional, local      this.#processDeps(dep, false, false, root, from, nmFolder)    }    for (const dep of node.externalDependencies) {      this.#processEdges(dep, true, root)      // nonOptional, external      this.#processDeps(dep, false, true, root, from, nmFolder)    }    for (const dep of node.externalOptionalDependencies) {      this.#processEdges(dep, true, root)      // optional, external      this.#processDeps(dep, true, true, root, from, nmFolder)    }  }   #processDeps (dep, optional, external, root, from, nmFolder) {    const toKey = getKey(dep)     let target    if (external) {      const toLocation = join('node_modules', '.store', toKey, 'node_modules', dep.packageName)      target = root.children.get(toLocation)    } else {      target = root.inventory.get(dep.localLocation)    }    // TODO: we should no-op is an edge has already been created with the same fromKey and toKey    /* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */    if (!target) {      return    }     if (dep.package.bin) {      for (const bn in dep.package.bin) {        target.binPaths.push(join(dep.root.localPath, nmFolder, '.bin', bn))      }    }     const pkg = {      _id: dep.package._id,      bin: target.package.bin,      bundleDependencies: undefined,      deprecated: undefined,      scripts: dep.package.scripts,      version: dep.package.version,    }    const link = new IsolatedLink({      isStoreLink: true,      location: join(nmFolder, dep.name),      name: toKey,      optional,      parent: root,      package: pkg,      path: join(dep.root.localPath, nmFolder, dep.name),      realpath: target.path,      resolved: external ? `file:.store/${toKey}/node_modules/${dep.packageName}` : dep.resolved,      root,      target,    })    // XXX top is from place-dep not lib/link.js    link.top = { path: dep.root.localPath }    const newEdge1 = { optional, from, to: link }    from.edgesOut.set(dep.name, newEdge1)    link.edgesIn.add(newEdge1)    const newEdge2 = { optional: false, from: link, to: target }    link.edgesOut.set(dep.name, newEdge2)    target.edgesIn.add(newEdge2)    root.children.set(link.location, link)  }}