1 var assert = require('assert')
2 var crypto = require('crypto')
3 var fs = require('graceful-fs')
4 var path = require('path')
5 var url = require('url')
7 var chownr = require('chownr')
8 var dezalgo = require('dezalgo')
9 var hostedFromURL = require('hosted-git-info').fromUrl
10 var inflight = require('inflight')
11 var log = require('npmlog')
12 var mkdir = require('mkdirp')
13 var normalizeGitUrl = require('normalize-git-url')
14 var npa = require('npm-package-arg')
15 var realizePackageSpecifier = require('realize-package-specifier')
17 var addLocal = require('./add-local.js')
18 var correctMkdir = require('../utils/correct-mkdir.js')
19 var git = require('../utils/git.js')
20 var npm = require('../npm.js')
21 var rm = require('../utils/gently-rm.js')
23 var remotes = path.resolve(npm.config.get('cache'), '_git-remotes')
24 var templates = path.join(remotes, '_templates')
26 var VALID_VARIABLES = [
36 module.exports = addRemoteGit
37 function addRemoteGit (uri, _cb) {
38 assert(typeof uri === 'string', 'must have git URL')
39 assert(typeof _cb === 'function', 'must have callback')
42 log.verbose('addRemoteGit', 'caching', uri)
44 // the URL comes in exactly as it was passed on the command line, or as
45 // normalized by normalize-package-data / read-package-json / read-installed,
46 // so figure out what to do with it using hosted-git-info
47 var parsed = hostedFromURL(uri)
49 // normalize GitHub syntax to org/repo (for now)
51 if (parsed.type === 'github' && parsed.default === 'shortcut') {
54 from = parsed.toString()
57 log.verbose('addRemoteGit', from, 'is a repository hosted by', parsed.type)
59 // prefer explicit URLs to pushing everything through shortcuts
60 if (parsed.default !== 'shortcut') {
61 return tryClone(from, parsed.toString(), false, cb)
64 // try git:, then git+ssh:, then git+https: before failing
65 tryGitProto(from, parsed, cb)
67 // verify that this is a Git URL before continuing
69 if (parsed.type !== 'git') {
70 return cb(new Error(uri + 'is not a Git or GitHub URL'))
73 tryClone(parsed.rawSpec, uri, false, cb)
77 function tryGitProto (from, hostedInfo, cb) {
78 var gitURL = hostedInfo.git()
79 if (!gitURL) return trySSH(from, hostedInfo, cb)
81 log.silly('tryGitProto', 'attempting to clone', gitURL)
82 tryClone(from, gitURL, true, function (er) {
83 if (er) return tryHTTPS(from, hostedInfo, cb)
85 cb.apply(this, arguments)
89 function tryHTTPS (from, hostedInfo, cb) {
90 var httpsURL = hostedInfo.https()
92 return cb(new Error(from + ' can not be cloned via Git, SSH, or HTTPS'))
95 log.silly('tryHTTPS', 'attempting to clone', httpsURL)
96 tryClone(from, httpsURL, true, function (er) {
97 if (er) return trySSH(from, hostedInfo, cb)
99 cb.apply(this, arguments)
103 function trySSH (from, hostedInfo, cb) {
104 var sshURL = hostedInfo.ssh()
105 if (!sshURL) return tryHTTPS(from, hostedInfo, cb)
107 log.silly('trySSH', 'attempting to clone', sshURL)
108 tryClone(from, sshURL, false, cb)
111 function tryClone (from, combinedURL, silent, cb) {
112 log.silly('tryClone', 'cloning', from, 'via', combinedURL)
114 var normalized = normalizeGitUrl(combinedURL)
115 var cloneURL = normalized.url
116 var treeish = normalized.branch
118 // ensure that similarly-named remotes don't collide
119 var repoID = cloneURL.replace(/[^a-zA-Z0-9]+/g, '-') + '-' +
120 crypto.createHash('sha1').update(combinedURL).digest('hex').slice(0, 8)
121 var cachedRemote = path.join(remotes, repoID)
123 cb = inflight(repoID, cb)
125 return log.verbose('tryClone', repoID, 'already in flight; waiting')
127 log.verbose('tryClone', repoID, 'not in flight; caching')
129 // initialize the remotes cache with the correct perms
130 getGitDir(function (er) {
131 if (er) return cb(er)
132 fs.stat(cachedRemote, function (er, s) {
133 if (er) return mirrorRemote(from, cloneURL, treeish, cachedRemote, silent, finish)
134 if (!s.isDirectory()) return resetRemote(from, cloneURL, treeish, cachedRemote, finish)
136 validateExistingRemote(from, cloneURL, treeish, cachedRemote, finish)
139 // always set permissions on the cached remote
140 function finish (er, data) {
141 if (er) return cb(er, data)
142 addModeRecursive(cachedRemote, npm.modes.file, function (er) {
149 // don't try too hard to hold on to a remote
150 function resetRemote (from, cloneURL, treeish, cachedRemote, cb) {
151 log.info('resetRemote', 'resetting', cachedRemote, 'for', from)
152 rm(cachedRemote, function (er) {
153 if (er) return cb(er)
154 mirrorRemote(from, cloneURL, treeish, cachedRemote, false, cb)
158 // reuse a cached remote when possible, but nuke it if it's in an
159 // inconsistent state
160 function validateExistingRemote (from, cloneURL, treeish, cachedRemote, cb) {
162 ['config', '--get', 'remote.origin.url'],
163 { cwd: cachedRemote, env: gitEnv() },
164 function (er, stdout, stderr) {
167 originURL = stdout.trim()
168 log.silly('validateExistingRemote', from, 'remote.origin.url:', originURL)
171 if (stderr) stderr = stderr.trim()
173 log.warn('addRemoteGit', from, 'resetting remote', cachedRemote, 'because of error:', stderr || er)
174 return resetRemote(from, cloneURL, treeish, cachedRemote, cb)
175 } else if (cloneURL !== originURL) {
179 'pre-existing cached repo', cachedRemote, 'points to', originURL, 'and not', cloneURL
181 return resetRemote(from, cloneURL, treeish, cachedRemote, cb)
184 log.verbose('validateExistingRemote', from, 'is updating existing cached remote', cachedRemote)
185 updateRemote(from, cloneURL, treeish, cachedRemote, cb)
190 // make a complete bare mirror of the remote repo
191 // NOTE: npm uses a blank template directory to prevent weird inconsistencies
192 // https://github.com/npm/npm/issues/5867
193 function mirrorRemote (from, cloneURL, treeish, cachedRemote, silent, cb) {
194 mkdir(cachedRemote, function (er) {
195 if (er) return cb(er)
199 '--template=' + templates,
201 cloneURL, cachedRemote
204 ['clone', '--template=' + templates, '--mirror', cloneURL, cachedRemote],
205 { cwd: cachedRemote, env: gitEnv() },
206 function (er, stdout, stderr) {
208 var combined = (stdout + '\n' + stderr).trim()
209 var command = 'git ' + args.join(' ') + ':'
211 log.verbose(command, combined)
213 log.error(command, combined)
217 log.verbose('mirrorRemote', from, 'git clone ' + cloneURL, stdout.trim())
218 setPermissions(from, cloneURL, treeish, cachedRemote, cb)
224 function setPermissions (from, cloneURL, treeish, cachedRemote, cb) {
225 if (process.platform === 'win32') {
226 log.verbose('setPermissions', from, 'skipping chownr on Windows')
227 resolveHead(from, cloneURL, treeish, cachedRemote, cb)
229 getGitDir(function (er, cs) {
231 log.error('setPermissions', from, 'could not get cache stat')
235 chownr(cachedRemote, cs.uid, cs.gid, function (er) {
239 'Failed to change git repository ownership under npm cache for',
245 log.verbose('setPermissions', from, 'set permissions on', cachedRemote)
246 resolveHead(from, cloneURL, treeish, cachedRemote, cb)
252 // always fetch the origin, even right after mirroring, because this way
253 // permissions will get set correctly
254 function updateRemote (from, cloneURL, treeish, cachedRemote, cb) {
256 ['fetch', '-a', 'origin'],
257 { cwd: cachedRemote, env: gitEnv() },
258 function (er, stdout, stderr) {
260 var combined = (stdout + '\n' + stderr).trim()
261 log.error('git fetch -a origin (' + cloneURL + ')', combined)
264 log.verbose('updateRemote', 'git fetch -a origin (' + cloneURL + ')', stdout.trim())
266 setPermissions(from, cloneURL, treeish, cachedRemote, cb)
271 // branches and tags are both symbolic labels that can be attached to different
272 // commits, so resolve the commit-ish to the current actual treeish the label
275 // important for shrinkwrap
276 function resolveHead (from, cloneURL, treeish, cachedRemote, cb) {
277 log.verbose('resolveHead', from, 'original treeish:', treeish)
278 var args = ['rev-list', '-n1', treeish]
281 { cwd: cachedRemote, env: gitEnv() },
282 function (er, stdout, stderr) {
284 log.error('git ' + args.join(' ') + ':', stderr)
288 var resolvedTreeish = stdout.trim()
289 log.silly('resolveHead', from, 'resolved treeish:', resolvedTreeish)
291 var resolvedURL = getResolved(cloneURL, resolvedTreeish)
294 'unable to clone ' + from + ' because git clone string ' +
295 cloneURL + ' is in a form npm can\'t handle'
298 log.verbose('resolveHead', from, 'resolved Git URL:', resolvedURL)
300 // generate a unique filename
301 var tmpdir = path.join(
303 'git-cache-' + crypto.pseudoRandomBytes(6).toString('hex'),
306 log.silly('resolveHead', 'Git working directory:', tmpdir)
308 mkdir(tmpdir, function (er) {
309 if (er) return cb(er)
311 cloneResolved(from, resolvedURL, resolvedTreeish, cachedRemote, tmpdir, cb)
317 // make a clone from the mirrored cache so we have a temporary directory in
318 // which we can check out the resolved treeish
319 function cloneResolved (from, resolvedURL, resolvedTreeish, cachedRemote, tmpdir, cb) {
320 var args = ['clone', cachedRemote, tmpdir]
323 { cwd: cachedRemote, env: gitEnv() },
324 function (er, stdout, stderr) {
325 stdout = (stdout + '\n' + stderr).trim()
327 log.error('git ' + args.join(' ') + ':', stderr)
330 log.verbose('cloneResolved', from, 'clone', stdout)
332 checkoutTreeish(from, resolvedURL, resolvedTreeish, tmpdir, cb)
337 // there is no safe way to do a one-step clone to a treeish that isn't
338 // guaranteed to be a branch, so explicitly check out the treeish once it's
340 function checkoutTreeish (from, resolvedURL, resolvedTreeish, tmpdir, cb) {
341 var args = ['checkout', resolvedTreeish]
344 { cwd: tmpdir, env: gitEnv() },
345 function (er, stdout, stderr) {
346 stdout = (stdout + '\n' + stderr).trim()
348 log.error('git ' + args.join(' ') + ':', stderr)
351 log.verbose('checkoutTreeish', from, 'checkout', stdout)
353 // convince addLocal that the checkout is a local dependency
354 realizePackageSpecifier(tmpdir, function (er, spec) {
356 log.error('addRemoteGit', 'Failed to map', tmpdir, 'to a package specifier')
360 // ensure pack logic is applied
361 // https://github.com/npm/npm/issues/6400
362 addLocal(spec, null, function (er, data) {
364 if (npm.config.get('save-exact')) {
365 log.verbose('addRemoteGit', 'data._from:', resolvedURL, '(save-exact)')
366 data._from = resolvedURL
368 log.verbose('addRemoteGit', 'data._from:', from)
372 log.verbose('addRemoteGit', 'data._resolved:', resolvedURL)
373 data._resolved = resolvedURL
383 function getGitDir (cb) {
384 correctMkdir(remotes, function (er, stats) {
385 if (er) return cb(er)
387 // We don't need global templates when cloning. Use an empty directory for
388 // the templates, creating it (and setting its permissions) if necessary.
389 mkdir(templates, function (er) {
390 if (er) return cb(er)
392 // Ensure that both the template and remotes directories have the correct
394 fs.chown(templates, stats.uid, stats.gid, function (er) {
403 // git responds to env vars in some weird ways in post-receive hooks
404 // so don't carry those along.
405 if (gitEnv_) return gitEnv_
407 // allow users to override npm's insistence on not prompting for
408 // passphrases, but default to just failing when credentials
410 gitEnv_ = { GIT_ASKPASS: 'echo' }
412 for (var k in process.env) {
413 if (!~VALID_VARIABLES.indexOf(k) && k.match(/^GIT/)) continue
414 gitEnv_[k] = process.env[k]
419 addRemoteGit.getResolved = getResolved
420 function getResolved (uri, treeish) {
421 // normalize hosted-git-info clone URLs back into regular URLs
422 // this will only work on URLs that hosted-git-info recognizes
423 // https://github.com/npm/npm/issues/7961
424 var rehydrated = hostedFromURL(uri)
425 if (rehydrated) uri = rehydrated.toString()
427 var parsed = url.parse(uri)
429 // Checks for known protocols:
430 // http:, https:, ssh:, and git:, with optional git+ prefix.
431 if (!parsed.protocol ||
432 !parsed.protocol.match(/^(((git\+)?(https?|ssh))|git|file):$/)) {
433 uri = 'git+ssh://' + uri
436 if (!/^git[+:]/.test(uri)) {
440 // Not all URIs are actually URIs, so use regex for the treeish.
441 return uri.replace(/(?:#.*)?$/, '#' + treeish)
444 // similar to chmodr except it add permissions rather than overwriting them
445 // adapted from https://github.com/isaacs/chmodr/blob/master/chmodr.js
446 function addModeRecursive (cachedRemote, mode, cb) {
447 fs.readdir(cachedRemote, function (er, children) {
448 // Any error other than ENOTDIR means it's not readable, or doesn't exist.
450 if (er && er.code !== 'ENOTDIR') return cb(er)
451 if (er || !children.length) return addMode(cachedRemote, mode, cb)
453 var len = children.length
455 children.forEach(function (child) {
456 addModeRecursive(path.resolve(cachedRemote, child), mode, then)
460 if (errState) return undefined
461 if (er) return cb(errState = er)
462 if (--len === 0) return addMode(cachedRemote, dirMode(mode), cb)
467 function addMode (cachedRemote, mode, cb) {
468 fs.stat(cachedRemote, function (er, stats) {
469 if (er) return cb(er)
470 mode = stats.mode | mode
471 fs.chmod(cachedRemote, mode, cb)
475 // taken from https://github.com/isaacs/chmodr/blob/master/chmodr.js
476 function dirMode (mode) {
477 if (mode & parseInt('0400', 8)) mode |= parseInt('0100', 8)
478 if (mode & parseInt('040', 8)) mode |= parseInt('010', 8)
479 if (mode & parseInt('04', 8)) mode |= parseInt('01', 8)