Automatically detecting smart Git hosts

lib/core/resolvers/GitHubResolver.js

@@ -1,6 +1,7 @@
 var util = require('util');
 var path = require('path');
 var mout = require('mout');
+var Q = require('q');
 var GitRemoteResolver = require('./GitRemoteResolver');
 var download = require('../../util/download');
 var extract = require('../../util/extract');
@@ -37,7 +38,9 @@ function GitHubResolver(decEndpoint, config, logger) {
     }
 
     // Enable shallow clones for GitHub repos
-    this._shallowClone = true;
+    this._shallowClone = function() {
+        return Q.resolve(true);
+    };
 }
 
 util.inherits(GitHubResolver, GitRemoteResolver);

lib/core/resolvers/GitRemoteResolver.js

@@ -26,8 +26,10 @@ function GitRemoteResolver(decEndpoint, config, logger) {
         this._host = url.parse(this._source).host;
     }
 
-    // Disable shallow clones
-    this._shallowClone = false;
+    this._remote = url.parse(this._source);
+
+    // Verify whether the server supports shallow cloning
+    this._shallowClone = this._supportsShallowCloning;
 }
 
 util.inherits(GitRemoteResolver, GitResolver);
@@ -115,34 +117,36 @@ GitRemoteResolver.prototype._fastClone = function (resolution) {
     branch = resolution.tag || resolution.branch;
     args = ['clone',  this._source, '-b', branch, '--progress', '.'];
 
-    // If the host does not support shallow clones, we don't use --depth=1
-    if (this._shallowClone && !GitRemoteResolver._noShallow.get(this._host)) {
-        args.push('--depth', 1);
-    }
-
-    return cmd('git', args, { cwd: this._tempDir })
-    .spread(function (stdout, stderr) {
-        // Only after 1.7.10 --branch accepts tags
-        // Detect those cases and inform the user to update git otherwise it's
-        // a lot slower than newer versions
-        if (!/branch .+? not found/i.test(stderr)) {
-            return;
+    return this._shallowClone().then(function (shallowCloningSupported) {
+        // If the host does not support shallow clones, we don't use --depth=1
+        if (shallowCloningSupported && !GitRemoteResolver._noShallow.get(this._host)) {
+            args.push('--depth', 1);
         }
 
-        that._logger.warn('old-git', 'It seems you are using an old version of git, it will be slower and propitious to errors!');
-        return cmd('git', ['checkout', resolution.commit], { cwd: that._tempDir });
-    }, function (err) {
-        // Some git servers do not support shallow clones
-        // When that happens, we mark this host and try again
-        if (!GitRemoteResolver._noShallow.has(that._source) &&
-            err.details &&
-            /(rpc failed|shallow|--depth)/i.test(err.details)
-        ) {
-            GitRemoteResolver._noShallow.set(that._host, true);
-            return that._fastClone(resolution);
-        }
+        return cmd('git', args, { cwd: that._tempDir })
+        .spread(function (stdout, stderr) {
+            // Only after 1.7.10 --branch accepts tags
+            // Detect those cases and inform the user to update git otherwise it's
+            // a lot slower than newer versions
+            if (!/branch .+? not found/i.test(stderr)) {
+                return;
+            }
 
-        throw err;
+            that._logger.warn('old-git', 'It seems you are using an old version of git, it will be slower and propitious to errors!');
+            return cmd('git', ['checkout', resolution.commit], { cwd: that._tempDir });
+        }, function (err) {
+            // Some git servers do not support shallow clones
+            // When that happens, we mark this host and try again
+            if (!GitRemoteResolver._noShallow.has(that._source) &&
+                err.details &&
+                /(rpc failed|shallow|--depth)/i.test(err.details)
+                ) {
+                GitRemoteResolver._noShallow.set(that._host, true);
+                return that._fastClone(resolution);
+            }
+
+            throw err;
+        });
     });
 };
 
@@ -160,6 +164,74 @@ GitRemoteResolver.prototype._suggestProxyWorkaround = function (err) {
     }
 };
 
+// Verifies whether the server supports shallow cloning.
+// This is done according to the rules found in the following links:
+// * https://github.com/dimitri/el-get/pull/1921/files
+// * http://stackoverflow.com/questions/9270488/is-it-possible-to-detect-whether-a-http-git-remote-is-smart-or-dumb
+//
+// Summary of the rules:
+// * Protocols like ssh or git always support shallow cloning
+// * HTTP-based protocols can be verified by sending a HEAD or GET request to the URI (appended to the URL of the Git repo):
+//       /info/refs?service=git-upload-pack
+// * If the server responds with a 'Content-Type' header of 'application/x-git-upload-pack-advertisement',
+//      the server supports shallow cloning ("smart server")
+// * If the server responds with a different content type, the server does not support shallow cloning ("dumb server")
+// * Instead of doing the HEAD or GET request using an HTTP client, we're letting Git and Curl do the heavy lifting.
+//      Calling Git with the GIT_CURL_VERBOSE=2 env variable will provide the Git and Curl output, which includes
+//      the content type. This has the advantage that Git will take care of using stored credentials and any additional
+//      negotiation that needs to take place.
+//
+// The above should cover most cases, including BitBucket.
+GitRemoteResolver.prototype._supportsShallowCloning = function () {
+    var value = true;
+
+    // Verify that the remote could be parsed and that a protocol is set
+    // This case is unlikely, but let's still cover it.
+    if (this._remote == null || this._remote.protocol == null) {
+        return Q.resolve(false);
+    }
+
+    // Check for protocol - the remote check for hosts supporting shallow cloning is only required for
+    // HTTP or HTTPS, not for Git or SSH.
+    // Also check for hosts that have been checked in a previous request and have been found to support
+    // shallow cloning.
+    if (mout.string.startsWith(this._remote.protocol, 'http')
+            && !GitRemoteResolver._canShallow.get(this._host)) {
+        // Provide GIT_CURL_VERBOSE=2 environment variable to capture curl output.
+        // Calling ls-remote includes a call to the git-upload-pack service, which returns the content type in the response.
+        var processEnv = mout.object.merge(process.env, { 'GIT_CURL_VERBOSE': 2 });
+
+        value = cmd('git', ['ls-remote', '--heads', this._source], {
+            env: processEnv
+        })
+        .spread(function (stdout, stderr) {
+            // Check stderr for content-type, ignore stdout
+            var isSmartServer;
+
+            // If the content type is 'x-git', then the server supports shallow cloning
+            isSmartServer = mout.string.contains(stderr,
+                'Content-Type: application/x-git-upload-pack-advertisement');
+
+            this._logger.debug('detect-smart-git', 'Smart Git host detected: ' + isSmartServer);
+
+            if (isSmartServer) {
+                // Cache this host
+                GitRemoteResolver._canShallow.set(this._host, true);
+            }
+
+            return isSmartServer;
+        }.bind(this));
+    }
+    else {
+        // One of the following cases:
+        // * A non-HTTP/HTTPS protocol
+        // * A host that has been checked before and that supports shallow cloning
+        return Q.resolve(true);
+    }
+
+    return value;
+};
+
 // ------------------------------
 
 // Grab refs remotely
@@ -198,4 +270,7 @@ GitRemoteResolver.refs = function (source) {
 // Store hosts that do not support shallow clones here
 GitRemoteResolver._noShallow = new LRU({ max: 50, maxAge: 5 * 60 * 1000 });
 
+// Store hosts that support shallow clones here
+GitRemoteResolver._canShallow = new LRU({ max: 50, maxAge: 5 * 60 * 1000 });
+
 module.exports = GitRemoteResolver;