feat(bundler): extract, update, artifacts (#3058)

This completes the work of adding basic Ruby/Bundler support to Renovate. It will now find all Gemfiles in a repository, extract dependencies from them, look up results on Rubygems, and raise PRs if updates are found.

Closes #932

lib/config/definitions.js

@@ -1085,7 +1085,9 @@ const options = [
     stage: 'package',
     type: 'json',
     default: {
-      fileMatch: [],
+      enabled: false,
+      fileMatch: ['(^|/)Gemfile$'],
+      versionScheme: 'ruby',
     },
     mergeable: true,
   },

lib/manager/bundler/artifacts.js

@@ -1,19 +1,110 @@
+/* istanbul ignore file */
+
+const { exec } = require('child-process-promise');
+const fs = require('fs-extra');
+const upath = require('upath');
+
 module.exports = {
   getArtifacts,
 };
 
-/*
- *  The getArtifacts() function is optional and necessary only if it is necessary to update "artifacts"
- *  after updating package files. Artifacts are files such as lock files or checksum files.
- *  Usually this will require running a child process command to produce an update.
- */
-
 async function getArtifacts(
   packageFileName,
   updatedDeps,
   newPackageFileContent,
   config
 ) {
-  await logger.debug({ config }, `composer.getArtifacts(${packageFileName})`);
-  return null;
+  logger.debug(`bundler.getArtifacts(${packageFileName})`);
+  const lockFileName = packageFileName + '.lock';
+  const existingLockFileContent = await platform.getFile(lockFileName);
+  if (!existingLockFileContent) {
+    logger.debug('No Gemfile.lock found');
+    return null;
+  }
+  const cwd = upath.join(config.localDir, upath.dirname(packageFileName));
+  let stdout;
+  let stderr;
+  try {
+    const localPackageFileName = upath.join(config.localDir, packageFileName);
+    await fs.outputFile(localPackageFileName, newPackageFileContent);
+    const localLockFileName = upath.join(config.localDir, lockFileName);
+    if (!config.gitFs) {
+      await fs.outputFile(localLockFileName, existingLockFileContent);
+      const fileList = await platform.getFileList();
+      const gemspecs = fileList.filter(file => file.endsWith('.gemspec'));
+      for (const gemspec of gemspecs) {
+        const content = await platform.getFile(gemspec);
+        await fs.outputFile(upath.join(config.localDir, gemspec), content);
+      }
+    }
+    const env =
+      global.trustLevel === 'high'
+        ? process.env
+        : {
+            HOME: process.env.HOME,
+            PATH: process.env.PATH,
+          };
+    const startTime = process.hrtime();
+    let cmd;
+    if (config.binarySource === 'docker') {
+      logger.info('Running bundler via docker');
+      cmd = `docker run --rm `;
+      const volumes = [config.localDir];
+      cmd += volumes.map(v => `-v ${v}:${v} `).join('');
+      const envVars = [];
+      cmd += envVars.map(e => `-e ${e} `);
+      cmd += `-w ${cwd} `;
+      cmd += `renovate/bundler bundler`;
+    } else {
+      logger.info('Running bundler via global bundler');
+      cmd = 'bundler';
+    }
+    const args = 'lock';
+    logger.debug({ cmd, args }, 'bundler command');
+    ({ stdout, stderr } = await exec(`${cmd} ${args}`, {
+      cwd,
+      shell: true,
+      env,
+    }));
+    const duration = process.hrtime(startTime);
+    const seconds = Math.round(duration[0] + duration[1] / 1e9);
+    logger.info(
+      { seconds, type: 'Gemfile.lock', stdout, stderr },
+      'Generated lockfile'
+    );
+    // istanbul ignore if
+    if (config.gitFs) {
+      const status = await platform.getRepoStatus();
+      if (!status.modified.includes(lockFileName)) {
+        return null;
+      }
+    } else {
+      const newLockFileContent = await fs.readFile(localLockFileName, 'utf8');
+
+      if (newLockFileContent === existingLockFileContent) {
+        logger.debug('Gemfile.lock is unchanged');
+        return null;
+      }
+    }
+    logger.debug('Returning updated Gemfile.lock');
+    return {
+      file: {
+        name: lockFileName,
+        contents: await fs.readFile(localLockFileName, 'utf8'),
+      },
+    };
+  } catch (err) {
+    if (
+      err.stdout &&
+      err.stdout.includes('No such file or directory') &&
+      !config.gitFs
+    ) {
+      throw new Error('bundler-fs');
+    }
+    logger.info(
+      { err, message: err.message },
+      'Failed to generate bundler.lock (unknown error)'
+    );
+    throw new Error('bundler-unknown');
+  }
 }

lib/manager/bundler/extract.js

@@ -1,18 +1,143 @@
+const { isValid } = require('../../versioning/ruby');
+
 module.exports = {
   extractPackageFile,
 };
 
-/*
- * The extractPackageFile() function is mandatory unless extractAllPackageFiles() is used instead.
- *
- * Use extractPackageFile() if it is OK to parse/extract package files in parallel independently.
- *
- * Here are examples of when extractAllPackageFiles has been necessary to be used instead:
- *  - for npm/yarn/lerna, "monorepos" can have links between package files and logic requiring us to selectively ignore "internal" dependencies within the same repository
- *  - for gradle, we use a third party CLI tool to extract all dependencies at once and so it should not be called independently on each package file separately
- */
-
-function extractPackageFile(content, fileName) {
-  logger.trace(`bundler.extractPackageFile(${fileName})`);
-  return null;
+function extractPackageFile(content) {
+  const res = {
+    registryUrls: [],
+    deps: [],
+  };
+  const lines = content.split('\n');
+  for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
+    const line = lines[lineNumber];
+    const sourceMatch = line.match(/^source "([^"]+)"\s*$/);
+    if (sourceMatch) {
+      res.registryUrls.push(sourceMatch[1]);
+    }
+    const gemMatch = line.match(/^gem "([^"]+)"(,\s+"([^"]+)"){0,2}/);
+    if (gemMatch) {
+      const dep = {
+        depName: gemMatch[1],
+        lineNumber,
+      };
+      if (gemMatch[3]) {
+        dep.currentValue = gemMatch[0]
+          .substring(`gem "${dep.depName}",`.length)
+          .replace(/"/g, '')
+          .trim();
+        if (!isValid(dep.currentValue)) {
+          dep.skipReason = 'invalid-value';
+        }
+      } else {
+        dep.skipReason = 'no-version';
+      }
+      if (!dep.skipReason) {
+        dep.purl = 'pkg:rubygems/' + dep.depName;
+      }
+      res.deps.push(dep);
+    }
+    const groupMatch = line.match(/^group\s+(.*?)\s+do/);
+    if (groupMatch) {
+      const depTypes = groupMatch[1]
+        .split(',')
+        .map(group => group.trim())
+        .map(group => group.replace(/^:/, ''));
+      const groupLineNumber = lineNumber;
+      let groupContent = '';
+      let groupLine = '';
+      while (lineNumber < lines.length && groupLine !== 'end') {
+        lineNumber += 1;
+        groupLine = lines[lineNumber];
+        if (groupLine !== 'end') {
+          groupContent += groupLine.replace(/^ {2}/, '') + '\n';
+        }
+      }
+      const groupRes = extractPackageFile(groupContent);
+      if (groupRes) {
+        res.deps = res.deps.concat(
+          groupRes.deps.map(dep => ({
+            ...dep,
+            depTypes,
+            lineNumber: dep.lineNumber + groupLineNumber + 1,
+          }))
+        );
+      }
+    }
+    const sourceBlockMatch = line.match(/^source\s+"(.*?)"\s+do/);
+    if (sourceBlockMatch) {
+      const repositoryUrl = sourceBlockMatch[1];
+      const sourceLineNumber = lineNumber;
+      let sourceContent = '';
+      let sourceLine = '';
+      while (lineNumber < lines.length && sourceLine !== 'end') {
+        lineNumber += 1;
+        sourceLine = lines[lineNumber];
+        if (sourceLine !== 'end') {
+          sourceContent += sourceLine.replace(/^ {2}/, '') + '\n';
+        }
+      }
+      const sourceRes = extractPackageFile(sourceContent);
+      if (sourceRes) {
+        res.deps = res.deps.concat(
+          sourceRes.deps.map(dep => ({
+            ...dep,
+            registryUrls: [repositoryUrl],
+            lineNumber: dep.lineNumber + sourceLineNumber + 1,
+          }))
+        );
+      }
+    }
+    const platformsMatch = line.match(/^platforms\s+(.*?)\s+do/);
+    if (platformsMatch) {
+      const platformsLineNumber = lineNumber;
+      let platformsContent = '';
+      let platformsLine = '';
+      while (lineNumber < lines.length && platformsLine !== 'end') {
+        lineNumber += 1;
+        platformsLine = lines[lineNumber];
+        if (platformsLine !== 'end') {
+          platformsContent += platformsLine.replace(/^ {2}/, '') + '\n';
+        }
+      }
+      const platformsRes = extractPackageFile(platformsContent);
+      if (platformsRes) {
+        res.deps = res.deps.concat(
+          // eslint-disable-next-line no-loop-func
+          platformsRes.deps.map(dep => ({
+            ...dep,
+            lineNumber: dep.lineNumber + platformsLineNumber + 1,
+          }))
+        );
+      }
+    }
+    const ifMatch = line.match(/^if\s+(.*?)/);
+    if (ifMatch) {
+      const ifLineNumber = lineNumber;
+      let ifContent = '';
+      let ifLine = '';
+      while (lineNumber < lines.length && ifLine !== 'end') {
+        lineNumber += 1;
+        ifLine = lines[lineNumber];
+        if (ifLine !== 'end') {
+          ifContent += ifLine.replace(/^ {2}/, '') + '\n';
+        }
+      }
+      const ifRes = extractPackageFile(ifContent);
+      if (ifRes) {
+        res.deps = res.deps.concat(
+          // eslint-disable-next-line no-loop-func
+          ifRes.deps.map(dep => ({
+            ...dep,
+            lineNumber: dep.lineNumber + ifLineNumber + 1,
+          }))
+        );
+      }
+    }
+  }
+  if (!res.deps.length && !res.registryUrls.length) {
+    return null;
+  }
+  return res;
 }

lib/manager/bundler/update.js

@@ -9,7 +9,29 @@ module.exports = {
  */
 
 function updateDependency(currentFileContent, upgrade) {
-  logger.debug({ config: upgrade }, 'bundler.updateDependency()');
-  // TODO
-  return currentFileContent;
+  try {
+    const lines = currentFileContent.split('\n');
+    const lineToChange = lines[upgrade.lineNumber];
+    if (!lineToChange.includes(upgrade.depName)) {
+      logger.debug('No gem match on line');
+      return null;
+    }
+    const newValue = upgrade.newValue
+      .split(',')
+      .map(part => `, "${part.trim()}"`)
+      .join('');
+    const newLine = lineToChange.replace(
+      /(gem "[^"]+")(,\s+"[^"]+"){0,2}/,
+      `$1${newValue}`
+    );
+    if (newLine === lineToChange) {
+      logger.debug('No changes necessary');
+      return currentFileContent;
+    }
+    lines[upgrade.lineNumber] = newLine;
+    return lines.join('\n');
+  } catch (err) {
+    logger.info({ err }, 'Error setting new Gemfile value');
+    return null;
+  }
 }

lib/workers/branch/index.js

@@ -323,6 +323,12 @@ async function processBranch(branchConfig, prHourlyLimitReached, packageFiles) {
     }
     if (err.message === 'update-failure') {
       logger.warn('Error updating branch: update failure');
+    } else if (err.message === 'bundler-fs') {
+      logger.warn(
+        'It is necessary to run Renovate in gitFs mode - contact your bot administrator'
+      );
+    } else if (err.message === 'bundler-unknown') {
+      logger.warn('Unknown bundler error');
     } else if (
       err.message !== 'registry-failure' &&
       err.message !== 'platform-failure'