Release v0.4.0 (#523) (#519) (#516) (#534) (#510) (#517) (#535)

* No await runner (#523) * cml-publish mime and repo (#519) * cml-publish mime and repo * cleanup * image/svg+xml * buffer * resolve * text buffer outputs plain * native bolean * snapshot * Fix pipe reading logic on Windows (#516) The fs.readSync function throws EOF instead of returning 0 like in other systems. See nodejs/node#35997 for more information. * minor doc fixes (#534) * cml-pr command (#510) * test-cml-pr * packages * pre-release * log isci * log isci * user_name * log gitlab url * log gitlab url * short 8 * len paths * no undef * git * files * files * files * files * files * files * files * files * files * files * files * files * files * files * files * files * files * sha2 * sha2 * log filepath * log filepath * log filepath * exists and clean * SHA vs SHA2 * pre-release * pre-release 2 * pre-release 3 * pre-release 3 * pre-release 3 * pre-release 3 * pre-release 3 * pre-release 3 * git back * git back * git back * release * REPO_TOKEN * html_url * cml pr defaults * cml pr defaults * test strict again * CI * vars * shorter params * snapshots and olivaw * test david * BB fix * snapshots * No git sha dependencies (#517) * check head_sha * 0.4.0 Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> Co-authored-by: Casper da Costa-Luis <work@cdcl.ml>
iterative · May 19, 2021 · d716d30 · d716d30
1 parent c0f3158
commit d716d30
Show file tree

Hide file tree

Showing 21 changed files with 619 additions and 504 deletions.
diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml
@@ -1,3 +1,4 @@
+name: GitLab
 on:
   workflow_dispatch:
   pull_request:

diff --git a/README.md b/README.md
@@ -93,7 +93,7 @@ jobs:
 ```
 
 We helpfully provide CML and other useful libraries pre-installed on our
-[custom Docker images](https://github.com/iterative/cml/blob/master/docker/Dockerfile).
+[custom Docker images](https://github.com/iterative/cml/blob/master/Dockerfile).
 In the above example, uncommenting the field
 `container: docker://dvcorg/cml:0-dvc2-base1-gpu` will make the GitHub Actions
 runner pull the CML Docker image. The image already has NodeJS, Python 3, DVC

diff --git a/bin/cml-pr.js b/bin/cml-pr.js
@@ -6,20 +6,27 @@ console.log = console.error;
 const yargs = require('yargs');
 const decamelize = require('decamelize-keys');
 
-const CML = require('../src/cml');
+const CML = require('../src/cml').default;
+const { GIT_REMOTE, GIT_USER_NAME, GIT_USER_EMAIL } = require('../src/cml');
 
 const run = async (opts) => {
   const globs = opts._.length ? opts._ : undefined;
   const cml = new CML(opts);
-  print(await cml.pr_create({ ...opts, globs }));
+  print((await cml.pr_create({ ...opts, globs })) || '');
 };
 
 const opts = decamelize(
   yargs
     .strict()
-    .usage('Usage: $0 <path to markdown file>')
+    .usage('Usage: $0 <glob path> ... <glob path>')
     .describe('md', 'Output in markdown format [](url).')
     .boolean('md')
+    .default('remote', GIT_REMOTE)
+    .describe('remote', 'Sets git remote.')
+    .default('user-email', GIT_USER_EMAIL)
+    .describe('user-email', 'Sets git user email.')
+    .default('user-name', GIT_USER_NAME)
+    .describe('user-name', 'Sets git user name.')
     .default('repo')
     .describe(
       'repo',

diff --git a/bin/cml-pr.test.js b/bin/cml-pr.test.js
@@ -0,0 +1,27 @@
+jest.setTimeout(200000);
+
+const { exec } = require('../src/utils');
+
+describe('CML e2e', () => {
+  test('cml-publish -h', async () => {
+    const output = await exec(`echo none | node ./bin/cml-pr.js -h`);
+
+    expect(output).toMatchInlineSnapshot(`
+      "Usage: cml-pr.js <glob path> ... <glob path>
+
+      Options:
+        --version     Show version number                                    [boolean]
+        --md          Output in markdown format [](url).                     [boolean]
+        --remote      Sets git remote.                             [default: \\"origin\\"]
+        --user-email  Sets git user email.            [default: \\"olivaw@iterative.ai\\"]
+        --user-name   Sets git user name.                     [default: \\"Olivaw[bot]\\"]
+        --repo        Specifies the repo to be used. If not specified is extracted
+                      from the CI ENV.
+        --token       Personal access token to be used. If not specified in extracted
+                      from ENV REPO_TOKEN.
+        --driver      If not specify it infers it from the ENV.
+                                                         [choices: \\"github\\", \\"gitlab\\"]
+        -h            Show help                                              [boolean]"
+    `);
+  });
+});
diff --git a/bin/cml-publish.js b/bin/cml-publish.js
@@ -6,28 +6,22 @@ console.log = console.error;
 const fs = require('fs').promises;
 const pipe_args = require('../src/pipe-args');
 const yargs = require('yargs');
+const decamelize = require('decamelize-keys');
 
-const CML = require('../src/cml');
+const CML = require('../src/cml').default;
 
 const run = async (opts) => {
-  const {
-    data,
-    file,
-    'gitlab-uploads': gitlab_uploads,
-    'rm-watermark': rm_watermark
-  } = opts;
+  const { data, file, repo, native } = opts;
 
   const path = opts._[0];
-  let buffer;
-  if (data) buffer = Buffer.from(data, 'binary');
+  const buffer = data ? Buffer.from(data, 'binary') : null;
+
+  const cml = new CML({ ...opts, repo: native ? repo : 'cml' });
 
-  const cml = new CML(opts);
   const output = await cml.publish({
     ...opts,
     buffer,
-    path,
-    gitlab_uploads,
-    rm_watermark
+    path
   });
 
   if (!file) print(output);
@@ -36,51 +30,53 @@ const run = async (opts) => {
 
 pipe_args.load('binary');
 const data = pipe_args.piped_arg();
-const argv = yargs
-  .strict()
-  .usage(`Usage: $0 <path to file>`)
-  .describe('md', 'Output in markdown format [title || name](url).')
-  .boolean('md')
-  .describe('md', 'Output in markdown format [title || name](url).')
-  .default('title')
-  .describe('title', 'Markdown title [title](url) or ![](url title).')
-  .alias('title', 't')
-  .boolean('gitlab-uploads')
-  .describe(
-    'gitlab-uploads',
-    'Uses GitLab uploads instead of CML storage. Use GitLab uploads to get around CML size limitations for hosting artifacts persistently. Only available for GitLab CI.'
-  )
-  .deprecateOption('gitlab-uploads', 'Use --native instead')
-  .boolean('native')
-  .describe(
-    'native',
-    "Uses driver's native capabilities to upload assets instead of CML's storage."
-  )
-  .boolean('rm-watermark')
-  .describe('rm-watermark', 'Avoid CML watermark.')
-  .default('file')
-  .describe(
-    'file',
-    'Append the output to the given file. Create it if does not exist.'
-  )
-  .alias('file', 'f')
-  .default('repo')
-  .describe(
-    'repo',
-    'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
-  )
-  .default('token')
-  .describe(
-    'token',
-    'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.'
-  )
-  .default('driver')
-  .choices('driver', ['github', 'gitlab'])
-  .describe('driver', 'If not specify it infers it from the ENV.')
-  .help('h')
-  .demand(data ? 0 : 1).argv;
+const opts = decamelize(
+  yargs
+    .strict()
+    .usage(`Usage: $0 <path to file>`)
+    .describe('md', 'Output in markdown format [title || name](url).')
+    .boolean('md')
+    .describe('md', 'Output in markdown format [title || name](url).')
+    .default('title')
+    .describe('title', 'Markdown title [title](url) or ![](url title).')
+    .alias('title', 't')
+    .boolean('native')
+    .describe(
+      'native',
+      "Uses driver's native capabilities to upload assets instead of CML's storage. Currently only available for GitLab CI."
+    )
+    .alias('native', 'gitlab-uploads')
+    .boolean('rm-watermark')
+    .describe('rm-watermark', 'Avoid CML watermark.')
+    .default('mime-type')
+    .describe(
+      'mime-type',
+      'Specifies the mime-type. If not set guess it from the content.'
+    )
+    .default('file')
+    .describe(
+      'file',
+      'Append the output to the given file. Create it if does not exist.'
+    )
+    .alias('file', 'f')
+    .default('repo')
+    .describe(
+      'repo',
+      'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+    )
+    .default('token')
+    .describe(
+      'token',
+      'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.'
+    )
+    .default('driver')
+    .choices('driver', ['github', 'gitlab'])
+    .describe('driver', 'If not specify it infers it from the ENV.')
+    .help('h')
+    .demand(data ? 0 : 1).argv
+);
 
-run({ ...argv, data }).catch((e) => {
+run({ ...opts, data }).catch((e) => {
   console.error(e);
   process.exit(1);
 });
diff --git a/bin/cml-publish.test.js b/bin/cml-publish.test.js
@@ -11,26 +11,26 @@ describe('CML e2e', () => {
       "Usage: cml-publish.js <path to file>
 
       Options:
-        --version         Show version number                                [boolean]
-        --md              Output in markdown format [title || name](url).    [boolean]
-        --title, -t       Markdown title [title](url) or ![](url title).
-        --gitlab-uploads  Uses GitLab uploads instead of CML storage. Use GitLab
-                          uploads to get around CML size limitations for hosting
-                          artifacts persistently. Only available for GitLab CI.
-                                          [deprecated: Use --native instead] [boolean]
-        --native          Uses driver's native capabilities to upload assets instead
-                          of CML's storage.                                  [boolean]
-        --rm-watermark    Avoid CML watermark.                               [boolean]
-        --file, -f        Append the output to the given file. Create it if does not
-                          exist.
-        --repo            Specifies the repo to be used. If not specified is extracted
-                          from the CI ENV.
-        --token           Personal access token to be used. If not specified,
-                          extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN,
-                          or BITBUCKET_TOKEN.
-        --driver          If not specify it infers it from the ENV.
+        --version                   Show version number                      [boolean]
+        --md                        Output in markdown format [title || name](url).
+                                                                             [boolean]
+        --title, -t                 Markdown title [title](url) or ![](url title).
+        --native, --gitlab-uploads  Uses driver's native capabilities to upload assets
+                                    instead of CML's storage. Currently only available
+                                    for GitLab CI.                           [boolean]
+        --rm-watermark              Avoid CML watermark.                     [boolean]
+        --mime-type                 Specifies the mime-type. If not set guess it from
+                                    the content.
+        --file, -f                  Append the output to the given file. Create it if
+                                    does not exist.
+        --repo                      Specifies the repo to be used. If not specified is
+                                    extracted from the CI ENV.
+        --token                     Personal access token to be used. If not
+                                    specified, extracted from ENV REPO_TOKEN,
+                                    GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.
+        --driver                    If not specify it infers it from the ENV.
                                                          [choices: \\"github\\", \\"gitlab\\"]
-        -h                Show help                                          [boolean]"
+        -h                          Show help                                [boolean]"
     `);
   });
 
@@ -97,6 +97,15 @@ describe('CML e2e', () => {
     await fs.promises.unlink(file);
   });
 
+  test('cml-publish assets/vega-lite.json', async () => {
+    const output = await exec(
+      `echo none | node ./bin/cml-publish.js --mime-type=application/json assets/vega-lite.json`
+    );
+
+    expect(output.startsWith('https://')).toBe(true);
+    expect(output.endsWith('json')).toBe(true);
+  });
+
   test('cml-publish assets/test.svg in Gitlab storage', async () => {
     const { TEST_GITLAB_REPO: repo, TEST_GITLAB_TOKEN: token } = process.env;
 
@@ -113,9 +122,10 @@ describe('CML e2e', () => {
     ).rejects.toThrowError('ENOENT: no such file or directory, stat');
   });
 
-  test('echo invalid | cml-publish produces buffer mime type error', async () => {
-    await expect(
-      exec(`echo invalid | node ./bin/cml-publish.js`)
-    ).rejects.toThrowError('Failed guessing mime type of buffer');
+  test('echo text | cml-publish produces a plain text file', async () => {
+    const output = await exec(`echo none | node ./bin/cml-publish.js`);
+
+    expect(output.startsWith('https://')).toBe(true);
+    expect(output.endsWith('plain')).toBe(true);
   });
 });
diff --git a/bin/cml-runner.js b/bin/cml-runner.js
@@ -9,7 +9,7 @@ const decamelize = require('decamelize-keys');
 
 const { exec, randid, sleep } = require('../src/utils');
 const tf = require('../src/terraform');
-const CML = require('../src/cml');
+const CML = require('../src/cml').default;
 
 const NAME = `cml-${randid()}`;
 const WORKDIR_BASE = `${homedir()}/.cml`;
@@ -193,11 +193,6 @@ const run_cloud = async (opts) => {
         }
 
         console.log(JSON.stringify(instance));
-
-        const {
-          attributes: { name }
-        } = instance;
-        await cml.await_runner({ name });
       }
     }
   }

diff --git a/bin/cml-send-comment.js b/bin/cml-send-comment.js
@@ -4,59 +4,50 @@ console.log = console.error;
 
 const fs = require('fs').promises;
 const yargs = require('yargs');
+const decamelize = require('decamelize-keys');
 
-const CML = require('../src/cml');
+const CML = require('../src/cml').default;
 
 const run = async (opts) => {
-  const {
-    'commit-sha': sha,
-    'head-sha': head_sha,
-    'rm-watermark': rm_watermark
-  } = opts;
   const path = opts._[0];
   const report = await fs.readFile(path, 'utf-8');
-
   const cml = new CML(opts);
-  await cml.comment_create({
-    report,
-    commit_sha: sha || head_sha,
-    rm_watermark
-  });
+  await cml.comment_create({ ...opts, report });
 };
 
-const argv = yargs
-  .strict()
-  .usage('Usage: $0 <path to markdown file>')
-  .default('commit-sha')
-  .describe(
-    'commit-sha',
-    'Commit SHA linked to this comment. Defaults to HEAD.'
-  )
-  .default('head-sha')
-  .describe('head-sha', 'Commit SHA linked to this comment. Defaults to HEAD.')
-  .deprecateOption('head-sha', 'Use commit-sha instead')
-  .boolean('rm-watermark')
-  .describe(
-    'rm-watermark',
-    'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.'
-  )
-  .default('repo')
-  .describe(
-    'repo',
-    'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
-  )
-  .default('token')
-  .describe(
-    'token',
-    'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.'
-  )
-  .default('driver')
-  .choices('driver', ['github', 'gitlab'])
-  .describe('driver', 'If not specify it infers it from the ENV.')
-  .help('h')
-  .demand(1).argv;
+const opts = decamelize(
+  yargs
+    .strict()
+    .usage('Usage: $0 <path to markdown file>')
+    .default('commit-sha')
+    .describe(
+      'commit-sha',
+      'Commit SHA linked to this comment. Defaults to HEAD.'
+    )
+    .alias('commit-sha', 'head-sha')
+    .boolean('rm-watermark')
+    .describe(
+      'rm-watermark',
+      'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.'
+    )
+    .default('repo')
+    .describe(
+      'repo',
+      'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+    )
+    .default('token')
+    .describe(
+      'token',
+      'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.'
+    )
+    .default('driver')
+    .choices('driver', ['github', 'gitlab'])
+    .describe('driver', 'If not specify it infers it from the ENV.')
+    .help('h')
+    .demand(1).argv
+);
 
-run(argv).catch((e) => {
+run(opts).catch((e) => {
   console.error(e);
   process.exit(1);
 });