From 90339b72604a0dcbfd3ff9f907c98203b3ee0e01 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Mon, 9 Jul 2018 15:29:44 -0400 Subject: [PATCH 1/2] Update to run new AppVeyor jobs AppVeyor builds got split into two jobs, `electron-x64` and `electron-ia32` --- script/ci-release-build.js | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/script/ci-release-build.js b/script/ci-release-build.js index d543c6d27f00c..a2acc74862893 100644 --- a/script/ci-release-build.js +++ b/script/ci-release-build.js @@ -3,6 +3,11 @@ const request = require('request') const buildAppVeyorURL = 'https://windows-ci.electronjs.org/api/builds' const vstsURL = 'https://github.visualstudio.com/electron/_apis/build' +const appVeyorJobs = { + 'electron-x64': 'electron', + 'electron-ia32': 'electron-39ng6' +} + const circleCIJobs = [ 'electron-linux-arm', 'electron-linux-arm64', @@ -71,9 +76,19 @@ async function circleCIcall (buildUrl, targetBranch, job, options) { console.log(`CircleCI release build request for ${job} successful. Check ${circleResponse.build_url} for status.`) } -async function buildAppVeyor (targetBranch, options) { - console.log(`Triggering AppVeyor to run build on branch: ${targetBranch} with release flag.`) +function buildAppVeyor (targetBranch, options) { assert(process.env.APPVEYOR_TOKEN, 'APPVEYOR_TOKEN not found in environment') + const validJobs = Object.keys(appVeyorJobs) + if (options.job) { + assert(validJobs.includes(options.job), `Unknown AppVeyor CI job name: ${options.job}. Valid values are: ${validJobs}.`) + callAppVeyor(targetBranch, options.job, options) + } else { + validJobs.forEach((job) => callAppVeyor(targetBranch, job, options)) + } +} + +async function callAppVeyor (targetBranch, job, options) { + console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`) let environmentVariables = {} if (options.ghRelease) { @@ -96,7 +111,7 @@ async function buildAppVeyor (targetBranch, options) { }, body: JSON.stringify({ accountName: 'AppVeyor', - projectSlug: 'electron', + projectSlug: appVeyorJobs[job], branch: targetBranch, environmentVariables }), @@ -106,13 +121,13 @@ async function buildAppVeyor (targetBranch, options) { console.log('Error calling AppVeyor:', err) }) const buildUrl = `https://windows-ci.electronjs.org/project/AppVeyor/electron/build/${appVeyorResponse.version}` - console.log(`AppVeyor release build request successful. Check build status at ${buildUrl}`) + console.log(`AppVeyor release build request for ${job} successful. Check build status at ${buildUrl}`) } function buildCircleCI (targetBranch, options) { const circleBuildUrl = `https://circleci.com/api/v1.1/project/github/electron/electron/tree/${targetBranch}?circle-token=${process.env.CIRCLE_TOKEN}` if (options.job) { - assert(circleCIJobs.includes(options.job), `Unknown CI job name: ${options.job}.`) + assert(circleCIJobs.includes(options.job), `Unknown CircleCI job name: ${options.job}. Valid values are: ${circleCIJobs}.`) circleCIcall(circleBuildUrl, targetBranch, options.job, options) } else { circleCIJobs.forEach((job) => circleCIcall(circleBuildUrl, targetBranch, job, options)) @@ -121,7 +136,7 @@ function buildCircleCI (targetBranch, options) { async function buildVSTS (targetBranch, options) { if (options.job) { - assert(vstsJobs.includes(options.job), `Unknown CI job name: ${options.job}.`) + assert(vstsJobs.includes(options.job), `Unknown VSTS CI job name: ${options.job}. Valid values are: ${vstsJobs}.`) } console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`) assert(process.env.VSTS_TOKEN, 'VSTS_TOKEN not found in environment') From 2040095b9644a21eef6627c3012f1f20d9f2a763 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Mon, 9 Jul 2018 18:46:31 -0400 Subject: [PATCH 2/2] Move github release checking to node.js Our upload script was relying on an old python script to find the GitHub release for uploading. --- script/find-release.js | 37 +++++++++++++ script/lib/github.py | 76 -------------------------- script/upload.py | 117 +++++++++-------------------------------- 3 files changed, 63 insertions(+), 167 deletions(-) create mode 100644 script/find-release.js delete mode 100644 script/lib/github.py diff --git a/script/find-release.js b/script/find-release.js new file mode 100644 index 0000000000000..3cfd79aae1427 --- /dev/null +++ b/script/find-release.js @@ -0,0 +1,37 @@ +const GitHub = require('github') +const github = new GitHub() + +if (process.argv.length < 3) { + console.log('Usage: find-release version') + process.exit(1) +} + +const version = process.argv[2] + +async function findRelease () { + github.authenticate({type: 'token', token: process.env.ELECTRON_GITHUB_TOKEN}) + let releases = await github.repos.getReleases({ + owner: 'electron', + repo: 'electron' + }) + let targetRelease = releases.data.find(release => { + return release.tag_name === version + }) + let returnObject = {} + + if (targetRelease) { + returnObject = { + id: targetRelease.id, + draft: targetRelease.draft, + exists: true + } + } else { + returnObject = { + exists: false, + draft: false + } + } + console.log(JSON.stringify(returnObject)) +} + +findRelease() diff --git a/script/lib/github.py b/script/lib/github.py deleted file mode 100644 index def7be585e8a0..0000000000000 --- a/script/lib/github.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -import json -import os -import re -import sys - -REQUESTS_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', - 'vendor', 'requests')) -sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib')) -sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib.linux-x86_64-2.7')) -import requests - -GITHUB_URL = 'https://api.github.com' -GITHUB_UPLOAD_ASSET_URL = 'https://uploads.github.com' - -class GitHub: - def __init__(self, access_token): - self._authorization = 'token %s' % access_token - - pattern = '^/repos/{0}/{0}/releases/{1}/assets$'.format('[^/]+', '[0-9]+') - self._releases_upload_api_pattern = re.compile(pattern) - - def __getattr__(self, attr): - return _Callable(self, '/%s' % attr) - - def send(self, method, path, **kw): - if not 'headers' in kw: - kw['headers'] = dict() - headers = kw['headers'] - headers['Authorization'] = self._authorization - headers['Accept'] = 'application/vnd.github.manifold-preview' - - # Switch to a different domain for the releases uploading API. - if self._releases_upload_api_pattern.match(path): - url = '%s%s' % (GITHUB_UPLOAD_ASSET_URL, path) - else: - url = '%s%s' % (GITHUB_URL, path) - # Data are sent in JSON format. - if 'data' in kw: - kw['data'] = json.dumps(kw['data']) - - r = getattr(requests, method)(url, **kw).json() - if 'message' in r: - raise Exception(json.dumps(r, indent=2, separators=(',', ': '))) - return r - - -class _Executable: - def __init__(self, gh, method, path): - self._gh = gh - self._method = method - self._path = path - - def __call__(self, **kw): - return self._gh.send(self._method, self._path, **kw) - - -class _Callable(object): - def __init__(self, gh, name): - self._gh = gh - self._name = name - - def __call__(self, *args): - if len(args) == 0: - return self - - name = '%s/%s' % (self._name, '/'.join([str(arg) for arg in args])) - return _Callable(self._gh, name) - - def __getattr__(self, attr): - if attr in ['get', 'put', 'post', 'patch', 'delete']: - return _Executable(self._gh, attr, self._name) - - name = '%s/%s' % (self._name, attr) - return _Callable(self._gh, name) diff --git a/script/upload.py b/script/upload.py index b188265fc11f0..6d71fc69d4d70 100755 --- a/script/upload.py +++ b/script/upload.py @@ -4,6 +4,7 @@ import datetime import errno import hashlib +import json import os import subprocess import sys @@ -14,7 +15,6 @@ get_zip_name from lib.util import electron_gyp, execute, get_electron_version, \ parse_version, scoped_cwd, s3put -from lib.github import GitHub ELECTRON_REPO = 'electron/electron' @@ -50,58 +50,40 @@ def main(): sys.stderr.flush() return 1 - github = GitHub(auth_token()) - releases = github.repos(ELECTRON_REPO).releases.get() tag_exists = False - release = None - for r in releases: - if not r['draft'] and r['tag_name'] == args.version: - release = r - tag_exists = True - break + release = get_release(args.version) + if not release['draft']: + tag_exists = True if not args.upload_to_s3: + assert release['exists'], 'Release does not exist; cannot upload to GitHub!' assert tag_exists == args.overwrite, \ 'You have to pass --overwrite to overwrite a published release' - if not args.overwrite: - release = create_or_get_release_draft(github, releases, args.version, - tag_exists) - elif release is None: - release = dict(tag_name=args.version) - - # Upload Electron with GitHub Releases API. - upload_electron(github, release, os.path.join(DIST_DIR, DIST_NAME), args) + # Upload Electron files. + upload_electron(release, os.path.join(DIST_DIR, DIST_NAME), args) if get_target_arch() != 'mips64el': - upload_electron(github, release, os.path.join(DIST_DIR, SYMBOLS_NAME), args) + upload_electron(release, os.path.join(DIST_DIR, SYMBOLS_NAME), args) if PLATFORM == 'darwin': - upload_electron(github, release, os.path.join(DIST_DIR, - 'electron-api.json'), args) - upload_electron(github, release, os.path.join(DIST_DIR, 'electron.d.ts'), - args) - upload_electron(github, release, os.path.join(DIST_DIR, DSYM_NAME), - args) + upload_electron(release, os.path.join(DIST_DIR, 'electron-api.json'), args) + upload_electron(release, os.path.join(DIST_DIR, 'electron.d.ts'), args) + upload_electron(release, os.path.join(DIST_DIR, DSYM_NAME), args) elif PLATFORM == 'win32': - upload_electron(github, release, os.path.join(DIST_DIR, PDB_NAME), - args) + upload_electron(release, os.path.join(DIST_DIR, PDB_NAME), args) # Upload free version of ffmpeg. ffmpeg = get_zip_name('ffmpeg', ELECTRON_VERSION) - upload_electron(github, release, os.path.join(DIST_DIR, ffmpeg), - args) + upload_electron(release, os.path.join(DIST_DIR, ffmpeg), args) chromedriver = get_zip_name('chromedriver', ELECTRON_VERSION) - upload_electron(github, release, os.path.join(DIST_DIR, chromedriver), - args) + upload_electron(release, os.path.join(DIST_DIR, chromedriver), args) mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION) - upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot), - args) + upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args) if get_target_arch().startswith('arm'): # Upload the x64 binary for arm/arm64 mksnapshot mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64') - upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot), - args) + upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args) if not tag_exists and not args.upload_to_s3: # Upload symbols to symbol server. @@ -167,79 +149,26 @@ def dist_newer_than_head(): return dist_time > int(head_time) -def get_text_with_editor(name): - editor = os.environ.get('EDITOR', 'nano') - initial_message = '\n# Please enter the body of your release note for %s.' \ - % name - - t = tempfile.NamedTemporaryFile(suffix='.tmp', delete=False) - t.write(initial_message) - t.close() - subprocess.call([editor, t.name]) - - text = '' - for line in open(t.name, 'r'): - if len(line) == 0 or line[0] != '#': - text += line - - os.unlink(t.name) - return text - -def create_or_get_release_draft(github, releases, tag, tag_exists): - # Search for existing draft. - for release in releases: - if release['draft'] and release['tag_name'] == tag: - return release - - if tag_exists: - tag = 'do-not-publish-me' - return create_release_draft(github, tag) - - -def create_release_draft(github, tag): - name = '{0} {1} beta'.format(PROJECT_NAME, tag) - if os.environ.has_key('CI'): - body = '(placeholder)' - else: - body = get_text_with_editor(name) - if body == '': - sys.stderr.write('Quit due to empty release note.\n') - sys.exit(0) - - data = dict(tag_name=tag, name=name, body=body, draft=True, prerelease=True) - r = github.repos(ELECTRON_REPO).releases.post(data=data) - return r - - -def upload_electron(github, release, file_path, args): +def upload_electron(release, file_path, args): filename = os.path.basename(file_path) # if upload_to_s3 is set, skip github upload. if args.upload_to_s3: bucket, access_key, secret_key = s3_config() - key_prefix = 'electron-artifacts/{0}_{1}'.format(release['tag_name'], + key_prefix = 'electron-artifacts/{0}_{1}'.format(args.version, args.upload_timestamp) s3put(bucket, access_key, secret_key, os.path.dirname(file_path), key_prefix, [file_path]) - upload_sha256_checksum(release['tag_name'], file_path, key_prefix) + upload_sha256_checksum(args.version, file_path, key_prefix) s3url = 'https://gh-contractor-zcbenz.s3.amazonaws.com' print '{0} uploaded to {1}/{2}/{0}'.format(filename, s3url, key_prefix) return - # Delete the original file before uploading in CI. - if os.environ.has_key('CI'): - try: - for asset in release['assets']: - if asset['name'] == filename: - github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete() - except Exception: - pass - # Upload the file. upload_io_to_github(release, filename, file_path) # Upload the checksum file. - upload_sha256_checksum(release['tag_name'], file_path) + upload_sha256_checksum(args.version, file_path) def upload_io_to_github(release, filename, filepath): @@ -273,6 +202,12 @@ def auth_token(): return token +def get_release(version): + script_path = os.path.join(SOURCE_ROOT, 'script', 'find-release.js') + release_info = execute(['node', script_path, version]) + release = json.loads(release_info) + return release + if __name__ == '__main__': import sys sys.exit(main())