/
azure-cron.yml
221 lines (201 loc) · 9.06 KB
/
azure-cron.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Azure Pipelines file, see https://aka.ms/yaml
# Do not run on PRs
pr: none
# Do not run on merge to main
trigger: none
# Do run on a schedule (hourly)
#
# This is currently (2019-08-15) broken on Azure for GitHub-hosted repos. It
# does, however, work as expected for Azure-hosted repos. As a workaround, we
# have created a repo inside Azure that contains an `azure-pipelines.yml` file
# that just triggers this job.
#
# When the situation is resolved, delete that repo in Azure and uncomment the
# following. In the meantime, this should stay commented so we avoid running
# jobs twice when Azure fixes this issue.
#schedules:
#- cron: "0 * * * *"
# displayName: hourly cron
# branches:
# include:
# - main
# always: true
jobs:
- job: fix_bazel_cache
timeoutInMinutes: 120
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: ci/bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
cd sdk
eval "$(dev-env/bin/dade assist)"
bazel build //ci/cron:cron
key=$(mktemp)
cleanup="rm -rf $key ~/.config/gcloud"
trap "$cleanup" EXIT
echo "$GCRED" > $key
gcloud auth activate-service-account --key-file=$key
export BOTO_CONFIG=/dev/null
./bazel-bin/ci/cron/cron bazel-cache --age 75 --delete --cache-suffix '**'
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: ci/tell-slack-failed.yml
- job: docker_image
timeoutInMinutes: 60
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: |
set -euo pipefail
eval "$(cd sdk; dev-env/bin/dade-assist)"
cd sdk
HEAD=$(git rev-parse HEAD)
while ! nix-build --no-out-link -A tools.sed -A tools.jq -A tools.curl -A tools.base64 nix; do :; done
trap 'rm -rf ~/.docker' EXIT
echo $DOCKER_PASSWORD | docker login --username $DOCKER_LOGIN --password-stdin
echo $DOCKER_CONTENT_TRUST_KEY | base64 -d > ~/.docker/da_automation.key
chmod 600 ~/.docker/da_automation.key
docker trust key load ~/.docker/da_automation.key --name $DOCKER_CONTENT_TRUST_USERNAME
RELEASES=$(curl https://api.github.com/repos/digital-asset/daml/releases -sSfL | jq -r '.[] | .tag_name')
DIR=$(pwd)
VERSIONS=$(curl 'https://hub.docker.com/v2/repositories/digitalasset/daml-sdk/tags/?page_size=10000' -sSfL)
# Our docker tags should be stable. Therefore, we only build the image if it has not already
# been built before and we checkout the Dockerfile for the release tag.
# We do not update docker images for older releases so only docker images for SDK releases
# >= 0.13.43 are built this way.
for version in $(echo $RELEASES | sed -e 's/ /\n/g'); do
LAST_UPDATE=$(echo $VERSIONS | jq -r '.results[] | select(.name == "'${version#v}'") | .last_updated')
if [[ -n "$LAST_UPDATE" ]]; then
echo "${version#v} already exists, skipping."
else
echo "Building version ${version#v}..."
#git checkout "$version"
cd ci/docker/daml-sdk
docker build -t digitalasset/daml-sdk:${version#v} --build-arg VERSION=${version#v} .
#git checkout Dockerfile
# Despite the name not suggesting it at all, this actually signs
# _and pushes_ the image; see
# https://docs.docker.com/engine/security/trust/#signing-images-with-docker-content-trust
docker trust sign digitalasset/daml-sdk:${version#v}
cd "$DIR"
git checkout $HEAD
echo "Done."
fi
done
env:
DOCKER_LOGIN: $(DOCKER_LOGIN)
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
DOCKER_CONTENT_TRUST_KEY: $(DOCKER_CONTENT_TRUST_KEY)
DOCKER_CONTENT_TRUST_USERNAME: $(DOCKER_CONTENT_TRUST_USERNAME)
# Does not appear explicitly in the script, but is used by
# docker trust key load
DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: $(DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE)
- template: ci/tell-slack-failed.yml
- job: vscode_marketplace
timeoutInMinutes: 10
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: |
set -euo pipefail
eval "$(cd sdk; dev-env/bin/dade-assist)"
AUTH=$(echo -n "OAuth:${MARKETPLACE_TOKEN}" | base64 -w0)
MARKET=$(curl -H "Authorization: Basic $AUTH" \
-H "Accept: application/json;api-version=5.0-preview.2" \
-sSfL \
"https://marketplace.visualstudio.com/_apis/gallery/publishers/DigitalAssetHoldingsLLC/extensions/daml?flags=1" \
| jq -r '.versions[0].version')
# This jq expression should ensure that we always upload the
# highest-number version. Here is how this works:
#
# 1. The GitHub API documentation does not specify the order for the
# "list releases" endpoint, but does specify that the "latest"
# endpoint returns the release that points to the most recent commit.
# Assuming the same sort order is applied for the list endpoint
# (which empirically seems to hold so far), this means that they may
# be out-of-order wrt version numbers, e.g. 1.1.0 may appear after
# 1.0.2.
# 2. The `.tag_name | .[1:] | split (".") | map(tonumber)` part will
# turn "v1.0.2" into an array [1, 0, 2].
# 3. jq documents its sort method to sort numbers in numeric order
# and arrays in lexical order (ascending in both cases).
#
# This is required because, while the VSCode Marketplace does show
# _a_ version number, it doesn't handle versions at all: we can only
# have one version on the marketplace at any given time, and any
# upload replaces the existing version.
GITHUB=$(curl https://api.github.com/repos/digital-asset/daml/releases -sSfL \
| jq -r '. | map(select(.prerelease == false)
| .tag_name
| .[1:]
| split (".")
| map(tonumber))
| sort
| reverse
| .[0]
| map(tostring)
| join(".")')
if [[ "$GITHUB" != "$MARKET" ]] && git merge-base --is-ancestor 798e96c9b9034eac85ace786b9e1955cf380285c v$GITHUB; then
echo "Publishing $GITHUB to VSCode Marketplace"
git checkout v$GITHUB
(
if [ -d sdk ]; then
cd sdk
fi
cp LICENSE compiler/daml-extension
trap "rm -rf $PWD/compiler/daml-extension/LICENSE" EXIT
cd compiler/daml-extension
sed -i "s/__VERSION__/$GITHUB/" package.json
# This produces out/src/extension.js
bazel run --run_under="cd $PWD &&" @nodejs//:yarn install
bazel run --run_under="cd $PWD &&" @nodejs//:yarn compile
bazel run --run_under="cd $PWD && " @daml_extension_deps//vsce/bin:vsce -- publish --yarn $GITHUB -p $MARKETPLACE_TOKEN
)
else
if [[ "$GITHUB" == "$MARKET" ]]; then
echo "Version on marketplace is already the latest ($GITHUB)."
else
echo "Latest version is not ready for marketplace publication."
fi
fi
env:
MARKETPLACE_TOKEN: $(VSCODE_MARKETPLACE_TOKEN)
- template: ci/tell-slack-failed.yml
- job: download_stats
timeoutInMinutes: 10
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
steps:
- checkout: self
- bash: |
set -euo pipefail
eval "$(cd sdk; dev-env/bin/dade-assist)"
STATS=$(mktemp)
curl https://api.github.com/repos/digital-asset/daml/releases -sSfL | gzip -9 > $STATS
GCS_KEY=$(mktemp)
cleanup () {
rm -f $GCS_KEY
}
trap cleanup EXIT
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
gcloud auth activate-service-account --key-file=$GCS_KEY
BOTO_CONFIG=/dev/null gsutil cp $STATS gs://daml-data/downloads/$(date -u +%Y%m%d_%H%M%SZ).json.gz
env:
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: ci/tell-slack-failed.yml
- template: ci/refresh-get-daml-com.yml