forked from rucio/rucio
186 lines (183 loc) · 8 KB
/
autotest.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
name: Run all the autotests
on:
pull_request:
push:
schedule:
- cron: '0 3 * * *'
jobs:
add_header:
if: github.repository_owner == 'rucio' || github.event_name != 'schedule'
name: Add header lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
# We need the history to determine the file creation date.
fetch-depth: 0
- name: Check headers
shell: bash
run: |
python3 tools/add_header --dry-run --disable-progress-bar
python_ruff:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ruff
- name: Run Ruff
run: ruff check --output-format=github .
python_annotations:
if: |
!startsWith(github.ref_name, 'release')
&& !startsWith(github.base_ref, 'release')
&& github.event_name != 'schedule'
&& github.ref_type != 'tag'
name: Check Python Type Annotations
runs-on: ubuntu-latest
env:
UPDATED_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE: ../updated_type_annotation_report.txt
INITIAL_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE: ../initial_type_annotation_report.txt
steps:
- uses: actions/checkout@v4
- name: Count current number of missing type annotations
id: count_updated
shell: bash
run: |
pip install flake8 flake8-annotations
source tools/count_missing_type_annotations_utils.sh
create_missing_python_type_annotations_report ${{ env.UPDATED_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }}
echo "The current number of missing python type annotations: $(wc -l < ${{ env.UPDATED_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }})"
- name: Checkout ancestor commit with rucio/master
uses: ./.github/actions/checkout_ancestor_commit
with:
paths_to_copy: |
tools/count_missing_type_annotations.sh
tools/count_missing_type_annotations_utils.sh
- name: Count initial number of missing type annotations
id: count_initial
shell: bash
run: |
source tools/count_missing_type_annotations_utils.sh
create_missing_python_type_annotations_report ${{ env.INITIAL_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }}
echo "The initial number of missing python type annotations: $(wc -l < ${{ env.INITIAL_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }})"
- name: Diff of missing annotations
shell: bash
run: |
diff ${{ env.INITIAL_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }} ${{ env.UPDATED_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }} || true
- name: Compare numbers of missing type annotations
shell: bash
run: |
INITIAL_NUMBER_OF_MISSING_TYPE_ANNOTATIONS=$(wc -l < ${{ env.INITIAL_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }})
UPDATED_NUMBER_OF_MISSING_TYPE_ANNOTATIONS=$(wc -l < ${{ env.UPDATED_PYTHON_TYPE_ANNOTATIONS_REPORT_FILE }})
echo "The initial number of missing type annotations is: $INITIAL_NUMBER_OF_MISSING_TYPE_ANNOTATIONS"
echo "The updated number of missing type annotations is: $UPDATED_NUMBER_OF_MISSING_TYPE_ANNOTATIONS"
if [ $UPDATED_NUMBER_OF_MISSING_TYPE_ANNOTATIONS -gt $INITIAL_NUMBER_OF_MISSING_TYPE_ANNOTATIONS ]; then
echo "The number of missing python type annotations should never increase! This way we ensure that new functions are type annotated."
echo "Look into the first lines of 'Diff of missing type annotations' to get a reference point what function might be missing type annotations."
echo "**TL;DR** New code has to be type annotated, old code should be migrated. Look into Best Practices[1] for specific instructions on how to use it in our repository."
echo "Look into the Rucio Type Annotations Guide[2] to get some help on why, what and how to add type annotations."
echo
echo "[1] https://codimd.web.cern.ch/6-SU3cTpQSWRK6FHkM7mAA#Best-Practices"
echo "[2] https://codimd.web.cern.ch/6-SU3cTpQSWRK6FHkM7mAA#"
exit 1
fi
python_pyright:
if: |
!startsWith(github.ref_name, 'release')
&& !startsWith(github.base_ref, 'release')
&& github.event_name != 'schedule'
&& github.ref_type != 'tag'
name: Python type check (Pyright)
runs-on: ubuntu-latest
env:
PYRIGHT_CURRENT_REPORT: ../pyright_current_report.json
PYRIGHT_ANCESTOR_REPORT: ../pyright_ancestor_report.json
steps:
- uses: actions/checkout@v4
- name: Install dependencies
run: |
sudo apt-get update && sudo apt-get install -y npm libkrb5-dev libxmlsec1-dev git
npm install --global pyright
python -m pip --no-cache-dir install --upgrade pip
python -m pip --no-cache-dir install --upgrade -r requirements.txt
- name: Make pyright report of current commit
run: |
tools/run_pyright.sh generate ${{ env.PYRIGHT_CURRENT_REPORT }}
- name: Show the current Pyright report
run: cat ${{ env.PYRIGHT_CURRENT_REPORT }}
- name: Checkout ancestor commit with rucio/master
uses: ./.github/actions/checkout_ancestor_commit
with:
paths_to_copy: |
pyrightconfig.json
tools/run_pyright.sh
tools/run_pyright/
- name: Make pyright report of ancestor commit
run: |
tools/run_pyright.sh generate ${{ env.PYRIGHT_ANCESTOR_REPORT }}
- name: Show the ancestor Pyright report
run: cat ${{ env.PYRIGHT_ANCESTOR_REPORT }}
- name: Compare reports
run: |
tools/run_pyright.sh compare --Werror \
${{ env.PYRIGHT_ANCESTOR_REPORT }} \
${{ env.PYRIGHT_CURRENT_REPORT }}
setup:
if: github.repository_owner == 'rucio' || github.event_name != 'schedule'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Update pip
run: python3 -m pip install -U pip setuptools
- name: Install python requirements for matrix_parser.py
run: python3 -m pip install -U PyYAML
- name: Identify branch
id: branch
run: python3 tools/github/workflow/set_branch_output_env.py
- name: Identify Matrix
id: matrix
run: |
EVENT="${{ github.event_name }}"
if [ $EVENT != 'schedule' ]; then
echo "matrix=$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix.yml)" >> $GITHUB_OUTPUT
else
echo "matrix=$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix_nightly.yml)" >> $GITHUB_OUTPUT
fi
outputs:
branch: ${{ steps.branch.outputs.branch }}
matrix: ${{ steps.matrix.outputs.matrix }}
test:
needs: setup
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cfg: ${{ fromJson(needs.setup.outputs.matrix) }}
steps:
- uses: actions/checkout@v4
- name: Build images
id: images
shell: bash
run: |
docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
i=0; until [ "$i" -ge 3 ]; do
IMAGES=$(echo '${{ toJson(matrix.cfg) }}' | ./tools/test/build_images.py \
--cache-repo ghcr.io/${{ github.repository }} \
--branch "${{ needs.setup.outputs.branch }}" ./etc/docker/test || echo "")
if [[ -n $IMAGES ]]; then break;
else
i=$((i+1)); sleep 5;
echo "::warning::Building images failed, retrying…"
fi
done
docker logout https://ghcr.io
if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi
echo "images=$IMAGES" >> $GITHUB_OUTPUT
- name: Run test with cfg
run: 'echo ''{"matrix": ${{ toJson(matrix.cfg) }}, "images": ${{ steps.images.outputs.images }} }'' | ./tools/test/run_tests.py'