-
Notifications
You must be signed in to change notification settings - Fork 8
/
generate-debian-versions.py
executable file
·203 lines (142 loc) · 5.15 KB
/
generate-debian-versions.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
#!/usr/bin/env python3
import json
import operator
import os
import subprocess
import urllib.request
import zipfile
from pathlib import Path
# this requires being run on an OS that has a version of "dpkg" which supports the
# "--compare-versions" option; also make sure to consider the version of dpkg being
# used in case there are changes to the comparing logic (last run with 1.19.7).
#
# also note that because of the large amount of versions being used there is
# significant overhead in having to use a subprocess, so this generator caches
# the results of said subprocess calls; a typical no-cache run takes about 5+
# minutes whereas with the cache it only takes seconds.
def download_debian_db():
urllib.request.urlretrieve("https://osv-vulnerabilities.storage.googleapis.com/Debian/all.zip", "debian-db.zip")
def extract_packages_with_versions(osvs):
dict = {}
for osv in osvs:
for affected in osv['affected']:
package = affected['package']['name']
if package not in dict:
dict[package] = []
for version in affected.get('versions', []):
dict[package].append(DebianVersion(version))
# deduplicate and sort the versions for each package
for package in dict:
dict[package] = sorted(list(dict.fromkeys(dict[package])))
return dict
class DebianVersionComparer:
def __init__(self, cache_path):
self.cache_path = Path(cache_path)
self.cache = {}
self._load_cache()
def _load_cache(self):
if self.cache_path:
self.cache_path.touch()
with open(self.cache_path, "r") as f:
lines = f.readlines()
for line in lines:
line = line.strip()
key, result = line.split(",")
if result == "True":
self.cache[key] = True
continue
if result == "False":
self.cache[key] = False
continue
print(f"ignoring invalid cache entry '{line}'")
def _save_to_cache(self, key, result):
self.cache[key] = result
if self.cache_path:
self.cache_path.touch()
with open(self.cache_path, "a") as f:
f.write(f"{key},{result}\n")
def compare(self, a, op, b):
key = f"{a} {op} {b}"
if key in self.cache:
return self.cache[key]
cmd = ["dpkg", "--compare-versions", a, op, b]
out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if out.stdout:
print(out.stdout.decode('utf-8'))
if out.stderr:
print(out.stderr.decode('utf-8'))
r = out.returncode == 0
self._save_to_cache(key, r)
return r
debian_comparer = DebianVersionComparer("/tmp/debian-versions-generator-cache.csv")
class DebianVersion:
def __str__(self):
return self.version
def __hash__(self):
return hash(self.version)
def __init__(self, version):
self.version = version
def __lt__(self, other):
return debian_comparer.compare(self.version, 'lt', other.version)
def __gt__(self, other):
return debian_comparer.compare(self.version, 'gt', other.version)
def __eq__(self, other):
return debian_comparer.compare(self.version, 'eq', other.version)
def compare(v1, relate, v2):
ops = {'<': operator.lt, '=': operator.eq, '>': operator.gt}
return ops[relate](v1, v2)
def compare_versions(lines, select="all"):
has_any_failed = False
for line in lines:
line = line.strip()
if line == "" or line.startswith('#') or line.startswith('//'):
continue
v1, op, v2 = line.strip().split(" ")
r = compare(DebianVersion(v1), op, DebianVersion(v2))
if not r:
has_any_failed = r
if select == "failures" and r:
continue
if select == "successes" and not r:
continue
color = '\033[92m' if r else '\033[91m'
rs = "T" if r else "F"
print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m")
return has_any_failed
def compare_versions_in_file(filepath, select="all"):
with open(filepath) as f:
lines = f.readlines()
return compare_versions(lines, select)
def generate_version_compares(versions):
comparisons = []
for i, version in enumerate(versions):
if i == 0:
continue
comparisons.append(f"{versions[i - 1]} < {version}\n")
return comparisons
def generate_package_compares(packages):
comparisons = []
for package in packages:
versions = packages[package]
comparisons.extend(generate_version_compares(versions))
# return comparisons
return list(dict.fromkeys(comparisons))
def fetch_packages_versions():
download_debian_db()
osvs = []
with zipfile.ZipFile('debian-db.zip') as db:
for fname in db.namelist():
with db.open(fname) as osv:
osvs.append(json.loads(osv.read().decode('utf-8')))
return extract_packages_with_versions(osvs)
outfile = "pkg/semantic/fixtures/debian-versions-generated.txt"
packs = fetch_packages_versions()
with open(outfile, "w") as f:
f.writelines(generate_package_compares(packs))
f.write("\n")
# set this to either "failures" or "successes" to only have those comparison results
# printed; setting it to anything else will have all comparison results printed
show = os.environ.get("VERSION_GENERATOR_PRINT", "failures")
did_any_fail = compare_versions_in_file(outfile, show)
if did_any_fail:
sys.exit(1)