-
Notifications
You must be signed in to change notification settings - Fork 3
/
sorri
505 lines (442 loc) · 17.2 KB
/
sorri
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
# vim: ft=bash
# sorri: a Simpler lORRI
#
# This is a simpler implementation of Tweag's lorri:
# https://github.com/target/lorri
#
# TODO: document inputs and env variables
#
# sorri reuses lorri's tricks for figuring out the files to track for changes,
# but uses direnv's own mechanism for actually tracking those files.
# sorri uses a local cache at '~/.cache/sorri/<project>/v<sorri version>/'.
# Each entry is a directory containing two files:
#
# ~/.cache/sorri/niv/v1/
# └── 0716a121e4f986f9f8cf11f7c579d332
# ├── link -> /nix/store/jfzkisfgmv3qgpzz3i8nai12y1cry77v-nix-shell
# └── manifest
#
# `link` is the result of a previous evaluation. `manifest` is used to find
# that result of a previous evaluation. The directory name
# (0716a121e4f986f9f8cf11f7c579d332 above) is the hash of the `manifest`.
#
# `link` is a symlink to a shell script that sets a shell's variables.
#
# cat ~/.cache/sorri/niv/v1/0716a121e4f986f9f8cf11f7c579d332/link
# declare -x AR_x86_64_apple_darwin="/nix/store/amsm28x2hnsgp8c0nm4glkjc2gw2l9kw-cctools-binutils-darwin-927.0.2/bin/ar"
# declare -x BZ2_LIB_DIR="/nix/store/7yikqcm4v4b57xv3cqknhdnf0p1aakxp-bzip2-1.0.6.0.1/lib"
# declare -x BZ2_STATIC="1"
# declare -x CARGO_BUILD_TARGET="x86_64-apple-darwin"
# declare -x CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_LINKER="/nix/store/swiic36rl7njy6bfll5z0afl42c9q4s5-lld-9.0.1/bin/lld"
#
# `manifest` is a list of files used for an evaluation alongside their checksums:
#
# $ cat ~/.cache/sorri/niv/v1/0716a121e4f986f9f8cf11f7c579d332/manifest
# /Users/nicolas/niv/shell.nix:029451f2a9bee59f4ce002bdbdf20554
# /Users/nicolas/niv/nix/default.nix:7ff8c9138044fc7e31f1d4ed2bf1c0ba
# /Users/nicolas/niv/nix/overlays/buf/default.nix:c4a24e0bba0178b73f0211d0f26147e6
# ...
#
# sorri first checks the existing cache entries (sorri/niv/v1/0716...,
# etc); if it finds a cache entry with a manifest where all the _manifest_
# entries (nix/default.nix:7ff...) match local files, the link is loaded; if no
# manifest matches, a new entry is created and loaded.
# NOTES:
# we use some functions from direnv's stdlib:
# - watch_file <foo>: updates $DIRENV_WATCHES to tell direnv to watch <foo>
# - expand_path: similar to realpath from coreutils
# Print the line iff SORRI_DEBUG is set and not empty
sorri_debug() {
if [ -n "${SORRI_DEBUG:-}" ]; then echo "debug:" "$@"; fi
}
sorri_log() {
echo "sorri:" "$@"
}
sorri_log_bold() {
tput bold
echo "sorri:" "!!!!" "$@" "!!!!"
tput sgr0
}
# Print in red and return with 1
sorri_abort() {
tput setaf 1
echo sorri: ERROR: "$@"
echo sorri: please run "'direnv allow'" to reload the shell
tput sgr0
exit 1
}
# Removes duplicate lines in place
sorri_remove_duplicates() {
file="$1"
tmpfile=$(mktemp)
sort <"$file" | uniq >"$tmpfile"
mv "$tmpfile" "$file"
}
# Adds the given file to the specified manifest:
# echo "foo.nix:<hash of foo.nix>" >> manifest
sorri_add_to_manifest() {
{
expand_path "$1" | tr -d '\n'
echo -n ":"
nix-hash "$1"
} >>"$2"
}
# Parses a Nix -vv log file and creates a manifest
sorri_create_manifest_from_logs() {
logfile="$1" # The path to the logfile
manifest="$2" # The path to the manifest (will be created)
while IFS= read -r line; do
case $line in
trace*)
# shellcheck disable=2001
copied=$(echo "$line" | sed 's/^trace: file read: '"'"'\([^'"'"']*\)'"'"'.*/\1/')
sorri_debug "found trace $copied"
if ! [[ $copied == /nix/store* ]]; then
sorri_add_to_manifest "$copied" "$manifest"
fi
;;
copied*)
# shellcheck disable=2001
copied=$(echo "$line" | sed 's/^copied source '"'"'\([^'"'"']*\)'"'"'.*/\1/')
sorri_debug "found copied $copied"
if ! [[ $copied == /nix/store* ]]; then
sorri_add_to_manifest "$copied" "$manifest"
fi
;;
evaluating*)
# shellcheck disable=2001
copied=$(echo "$line" | sed 's/^evaluating file '"'"'\([^'"'"']*\)'"'"'.*/\1/')
sorri_debug "found evaluated $copied"
# skip files if they're in the store (i.e. immutable)
if ! [[ $copied == /nix/store* ]]; then
# when evaluating a `default.nix`, Nix sometimes prints the
# path to the file, and sometimes to the directory...
if [ -d "$copied" ]; then
sorri_add_to_manifest "$copied/default.nix" "$manifest"
else
sorri_add_to_manifest "$copied" "$manifest"
fi
fi
;;
esac
done <"$logfile"
sorri_remove_duplicates "$manifest"
}
# Wrapper function for creating a new manifest based on the files currently
# present in the source tree.
# NOTE: The manifest (and link) is created atomically meaning this works fine
# if two shells are opened concurrently
sorri_create_manifest() {
sorri_debug creating manifest for "$PWD"
evallogs=$(mktemp)
# A nix wrapper that imports ./shell.nix. It modifies the resulting
# derivation in two ways:
# - The builder is replaced with a bash function that calls `export >
# $out`, which effectively writes all the environment variables to $out.
# The variables can then be imported by sourcing this file.
# - The readFile and readDir builtins are overriden to print their
# arguments whenever they are called (so that we can parse that and track
# those files)
# TODO: use the same tricks for getEnv
local shellnix;
shellnix=$(cat <<EOF
let
overrides = {
import = scopedImport overrides;
scopedImport = x: builtins.scopedImport (overrides // x);
builtins = builtins // {
readFile = file: builtins.trace "file read: '\${toString file}'" (builtins.readFile file);
readDir = path: builtins.trace "file read: '\${toString path}'" (builtins.readDir path);
};
};
# TODO: how do we deal with shellHook s?
# if the shell hook sets a variable, then it should be handled by the shell
# If it does other stuff then this is not gonna work since direnv runs this
# in a subshell.
builder = builtins.toFile "foo-bidou" ''
[ -e \$stdenv/setup ] && . \$stdenv/setup
export > \$out
'';
imported =
let
raw = overrides.scopedImport overrides $(expand_path ./shell.nix);
in
if builtins.isFunction raw
then raw {}
else raw;
in
derivation (
imported.drvAttrs // {
args = [ "-e" builder ];
}
)
EOF
)
# The resulting link to the shell build (is used as a GC root)
buildout=$(mktemp -d)/result
sorri_log building shell, this may take a while
# We keep lines like these:
# 'copied source /niv/src to ...': source trees and files imported to the store
# 'evaluating file foo.nix ...' Nix files used for eval
# 'trace: file read: sources.json...' files from readFile & readDir
keepem=(grep -E "^copied source|^evaluating file|^trace: file read:")
# we drop all the lines like the above but that reference files in the
# store; those files are immutable so we don't want to watch them for
# changes
dropem=(grep -vE "^copied source '/nix|^evaluating file '/nix|^trace: file read: '/nix")
if [ -n "${SORRI_DEBUG:-}" ]; then
nix-build -E "$shellnix" -o "$buildout" -vv \
2> >(tee -a >("${keepem[@]}" | "${dropem[@]}" >"$evallogs")) || sorri_abort nix-build failed
else
logs=$(mktemp)
nix-build -E "$shellnix" -o "$buildout" -vv --max-jobs 8 \
2> >(tee -a "$logs" > >("${keepem[@]}" | "${dropem[@]}" >"$evallogs")) >/dev/null \
|| sorri_abort nix-build failed, logs can be found at "${logs}:"$'\n'"---"$'\n'"$(tail -n 5 "$logs")"$'\n'"---"
rm "$logs"
fi
sorri_debug build finished "$buildout"
tmpmanifest=$(mktemp)
sorri_create_manifest_from_logs "$evallogs" "$tmpmanifest"
# The identifier for this new cache
manifest_hash=$(nix-hash "$tmpmanifest" | tr -d '\n')
mkdir -p "$SORRI_CACHE_DIR/$manifest_hash"
# create the file atomically
mv -f "$tmpmanifest" "$SORRI_CACHE_DIR/$manifest_hash/manifest"
link="$SORRI_CACHE_DIR/$manifest_hash/link"
mv -f "$buildout" "$link"
rmdir "$(dirname "$buildout")"
# Register the shell build as a GC root
nix-store --indirect --add-root "$link" -r "$link"
sorri_log created cached shell "$manifest_hash"
sorri_import_link_of "$SORRI_CACHE_DIR/$manifest_hash"
}
# Load the environment variables saved in a cache entry by importing the link
# file
sorri_import_link_of() {
manifest="$1/manifest"
if [ ! -f "$manifest" ]; then
sorri_abort no manifest found at "$manifest"
fi
link="$1"/link
if [ ! -f "$link" ]; then
sorri_abort no link found at "$link"
fi
sorri_debug importing manifest "$manifest" and link "$link"
# read the manifest line by line and issue direnv `watch_file` calls for
# every file
while IFS= read -r watched; do
watched_file=${watched%:*}
sorri_debug adding file "$watched_file" to watch
watch_file "$watched_file"
done <"$manifest"
# this overrides Bash's 'declare -x'. The 'link' is a bash that calls
# 'declare -x' (== export) on every environment variable in the built
# shell, but there are some variables (PATH, HOME) that we don't actually
# want to inherit from the shell.
function declare() {
if [ "$1" == "-x" ]; then shift; fi
# Some variables require special handling.
case "$1" in
# vars from: https://github.com/NixOS/nix/blob/92d08c02c84be34ec0df56ed718526c382845d1a/src/nix-build/nix-build.cc#L100
"HOME="*) ;;
"USER="*) ;;
"LOGNAME="*) ;;
"DISPLAY="*) ;;
"PATH="*)
# here we don't use PATH_add from direnv because it's too slow
# https://github.com/direnv/direnv/issues/671
PATH="${1#PATH=}:$PATH";;
"TERM="*) ;;
"IN_NIX_SHELL="*) ;;
"TZ="*) ;;
"PAGER="*) ;;
"NIX_BUILD_SHELL="*) ;;
"SHLVL="*) ;;
# vars from: https://github.com/NixOS/nix/blob/92d08c02c84be34ec0df56ed718526c382845d1a/src/nix-build/nix-build.cc#L385
"TEMPDIR="*) ;;
"TMPDIR="*) ;;
"TEMP="*) ;;
"TMP="*) ;;
# vars from: https://github.com/NixOS/nix/blob/92d08c02c84be34ec0df56ed718526c382845d1a/src/nix-build/nix-build.cc#L421
"NIX_ENFORCE_PURITY="*) ;;
# vars from: https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html (last checked: 2019-09-26)
# reported in https://github.com/target/lorri/issues/153
"OLDPWD="*) ;;
"PWD="*) ;;
"SHELL="*) ;;
# some stuff we don't want set
# TODO: find a proper way to deal with this
"__darwinAllowLocalNetworking="*) ;;
"__impureHostDeps="*) ;;
"__propagatedImpureHostDeps="*) ;;
"__propagatedSandboxProfile"*) ;;
"__sandboxProfile="*) ;;
"allowSubstitutes="*) ;;
"buildInputs="*) ;;
"buildPhase"*) ;;
"builder="*) ;;
"checkPhase="*) ;;
"cmakeFlags="*) ;;
"configureFlags="*) ;;
"depsBuildBuild="*) ;;
"depsBuildBuildPropagated="*) ;;
"depsBuildTarget="*) ;;
"depsBuildTargetPropagated="*) ;;
"depsHostHost="*) ;;
"depsHostHostPropagated="*) ;;
"depsTargetTarget="*) ;;
"depsTargetTargetPropagated="*) ;;
"doCheck="*) ;;
"doInstallCheck="*) ;;
"dontDisableStatic="*) ;;
"gl_cv"*) ;;
"installPhase="*) ;;
"mesonFlags="*) ;;
"name="*) ;;
"nativeBuildInputs="*) ;;
"nobuildPhase="*) ;;
"out="*) ;;
"outputs="*) ;;
"patches="*) ;;
"phases="*) ;;
"postUnpack="*) ;;
"preferLocalBuild="*) ;;
"propagatedBuildInputs="*) ;;
"propagatedNativeBuildInputs="*) ;;
"rs="*) ;;
"shell="*) ;;
"shellHook="*) ;;
"src="*) ;;
"stdenv="*) ;;
"strictDeps="*) ;;
"system="*) ;;
"version="*) ;;
# pretty sure these can stay the same
"NIX_SSL_CERT_FILE="*) ;;
"SSL_CERT_FILE="*) ;;
*) export "${@?}" ;;
esac
}
# shellcheck disable=1090
. "$link"
unset declare
}
# Checks if a particular cache entry can be used by comparing the tracked files
# and their checksums.
sorri_check_manifest_of() {
sorri_debug "looking for manifest in $1"
if [ ! -f "$1"/manifest ]; then
sorri_abort "error: no manifest in $1"
fi
# loop over the entries in the manifest, exiting if one doesn't match the
# local file it references.
ok=true
while IFS= read -r watched; do
sorri_debug "read: $watched"
watched_file=${watched%:*}
watched_hash=${watched#*:}
sorri_debug "file: '$watched_file'"
sorri_debug "hash: '$watched_hash'"
if [ -f "$watched_file" ] \
&& [ "$(nix-hash "$watched_file" | tr -d '\n')" == "$watched_hash" ]; then
sorri_debug "$watched_file" "($watched_hash)" "ok"
else
sorri_debug "$watched_file" "($watched_hash)" "not ok"
sorri_debug giving up on "$1"
ok=false
break
fi
done <"$1/manifest"
"$ok"
}
# Lists the directories at "$1", most recent first.
sorri_find_recent_first() {
if find --help 2>/dev/null | grep GNU >/dev/null; then
# this assumes find and stat are the GNU variants
find "$1" \
-maxdepth 1 -mindepth 1 \
-type d -printf "%T+\t%p\n" \
| sort -r \
| cut -f 2-
elif stat --help 2>/dev/null | grep GNU>/dev/null; then
# this assumes BSD find and GNU stat
find "$1" \
-maxdepth 1 -mindepth 1 \
-type d -exec sh -c 'stat -c "%Y {}" {}' \; \
| sort -rn \
| cut -d ' ' -f 2-
else
# this assumes find and stat are the Darwin variants
find "$1" \
-maxdepth 1 -mindepth 1 \
-type d -exec stat -lt "%Y-%m-%d" {} \+ \
| cut -d' ' -f6- \
| sort -rn \
| cut -d ' ' -f 2-
fi
}
# removes all cache entries except the n most recent ones
# (LRU style)
sorri_prune_old_entries() {
local n_to_keep=${1:-5}
while IFS= read -r entry; do
sorri_log removing old cache entry "$entry"
# here we avoid rm -rf at all cost in case anything goes wrong with
# "$entry"'s content.
rm "$entry"/manifest
rm "$entry"/link
rmdir "$entry"
done < <(sorri_find_recent_first "$SORRI_CACHE_DIR" | tail -n +"$(( n_to_keep + 1 ))")
}
sorri_main() {
if [[ $# == 0 ]]
then
SORRI_CACHE_NAME="${SORRI_CACHE_NAME:-global}"
elif [[ $# == 1 ]]
then
SORRI_CACHE_NAME="$1"
else
sorri_abort "OH NOOOO"
fi
sorri_debug SORRI_CACHE_NAME "$SORRI_CACHE_NAME"
# ~/.cache/sorri/<project>/v42
SORRI_CACHE_DIR_PREFIX="${SORRI_CACHE_DIR_PREFIX:-$HOME/.cache/sorri/${SORRI_CACHE_NAME}}"
sorri_debug SORRI_CACHE_DIR_PREFIX "$SORRI_CACHE_DIR_PREFIX"
# NOTE: change version here
# ~/.cache/sorri/<project>/v42
SORRI_CACHE_DIR="${SORRI_CACHE_DIR_PREFIX}/v2"
sorri_debug SORRI_CACHE_DIR "$SORRI_CACHE_DIR"
mkdir -p "$SORRI_CACHE_DIR"
# If there are old entries, then tell user to delete it to avoid zombie
# roots
while IFS= read -r old_cache_entry; do
sorri_log_bold please delete "$old_cache_entry" unless you plan on going back to older sorri versions
done < <(find "$SORRI_CACHE_DIR_PREFIX" -mindepth 1 -maxdepth 1 -type d -not -wholename "$SORRI_CACHE_DIR")
if ! command -v nix &>/dev/null; then
sorri_abort nix executable not found
fi
# The Nix evaluation may be using `lib.inNixShell`, so we play the game
export IN_NIX_SHELL=impure
accepted=""
sorri_log looking for matching cached shell in "$SORRI_CACHE_DIR"
while IFS= read -r candidate; do
sorri_debug checking manifest "$candidate"
if sorri_check_manifest_of "$candidate"; then
sorri_debug accepting sorri cache "$candidate"
touch "$candidate" # label as most recently used
accepted="$candidate"
break
fi
done < <(sorri_find_recent_first "$SORRI_CACHE_DIR")
if [ -n "$accepted" ]; then
sorri_log using cache created "$(date -r "$accepted")" "($(basename "$accepted"))"
sorri_import_link_of "$accepted"
else
sorri_log no candidate accepted, creating manifest
sorri_create_manifest
# we only keep the 5 latest entries to avoid superfluous cruft in $TMP and
# Nix GC roots.
sorri_prune_old_entries 5
fi
}
sorri_main "$@"