Skip to content

Commit

Permalink
tools: log and verify sha256sum
Browse files Browse the repository at this point in the history
PR-URL: #48088
Refs: nodejs/security-wg#973
Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
  • Loading branch information
fasenderos authored and danielleadams committed Jul 6, 2023
1 parent 9ffe061 commit 5767844
Show file tree
Hide file tree
Showing 15 changed files with 219 additions and 15 deletions.
6 changes: 5 additions & 1 deletion tools/dep_updaters/update-ada.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd)
DEPS_DIR="$BASE_DIR/deps"
ADA_VERSION=$1

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

if [ "$#" -le 0 ]; then
echo "Error: please provide an ada version to update to"
echo " e.g. $0 1.0.0"
Expand All @@ -25,13 +28,14 @@ cleanup () {
trap cleanup INT TERM EXIT

ADA_REF="v$ADA_VERSION"
ADA_ZIP="ada-$ADA_VERSION.zip"
ADA_ZIP="ada-$ADA_REF.zip"
ADA_LICENSE="LICENSE-MIT"

cd "$WORKSPACE"

echo "Fetching ada source archive..."
curl -sL -o "$ADA_ZIP" "https://github.com/ada-url/ada/releases/download/$ADA_REF/singleheader.zip"
log_and_verify_sha256sum "ada" "$ADA_ZIP"
unzip "$ADA_ZIP"
rm "$ADA_ZIP"

Expand Down
10 changes: 9 additions & 1 deletion tools/dep_updaters/update-base64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/aklomp/base64/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -39,8 +42,13 @@ trap cleanup INT TERM EXIT

cd "$WORKSPACE"

BASE64_TARBALL="base64-v$NEW_VERSION.tar.gz"

echo "Fetching base64 source archive"
curl -sL "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION" | tar xzf -
curl -sL -o "$BASE64_TARBALL" "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION"
log_and_verify_sha256sum "base64" "$BASE64_TARBALL"
gzip -dc "$BASE64_TARBALL" | tar xf -
rm "$BASE64_TARBALL"
mv aklomp-base64-* base64

echo "Replacing existing base64"
Expand Down
8 changes: 6 additions & 2 deletions tools/dep_updaters/update-brotli.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/google/brotli/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -44,10 +47,11 @@ trap cleanup INT TERM EXIT

cd "$WORKSPACE"

BROTLI_TARBALL="v$NEW_VERSION.tar.gz"
BROTLI_TARBALL="brotli-v$NEW_VERSION.tar.gz"

echo "Fetching brotli source archive"
curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/$BROTLI_TARBALL"
curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/v$NEW_VERSION.tar.gz"
log_and_verify_sha256sum "brotli" "$BROTLI_TARBALL"
gzip -dc "$BROTLI_TARBALL" | tar xf -
rm "$BROTLI_TARBALL"
mv "brotli-$NEW_VERSION" "brotli"
Expand Down
8 changes: 7 additions & 1 deletion tools/dep_updaters/update-c-ares.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/c-ares/c-ares/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -43,7 +46,10 @@ ARES_TARBALL="c-ares-$NEW_VERSION.tar.gz"
cd "$WORKSPACE"

echo "Fetching c-ares source archive"
curl -sL "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL" | tar xz
curl -sL -o "$ARES_TARBALL" "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL"
log_and_verify_sha256sum "c-ares" "$ARES_TARBALL"
gzip -dc "$ARES_TARBALL" | tar xf -
rm "$ARES_TARBALL"
mv "c-ares-$NEW_VERSION" cares

echo "Removing tests"
Expand Down
10 changes: 9 additions & 1 deletion tools/dep_updaters/update-libuv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/libuv/libuv/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -45,8 +48,13 @@ trap cleanup INT TERM EXIT

cd "$WORKSPACE"

LIBUV_TARBALL="libuv-v$NEW_VERSION.tar.gz"

echo "Fetching libuv source archive..."
curl -sL "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION" | tar xzf -
curl -sL -o "$LIBUV_TARBALL" "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION"
log_and_verify_sha256sum "libuv" "$LIBUV_TARBALL"
gzip -dc "$LIBUV_TARBALL" | tar xf -
rm "$LIBUV_TARBALL"
mv libuv-libuv-* uv

echo "Replacing existing libuv (except GYP build files)"
Expand Down
14 changes: 9 additions & 5 deletions tools/dep_updaters/update-llhttp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ DEPS_DIR="${BASE_DIR}/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/nodejs/llhttp/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -52,19 +55,20 @@ if echo "$NEW_VERSION" | grep -qs "/" ; then # Download a release
echo "Checking out branch $BRANCH ..."
git checkout "$BRANCH"

echo "Building llhtttp ..."
echo "Building llhttp ..."
npm install
make release

echo "Copying llhtttp release ..."
echo "Copying llhttp release ..."
rm -rf "$DEPS_DIR/llhttp"
cp -a release "$DEPS_DIR/llhttp"
else
echo "Download llhttp release $NEW_VERSION ..."
curl -sL -o llhttp.tar.gz "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz"
gzip -dc llhttp.tar.gz | tar xf -
LLHTTP_TARBALL="llhttp-v$NEW_VERSION.tar.gz"
curl -sL -o "$LLHTTP_TARBALL" "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz"
gzip -dc "$LLHTTP_TARBALL" | tar xf -

echo "Copying llhtttp release ..."
echo "Copying llhttp release ..."
rm -rf "$DEPS_DIR/llhttp"
cp -a "llhttp-release-v$NEW_VERSION" "$DEPS_DIR/llhttp"
fi
Expand Down
8 changes: 8 additions & 0 deletions tools/dep_updaters/update-nghttp2.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/nghttp2/nghttp2/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -44,6 +47,11 @@ cd "$WORKSPACE"

echo "Fetching nghttp2 source archive"
curl -sL -o "$NGHTTP2_TARBALL" "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/$NGHTTP2_TARBALL"

DEPOSITED_CHECKSUM=$(curl -sL "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/checksums.txt" | grep "$NGHTTP2_TARBALL")

log_and_verify_sha256sum "nghttp2" "$NGHTTP2_TARBALL" "$DEPOSITED_CHECKSUM"

gzip -dc "$NGHTTP2_TARBALL" | tar xf -
rm "$NGHTTP2_TARBALL"
mv "nghttp2-$NEW_VERSION" nghttp2
Expand Down
4 changes: 4 additions & 0 deletions tools/dep_updaters/update-nghttp3.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/ngtcp2/nghttp3/releases');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -44,6 +47,7 @@ cd "$WORKSPACE"

echo "Fetching nghttp3 source archive..."
curl -sL -o "$NGHTTP3_ZIP.zip" "https://github.com/ngtcp2/nghttp3/archive/refs/tags/$NGHTTP3_REF.zip"
log_and_verify_sha256sum "nghttp3" "$NGHTTP3_ZIP.zip"
unzip "$NGHTTP3_ZIP.zip"
rm "$NGHTTP3_ZIP.zip"
mv "$NGHTTP3_ZIP" nghttp3
Expand Down
4 changes: 4 additions & 0 deletions tools/dep_updaters/update-ngtcp2.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/ngtcp2/ngtcp2/releases');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -44,6 +47,7 @@ cd "$WORKSPACE"

echo "Fetching ngtcp2 source archive..."
curl -sL -o "$NGTCP2_ZIP.zip" "https://github.com/ngtcp2/ngtcp2/archive/refs/tags/$NGTCP2_REF.zip"
log_and_verify_sha256sum "ngtcp2" "$NGTCP2_ZIP.zip"
unzip "$NGTCP2_ZIP.zip"
rm "$NGTCP2_ZIP.zip"
mv "$NGTCP2_ZIP" ngtcp2
Expand Down
7 changes: 6 additions & 1 deletion tools/dep_updaters/update-npm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NPM="$DEPS_DIR/npm/bin/npm-cli.js"

NPM_VERSION=$1
Expand All @@ -30,12 +33,14 @@ trap cleanup INT TERM EXIT

cd "$WORKSPACE"

NPM_TGZ=npm.tgz
NPM_TGZ="npm-v$NPM_VERSION.tar.gz"

NPM_TARBALL="$($NODE "$NPM" view npm@"$NPM_VERSION" dist.tarball)"

curl -s "$NPM_TARBALL" > "$NPM_TGZ"

log_and_verify_sha256sum "npm" "$NPM_TGZ"

rm -rf "$DEPS_DIR/npm"

mkdir "$DEPS_DIR/npm"
Expand Down
105 changes: 105 additions & 0 deletions tools/dep_updaters/update-openssl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#!/bin/sh
set -e
# Shell script to update OpenSSL in the source tree to a specific version
# Based on https://github.com/nodejs/node/blob/main/doc/contributing/maintaining-openssl.md

cleanup() {
EXIT_CODE=$?
[ -d "$WORKSPACE" ] && rm -rf "$WORKSPACE"
exit $EXIT_CODE
}

download() {
if [ -z "$1" ]; then
echo "Error: please provide an OpenSSL version to update to"
echo " e.g. ./$0 download 3.0.7+quic1"
exit 1
fi

OPENSSL_VERSION=$1
echo "Making temporary workspace..."
WORKSPACE=$(mktemp -d 2> /dev/null || mktemp -d -t 'tmp')

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

cd "$WORKSPACE"

echo "Fetching OpenSSL source archive..."
OPENSSL_TARBALL="openssl-v$OPENSSL_VERSION.tar.gz"
curl -sL -o "$OPENSSL_TARBALL" "https://api.github.com/repos/quictls/openssl/tarball/openssl-$OPENSSL_VERSION"
log_and_verify_sha256sum "openssl" "$OPENSSL_TARBALL"
gzip -dc "$OPENSSL_TARBALL" | tar xf -
rm "$OPENSSL_TARBALL"
mv quictls-openssl-* openssl

echo "Replacing existing OpenSSL..."
rm -rf "$DEPS_DIR/openssl/openssl"
mv "$WORKSPACE/openssl" "$DEPS_DIR/openssl/"

echo "All done!"
echo ""
echo "Please git add openssl, and commit the new version:"
echo ""
echo "$ git add -A deps/openssl/openssl"
echo "$ git commit -m \"deps: upgrade openssl sources to quictls/openssl-$OPENSSL_VERSION\""
echo ""
}

regenerate() {
command -v perl >/dev/null 2>&1 || { echo >&2 "Error: 'Perl' required but not installed."; exit 1; }
command -v nasm >/dev/null 2>&1 || { echo >&2 "Error: 'nasm' required but not installed."; exit 1; }
command -v as >/dev/null 2>&1 || { echo >&2 "Error: 'GNU as' required but not installed."; exit 1; }
perl -e "use Text::Template">/dev/null 2>&1 || { echo >&2 "Error: 'Text::Template' Perl module required but not installed."; exit 1; }

echo "Regenerating platform-dependent files..."

make -C "$DEPS_DIR/openssl/config" clean
# Needed for compatibility with nasm on 32-bit Windows
# See https://github.com/nodejs/node/blob/main/doc/contributing/maintaining-openssl.md#2-execute-make-in-depsopensslconfig-directory
sed -i 's/#ifdef/%ifdef/g' "$DEPS_DIR/openssl/openssl/crypto/perlasm/x86asm.pl"
sed -i 's/#endif/%endif/g' "$DEPS_DIR/openssl/openssl/crypto/perlasm/x86asm.pl"
make -C "$DEPS_DIR/openssl/config"

echo "All done!"
echo ""
echo "Please commit the regenerated files:"
echo ""
echo "$ git add -A deps/openssl/config/archs deps/openssl/openssl"
echo "$ git commit -m \"deps: update archs files for openssl\""
echo ""
}

help() {
echo "Shell script to update OpenSSL in the source tree to a specific version"
echo "Sub-commands:"
printf "%-23s %s\n" "help" "show help menu and commands"
printf "%-23s %s\n" "download" "download and replace OpenSSL source code with new version"
printf "%-23s %s\n" "regenerate" "regenerate platform-specific files"
echo ""
exit "${1:-0}"
}

main() {
if [ ${#} -eq 0 ]; then
help 0
fi

trap cleanup INT TERM EXIT

BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd)
DEPS_DIR="$BASE_DIR/deps"

case ${1} in
help | download | regenerate )
$1 "${2}"
;;
* )
echo "unknown command: $1"
help 1
exit 1
;;
esac
}

main "$@"
6 changes: 5 additions & 1 deletion tools/dep_updaters/update-simdutf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/simdutf/simdutf/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -36,13 +39,14 @@ cleanup () {
trap cleanup INT TERM EXIT

SIMDUTF_REF="v$NEW_VERSION"
SIMDUTF_ZIP="simdutf-$NEW_VERSION.zip"
SIMDUTF_ZIP="simdutf-$SIMDUTF_REF.zip"
SIMDUTF_LICENSE="LICENSE-MIT"

cd "$WORKSPACE"

echo "Fetching simdutf source archive..."
curl -sL -o "$SIMDUTF_ZIP" "https://github.com/simdutf/simdutf/releases/download/$SIMDUTF_REF/singleheader.zip"
log_and_verify_sha256sum "simdutf" "$SIMDUTF_ZIP"
unzip "$SIMDUTF_ZIP"
rm "$SIMDUTF_ZIP"
rm ./*_demo.cpp
Expand Down
5 changes: 5 additions & 0 deletions tools/dep_updaters/update-uvwasi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
[ -x "$NODE" ] || NODE=$(command -v node)

# shellcheck disable=SC1091
. "$BASE_DIR/tools/dep_updaters/utils.sh"

NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
const res = await fetch('https://api.github.com/repos/nodejs/uvwasi/releases/latest');
if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
Expand Down Expand Up @@ -46,6 +49,8 @@ cd "$WORKSPACE"
echo "Fetching UVWASI source archive..."
curl -sL -o "$UVWASI_ZIP.zip" "https://github.com/nodejs/uvwasi/archive/refs/tags/v$NEW_VERSION.zip"

log_and_verify_sha256sum "uvwasi" "$UVWASI_ZIP.zip"

echo "Moving existing GYP build file"
mv "$DEPS_DIR/uvwasi/"*.gyp "$WORKSPACE/"
rm -rf "$DEPS_DIR/uvwasi/"
Expand Down

0 comments on commit 5767844

Please sign in to comment.