tools: log and verify sha256sum

fasenderos · web-flow · commit 847b9e08849f · 2023-05-25T19:46:07.000Z
PR-URL: #48088 Refs: nodejs/security-wg#973 Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com> Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
diff --git a/tools/dep_updaters/update-ada.sh b/tools/dep_updaters/update-ada.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/ada-url/ada/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -37,13 +40,14 @@ cleanup () {
 trap cleanup INT TERM EXIT
 
 ADA_REF="v$NEW_VERSION"
-ADA_ZIP="ada-$NEW_VERSION.zip"
+ADA_ZIP="ada-$ADA_REF.zip"
 ADA_LICENSE="LICENSE-MIT"
 
 cd "$WORKSPACE"
 
 echo "Fetching ada source archive..."
 curl -sL -o "$ADA_ZIP" "https://github.com/ada-url/ada/releases/download/$ADA_REF/singleheader.zip"
+log_and_verify_sha256sum "ada" "$ADA_ZIP"
 unzip "$ADA_ZIP"
 rm "$ADA_ZIP"
 
diff --git a/tools/dep_updaters/update-base64.sh b/tools/dep_updaters/update-base64.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/aklomp/base64/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -39,8 +42,13 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
+BASE64_TARBALL="base64-v$NEW_VERSION.tar.gz"
+
 echo "Fetching base64 source archive"
-curl -sL "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION" | tar xzf -
+curl -sL -o "$BASE64_TARBALL" "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION"
+log_and_verify_sha256sum "base64" "$BASE64_TARBALL"
+gzip -dc "$BASE64_TARBALL" | tar xf -
+rm "$BASE64_TARBALL"
 mv aklomp-base64-* base64
 
 echo "Replacing existing base64"
diff --git a/tools/dep_updaters/update-brotli.sh b/tools/dep_updaters/update-brotli.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/google/brotli/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,10 +47,11 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
-BROTLI_TARBALL="v$NEW_VERSION.tar.gz"
+BROTLI_TARBALL="brotli-v$NEW_VERSION.tar.gz"
 
 echo "Fetching brotli source archive"
-curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/$BROTLI_TARBALL"
+curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/v$NEW_VERSION.tar.gz"
+log_and_verify_sha256sum "brotli" "$BROTLI_TARBALL"
 gzip -dc "$BROTLI_TARBALL" | tar xf -
 rm "$BROTLI_TARBALL"
 mv "brotli-$NEW_VERSION" "brotli"
diff --git a/tools/dep_updaters/update-c-ares.sh b/tools/dep_updaters/update-c-ares.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/c-ares/c-ares/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -43,7 +46,10 @@ ARES_TARBALL="c-ares-$NEW_VERSION.tar.gz"
 cd "$WORKSPACE"
 
 echo "Fetching c-ares source archive"
-curl -sL "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL" | tar xz
+curl -sL -o "$ARES_TARBALL" "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL"
+log_and_verify_sha256sum "c-ares" "$ARES_TARBALL"
+gzip -dc "$ARES_TARBALL" | tar xf -
+rm "$ARES_TARBALL"
 mv "c-ares-$NEW_VERSION" cares
 
 echo "Removing tests"
diff --git a/tools/dep_updaters/update-libuv.sh b/tools/dep_updaters/update-libuv.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/libuv/libuv/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -46,8 +49,13 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
+LIBUV_TARBALL="libuv-v$NEW_VERSION.tar.gz"
+
 echo "Fetching libuv source archive..."
-curl -sL "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION" | tar xzf -
+curl -sL -o "$LIBUV_TARBALL" "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION"
+log_and_verify_sha256sum "libuv" "$LIBUV_TARBALL"
+gzip -dc "$LIBUV_TARBALL" | tar xf -
+rm "$LIBUV_TARBALL"
 mv libuv-libuv-* uv
 
 echo "Replacing existing libuv (except GYP build files)"
diff --git a/tools/dep_updaters/update-llhttp.sh b/tools/dep_updaters/update-llhttp.sh
@@ -9,6 +9,9 @@ DEPS_DIR="${BASE_DIR}/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/nodejs/llhttp/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -52,19 +55,20 @@ if echo "$NEW_VERSION" | grep -qs "/" ; then # Download a release
   echo "Checking out branch $BRANCH ..."
   git checkout "$BRANCH"
 
-  echo "Building llhtttp ..."
+  echo "Building llhttp ..."
   npm install
   make release
 
-  echo "Copying llhtttp release ..."
+  echo "Copying llhttp release ..."
   rm -rf "$DEPS_DIR/llhttp"
   cp -a release "$DEPS_DIR/llhttp"
 else
   echo "Download llhttp release $NEW_VERSION ..."
-  curl -sL -o llhttp.tar.gz "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz"
-  gzip -dc llhttp.tar.gz | tar xf -
+  LLHTTP_TARBALL="llhttp-v$NEW_VERSION.tar.gz"
+  curl -sL -o "$LLHTTP_TARBALL" "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz"
+  gzip -dc "$LLHTTP_TARBALL" | tar xf -
 
-  echo "Copying llhtttp release ..."
+  echo "Copying llhttp release ..."
   rm -rf "$DEPS_DIR/llhttp"
   cp -a "llhttp-release-v$NEW_VERSION" "$DEPS_DIR/llhttp"
 fi
diff --git a/tools/dep_updaters/update-nghttp2.sh b/tools/dep_updaters/update-nghttp2.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/nghttp2/nghttp2/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,6 +47,11 @@ cd "$WORKSPACE"
 
 echo "Fetching nghttp2 source archive"
 curl -sL -o "$NGHTTP2_TARBALL" "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/$NGHTTP2_TARBALL"
+
+DEPOSITED_CHECKSUM=$(curl -sL "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/checksums.txt" | grep "$NGHTTP2_TARBALL")
+
+log_and_verify_sha256sum "nghttp2" "$NGHTTP2_TARBALL" "$DEPOSITED_CHECKSUM"
+
 gzip -dc "$NGHTTP2_TARBALL" | tar xf -
 rm "$NGHTTP2_TARBALL"
 mv "nghttp2-$NEW_VERSION" nghttp2
diff --git a/tools/dep_updaters/update-nghttp3.sh b/tools/dep_updaters/update-nghttp3.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/ngtcp2/nghttp3/releases');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,6 +47,7 @@ cd "$WORKSPACE"
 
 echo "Fetching nghttp3 source archive..."
 curl -sL -o "$NGHTTP3_ZIP.zip" "https://github.com/ngtcp2/nghttp3/archive/refs/tags/$NGHTTP3_REF.zip"
+log_and_verify_sha256sum "nghttp3" "$NGHTTP3_ZIP.zip"
 unzip "$NGHTTP3_ZIP.zip"
 rm "$NGHTTP3_ZIP.zip"
 mv "$NGHTTP3_ZIP" nghttp3
diff --git a/tools/dep_updaters/update-ngtcp2.sh b/tools/dep_updaters/update-ngtcp2.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/ngtcp2/ngtcp2/releases');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,6 +47,7 @@ cd "$WORKSPACE"
 
 echo "Fetching ngtcp2 source archive..."
 curl -sL -o "$NGTCP2_ZIP.zip" "https://github.com/ngtcp2/ngtcp2/archive/refs/tags/$NGTCP2_REF.zip"
+log_and_verify_sha256sum "ngtcp2" "$NGTCP2_ZIP.zip"
 unzip "$NGTCP2_ZIP.zip"
 rm "$NGTCP2_ZIP.zip"
 mv "$NGTCP2_ZIP" ngtcp2
diff --git a/tools/dep_updaters/update-npm.sh b/tools/dep_updaters/update-npm.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NPM="$DEPS_DIR/npm/bin/npm-cli.js"
 
 NPM_VERSION=$1
@@ -30,12 +33,14 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
-NPM_TGZ=npm.tgz
+NPM_TGZ="npm-v$NPM_VERSION.tar.gz"
 
 NPM_TARBALL="$($NODE "$NPM" view npm@"$NPM_VERSION" dist.tarball)"
 
 curl -s "$NPM_TARBALL" > "$NPM_TGZ"
 
+log_and_verify_sha256sum "npm" "$NPM_TGZ"
+
 rm -rf "$DEPS_DIR/npm"
 
 mkdir "$DEPS_DIR/npm"
diff --git a/tools/dep_updaters/update-openssl.sh b/tools/dep_updaters/update-openssl.sh
@@ -20,11 +20,17 @@ download() {
   echo "Making temporary workspace..."
   WORKSPACE=$(mktemp -d 2> /dev/null || mktemp -d -t 'tmp')
 
+  # shellcheck disable=SC1091
+  . "$BASE_DIR/tools/dep_updaters/utils.sh"
 
   cd "$WORKSPACE"
 
   echo "Fetching OpenSSL source archive..."
-  curl -sL "https://api.github.com/repos/quictls/openssl/tarball/openssl-$OPENSSL_VERSION" | tar xzf -
+  OPENSSL_TARBALL="openssl-v$OPENSSL_VERSION.tar.gz"
+  curl -sL -o "$OPENSSL_TARBALL" "https://api.github.com/repos/quictls/openssl/tarball/openssl-$OPENSSL_VERSION"
+  log_and_verify_sha256sum "openssl" "$OPENSSL_TARBALL"
+  gzip -dc "$OPENSSL_TARBALL" | tar xf -
+  rm "$OPENSSL_TARBALL"
   mv quictls-openssl-* openssl
 
   echo "Replacing existing OpenSSL..."
diff --git a/tools/dep_updaters/update-simdutf.sh b/tools/dep_updaters/update-simdutf.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/simdutf/simdutf/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -36,13 +39,14 @@ cleanup () {
 trap cleanup INT TERM EXIT
 
 SIMDUTF_REF="v$NEW_VERSION"
-SIMDUTF_ZIP="simdutf-$NEW_VERSION.zip"
+SIMDUTF_ZIP="simdutf-$SIMDUTF_REF.zip"
 SIMDUTF_LICENSE="LICENSE-MIT"
 
 cd "$WORKSPACE"
 
 echo "Fetching simdutf source archive..."
 curl -sL -o "$SIMDUTF_ZIP" "https://github.com/simdutf/simdutf/releases/download/$SIMDUTF_REF/singleheader.zip"
+log_and_verify_sha256sum "simdutf" "$SIMDUTF_ZIP"
 unzip "$SIMDUTF_ZIP"
 rm "$SIMDUTF_ZIP"
 rm ./*_demo.cpp
diff --git a/tools/dep_updaters/update-uvwasi.sh b/tools/dep_updaters/update-uvwasi.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/nodejs/uvwasi/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -46,6 +49,8 @@ cd "$WORKSPACE"
 echo "Fetching UVWASI source archive..."
 curl -sL -o "$UVWASI_ZIP.zip" "https://github.com/nodejs/uvwasi/archive/refs/tags/v$NEW_VERSION.zip"
 
+log_and_verify_sha256sum "uvwasi" "$UVWASI_ZIP.zip"
+
 echo "Moving existing GYP build file"
 mv "$DEPS_DIR/uvwasi/"*.gyp "$WORKSPACE/"
 rm -rf "$DEPS_DIR/uvwasi/"
diff --git a/tools/dep_updaters/update-zlib.sh b/tools/dep_updaters/update-zlib.sh
@@ -7,6 +7,9 @@ set -e
 BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd)
 DEPS_DIR="$BASE_DIR/deps"
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 echo "Comparing latest upstream with current revision"
 
 git fetch https://chromium.googlesource.com/chromium/src/third_party/zlib.git HEAD
@@ -49,10 +52,12 @@ cd "$WORKSPACE"
 
 mkdir zlib
 
-ZLIB_TARBALL=zlib.tar.gz
+ZLIB_TARBALL="zlib-v$NEW_VERSION.tar.gz"
 
 echo "Fetching zlib source archive"
-curl -sL -o $ZLIB_TARBALL https://chromium.googlesource.com/chromium/src/+archive/refs/heads/main/third_party/$ZLIB_TARBALL
+curl -sL -o "$ZLIB_TARBALL" https://chromium.googlesource.com/chromium/src/+archive/refs/heads/main/third_party/zlib.tar.gz
+
+log_and_verify_sha256sum "zlib" "$ZLIB_TARBALL"
 
 gzip -dc "$ZLIB_TARBALL" | tar xf - -C zlib/
 
diff --git a/tools/dep_updaters/utils.sh b/tools/dep_updaters/utils.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# This function logs the archive checksum and, if provided, compares it with
+# the deposited checksum
+#
+# $1 is the package name e.g. 'acorn', 'ada', 'base64' etc. See that file
+# for a complete list of package name
+# $2 is the downloaded archive
+# $3 (optional) is the deposited sha256 cheksum. When provided, it is checked
+# against the checksum generated from the archive
+log_and_verify_sha256sum() {
+  package_name="$1"
+  archive="$2"
+  checksum="$3"
+  bsd_formatted_checksum=$(sha256sum --tag "$archive")
+  if [ -z "$3" ]; then
+    echo "$bsd_formatted_checksum"
+  else
+    archive_checksum=$(sha256sum "$archive")
+    if [ "$checksum" = "$archive_checksum" ]; then
+      echo "Valid $package_name checksum"
+      echo "$bsd_formatted_checksum"
+    else
+      echo "ERROR - Invalid $package_name checksum:"
+      echo "deposited: $checksum"
+      echo "generated: $archive_checksum"
+      exit 1
+    fi
+  fi
+}
