Merge pull request #213 from infosiftr/jq-template

Add initial jq-based templating engine

Author: yosifkit

.gitattributes

@@ -0,0 +1,3 @@
+**/Dockerfile            linguist-generated
+**/docker-entrypoint.sh  linguist-generated
+Dockerfile.template      linguist-language=Dockerfile

.github/workflows/verify-templating.yml

@@ -0,0 +1,22 @@
+name: Verify Templating
+
+on:
+  pull_request:
+  push:
+
+defaults:
+  run:
+    shell: 'bash -Eeuo pipefail -x {0}'
+
+jobs:
+  apply-templates:
+    name: Check For Uncomitted Changes
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Apply Templates
+        run: ./apply-templates.sh
+      - name: Check Git Status
+        run: |
+          status="$(git status --short)"
+          [ -z "$status" ]

.gitignore

@@ -0,0 +1 @@
+.jq-template.awk

2.1/Dockerfile

@@ -1,4 +1,4 @@
-FROM adoptopenjdk:%%JAVA_VERSION%%-jre-hotspot-bionic
+FROM adoptopenjdk:{{ .java }}-jre-hotspot-bionic
 
 # explicitly set user/group IDs
 RUN set -eux; \
@@ -13,7 +13,15 @@ RUN set -eux; \
 # "free" is used by cassandra-env.sh
 		procps \
 # "cqlsh" needs a python interpreter
+{{
+	# python3 is only supported in 4.0+
+	# https://issues.apache.org/jira/browse/CASSANDRA-10190
+	if env.version | split(".") | .[0] | tonumber < 4 then (
+-}}
+		python \
+{{ ) else ( -}}
 		python3 \
+{{ ) end -}}
 # "ip" is not required by Cassandra itself, but is commonly used in scripting Cassandra's configuration (since it is so fixated on explicit IP addresses)
 		iproute2 \
 # Cassandra will automatically use numactl if available
@@ -59,8 +67,8 @@ ENV GPG_KEYS \
 # gpg: key E91335D77E3E87CB: public key "Michael Semb Wever <[email protected]>" imported
 	A4C465FEA0C552561A392A61E91335D77E3E87CB
 
-ENV CASSANDRA_VERSION %%CASSANDRA_VERSION%%
-ENV CASSANDRA_SHA512 %%CASSANDRA_SHA512%%
+ENV CASSANDRA_VERSION {{ .version }}
+ENV CASSANDRA_SHA512 {{ .sha512 }}
 
 RUN set -eux; \
 	savedAptMark="$(apt-mark showmanual)"; \
@@ -142,7 +150,9 @@ RUN set -eux; \
 VOLUME /var/lib/cassandra
 
 COPY docker-entrypoint.sh /usr/local/bin/
+{{ if env.version | split(".") | .[0] | tonumber < 4 then ( -}}
 RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
+{{ ) else "" end -}}
 ENTRYPOINT ["docker-entrypoint.sh"]
 
 # 7000: intra-node communication

2.2/Dockerfile

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[ -f versions.json ] # run "versions.sh" first
+
+jqt='.jq-template.awk'
+if [ -n "${BASHBREW_SCRIPTS:-}" ]; then
+	jqt="$BASHBREW_SCRIPTS/jq-template.awk"
+elif [ "$BASH_SOURCE" -nt "$jqt" ]; then
+	wget -qO "$jqt" 'https://github.com/docker-library/bashbrew/raw/ac3e8e9541cb362a579b05bec41dd40d1df1c6e6/scripts/jq-template.awk'
+fi
+
+if [ "$#" -eq 0 ]; then
+	versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)"
+	eval "set -- $versions"
+fi
+
+generated_warning() {
+	cat <<-EOH
+		#
+		# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh"
+		#
+		# PLEASE DO NOT EDIT IT DIRECTLY.
+		#
+
+	EOH
+}
+
+for version; do
+	export version
+
+	echo "processing $version ..."
+
+	{
+		generated_warning
+		gawk -f "$jqt" Dockerfile.template
+	} > "$version/Dockerfile"
+
+	cp -a docker-entrypoint.sh "$version/"
+done

3.0/Dockerfile

@@ -9,8 +9,13 @@ declare -A aliases=(
 self="$(basename "$BASH_SOURCE")"
 cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
 
-versions=( */ )
-versions=( "${versions[@]%/}" )
+if [ "$#" -eq 0 ]; then
+	versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)"
+	eval "set -- $versions"
+fi
+
+# sort version numbers with highest first
+IFS=$'\n'; set -- $(sort -rV <<<"$*"); unset IFS
 
 # get the most recent commit which modified any of "$@"
 fileCommit() {
@@ -65,10 +70,10 @@ join() {
 	echo "${out#$sep}"
 }
 
-for version in "${versions[@]}"; do
-	commit="$(dirCommit "$version")"
+for version; do
+	export version
 
-	fullVersion="$(git show "$commit":"$version/Dockerfile" | awk '$1 == "ENV" && $2 == "CASSANDRA_VERSION" { print $3; exit }')"
+	fullVersion="$(jq -r '.[env.version].version' versions.json)"
 
 	versionAliases=( $fullVersion )
 	if [ "$version" != "$fullVersion" ]; then
@@ -100,6 +105,8 @@ for version in "${versions[@]}"; do
 
 	arches="$(xargs -n1 <<<"$arches" | sort)"
 
+	commit="$(dirCommit "$version")"
+
 	echo
 	cat <<-EOE
 		Tags: $(join ', ' "${versionAliases[@]}")

3.11/Dockerfile

@@ -3,64 +3,5 @@ set -Eeuo pipefail
 
 cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
 
-defaultJavaVersion='11'
-declare -A javaVersions=(
-	[2.1]='8'
-	[2.2]='8'
-	[3.0]='8'
-	[3.11]='8'
-)
-
-versions=( "$@" )
-if [ ${#versions[@]} -eq 0 ]; then
-	versions=( */ )
-fi
-versions=( "${versions[@]%/}" )
-
-for version in "${versions[@]}"; do
-	possibleVersions=( $(
-		git ls-remote --tags 'https://gitbox.apache.org/repos/asf/cassandra.git' "refs/tags/cassandra-$version*" \
-			| cut -d/ -f3- \
-			| cut -d^ -f1 \
-			| cut -d- -f2- \
-			| sort -urV
-	) )
-
-	fullVersion=
-	sha512=
-	for possibleVersion in "${possibleVersions[@]}"; do
-		if sha512="$(wget -qO- "https://downloads.apache.org/cassandra/$possibleVersion/apache-cassandra-$possibleVersion-bin.tar.gz.sha512" | grep -oE '[a-f0-9]{128}')" && [ -n "$sha512" ]; then
-			fullVersion="$possibleVersion"
-			break
-		fi
-	done
-	if [ -z "$fullVersion" ]; then
-		echo >&2 "error: failed to find full version for $version"
-		exit 1
-	fi
-
-	echo "$version: $fullVersion"
-
-	javaVersion="${javaVersions[$version]:-$defaultJavaVersion}"
-	cp -a docker-entrypoint.sh "$version/"
-	sed \
-		-e "s/%%CASSANDRA_VERSION%%/$fullVersion/g" \
-		-e "s/%%CASSANDRA_SHA512%%/$sha512/g" \
-		-e "s/%%JAVA_VERSION%%/$javaVersion/g" \
-		Dockerfile.template > "$version/Dockerfile"
-
-	# remove the "/docker-entrypoint.sh" backwards-compatibility symlink in Cassandra 3.12+
-	case "$version" in
-		2.*|3.0|3.11) ;;
-		*) sed -i '/^RUN .* \/docker-entrypoint.sh # backwards compat$/d' "$version/Dockerfile" ;;
-	esac
-	# TODO once Cassandra 2.x and 3.x are deprecated, we should remove this from the template itself (and remove this code too)
-
-	# python3 is only supported in 4.0+
-	# https://issues.apache.org/jira/browse/CASSANDRA-10190
-	case "$version" in
-		2.* | 3.*)
-			sed -i 's/python3/python/g' "$version/Dockerfile"
-			;;
-	esac
-done
+./versions.sh "$@"
+./apply-templates.sh "$@"

4.0/Dockerfile

@@ -0,0 +1,27 @@
+{
+  "2.1": {
+    "java": "8",
+    "sha512": "ea2c35f3f9ce2be8eb96df603927c6867613f3e2215132145da2c3015dd1ddf1370d7196ef4485002e7a54fe22c32abd9c659102057f9297b69113bf83d96d0b",
+    "version": "2.1.21"
+  },
+  "2.2": {
+    "java": "8",
+    "sha512": "db2026342e876caf790833d49f7ab1a2fbba39bf380384ef66e2da4913f537690a56c97cb2f6ea17f667a0d34aeb406fa658db02aec1121a5ba7134ab59a5cfb",
+    "version": "2.2.16"
+  },
+  "3.0": {
+    "java": "8",
+    "sha512": "905ceb18da3391353b3904cdf243b30738167358655d80398d729b8e67fe5b3f1f10a51db19e30c42f7c511b564671785e7654b0892fc79a0fb7c4fda96dbe7b",
+    "version": "3.0.21"
+  },
+  "3.11": {
+    "java": "8",
+    "sha512": "c23a51d2d583e707fae8b0e0c413a6287a47af131650715cb3ae8d404b9958fca531c6d73cd196bde5054d2485cc12b6bf195de173f1baa647ac4cc012d9aecd",
+    "version": "3.11.7"
+  },
+  "4.0": {
+    "java": "11",
+    "sha512": "240ae95f78de172333eee865f01b838433845fbd0dceea0eb91ea3a419873f74c5e266cfb62553fa0260849afa7ec5cc65335d037d1455e36b96ddc0f18effc7",
+    "version": "4.0-beta1"
+  }
+}

Dockerfile.template

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+defaultJavaVersion='11'
+declare -A javaVersions=(
+	[2.1]='8'
+	[2.2]='8'
+	[3.0]='8'
+	[3.11]='8'
+)
+
+cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
+
+versions=( "$@" )
+if [ ${#versions[@]} -eq 0 ]; then
+	versions=( */ )
+	json='{}'
+else
+	json="$(< versions.json)"
+fi
+versions=( "${versions[@]%/}" )
+
+for version in "${versions[@]}"; do
+	export version
+
+	possibleVersions=( $(
+		git ls-remote --tags 'https://gitbox.apache.org/repos/asf/cassandra.git' "refs/tags/cassandra-$version*" \
+			| cut -d/ -f3- \
+			| cut -d^ -f1 \
+			| cut -d- -f2- \
+			| sort -urV
+	) )
+
+	fullVersion=
+	sha512=
+	for possibleVersion in "${possibleVersions[@]}"; do
+		if sha512="$(wget -qO- "https://downloads.apache.org/cassandra/$possibleVersion/apache-cassandra-$possibleVersion-bin.tar.gz.sha512" | grep -oE '[a-f0-9]{128}')" && [ -n "$sha512" ]; then
+			fullVersion="$possibleVersion"
+			break
+		fi
+	done
+	if [ -z "$fullVersion" ]; then
+		echo >&2 "error: failed to find full version for $version"
+		exit 1
+	fi
+	export fullVersion sha512
+
+	export javaVersion="${javaVersions[$version]:-$defaultJavaVersion}"
+
+	echo "$version: $fullVersion"
+
+	json="$(jq <<<"$json" -c '
+		.[env.version] = {
+			version: env.fullVersion,
+			sha512: env.sha512,
+			java: env.javaVersion,
+		}
+	')"
+done
+
+jq <<<"$json" -S . > versions.json