From 195328f4e51c0175682998d7a15c6ed54a96aa85 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 16 May 2023 01:30:10 +0300 Subject: [PATCH 1/3] chore: log upgrade failures remotely to project table --- .../admin_api_scripts/pg_upgrade_common.sh | 63 +++++++++++++++++++ .../admin_api_scripts/pg_upgrade_complete.sh | 50 +++++++++------ .../admin_api_scripts/pg_upgrade_initiate.sh | 15 +++-- 3 files changed, 104 insertions(+), 24 deletions(-) create mode 100755 ansible/files/admin_api_scripts/pg_upgrade_common.sh mode change 100644 => 100755 ansible/files/admin_api_scripts/pg_upgrade_complete.sh diff --git a/ansible/files/admin_api_scripts/pg_upgrade_common.sh b/ansible/files/admin_api_scripts/pg_upgrade_common.sh new file mode 100755 index 000000000..98232bbae --- /dev/null +++ b/ansible/files/admin_api_scripts/pg_upgrade_common.sh @@ -0,0 +1,63 @@ +#! /usr/bin/env bash + +# Common functions and variables used by pg_upgrade_initiate.sh and pg_upgrade_complete.sh + +REPORTING_PROJECT_REF="ihmaxnjpcccasmrbkpvo" +REPORTING_CREDENTIALS_FILE="/root/upgrade-reporting-credentials" + +REPORTING_ANON_KEY="" +if [ -f "$REPORTING_CREDENTIALS_FILE" ]; then + REPORTING_ANON_KEY=$(cat "$REPORTING_CREDENTIALS_FILE") +fi + +function run_sql { + psql -h localhost -U supabase_admin -d postgres "$@" +} + +function ship_logs { + LOG_FILE=$1 + + if [ -z "$REPORTING_ANON_KEY" ]; then + echo "No reporting key found. Skipping log upload." + return 0 + fi + + if [ ! -f "$LOG_FILE" ]; then + echo "No log file found. Skipping log upload." + return 0 + fi + + if [ ! -s "$LOG_FILE" ]; then + echo "Log file is empty. Skipping log upload." + return 0 + fi + + HOSTNAME=$(hostname) + DERIVED_REF="${HOSTNAME##*-}" + + printf -v BODY '{ "ref": "%s", "step": "%s", "content": %s }' "$DERIVED_REF" "completion" "$(cat "$LOG_FILE" | jq -Rs '.')" + curl -s -X POST "https://$REPORTING_PROJECT_REF.supabase.co/rest/v1/error_logs" \ + -H "apikey: ${REPORTING_ANON_KEY}" \ + -H 'Content-type: application/json' \ + -d "$BODY" +} + +function retry { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** (count + 1))) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + echo "Command $* exited with code $exit, retrying..." + sleep $wait + else + echo "Command $* exited with code $exit, no more retries left." + return $exit + fi + done + return 0 +} \ No newline at end of file diff --git a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh old mode 100644 new mode 100755 index 5e0330c2f..e8a09a6b8 --- a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh @@ -7,16 +7,19 @@ set -eEuo pipefail -run_sql() { - psql -h localhost -U supabase_admin -d postgres "$@" -} +# shellcheck disable=SC1091 +source ./pg_upgrade_common.sh + +LOG_FILE="/tmp/pg-upgrade-complete.log" -cleanup() { +function cleanup { UPGRADE_STATUS=${1:-"failed"} EXIT_CODE=${?:-0} echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status + ship_logs "$LOG_FILE" || true + exit "$EXIT_CODE" } @@ -29,18 +32,34 @@ function complete_pg_upgrade { echo "running" > /tmp/pg-upgrade-status echo "1. Mounting data disk" - mount -a -v + retry 3 mount -a -v # copying custom configurations echo "2. Copying custom configurations" - cp -R /data/conf/* /etc/postgresql-custom/ - chown -R postgres:postgres /var/lib/postgresql/data - chown -R postgres:postgres /data/pgdata + retry 3 copy_configs echo "3. Starting postgresql" - service postgresql start + retry 3 service postgresql start echo "4. Running generated SQL files" + retry 3 run_generated_sql + + sleep 5 + + echo "5. Restarting postgresql" + retry 3 service postgresql restart + + echo "6. Starting vacuum analyze" + retry 3 start_vacuum_analyze +} + +function copy_configs { + cp -R /data/conf/* /etc/postgresql-custom/ + chown -R postgres:postgres /var/lib/postgresql/data + chown -R postgres:postgres /data/pgdata +} + +function run_generated_sql { if [ -d /data/sql ]; then for FILE in /data/sql/*.sql; do if [ -f "$FILE" ]; then @@ -48,22 +67,15 @@ function complete_pg_upgrade { fi done fi - - sleep 5 - - echo "5. Restarting postgresql" - service postgresql restart - - echo "6. Starting vacuum analyze" - start_vacuum_analyze } function start_vacuum_analyze { + echo "complete" > /tmp/pg-upgrade-status su -c 'vacuumdb --all --analyze-in-stages' -s "$SHELL" postgres echo "Upgrade job completed" - cleanup "complete" } trap cleanup ERR -complete_pg_upgrade >>/var/log/pg-upgrade-complete.log 2>&1 & + +complete_pg_upgrade >> $LOG_FILE 2>&1 & diff --git a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh index ce757bbee..fc761cfbf 100644 --- a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh @@ -27,6 +27,11 @@ PG13_EXTENSIONS_TO_DISABLE=( set -eEuo pipefail +# shellcheck disable=SC1091 +source ./pg_upgrade_common.sh + +LOG_FILE="/var/log/pg-upgrade-initiate.log" + PGVERSION=$1 IS_DRY_RUN=${2:-false} if [ "$IS_DRY_RUN" != false ]; then @@ -35,12 +40,9 @@ fi MOUNT_POINT="/data_migration" -run_sql() { - psql -h localhost -U supabase_admin -d postgres "$@" -} - POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") + # If upgrading from older major PG versions, disable specific extensions if [[ "$OLD_PGVERSION" =~ 14* ]]; then EXTENSIONS_TO_DISABLE+=("${PG14_EXTENSIONS_TO_DISABLE[@]}") @@ -61,6 +63,9 @@ cleanup() { if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then echo "Copying pg_upgrade output to /var/log" cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ || true + ship_logs "$LOG_FILE" || true + tail -n +1 /var/log/pg_upgrade_output.d/*/* > /var/log/pg_upgrade_output.d/pg_upgrade.log || true + ship_logs "/var/log/pg_upgrade_output.d/pg_upgrade.log" || true fi if [ -L /var/lib/postgresql ]; then @@ -279,6 +284,6 @@ echo "running" > /tmp/pg-upgrade-status if [ "$IS_DRY_RUN" = true ]; then initiate_upgrade else - initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1 & + initiate_upgrade >> "$LOG_FILE" 2>&1 & echo "Upgrade initiate job completed" fi From f4c290f133a9f95c2116c3e68b3ce56495274848 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 16 May 2023 13:28:56 +0300 Subject: [PATCH 2/3] chore: nits --- ansible/files/admin_api_scripts/pg_upgrade_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_common.sh b/ansible/files/admin_api_scripts/pg_upgrade_common.sh index 98232bbae..af7644b28 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_common.sh @@ -36,7 +36,7 @@ function ship_logs { DERIVED_REF="${HOSTNAME##*-}" printf -v BODY '{ "ref": "%s", "step": "%s", "content": %s }' "$DERIVED_REF" "completion" "$(cat "$LOG_FILE" | jq -Rs '.')" - curl -s -X POST "https://$REPORTING_PROJECT_REF.supabase.co/rest/v1/error_logs" \ + curl -sf -X POST "https://$REPORTING_PROJECT_REF.supabase.co/rest/v1/error_logs" \ -H "apikey: ${REPORTING_ANON_KEY}" \ -H 'Content-type: application/json' \ -d "$BODY" @@ -60,4 +60,4 @@ function retry { fi done return 0 -} \ No newline at end of file +} From 1cf703df0ce81a22655b4f415d2db3ff62f93eb9 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 16 May 2023 13:30:27 +0300 Subject: [PATCH 3/3] chore bump version --- common.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 058bd9d1b..0c9379378 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.0.79" +postgres-version = "15.1.0.80"