treefmt v2.4.0traversed 940 files emitted 234 files for processing formatted 234 files (4 changed) in 1.319s M ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh M ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh M ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh M ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index 671c392..1cbe9f2 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash -# Common functions and variables used by +# Common functions and variables used by # prepare.sh, initiate.sh and complete.sh #staging/prod diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index e31a74a..ecea3de 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -133,11 +133,11 @@ cleanup() { # can fail under set -e, or the restart below would be refused. unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d" - # jparem : $LOG_FILE exists, even if directory - # ${MOUNT_POINT}/pgdata/pg_upgrade_output.d/ does not! - # so removing the ship-ing of the log here rather than the conditional block - # down there.. - # https://linear.app/supabase/issue/MPG-1531/initiatesh-call-ship-logs-no-matter-what + # jparem : $LOG_FILE exists, even if directory + # ${MOUNT_POINT}/pgdata/pg_upgrade_output.d/ does not! + # so removing the ship-ing of the log here rather than the conditional block + # down there.. + # https://linear.app/supabase/issue/MPG-1531/initiatesh-call-ship-logs-no-matter-what ship_logs "$LOG_FILE" || true if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then @@ -657,71 +657,71 @@ EOF CI_stop_postgres fi - #CHECKSUM_FLAG="--data-checksums" if the SOURCE cluster has checksums - #CHECKSUM_FLAG="" if the SOURCE doesn't have them (yet) - # in this case the following code adds them + #CHECKSUM_FLAG="--data-checksums" if the SOURCE cluster has checksums + #CHECKSUM_FLAG="" if the SOURCE doesn't have them (yet) + # in this case the following code adds them - if [[ "$CHECKSUM_FLAG" == "" ]]; then + if [[ $CHECKSUM_FLAG == "" ]]; then - # old db hasn't got checksum, we will enable it its overall size - # (${PGDATAOLD}) is <= 20 Gb + # old db hasn't got checksum, we will enable it its overall size + # (${PGDATAOLD}) is <= 20 Gb - # 20 gb = 20 x 1024 x 1024 = 20971520 - MAX_SIZE=20971520 - - PGDATAOLD_SIZE=$(du -sk "${PGDATAOLD}" | cut -f1) + # 20 gb = 20 x 1024 x 1024 = 20971520 + MAX_SIZE=20971520 - if [[ "$PGDATAOLD_SIZE" -le "$MAX_SIZE" ]]; then + PGDATAOLD_SIZE=$(du -sk "${PGDATAOLD}" | cut -f1) - log "9.1 adding checksums to SOURCE PG (<20 Gb)" + if [[ $PGDATAOLD_SIZE -le $MAX_SIZE ]]; then - # we're adding a timer: the "pg_checksum -e" operation on ${PGDATAOLD} must - # not run for more than 10 minutes otherwise we exist this code on error - # ofc, if the pg_checksums detects a physical corruption, we have to exit - # the program with a consistent message explaining the corruption - - MAX_DURATION="10m" - KILL_AFTER="15s" + log "9.1 adding checksums to SOURCE PG (<20 Gb)" - timeout -k "${KILL_AFTER}" "${MAX_DURATION}" ${PGBINOLD}/pg_checksums -e "${PGDATAOLD}" || STATUS=$? + # we're adding a timer: the "pg_checksum -e" operation on ${PGDATAOLD} must + # not run for more than 10 minutes otherwise we exist this code on error + # ofc, if the pg_checksums detects a physical corruption, we have to exit + # the program with a consistent message explaining the corruption - # if everything is fine, $STATUS hasn't been defined, - # so we initialize it with 0 because it's a success - STATUS=${STATUS:-0} + MAX_DURATION="10m" + KILL_AFTER="15s" + + timeout -k "${KILL_AFTER}" "${MAX_DURATION}" ${PGBINOLD}/pg_checksums -e "${PGDATAOLD}" || STATUS=$? + + # if everything is fine, $STATUS hasn't been defined, + # so we initialize it with 0 because it's a success + STATUS=${STATUS:-0} # we'll manage ERROR codes down there, to actually log anything # properly, for later diagnostic reading the $LOG_FILE into project # where it has been shipped ship_logs() in common.sh - case $STATUS in - 0) - log "adding checksums SUCCEEDED." - ;; - 124|137) - log "adding checksums FAILED. Max time (${MAX_DURATION}) reached! Process killed." - exit 124 - ;; - *) - log "adding checksum FAILED with Error code ${STATUS}" - exit "$STATUS" - ;; - esac - - # we're here if everything went well with STATUS=0 - # so now, checksums are enabled on the SOURCE! - CHECKSUM_FLAG="--data-checksums" - - else - - log "9.1 did *not* add checksum to SOURCE PG: size $PGDATAOLD_SIZE > 20 Gb!" - - fi - - else - - log "9.1 checksums already present on SOURCE" - - fi + case $STATUS in + 0) + log "adding checksums SUCCEEDED." + ;; + 124 | 137) + log "adding checksums FAILED. Max time (${MAX_DURATION}) reached! Process killed." + exit 124 + ;; + *) + log "adding checksum FAILED with Error code ${STATUS}" + exit "$STATUS" + ;; + esac + + # we're here if everything went well with STATUS=0 + # so now, checksums are enabled on the SOURCE! + CHECKSUM_FLAG="--data-checksums" + + else + + log "9.1 did *not* add checksum to SOURCE PG: size $PGDATAOLD_SIZE > 20 Gb!" + + fi + + else + + log "9.1 checksums already present on SOURCE" + + fi log "10. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh index 12c8e19..2583295 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh @@ -22,7 +22,7 @@ chmod 640 "$LOG_FILE" # file, a previous attempt has run and failed. RETRY=0 if grep -qF "prepare.sh started" "$LOG_FILE"; then - RETRY=1 + RETRY=1 fi # From here on, everything written to stdout/stderr (log messages, but also @@ -40,8 +40,8 @@ trap 'log "prepare.sh FAILED at line $LINENO: \"$BASH_COMMAND\" (exit code $?)"' # Source the shared helpers (log and ship_logs functions) # shellcheck disable=SC1091 source "$SCRIPT_DIR/common.sh" || { - log "cannot source $SCRIPT_DIR/common.sh" - exit 1 + log "cannot source $SCRIPT_DIR/common.sh" + exit 1 } # ship_logs reads these variables, and "set -u" would abort the script on an @@ -57,16 +57,16 @@ source "$SCRIPT_DIR/common.sh" || { # retry, so the caller handles the "gave up" case. # The script's exit code is always preserved for the caller. on_exit() { - local rc=$? - trap - EXIT ERR # avoid re-entering the traps from inside this handler - - if [ "$rc" -eq 0 ]; then - log "prepare.sh ended with no errors." - ship_logs "$LOG_FILE" completion || log "Log upload failed (non-blocking)." - else - ship_logs "$LOG_FILE" prepare-failed || log "Log upload failed (non-blocking)." - fi - exit "$rc" + local rc=$? + trap - EXIT ERR # avoid re-entering the traps from inside this handler + + if [ "$rc" -eq 0 ]; then + log "prepare.sh ended with no errors." + ship_logs "$LOG_FILE" completion || log "Log upload failed (non-blocking)." + else + ship_logs "$LOG_FILE" prepare-failed || log "Log upload failed (non-blocking)." + fi + exit "$rc" } trap on_exit EXIT @@ -75,7 +75,10 @@ trap on_exit EXIT log "prepare.sh started." # check that /data is actually mounted -mountpoint -q /data || { log "/data is not mounted"; exit 1; } +mountpoint -q /data || { + log "/data is not mounted" + exit 1 +} # stop postgresql: "systemctl stop blocks until the unit has fully stopped" # https://linear.app/supabase/issue/MPG-1283/harden-a-bit-preparesh#comment-cea7446f @@ -84,17 +87,19 @@ systemctl stop postgresql # ...but let's still verify that the service is explicitly stopped if systemctl is-active --quiet postgresql; then - log "postgresql service is still active!"; exit 1 + log "postgresql service is still active!" + exit 1 fi # ...and that there are no remaining postgres processes if pgrep -x postgres >/dev/null; then - log "postgres processes are still alive!"; exit 1 + log "postgres processes are still alive!" + exit 1 fi # copy the pgsodium root key with proper owner and permissions install -m 600 -o postgres -g postgres \ - /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key + /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key # flush cached writes to persistent storage sync @@ -102,12 +107,12 @@ sync # Unmount /data. Capture the native umount error message and route it through # "log" so that it is timestamped like the rest of the log entries. if ! umount_out=$(umount /data 2>&1); then - log "Unable to unmount /data: $umount_out" - # List the processes still using /data ("|| true" so a fuser failure - # does not trigger the ERR trap and mask the real cause) - log "Processes using /data:" - log "$(ls -l /proc/*/fd /proc/*/cwd 2>/dev/null | grep ' /data' || true)" - exit 1 + log "Unable to unmount /data: $umount_out" + # List the processes still using /data ("|| true" so a fuser failure + # does not trigger the ERR trap and mask the real cause) + log "Processes using /data:" + log "$(ls -l /proc/*/fd /proc/*/cwd 2>/dev/null | grep ' /data' || true)" + exit 1 fi # Reaching this point means success: the final message and the log upload diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh index c40fe69..1537a5c 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh @@ -4,9 +4,9 @@ # file copy. Sourced by initiate.sh and tests/pg_upgrade/test-start-guard.sh. # Expects log, IS_CI, PGDATAOLD and PGDATANEW from the caller. -# From step 9 until cleanup, nothing may start the source postgres: -# -# If it has to run, "pg_checksums -e" will operate on the source postgres, and +# From step 9 until cleanup, nothing may start the source postgres: +# +# If it has to run, "pg_checksums -e" will operate on the source postgres, and # this one has to stay stopped during the whole operation # # pg_upgrade copies the data files without a postmaster holding the directory,