this derivation will be built: /nix/store/5ljcq0glvmp2v48xhnglk9wh4cingivm-treefmt-check.drv building '/nix/store/5ljcq0glvmp2v48xhnglk9wh4cingivm-treefmt-check.drv' treefmt-check> treefmt v2.4.0traversed 940 files treefmt-check> emitted 234 files for processing treefmt-check> formatted 234 files (4 changed) in 580ms treefmt-check> M ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh treefmt-check> M ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh treefmt-check> M ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh treefmt-check> M ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh treefmt-check> diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh treefmt-check> index 671c392..1cbe9f2 100755 treefmt-check> --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh treefmt-check> +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh treefmt-check> @@ -1,6 +1,6 @@ treefmt-check> #! /usr/bin/env bash treefmt-check> treefmt-check> -# Common functions and variables used by treefmt-check> +# Common functions and variables used by treefmt-check> # prepare.sh, initiate.sh and complete.sh treefmt-check> treefmt-check> #staging/prod treefmt-check> diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh treefmt-check> index e31a74a..ecea3de 100755 treefmt-check> --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh treefmt-check> +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh treefmt-check> @@ -133,11 +133,11 @@ cleanup() { treefmt-check> # can fail under set -e, or the restart below would be refused. treefmt-check> unblock_postgres_start || log "WARNING: failed to lift the postgres start block; check /run/systemd/system/postgresql.service.d" treefmt-check> treefmt-check> - # jparem : $LOG_FILE exists, even if directory treefmt-check> - # ${MOUNT_POINT}/pgdata/pg_upgrade_output.d/ does not! treefmt-check> - # so removing the ship-ing of the log here rather than the conditional block treefmt-check> - # down there.. treefmt-check> - # https://linear.app/supabase/issue/MPG-1531/initiatesh-call-ship-logs-no-matter-what treefmt-check> + # jparem : $LOG_FILE exists, even if directory treefmt-check> + # ${MOUNT_POINT}/pgdata/pg_upgrade_output.d/ does not! treefmt-check> + # so removing the ship-ing of the log here rather than the conditional block treefmt-check> + # down there.. treefmt-check> + # https://linear.app/supabase/issue/MPG-1531/initiatesh-call-ship-logs-no-matter-what treefmt-check> ship_logs "$LOG_FILE" || true treefmt-check> treefmt-check> if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then treefmt-check> @@ -657,71 +657,71 @@ EOF treefmt-check> CI_stop_postgres treefmt-check> fi treefmt-check> treefmt-check> - #CHECKSUM_FLAG="--data-checksums" if the SOURCE cluster has checksums treefmt-check> - #CHECKSUM_FLAG="" if the SOURCE doesn't have them (yet) treefmt-check> - # in this case the following code adds them treefmt-check> + #CHECKSUM_FLAG="--data-checksums" if the SOURCE cluster has checksums treefmt-check> + #CHECKSUM_FLAG="" if the SOURCE doesn't have them (yet) treefmt-check> + # in this case the following code adds them treefmt-check> treefmt-check> - if [[ "$CHECKSUM_FLAG" == "" ]]; then treefmt-check> + if [[ $CHECKSUM_FLAG == "" ]]; then treefmt-check> treefmt-check> - # old db hasn't got checksum, we will enable it its overall size treefmt-check> - # (${PGDATAOLD}) is <= 20 Gb treefmt-check> + # old db hasn't got checksum, we will enable it its overall size treefmt-check> + # (${PGDATAOLD}) is <= 20 Gb treefmt-check> treefmt-check> - # 20 gb = 20 x 1024 x 1024 = 20971520 treefmt-check> - MAX_SIZE=20971520 treefmt-check> - treefmt-check> - PGDATAOLD_SIZE=$(du -sk "${PGDATAOLD}" | cut -f1) treefmt-check> + # 20 gb = 20 x 1024 x 1024 = 20971520 treefmt-check> + MAX_SIZE=20971520 treefmt-check> treefmt-check> - if [[ "$PGDATAOLD_SIZE" -le "$MAX_SIZE" ]]; then treefmt-check> + PGDATAOLD_SIZE=$(du -sk "${PGDATAOLD}" | cut -f1) treefmt-check> treefmt-check> - log "9.1 adding checksums to SOURCE PG (<20 Gb)" treefmt-check> + if [[ $PGDATAOLD_SIZE -le $MAX_SIZE ]]; then treefmt-check> treefmt-check> - # we're adding a timer: the "pg_checksum -e" operation on ${PGDATAOLD} must treefmt-check> - # not run for more than 10 minutes otherwise we exist this code on error treefmt-check> - # ofc, if the pg_checksums detects a physical corruption, we have to exit treefmt-check> - # the program with a consistent message explaining the corruption treefmt-check> - treefmt-check> - MAX_DURATION="10m" treefmt-check> - KILL_AFTER="15s" treefmt-check> + log "9.1 adding checksums to SOURCE PG (<20 Gb)" treefmt-check> treefmt-check> - timeout -k "${KILL_AFTER}" "${MAX_DURATION}" ${PGBINOLD}/pg_checksums -e "${PGDATAOLD}" || STATUS=$? treefmt-check> + # we're adding a timer: the "pg_checksum -e" operation on ${PGDATAOLD} must treefmt-check> + # not run for more than 10 minutes otherwise we exist this code on error treefmt-check> + # ofc, if the pg_checksums detects a physical corruption, we have to exit treefmt-check> + # the program with a consistent message explaining the corruption treefmt-check> treefmt-check> - # if everything is fine, $STATUS hasn't been defined, treefmt-check> - # so we initialize it with 0 because it's a success treefmt-check> - STATUS=${STATUS:-0} treefmt-check> + MAX_DURATION="10m" treefmt-check> + KILL_AFTER="15s" treefmt-check> + treefmt-check> + timeout -k "${KILL_AFTER}" "${MAX_DURATION}" ${PGBINOLD}/pg_checksums -e "${PGDATAOLD}" || STATUS=$? treefmt-check> + treefmt-check> + # if everything is fine, $STATUS hasn't been defined, treefmt-check> + # so we initialize it with 0 because it's a success treefmt-check> + STATUS=${STATUS:-0} treefmt-check> treefmt-check> # we'll manage ERROR codes down there, to actually log anything treefmt-check> # properly, for later diagnostic reading the $LOG_FILE into project treefmt-check> # where it has been shipped ship_logs() in common.sh treefmt-check> treefmt-check> - case $STATUS in treefmt-check> - 0) treefmt-check> - log "adding checksums SUCCEEDED." treefmt-check> - ;; treefmt-check> - 124|137) treefmt-check> - log "adding checksums FAILED. Max time (${MAX_DURATION}) reached! Process killed." treefmt-check> - exit 124 treefmt-check> - ;; treefmt-check> - *) treefmt-check> - log "adding checksum FAILED with Error code ${STATUS}" treefmt-check> - exit "$STATUS" treefmt-check> - ;; treefmt-check> - esac treefmt-check> - treefmt-check> - # we're here if everything went well with STATUS=0 treefmt-check> - # so now, checksums are enabled on the SOURCE! treefmt-check> - CHECKSUM_FLAG="--data-checksums" treefmt-check> - treefmt-check> - else treefmt-check> - treefmt-check> - log "9.1 did *not* add checksum to SOURCE PG: size $PGDATAOLD_SIZE > 20 Gb!" treefmt-check> - treefmt-check> - fi treefmt-check> - treefmt-check> - else treefmt-check> - treefmt-check> - log "9.1 checksums already present on SOURCE" treefmt-check> - treefmt-check> - fi treefmt-check> + case $STATUS in treefmt-check> + 0) treefmt-check> + log "adding checksums SUCCEEDED." treefmt-check> + ;; treefmt-check> + 124 | 137) treefmt-check> + log "adding checksums FAILED. Max time (${MAX_DURATION}) reached! Process killed." treefmt-check> + exit 124 treefmt-check> + ;; treefmt-check> + *) treefmt-check> + log "adding checksum FAILED with Error code ${STATUS}" treefmt-check> + exit "$STATUS" treefmt-check> + ;; treefmt-check> + esac treefmt-check> + treefmt-check> + # we're here if everything went well with STATUS=0 treefmt-check> + # so now, checksums are enabled on the SOURCE! treefmt-check> + CHECKSUM_FLAG="--data-checksums" treefmt-check> + treefmt-check> + else treefmt-check> + treefmt-check> + log "9.1 did *not* add checksum to SOURCE PG: size $PGDATAOLD_SIZE > 20 Gb!" treefmt-check> + treefmt-check> + fi treefmt-check> + treefmt-check> + else treefmt-check> + treefmt-check> + log "9.1 checksums already present on SOURCE" treefmt-check> + treefmt-check> + fi treefmt-check> treefmt-check> log "10. Creating new data directory, initializing database" treefmt-check> chown -R postgres:postgres "$MOUNT_POINT/" treefmt-check> diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh treefmt-check> index 5ed5213..c4512db 100755 treefmt-check> --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh treefmt-check> +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh treefmt-check> @@ -22,7 +22,7 @@ chmod 640 "$LOG_FILE" treefmt-check> # file, a previous attempt has run and failed. treefmt-check> RETRY=0 treefmt-check> if grep -qF "prepare.sh started" "$LOG_FILE"; then treefmt-check> - RETRY=1 treefmt-check> + RETRY=1 treefmt-check> fi treefmt-check> treefmt-check> # From here on, everything written to stdout/stderr (log messages, but also treefmt-check> @@ -40,8 +40,8 @@ trap 'log "prepare.sh FAILED at line $LINENO: \"$BASH_COMMAND\" (exit code $?)"' treefmt-check> # Source the shared helpers (log and ship_logs functions) treefmt-check> # shellcheck disable=SC1091 treefmt-check> source "$SCRIPT_DIR/common.sh" || { treefmt-check> - log "cannot source $SCRIPT_DIR/common.sh" treefmt-check> - exit 1 treefmt-check> + log "cannot source $SCRIPT_DIR/common.sh" treefmt-check> + exit 1 treefmt-check> } treefmt-check> treefmt-check> # ship_logs reads these variables, and "set -u" would abort the script on an treefmt-check> @@ -57,23 +57,23 @@ source "$SCRIPT_DIR/common.sh" || { treefmt-check> # retry, so the caller handles the "gave up" case. treefmt-check> # The script's exit code is always preserved for the caller. treefmt-check> on_exit() { treefmt-check> - local rc=$? treefmt-check> - trap - EXIT ERR # avoid re-entering the traps from inside this handler treefmt-check> - treefmt-check> - if [ "$rc" -eq 0 ]; then treefmt-check> - if [ "$RETRY" -eq 0 ]; then treefmt-check> - log "prepare.sh ended with no errors." treefmt-check> - else treefmt-check> - log "prepare.sh finally ended without error this time." treefmt-check> - fi treefmt-check> - # Single upload of the log file; never fatal treefmt-check> - if declare -F ship_logs >/dev/null; then treefmt-check> - ship_logs "$LOG_FILE" || log "Log upload failed (non-blocking)." treefmt-check> - else treefmt-check> - log "ship_logs is not available, log upload skipped." treefmt-check> - fi treefmt-check> - fi treefmt-check> - exit "$rc" treefmt-check> + local rc=$? treefmt-check> + trap - EXIT ERR # avoid re-entering the traps from inside this handler treefmt-check> + treefmt-check> + if [ "$rc" -eq 0 ]; then treefmt-check> + if [ "$RETRY" -eq 0 ]; then treefmt-check> + log "prepare.sh ended with no errors." treefmt-check> + else treefmt-check> + log "prepare.sh finally ended without error this time." treefmt-check> + fi treefmt-check> + # Single upload of the log file; never fatal treefmt-check> + if declare -F ship_logs >/dev/null; then treefmt-check> + ship_logs "$LOG_FILE" || log "Log upload failed (non-blocking)." treefmt-check> + else treefmt-check> + log "ship_logs is not available, log upload skipped." treefmt-check> + fi treefmt-check> + fi treefmt-check> + exit "$rc" treefmt-check> } treefmt-check> trap on_exit EXIT treefmt-check> treefmt-check> @@ -82,7 +82,10 @@ trap on_exit EXIT treefmt-check> log "prepare.sh started." treefmt-check> treefmt-check> # check that /data is actually mounted treefmt-check> -mountpoint -q /data || { log "/data is not mounted"; exit 1; } treefmt-check> +mountpoint -q /data || { treefmt-check> + log "/data is not mounted" treefmt-check> + exit 1 treefmt-check> +} treefmt-check> treefmt-check> # stop postgresql: "systemctl stop blocks until the unit has fully stopped" treefmt-check> # https://linear.app/supabase/issue/MPG-1283/harden-a-bit-preparesh#comment-cea7446f treefmt-check> @@ -91,17 +94,19 @@ systemctl stop postgresql treefmt-check> treefmt-check> # ...but let's still verify that the service is explicitly stopped treefmt-check> if systemctl is-active --quiet postgresql; then treefmt-check> - log "postgresql service is still active!"; exit 1 treefmt-check> + log "postgresql service is still active!" treefmt-check> + exit 1 treefmt-check> fi treefmt-check> treefmt-check> # ...and that there are no remaining postgres processes treefmt-check> if pgrep -x postgres >/dev/null; then treefmt-check> - log "postgres processes are still alive!"; exit 1 treefmt-check> + log "postgres processes are still alive!" treefmt-check> + exit 1 treefmt-check> fi treefmt-check> treefmt-check> # copy the pgsodium root key with proper owner and permissions treefmt-check> install -m 600 -o postgres -g postgres \ treefmt-check> - /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key treefmt-check> + /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key treefmt-check> treefmt-check> # flush cached writes to persistent storage treefmt-check> sync treefmt-check> @@ -109,12 +114,12 @@ sync treefmt-check> # Unmount /data. Capture the native umount error message and route it through treefmt-check> # "log" so that it is timestamped like the rest of the log entries. treefmt-check> if ! umount_out=$(umount /data 2>&1); then treefmt-check> - log "Unable to unmount /data: $umount_out" treefmt-check> - # List the processes still using /data ("|| true" so a fuser failure treefmt-check> - # does not trigger the ERR trap and mask the real cause) treefmt-check> - log "Processes using /data:" treefmt-check> - log "$(ls -l /proc/*/fd /proc/*/cwd 2>/dev/null | grep ' /data' || true)" treefmt-check> - exit 1 treefmt-check> + log "Unable to unmount /data: $umount_out" treefmt-check> + # List the processes still using /data ("|| true" so a fuser failure treefmt-check> + # does not trigger the ERR trap and mask the real cause) treefmt-check> + log "Processes using /data:" treefmt-check> + log "$(ls -l /proc/*/fd /proc/*/cwd 2>/dev/null | grep ' /data' || true)" treefmt-check> + exit 1 treefmt-check> fi treefmt-check> treefmt-check> # Reaching this point means success: the final message and the log upload treefmt-check> diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh treefmt-check> index c40fe69..1537a5c 100755 treefmt-check> --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh treefmt-check> +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh treefmt-check> @@ -4,9 +4,9 @@ treefmt-check> # file copy. Sourced by initiate.sh and tests/pg_upgrade/test-start-guard.sh. treefmt-check> # Expects log, IS_CI, PGDATAOLD and PGDATANEW from the caller. treefmt-check> treefmt-check> -# From step 9 until cleanup, nothing may start the source postgres: treefmt-check> -# treefmt-check> -# If it has to run, "pg_checksums -e" will operate on the source postgres, and treefmt-check> +# From step 9 until cleanup, nothing may start the source postgres: treefmt-check> +# treefmt-check> +# If it has to run, "pg_checksums -e" will operate on the source postgres, and treefmt-check> # this one has to stay stopped during the whole operation treefmt-check> # treefmt-check> # pg_upgrade copies the data files without a postmaster holding the directory, error: Cannot build '/nix/store/5ljcq0glvmp2v48xhnglk9wh4cingivm-treefmt-check.drv'. Reason: builder failed with exit code 1. Output paths: /nix/store/9c6lj79prv2hyamsd4lm4lj5nlj2dzy3-treefmt-check Last 25 log lines: > + # List the processes still using /data ("|| true" so a fuser failure > + # does not trigger the ERR trap and mask the real cause) > + log "Processes using /data:" > + log "$(ls -l /proc/*/fd /proc/*/cwd 2>/dev/null | grep ' /data' || true)" > + exit 1 > fi > > # Reaching this point means success: the final message and the log upload > diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh > index c40fe69..1537a5c 100755 > --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh > +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/start_guard.sh > @@ -4,9 +4,9 @@ > # file copy. Sourced by initiate.sh and tests/pg_upgrade/test-start-guard.sh. > # Expects log, IS_CI, PGDATAOLD and PGDATANEW from the caller. > > -# From step 9 until cleanup, nothing may start the source postgres: > -# > -# If it has to run, "pg_checksums -e" will operate on the source postgres, and > +# From step 9 until cleanup, nothing may start the source postgres: > +# > +# If it has to run, "pg_checksums -e" will operate on the source postgres, and > # this one has to stay stopped during the whole operation > # > # pg_upgrade copies the data files without a postmaster holding the directory, For full logs, run: nix log /nix/store/5ljcq0glvmp2v48xhnglk9wh4cingivm-treefmt-check.drv