From cfbda7c3cde7156eede02c36407093cd0538d9c3 Mon Sep 17 00:00:00 2001 From: Ricearoni1245 Date: Mon, 2 Mar 2026 13:56:23 -0600 Subject: [PATCH] diagnose and fix: removed rm for skymoney data in deploy) --- .env | 4 +- .env.example | 2 + .gitea/workflows/deploy.yml | 6 ++- docs/production-db-recovery-runbook.md | 15 +++++++- docs/production-operations-policy.md | 38 +++++++++++++++++++ scripts/guard-prod-volume.sh | 32 ++++++++++++++++ .../post-deployment-verification-checklist.md | 2 + 7 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 docs/production-operations-policy.md create mode 100644 scripts/guard-prod-volume.sh diff --git a/.env b/.env index b8afcd7..9852a1b 100644 --- a/.env +++ b/.env @@ -45,4 +45,6 @@ PASSWORD_RESET_CONFIRM_RATE_LIMIT_PER_MINUTE=10 EXPECTED_PROD_DB_HOST=postgres EXPECTED_PROD_DB_NAME=skymoney EXPECTED_BACKUP_DB_HOST=127.0.0.1 -EXPECTED_BACKUP_DB_NAME=skymoney \ No newline at end of file +EXPECTED_BACKUP_DB_NAME=skymoney +PROD_DB_VOLUME_NAME=skymoney_pgdata +ALLOW_EMPTY_PROD_VOLUME=0 \ No newline at end of file diff --git a/.env.example b/.env.example index 73027ab..5c2d091 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,8 @@ EXPECTED_PROD_DB_HOST=postgres EXPECTED_PROD_DB_NAME=skymoney EXPECTED_BACKUP_DB_HOST=127.0.0.1 EXPECTED_BACKUP_DB_NAME=skymoney +PROD_DB_VOLUME_NAME=skymoney_pgdata +ALLOW_EMPTY_PROD_VOLUME=0 ARCHIVE_EXISTING_RESTORE_DB=1 RESTORE_ARCHIVE_DIR=./backups/restore-archives diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index bd7557c..6e0bc19 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -53,8 +53,12 @@ jobs: # Validate migration target before touching containers export EXPECTED_PROD_DB_HOST="${EXPECTED_PROD_DB_HOST:-postgres}" export EXPECTED_PROD_DB_NAME="${EXPECTED_PROD_DB_NAME:-skymoney}" - chmod +x ./scripts/validate-prod-db-target.sh ./scripts/backup.sh + chmod +x ./scripts/validate-prod-db-target.sh ./scripts/guard-prod-volume.sh ./scripts/backup.sh bash ./scripts/validate-prod-db-target.sh + PROD_DB_VOLUME_NAME="${PROD_DB_VOLUME_NAME:-skymoney_pgdata}" \ + ALLOW_EMPTY_PROD_VOLUME="${ALLOW_EMPTY_PROD_VOLUME:-0}" \ + DOCKER_CMD="sudo docker" \ + bash ./scripts/guard-prod-volume.sh # Build and start all services sudo docker-compose -p skymoney up -d --build diff --git a/docs/production-db-recovery-runbook.md b/docs/production-db-recovery-runbook.md index a74957d..5ff05f4 100644 --- a/docs/production-db-recovery-runbook.md +++ b/docs/production-db-recovery-runbook.md @@ -188,8 +188,19 @@ psql "postgres://:@127.0.0.1:5432/skymoney" \ 1. `docker-compose.yml` pins volume name: `skymoney_pgdata`. 2. Deploy workflow sets `COMPOSE_PROJECT_NAME=skymoney`. 3. Deploy workflow runs `scripts/validate-prod-db-target.sh`. -4. Deploy workflow runs pre-migration `scripts/backup.sh`. -5. Deploy workflow uses `prisma migrate deploy` only. +4. Deploy workflow runs `scripts/guard-prod-volume.sh` and blocks deploy when prod volume is missing/empty. +5. Deploy workflow runs pre-migration `scripts/backup.sh`. +6. Deploy workflow uses `prisma migrate deploy` only. + +### Intentional rebuild override + +If you intentionally need to initialize an empty production volume (rare), set: + +```bash +ALLOW_EMPTY_PROD_VOLUME=1 +``` + +for one deploy run only, then reset it back to `0`. ## Quarterly drill requirement diff --git a/docs/production-operations-policy.md b/docs/production-operations-policy.md new file mode 100644 index 0000000..ea9ac85 --- /dev/null +++ b/docs/production-operations-policy.md @@ -0,0 +1,38 @@ +# Production Operations Policy + +Last updated: March 2, 2026 + +## Purpose + +Prevent destructive production actions that can cause irreversible data loss. + +## Hard bans in production + +Never run these commands against production: + +1. `docker volume rm skymoney_pgdata` +2. `docker compose down -v` / `docker-compose down -v` +3. `prisma migrate reset` +4. `prisma migrate dev` +5. `prisma db push --accept-data-loss` + +## Allowed migration path + +1. `prisma migrate deploy` only. +2. Mandatory pre-migration backup (`scripts/backup.sh`). +3. DB target validation (`scripts/validate-prod-db-target.sh`). +4. Volume guard (`scripts/guard-prod-volume.sh`). + +## Operator controls + +1. Prefer constrained sudoers permissions over broad `sudo docker`. +2. Keep all manual production commands logged in an incident/change ticket. +3. Require peer confirmation before any storage/volume action. + +## Intentional rebuild exception + +Only for explicit rebuild events: + +1. Set `ALLOW_EMPTY_PROD_VOLUME=1` for one deploy run. +2. Record reason and approver. +3. Reset `ALLOW_EMPTY_PROD_VOLUME=0` immediately afterward. diff --git a/scripts/guard-prod-volume.sh b/scripts/guard-prod-volume.sh new file mode 100644 index 0000000..b7c3aa5 --- /dev/null +++ b/scripts/guard-prod-volume.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +VOLUME_NAME="${PROD_DB_VOLUME_NAME:-skymoney_pgdata}" +ALLOW_EMPTY="${ALLOW_EMPTY_PROD_VOLUME:-0}" +DOCKER_CMD="${DOCKER_CMD:-docker}" + +if ! $DOCKER_CMD volume inspect "$VOLUME_NAME" >/dev/null 2>&1; then + if [[ "$ALLOW_EMPTY" == "1" ]]; then + echo "WARN: volume '$VOLUME_NAME' is missing, but ALLOW_EMPTY_PROD_VOLUME=1; continuing." + exit 0 + fi + echo "ERROR: required volume '$VOLUME_NAME' does not exist." + echo "Refusing deploy because this can indicate destructive data loss (e.g., volume deletion)." + echo "If this is an intentional first-time init, set ALLOW_EMPTY_PROD_VOLUME=1 for one run." + exit 1 +fi + +if $DOCKER_CMD run --rm -v "${VOLUME_NAME}:/var/lib/postgresql/data" alpine sh -lc "test -f /var/lib/postgresql/data/PG_VERSION"; then + echo "Production volume guard passed: '$VOLUME_NAME' contains PostgreSQL data." + exit 0 +fi + +if [[ "$ALLOW_EMPTY" == "1" ]]; then + echo "WARN: volume '$VOLUME_NAME' is empty/uninitialized, but ALLOW_EMPTY_PROD_VOLUME=1; continuing." + exit 0 +fi + +echo "ERROR: volume '$VOLUME_NAME' exists but appears empty/uninitialized (missing PG_VERSION)." +echo "Refusing deploy to prevent silent database re-initialization." +echo "If this is an intentional rebuild, set ALLOW_EMPTY_PROD_VOLUME=1 for one run." +exit 1 diff --git a/tests-results-for-OWASP/post-deployment-verification-checklist.md b/tests-results-for-OWASP/post-deployment-verification-checklist.md index ab151f3..bf6002a 100644 --- a/tests-results-for-OWASP/post-deployment-verification-checklist.md +++ b/tests-results-for-OWASP/post-deployment-verification-checklist.md @@ -22,6 +22,7 @@ Expected: - `COMPOSE_PROJECT_NAME=skymoney` is set for deploy runtime. - `docker-compose.yml` volume `pgdata` is pinned to `skymoney_pgdata`. - `scripts/validate-prod-db-target.sh` passes for current `.env`. +- `scripts/guard-prod-volume.sh` passes (or explicit one-time rebuild override is documented). - deploy runbook acknowledges forbidden destructive commands in prod: - `prisma migrate reset` - `prisma migrate dev` @@ -36,6 +37,7 @@ Expected: docker ps --format '{{.Names}}' docker inspect --format '{{json .Mounts}}' docker volume ls | grep -E 'pgdata|skymoney|postgres' +PROD_DB_VOLUME_NAME=skymoney_pgdata ALLOW_EMPTY_PROD_VOLUME=0 DOCKER_CMD="sudo docker" bash ./scripts/guard-prod-volume.sh ``` Expected: