diagnose and fix: removed rm for skymoney data in deploy)
This commit is contained in:
4
.env
4
.env
@@ -45,4 +45,6 @@ PASSWORD_RESET_CONFIRM_RATE_LIMIT_PER_MINUTE=10
|
||||
EXPECTED_PROD_DB_HOST=postgres
|
||||
EXPECTED_PROD_DB_NAME=skymoney
|
||||
EXPECTED_BACKUP_DB_HOST=127.0.0.1
|
||||
EXPECTED_BACKUP_DB_NAME=skymoney
|
||||
EXPECTED_BACKUP_DB_NAME=skymoney
|
||||
PROD_DB_VOLUME_NAME=skymoney_pgdata
|
||||
ALLOW_EMPTY_PROD_VOLUME=0
|
||||
@@ -23,6 +23,8 @@ EXPECTED_PROD_DB_HOST=postgres
|
||||
EXPECTED_PROD_DB_NAME=skymoney
|
||||
EXPECTED_BACKUP_DB_HOST=127.0.0.1
|
||||
EXPECTED_BACKUP_DB_NAME=skymoney
|
||||
PROD_DB_VOLUME_NAME=skymoney_pgdata
|
||||
ALLOW_EMPTY_PROD_VOLUME=0
|
||||
ARCHIVE_EXISTING_RESTORE_DB=1
|
||||
RESTORE_ARCHIVE_DIR=./backups/restore-archives
|
||||
|
||||
|
||||
@@ -53,8 +53,12 @@ jobs:
|
||||
# Validate migration target before touching containers
|
||||
export EXPECTED_PROD_DB_HOST="${EXPECTED_PROD_DB_HOST:-postgres}"
|
||||
export EXPECTED_PROD_DB_NAME="${EXPECTED_PROD_DB_NAME:-skymoney}"
|
||||
chmod +x ./scripts/validate-prod-db-target.sh ./scripts/backup.sh
|
||||
chmod +x ./scripts/validate-prod-db-target.sh ./scripts/guard-prod-volume.sh ./scripts/backup.sh
|
||||
bash ./scripts/validate-prod-db-target.sh
|
||||
PROD_DB_VOLUME_NAME="${PROD_DB_VOLUME_NAME:-skymoney_pgdata}" \
|
||||
ALLOW_EMPTY_PROD_VOLUME="${ALLOW_EMPTY_PROD_VOLUME:-0}" \
|
||||
DOCKER_CMD="sudo docker" \
|
||||
bash ./scripts/guard-prod-volume.sh
|
||||
|
||||
# Build and start all services
|
||||
sudo docker-compose -p skymoney up -d --build
|
||||
|
||||
@@ -188,8 +188,19 @@ psql "postgres://<admin-user>:<admin-pass>@127.0.0.1:5432/skymoney" \
|
||||
1. `docker-compose.yml` pins volume name: `skymoney_pgdata`.
|
||||
2. Deploy workflow sets `COMPOSE_PROJECT_NAME=skymoney`.
|
||||
3. Deploy workflow runs `scripts/validate-prod-db-target.sh`.
|
||||
4. Deploy workflow runs pre-migration `scripts/backup.sh`.
|
||||
5. Deploy workflow uses `prisma migrate deploy` only.
|
||||
4. Deploy workflow runs `scripts/guard-prod-volume.sh` and blocks deploy when prod volume is missing/empty.
|
||||
5. Deploy workflow runs pre-migration `scripts/backup.sh`.
|
||||
6. Deploy workflow uses `prisma migrate deploy` only.
|
||||
|
||||
### Intentional rebuild override
|
||||
|
||||
If you intentionally need to initialize an empty production volume (rare), set:
|
||||
|
||||
```bash
|
||||
ALLOW_EMPTY_PROD_VOLUME=1
|
||||
```
|
||||
|
||||
for one deploy run only, then reset it back to `0`.
|
||||
|
||||
## Quarterly drill requirement
|
||||
|
||||
|
||||
38
docs/production-operations-policy.md
Normal file
38
docs/production-operations-policy.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# Production Operations Policy
|
||||
|
||||
Last updated: March 2, 2026
|
||||
|
||||
## Purpose
|
||||
|
||||
Prevent destructive production actions that can cause irreversible data loss.
|
||||
|
||||
## Hard bans in production
|
||||
|
||||
Never run these commands against production:
|
||||
|
||||
1. `docker volume rm skymoney_pgdata`
|
||||
2. `docker compose down -v` / `docker-compose down -v`
|
||||
3. `prisma migrate reset`
|
||||
4. `prisma migrate dev`
|
||||
5. `prisma db push --accept-data-loss`
|
||||
|
||||
## Allowed migration path
|
||||
|
||||
1. `prisma migrate deploy` only.
|
||||
2. Mandatory pre-migration backup (`scripts/backup.sh`).
|
||||
3. DB target validation (`scripts/validate-prod-db-target.sh`).
|
||||
4. Volume guard (`scripts/guard-prod-volume.sh`).
|
||||
|
||||
## Operator controls
|
||||
|
||||
1. Prefer constrained sudoers permissions over broad `sudo docker`.
|
||||
2. Keep all manual production commands logged in an incident/change ticket.
|
||||
3. Require peer confirmation before any storage/volume action.
|
||||
|
||||
## Intentional rebuild exception
|
||||
|
||||
Only for explicit rebuild events:
|
||||
|
||||
1. Set `ALLOW_EMPTY_PROD_VOLUME=1` for one deploy run.
|
||||
2. Record reason and approver.
|
||||
3. Reset `ALLOW_EMPTY_PROD_VOLUME=0` immediately afterward.
|
||||
32
scripts/guard-prod-volume.sh
Normal file
32
scripts/guard-prod-volume.sh
Normal file
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
VOLUME_NAME="${PROD_DB_VOLUME_NAME:-skymoney_pgdata}"
|
||||
ALLOW_EMPTY="${ALLOW_EMPTY_PROD_VOLUME:-0}"
|
||||
DOCKER_CMD="${DOCKER_CMD:-docker}"
|
||||
|
||||
if ! $DOCKER_CMD volume inspect "$VOLUME_NAME" >/dev/null 2>&1; then
|
||||
if [[ "$ALLOW_EMPTY" == "1" ]]; then
|
||||
echo "WARN: volume '$VOLUME_NAME' is missing, but ALLOW_EMPTY_PROD_VOLUME=1; continuing."
|
||||
exit 0
|
||||
fi
|
||||
echo "ERROR: required volume '$VOLUME_NAME' does not exist."
|
||||
echo "Refusing deploy because this can indicate destructive data loss (e.g., volume deletion)."
|
||||
echo "If this is an intentional first-time init, set ALLOW_EMPTY_PROD_VOLUME=1 for one run."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if $DOCKER_CMD run --rm -v "${VOLUME_NAME}:/var/lib/postgresql/data" alpine sh -lc "test -f /var/lib/postgresql/data/PG_VERSION"; then
|
||||
echo "Production volume guard passed: '$VOLUME_NAME' contains PostgreSQL data."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$ALLOW_EMPTY" == "1" ]]; then
|
||||
echo "WARN: volume '$VOLUME_NAME' is empty/uninitialized, but ALLOW_EMPTY_PROD_VOLUME=1; continuing."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "ERROR: volume '$VOLUME_NAME' exists but appears empty/uninitialized (missing PG_VERSION)."
|
||||
echo "Refusing deploy to prevent silent database re-initialization."
|
||||
echo "If this is an intentional rebuild, set ALLOW_EMPTY_PROD_VOLUME=1 for one run."
|
||||
exit 1
|
||||
@@ -22,6 +22,7 @@ Expected:
|
||||
- `COMPOSE_PROJECT_NAME=skymoney` is set for deploy runtime.
|
||||
- `docker-compose.yml` volume `pgdata` is pinned to `skymoney_pgdata`.
|
||||
- `scripts/validate-prod-db-target.sh` passes for current `.env`.
|
||||
- `scripts/guard-prod-volume.sh` passes (or explicit one-time rebuild override is documented).
|
||||
- deploy runbook acknowledges forbidden destructive commands in prod:
|
||||
- `prisma migrate reset`
|
||||
- `prisma migrate dev`
|
||||
@@ -36,6 +37,7 @@ Expected:
|
||||
docker ps --format '{{.Names}}'
|
||||
docker inspect <postgres-container> --format '{{json .Mounts}}'
|
||||
docker volume ls | grep -E 'pgdata|skymoney|postgres'
|
||||
PROD_DB_VOLUME_NAME=skymoney_pgdata ALLOW_EMPTY_PROD_VOLUME=0 DOCKER_CMD="sudo docker" bash ./scripts/guard-prod-volume.sh
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
Reference in New Issue
Block a user