Added health check and deployment setup for one container per customer. Fixes #12
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
Executable
+458
@@ -0,0 +1,458 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Management of a Rezervovator stack with one container per customer.
|
||||
# See README.md in this directory.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="$DEPLOY_DIR/.env"
|
||||
CUSTOMERS_DIR="$DEPLOY_DIR/customers"
|
||||
BACKUPS_DIR="$DEPLOY_DIR/backups"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") <command> [arguments]
|
||||
|
||||
init Create .env and start the proxy and PostgreSQL
|
||||
add <name> <domain> [from] Create a customer: database, config, volume, container.
|
||||
[from] is the sender address of e-mails, default noreply@<domain>
|
||||
list List customers and the state of their containers
|
||||
upgrade <name>...|--all Recreate containers from REZERVATOR_IMAGE, one by one.
|
||||
Backs up first, rolls back and stops when one fails.
|
||||
backup <name>...|--all Dump the database and the data volume to backups/<name>/
|
||||
restore <name> <timestamp> Restore a customer from backups/<name>/<timestamp>-*
|
||||
remove <name> [--purge] Stop and remove the container. With --purge also the database,
|
||||
the data volume and the config (asks for confirmation).
|
||||
|
||||
<name> is lowercase letters, digits and dashes, starting with a letter.
|
||||
EOF
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "Error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
info() {
|
||||
echo "==> $*"
|
||||
}
|
||||
|
||||
random_string() {
|
||||
# $1 = number of bytes, output is twice as many hex characters
|
||||
openssl rand -hex "$1"
|
||||
}
|
||||
|
||||
load_env() {
|
||||
[ -f "$ENV_FILE" ] || die "$ENV_FILE not found, run '$(basename "$0") init' first"
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
PREFIX="${REZERVATOR_PREFIX:-rezervator}"
|
||||
IMAGE="${REZERVATOR_IMAGE:-rezervator:latest}"
|
||||
TLS="${REZERVATOR_TLS:-true}"
|
||||
PG_CONTAINER="$PREFIX-postgres"
|
||||
NETWORK="$PREFIX"
|
||||
CUSTOMER_MAX_CONNECTIONS="${CUSTOMER_MAX_CONNECTIONS:-5}"
|
||||
}
|
||||
|
||||
compose() {
|
||||
docker compose --project-directory "$DEPLOY_DIR" --env-file "$ENV_FILE" "$@"
|
||||
}
|
||||
|
||||
check_name() {
|
||||
[[ "$1" =~ ^[a-z][a-z0-9-]{0,30}$ ]] || die "Invalid customer name '$1'"
|
||||
}
|
||||
|
||||
container_name() { echo "$PREFIX-app-$1"; }
|
||||
volume_name() { echo "$PREFIX-data-$1"; }
|
||||
# Database and role have the same name
|
||||
db_name() { echo "rz_${1//-/_}"; }
|
||||
|
||||
customer_exists() {
|
||||
[ -f "$CUSTOMERS_DIR/$1/customer.env" ]
|
||||
}
|
||||
|
||||
require_customer() {
|
||||
check_name "$1"
|
||||
customer_exists "$1" || die "Unknown customer '$1'"
|
||||
}
|
||||
|
||||
all_customers() {
|
||||
[ -d "$CUSTOMERS_DIR" ] || return 0
|
||||
for dir in "$CUSTOMERS_DIR"/*/; do
|
||||
[ -f "${dir}customer.env" ] && basename "$dir"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
psql_admin() {
|
||||
docker exec -i "$PG_CONTAINER" psql -U postgres -v ON_ERROR_STOP=1 -q "$@"
|
||||
}
|
||||
|
||||
wait_for_postgres() {
|
||||
for _ in $(seq 1 60); do
|
||||
# Over TCP: while the image initializes a new cluster, its temporary server listens on the socket only
|
||||
if docker exec "$PG_CONTAINER" pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
die "PostgreSQL in container $PG_CONTAINER is not ready"
|
||||
}
|
||||
|
||||
# Starts the container of a customer. $1 = name, $2 = image, $3 = initial admin password (optional)
|
||||
run_container() {
|
||||
local name="$1" image="$2" admin_password="${3:-}"
|
||||
local container router domain
|
||||
container="$(container_name "$name")"
|
||||
router="$PREFIX-$name"
|
||||
# shellcheck disable=SC1090
|
||||
domain="$(. "$CUSTOMERS_DIR/$name/customer.env" && echo "$DOMAIN")"
|
||||
|
||||
local args=(
|
||||
--detach
|
||||
--name "$container"
|
||||
--restart unless-stopped
|
||||
--network "$NETWORK"
|
||||
--volume "$CUSTOMERS_DIR/$name/config.toml:/app/config/config.toml:ro"
|
||||
--volume "$(volume_name "$name"):/app/data"
|
||||
--label "rezervator.stack=$PREFIX"
|
||||
--label "rezervator.customer=$name"
|
||||
--label "rezervator.domain=$domain"
|
||||
--label "traefik.enable=true"
|
||||
--label "traefik.http.services.$router.loadbalancer.server.port=3000"
|
||||
)
|
||||
|
||||
if [ "$TLS" = "true" ]; then
|
||||
args+=(
|
||||
--label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
|
||||
--label "traefik.http.routers.$router.entrypoints=websecure"
|
||||
--label "traefik.http.routers.$router.tls.certresolver=le"
|
||||
--label "traefik.http.routers.$router-http.rule=Host(\`$domain\`)"
|
||||
--label "traefik.http.routers.$router-http.entrypoints=web"
|
||||
--label "traefik.http.routers.$router-http.middlewares=$router-https"
|
||||
--label "traefik.http.middlewares.$router-https.redirectscheme.scheme=https"
|
||||
)
|
||||
else
|
||||
args+=(
|
||||
--label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
|
||||
--label "traefik.http.routers.$router.entrypoints=web"
|
||||
)
|
||||
fi
|
||||
|
||||
if [ -n "$admin_password" ]; then
|
||||
args+=(--env "REZERVATOR_ADMIN_PASSWORD=$admin_password")
|
||||
fi
|
||||
|
||||
docker run "${args[@]}" "$image" >/dev/null
|
||||
}
|
||||
|
||||
# Waits until the container reports healthy. Returns 1 when it stops or stays unhealthy.
|
||||
wait_healthy() {
|
||||
local container="$1" status
|
||||
for _ in $(seq 1 90); do
|
||||
status="$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null || echo "missing")"
|
||||
case "$status" in
|
||||
# The proxy picks a new container up a moment after it turns healthy
|
||||
"running healthy") sleep 3; return 0 ;;
|
||||
# Image without a health check: running for a while is the best we can tell
|
||||
"running none") sleep 5
|
||||
[ "$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)" = "running" ] && return 0
|
||||
return 1 ;;
|
||||
"running starting") ;;
|
||||
"running unhealthy") return 1 ;;
|
||||
running*) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd_init() {
|
||||
command -v docker >/dev/null || die "docker not found"
|
||||
command -v openssl >/dev/null || die "openssl not found"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
sed "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$(random_string 24)/" "$DEPLOY_DIR/.env.example" > "$ENV_FILE"
|
||||
chmod 600 "$ENV_FILE"
|
||||
info "Created $ENV_FILE with a generated PostgreSQL password. Review it (image, TLS, ACME_EMAIL) and run init again."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
load_env
|
||||
if [ "$TLS" = "true" ] && [ -z "${ACME_EMAIL:-}" ]; then
|
||||
die "REZERVATOR_TLS=true needs ACME_EMAIL in $ENV_FILE"
|
||||
fi
|
||||
|
||||
mkdir -p "$CUSTOMERS_DIR" "$BACKUPS_DIR"
|
||||
chmod 700 "$CUSTOMERS_DIR" "$BACKUPS_DIR"
|
||||
compose up --detach
|
||||
wait_for_postgres
|
||||
info "Proxy and PostgreSQL are running"
|
||||
}
|
||||
|
||||
cmd_add() {
|
||||
[ $# -ge 2 ] || die "Usage: add <name> <domain> [from]"
|
||||
local name="$1" domain="$2" mail_from="${3:-noreply@$2}"
|
||||
load_env
|
||||
check_name "$name"
|
||||
[[ "$domain" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "Invalid domain '$domain'"
|
||||
[[ "$mail_from" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$ ]] || die "Invalid sender address '$mail_from'"
|
||||
customer_exists "$name" && die "Customer '$name' already exists"
|
||||
docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
|
||||
wait_for_postgres
|
||||
|
||||
local db db_password admin_password dir
|
||||
db="$(db_name "$name")"
|
||||
db_password="$(random_string 24)"
|
||||
admin_password="$(random_string 9)"
|
||||
dir="$CUSTOMERS_DIR/$name"
|
||||
|
||||
if [ "$(psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname = '$db'")" = "1" ]; then
|
||||
die "Database $db already exists"
|
||||
fi
|
||||
|
||||
info "Creating database $db"
|
||||
psql_admin <<EOF
|
||||
CREATE ROLE "$db" LOGIN PASSWORD '$db_password';
|
||||
CREATE DATABASE "$db" OWNER "$db";
|
||||
REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
|
||||
EOF
|
||||
|
||||
info "Writing config to $dir"
|
||||
mkdir -p "$dir"
|
||||
chmod 700 "$dir"
|
||||
sed -e "s|@SESSION_KEY@|$(random_string 48)|" \
|
||||
-e "s|@DB_USER@|$db|" \
|
||||
-e "s|@DB_PASSWORD@|$db_password|" \
|
||||
-e "s|@DB_NAME@|$db|" \
|
||||
-e "s|@DB_HOST@|$PG_CONTAINER|" \
|
||||
-e "s|@MAX_CONNECTIONS@|$CUSTOMER_MAX_CONNECTIONS|" \
|
||||
-e "s|@MAIL_FROM@|$mail_from|" \
|
||||
"$DEPLOY_DIR/config.template.toml" > "$dir/config.toml"
|
||||
# The application in the container runs as another user and must be able to read it
|
||||
chmod 644 "$dir/config.toml"
|
||||
printf 'DOMAIN=%s\n' "$domain" > "$dir/customer.env"
|
||||
|
||||
info "Starting container $(container_name "$name")"
|
||||
docker volume create "$(volume_name "$name")" >/dev/null
|
||||
run_container "$name" "$IMAGE" "$admin_password"
|
||||
|
||||
if ! wait_healthy "$(container_name "$name")"; then
|
||||
docker logs --tail 20 "$(container_name "$name")" >&2 || true
|
||||
die "Container $(container_name "$name") did not become healthy. Fix the problem and run 'remove $name --purge' before adding it again."
|
||||
fi
|
||||
|
||||
local scheme="http"
|
||||
[ "$TLS" = "true" ] && scheme="https"
|
||||
cat <<EOF
|
||||
|
||||
Customer '$name' is running at $scheme://$domain
|
||||
|
||||
Admin login: admin
|
||||
Admin password: $admin_password
|
||||
|
||||
The password is shown only now. Hand it over and have it changed after the first login.
|
||||
EOF
|
||||
}
|
||||
|
||||
cmd_list() {
|
||||
load_env
|
||||
printf '%-20s %-34s %-12s %s\n' "CUSTOMER" "DOMAIN" "STATE" "IMAGE"
|
||||
local name domain state image
|
||||
for name in $(all_customers); do
|
||||
# shellcheck disable=SC1090
|
||||
domain="$(. "$CUSTOMERS_DIR/$name/customer.env" && echo "$DOMAIN")"
|
||||
state="$(docker inspect --format '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$(container_name "$name")" 2>/dev/null || echo "no container")"
|
||||
image="$(docker inspect --format '{{.Image}}' "$(container_name "$name")" 2>/dev/null || true)"
|
||||
if [ -n "$image" ]; then
|
||||
# Tag of the image when it still has one, otherwise the short id
|
||||
image="$(docker image inspect --format '{{if .RepoTags}}{{index .RepoTags 0}}{{else}}{{slice .Id 7 19}}{{end}}' "$image" 2>/dev/null || echo "${image:7:12}")"
|
||||
else
|
||||
image="-"
|
||||
fi
|
||||
printf '%-20s %-34s %-12s %s\n' "$name" "$domain" "$state" "$image"
|
||||
done
|
||||
}
|
||||
|
||||
# Dumps the database and the data volume. $1 = name. Prints the timestamp of the backup.
|
||||
backup_customer() {
|
||||
local name="$1" db dir timestamp
|
||||
db="$(db_name "$name")"
|
||||
dir="$BACKUPS_DIR/$name"
|
||||
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
mkdir -p "$dir"
|
||||
chmod 700 "$dir"
|
||||
|
||||
docker exec "$PG_CONTAINER" pg_dump -U postgres --no-owner --no-privileges "$db" | gzip > "$dir/$timestamp-db.sql.gz"
|
||||
# Any image with tar would do; the application image is certainly present
|
||||
docker run --rm --volume "$(volume_name "$name"):/data:ro" --entrypoint tar "$IMAGE" \
|
||||
czf - -C /data . > "$dir/$timestamp-data.tar.gz"
|
||||
|
||||
# An empty or truncated dump must not pass for a backup
|
||||
gzip -t "$dir/$timestamp-db.sql.gz" && gzip -t "$dir/$timestamp-data.tar.gz"
|
||||
[ "$(gzip -dc "$dir/$timestamp-db.sql.gz" | wc -c)" -gt 0 ] || die "Database dump of '$name' is empty"
|
||||
|
||||
echo "$timestamp"
|
||||
}
|
||||
|
||||
cmd_backup() {
|
||||
[ $# -ge 1 ] || die "Usage: backup <name>...|--all"
|
||||
load_env
|
||||
wait_for_postgres
|
||||
local names=("$@") name timestamp
|
||||
if [ "$1" = "--all" ]; then
|
||||
# shellcheck disable=SC2207
|
||||
names=($(all_customers))
|
||||
fi
|
||||
[ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
|
||||
for name in "${names[@]}"; do
|
||||
require_customer "$name"
|
||||
timestamp="$(backup_customer "$name")"
|
||||
info "Backup of '$name': $BACKUPS_DIR/$name/$timestamp-db.sql.gz, $timestamp-data.tar.gz"
|
||||
done
|
||||
}
|
||||
|
||||
# Replaces the database and the data volume by a backup. The container must be stopped.
|
||||
restore_customer() {
|
||||
local name="$1" timestamp="$2" db dir
|
||||
db="$(db_name "$name")"
|
||||
dir="$BACKUPS_DIR/$name"
|
||||
|
||||
psql_admin <<EOF
|
||||
DROP DATABASE IF EXISTS "$db" WITH (FORCE);
|
||||
CREATE DATABASE "$db" OWNER "$db";
|
||||
REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
|
||||
EOF
|
||||
gzip -dc "$dir/$timestamp-db.sql.gz" | docker exec -i "$PG_CONTAINER" psql -U "$db" -d "$db" -v ON_ERROR_STOP=1 -q >/dev/null
|
||||
|
||||
docker run --rm -i --volume "$(volume_name "$name"):/data" --entrypoint sh "$IMAGE" \
|
||||
-c 'find /data -mindepth 1 -delete && tar xzf - -C /data' < "$dir/$timestamp-data.tar.gz"
|
||||
}
|
||||
|
||||
cmd_restore() {
|
||||
[ $# -eq 2 ] || die "Usage: restore <name> <timestamp>"
|
||||
local name="$1" timestamp="$2" container image
|
||||
load_env
|
||||
require_customer "$name"
|
||||
[ -f "$BACKUPS_DIR/$name/$timestamp-db.sql.gz" ] && [ -f "$BACKUPS_DIR/$name/$timestamp-data.tar.gz" ] \
|
||||
|| die "Backup $timestamp of '$name' not found in $BACKUPS_DIR/$name"
|
||||
wait_for_postgres
|
||||
|
||||
container="$(container_name "$name")"
|
||||
# Keep the image the customer runs now; the configured one may be newer than the backup
|
||||
image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || echo "$IMAGE")"
|
||||
|
||||
info "Stopping $container"
|
||||
docker rm --force "$container" >/dev/null 2>&1 || true
|
||||
info "Restoring '$name' from backup $timestamp"
|
||||
restore_customer "$name" "$timestamp"
|
||||
run_container "$name" "$image"
|
||||
wait_healthy "$container" || die "Container $container is not healthy after the restore, see 'docker logs $container'"
|
||||
info "Customer '$name' restored and running"
|
||||
}
|
||||
|
||||
cmd_upgrade() {
|
||||
[ $# -ge 1 ] || die "Usage: upgrade <name>...|--all"
|
||||
load_env
|
||||
docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
|
||||
wait_for_postgres
|
||||
|
||||
local names=("$@") name container old_image new_image timestamp
|
||||
if [ "$1" = "--all" ]; then
|
||||
# shellcheck disable=SC2207
|
||||
names=($(all_customers))
|
||||
fi
|
||||
[ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
|
||||
new_image="$(docker image inspect --format '{{.Id}}' "$IMAGE")"
|
||||
|
||||
for name in "${names[@]}"; do
|
||||
require_customer "$name"
|
||||
container="$(container_name "$name")"
|
||||
old_image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)"
|
||||
|
||||
if [ "$old_image" = "$new_image" ]; then
|
||||
info "'$name' already runs $IMAGE, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
info "Upgrading '$name'"
|
||||
timestamp="$(backup_customer "$name")"
|
||||
docker rm --force "$container" >/dev/null 2>&1 || true
|
||||
run_container "$name" "$IMAGE"
|
||||
|
||||
if wait_healthy "$container"; then
|
||||
info "'$name' upgraded (backup before the upgrade: $timestamp)"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Upgrade of '$name' failed, last log lines of the new container:" >&2
|
||||
docker logs --tail 20 "$container" >&2 || true
|
||||
docker rm --force "$container" >/dev/null 2>&1 || true
|
||||
|
||||
if [ -z "$old_image" ]; then
|
||||
die "'$name' had no container before, nothing to roll back to. Backup: $timestamp"
|
||||
fi
|
||||
|
||||
# Migrations of the new version may have changed the database, so the old version
|
||||
# gets the database from before the upgrade
|
||||
info "Rolling '$name' back to the previous image and to backup $timestamp"
|
||||
restore_customer "$name" "$timestamp"
|
||||
run_container "$name" "$old_image"
|
||||
if wait_healthy "$container"; then
|
||||
die "'$name' was rolled back. Upgrade stopped, the remaining customers were not touched."
|
||||
fi
|
||||
die "'$name' is DOWN: the rollback did not become healthy either. Backup: $timestamp"
|
||||
done
|
||||
}
|
||||
|
||||
cmd_remove() {
|
||||
[ $# -ge 1 ] || die "Usage: remove <name> [--purge]"
|
||||
local name="$1" purge="${2:-}" db answer
|
||||
load_env
|
||||
require_customer "$name"
|
||||
|
||||
info "Removing container $(container_name "$name")"
|
||||
docker rm --force "$(container_name "$name")" >/dev/null 2>&1 || true
|
||||
|
||||
if [ "$purge" != "--purge" ]; then
|
||||
info "Database, data volume and config of '$name' were kept. Start it again with 'upgrade $name'."
|
||||
return 0
|
||||
fi
|
||||
|
||||
read -r -p "This deletes the database, uploaded files and config of '$name'. Type the name to confirm: " answer
|
||||
[ "$answer" = "$name" ] || die "Not confirmed, nothing else was deleted"
|
||||
|
||||
wait_for_postgres
|
||||
db="$(db_name "$name")"
|
||||
psql_admin <<EOF
|
||||
DROP DATABASE IF EXISTS "$db" WITH (FORCE);
|
||||
DROP ROLE IF EXISTS "$db";
|
||||
EOF
|
||||
docker volume rm "$(volume_name "$name")" >/dev/null 2>&1 || true
|
||||
rm -rf "${CUSTOMERS_DIR:?}/$name"
|
||||
info "Customer '$name' deleted. Backups in $BACKUPS_DIR/$name were kept."
|
||||
}
|
||||
|
||||
main() {
|
||||
[ $# -ge 1 ] || { usage; exit 1; }
|
||||
local command="$1"
|
||||
shift
|
||||
case "$command" in
|
||||
init) cmd_init "$@" ;;
|
||||
add) cmd_add "$@" ;;
|
||||
list) cmd_list "$@" ;;
|
||||
upgrade) cmd_upgrade "$@" ;;
|
||||
backup) cmd_backup "$@" ;;
|
||||
restore) cmd_restore "$@" ;;
|
||||
remove) cmd_remove "$@" ;;
|
||||
-h|--help|help) usage ;;
|
||||
*) usage; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user