Added health check and deployment setup for one container per customer. Fixes #12

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 16:34:40 +02:00
parent c1d998878a
commit 156fe1a0f4
14 changed files with 799 additions and 4 deletions
@@ -0,0 +1,25 @@
+# Settings of the stack. Copy to .env (./rezervator.sh init does it and generates the password).
+
+# Prefix of container, network and volume names. Change it to run a second stack on the same host.
+REZERVATOR_PREFIX=rezervator
+
+# Image the customer containers are started from
+REZERVATOR_IMAGE=rezervator:latest
+
+# true: HTTPS with Let's Encrypt certificates, HTTP is redirected. Needs public DNS records
+#       pointing to this host and ACME_EMAIL.
+# false: plain HTTP only, for local testing or behind another TLS terminating proxy.
+REZERVATOR_TLS=true
+ACME_EMAIL=
+
+# Ports published by the proxy
+HTTP_PORT=80
+HTTPS_PORT=443
+
+# Password of the PostgreSQL superuser. Generated by ./rezervator.sh init.
+POSTGRES_PASSWORD=
+
+# Connection limits. PG_MAX_CONNECTIONS must be above
+# (number of customers) x CUSTOMER_MAX_CONNECTIONS, or put PgBouncer in front of PostgreSQL.
+PG_MAX_CONNECTIONS=300
+CUSTOMER_MAX_CONNECTIONS=5
@@ -0,0 +1,96 @@
+# Running Rezervovator for many customers
+
+The application serves one company per instance. To host many customers, every customer gets their own container from the same image, with their own database, configuration and data volume. This directory contains what is needed to run that on one Docker host:
+
+- `docker-compose.yml` — the shared part: the reverse proxy (Traefik) and one PostgreSQL server
+- `rezervator.sh` — creates, upgrades, backs up, restores and removes customers
+- `config.template.toml` — template of a customer's configuration
+- `.env.example` — settings of the stack
+
+```text
+                      ┌─ rezervator-app-alpha ── database rz_alpha, volume rezervator-data-alpha
+internet ── proxy ────┼─ rezervator-app-beta  ── database rz_beta,  volume rezervator-data-beta
+          (TLS, by    └─ ...
+           domain)             all databases live in the rezervator-postgres container
+```
+
+## Requirements
+
+- Docker with the Compose plugin, `bash` and `openssl` on the host
+- The application image on the host (`docker build -t rezervator:latest .` in the repository root, or pulled from a registry)
+- For HTTPS: a DNS record for every customer domain pointing to the host, and ports 80 and 443 reachable from the internet
+
+## First setup
+
+```sh
+./rezervator.sh init        # creates .env with a generated PostgreSQL password and stops
+$EDITOR .env                # image, TLS, ACME_EMAIL, ports
+$EDITOR config.template.toml  # the [mailing] section: your SMTP server
+./rezervator.sh init        # starts the proxy and PostgreSQL
+```
+
+`REZERVATOR_TLS=true` gets a Let's Encrypt certificate for every customer domain and redirects HTTP to HTTPS. Use `false` only for local testing or behind another proxy that terminates TLS: the session cookie is marked `Secure`, so logging in over plain HTTP works only on `localhost` addresses.
+
+## Adding a customer
+
+```sh
+./rezervator.sh add alpha rezervace.alpha.example
+```
+
+This creates the database and its user, writes `customers/alpha/config.toml` with a unique session key and database password, creates the data volume, starts the container and waits until it is healthy. At the end it prints the generated password of the `admin` account — it is shown only once.
+
+An optional third argument sets the sender address of e-mails (default `noreply@<domain>`).
+
+`./rezervator.sh list` shows all customers and the state of their containers.
+
+## Upgrading
+
+Build or pull the new image under the name set in `REZERVATOR_IMAGE`, then:
+
+```sh
+./rezervator.sh upgrade alpha     # try one customer first
+./rezervator.sh upgrade --all
+```
+
+Customers are upgraded one by one. For each one the script makes a backup, replaces the container (database migrations run when the new version starts) and waits for it to become healthy. Customers that already run the image are skipped, so an interrupted upgrade can simply be run again.
+
+If a customer does not become healthy, the script puts back the database and data from the backup it just made, starts the previous image again and stops. The remaining customers are not touched. The customer is unavailable for the duration of the attempt.
+
+## Backup and restore
+
+```sh
+./rezervator.sh backup --all
+./rezervator.sh restore alpha 20260101T020000Z
+```
+
+A backup is two files in `backups/<name>/`: `<timestamp>-db.sql.gz` (database dump) and `<timestamp>-data.tar.gz` (uploaded banner and generated styles). `restore` replaces both the database and the data volume, and keeps the image the customer was running.
+
+The script does not schedule backups, does not delete old ones and does not copy them off the host. Run `backup --all` from cron, and ship `backups/`, `customers/` and `.env` somewhere else — `customers/` and `.env` hold the passwords and session keys and cannot be regenerated.
+
+## Removing a customer
+
+```sh
+./rezervator.sh remove alpha            # stops the container, keeps everything else
+./rezervator.sh remove alpha --purge    # also deletes the database, data volume and config
+```
+
+After a plain `remove`, `upgrade alpha` starts the customer again. `--purge` asks for the customer name as confirmation and keeps existing backups.
+
+## Health and monitoring
+
+Every instance answers `GET /health` with 200 when it can reach its database and with 503 otherwise. The image uses it as its Docker health check, and Traefik sends traffic only to healthy containers. Point an external monitor to `https://<domain>/health`.
+
+## Sizing
+
+- One idle instance used about 7 MiB of memory in a test, about 12 MiB after some use. PostgreSQL itself needs far more than that.
+- Every instance may open up to `CUSTOMER_MAX_CONNECTIONS` database connections (5 by default; it is written into the customer's config when the customer is added) and closes idle ones. `PG_MAX_CONNECTIONS` (300 by default) must stay above the number of customers times that limit. Beyond that, put PgBouncer in front of PostgreSQL.
+
+## Security notes
+
+- `customers/` and `.env` contain secrets and are readable only by the user who ran the script. Keep them out of version control (they are in `.gitignore`).
+- The proxy has read access to the Docker socket to discover containers. That is equivalent to root on the host; do not expose the proxy's API or dashboard.
+- Customers share one PostgreSQL server but each has its own database and role, without access to the others.
+
+## Running a second stack
+
+Change `REZERVATOR_PREFIX` (and the ports) in `.env` of another copy of this directory. All container, network and volume names are derived from the prefix, and each proxy only sees the containers of its own stack.
@@ -0,0 +1,31 @@
+# Template of the configuration of one customer instance. ./rezervator.sh add fills in the
+# @PLACEHOLDERS@ and writes the result to customers/<name>/config.toml.
+# Edit the [mailing] section before adding the first customer.
+
+[session]
+key = '@SESSION_KEY@'
+
+[network]
+bind_ip = '0.0.0.0'
+port = 3000
+
+[database]
+user = '@DB_USER@'
+password = '@DB_PASSWORD@'
+db_name = '@DB_NAME@'
+host = '@DB_HOST@'
+port = 5432
+max_connections = @MAX_CONNECTIONS@
+
+[mailing]
+transport = "Smtp"
+from = "@MAIL_FROM@"
+server = "smtp.example.com"
+#hello_name = "host.example.com"
+port = 587
+user = "smtp-user"
+password = "smtp-password"
+tls = true
+
+[logging]
+severity = "info"
@@ -0,0 +1,57 @@
+# Shared infrastructure for running one Rezervovator container per customer:
+# the reverse proxy and the PostgreSQL server. Customer containers are not listed
+# here, they are created with ./rezervator.sh (see README.md in this directory).
+
+name: ${REZERVATOR_PREFIX:-rezervator}
+
+services:
+  proxy:
+    image: traefik:v3
+    container_name: ${REZERVATOR_PREFIX:-rezervator}-proxy
+    restart: unless-stopped
+    command:
+      - --providers.docker=true
+      - --providers.docker.exposedbydefault=false
+      - --providers.docker.network=${REZERVATOR_PREFIX:-rezervator}
+      # Only containers of this stack, so two stacks on one host don't mix
+      - --providers.docker.constraints=Label(`rezervator.stack`,`${REZERVATOR_PREFIX:-rezervator}`)
+      - --entrypoints.web.address=:80
+      - --entrypoints.websecure.address=:443
+      # Let's Encrypt certificates, used when REZERVATOR_TLS=true
+      - --certificatesresolvers.le.acme.email=${ACME_EMAIL:-}
+      - --certificatesresolvers.le.acme.storage=/letsencrypt/acme.json
+      - --certificatesresolvers.le.acme.httpchallenge.entrypoint=web
+    ports:
+      - "${HTTP_PORT:-80}:80"
+      - "${HTTPS_PORT:-443}:443"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - letsencrypt:/letsencrypt
+    networks:
+      - rezervator
+
+  postgres:
+    image: postgres:17
+    container_name: ${REZERVATOR_PREFIX:-rezervator}-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env (./rezervator.sh init creates it)}
+    # Every customer instance may open up to CUSTOMER_MAX_CONNECTIONS connections
+    command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-300}"]
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres"]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+    networks:
+      - rezervator
+
+networks:
+  rezervator:
+    name: ${REZERVATOR_PREFIX:-rezervator}
+
+volumes:
+  letsencrypt:
+  postgres-data:
@@ -0,0 +1,458 @@
+#!/usr/bin/env bash
+#
+# Management of a Rezervovator stack with one container per customer.
+# See README.md in this directory.
+
+set -euo pipefail
+
+DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="$DEPLOY_DIR/.env"
+CUSTOMERS_DIR="$DEPLOY_DIR/customers"
+BACKUPS_DIR="$DEPLOY_DIR/backups"
+
+usage() {
+    cat <<EOF
+Usage: $(basename "$0") <command> [arguments]
+
+  init                         Create .env and start the proxy and PostgreSQL
+  add <name> <domain> [from]   Create a customer: database, config, volume, container.
+                               [from] is the sender address of e-mails, default noreply@<domain>
+  list                         List customers and the state of their containers
+  upgrade <name>...|--all      Recreate containers from REZERVATOR_IMAGE, one by one.
+                               Backs up first, rolls back and stops when one fails.
+  backup <name>...|--all       Dump the database and the data volume to backups/<name>/
+  restore <name> <timestamp>   Restore a customer from backups/<name>/<timestamp>-*
+  remove <name> [--purge]      Stop and remove the container. With --purge also the database,
+                               the data volume and the config (asks for confirmation).
+
+<name> is lowercase letters, digits and dashes, starting with a letter.
+EOF
+}
+
+die() {
+    echo "Error: $*" >&2
+    exit 1
+}
+
+info() {
+    echo "==> $*"
+}
+
+random_string() {
+    # $1 = number of bytes, output is twice as many hex characters
+    openssl rand -hex "$1"
+}
+
+load_env() {
+    [ -f "$ENV_FILE" ] || die "$ENV_FILE not found, run '$(basename "$0") init' first"
+    set -a
+    # shellcheck disable=SC1090
+    . "$ENV_FILE"
+    set +a
+
+    PREFIX="${REZERVATOR_PREFIX:-rezervator}"
+    IMAGE="${REZERVATOR_IMAGE:-rezervator:latest}"
+    TLS="${REZERVATOR_TLS:-true}"
+    PG_CONTAINER="$PREFIX-postgres"
+    NETWORK="$PREFIX"
+    CUSTOMER_MAX_CONNECTIONS="${CUSTOMER_MAX_CONNECTIONS:-5}"
+}
+
+compose() {
+    docker compose --project-directory "$DEPLOY_DIR" --env-file "$ENV_FILE" "$@"
+}
+
+check_name() {
+    [[ "$1" =~ ^[a-z][a-z0-9-]{0,30}$ ]] || die "Invalid customer name '$1'"
+}
+
+container_name() { echo "$PREFIX-app-$1"; }
+volume_name() { echo "$PREFIX-data-$1"; }
+# Database and role have the same name
+db_name() { echo "rz_${1//-/_}"; }
+
+customer_exists() {
+    [ -f "$CUSTOMERS_DIR/$1/customer.env" ]
+}
+
+require_customer() {
+    check_name "$1"
+    customer_exists "$1" || die "Unknown customer '$1'"
+}
+
+all_customers() {
+    [ -d "$CUSTOMERS_DIR" ] || return 0
+    for dir in "$CUSTOMERS_DIR"/*/; do
+        [ -f "${dir}customer.env" ] && basename "$dir"
+    done
+    return 0
+}
+
+psql_admin() {
+    docker exec -i "$PG_CONTAINER" psql -U postgres -v ON_ERROR_STOP=1 -q "$@"
+}
+
+wait_for_postgres() {
+    for _ in $(seq 1 60); do
+        # Over TCP: while the image initializes a new cluster, its temporary server listens on the socket only
+        if docker exec "$PG_CONTAINER" pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
+            return 0
+        fi
+        sleep 1
+    done
+    die "PostgreSQL in container $PG_CONTAINER is not ready"
+}
+
+# Starts the container of a customer. $1 = name, $2 = image, $3 = initial admin password (optional)
+run_container() {
+    local name="$1" image="$2" admin_password="${3:-}"
+    local container router domain
+    container="$(container_name "$name")"
+    router="$PREFIX-$name"
+    # shellcheck disable=SC1090
+    domain="$(. "$CUSTOMERS_DIR/$name/customer.env" && echo "$DOMAIN")"
+
+    local args=(
+        --detach
+        --name "$container"
+        --restart unless-stopped
+        --network "$NETWORK"
+        --volume "$CUSTOMERS_DIR/$name/config.toml:/app/config/config.toml:ro"
+        --volume "$(volume_name "$name"):/app/data"
+        --label "rezervator.stack=$PREFIX"
+        --label "rezervator.customer=$name"
+        --label "rezervator.domain=$domain"
+        --label "traefik.enable=true"
+        --label "traefik.http.services.$router.loadbalancer.server.port=3000"
+    )
+
+    if [ "$TLS" = "true" ]; then
+        args+=(
+            --label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
+            --label "traefik.http.routers.$router.entrypoints=websecure"
+            --label "traefik.http.routers.$router.tls.certresolver=le"
+            --label "traefik.http.routers.$router-http.rule=Host(\`$domain\`)"
+            --label "traefik.http.routers.$router-http.entrypoints=web"
+            --label "traefik.http.routers.$router-http.middlewares=$router-https"
+            --label "traefik.http.middlewares.$router-https.redirectscheme.scheme=https"
+        )
+    else
+        args+=(
+            --label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
+            --label "traefik.http.routers.$router.entrypoints=web"
+        )
+    fi
+
+    if [ -n "$admin_password" ]; then
+        args+=(--env "REZERVATOR_ADMIN_PASSWORD=$admin_password")
+    fi
+
+    docker run "${args[@]}" "$image" >/dev/null
+}
+
+# Waits until the container reports healthy. Returns 1 when it stops or stays unhealthy.
+wait_healthy() {
+    local container="$1" status
+    for _ in $(seq 1 90); do
+        status="$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null || echo "missing")"
+        case "$status" in
+            # The proxy picks a new container up a moment after it turns healthy
+            "running healthy") sleep 3; return 0 ;;
+            # Image without a health check: running for a while is the best we can tell
+            "running none") sleep 5
+                [ "$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)" = "running" ] && return 0
+                return 1 ;;
+            "running starting") ;;
+            "running unhealthy") return 1 ;;
+            running*) ;;
+            *) return 1 ;;
+        esac
+        sleep 2
+    done
+    return 1
+}
+
+cmd_init() {
+    command -v docker >/dev/null || die "docker not found"
+    command -v openssl >/dev/null || die "openssl not found"
+
+    if [ ! -f "$ENV_FILE" ]; then
+        sed "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$(random_string 24)/" "$DEPLOY_DIR/.env.example" > "$ENV_FILE"
+        chmod 600 "$ENV_FILE"
+        info "Created $ENV_FILE with a generated PostgreSQL password. Review it (image, TLS, ACME_EMAIL) and run init again."
+        exit 0
+    fi
+
+    load_env
+    if [ "$TLS" = "true" ] && [ -z "${ACME_EMAIL:-}" ]; then
+        die "REZERVATOR_TLS=true needs ACME_EMAIL in $ENV_FILE"
+    fi
+
+    mkdir -p "$CUSTOMERS_DIR" "$BACKUPS_DIR"
+    chmod 700 "$CUSTOMERS_DIR" "$BACKUPS_DIR"
+    compose up --detach
+    wait_for_postgres
+    info "Proxy and PostgreSQL are running"
+}
+
+cmd_add() {
+    [ $# -ge 2 ] || die "Usage: add <name> <domain> [from]"
+    local name="$1" domain="$2" mail_from="${3:-noreply@$2}"
+    load_env
+    check_name "$name"
+    [[ "$domain" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "Invalid domain '$domain'"
+    [[ "$mail_from" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$ ]] || die "Invalid sender address '$mail_from'"
+    customer_exists "$name" && die "Customer '$name' already exists"
+    docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
+    wait_for_postgres
+
+    local db db_password admin_password dir
+    db="$(db_name "$name")"
+    db_password="$(random_string 24)"
+    admin_password="$(random_string 9)"
+    dir="$CUSTOMERS_DIR/$name"
+
+    if [ "$(psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname = '$db'")" = "1" ]; then
+        die "Database $db already exists"
+    fi
+
+    info "Creating database $db"
+    psql_admin <<EOF
+CREATE ROLE "$db" LOGIN PASSWORD '$db_password';
+CREATE DATABASE "$db" OWNER "$db";
+REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
+EOF
+
+    info "Writing config to $dir"
+    mkdir -p "$dir"
+    chmod 700 "$dir"
+    sed -e "s|@SESSION_KEY@|$(random_string 48)|" \
+        -e "s|@DB_USER@|$db|" \
+        -e "s|@DB_PASSWORD@|$db_password|" \
+        -e "s|@DB_NAME@|$db|" \
+        -e "s|@DB_HOST@|$PG_CONTAINER|" \
+        -e "s|@MAX_CONNECTIONS@|$CUSTOMER_MAX_CONNECTIONS|" \
+        -e "s|@MAIL_FROM@|$mail_from|" \
+        "$DEPLOY_DIR/config.template.toml" > "$dir/config.toml"
+    # The application in the container runs as another user and must be able to read it
+    chmod 644 "$dir/config.toml"
+    printf 'DOMAIN=%s\n' "$domain" > "$dir/customer.env"
+
+    info "Starting container $(container_name "$name")"
+    docker volume create "$(volume_name "$name")" >/dev/null
+    run_container "$name" "$IMAGE" "$admin_password"
+
+    if ! wait_healthy "$(container_name "$name")"; then
+        docker logs --tail 20 "$(container_name "$name")" >&2 || true
+        die "Container $(container_name "$name") did not become healthy. Fix the problem and run 'remove $name --purge' before adding it again."
+    fi
+
+    local scheme="http"
+    [ "$TLS" = "true" ] && scheme="https"
+    cat <<EOF
+
+Customer '$name' is running at $scheme://$domain
+
+  Admin login:    admin
+  Admin password: $admin_password
+
+The password is shown only now. Hand it over and have it changed after the first login.
+EOF
+}
+
+cmd_list() {
+    load_env
+    printf '%-20s %-34s %-12s %s\n' "CUSTOMER" "DOMAIN" "STATE" "IMAGE"
+    local name domain state image
+    for name in $(all_customers); do
+        # shellcheck disable=SC1090
+        domain="$(. "$CUSTOMERS_DIR/$name/customer.env" && echo "$DOMAIN")"
+        state="$(docker inspect --format '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$(container_name "$name")" 2>/dev/null || echo "no container")"
+        image="$(docker inspect --format '{{.Image}}' "$(container_name "$name")" 2>/dev/null || true)"
+        if [ -n "$image" ]; then
+            # Tag of the image when it still has one, otherwise the short id
+            image="$(docker image inspect --format '{{if .RepoTags}}{{index .RepoTags 0}}{{else}}{{slice .Id 7 19}}{{end}}' "$image" 2>/dev/null || echo "${image:7:12}")"
+        else
+            image="-"
+        fi
+        printf '%-20s %-34s %-12s %s\n' "$name" "$domain" "$state" "$image"
+    done
+}
+
+# Dumps the database and the data volume. $1 = name. Prints the timestamp of the backup.
+backup_customer() {
+    local name="$1" db dir timestamp
+    db="$(db_name "$name")"
+    dir="$BACKUPS_DIR/$name"
+    timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+    mkdir -p "$dir"
+    chmod 700 "$dir"
+
+    docker exec "$PG_CONTAINER" pg_dump -U postgres --no-owner --no-privileges "$db" | gzip > "$dir/$timestamp-db.sql.gz"
+    # Any image with tar would do; the application image is certainly present
+    docker run --rm --volume "$(volume_name "$name"):/data:ro" --entrypoint tar "$IMAGE" \
+        czf - -C /data . > "$dir/$timestamp-data.tar.gz"
+
+    # An empty or truncated dump must not pass for a backup
+    gzip -t "$dir/$timestamp-db.sql.gz" && gzip -t "$dir/$timestamp-data.tar.gz"
+    [ "$(gzip -dc "$dir/$timestamp-db.sql.gz" | wc -c)" -gt 0 ] || die "Database dump of '$name' is empty"
+
+    echo "$timestamp"
+}
+
+cmd_backup() {
+    [ $# -ge 1 ] || die "Usage: backup <name>...|--all"
+    load_env
+    wait_for_postgres
+    local names=("$@") name timestamp
+    if [ "$1" = "--all" ]; then
+        # shellcheck disable=SC2207
+        names=($(all_customers))
+    fi
+    [ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
+    for name in "${names[@]}"; do
+        require_customer "$name"
+        timestamp="$(backup_customer "$name")"
+        info "Backup of '$name': $BACKUPS_DIR/$name/$timestamp-db.sql.gz, $timestamp-data.tar.gz"
+    done
+}
+
+# Replaces the database and the data volume by a backup. The container must be stopped.
+restore_customer() {
+    local name="$1" timestamp="$2" db dir
+    db="$(db_name "$name")"
+    dir="$BACKUPS_DIR/$name"
+
+    psql_admin <<EOF
+DROP DATABASE IF EXISTS "$db" WITH (FORCE);
+CREATE DATABASE "$db" OWNER "$db";
+REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
+EOF
+    gzip -dc "$dir/$timestamp-db.sql.gz" | docker exec -i "$PG_CONTAINER" psql -U "$db" -d "$db" -v ON_ERROR_STOP=1 -q >/dev/null
+
+    docker run --rm -i --volume "$(volume_name "$name"):/data" --entrypoint sh "$IMAGE" \
+        -c 'find /data -mindepth 1 -delete && tar xzf - -C /data' < "$dir/$timestamp-data.tar.gz"
+}
+
+cmd_restore() {
+    [ $# -eq 2 ] || die "Usage: restore <name> <timestamp>"
+    local name="$1" timestamp="$2" container image
+    load_env
+    require_customer "$name"
+    [ -f "$BACKUPS_DIR/$name/$timestamp-db.sql.gz" ] && [ -f "$BACKUPS_DIR/$name/$timestamp-data.tar.gz" ] \
+        || die "Backup $timestamp of '$name' not found in $BACKUPS_DIR/$name"
+    wait_for_postgres
+
+    container="$(container_name "$name")"
+    # Keep the image the customer runs now; the configured one may be newer than the backup
+    image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || echo "$IMAGE")"
+
+    info "Stopping $container"
+    docker rm --force "$container" >/dev/null 2>&1 || true
+    info "Restoring '$name' from backup $timestamp"
+    restore_customer "$name" "$timestamp"
+    run_container "$name" "$image"
+    wait_healthy "$container" || die "Container $container is not healthy after the restore, see 'docker logs $container'"
+    info "Customer '$name' restored and running"
+}
+
+cmd_upgrade() {
+    [ $# -ge 1 ] || die "Usage: upgrade <name>...|--all"
+    load_env
+    docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
+    wait_for_postgres
+
+    local names=("$@") name container old_image new_image timestamp
+    if [ "$1" = "--all" ]; then
+        # shellcheck disable=SC2207
+        names=($(all_customers))
+    fi
+    [ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
+    new_image="$(docker image inspect --format '{{.Id}}' "$IMAGE")"
+
+    for name in "${names[@]}"; do
+        require_customer "$name"
+        container="$(container_name "$name")"
+        old_image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)"
+
+        if [ "$old_image" = "$new_image" ]; then
+            info "'$name' already runs $IMAGE, skipping"
+            continue
+        fi
+
+        info "Upgrading '$name'"
+        timestamp="$(backup_customer "$name")"
+        docker rm --force "$container" >/dev/null 2>&1 || true
+        run_container "$name" "$IMAGE"
+
+        if wait_healthy "$container"; then
+            info "'$name' upgraded (backup before the upgrade: $timestamp)"
+            continue
+        fi
+
+        echo "Upgrade of '$name' failed, last log lines of the new container:" >&2
+        docker logs --tail 20 "$container" >&2 || true
+        docker rm --force "$container" >/dev/null 2>&1 || true
+
+        if [ -z "$old_image" ]; then
+            die "'$name' had no container before, nothing to roll back to. Backup: $timestamp"
+        fi
+
+        # Migrations of the new version may have changed the database, so the old version
+        # gets the database from before the upgrade
+        info "Rolling '$name' back to the previous image and to backup $timestamp"
+        restore_customer "$name" "$timestamp"
+        run_container "$name" "$old_image"
+        if wait_healthy "$container"; then
+            die "'$name' was rolled back. Upgrade stopped, the remaining customers were not touched."
+        fi
+        die "'$name' is DOWN: the rollback did not become healthy either. Backup: $timestamp"
+    done
+}
+
+cmd_remove() {
+    [ $# -ge 1 ] || die "Usage: remove <name> [--purge]"
+    local name="$1" purge="${2:-}" db answer
+    load_env
+    require_customer "$name"
+
+    info "Removing container $(container_name "$name")"
+    docker rm --force "$(container_name "$name")" >/dev/null 2>&1 || true
+
+    if [ "$purge" != "--purge" ]; then
+        info "Database, data volume and config of '$name' were kept. Start it again with 'upgrade $name'."
+        return 0
+    fi
+
+    read -r -p "This deletes the database, uploaded files and config of '$name'. Type the name to confirm: " answer
+    [ "$answer" = "$name" ] || die "Not confirmed, nothing else was deleted"
+
+    wait_for_postgres
+    db="$(db_name "$name")"
+    psql_admin <<EOF
+DROP DATABASE IF EXISTS "$db" WITH (FORCE);
+DROP ROLE IF EXISTS "$db";
+EOF
+    docker volume rm "$(volume_name "$name")" >/dev/null 2>&1 || true
+    rm -rf "${CUSTOMERS_DIR:?}/$name"
+    info "Customer '$name' deleted. Backups in $BACKUPS_DIR/$name were kept."
+}
+
+main() {
+    [ $# -ge 1 ] || { usage; exit 1; }
+    local command="$1"
+    shift
+    case "$command" in
+        init) cmd_init "$@" ;;
+        add) cmd_add "$@" ;;
+        list) cmd_list "$@" ;;
+        upgrade) cmd_upgrade "$@" ;;
+        backup) cmd_backup "$@" ;;
+        restore) cmd_restore "$@" ;;
+        remove) cmd_remove "$@" ;;
+        -h|--help|help) usage ;;
+        *) usage; exit 1 ;;
+    esac
+}
+
+main "$@"