commit 9e16977b6ac2bcf0915fdc6954a0d10534b83c5e equwal <truex@equwal.com> 2026-09-22 17:21:33 -0700 Add burst mode: Terraform for workers on the cheapest cloud infra/burst holds everything that burst mode needs. Nothing in it runs until the operator turns it on (infra/burst/README.md). - state/: the shared state, applied once on the web server. It installs Postgres and Redis on 127.0.0.1 and a user that can only forward those two ports (setup-web-server.sh), and it makes the Cloudflare R2 bucket and a token for that bucket only. R2 charges no transfer fee, so workers on any cloud and customers read files at no extra cost. - workers/aws/ and workers/hetzner/: one spot instance or one server for each worker. The burst controller writes their variables and applies them. No inbound port on either cloud. On AWS, a power-off ends the instance. - cloud-init.yaml.tftpl: the first boot of a worker on either cloud. It opens an autossh tunnel to Postgres and Redis, installs the app at the web server's commit from the bucket and the speech stack at the web server's versions, starts the worker on the paid queue, and powers the machine off after 90 idle minutes if the controller is gone. - systemd/: the controller timer, and the web server's own worker with a memory limit that protects Postgres. The Terraform tests (terraform test, mock providers, no account) check the bucket-only token, the pinned host key of the web server, the tunnel and database addresses, the spot settings, the closed firewalls and the worker names. set-secret.sh can now write a key to another file, for the cloud keys of burst mode. .gitattributes keeps LF line endings for the Terraform, template and unit files. A CR from a Windows checkout would reach the cloud-init file, the env files and the shell scripts of a worker.
.gitattributes | 8 + .gitignore | 7 + README.md | 8 + infra/burst/README.md | 287 +++++++++++++++++++++ infra/burst/cloud-init.yaml.tftpl | 163 ++++++++++++ infra/burst/state/.terraform.lock.hcl | 61 +++++ infra/burst/state/main.tf | 124 +++++++++ infra/burst/state/outputs.tf | 54 ++++ infra/burst/state/setup-web-server.sh | 76 ++++++ infra/burst/state/tests/host_key.pub | 1 + infra/burst/state/tests/state.tftest.hcl | 109 ++++++++ infra/burst/state/variables.tf | 48 ++++ infra/burst/systemd/subplz-burst.service | 15 ++ infra/burst/systemd/subplz-burst.timer | 9 + infra/burst/systemd/subplz-worker.service | 21 ++ infra/burst/worker-constraints.txt | 89 +++++++ infra/burst/workers/aws/.terraform.lock.hcl | 26 ++ infra/burst/workers/aws/iam-policy.json | 60 +++++ infra/burst/workers/aws/main.tf | 128 +++++++++ infra/burst/workers/aws/outputs.tf | 4 + infra/burst/workers/aws/tests/aws.tftest.hcl | 109 ++++++++ infra/burst/workers/aws/variables.tf | 48 ++++ infra/burst/workers/hetzner/.terraform.lock.hcl | 23 ++ infra/burst/workers/hetzner/main.tf | 71 +++++ infra/burst/workers/hetzner/outputs.tf | 4 + .../burst/workers/hetzner/tests/hetzner.tftest.hcl | 66 +++++ infra/burst/workers/hetzner/variables.tf | 43 +++ tools/set-secret.sh | 19 +- 28 files changed, 1673 insertions(+), 8 deletions(-)
diff --git a/.gitattributes b/.gitattributes index dfdb8b7..a765581 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,9 @@ *.sh text eol=lf +# Terraform, templates and unit files run on Linux: a CR would reach the +# cloud-init file, the env files and the shell scripts of a worker. +*.tf text eol=lf +*.tftpl text eol=lf +*.hcl text eol=lf +*.service text eol=lf +*.timer text eol=lf +*.pub text eol=lf diff --git a/.gitignore b/.gitignore index a846ed3..42c6d87 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,10 @@ tests/engine/excerpt-local/ # Test tooling for the browser engine node_modules/ + +# Terraform (infra/burst): providers, and state files, which hold passwords. +# The state lives in /var/lib/subplz-burst on the web server, not here. +.terraform/ +*.tfstate +*.tfstate.* +crash.log diff --git a/README.md b/README.md index 33c5b09..aaed18c 100644 --- a/README.md +++ b/README.md @@ -490,6 +490,14 @@ With an external queue the API copies staged uploads into shared storage before enqueuing, and the worker pulls them down, so the API and workers do not need a shared filesystem. +Paid jobs and free jobs go to separate queues. `SUBPLZ_WEB_WORKER_QUEUES` +sets which a worker takes: `paid,free` (the default) or `paid`. + +**Burst mode** adds workers on the cheapest cloud (AWS spot or Hetzner) when +paid jobs wait, and removes them when they are idle. It is built with +Terraform and is off until the operator turns it on: see +[`infra/burst/README.md`](infra/burst/README.md). + ### Before going public - Set the Stripe keys, the webhook and SMTP (see *Turning payments on*). diff --git a/infra/burst/README.md b/infra/burst/README.md new file mode 100644 index 0000000..e6ab776 --- /dev/null +++ b/infra/burst/README.md @@ -0,0 +1,287 @@ +# Burst mode: more workers, on the cheapest cloud, when paid jobs wait + +Burst mode is **off**. Nothing here runs, and nothing here costs money, until +the owner turns it on with the procedure below. + +## What it does + +Today the web server does each server job itself, one at a time. When many +paid jobs wait (a 500-book pack, an unlimited plan), buyers wait for hours. + +In burst mode: + +- The web server keeps the data: Postgres and Redis on its own 127.0.0.1, and + its own worker, which takes paid jobs first and then free ones. +- The files go to a Cloudflare R2 bucket. R2 charges no transfer fee, to any + cloud or to a customer who downloads a result. +- Once a minute the burst controller (`python -m tools.burst`) reads the paid + queue. When paid jobs wait, it reads the live offers of each cloud that has + keys: the newest AWS spot prices, and the Hetzner prices of the server types + that are in stock. A new worker goes to the offer with the lowest cost per + book, inside the spend cap. +- A worker reaches Postgres and Redis through an SSH tunnel to the web server. + The tunnel user can forward those two ports and do nothing else. A worker + takes no inbound connection. +- A worker with no job for 10 minutes stops after its job, and Terraform + destroys its machine. A scale-in never kills a job. When a cloud takes a spot + machine back during a job, the job runs once more on another worker. +- Free credits never start a machine: burst workers take paid jobs only. + +``` + web server (RackNerd, US west) + Postgres + Redis on 127.0.0.1, own worker, controller + ^ | + SSH tunnel | | terraform apply + (2 ports) | v + burst workers on AWS spot (us-west-2) or Hetzner, 0..N + | + v + Cloudflare R2 bucket (inputs, results, app releases, backups) +``` + +## Money + +**Spend cap.** The controller keeps the cost of the burst machines of the last +30 days under a cap: + + cap = min($100, max($5, 30% of the net server revenue of the last 30 days)) + +The net revenue counts each pack for 30 days after its purchase, and each +active subscription, after the Stripe fee (2.9% + 30 cents, and 0.7% more for +a subscription). Free credits count for nothing. The $5 floor lets the first +buyer have a burst worker. The controller also keeps 2.5 hours of each running +worker in reserve. At the cap it starts no new machine, and running jobs +finish. Change the numbers with `SUBPLZ_BURST_CAP_SHARE`, +`SUBPLZ_BURST_CAP_FLOOR_USD` and `SUBPLZ_BURST_CAP_CEILING_USD`. + +**Other limits.** `SUBPLZ_BURST_MAX_WORKERS` (default 1). The AWS spot price +limit (`max_price`, $0.0625 an hour). The AWS spot limit of a new account (5 +vCPU: one worker). The Hetzner server limit. A machine powers itself off after +90 idle minutes, if the controller is gone. + +**Prices** (financial advocate, 2026-09-22; the controller reads the live ones): + +| Offer | Per hour | Compute per 10-hour book | Note | +|---|---|---|---| +| AWS spot, 4-vCPU x86 (c6a, c7a, m6a) | about $0.06, + $0.007 IPv4 and disk | about $0.14 | billed per second | +| AWS spot, 4-vCPU Graviton (c7g) | $0.041-0.056 | $0.10-0.14 | off until one test book: add to `SUBPLZ_BURST_AWS_TYPES` | +| Hetzner EU cx33 | about $0.016 | about $0.05 | often sold out; billed per started hour | +| Hetzner EU cpx32 | about $0.065 | $0.14-0.20 | 155 ms from the web server | +| Hetzner US cpx31 | about $0.115 | $0.25-0.35 | | +| R2 bucket | $0.015 per GB-month | | no transfer fee; 10 GB free | + +While burst mode is on and idle, only the R2 storage costs money. AWS charges +$0.09/GB for the results that its workers send to R2, above 100 GB a month: +when you pass that, set `SUBPLZ_BURST_AWS_EGRESS_USD_PER_BOOK` (about 0.09 x +the GB of a result), so that the price comparison includes it. + +## Parts + +| Path | What | +|---|---| +| `state/` | The web server's Postgres, Redis and tunnel user (`setup-web-server.sh`), the R2 bucket and its token. Apply once, on the web server. | +| `workers/aws/` | One spot instance for each AWS worker. Only the controller applies it. `iam-policy.json` is the policy of the controller's AWS key. | +| `workers/hetzner/` | One server for each Hetzner worker. Only the controller applies it. | +| `cloud-init.yaml.tftpl` | What a new machine does at first boot, on either cloud. | +| `worker-constraints.txt` | The package versions of the web server, so that a worker computes the same result. | +| `systemd/` | Units for the web server: the controller timer and the web server's own worker. | +| `../../tools/burst/` | The controller: `core.py` decides, `offers.py` reads the prices, `revenue.py` reads the revenue. | +| `../../tools/copy_database.py` | Copies SQLite to Postgres (once, when burst mode goes on) and back. | + +## Accounts (owner steps, once) + +Only the owner does these steps. Claude never types a key. + +1. **Cloudflare**: an account with R2 turned on (done on 2026-09-22). At + activation, make an API token for Terraform (My Profile, API Tokens, Create + Custom Token) with two permissions: Account, Workers R2 Storage, Edit; and + User, API Tokens, Edit. Write down the account ID (R2 page, right side). +2. **AWS**, for spot workers: an account on the Paid plan (the Free plan blocks + xlarge machines), with MFA on the root user. Make an IAM user + `subplz-burst` with the policy in `workers/aws/iam-policy.json`, and an + access key for it. +3. **Hetzner**, for Hetzner workers: an account, a project `subplz-burst` with + nothing else in it, and an API token (Read & Write) in that project. Check + the server limit of the project (Limits). + +The controller uses each cloud that has keys. One cloud is enough; two let it +switch to the cheaper one. + +## Turn burst mode on + +Do these steps as root on the web server. The site is down from step 8 to +step 12, about 10 minutes. + +1. Install Terraform: + + ```bash + wget -qO- https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com bookworm main" > /etc/apt/sources.list.d/hashicorp.list + apt-get update && apt-get install -y terraform + ``` + +2. Put the keys in `/etc/subplz-burst.env`. Each command asks for the value + with the input hidden. Leave out the lines of a cloud that you do not use: + + ```bash + cd /opt/subplz-web + tools/set-secret.sh CLOUDFLARE_API_TOKEN /etc/subplz-burst.env + tools/set-secret.sh SUBPLZ_BURST_AWS_ACCESS_KEY_ID /etc/subplz-burst.env + tools/set-secret.sh SUBPLZ_BURST_AWS_SECRET_ACCESS_KEY /etc/subplz-burst.env + tools/set-secret.sh HCLOUD_TOKEN /etc/subplz-burst.env + echo "SUBPLZ_BURST_MAX_WORKERS=1" >> /etc/subplz-burst.env + ``` + +3. Install the queue, storage and Postgres clients into the app, and make the + folder of the Terraform state (it holds passwords): + + ```bash + .venv/bin/pip install -c infra/burst/worker-constraints.txt redis rq boto3 "psycopg[binary]" + install -d -m 700 /var/lib/subplz-burst + ``` + +4. Make the shared state. This installs Postgres, Redis and the tunnel user on + this machine, and makes the bucket and its token. Read the plan before you + type `yes`: + + ```bash + set -a; . /etc/subplz-burst.env; set +a + S=/opt/subplz-web/infra/burst/state + terraform -chdir=$S init -backend-config=path=/var/lib/subplz-burst/state.tfstate + terraform -chdir=$S apply -var cloudflare_account_id=YOUR_ACCOUNT_ID + ``` + +5. Write the storage keys and the backup settings: + + ```bash + (umask 077 + terraform -chdir=$S output -raw storage_env > /etc/subplz-storage.env + terraform -chdir=$S output -raw rclone_env > /etc/subplz-rclone.env + echo "BUCKET=$(terraform -chdir=$S output -raw bucket)" > /etc/subplz-backup.env) + ``` + +6. Make the backups: + + ```bash + cp .env .env.bak-before-burst + cp data/subplz.db data/subplz.db.bak-before-burst + ``` + +7. Read the new database address: + + ```bash + DB=$(terraform -chdir=$S output -raw web_env | sed -n 's/^SUBPLZ_WEB_DATABASE_URL=//p') + ``` + +8. Stop the site: + + ```bash + systemctl stop subplz-web + ``` + +9. Copy the database to Postgres, and the files to the bucket: + + ```bash + .venv/bin/python -m tools.copy_database sqlite:////opt/subplz-web/data/subplz.db "$DB" + env $(cat /etc/subplz-rclone.env) rclone copy --progress data/artifacts "r2:$(terraform -chdir=$S output -raw bucket)/jobs" + ``` + +10. Tell the app to use the queue, Postgres and the bucket: + + ```bash + terraform -chdir=$S output -raw web_env >> .env + install -d /etc/systemd/system/subplz-web.service.d + printf '[Service]\nEnvironmentFile=/etc/subplz-storage.env\n' > /etc/systemd/system/subplz-web.service.d/storage.conf + cp infra/burst/systemd/subplz-worker.service infra/burst/systemd/subplz-burst.* /etc/systemd/system/ + systemctl daemon-reload + ``` + +11. Start the site, the web server's own worker, and the nightly backup: + + ```bash + systemctl start subplz-web + systemctl enable --now subplz-worker subplz-pgdump.timer + ``` + +12. Do a test job on the site with "Convert in this browser" off. Make sure + that it succeeds and that its files download. If it fails, turn burst mode + off (below) before you look for the cause. + +13. Prepare the worker modules of each cloud that has keys: + + ```bash + for c in aws hetzner; do + terraform -chdir=infra/burst/workers/$c init -backend-config=path=/var/lib/subplz-burst/workers-$c.tfstate + done + ``` + +14. Start the controller. Do a dry run first: it logs its decision, the offers + and the spend cap, and changes nothing: + + ```bash + set -a; . ./.env; . /etc/subplz-storage.env; . /etc/subplz-burst.env; set +a + .venv/bin/python -m tools.burst --dry-run + systemctl enable --now subplz-burst.timer + ``` + +15. Watch it: `journalctl -u subplz-burst -f`. A new worker needs about 10 + minutes to install before it takes a job. + +16. Measure one book on each cloud that the controller used. If a 10-hour book + takes much more or less than 2 hours on 4 vCPU, the price comparison is + off: tell Claude the times. + +## While burst mode is on + +- **Deploy**: after `git pull`, restart `subplz-web` and `subplz-worker`. The + controller sees the new commit. It stops the workers of the old commit after + their jobs and starts new ones. You do nothing else. +- **Workers**: change `SUBPLZ_BURST_MAX_WORKERS` in `/etc/subplz-burst.env`. + The next run uses it. With `0`, each burst worker stops after its job and its + machine goes. +- **AWS spot limit**: when paid jobs often wait for more than an hour, ask AWS + for more spot vCPU (Service Quotas, "All Standard Spot Instance Requests"), + then raise `SUBPLZ_BURST_AWS_VCPU_LIMIT` and `SUBPLZ_BURST_MAX_WORKERS`. +- **Keys**: do not change the passwords, the tunnel key or the bucket token + while a worker runs. Set the workers to 0 first, and wait until + `terraform -chdir=infra/burst/workers/<cloud> state list` shows nothing. +- **State**: `/var/lib/subplz-burst` is the only record of what Terraform made. + If it is lost, the machines still run and cost money: find them in the AWS + and Hetzner consoles (tag or label `subplz-burst-worker`) and delete them. +- **Storage**: nothing deletes files. R2 costs $0.015 per GB-month; watch the + bucket size in the Cloudflare dashboard. + +## Turn burst mode off + +1. Set `SUBPLZ_BURST_MAX_WORKERS=0`. Wait until the `state list` of each worker + module shows nothing. Then: + `systemctl disable --now subplz-burst.timer subplz-worker subplz-pgdump.timer`. +2. Stop the site. Copy Postgres to a new SQLite file, and the bucket to the + disk: + + ```bash + systemctl stop subplz-web + .venv/bin/python -m tools.copy_database "$DB" sqlite:////opt/subplz-web/data/subplz-from-postgres.db + env $(cat /etc/subplz-rclone.env) rclone copy --progress "r2:$(terraform -chdir=$S output -raw bucket)/jobs" data/artifacts + mv data/subplz.db data/subplz.db.bak-burst && mv data/subplz-from-postgres.db data/subplz.db + ``` + +3. Remove the burst lines from `.env` (the six lines that step 10 added), and + remove `/etc/systemd/system/subplz-web.service.d/storage.conf`. Then + `systemctl daemon-reload && systemctl start subplz-web`, and do a test job. +4. Only after the test job succeeds: remove the bucket and its token with + `terraform -chdir=$S destroy`. It deletes the bucket with its files. + Postgres and Redis stay installed on the web server, unused. + +## Security + +- Postgres and Redis listen on 127.0.0.1 only. A worker reaches them through + an SSH tunnel as `subplz-tunnel`, a user with no shell whose key can forward + those two ports and nothing else. The worker checks the web server's host key. +- A worker accepts no inbound connection (AWS security group and Hetzner + firewall without inbound rules). +- The bucket token can read and write that one bucket only. The AWS key can end + only instances tagged `subplz-burst-worker`. +- The state files and `/etc/subplz-*.env` hold secrets: root only, mode 0600. + A machine's user data holds the tunnel key and the database passwords; only + the cloud account and the machine itself can read it. diff --git a/infra/burst/cloud-init.yaml.tftpl b/infra/burst/cloud-init.yaml.tftpl new file mode 100644 index 0000000..9f84b13 --- /dev/null +++ b/infra/burst/cloud-init.yaml.tftpl @@ -0,0 +1,163 @@ +#cloud-config +# One burst worker, on any cloud: the subplz-web app at commit ${ref}, running +# worker.py on the paid queue. Postgres and Redis stay on the web server; the +# worker reaches them through an SSH tunnel, on its own 127.0.0.1. The worker +# modules (workers/aws, workers/hetzner) render this file. +hostname: ${name} +preserve_hostname: false +package_update: true +packages: + - autossh + - curl + - ffmpeg + - git + - python3-venv +users: + - name: subplz + system: true + homedir: /opt/subplz-web + shell: /usr/sbin/nologin +write_files: + - path: /etc/subplz-worker.env + owner: root:root + permissions: "0600" + content: | + SUBPLZ_WEB_QUEUE_BACKEND=redis + SUBPLZ_WEB_WORKER_QUEUES=paid + SUBPLZ_WEB_REDIS_URL=${redis_url} + SUBPLZ_WEB_DATABASE_URL=${database_url} + SUBPLZ_WEB_STORAGE_BACKEND=s3 + SUBPLZ_WEB_S3_BUCKET=${bucket} + SUBPLZ_WEB_S3_PREFIX=${s3_prefix} + SUBPLZ_WEB_DEVICE=cpu + SUBPLZ_WEB_THREADS=${threads} + HOME=/opt/subplz-web + ${indent(6, storage_env)} + - path: /etc/subplz/tunnel_key + permissions: "0600" + content: | + ${indent(6, tunnel_private_key)} + - path: /etc/subplz/known_hosts + permissions: "0644" + content: | + ${known_hosts} + - path: /etc/systemd/system/subplz-tunnel.service + content: | + [Unit] + Description=SSH tunnel to Postgres and Redis on the web server + After=network-online.target + Wants=network-online.target + + [Service] + User=subplz + Environment=AUTOSSH_GATETIME=0 + ExecStart=/usr/bin/autossh -M 0 -N -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o ExitOnForwardFailure=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/subplz/known_hosts -o IdentitiesOnly=yes -i /etc/subplz/tunnel_key -p ${ssh_port} -L 127.0.0.1:5432:127.0.0.1:5432 -L 127.0.0.1:6379:127.0.0.1:6379 subplz-tunnel@${web_server_ip} + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target + - path: /etc/systemd/system/subplz-worker.service + content: | + [Unit] + Description=subplz-web burst worker + After=subplz-tunnel.service + Requires=subplz-tunnel.service + + [Service] + User=subplz + WorkingDirectory=/opt/subplz-web + EnvironmentFile=/etc/subplz-worker.env + Environment=PATH=/opt/subplz-web/.venv/bin:/usr/local/bin:/usr/bin:/bin + ExecStart=/opt/subplz-web/.venv/bin/python worker.py + # After the shutdown command of the burst controller, rq finishes its job + # and exits with 0. Then the worker must stay stopped. + Restart=on-failure + RestartSec=10 + + [Install] + WantedBy=multi-user.target + - path: /etc/systemd/system/subplz-idle-check.service + content: | + [Unit] + Description=Power off this burst machine when its worker is idle too long + + [Service] + Type=oneshot + EnvironmentFile=/etc/subplz-worker.env + ExecStart=/bin/bash /opt/subplz-install/idle-check.sh + - path: /etc/systemd/system/subplz-idle-check.timer + content: | + [Unit] + Description=Check each 10 minutes whether the worker is idle + + [Timer] + OnActiveSec=10min + OnUnitActiveSec=10min + + [Install] + WantedBy=timers.target + - path: /opt/subplz-install/idle-check.sh + permissions: "0700" + content: | + #!/bin/bash + # A safety net for the burst controller: if the controller is gone, this + # machine still stops. After ${idle_checks} idle checks in a row (10 minutes + # each), or when Redis cannot be reached, power off. On AWS a power-off + # ends the machine and its bill; on Hetzner the controller then deletes it. + state_file=/var/lib/subplz-idle-count + state=$(/opt/subplz-web/.venv/bin/python - <<'PY' 2>/dev/null || echo idle + import os, socket + from redis import Redis + from rq import Worker + conn = Redis.from_url(os.environ["SUBPLZ_WEB_REDIS_URL"], socket_timeout=10) + host = socket.gethostname() + states = [w.get_state() for w in Worker.all(connection=conn) if w.hostname == host] + print("busy" if "busy" in states else "idle") + PY + ) + if [ "$state" = "busy" ]; then + echo 0 > "$state_file" + exit 0 + fi + count=$(( $(cat "$state_file" 2>/dev/null || echo 0) + 1 )) + echo "$count" > "$state_file" + if [ "$count" -ge ${idle_checks} ]; then + systemctl poweroff + fi + - path: /opt/subplz-install/constraints.txt + content: | + ${indent(6, constraints)} + - path: /opt/subplz-install/install.sh + permissions: "0700" + content: | + #!/bin/bash + # Install the app from the bucket (the repository is private), then the + # speech stack at the versions of the web server, then start the tunnel, + # the worker and the idle check. + set -euo pipefail + set -a + . /etc/subplz-worker.env + set +a + chown subplz /etc/subplz/tunnel_key + cd /opt/subplz-install + curl --fail --silent --show-error --retry 5 \ + --aws-sigv4 "aws:amz:$AWS_DEFAULT_REGION:s3" \ + --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" \ + -o app.tar.gz "$AWS_ENDPOINT_URL_S3/$SUBPLZ_WEB_S3_BUCKET/releases/${ref}.tar.gz" + mkdir -p /opt/subplz-web + tar -xzf app.tar.gz -C /opt/subplz-web + python3 -m venv /opt/subplz-web/.venv + pip=/opt/subplz-web/.venv/bin/pip + # CPU builds of torch and torchaudio both come from the PyTorch index. A + # torchaudio from PyPI is a CUDA build and fails at run time. Their + # dependencies come from PyPI in the next step, at the pinned versions. + $pip install --quiet --no-deps --index-url https://download.pytorch.org/whl/cpu \ + -c constraints.txt torch torchaudio + $pip install --quiet -c constraints.txt torch torchaudio "${subplz_spec}" \ + -r /opt/subplz-web/requirements.txt redis rq boto3 "psycopg[binary]" + chown -R subplz:subplz /opt/subplz-web + systemctl daemon-reload + systemctl enable --now subplz-tunnel.service subplz-worker.service subplz-idle-check.timer +runcmd: + - [bash, /opt/subplz-install/install.sh] diff --git a/infra/burst/state/.terraform.lock.hcl b/infra/burst/state/.terraform.lock.hcl new file mode 100644 index 0000000..13a5790 --- /dev/null +++ b/infra/burst/state/.terraform.lock.hcl @@ -0,0 +1,61 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/cloudflare/cloudflare" { + version = "5.25.0" + constraints = "~> 5.25" + hashes = [ + "h1:JkPKsRlBVN9yY/kN92pjEBmnaI+puaB63gPM3igAZw4=", + "zh:403d477c0e4c05a9f9886ef7e750dcc3488be7cd2210207e98b4a731bc499159", + "zh:4a2a4659c3273ddb1437a34865eac21823a117557b050786ba510ac5f34047cf", + "zh:6e4a2f6ed105b5353c7b56962d00f923873dd68af6509b9b9f595057edba1915", + "zh:aadd2051c1eb01e7b518d4808de7f5192b2ecbc841d70924785ee3385a889b9e", + "zh:ab0322bcb0d4e465f8603b39f7e743c836b22ae775c03ed677ad999a6a1b8bf4", + "zh:c47e38911c92e0b2ffe7be1d36d96b79dd2af79bc61745d6afe5a730cae9f287", + "zh:df3f13ae57104ce2f7619cc6c050ab90a5a3299470cdbd2ab8dd3809b84edcb8", + "zh:f809ab383cca0a5f83072981c64208cbd7fa67e986a86ee02dd2c82333221e32", + "zh:fb7de9d22b02367036c146bd683c8968ec4bbb56aca3185fa84146d118a2bf96", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.1" + constraints = "~> 3.6" + hashes = [ + "h1:7uiStw0Rl9KOdX5UNMG/sp9nyadoD4LZekQTiYlYPhE=", + "zh:05f4734c1f0be840b711b3eff259ebc5fca436784c728955b1678078466f48d7", + "zh:0b91bf19371d012434eba1deeb6aab77158def9b39601dcbd94450b3974a2a26", + "zh:0ee6eacd47ec00183d55d726a4b6c4ce951a199f944bf22f1aa58392ebdfa7a2", + "zh:19388a4074b76a89a43a6c8328d7ae8ee2e7de3d346af51e80d3e6d3d12925f1", + "zh:23e74d48c5e2ac2e823fd527f49fee9db37d32a1990c9e3bf126ead697b843eb", + "zh:3cabf7fbd096c520064aae3aba61aba670af83ab91291a71fa1b1332929c2b7f", + "zh:5c0a3b8af0be60be4eca12ddee385cfa8babc1ec8e98cdf9de2f2274c73eabfa", + "zh:60b4f8a8ef18f52bf8e19215229dae408bee732825964092db7c989fd2de4097", + "zh:7359015acfedcbd6366f2329c854cf8d3c8ca5cd0faa89d2d37db358d6eba6c5", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b38758402f0e13a1071162da28994023cd2ac676e54af350c9ffd8dfa73fa7b", + "zh:7c7fbb8895eb75bb4de1f933e98553bd99c8d048c89a925ddba490aa5a67f7dc", + "zh:8c2b8c6a7ccdec16b73e2fb9f3700ea097f58c592571e4c5de60c93d2301732c", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.4.1" + constraints = "~> 4.0" + hashes = [ + "h1:BHiQRPHBNJKfcr+kzFko7BZ2wWqsDx4U8A0MPDQRbPk=", + "zh:0855f9a980426a0af2c4415fb23cccbbb88c2d1dd40d83da6f45622c075b398b", + "zh:189227fdae28e547b1d988f5d83c4280fce02e3152d8b62aa810047e2f9f7e59", + "zh:2557db4f493aa90b3683905d9eeb103e55a3facc4ccda763d6de6025116ae121", + "zh:2cfd8d6a543ac61c6e7c0461839ab89bb8da1ba6c8090ab946b1cf47f100d7fa", + "zh:5c04cf2a683ea60a0072c59bbcf415f232c382c891b3aaeb9bbbb28213e0d878", + "zh:6681e35d50328bc11a49bb61c002121353f6a844cc6d63a0c3db57745ec38a56", + "zh:693b810e438bb1dfbc13a42629597284ac1eb08eb30f46f18738d7a9bb768d34", + "zh:77c4425f75346b23817837f3947b6ab7e765308abbf573fec5d2cafd4b82f142", + "zh:b3fd64ec1cecea98c88d2332dbeac40498ce2506e3235bdbd1afdfbaa47bb758", + "zh:c06224b5797b129a84e2514be43f2b140622928fe9c92b55d7941be3296ea289", + "zh:debc830ec123f27944c6170d76cb4ea3542845896960bb9fc530702774df328e", + "zh:f300fcc909014550fd74b2b751665dcc049f874e7a475903f65671baf2c2d86d", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/infra/burst/state/main.tf b/infra/burst/state/main.tf new file mode 100644 index 0000000..b317b5d --- /dev/null +++ b/infra/burst/state/main.tf @@ -0,0 +1,124 @@ +# The shared state of burst mode: Postgres and Redis on the web server, the +# file bucket (Cloudflare R2), and the key of the SSH tunnel that each burst +# worker uses to reach Postgres and Redis. Workers can come from any cloud; +# this module does not depend on one. +# +# Run it ON THE WEB SERVER, as root: the setup step changes this machine +# (see setup-web-server.sh). See ../README.md for when and how. + +terraform { + required_version = ">= 1.9" + required_providers { + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 5.25" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } + } + # The state holds passwords and keys: /var/lib/subplz-burst, root only. + backend "local" {} +} + +# Reads CLOUDFLARE_API_TOKEN. +provider "cloudflare" {} + +resource "random_password" "postgres" { + length = 32 + special = false +} + +resource "random_password" "redis" { + length = 32 + special = false +} + +# One key for every worker: it can open the two tunnels and nothing else. +resource "tls_private_key" "tunnel" { + algorithm = "ED25519" +} + +resource "cloudflare_r2_bucket" "files" { + account_id = var.cloudflare_account_id + name = var.bucket_name + location = var.bucket_location +} + +# The permission groups "read objects" and "write objects" of R2, found by name +# unless var.r2_permission_group_ids gives them. +data "cloudflare_api_token_permission_groups_list" "r2_read" { + count = var.r2_permission_group_ids == null ? 1 : 0 + name = "Workers%20R2%20Storage%20Bucket%20Item%20Read" +} + +data "cloudflare_api_token_permission_groups_list" "r2_write" { + count = var.r2_permission_group_ids == null ? 1 : 0 + name = "Workers%20R2%20Storage%20Bucket%20Item%20Write" +} + +# The app and the workers get a key for this bucket only. R2 takes the token +# id as the S3 key id, and the SHA-256 of the token value as the S3 secret. +resource "cloudflare_api_token" "files" { + name = "${var.bucket_name}-app" + policies = [{ + effect = "allow" + permission_groups = [for id in local.r2_permission_groups : { id = id }] + resources = jsonencode({ + "com.cloudflare.edge.r2.bucket.${var.cloudflare_account_id}_default_${cloudflare_r2_bucket.files.name}" = "*" + }) + }] +} + +# Postgres and Redis on 127.0.0.1, and the tunnel user. The script is +# idempotent: a second run changes nothing. +resource "terraform_data" "web_server" { + triggers_replace = [ + filesha256("${path.module}/setup-web-server.sh"), + sha256(random_password.postgres.result), + sha256(random_password.redis.result), + tls_private_key.tunnel.public_key_openssh, + ] + + provisioner "local-exec" { + command = "bash ${path.module}/setup-web-server.sh" + environment = { + POSTGRES_PASSWORD = random_password.postgres.result + REDIS_PASSWORD = random_password.redis.result + TUNNEL_PUBLIC_KEY = tls_private_key.tunnel.public_key_openssh + } + } +} + +locals { + r2_permission_groups = var.r2_permission_group_ids != null ? var.r2_permission_group_ids : concat( + [for d in data.cloudflare_api_token_permission_groups_list.r2_read : d.result[0].id], + [for d in data.cloudflare_api_token_permission_groups_list.r2_write : d.result[0].id], + ) + s3_endpoint = "https://${var.cloudflare_account_id}.r2.cloudflarestorage.com" + s3_key_id = cloudflare_api_token.files.id + s3_secret = sha256(cloudflare_api_token.files.value) + # Through the tunnel, a worker sees Postgres and Redis on its own 127.0.0.1, + # as the web server does. + database_url = "postgresql+psycopg://subplz:${random_password.postgres.result}@127.0.0.1:5432/subplz" + redis_url = "redis://:${random_password.redis.result}@127.0.0.1:6379/0" + known_hosts = ( + var.ssh_port == 22 + ? "${var.web_server_ip} ${trimspace(file(var.host_key_file))}" + : "[${var.web_server_ip}]:${var.ssh_port} ${trimspace(file(var.host_key_file))}" + ) + # boto3 1.36 and later send checksums that some S3 services refuse. + storage_env = <<-EOT + AWS_ACCESS_KEY_ID=${local.s3_key_id} + AWS_SECRET_ACCESS_KEY=${local.s3_secret} + AWS_ENDPOINT_URL_S3=${local.s3_endpoint} + AWS_DEFAULT_REGION=auto + AWS_REQUEST_CHECKSUM_CALCULATION=when_required + AWS_RESPONSE_CHECKSUM_VALIDATION=when_required + EOT +} diff --git a/infra/burst/state/outputs.tf b/infra/burst/state/outputs.tf new file mode 100644 index 0000000..02c10b2 --- /dev/null +++ b/infra/burst/state/outputs.tf @@ -0,0 +1,54 @@ +# What the worker modules read (through terraform_remote_state). +output "worker" { + description = "Settings of a burst worker, on any cloud." + sensitive = true + value = { + web_server_ip = var.web_server_ip + ssh_port = var.ssh_port + known_hosts = local.known_hosts + tunnel_private_key = tls_private_key.tunnel.private_key_openssh + database_url = local.database_url + redis_url = local.redis_url + bucket = cloudflare_r2_bucket.files.name + s3_prefix = "jobs/" + storage_env = local.storage_env + } +} + +# The lines to add to /opt/subplz-web/.env on the web server when burst mode goes on. +output "web_env" { + description = "Settings of the app on the web server (append to /opt/subplz-web/.env)." + sensitive = true + value = <<-EOT + SUBPLZ_WEB_QUEUE_BACKEND=redis + SUBPLZ_WEB_REDIS_URL=${local.redis_url} + SUBPLZ_WEB_DATABASE_URL=${local.database_url} + SUBPLZ_WEB_STORAGE_BACKEND=s3 + SUBPLZ_WEB_S3_BUCKET=${cloudflare_r2_bucket.files.name} + SUBPLZ_WEB_S3_PREFIX=jobs/ + EOT +} + +# boto3 reads these from the process environment, not from .env: they go to +# /etc/subplz-storage.env, which the systemd units load. +output "storage_env" { + description = "Storage keys for the web server (write to /etc/subplz-storage.env, mode 0600)." + sensitive = true + value = local.storage_env +} + +output "rclone_env" { + description = "Environment for rclone: the one-time copy of the files, and the nightly Postgres backup." + sensitive = true + value = <<-EOT + RCLONE_CONFIG_R2_TYPE=s3 + RCLONE_CONFIG_R2_PROVIDER=Cloudflare + RCLONE_CONFIG_R2_ENDPOINT=${local.s3_endpoint} + RCLONE_CONFIG_R2_ACCESS_KEY_ID=${local.s3_key_id} + RCLONE_CONFIG_R2_SECRET_ACCESS_KEY=${local.s3_secret} + EOT +} + +output "bucket" { + value = cloudflare_r2_bucket.files.name +} diff --git a/infra/burst/state/setup-web-server.sh b/infra/burst/state/setup-web-server.sh new file mode 100644 index 0000000..447c640 --- /dev/null +++ b/infra/burst/state/setup-web-server.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Set up the shared state of burst mode on the web server (Debian 12): +# +# - Postgres on 127.0.0.1: role and database "subplz". +# - Redis on 127.0.0.1, with a password, and a journal on disk. +# - The user "subplz-tunnel": burst workers log in as this user only to +# forward two ports (Postgres and Redis). No shell, no other port. +# - A nightly Postgres backup to the bucket (enabled by the runbook). +# +# Terraform (main.tf) runs this script on the web server itself. It is +# idempotent: a second run changes nothing. Input: POSTGRES_PASSWORD, +# REDIS_PASSWORD, TUNNEL_PUBLIC_KEY in the environment. +set -euo pipefail +: "${POSTGRES_PASSWORD:?}" "${REDIS_PASSWORD:?}" "${TUNNEL_PUBLIC_KEY:?}" + +export DEBIAN_FRONTEND=noninteractive +apt-get update -q +apt-get install -y -q postgresql redis-server rclone + +# --- Postgres: the Debian package listens on localhost only. ----------------- +runuser -u postgres -- psql -q -v ON_ERROR_STOP=1 -v pw="$POSTGRES_PASSWORD" <<'SQL' +SELECT 'CREATE ROLE subplz LOGIN' WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'subplz')\gexec +ALTER ROLE subplz PASSWORD :'pw'; +SELECT 'CREATE DATABASE subplz OWNER subplz' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'subplz')\gexec +SQL + +# --- Redis: localhost only, a password, and a journal (the queue survives a restart). +conf=/etc/redis/redis.conf +set_conf() { + if grep -q "^$1 " "$conf"; then + sed -i "s|^$1 .*|$1 $2|" "$conf" + else + echo "$1 $2" >> "$conf" + fi +} +set_conf bind "127.0.0.1 -::1" +set_conf requirepass "$REDIS_PASSWORD" +set_conf appendonly yes +systemctl restart redis-server + +# --- The tunnel user. --------------------------------------------------------- +if ! id subplz-tunnel >/dev/null 2>&1; then + useradd --system --create-home --shell /usr/sbin/nologin subplz-tunnel +fi +home=$(getent passwd subplz-tunnel | cut -d: -f6) +install -d -m 700 -o subplz-tunnel -g subplz-tunnel "$home/.ssh" +printf 'restrict,port-forwarding,permitopen="127.0.0.1:5432",permitopen="127.0.0.1:6379" %s\n' \ + "$TUNNEL_PUBLIC_KEY" > "$home/.ssh/authorized_keys" +chown subplz-tunnel:subplz-tunnel "$home/.ssh/authorized_keys" +chmod 600 "$home/.ssh/authorized_keys" + +# --- Nightly backup of Postgres to the bucket. The runbook writes +# /etc/subplz-rclone.env and /etc/subplz-backup.env, then enables the timer. +cat > /etc/systemd/system/subplz-pgdump.service <<'UNIT' +[Unit] +Description=Back up the subplz Postgres database to the bucket + +[Service] +Type=oneshot +EnvironmentFile=/etc/subplz-rclone.env +EnvironmentFile=/etc/subplz-backup.env +ExecStart=/bin/bash -c 'set -o pipefail; runuser -u postgres -- pg_dump -Fc subplz | rclone rcat "r2:$BUCKET/backups/subplz-$(date -u +%%Y%%m%%d).dump"' +UNIT +cat > /etc/systemd/system/subplz-pgdump.timer <<'UNIT' +[Unit] +Description=Back up the subplz Postgres database each night + +[Timer] +OnCalendar=*-*-* 04:30:00 +Persistent=true + +[Install] +WantedBy=timers.target +UNIT +systemctl daemon-reload +echo "web server ready: Postgres, Redis, tunnel user" diff --git a/infra/burst/state/tests/host_key.pub b/infra/burst/state/tests/host_key.pub new file mode 100644 index 0000000..e1482e4 --- /dev/null +++ b/infra/burst/state/tests/host_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeHostKeyForTestsOnly0000000000000000000 root@web diff --git a/infra/burst/state/tests/state.tftest.hcl b/infra/burst/state/tests/state.tftest.hcl new file mode 100644 index 0000000..263352a --- /dev/null +++ b/infra/burst/state/tests/state.tftest.hcl @@ -0,0 +1,109 @@ +# terraform test with mock providers: no account, no cost. Only plans run: +# an apply would run setup-web-server.sh on this machine. + +mock_provider "cloudflare" {} + +mock_provider "random" {} +mock_provider "tls" {} + +# Values that exist only after apply, known during plan for the tests. +override_resource { + target = random_password.postgres + override_during = plan + values = { result = "pgpw" } +} + +override_resource { + target = random_password.redis + override_during = plan + values = { result = "redispw" } +} + +override_resource { + target = cloudflare_api_token.files + override_during = plan + values = { id = "token-id", value = "token-value" } +} + +variables { + cloudflare_account_id = "0123456789abcdef0123456789abcdef" + host_key_file = "tests/host_key.pub" + # The test framework cannot mock the nested result of the permission lookup. + r2_permission_group_ids = ["read-group", "write-group"] +} + +run "the_token_can_touch_the_bucket_and_nothing_else" { + command = plan + + assert { + condition = jsondecode(cloudflare_api_token.files.policies[0].resources) == { + "com.cloudflare.edge.r2.bucket.0123456789abcdef0123456789abcdef_default_subplz-files" = "*" + } + error_message = "The app token must be scoped to the one bucket." + } + assert { + condition = [for g in cloudflare_api_token.files.policies[0].permission_groups : g.id] == ["read-group", "write-group"] + error_message = "The app token must read and write objects, with no other permission." + } + assert { + condition = cloudflare_r2_bucket.files.location == "wnam" + error_message = "The bucket must be near the web server (western North America)." + } +} + +run "workers_accept_only_the_web_server_host_key" { + command = plan + + assert { + condition = startswith(local.known_hosts, "198.44.53.27 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeHostKey") + error_message = "known_hosts must pin the web server's host key." + } +} + +run "a_port_other_than_22_is_written_the_ssh_way" { + command = plan + + variables { + ssh_port = 2222 + } + + assert { + condition = startswith(local.known_hosts, "[198.44.53.27]:2222 ssh-ed25519 ") + error_message = "known_hosts needs [host]:port for a port other than 22." + } +} + +run "the_databases_are_reached_on_127_0_0_1" { + command = plan + + assert { + condition = startswith(local.database_url, "postgresql+psycopg://subplz:") && endswith(local.database_url, "@127.0.0.1:5432/subplz") + error_message = "Postgres must be reached through the tunnel on 127.0.0.1." + } + assert { + condition = startswith(local.redis_url, "redis://:") && endswith(local.redis_url, "@127.0.0.1:6379/0") + error_message = "Redis must be reached through the tunnel on 127.0.0.1." + } + assert { + condition = local.database_url == "postgresql+psycopg://subplz:pgpw@127.0.0.1:5432/subplz" && local.redis_url == "redis://:redispw@127.0.0.1:6379/0" + error_message = "The URLs must carry the generated passwords." + } + assert { + condition = strcontains(local.storage_env, "AWS_ACCESS_KEY_ID=token-id\n") && strcontains(local.storage_env, "AWS_SECRET_ACCESS_KEY=${sha256("token-value")}\n") + error_message = "R2 takes the token id as the key id, and the SHA-256 of the token value as the secret." + } + assert { + condition = strcontains(local.storage_env, "AWS_ENDPOINT_URL_S3=https://0123456789abcdef0123456789abcdef.r2.cloudflarestorage.com\n") + error_message = "The storage endpoint must be the account's R2 endpoint." + } +} + +run "a_bad_account_id_is_refused" { + command = plan + + variables { + cloudflare_account_id = "not-an-id" + } + + expect_failures = [var.cloudflare_account_id] +} diff --git a/infra/burst/state/variables.tf b/infra/burst/state/variables.tf new file mode 100644 index 0000000..6d52825 --- /dev/null +++ b/infra/burst/state/variables.tf @@ -0,0 +1,48 @@ +variable "cloudflare_account_id" { + description = "The Cloudflare account that holds the bucket (R2, right-hand column)." + type = string + + validation { + condition = can(regex("^[0-9a-f]{32}$", var.cloudflare_account_id)) + error_message = "cloudflare_account_id must be the 32-character account ID." + } +} + +variable "r2_permission_group_ids" { + description = "Ids of the R2 permission groups that read and write objects. Null: look them up by name." + type = list(string) + default = null +} + +variable "bucket_name" { + type = string + default = "subplz-files" +} + +variable "bucket_location" { + description = "R2 location hint. wnam: western North America, near the web server." + type = string + default = "wnam" +} + +variable "web_server_ip" { + description = "Public IPv4 address of the web server. Burst workers open their SSH tunnel to it." + type = string + default = "198.44.53.27" + + validation { + condition = can(cidrhost("${var.web_server_ip}/32", 0)) + error_message = "web_server_ip must be one IPv4 address." + } +} + +variable "ssh_port" { + type = number + default = 22 +} + +variable "host_key_file" { + description = "The web server's SSH host key. Workers accept this key only." + type = string + default = "/etc/ssh/ssh_host_ed25519_key.pub" +} diff --git a/infra/burst/systemd/subplz-burst.service b/infra/burst/systemd/subplz-burst.service new file mode 100644 index 0000000..461c6b2 --- /dev/null +++ b/infra/burst/systemd/subplz-burst.service @@ -0,0 +1,15 @@ +[Unit] +Description=subplz-web burst controller: start burst workers on the cheapest cloud +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +WorkingDirectory=/opt/subplz-web +# The app settings (SUBPLZ_WEB_REDIS_URL, SUBPLZ_WEB_S3_BUCKET, the database), +# the storage keys, and the cloud keys for Terraform (see infra/burst/README.md). +EnvironmentFile=/opt/subplz-web/.env +EnvironmentFile=/etc/subplz-storage.env +EnvironmentFile=/etc/subplz-burst.env +ExecStart=/opt/subplz-web/.venv/bin/python -m tools.burst diff --git a/infra/burst/systemd/subplz-burst.timer b/infra/burst/systemd/subplz-burst.timer new file mode 100644 index 0000000..8984846 --- /dev/null +++ b/infra/burst/systemd/subplz-burst.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Run the subplz-web burst controller once a minute + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min + +[Install] +WantedBy=timers.target diff --git a/infra/burst/systemd/subplz-worker.service b/infra/burst/systemd/subplz-worker.service new file mode 100644 index 0000000..a71cc09 --- /dev/null +++ b/infra/burst/systemd/subplz-worker.service @@ -0,0 +1,21 @@ +[Unit] +Description=subplz-web worker on the web server +After=network-online.target subplz-web.service +Wants=network-online.target + +[Service] +User=subplz +WorkingDirectory=/opt/subplz-web +EnvironmentFile=/opt/subplz-web/.env +EnvironmentFile=/etc/subplz-storage.env +Environment=PATH=/opt/subplz-web/.venv/bin:/usr/local/bin:/usr/bin:/bin +Environment=HOME=/opt/subplz-web +ExecStart=/opt/subplz-web/.venv/bin/python worker.py +# Postgres and Redis run on this machine too: if memory runs out, the kernel +# must stop this worker, not the database. +MemoryMax=2800M +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/infra/burst/worker-constraints.txt b/infra/burst/worker-constraints.txt new file mode 100644 index 0000000..20e081f --- /dev/null +++ b/infra/burst/worker-constraints.txt @@ -0,0 +1,89 @@ +# Exact versions of the web server's venv (pip freeze on 2026-09-22), so that +# a burst worker computes what the web server computes. Also the queue, storage +# and Postgres clients, which the web server installs when it turns burst on. +# pip, setuptools and wheel are left out: pip manages them itself. subplz comes +# from git and is pinned in variables.tf (subplz_spec): a constraint cannot be a URL. +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +av==18.1.0 +beautifulsoup4==4.13.5 +biopython==1.88 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +ctranslate2==4.6.0 +EbookLib==0.20 +emoji==2.16.0 +fastapi==0.141.1 +faster-whisper==1.2.0 +ffmpeg-python==0.2.0 +filelock==3.32.3 +flatbuffers==25.12.19 +fsspec==2026.7.0 +future==1.0.0 +greenlet==3.5.6 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httptools==0.8.0 +httpx==0.28.1 +huggingface_hub==1.32.0 +idna==3.20 +Jinja2==3.1.6 +lingua-language-detector==2.1.1 +llvmlite==0.49.0 +loguru==0.7.3 +lxml==6.1.3 +MarkupSafe==3.0.3 +mobi==0.4.1 +more-itertools==11.1.0 +mpmath==1.3.0 +natsort==8.4.0 +networkx==3.6.1 +numba==0.67.0 +numpy==1.26.4 +onnxruntime==1.30.0 +openai-whisper==20250625 +packaging==26.3 +protobuf==7.36.2 +pycountry==24.6.1 +pydantic==2.13.5 +pydantic-settings==2.15.0 +pydantic_core==2.46.5 +pysbd==0.3.4 +python-dotenv==1.2.3 +python-multipart==0.0.32 +PyYAML==6.0.3 +RapidFuzz==3.12.2 +regex==2024.11.6 +requests==2.34.2 +six==1.17.0 +soupsieve==2.9.2 +SQLAlchemy==2.0.54 +stable-ts==2.19.1 +standard-imghdr==3.13.0 +stanza==1.10.1 +starlette==1.6.0 +stripe==15.6.1 +sympy==1.14.0 +tiktoken==0.14.0 +tokenizers==0.23.2 +torch==2.14.0+cpu +torchaudio==2.11.0+cpu +tqdm==4.67.3 +triton==3.8.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.53.0 +uvloop==0.22.1 +watchdog==6.0.0 +watchfiles==1.2.0 +wcwidth==0.8.4 +websockets==17.1 +redis==8.1.0 +rq==2.12.0 +boto3==1.43.100 +psycopg==3.3.6 +psycopg-binary==3.3.6 diff --git a/infra/burst/workers/aws/.terraform.lock.hcl b/infra/burst/workers/aws/.terraform.lock.hcl new file mode 100644 index 0000000..5764421 --- /dev/null +++ b/infra/burst/workers/aws/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.66.0" + constraints = "~> 6.66" + hashes = [ + "h1:xehZnyesOrJ1/R9tmnRSu7FRkwoDDKelEHI3WdnJ72g=", + "zh:156fe7164a3d26ef6b35734c43e99fb198df90575ed897d1182b8e930b8cd523", + "zh:1af52b22b35be00f8d16e3ebebff9fa699ec4db2ef69e6032ba5c536f80c03d9", + "zh:2545a8478bd551fdc9694f6cc1a1ad24617f6736f8bde0ad6cae90987c65380f", + "zh:4070db1ee369ccb41cb610bfd887386bc0a9b9ecad60aeb4dbce58443d2519dd", + "zh:53da7d3c1840ef875c7d34e967732502a64fe677af0e78824773d4c15a8fe740", + "zh:576a93a28bf611a4de2a2e6ced697a41d5126b8fd31d30782b16797e410a9706", + "zh:58fed5fa9a033355b9d4f3092c817b70d934100e0d8678d6e4c93f3c9493d4e4", + "zh:6a9ca2f24e2ee9156dd785d159a850b35d190e9cf7eca21cb9582970c2db80cd", + "zh:729edd30f99cc16009deba5c013265b0c81eda261a3d0821cbd011d3287fd230", + "zh:7ae460049b75bd4aefee465ef7c53a01ac2df46d4d3e3ac00824afa8b5cb83fb", + "zh:9051fa85c8034ade8a57a5c6f232fd33da28f3800bb5aa40bc8625dbc5e27632", + "zh:906547e4319805e7acf7fbdf2bac28a4b1a7370790a2a430c7adb1b29bb934eb", + "zh:998f27410a66158a35ee5ed142c27e5b21fe8601941da55da2157f8042d6dcca", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c1804eff1dda0446dc2d215231015bb65a2fc6c3b7ba24584fe45f1ddd3fa9f", + "zh:b03ff5efdee310502aaaeb460144dc059bce72a0d8217e6b989099ef8aef9283", + ] +} diff --git a/infra/burst/workers/aws/iam-policy.json b/infra/burst/workers/aws/iam-policy.json new file mode 100644 index 0000000..99bf100 --- /dev/null +++ b/infra/burst/workers/aws/iam-policy.json @@ -0,0 +1,60 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadPricesImagesNetworkAndInstances", + "Effect": "Allow", + "Action": [ + "ec2:Describe*" + ], + "Resource": "*" + }, + { + "Sid": "StartWorkers", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:CreateTags" + ], + "Resource": "*" + }, + { + "Sid": "EndOnlyBurstWorkers", + "Effect": "Allow", + "Action": [ + "ec2:TerminateInstances", + "ec2:ModifyInstanceAttribute" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/Role": "subplz-burst-worker" + } + } + }, + { + "Sid": "TheWorkerSecurityGroup", + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:DeleteSecurityGroup", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:RevokeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": "*" + }, + { + "Sid": "TheSpotRoleOnce", + "Effect": "Allow", + "Action": "iam:CreateServiceLinkedRole", + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "spot.amazonaws.com" + } + } + } + ] +} diff --git a/infra/burst/workers/aws/main.tf b/infra/burst/workers/aws/main.tf new file mode 100644 index 0000000..a55691e --- /dev/null +++ b/infra/burst/workers/aws/main.tf @@ -0,0 +1,128 @@ +# Burst workers on AWS: one spot instance for each entry of var.workers. The +# burst controller (python -m tools.burst) writes var.workers and applies this +# when the set changes; do not apply it by hand while burst mode is on. + +terraform { + required_version = ">= 1.9" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.66" + } + } + backend "local" {} +} + +# Reads AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION. The controller +# sets them from SUBPLZ_BURST_AWS_* (see tools/burst/__main__.py). +provider "aws" {} + +data "terraform_remote_state" "state" { + backend = "local" + config = { + path = var.state_file + } +} + +locals { + state = data.terraform_remote_state.state.outputs.worker + # Graviton families (c7g, m6g ...) are arm64; the others are x86-64. + arch = { for name, w in var.workers : name => can(regex("^[a-z]+[0-9]+g[a-z]*\\.", w.machine)) ? "arm64" : "amd64" } +} + +data "aws_ami" "debian" { + for_each = toset(["amd64", "arm64"]) + most_recent = true + owners = ["136693071363"] # Debian + filter { + name = "name" + values = ["debian-12-${each.key}-*"] + } +} + +data "aws_vpc" "default" { + default = true +} + +data "aws_subnet" "zone" { + for_each = toset([for w in var.workers : w.zone]) + vpc_id = data.aws_vpc.default.id + availability_zone = each.key + default_for_az = true +} + +# No inbound port: a worker only connects out (the SSH tunnel, the bucket, the +# package indexes). +resource "aws_security_group" "worker" { + name = "subplz-burst-worker" + description = "subplz burst workers: outbound only" + vpc_id = data.aws_vpc.default.id + + # An empty list, not a missing one: Terraform then removes any inbound rule. + ingress = [] + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "worker" { + for_each = var.workers + + ami = data.aws_ami.debian[local.arch[each.key]].id + instance_type = each.value.machine + subnet_id = data.aws_subnet.zone[each.value.zone].id + vpc_security_group_ids = [aws_security_group.worker.id] + associate_public_ip_address = true + instance_initiated_shutdown_behavior = "terminate" # the idle check powers off: the bill ends + + instance_market_options { + market_type = "spot" + spot_options { + max_price = var.max_price + spot_instance_type = "one-time" + instance_interruption_behavior = "terminate" + } + } + + root_block_device { + volume_size = 20 + volume_type = "gp3" + } + + metadata_options { + http_tokens = "required" + } + + user_data = templatefile("${path.module}/../../cloud-init.yaml.tftpl", { + name = each.key + ref = each.value.ref + subplz_spec = var.subplz_spec + threads = var.threads + idle_checks = var.idle_checks + constraints = file("${path.module}/../../worker-constraints.txt") + web_server_ip = local.state.web_server_ip + ssh_port = local.state.ssh_port + known_hosts = local.state.known_hosts + tunnel_private_key = local.state.tunnel_private_key + database_url = local.state.database_url + redis_url = local.state.redis_url + bucket = local.state.bucket + s3_prefix = local.state.s3_prefix + storage_env = local.state.storage_env + }) + + tags = { + Name = each.key + Role = "subplz-burst-worker" + } + + # A change of these would replace a machine, and kill the job on it. New + # workers get the new values; the controller replaces old ones when idle. + lifecycle { + ignore_changes = [user_data, ami] + } +} diff --git a/infra/burst/workers/aws/outputs.tf b/infra/burst/workers/aws/outputs.tf new file mode 100644 index 0000000..a921d75 --- /dev/null +++ b/infra/burst/workers/aws/outputs.tf @@ -0,0 +1,4 @@ +output "workers" { + description = "Each worker on AWS and its public IPv4 address." + value = { for name, i in aws_instance.worker : name => i.public_ip } +} diff --git a/infra/burst/workers/aws/tests/aws.tftest.hcl b/infra/burst/workers/aws/tests/aws.tftest.hcl new file mode 100644 index 0000000..dc7c039 --- /dev/null +++ b/infra/burst/workers/aws/tests/aws.tftest.hcl @@ -0,0 +1,109 @@ +# terraform test with a mock provider: no account and no cost. + +mock_provider "aws" {} + +override_data { + target = data.terraform_remote_state.state + values = { + outputs = { + worker = { + web_server_ip = "198.44.53.27" + ssh_port = 22 + known_hosts = "198.44.53.27 ssh-ed25519 AAAAhostkey" + tunnel_private_key = "-----BEGIN OPENSSH PRIVATE KEY-----\nTUNNELKEY\n-----END OPENSSH PRIVATE KEY-----\n" + database_url = "postgresql+psycopg://subplz:pgpw@127.0.0.1:5432/subplz" + redis_url = "redis://:redispw@127.0.0.1:6379/0" + bucket = "subplz-files" + s3_prefix = "jobs/" + storage_env = "AWS_ACCESS_KEY_ID=kid\nAWS_SECRET_ACCESS_KEY=secret\nAWS_ENDPOINT_URL_S3=https://acct.r2.cloudflarestorage.com\nAWS_DEFAULT_REGION=auto\n" + } + } + } +} + +variables { + workers = { + "subplz-burst-1790000000-0" = { ref = "0123456789abcdef0123456789abcdef01234567", machine = "c6a.xlarge", zone = "us-west-2a" } + "subplz-burst-1790000000-1" = { ref = "89abcdef0123456789abcdef0123456789abcdef", machine = "c7g.xlarge", zone = "us-west-2b" } + } +} + +run "one_spot_instance_for_each_worker" { + command = plan + + assert { + condition = length(aws_instance.worker) == 2 + error_message = "Expected one instance for each worker." + } + assert { + condition = alltrue([for n, i in aws_instance.worker : + i.instance_type == var.workers[n].machine + && i.instance_market_options[0].market_type == "spot" + && i.instance_market_options[0].spot_options[0].max_price == "0.0625" + && i.instance_initiated_shutdown_behavior == "terminate" + ]) + error_message = "Each worker must be a spot instance of its type, with a price limit, that ends when it powers off." + } + assert { + condition = local.arch["subplz-burst-1790000000-0"] == "amd64" && local.arch["subplz-burst-1790000000-1"] == "arm64" + error_message = "A Graviton type must get the arm64 image, the others the amd64 image." + } + assert { + condition = length(aws_security_group.worker.ingress) == 0 + error_message = "A worker must take no inbound connection." + } +} + +run "the_cloud_config_names_the_host_and_opens_the_tunnel" { + command = plan + + assert { + condition = alltrue([for n, i in aws_instance.worker : yamldecode(i.user_data).hostname == n]) + error_message = "The host name must be the machine name: the controller matches rq workers by host name." + } + assert { + condition = alltrue([for n, i in aws_instance.worker : + strcontains(i.user_data, "releases/${var.workers[n].ref}.tar.gz") + && strcontains(i.user_data, "SUBPLZ_WEB_WORKER_QUEUES=paid\n") + && strcontains(i.user_data, "-L 127.0.0.1:5432:127.0.0.1:5432 -L 127.0.0.1:6379:127.0.0.1:6379 subplz-tunnel@198.44.53.27") + && strcontains(i.user_data, "AWS_ENDPOINT_URL_S3=https://acct.r2.cloudflarestorage.com\n") + ]) + error_message = "Each worker must run its own commit, take paid jobs, reach the databases through the tunnel, and use the bucket." + } + assert { + condition = alltrue([for i in aws_instance.worker : + contains([for f in yamldecode(i.user_data).write_files : f.content], + "-----BEGIN OPENSSH PRIVATE KEY-----\nTUNNELKEY\n-----END OPENSSH PRIVATE KEY-----\n") + ]) + error_message = "The tunnel key must be written whole." + } + assert { + condition = alltrue([for i in aws_instance.worker : + strcontains(one([for f in yamldecode(i.user_data).write_files : f.content if f.path == "/opt/subplz-install/idle-check.sh"]), + "-ge 9 ]")]) + error_message = "The idle check must power off after var.idle_checks idle checks." + } +} + +run "no_workers_means_no_instances" { + command = plan + + variables { + workers = {} + } + + assert { + condition = length(aws_instance.worker) == 0 + error_message = "An empty set must make no instances." + } +} + +run "a_name_without_the_prefix_is_refused" { + command = plan + + variables { + workers = { "web-1" = { ref = "0123456789abcdef0123456789abcdef01234567", machine = "c6a.xlarge", zone = "us-west-2a" } } + } + + expect_failures = [var.workers] +} diff --git a/infra/burst/workers/aws/variables.tf b/infra/burst/workers/aws/variables.tf new file mode 100644 index 0000000..03eb66d --- /dev/null +++ b/infra/burst/workers/aws/variables.tf @@ -0,0 +1,48 @@ +variable "workers" { + description = "The workers on AWS: machine name -> app commit, instance type, availability zone. The burst controller writes this." + type = map(object({ + ref = string + machine = string + zone = string + })) + default = {} + + validation { + condition = alltrue([for name in keys(var.workers) : can(regex("^subplz-burst-[0-9a-z-]{1,40}$", name))]) + error_message = "Each worker name must start with subplz-burst-: the controller finds its workers by that prefix." + } + validation { + condition = alltrue([for w in values(var.workers) : can(regex("^[0-9a-f]{40}$", w.ref))]) + error_message = "Each ref must be a full git commit hash." + } +} + +variable "max_price" { + description = "Highest spot price per hour, in USD. Above it AWS does not start the machine." + type = string + default = "0.0625" +} + +variable "threads" { + description = "Whisper threads: all vCPUs of an xlarge instance." + type = number + default = 4 +} + +variable "idle_checks" { + description = "Idle checks (10 minutes each) before a machine powers itself off, if the controller is gone." + type = number + default = 9 +} + +variable "subplz_spec" { + description = "The subplz package, pinned to the commit that the web server runs." + type = string + default = "subplz @ git+https://github.com/kanjieater/SubPlz.git@6be8b6fab185372b458bdde13c61f4da204dc6fc" +} + +variable "state_file" { + description = "Path of the state file of ../../state." + type = string + default = "/var/lib/subplz-burst/state.tfstate" +} diff --git a/infra/burst/workers/hetzner/.terraform.lock.hcl b/infra/burst/workers/hetzner/.terraform.lock.hcl new file mode 100644 index 0000000..eecbb7a --- /dev/null +++ b/infra/burst/workers/hetzner/.terraform.lock.hcl @@ -0,0 +1,23 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hetznercloud/hcloud" { + version = "1.69.0" + constraints = "~> 1.69" + hashes = [ + "h1:MLxpeLlRL97wDLqL+WNpb7L9Vk+g8aUHLjXspFPR+tg=", + "zh:023c0580c46b48a1dce8aeb5832ec9896ce53ee019a2b296d854873f92aad7d9", + "zh:02fa14eebb1c48b52b9acb33fb987425513537b70a50dac5639aecfe62c4b66f", + "zh:0f534ee078e5f0f3c21e99cc020c3a9be565fceed4936d88b978f7b7d4b981fc", + "zh:42dcf6859d097179c30cbe346931be731548a51ef7c73d08cd4af69a044bd60f", + "zh:6d319ee6c9e64437634f3e682196dab24025ea91a7332b40972279e9c6e67f64", + "zh:71a1dc59a47ab1b4127696e8047b54bb4bc8f92b8994ecc13509eaeb7e13a729", + "zh:781310b68c74dede2e225096d7ada9a0b4b9252b6cafa4dc1ff064dd9ea4c817", + "zh:7c4e0411e6e10ca6bd49c8e699c7c61cd9750d32d46ee5472d01faa30fbd222f", + "zh:c4c70587a7632aea65d6b1bb0bbe8da2145a7acb6d16a7dd0e0dbbde6195c8ca", + "zh:caaab2e9bb7cf8bb478b143bc0156c40729d720d3e2c77cf021e515fbeca3a6a", + "zh:cbd1c2040111b595cfa5daa201166560b4988eb0227dd6be17e8ae72a5ff1df0", + "zh:eba595371b98a66c1bf64dfb9f9256e6dc365e990e7202c50b6977fb3d24f822", + "zh:fd477dca1d5b0bc7f41cce6b9f3e3250b58597cc178a70fa3136717343ffeede", + ] +} diff --git a/infra/burst/workers/hetzner/main.tf b/infra/burst/workers/hetzner/main.tf new file mode 100644 index 0000000..2d4afa2 --- /dev/null +++ b/infra/burst/workers/hetzner/main.tf @@ -0,0 +1,71 @@ +# Burst workers on Hetzner Cloud: one server for each entry of var.workers. The +# burst controller (python -m tools.burst) writes var.workers and applies this +# when the set changes; do not apply it by hand while burst mode is on. + +terraform { + required_version = ">= 1.9" + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.69" + } + } + backend "local" {} +} + +# Reads HCLOUD_TOKEN. Keep the burst servers in a Hetzner project of their own. +provider "hcloud" {} + +data "terraform_remote_state" "state" { + backend = "local" + config = { + path = var.state_file + } +} + +locals { + state = data.terraform_remote_state.state.outputs.worker +} + +# No inbound rule: Hetzner then drops all inbound traffic. A worker only +# connects out (the SSH tunnel, the bucket, the package indexes). +resource "hcloud_firewall" "worker" { + name = "subplz-burst-worker" +} + +resource "hcloud_server" "worker" { + for_each = var.workers + + name = each.key + server_type = each.value.machine + location = each.value.zone + image = "debian-12" + firewall_ids = [hcloud_firewall.worker.id] + labels = { + role = "subplz-burst-worker" + } + + user_data = templatefile("${path.module}/../../cloud-init.yaml.tftpl", { + name = each.key + ref = each.value.ref + subplz_spec = var.subplz_spec + threads = var.threads + idle_checks = var.idle_checks + constraints = file("${path.module}/../../worker-constraints.txt") + web_server_ip = local.state.web_server_ip + ssh_port = local.state.ssh_port + known_hosts = local.state.known_hosts + tunnel_private_key = local.state.tunnel_private_key + database_url = local.state.database_url + redis_url = local.state.redis_url + bucket = local.state.bucket + s3_prefix = local.state.s3_prefix + storage_env = local.state.storage_env + }) + + # A change of these would replace a server, and kill the job on it. New + # workers get the new values; the controller replaces old ones when idle. + lifecycle { + ignore_changes = [user_data, image] + } +} diff --git a/infra/burst/workers/hetzner/outputs.tf b/infra/burst/workers/hetzner/outputs.tf new file mode 100644 index 0000000..203373b --- /dev/null +++ b/infra/burst/workers/hetzner/outputs.tf @@ -0,0 +1,4 @@ +output "workers" { + description = "Each worker on Hetzner and its public IPv4 address." + value = { for name, s in hcloud_server.worker : name => s.ipv4_address } +} diff --git a/infra/burst/workers/hetzner/tests/hetzner.tftest.hcl b/infra/burst/workers/hetzner/tests/hetzner.tftest.hcl new file mode 100644 index 0000000..660427f --- /dev/null +++ b/infra/burst/workers/hetzner/tests/hetzner.tftest.hcl @@ -0,0 +1,66 @@ +# terraform test with a mock provider: no account and no cost. + +mock_provider "hcloud" {} + +override_data { + target = data.terraform_remote_state.state + values = { + outputs = { + worker = { + web_server_ip = "198.44.53.27" + ssh_port = 22 + known_hosts = "198.44.53.27 ssh-ed25519 AAAAhostkey" + tunnel_private_key = "-----BEGIN OPENSSH PRIVATE KEY-----\nTUNNELKEY\n-----END OPENSSH PRIVATE KEY-----\n" + database_url = "postgresql+psycopg://subplz:pgpw@127.0.0.1:5432/subplz" + redis_url = "redis://:redispw@127.0.0.1:6379/0" + bucket = "subplz-files" + s3_prefix = "jobs/" + storage_env = "AWS_ACCESS_KEY_ID=kid\nAWS_SECRET_ACCESS_KEY=secret\nAWS_ENDPOINT_URL_S3=https://acct.r2.cloudflarestorage.com\nAWS_DEFAULT_REGION=auto\n" + } + } + } +} + +variables { + workers = { + "subplz-burst-1790000000-0" = { ref = "0123456789abcdef0123456789abcdef01234567", machine = "cx33", zone = "fsn1" } + } +} + +run "one_server_for_each_worker_behind_a_closed_firewall" { + command = plan + + assert { + condition = alltrue([for n, s in hcloud_server.worker : + s.server_type == "cx33" && s.location == "fsn1" && s.image == "debian-12" + ]) + error_message = "Each worker must be a Debian 12 server of its type, in its location." + } + assert { + condition = length(hcloud_firewall.worker.rule) == 0 + error_message = "The firewall must have no inbound rule, so that it drops all inbound traffic." + } +} + +run "the_cloud_config_names_the_host_and_opens_the_tunnel" { + command = plan + + assert { + condition = alltrue([for n, s in hcloud_server.worker : + yamldecode(s.user_data).hostname == n + && strcontains(s.user_data, "subplz-tunnel@198.44.53.27") + && strcontains(s.user_data, "SUBPLZ_WEB_WORKER_QUEUES=paid\n") + ]) + error_message = "The server must name itself, open the tunnel, and take paid jobs." + } +} + +run "a_short_ref_is_refused" { + command = plan + + variables { + workers = { "subplz-burst-1-0" = { ref = "0123abc", machine = "cx33", zone = "fsn1" } } + } + + expect_failures = [var.workers] +} diff --git a/infra/burst/workers/hetzner/variables.tf b/infra/burst/workers/hetzner/variables.tf new file mode 100644 index 0000000..bbb9b3b --- /dev/null +++ b/infra/burst/workers/hetzner/variables.tf @@ -0,0 +1,43 @@ +variable "workers" { + description = "The workers on Hetzner: server name -> app commit, server type, location. The burst controller writes this." + type = map(object({ + ref = string + machine = string + zone = string + })) + default = {} + + validation { + condition = alltrue([for name in keys(var.workers) : can(regex("^subplz-burst-[0-9a-z-]{1,40}$", name))]) + error_message = "Each worker name must start with subplz-burst-: the controller finds its workers by that prefix." + } + validation { + condition = alltrue([for w in values(var.workers) : can(regex("^[0-9a-f]{40}$", w.ref))]) + error_message = "Each ref must be a full git commit hash." + } +} + + +variable "threads" { + description = "Whisper threads: all vCPUs of a 4-vCPU server type." + type = number + default = 4 +} + +variable "idle_checks" { + description = "Idle checks (10 minutes each) before a machine powers itself off, if the controller is gone." + type = number + default = 9 +} + +variable "subplz_spec" { + description = "The subplz package, pinned to the commit that the web server runs." + type = string + default = "subplz @ git+https://github.com/kanjieater/SubPlz.git@6be8b6fab185372b458bdde13c61f4da204dc6fc" +} + +variable "state_file" { + description = "Path of the state file of ../../state." + type = string + default = "/var/lib/subplz-burst/state.tfstate" +} diff --git a/tools/set-secret.sh b/tools/set-secret.sh index eea3589..4043128 100755 --- a/tools/set-secret.sh +++ b/tools/set-secret.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash -# Put one secret into .env without it touching shell history, the process list -# or the screen. +# Put one secret into .env (or into FILE) without it touching shell history, +# the process list or the screen. # # tools/set-secret.sh SUBPLZ_WEB_STRIPE_SECRET_KEY +# tools/set-secret.sh DIGITALOCEAN_TOKEN /etc/subplz-burst.env set -euo pipefail -key="${1:?usage: set-secret.sh VARIABLE_NAME}" +key="${1:?usage: set-secret.sh VARIABLE_NAME [FILE]}" root="$(cd "$(dirname "$0")/.." && pwd)" -env_file="$root/.env" +env_file="${2:-$root/.env}" read -r -s -p "Value for $key (input hidden): " value echo @@ -20,9 +21,11 @@ printf '%s=%s\n' "$key" "$value" >> "$tmp" cat "$tmp" > "$env_file" rm -f "$tmp" -# The service does not run as root; keep the file readable by whoever owns the -# checkout, and by nobody else. -chown "$(stat -c %U:%G "$root")" "$env_file" 2>/dev/null || true +# The service does not run as root; keep .env readable by whoever owns the +# checkout, and by nobody else. Another FILE stays with root. +if [ -z "${2:-}" ]; then + chown "$(stat -c %U:%G "$root")" "$env_file" 2>/dev/null || true +fi chmod 600 "$env_file" -echo "$key saved. Apply it with: systemctl restart subplz-web" +echo "$key saved in $env_file. Restart the services that read it."