infra/burst/cloud-init.yaml.tftpl (6115 bytes)
1 #cloud-config 2 # One burst worker, on any cloud: the subplz-web app at commit ${ref}, running 3 # worker.py on the paid queue. Postgres and Redis stay on the web server; the 4 # worker reaches them through an SSH tunnel, on its own 127.0.0.1. The worker 5 # modules (workers/aws, workers/hetzner) render this file. 6 hostname: ${name} 7 preserve_hostname: false 8 package_update: true 9 packages: 10 - autossh 11 - curl 12 - ffmpeg 13 - git 14 - python3-venv 15 users: 16 - name: subplz 17 system: true 18 homedir: /opt/subplz-web 19 shell: /usr/sbin/nologin 20 write_files: 21 - path: /etc/subplz-worker.env 22 owner: root:root 23 permissions: "0600" 24 content: | 25 SUBPLZ_WEB_QUEUE_BACKEND=redis 26 SUBPLZ_WEB_WORKER_QUEUES=paid 27 SUBPLZ_WEB_REDIS_URL=${redis_url} 28 SUBPLZ_WEB_DATABASE_URL=${database_url} 29 SUBPLZ_WEB_STORAGE_BACKEND=s3 30 SUBPLZ_WEB_S3_BUCKET=${bucket} 31 SUBPLZ_WEB_S3_PREFIX=${s3_prefix} 32 SUBPLZ_WEB_DEVICE=cpu 33 SUBPLZ_WEB_THREADS=${threads} 34 HOME=/opt/subplz-web 35 ${indent(6, storage_env)} 36 - path: /etc/subplz/tunnel_key 37 permissions: "0600" 38 content: | 39 ${indent(6, tunnel_private_key)} 40 - path: /etc/subplz/known_hosts 41 permissions: "0644" 42 content: | 43 ${known_hosts} 44 - path: /etc/systemd/system/subplz-tunnel.service 45 content: | 46 [Unit] 47 Description=SSH tunnel to Postgres and Redis on the web server 48 After=network-online.target 49 Wants=network-online.target 50 51 [Service] 52 User=subplz 53 Environment=AUTOSSH_GATETIME=0 54 ExecStart=/usr/bin/autossh -M 0 -N -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o ExitOnForwardFailure=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/subplz/known_hosts -o IdentitiesOnly=yes -i /etc/subplz/tunnel_key -p ${ssh_port} -L 127.0.0.1:5432:127.0.0.1:5432 -L 127.0.0.1:6379:127.0.0.1:6379 subplz-tunnel@${web_server_ip} 55 Restart=always 56 RestartSec=5 57 58 [Install] 59 WantedBy=multi-user.target 60 - path: /etc/systemd/system/subplz-worker.service 61 content: | 62 [Unit] 63 Description=subplz-web burst worker 64 After=subplz-tunnel.service 65 Requires=subplz-tunnel.service 66 67 [Service] 68 User=subplz 69 WorkingDirectory=/opt/subplz-web 70 EnvironmentFile=/etc/subplz-worker.env 71 Environment=PATH=/opt/subplz-web/.venv/bin:/usr/local/bin:/usr/bin:/bin 72 ExecStart=/opt/subplz-web/.venv/bin/python worker.py 73 # After the shutdown command of the burst controller, rq finishes its job 74 # and exits with 0. Then the worker must stay stopped. 75 Restart=on-failure 76 RestartSec=10 77 78 [Install] 79 WantedBy=multi-user.target 80 - path: /etc/systemd/system/subplz-idle-check.service 81 content: | 82 [Unit] 83 Description=Power off this burst machine when its worker is idle too long 84 85 [Service] 86 Type=oneshot 87 EnvironmentFile=/etc/subplz-worker.env 88 ExecStart=/bin/bash /opt/subplz-install/idle-check.sh 89 - path: /etc/systemd/system/subplz-idle-check.timer 90 content: | 91 [Unit] 92 Description=Check each 10 minutes whether the worker is idle 93 94 [Timer] 95 OnActiveSec=10min 96 OnUnitActiveSec=10min 97 98 [Install] 99 WantedBy=timers.target 100 - path: /opt/subplz-install/idle-check.sh 101 permissions: "0700" 102 content: | 103 #!/bin/bash 104 # A safety net for the burst controller: if the controller is gone, this 105 # machine still stops. After ${idle_checks} idle checks in a row (10 minutes 106 # each), or when Redis cannot be reached, power off. On AWS a power-off 107 # ends the machine and its bill; on Hetzner the controller then deletes it. 108 state_file=/var/lib/subplz-idle-count 109 state=$(/opt/subplz-web/.venv/bin/python - <<'PY' 2>/dev/null || echo idle 110 import os, socket 111 from redis import Redis 112 from rq import Worker 113 conn = Redis.from_url(os.environ["SUBPLZ_WEB_REDIS_URL"], socket_timeout=10) 114 host = socket.gethostname() 115 states = [w.get_state() for w in Worker.all(connection=conn) if w.hostname == host] 116 print("busy" if "busy" in states else "idle") 117 PY 118 ) 119 if [ "$state" = "busy" ]; then 120 echo 0 > "$state_file" 121 exit 0 122 fi 123 count=$(( $(cat "$state_file" 2>/dev/null || echo 0) + 1 )) 124 echo "$count" > "$state_file" 125 if [ "$count" -ge ${idle_checks} ]; then 126 systemctl poweroff 127 fi 128 - path: /opt/subplz-install/constraints.txt 129 content: | 130 ${indent(6, constraints)} 131 - path: /opt/subplz-install/install.sh 132 permissions: "0700" 133 content: | 134 #!/bin/bash 135 # Install the app from the bucket (the repository is private), then the 136 # speech stack at the versions of the web server, then start the tunnel, 137 # the worker and the idle check. 138 set -euo pipefail 139 set -a 140 . /etc/subplz-worker.env 141 set +a 142 chown subplz /etc/subplz/tunnel_key 143 cd /opt/subplz-install 144 curl --fail --silent --show-error --retry 5 \ 145 --aws-sigv4 "aws:amz:$AWS_DEFAULT_REGION:s3" \ 146 --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" \ 147 -o app.tar.gz "$AWS_ENDPOINT_URL_S3/$SUBPLZ_WEB_S3_BUCKET/releases/${ref}.tar.gz" 148 mkdir -p /opt/subplz-web 149 tar -xzf app.tar.gz -C /opt/subplz-web 150 python3 -m venv /opt/subplz-web/.venv 151 pip=/opt/subplz-web/.venv/bin/pip 152 # CPU builds of torch and torchaudio both come from the PyTorch index. A 153 # torchaudio from PyPI is a CUDA build and fails at run time. Their 154 # dependencies come from PyPI in the next step, at the pinned versions. 155 $pip install --quiet --no-deps --index-url https://download.pytorch.org/whl/cpu \ 156 -c constraints.txt torch torchaudio 157 $pip install --quiet -c constraints.txt torch torchaudio "${subplz_spec}" \ 158 -r /opt/subplz-web/requirements.txt redis rq boto3 "psycopg[binary]" 159 chown -R subplz:subplz /opt/subplz-web 160 systemctl daemon-reload 161 systemctl enable --now subplz-tunnel.service subplz-worker.service subplz-idle-check.timer 162 runcmd: 163 - [bash, /opt/subplz-install/install.sh]