infra/burst/workers/aws/main.tf (3800 bytes)
1 # Burst workers on AWS: one spot instance for each entry of var.workers. The 2 # burst controller (python -m tools.burst) writes var.workers and applies this 3 # when the set changes; do not apply it by hand while burst mode is on. 4 5 terraform { 6 required_version = ">= 1.9" 7 required_providers { 8 aws = { 9 source = "hashicorp/aws" 10 version = "~> 6.66" 11 } 12 } 13 backend "local" {} 14 } 15 16 # Reads AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION. The controller 17 # sets them from SUBPLZ_BURST_AWS_* (see tools/burst/__main__.py). 18 provider "aws" {} 19 20 data "terraform_remote_state" "state" { 21 backend = "local" 22 config = { 23 path = var.state_file 24 } 25 } 26 27 locals { 28 state = data.terraform_remote_state.state.outputs.worker 29 # Graviton families (c7g, m6g ...) are arm64; the others are x86-64. 30 arch = { for name, w in var.workers : name => can(regex("^[a-z]+[0-9]+g[a-z]*\\.", w.machine)) ? "arm64" : "amd64" } 31 } 32 33 data "aws_ami" "debian" { 34 for_each = toset(["amd64", "arm64"]) 35 most_recent = true 36 owners = ["136693071363"] # Debian 37 filter { 38 name = "name" 39 values = ["debian-12-${each.key}-*"] 40 } 41 } 42 43 data "aws_vpc" "default" { 44 default = true 45 } 46 47 data "aws_subnet" "zone" { 48 for_each = toset([for w in var.workers : w.zone]) 49 vpc_id = data.aws_vpc.default.id 50 availability_zone = each.key 51 default_for_az = true 52 } 53 54 # No inbound port: a worker only connects out (the SSH tunnel, the bucket, the 55 # package indexes). 56 resource "aws_security_group" "worker" { 57 name = "subplz-burst-worker" 58 description = "subplz burst workers: outbound only" 59 vpc_id = data.aws_vpc.default.id 60 61 # An empty list, not a missing one: Terraform then removes any inbound rule. 62 ingress = [] 63 64 egress { 65 from_port = 0 66 to_port = 0 67 protocol = "-1" 68 cidr_blocks = ["0.0.0.0/0"] 69 } 70 } 71 72 resource "aws_instance" "worker" { 73 for_each = var.workers 74 75 ami = data.aws_ami.debian[local.arch[each.key]].id 76 instance_type = each.value.machine 77 subnet_id = data.aws_subnet.zone[each.value.zone].id 78 vpc_security_group_ids = [aws_security_group.worker.id] 79 associate_public_ip_address = true 80 instance_initiated_shutdown_behavior = "terminate" # the idle check powers off: the bill ends 81 82 instance_market_options { 83 market_type = "spot" 84 spot_options { 85 max_price = var.max_price 86 spot_instance_type = "one-time" 87 instance_interruption_behavior = "terminate" 88 } 89 } 90 91 root_block_device { 92 volume_size = 20 93 volume_type = "gp3" 94 } 95 96 metadata_options { 97 http_tokens = "required" 98 } 99 100 user_data = templatefile("${path.module}/../../cloud-init.yaml.tftpl", { 101 name = each.key 102 ref = each.value.ref 103 subplz_spec = var.subplz_spec 104 threads = var.threads 105 idle_checks = var.idle_checks 106 constraints = file("${path.module}/../../worker-constraints.txt") 107 web_server_ip = local.state.web_server_ip 108 ssh_port = local.state.ssh_port 109 known_hosts = local.state.known_hosts 110 tunnel_private_key = local.state.tunnel_private_key 111 database_url = local.state.database_url 112 redis_url = local.state.redis_url 113 bucket = local.state.bucket 114 s3_prefix = local.state.s3_prefix 115 storage_env = local.state.storage_env 116 }) 117 118 tags = { 119 Name = each.key 120 Role = "subplz-burst-worker" 121 } 122 123 # A change of these would replace a machine, and kill the job on it. New 124 # workers get the new values; the controller replaces old ones when idle. 125 lifecycle { 126 ignore_changes = [user_data, ami] 127 } 128 }