Provisioning DR Sandboxes with Pulumi

#!/usr/bin/env python3
"""Pulumi program: stand up an isolated, snapshot-backed restore-drill sandbox.

Run with `pulumi up`. The program resolves the newest eligible automated
snapshot at runtime with boto3, builds an isolated VPC/subnet plus a security
group that admits only the validation orchestrator CIDR, restores an Aurora
cluster from that snapshot, tags it for TTL garbage collection, and exports the
writer endpoint for the validation runner.

Snapshot resolution runs before any resource is declared, so a missing snapshot
raises and `pulumi up` aborts non-zero with no partial sandbox created.
"""
import boto3
from botocore.exceptions import BotoCoreError, ClientError

import pulumi
import pulumi_aws as aws

config = pulumi.Config()
source_cluster = config.require("sourceClusterId")
pitr_target_epoch = config.require_int("pitrTargetEpoch")
orchestrator_cidr = config.require("orchestratorCidr")
ttl_epoch = config.require_int("sandboxTtlEpoch")
region = config.get("primaryRegion") or "us-east-1"
drill_id = pulumi.get_stack()


def resolve_newest_snapshot(cluster_id: str, target_epoch: int, aws_region: str) -> dict:
    """Return the newest non-terminal automated snapshot at or before target_epoch."""
    client = boto3.client("rds", region_name=aws_region)
    paginator = client.get_paginator("describe_db_cluster_snapshots")
    eligible = []
    try:
        for page in paginator.paginate(DBClusterIdentifier=cluster_id, SnapshotType="automated"):
            for snap in page["DBClusterSnapshots"]:
                created = snap.get("SnapshotCreateTime")
                if created is None or snap["Status"] not in ("available", "creating", "copying"):
                    continue
                if created.timestamp() <= target_epoch:
                    eligible.append(snap)
    except (BotoCoreError, ClientError) as exc:
        raise pulumi.RunError(f"snapshot discovery failed for {cluster_id}: {exc}")
    if not eligible:
        raise pulumi.RunError(
            f"no automated snapshot for {cluster_id} at or before epoch {target_epoch}"
        )
    return max(eligible, key=lambda s: s["SnapshotCreateTime"])


snapshot = resolve_newest_snapshot(source_cluster, pitr_target_epoch, region)
snapshot_id = snapshot["DBClusterSnapshotIdentifier"]
kms_key_id = snapshot.get("KmsKeyId")
pulumi.log.info(f"resolved snapshot {snapshot_id} (status={snapshot['Status']})")

common_tags = {
    "Environment": "dr-sandbox",
    "DrillID": drill_id,
    "TTL": str(ttl_epoch),
    "ManagedBy": "pulumi",
}

vpc = aws.ec2.Vpc(
    "dr-sandbox-vpc",
    cidr_block="10.240.0.0/16",
    enable_dns_support=True,
    enable_dns_hostnames=True,
    tags={**common_tags, "Name": f"dr-sandbox-{drill_id}"},
)

azs = aws.get_availability_zones(state="available")

subnet_a = aws.ec2.Subnet(
    "dr-sandbox-subnet-a",
    vpc_id=vpc.id,
    cidr_block="10.240.1.0/24",
    availability_zone=azs.names[0],
    map_public_ip_on_launch=False,
    tags=common_tags,
)

subnet_b = aws.ec2.Subnet(
    "dr-sandbox-subnet-b",
    vpc_id=vpc.id,
    cidr_block="10.240.2.0/24",
    availability_zone=azs.names[1],
    map_public_ip_on_launch=False,
    tags=common_tags,
)

subnet_group = aws.rds.SubnetGroup(
    "dr-sandbox-subnet-group",
    subnet_ids=[subnet_a.id, subnet_b.id],
    tags=common_tags,
)

sandbox_sg = aws.ec2.SecurityGroup(
    "dr-sandbox-sg",
    vpc_id=vpc.id,
    description="Admit only the validation orchestrator CIDR to the DR sandbox",
    ingress=[aws.ec2.SecurityGroupIngressArgs(
        protocol="tcp",
        from_port=3306,
        to_port=3306,
        cidr_blocks=[orchestrator_cidr],
        description="Aurora/MySQL from the validation orchestrator only",
    )],
    egress=[aws.ec2.SecurityGroupEgressArgs(
        protocol="-1",
        from_port=0,
        to_port=0,
        cidr_blocks=["0.0.0.0/0"],
    )],
    tags=common_tags,
)

cluster = aws.rds.Cluster(
    "dr-sandbox-cluster",
    cluster_identifier=f"dr-sandbox-{drill_id}",
    engine="aurora-mysql",
    snapshot_identifier=snapshot_id,
    kms_key_id=kms_key_id,
    db_subnet_group_name=subnet_group.name,
    vpc_security_group_ids=[sandbox_sg.id],
    skip_final_snapshot=True,
    deletion_protection=False,
    apply_immediately=True,
    tags=common_tags,
)

instance = aws.rds.ClusterInstance(
    "dr-sandbox-instance",
    cluster_identifier=cluster.id,
    instance_class="db.r6g.large",
    engine="aurora-mysql",
    publicly_accessible=False,
    tags=common_tags,
)

pulumi.export("drill_id", drill_id)
pulumi.export("resolved_snapshot", snapshot_id)
pulumi.export("sandbox_endpoint", cluster.endpoint)
pulumi.export("sandbox_reader_endpoint", cluster.reader_endpoint)
pulumi.export("ttl_epoch", ttl_epoch)

#!/usr/bin/env python3
"""Reap DR sandboxes whose Pulumi stacks have passed their TTL.

Teardown path for the sandbox program. Uses the Pulumi Automation API: for every
stack in the local project, read its `ttl_epoch` output and run stack.destroy()
once wall-clock time has passed it. Run this from a scheduled job so no sandbox
outlives its drill window.

Exit codes:
    0  every expired sandbox destroyed (or none were due yet)
    1  one or more destroys failed
    2  usage / configuration error
"""
import sys
import time

from pulumi.automation import LocalWorkspace, select_stack
from pulumi.automation.errors import CommandError


def reap(work_dir: str) -> int:
    ws = LocalWorkspace(work_dir=work_dir)
    try:
        stacks = ws.list_stacks()
    except CommandError as exc:
        print(f"cannot list stacks in {work_dir}: {exc}", file=sys.stderr)
        return 2

    now = int(time.time())
    reaped = 0
    failures = 0
    for summary in stacks:
        stack = select_stack(stack_name=summary.name, work_dir=work_dir)
        ttl = stack.outputs().get("ttl_epoch")
        if ttl is None or int(ttl.value) > now:
            continue
        try:
            stack.destroy(on_output=print)
            reaped += 1
            print(f"destroyed expired sandbox stack {summary.name}")
        except CommandError as exc:
            print(f"destroy failed for {summary.name}: {exc}", file=sys.stderr)
            failures += 1

    print(f"reaped {reaped} sandbox(es); {failures} failure(s)")
    return 1 if failures else 0


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("usage: reap_expired_sandboxes.py <pulumi_project_work_dir>", file=sys.stderr)
        sys.exit(2)
    sys.exit(reap(sys.argv[1]))

Diagnostics:
  pulumi:pulumi:Stack (dr-sandbox-drill-20260718-01):
    resolved snapshot rds:prod-aurora-2026-07-18-06-00 (status=available)

     Type                        Name                          Status
 +   pulumi:pulumi:Stack         dr-sandbox-drill-20260718-01  created
 +   ├─ aws:ec2:Vpc              dr-sandbox-vpc                created
 +   ├─ aws:ec2:SecurityGroup    dr-sandbox-sg                 created
 +   ├─ aws:rds:Cluster          dr-sandbox-cluster            created
 +   └─ aws:rds:ClusterInstance  dr-sandbox-instance           created

Outputs:
    sandbox_endpoint: "dr-sandbox-drill-20260718-01.cluster-abc.us-east-1.rds.amazonaws.com"
    resolved_snapshot: "rds:prod-aurora-2026-07-18-06-00"
    ttl_epoch: 1752868800

Resources:
    + 7 created