🏡 index : cold-air-networks/chromium-setup.git

// hetzner-runner provisions an ephemeral Hetzner Cloud server for a
// Chromium build and guarantees it gets torn down afterward.
//
// Usage:
//
//	hetzner-runner up                  # create server, print IP, write state file
//	hetzner-runner down                # delete server + ssh key from state file
//	hetzner-runner run -- <cmd> [args] # up, ssh-exec cmd on the box, down (always)
//	hetzner-runner run -script build.sh -env GITEA_TOKEN -env MIRROR_READ_TOKEN
//	                                    # up, pipe build.sh to `bash -s` on the box
//	                                    # (env vars forwarded from this process's
//	                                    # own environment, not the CLI args), down (always)
//	hetzner-runner exec -- <cmd> [args]        # run against the server from a prior `up`,
//	hetzner-runner exec -script phase.sh ...   # no provisioning, no teardown -- lets a CI
//	                                            # workflow split a build into separate,
//	                                            # separately-visible steps sharing one box
//
// HCLOUD_TOKEN must be set in the environment. Never pass secrets as flags
// (shows up in process lists / CI logs) — use -env to forward them instead.
package main

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"crypto/rand"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/hetznercloud/hcloud-go/v2/hcloud"
	"golang.org/x/crypto/ssh"
)

const stateFileName = ".hetzner-runner-state.json"

// selfDestructScript is cloud-init user-data that runs on the box itself.
// It's a failsafe independent of this program's lifetime: if `down` never
// runs (laptop dies, network drops, CI job gets killed), the server deletes
// itself via the Hetzner API after maxLifetime regardless.
const selfDestructScript = `#cloud-config
write_files:
  - path: /root/.hcloud-token
    permissions: '0600'
    content: '%s'
runcmd:
  - [ bash, -c, "echo 'curl -sf -X DELETE -H \"Authorization: Bearer $(cat /root/.hcloud-token)\" https://api.hetzner.cloud/v1/servers/$(curl -sf http://169.254.169.254/hetzner/v1/metadata/instance-id)' | at now + %d hours" ]
`

type state struct {
	ServerID    int64  `json:"server_id"`
	ServerName  string `json:"server_name"`
	SSHKeyID    int64  `json:"ssh_key_id"`
	IPv4        string `json:"ipv4"`
	PrivKeyPath string `json:"priv_key_path"`
}

type config struct {
	serverType  string
	image       string
	location    string
	namePrefix  string
	maxLifetime time.Duration
	sshTimeout  time.Duration
}

func defaultConfig() config {
	return config{
		serverType:  envOr("HETZNER_SERVER_TYPE", "cpx51"),
		image:       envOr("HETZNER_IMAGE", "ubuntu-24.04"),
		location:    envOr("HETZNER_LOCATION", "hil"),
		namePrefix:  envOr("HETZNER_NAME_PREFIX", "chromium-build"),
		// Failsafe backstop for exactly the gap found in ci run #65
		// (2026-08-05): act_runner cancelling a step on its own job
		// timeout aborts the whole job immediately rather than
		// continuing to the `if: always()` teardown step, so this is
		// sometimes the *only* thing that deletes the server. Needs
		// comfortable margin beyond the job timeout (now 6h, doubled
		// from 3h the same day this was discovered) -- 8h left only 2h
		// of margin, uncomfortably close given provisioning/retry
		// overhead on top of the job's own 6h.
		maxLifetime: 10 * time.Hour,
		sshTimeout:  3 * time.Minute,
	}
}

func envOr(k, def string) string {
	if v := os.Getenv(k); v != "" {
		return v
	}
	return def
}

// dlog appends a timestamped line to a local log file on the runner host,
// independent of whatever Gitea's own log capture does. Confirmed twice
// (ci runs #62 and #64) that Gitea's job log can desync from what's
// actually happening -- showing stale/absent output, or the job marked
// failed, while the underlying process was still alive and healthy, or
// had already vanished with zero trace anywhere else (not in the job
// log, not in dmesg/OOM, not in the runner service's own journal, which
// only ever logs one line per task dispatch). This is a second,
// independent record of this process's own view of events -- what it was
// doing and when -- that survives regardless of what Gitea or act_runner
// think happened. Best-effort: a logging failure must never fail the
// actual operation.
func dlog(format string, args ...any) {
	path := filepath.Join(envOr("HOME", "."), "hetzner-runner-debug.log")
	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	line := fmt.Sprintf(format, args...)
	fmt.Fprintf(f, "%s pid=%d %s\n", time.Now().UTC().Format(time.RFC3339Nano), os.Getpid(), line)
}

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "usage: hetzner-runner <up|down|run> [flags]")
		os.Exit(2)
	}

	dlog("main: invoked with args=%v", os.Args[1:])
	defer dlog("main: exiting normally, args=%v", os.Args[1:])

	token := os.Getenv("HCLOUD_TOKEN")
	if token == "" {
		fatal("HCLOUD_TOKEN must be set in the environment")
	}
	client := hcloud.NewClient(hcloud.WithToken(token))
	cfg := defaultConfig()

	switch os.Args[1] {
	case "up":
		fs := flag.NewFlagSet("up", flag.ExitOnError)
		bindConfig(fs, &cfg)
		fs.Parse(os.Args[2:])
		dlog("up: starting, type=%s location=%s", cfg.serverType, cfg.location)
		st, err := up(context.Background(), client, token, cfg)
		if err != nil {
			dlog("up: FAILED: %v", err)
			fatal("up: %v", err)
		}
		saveState(st)
		dlog("up: succeeded, server_id=%d ip=%s", st.ServerID, st.IPv4)
		fmt.Printf("server ready: id=%d ip=%s\n", st.ServerID, st.IPv4)

	case "down":
		st, err := loadState()
		if err != nil {
			fatal("down: %v (nothing to tear down, or run from the same directory as `up`)", err)
		}
		dlog("down: starting, server_id=%d", st.ServerID)
		if err := down(context.Background(), client, st); err != nil {
			dlog("down: FAILED: %v", err)
			fatal("down: %v", err)
		}
		os.Remove(stateFileName)
		dlog("down: succeeded, server_id=%d", st.ServerID)
		fmt.Println("torn down")

	case "run":
		fs := flag.NewFlagSet("run", flag.ExitOnError)
		bindConfig(fs, &cfg)
		scriptPath, envNames := bindPayloadFlags(fs)
		fs.Parse(os.Args[2:])
		payload := buildPayload("run", scriptPath, envNames, fs.Args())
		os.Exit(runLifecycle(client, token, cfg, payload))

	case "exec":
		// Runs a command/script against an already-provisioned server (from
		// a prior `up`), without provisioning or tearing down -- lets a CI
		// workflow split a build into separately-visible steps (Sync, gn
		// gen, Build v8, ...) that all share one box, with `up`/`down` as
		// their own explicit steps (the last one gated on `if: always()`
		// for guaranteed cleanup).
		fs := flag.NewFlagSet("exec", flag.ExitOnError)
		scriptPath, envNames := bindPayloadFlags(fs)
		uploads := &stringSlice{}
		fs.Var(uploads, "upload", "local=remote file to copy to the box before running (repeatable)")
		downloads := &stringSlice{}
		fs.Var(downloads, "download", "remote=local file to copy from the box after running, only if it succeeded (repeatable)")
		fs.Parse(os.Args[2:])
		payload := buildPayload("exec", scriptPath, envNames, fs.Args())

		st, err := loadState()
		if err != nil {
			fatal("exec: %v (run `up` first)", err)
		}

		os.Exit(runExecCommand(client, st, payload, *uploads, *downloads, scriptPath))

	default:
		fmt.Fprintln(os.Stderr, "usage: hetzner-runner <up|down|run> [flags]")
		os.Exit(2)
	}
}

// bindPayloadFlags registers -script/-env on fs and returns pointers to
// read after fs.Parse. Shared by `run` and `exec`, which both accept either
// a -- command or a -script.
func bindPayloadFlags(fs *flag.FlagSet) (scriptPath *string, envNames *stringSlice) {
	scriptPath = fs.String("script", "", "local script to pipe to `bash -s` on the remote box, instead of a -- command")
	envNames = &stringSlice{}
	fs.Var(envNames, "env", "name of an env var to read from this process's environment and forward to the remote script (repeatable)")
	return scriptPath, envNames
}

// buildPayload builds env (from -env flags) unconditionally, regardless of
// whether -script or a plain -- command is used -- both need it forwarded
// to the remote side. Previously env was only ever built inside the
// -script branch; a plain -- command silently ignored -env entirely (no
// forwarding *and* no validation that the named vars even existed
// locally), found while diagnosing why chromium-fork's sccache/MinIO
// credentials (forwarded via -env against -- 'autoninja ...' steps) never
// reached the remote build at all.
func buildPayload(subcmd string, scriptPath *string, envNames *stringSlice, remainder []string) remotePayload {
	env := map[string]string{}
	for _, name := range *envNames {
		v, ok := os.LookupEnv(name)
		if !ok {
			fatal("%s: -env %s: not set in this process's environment", subcmd, name)
		}
		env[name] = v
	}

	if *scriptPath != "" {
		script, err := os.ReadFile(*scriptPath)
		if err != nil {
			fatal("%s: reading -script %q: %v", subcmd, *scriptPath, err)
		}
		return remotePayload{script: string(script), env: env}
	}
	if len(remainder) == 0 {
		fatal("%s: usage: hetzner-runner %s [flags] -- <remote command...>  (or -script <path>)", subcmd, subcmd)
	}
	return remotePayload{command: strings.Join(remainder, " "), env: env}
}

func bindConfig(fs *flag.FlagSet, cfg *config) {
	fs.StringVar(&cfg.serverType, "type", cfg.serverType, "Hetzner server type")
	fs.StringVar(&cfg.image, "image", cfg.image, "OS image")
	fs.StringVar(&cfg.location, "location", cfg.location, "Hetzner location")
	fs.DurationVar(&cfg.maxLifetime, "max-lifetime", cfg.maxLifetime, "failsafe: server self-deletes after this long even if `down` never runs")
	fs.DurationVar(&cfg.sshTimeout, "ssh-timeout", cfg.sshTimeout, "how long to wait for SSH to come up")
}

// remotePayload is either a plain command (`run -- <cmd>`) or a local
// script piped to `bash -s` on the remote box with forwarded env vars
// (`run -script ...`). Exactly one of command / script is set.
type remotePayload struct {
	command string
	script  string
	env     map[string]string // only used with script; forwarded via `export`, never argv
}

// stringSlice is a repeatable string flag (flag.Value).
type stringSlice []string

func (s *stringSlice) String() string     { return strings.Join(*s, ",") }
func (s *stringSlice) Set(v string) error { *s = append(*s, v); return nil }

// withHeartbeat runs fn, logging a heartbeat to the local debug log every
// 30s while it's in flight. If this process gets killed abruptly mid-call
// (the scenario dlog's pre/post-call logging can't help with, since
// there's no "after" to log), the last heartbeat timestamp pins down
// precisely when it was still alive and what it was doing -- evidence
// that survives independent of whatever Gitea's log capture shows.
func withHeartbeat(label string, fn func() (int, error)) (int, error) {
	stop := make(chan struct{})
	defer close(stop)
	go func() {
		ticker := time.NewTicker(30 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-stop:
				return
			case <-ticker.C:
				dlog("heartbeat: still executing %s", label)
			}
		}
	}()
	return fn()
}

// runLifecycle does up -> exec -> down, guaranteeing down runs even if the
// remote command fails or this process is interrupted (Ctrl-C, CI job
// cancellation).
// runExecCommand runs payload against an already-provisioned server (st),
// uploading any -upload files first.
//
// Unlike an ordinary step failure (a nonzero remote exit code -- left for
// the CI workflow's own separate, later "down" step to clean up, since one
// failing build phase shouldn't nuke a box other phases might still want),
// this proactively tears the server down itself if it hits a genuine
// Go-level error partway through (upload failure, SSH connection failure,
// or a panic) -- belt-and-suspenders against the gap found in ci run #65:
// act_runner cancelling a step on its own job timeout aborts the whole job
// immediately, skipping the `if: always()` teardown step entirely. A
// SIGKILL still can't be caught here either (nothing can), but this at
// least covers every OTHER way this process could die with an error
// instead of a clean exit, which previously fell through to nothing until
// the separate down step (or the cloud-init failsafe, hours later) caught
// it.
func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, uploads, downloads stringSlice, scriptPath *string) (code int) {
	succeeded := false
	defer func() {
		if succeeded {
			return
		}
		r := recover()
		dlog("exec: attempting cleanup after error (recovered_panic=%v)", r != nil)
		downCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
		defer cancel()
		if err := down(downCtx, client, st); err != nil {
			dlog("exec: cleanup-on-error teardown FAILED: %v", err)
			fmt.Fprintf(os.Stderr, "WARNING: cleanup-on-error teardown failed, server %d may still be running and billing: %v\n", st.ServerID, err)
			fmt.Fprintf(os.Stderr, "  manual cleanup: hcloud server delete %d\n", st.ServerID)
		} else {
			os.Remove(stateFileName)
			dlog("exec: cleanup-on-error teardown succeeded, server_id=%d", st.ServerID)
			fmt.Fprintln(os.Stderr, "note: tore down the server after an error in this step")
		}
		if r != nil {
			panic(r)
		}
	}()

	payloadDesc := payload.command
	if payload.script != "" {
		payloadDesc = fmt.Sprintf("-script %s (%d bytes)", *scriptPath, len(payload.script))
	}
	dlog("exec: starting against server_id=%d: %s", st.ServerID, payloadDesc)

	for _, u := range uploads {
		local, remote, ok := strings.Cut(u, "=")
		if !ok {
			fmt.Fprintf(os.Stderr, "exec: -upload %q: expected local=remote\n", u)
			return 2 // succeeded stays false: not a server-side error, but nothing ran either, so tear down for consistency
		}
		dlog("exec: uploading %s -> %s", local, remote)
		if err := uploadFile(st, local, remote); err != nil {
			dlog("exec: upload FAILED (Go-level error, cleaning up): %v", err)
			fmt.Fprintf(os.Stderr, "exec: uploading %s: %v\n", local, err)
			return 1
		}
	}

	var runErr error
	code, runErr = withHeartbeat(payloadDesc, func() (int, error) {
		if payload.script != "" {
			return sshExecScript(st, payload.script, payload.env)
		}
		return sshExec(st, payload.command, payload.env)
	})
	if runErr != nil {
		dlog("exec: Go-level FAILURE (cleaning up): %v", runErr)
		fmt.Fprintf(os.Stderr, "remote command failed: %v\n", runErr)
		return 1
	}

	// Only download if the command actually succeeded -- a nonzero exit
	// (e.g. the tar step itself failing) means there's nothing valid to
	// pull down.
	if code == 0 {
		for _, d := range downloads {
			remote, local, ok := strings.Cut(d, "=")
			if !ok {
				fmt.Fprintf(os.Stderr, "exec: -download %q: expected remote=local\n", d)
				return 2
			}
			dlog("exec: downloading %s -> %s", remote, local)
			if err := downloadFile(st, remote, local); err != nil {
				dlog("exec: download FAILED (Go-level error, cleaning up): %v", err)
				fmt.Fprintf(os.Stderr, "exec: downloading %s: %v\n", remote, err)
				return 1
			}
		}
	}

	// A normal nonzero exit code from the remote command is NOT a
	// Go-level error -- an ordinary step failure, left for the workflow's
	// own separate teardown step. Only Go-level errors (above) skip
	// marking succeeded, triggering cleanup here.
	succeeded = true
	dlog("exec: finished, exit code=%d", code)
	return code
}

func runLifecycle(client *hcloud.Client, token string, cfg config, payload remotePayload) (exitCode int) {
	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer cancel()
	go func() {
		<-ctx.Done()
		// Distinguishes "our own cancel() ran because we're exiting
		// normally" from "something external actually signaled us" --
		// ctx.Err() is context.Canceled either way, but this goroutine
		// only starts once, so if it fires before the deferred teardown
		// below has finished, that's real external signal delivery.
		dlog("runLifecycle: context cancelled (signal received or normal exit racing this check)")
	}()

	dlog("runLifecycle: starting up, type=%s location=%s", cfg.serverType, cfg.location)
	st, err := up(ctx, client, token, cfg)
	if err != nil {
		dlog("runLifecycle: up FAILED: %v", err)
		fatal("up: %v", err)
	}
	saveState(st)
	dlog("runLifecycle: up succeeded, server_id=%d ip=%s", st.ServerID, st.IPv4)

	defer func() {
		// Always tear down, regardless of how we got here.
		dlog("runLifecycle: tearing down server_id=%d", st.ServerID)
		downCtx, downCancel := context.WithTimeout(context.Background(), 2*time.Minute)
		defer downCancel()
		if err := down(downCtx, client, st); err != nil {
			dlog("runLifecycle: teardown FAILED: %v", err)
			fmt.Fprintf(os.Stderr, "WARNING: teardown failed, server %d (%s) may still be running and billing: %v\n", st.ServerID, st.IPv4, err)
			fmt.Fprintf(os.Stderr, "  manual cleanup: hcloud server delete %d\n", st.ServerID)
			return
		}
		os.Remove(stateFileName)
		dlog("runLifecycle: teardown succeeded, server_id=%d", st.ServerID)
		fmt.Println("torn down")
	}()

	var code int
	var label string
	if payload.script != "" {
		label = fmt.Sprintf("script (%d bytes)", len(payload.script))
		dlog("runLifecycle: running script (%d bytes, %d env vars)", len(payload.script), len(payload.env))
		fmt.Printf("server ready: id=%d ip=%s, running script (%d bytes, %d env vars forwarded)\n", st.ServerID, st.IPv4, len(payload.script), len(payload.env))
	} else {
		label = payload.command
		dlog("runLifecycle: running command: %s", payload.command)
		fmt.Printf("server ready: id=%d ip=%s, running: %s\n", st.ServerID, st.IPv4, payload.command)
	}
	code, err = withHeartbeat(label, func() (int, error) {
		if payload.script != "" {
			return sshExecScript(st, payload.script, payload.env)
		}
		return sshExec(st, payload.command, payload.env)
	})
	if err != nil {
		dlog("runLifecycle: remote command FAILED: %v", err)
		fmt.Fprintf(os.Stderr, "remote command failed: %v\n", err)
	} else {
		dlog("runLifecycle: remote command finished, exit code=%d", code)
	}
	return code
}

func up(ctx context.Context, client *hcloud.Client, token string, cfg config) (*state, error) {
	pub, privPath, err := ensureEphemeralKey()
	if err != nil {
		return nil, fmt.Errorf("generating ephemeral ssh key: %w", err)
	}

	name := fmt.Sprintf("%s-%s", cfg.namePrefix, time.Now().Format("20060102-150405"))

	keyResult, _, err := client.SSHKey.Create(ctx, hcloud.SSHKeyCreateOpts{
		Name:      name,
		PublicKey: pub,
	})
	if err != nil {
		return nil, fmt.Errorf("uploading ephemeral ssh key: %w", err)
	}

	serverType, _, err := client.ServerType.GetByName(ctx, cfg.serverType)
	if err != nil || serverType == nil {
		return nil, fmt.Errorf("looking up server type %q: %w", cfg.serverType, err)
	}
	image, _, err := client.Image.GetByNameAndArchitecture(ctx, cfg.image, serverType.Architecture)
	if err != nil || image == nil {
		return nil, fmt.Errorf("looking up image %q: %w", cfg.image, err)
	}
	location, _, err := client.Location.GetByName(ctx, cfg.location)
	if err != nil || location == nil {
		return nil, fmt.Errorf("looking up location %q: %w", cfg.location, err)
	}

	userData := fmt.Sprintf(selfDestructScript, token, int(cfg.maxLifetime.Hours()))

	result, _, err := client.Server.Create(ctx, hcloud.ServerCreateOpts{
		Name:       name,
		ServerType: serverType,
		Image:      image,
		Location:   location,
		SSHKeys:    []*hcloud.SSHKey{keyResult},
		UserData:   userData,
	})
	if err != nil {
		return nil, fmt.Errorf("creating server: %w", err)
	}

	if err := waitForAction(ctx, client, result.Action); err != nil {
		return nil, fmt.Errorf("waiting for server creation: %w", err)
	}

	st := &state{
		ServerID:    result.Server.ID,
		ServerName:  name,
		SSHKeyID:    keyResult.ID,
		IPv4:        result.Server.PublicNet.IPv4.IP.String(),
		PrivKeyPath: privPath,
	}

	if err := waitForSSH(st, cfg.sshTimeout); err != nil {
		// Best-effort cleanup on partial failure so we don't leak a
		// server the caller doesn't know the ID of yet.
		_ = down(ctx, client, st)
		return nil, fmt.Errorf("waiting for ssh: %w", err)
	}

	return st, nil
}

func down(ctx context.Context, client *hcloud.Client, st *state) error {
	var errs []string
	serverDeleted := true
	if st.ServerID != 0 {
		if _, _, err := client.Server.DeleteWithResult(ctx, &hcloud.Server{ID: st.ServerID}); err != nil {
			errs = append(errs, fmt.Sprintf("delete server %d: %v", st.ServerID, err))
			serverDeleted = false
		}
	}
	if st.SSHKeyID != 0 {
		if _, err := client.SSHKey.Delete(ctx, &hcloud.SSHKey{ID: st.SSHKeyID}); err != nil {
			errs = append(errs, fmt.Sprintf("delete ssh key %d: %v", st.SSHKeyID, err))
		}
	}
	// Only remove the local private key once the server itself is
	// confirmed gone. Removing it unconditionally (as this used to do)
	// meant a failed/errored server deletion still discarded the one
	// thing that could get back into the now-orphaned box to inspect or
	// manually clean it up -- confirmed missing in exactly this scenario
	// investigating ci run #62 (2026-08-05): server still alive, local
	// key already gone, no way back in.
	if st.PrivKeyPath != "" && serverDeleted {
		os.Remove(st.PrivKeyPath)
	}
	if len(errs) > 0 {
		return fmt.Errorf("%s", strings.Join(errs, "; "))
	}
	return nil
}

func waitForAction(ctx context.Context, client *hcloud.Client, action *hcloud.Action) error {
	progress, errs := client.Action.WatchProgress(ctx, action)
	for {
		select {
		case <-progress:
		case err, ok := <-errs:
			if !ok {
				return nil
			}
			if err != nil {
				return err
			}
		}
	}
}

func waitForSSH(st *state, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	var lastErr error
	for time.Now().Before(deadline) {
		conn, err := net.DialTimeout("tcp", net.JoinHostPort(st.IPv4, "22"), 5*time.Second)
		if err == nil {
			conn.Close()
			// TCP is up; give sshd + cloud-init user creation a moment,
			// then confirm with a real SSH handshake + command.
			time.Sleep(3 * time.Second)
			if code, err := sshExec(st, "true", nil); err == nil && code == 0 {
				return nil
			} else {
				lastErr = err
			}
		} else {
			lastErr = err
		}
		time.Sleep(5 * time.Second)
	}
	return fmt.Errorf("timed out after %s, last error: %v", timeout, lastErr)
}

// sshExec runs command via a plain SSH exec request (no stdin -- see
// sshSession/sshExecScript for why that's what makes a PTY here safe).
//
// env is prefixed onto the command as `export KEY='value'; ...`. Found
// missing entirely (env silently accepted by the CLI flags, validated as
// present locally, then never actually forwarded) while diagnosing why
// chromium-fork's sccache/MinIO credentials never reached the remote
// `autoninja` invocations in ci runs #69/#70 -- every `-- <cmd>` step
// using `-env AWS_ACCESS_KEY_ID -env AWS_SECRET_ACCESS_KEY` had those
// values validated against *this* process's environment and then
// dropped, never exported into the remote command's. sshExecScript
// (the -script path) already did this correctly; -- <cmd> never did.
func sshExec(st *state, command string, env map[string]string) (int, error) {
	session, client, err := sshSession(st, true)
	if err != nil {
		return -1, err
	}
	defer client.Close()
	defer session.Close()

	var buf strings.Builder
	for k, v := range env {
		fmt.Fprintf(&buf, "export %s=%s; ", k, shellQuote(v))
	}
	buf.WriteString(command)

	session.Stdout = os.Stdout
	session.Stderr = os.Stderr
	return runSession(session, buf.String())
}

// uploadFile copies a local file to the remote box. See uploadBytes for
// the transfer mechanism.
func uploadFile(st *state, localPath, remotePath string) error {
	content, err := os.ReadFile(localPath)
	if err != nil {
		return err
	}
	return uploadBytes(st, content, remotePath)
}

// uploadBytes writes content to remotePath by piping it to `cat >
// remotePath` over a plain (non-PTY) exec session -- no SFTP subsystem
// required. Deliberately non-PTY: a plain pipe has no fixed-size line
// buffer the way a PTY's canonical-mode input queue does (historically
// 4KB on Linux) and won't silently truncate a multi-KB write -- see
// sshExecScript's doc comment for why that distinction turned out to
// matter a great deal.
//
// Uses Start()+StdinPipe()+explicit Close() rather than Run()'s automatic
// Stdin-reader-copying, so the exact moment EOF is signaled to the remote
// `cat` is under our control instead of implicit in the library's internal
// copy goroutine.
func uploadBytes(st *state, content []byte, remotePath string) error {
	session, client, err := sshSession(st, false)
	if err != nil {
		return err
	}
	defer client.Close()
	defer session.Close()

	stdin, err := session.StdinPipe()
	if err != nil {
		return err
	}
	var out bytes.Buffer
	session.Stdout = &out
	session.Stderr = &out

	if err := session.Start(fmt.Sprintf("cat > %s", shellQuote(remotePath))); err != nil {
		return err
	}
	if _, err := stdin.Write(content); err != nil {
		return err
	}
	if err := stdin.Close(); err != nil {
		return err
	}

	if err := session.Wait(); err != nil {
		if exitErr, ok := err.(*ssh.ExitError); ok {
			return fmt.Errorf("remote `cat > %s` exited %d: %s", remotePath, exitErr.ExitStatus(), out.String())
		}
		return err
	}
	return nil
}

// downloadFile is uploadBytes's mirror image: streams remotePath's bytes
// (via a plain, non-PTY `cat remotePath`) straight to a local file,
// without buffering the whole thing in memory first -- build artifacts
// pulled this way can be hundreds of MB to a few GB, and hetzner-runner
// itself runs on the same host (mainframe) as the destination disk, so
// this is the direct path rather than routing through any intermediate
// storage.
func downloadFile(st *state, remotePath, localPath string) error {
	session, client, err := sshSession(st, false)
	if err != nil {
		return err
	}
	defer client.Close()
	defer session.Close()

	if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
		return fmt.Errorf("creating local directory for %s: %w", localPath, err)
	}
	f, err := os.Create(localPath)
	if err != nil {
		return err
	}
	defer f.Close()

	var stderrBuf bytes.Buffer
	session.Stdout = f
	session.Stderr = &stderrBuf

	if err := session.Run(fmt.Sprintf("cat %s", shellQuote(remotePath))); err != nil {
		if exitErr, ok := err.(*ssh.ExitError); ok {
			return fmt.Errorf("remote `cat %s` exited %d: %s", remotePath, exitErr.ExitStatus(), stderrBuf.String())
		}
		return err
	}
	return nil
}

// sshExecScript runs script on the remote box by uploading it as a file
// and executing it, rather than piping it to `bash -s` over stdin.
//
// This replaced two earlier, progressively-less-wrong attempts, both
// disproven in production:
//  1. `session.Run("bash -s")` with Stdin set to a Reader -- hung
//     indefinitely after the script's real work finished (ci runs #56,
//     #60): PTY stdin EOF is not a reliable "hang up" signal, since a PTY
//     is fundamentally for interactive sessions where a pause in input
//     isn't supposed to mean "terminate". Confirmed independently outside
//     this codebase: golang/go#6088 (identical x/crypto/ssh symptom) and
//     net-ssh/net-ssh#460 (different language, same symptom, confirms
//     removing request_pty fixes it).
//  2. Switching to Start()+StdinPipe()+explicit Close(), appending
//     `exit $?` so bash would terminate on its own instead of depending on
//     EOF at all -- this fixed short scripts but a real sync.sh run (ci
//     run #63) still hung, this time *mid-script*: the process was found
//     blocked in do_select() on its stdin fd, and directories the script
//     creates early on (depot_tools clone) didn't exist yet even though
//     later output (an apt-get install several lines further down) had
//     already reached the log. That means the script was truncated
//     partway through delivery, not just missing its termination signal --
//     consistent with a PTY's small, fixed-size canonical-mode input queue
//     (historically 4KB on Linux) silently dropping the tail of a
//     multi-KB single write() that arrives faster than bash drains it. A
//     plain pipe (as uploadBytes uses) doesn't have this limitation.
//
// Uploading the script and executing it as a file sidesteps this whole
// class of problem: no script content ever goes through a PTY's stdin at
// all, only the plain, already-proven-reliable uploadBytes path. The PTY
// is still used for the *execution* (via sshExec, for live output
// streaming), but that session has no stdin data of its own -- a plain
// command with nothing written to its stdin has no EOF ambiguity to begin
// with.
func sshExecScript(st *state, script string, env map[string]string) (int, error) {
	var buf strings.Builder
	buf.WriteString("#!/usr/bin/env bash\n")
	for k, v := range env {
		fmt.Fprintf(&buf, "export %s=%s\n", k, shellQuote(v))
	}
	buf.WriteString(script)

	remotePath := fmt.Sprintf("/root/.hetzner-runner-script-%d.sh", time.Now().UnixNano())
	if err := uploadBytes(st, []byte(buf.String()), remotePath); err != nil {
		return -1, fmt.Errorf("uploading script: %w", err)
	}
	code, err := sshExec(st, fmt.Sprintf("bash %s", shellQuote(remotePath)), nil)
	if rmCode, rmErr := sshExec(st, fmt.Sprintf("rm -f %s", shellQuote(remotePath)), nil); rmErr == nil && rmCode != 0 {
		fmt.Fprintf(os.Stderr, "warning: failed to remove remote script %s\n", remotePath)
	}
	return code, err
}

func shellQuote(v string) string {
	return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'"
}

// sshSession opens a new SSH session on the box described by st.
//
// wantPTY controls whether a PTY is requested:
//   - true for anything running a real command whose output we want to
//     watch live (build.sh phases, ad-hoc exec) -- a PTY makes the remote
//     side (ninja, gclient, python) think it's talking to a terminal, so
//     it line-buffers instead of block-buffering for a non-tty pipe.
//     Without this, output for a multi-hour build can arrive in laggy
//     multi-KB chunks instead of streaming as it happens. Combines stderr
//     into stdout, which is fine for a CI log.
//   - false for raw data transfer (uploadFile's `cat > file`) -- a PTY's
//     canonical-mode line discipline is actively wrong there: it can both
//     hang waiting for a line-oriented EOF signal a plain channel-close
//     doesn't reliably deliver, and reinterpret control bytes (Ctrl-D,
//     Ctrl-C, ...) that happen to appear in otherwise-binary-safe content.
func sshSession(st *state, wantPTY bool) (*ssh.Session, *ssh.Client, error) {
	key, err := os.ReadFile(st.PrivKeyPath)
	if err != nil {
		return nil, nil, err
	}
	signer, err := ssh.ParsePrivateKey(key)
	if err != nil {
		return nil, nil, err
	}
	sshCfg := &ssh.ClientConfig{
		User:            "root",
		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(), // ephemeral box, TOFU risk accepted for build workloads
		Timeout:         10 * time.Second,
	}
	client, err := ssh.Dial("tcp", net.JoinHostPort(st.IPv4, "22"), sshCfg)
	if err != nil {
		return nil, nil, err
	}
	session, err := client.NewSession()
	if err != nil {
		client.Close()
		return nil, nil, err
	}

	if wantPTY {
		modes := ssh.TerminalModes{
			ssh.ECHO:          0,
			ssh.TTY_OP_ISPEED: 14400,
			ssh.TTY_OP_OSPEED: 14400,
		}
		if err := session.RequestPty("xterm", 40, 200, modes); err != nil {
			session.Close()
			client.Close()
			return nil, nil, fmt.Errorf("requesting pty: %w", err)
		}
	}

	return session, client, nil
}

func runSession(session *ssh.Session, command string) (int, error) {
	if err := session.Run(command); err != nil {
		if exitErr, ok := err.(*ssh.ExitError); ok {
			return exitErr.ExitStatus(), nil
		}
		return -1, err
	}
	return 0, nil
}

func ensureEphemeralKey() (pubAuthorizedKeys string, privPath string, err error) {
	dir, err := os.MkdirTemp("", "hetzner-runner-key-*")
	if err != nil {
		return "", "", err
	}
	pub, priv, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		return "", "", err
	}
	sshPub, err := ssh.NewPublicKey(pub)
	if err != nil {
		return "", "", err
	}
	pkcs8, err := x509.MarshalPKCS8PrivateKey(priv)
	if err != nil {
		return "", "", err
	}
	privPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8})

	privPath = filepath.Join(dir, "id_ed25519")
	if err := os.WriteFile(privPath, privPEM, 0600); err != nil {
		return "", "", err
	}
	return string(ssh.MarshalAuthorizedKey(sshPub)), privPath, nil
}

func saveState(st *state) {
	f, err := os.Create(stateFileName)
	if err != nil {
		fatal("writing state file: %v", err)
	}
	defer f.Close()
	json.NewEncoder(f).Encode(st)
}

func loadState() (*state, error) {
	f, err := os.Open(stateFileName)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	var st state
	if err := json.NewDecoder(f).Decode(&st); err != nil {
		return nil, err
	}
	return &st, nil
}

func fatal(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "error: "+format+"\n", args...)
	os.Exit(1)
}