git » dnss » commit 7e142e5

diff --git a/internal/httpresolver/resolver.go b/internal/httpresolver/resolver.go
index 1d1339d..3624058 100644
--- a/internal/httpresolver/resolver.go
+++ b/internal/httpresolver/resolver.go
@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"sync"
 	"time"
 
 	"blitiri.com.ar/go/dnss/internal/dnsserver"
@@ -26,18 +25,17 @@ import (
 // httpsResolver implements the dnsserver.Resolver interface by querying a
 // server via DNS over HTTPS (DoH, RFC 8484).
 type httpsResolver struct {
-	Upstream  *url.URL
-	CAFile    string
-	tlsConfig *tls.Config
+	Upstream *url.URL
+	CAFile   string
 
 	// net.Resolver that will contact the server at --fallback_upstream for
 	// DNS resolutions.
 	fallbackResolver *net.Resolver
 
-	mu       sync.Mutex
-	client   *http.Client
-	firstErr time.Time
-	tr       *trace.Trace
+	client *http.Client
+
+	// Trace to keep track of client errors, for debugging.
+	tr *trace.Trace
 }
 
 var errAppendingCerts = errors.New("error appending certificates")
@@ -84,127 +82,63 @@ func NewDoH(upstream *url.URL, caFile, fallback string) *httpsResolver {
 func (r *httpsResolver) Init() error {
 	// If CAFile is empty, we're ok with the defaults (use the system default
 	// CA database).
+	var tlsConfig *tls.Config
 	if r.CAFile != "" {
 		pool, err := loadCertPool(r.CAFile)
 		if err != nil {
 			return err
 		}
 
-		r.tlsConfig = &tls.Config{
+		tlsConfig = &tls.Config{
 			RootCAs: pool,
 		}
 	}
 
-	client, err := r.newClient()
-
-	r.mu.Lock()
-	r.client = client
+	r.client = newClient(tlsConfig, r.fallbackResolver)
 	r.tr = trace.New("httpresolver.Client", r.Upstream.String())
-	r.tr.Printf("Init complete, client: %p", r.client)
-	r.mu.Unlock()
+	r.tr.Printf("Init complete")
 
-	return err
+	return nil
 }
 
-func (r *httpsResolver) newClient() (*http.Client, error) {
-	transport := &http.Transport{
-		TLSClientConfig: r.tlsConfig,
-
-		// Take the semi-standard proxy settings from the environment.
-		Proxy: http.ProxyFromEnvironment,
-
-		// Drop connections after 30s idle.
-		// This helps prevent connection pile-up on frequent client rotations,
-		// which can happen with intermittent network issues.
-		IdleConnTimeout: 30 * time.Second,
-
-		// Reasonable defaults, based on http.DefaultTransport.
-		DialContext: (&net.Dialer{
-			Timeout:   10 * time.Second,
-			KeepAlive: 1 * time.Second,
-			Resolver:  r.fallbackResolver,
-		}).DialContext,
-		ForceAttemptHTTP2:     true,
-		MaxIdleConns:          10,
-		TLSHandshakeTimeout:   4 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-	}
-
-	client := &http.Client{
+func newClient(tlsConfig *tls.Config, resolver *net.Resolver) *http.Client {
+	// Start from the default transport, which has reasonable settings (e.g.
+	// for timeouts, and taking proxy settings from the environment).
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = tlsConfig
+
+	// Use our resolver to find the upstream's address.
+	// Limit the dial timeout, as queries time out sooner anyway. Note dials
+	// can outlive the queries that started them, so the connection may still
+	// be useful for later queries.
+	transport.DialContext = (&net.Dialer{
+		Timeout:  10 * time.Second,
+		Resolver: resolver,
+	}).DialContext
+
+	// Enable HTTP/2 health checks: if we haven't received anything on a
+	// connection for 5s, send a ping, and close the connection if there is no
+	// response within 3s.
+	// This detects dead connections (e.g. when the network changes, or after
+	// an outage), which otherwise the transport would keep using for a long
+	// time.
+	transport.HTTP2 = &http.HTTP2Config{
+		SendPingTimeout: 5 * time.Second,
+		PingTimeout:     3 * time.Second,
+	}
+
+	return &http.Client{
 		// Give our HTTP requests 4 second timeouts: DNS usually doesn't wait
 		// that long anyway, but this helps with slow connections.
 		Timeout: 4 * time.Second,
 
 		Transport: transport,
 	}
-
-	return client, nil
-}
-
-func (r *httpsResolver) setClientError(err error) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	if err == nil {
-		r.firstErr = time.Time{}
-	} else {
-		if r.firstErr.IsZero() {
-			r.firstErr = time.Now()
-		}
-		r.tr.Printf("Client error: %v", err)
-	}
-}
-
-func (r *httpsResolver) Maintain(ctx context.Context) {
-	ticker := time.NewTicker(2 * time.Second)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-ticker.C:
-			r.maybeRotateClient()
-		}
-	}
 }
 
-func (r *httpsResolver) maybeRotateClient() {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	if r.firstErr.IsZero() {
-		return
-	}
-
-	// If we've seen errors for the last 10s, rotate the client.
-	// This is unfortunately needed because the Go HTTP/2 transport will
-	// insist on using a dead connection for a long time, and cannot be told
-	// to close it. This causes problems when the computer changes connections
-	// (e.g. switch wifi network) or is having intermittent network issues.
-	// This workaround works because a new client will initiate a new
-	// connection, and the old one will die in the background.
-	// The time chosen here combines with the transport timeouts set above, so
-	// we never have too many in-flight connections.
-	if time.Since(r.firstErr) > 10*time.Second {
-		// Close the old trace, and create a new one.
-		// This makes it easier to analyze the client behaviour in the traces.
-		r.tr.Errorf("Rotating client after %s of errors: %p",
-			time.Since(r.firstErr), r.client)
-		r.tr.Finish()
-
-		r.tr = trace.New("httpresolver.Client", r.Upstream.String())
-		client, err := r.newClient()
-		if err != nil {
-			r.tr.Errorf("Error creating new client: %v", err)
-			return
-		}
-
-		r.client = client
-		r.firstErr = time.Time{}
-		r.tr.Printf("Rotated client: %p", r.client)
-	}
-}
+// Maintain the resolver. There's nothing to do, as the HTTP transport takes
+// care of the connections.
+func (r *httpsResolver) Maintain(ctx context.Context) {}
 
 func (r *httpsResolver) Query(req *dns.Msg, tr *trace.Trace) (*dns.Msg, error) {
 	packed, err := req.Pack()
@@ -218,16 +152,12 @@ func (r *httpsResolver) Query(req *dns.Msg, tr *trace.Trace) (*dns.Msg, error) {
 
 	// TODO: Accept header.
 
-	r.mu.Lock()
-	client := r.client
-	r.mu.Unlock()
-
-	hr, err := client.Post(
+	hr, err := r.client.Post(
 		r.Upstream.String(),
 		"application/dns-message",
 		bytes.NewReader(packed))
-	r.setClientError(err)
 	if err != nil {
+		r.tr.Printf("Client error: %v", err)
 		return nil, fmt.Errorf("POST failed: %w", err)
 	}
 	tr.Printf("%s  %s", hr.Proto, hr.Status)
diff --git a/tests/dnsprobe.go b/tests/dnsprobe.go
new file mode 100644
index 0000000..f3144eb
--- /dev/null
+++ b/tests/dnsprobe.go
@@ -0,0 +1,81 @@
+//go:build ignore
+
+// dnsprobe sends DNS queries to a server at a regular interval, and logs the
+// result of each one. It's used by outage.sh to measure how dnss behaves
+// during and after network outages.
+//
+// Each query is for a different name, so the answers can't come from a cache.
+//
+// It prints one line per query, as they complete:
+//
+//	<send time, in unix milliseconds> <latency in milliseconds> <result>
+//
+// The result is "ok" if we got a NOERROR or NXDOMAIN reply, or a short
+// description of the failure otherwise.
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"math/rand/v2"
+	"net"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+var (
+	server   = flag.String("server", "127.0.0.1:1053", "DNS server to query")
+	interval = flag.Duration("interval", 200*time.Millisecond,
+		"time between queries")
+	timeout = flag.Duration("timeout", 2*time.Second, "timeout for each query")
+	domain  = flag.String("domain", "probe.test.", "domain to query under")
+)
+
+var outMu sync.Mutex
+
+func main() {
+	flag.Parse()
+
+	c := &dns.Client{
+		Net:     "udp",
+		Timeout: *timeout,
+	}
+
+	// Use a random prefix, so names don't repeat across runs either.
+	prefix := rand.Uint32()
+
+	ticker := time.NewTicker(*interval)
+	for i := 0; ; i++ {
+		go probe(c, fmt.Sprintf("%x-%d.%s", prefix, i, *domain))
+		<-ticker.C
+	}
+}
+
+func probe(c *dns.Client, name string) {
+	m := &dns.Msg{}
+	m.SetQuestion(name, dns.TypeA)
+
+	start := time.Now()
+	r, _, err := c.Exchange(m, *server)
+	latency := time.Since(start)
+
+	result := "ok"
+	var netErr net.Error
+	switch {
+	case errors.As(err, &netErr) && netErr.Timeout():
+		result = "timeout"
+	case err != nil:
+		result = "error: " + err.Error()
+	case r.Rcode != dns.RcodeSuccess && r.Rcode != dns.RcodeNameError:
+		result = dns.RcodeToString[r.Rcode]
+	}
+
+	outMu.Lock()
+	fmt.Fprintf(os.Stdout, "%d %d %s\n",
+		start.UnixMilli(), latency.Milliseconds(), result)
+	outMu.Unlock()
+}
diff --git a/tests/outage.sh b/tests/outage.sh
new file mode 100755
index 0000000..882cb20
--- /dev/null
+++ b/tests/outage.sh
@@ -0,0 +1,452 @@
+#!/bin/bash
+#
+# Tests how dnss behaves when the network goes away, and then comes back.
+#
+# It runs dnss (in DNS-to-HTTPS mode) in a simulated network, and while
+# continuously sending queries to it, it introduces different kinds of
+# outages (e.g. the link going down, or the upstream becoming unreachable).
+# Then it restores the network, and measures how long it takes for dnss to
+# successfully resolve queries again.
+#
+# The network is simulated using Linux network namespaces, inside an
+# unprivileged user namespace, so it doesn't require root.
+#
+#   client ns               router ns                 upstream ns
+#   dnss (DNS-to-HTTPS)     forwards between          dnss (HTTPS-to-DNS)
+#   dnsprobe                both links                minidns
+#      c0 -------------------- r0      r1 -------------------- u0
+#   10.0.1.2              10.0.1.1     10.0.2.1             10.0.2.2
+#
+# Usage:
+#   tests/outage.sh [scenario...]
+#
+# Scenarios (default: all of them):
+#   linkdown          The client's link goes down (e.g. wifi disconnects).
+#   blackhole         The router silently drops traffic to the upstream (e.g.
+#                     an ISP outage, the local network is still up).
+#   ipchange          The client's address goes away, and comes back as a
+#                     different one (e.g. switching networks or VPN, or NAT
+#                     state loss). Existing connections are dead.
+#   upstream-restart  The upstream DoH server goes down and comes back.
+#   start-offline     dnss starts with the client's link down.
+#   powercut          dnss starts with the client's link down. Half way
+#                     through the outage, the link comes up but the upstream
+#                     is still unreachable (e.g. after a power cut, the local
+#                     router comes up before its internet connection does).
+#
+# Environment variables:
+#   DNSS_BIN        dnss binary to test (default: build from the tree).
+#   DURATIONS       Outage durations in seconds (default: "5 60").
+#   RECOVERY_LIMIT  Seconds we allow for recovery (default: 30).
+#   DNSS_FLAGS      Extra flags to pass to the dnss being tested.
+#   DNSS_ENV        Extra environment for the dnss being tested (e.g.
+#                   "GODEBUG=http2debug=1").
+#   PROBE_INTERVAL  Time between queries sent to dnss (default: 200ms).
+#   STABLE_COUNT    Number of consecutive successful queries needed to
+#                   consider dnss recovered (default: 10).
+#   UPSTREAM        How dnss refers to the upstream: "name" (default) uses a
+#                   hostname, which needs to be resolved via the fallback
+#                   resolver; "ip" uses the IP address directly.
+#   OUTAGE_DIR      Working directory (default: .outage). Using different
+#                   ones allows running multiple instances in parallel.
+#
+# Results are printed at the end. Logs for each run are left in
+# $OUTAGE_DIR/runs/<scenario>-<duration>/.
+
+set -e
+
+# The tests are run from the repository root.
+cd "$(realpath "$(dirname "$0")/../" )"
+
+ALL_SCENARIOS="linkdown blackhole ipchange upstream-restart start-offline
+	powercut"
+DURATIONS="${DURATIONS:-5 60}"
+RECOVERY_LIMIT="${RECOVERY_LIMIT:-30}"
+PROBE_INTERVAL="${PROBE_INTERVAL:-200ms}"
+STABLE_COUNT="${STABLE_COUNT:-10}"
+UPSTREAM="${UPSTREAM:-name}"
+
+W="${OUTAGE_DIR:-.outage}"
+
+#####################################
+# Outer stage: build everything, then re-run ourselves inside the namespaces.
+# We need to build outside, as there is no network access inside.
+
+if [ "$OUTAGE_INSIDE_NS" != "1" ]; then
+	rm -rf "$W"
+	mkdir -p "$W/certs"
+
+	echo "## Building"
+	go build -o "$W/dnss" .
+	go build -o "$W/minidns" tests/minidns.go
+	go build -o "$W/dnsprobe" tests/dnsprobe.go
+
+	if [ "$DNSS_BIN" == "" ]; then
+		DNSS_BIN="$W/dnss"
+	fi
+	DNSS_BIN="$(realpath "$DNSS_BIN")"
+
+	GENERATE_CERT="$PWD/tests/generate_cert.go"
+	(
+		cd "$W/certs"
+		go run "$GENERATE_CERT" -ca -duration=24h \
+			--host=upstream.test,10.0.2.2
+	) > /dev/null
+
+	export OUTAGE_INSIDE_NS=1 DNSS_BIN
+	exec unshare --user --map-root-user --net "$0" "$@"
+fi
+
+#####################################
+# Inner stage: we are root in our own user and network namespaces.
+# This (initial) network namespace is the client's.
+
+# Kill all our background processes on exit.
+# Note we can't use "kill 0", as we may share the process group with our
+# caller. Because of this, background processes must be started directly
+# (and not via shell functions), so they are tracked as jobs.
+trap "exit 2" INT TERM
+trap 'kill $(jobs -p) $ROUTER_NS $UPSTREAM_NS 2> /dev/null' EXIT
+
+SCENARIOS="${*:-$ALL_SCENARIOS}"
+
+# Current time, in unix milliseconds.
+function now() {
+	date +%s%3N
+}
+
+function log() {
+	echo "$(date +%T.%3N)  $*"
+}
+
+# Create a new network namespace, and print the pid of a process inside it,
+# so we can refer to it.
+# The process' output must be redirected, otherwise the caller's command
+# substitution would wait for it to finish.
+function new_ns() {
+	unshare --net sleep infinity > /dev/null 2>&1 &
+	echo $!
+}
+
+ROUTER_NS=$(new_ns)
+UPSTREAM_NS=$(new_ns)
+
+# Run a command in the router or upstream namespaces.
+function inr() { nsenter -t "$ROUTER_NS" -n "$@"; }
+function inu() { nsenter -t "$UPSTREAM_NS" -n "$@"; }
+
+CLIENT_IP=10.0.1.2
+
+function setup_network() {
+	ip link set lo up
+	inr ip link set lo up
+	inu ip link set lo up
+
+	# client <-> router.
+	ip link add c0 type veth peer name r0
+	ip link set r0 netns "$ROUTER_NS"
+	ip addr add $CLIENT_IP/24 dev c0
+	ip link set c0 up
+	ip route add default via 10.0.1.1
+	inr ip addr add 10.0.1.1/24 dev r0
+	inr ip link set r0 up
+
+	# router <-> upstream.
+	inr ip link add r1 type veth peer name u0
+	inr ip link set u0 netns "$UPSTREAM_NS"
+	inr ip addr add 10.0.2.1/24 dev r1
+	inr ip link set r1 up
+	inu ip addr add 10.0.2.2/24 dev u0
+	inu ip link set u0 up
+	inu ip route add default via 10.0.2.1
+
+	inr sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
+}
+
+function start_minidns() {
+	cat > $W/zones <<EOF
+upstream.test A 10.0.2.2
+EOF
+	nsenter -t "$UPSTREAM_NS" -n \
+		$W/minidns -addr ":53" -zones $W/zones > $W/minidns.log 2>&1 &
+}
+
+function start_upstream_dnss() {
+	nsenter -t "$UPSTREAM_NS" -n "$W/dnss" -enable_https_to_dns \
+		-dns_upstream "10.0.2.2:53" \
+		-https_key $W/certs/privkey.pem \
+		-https_cert $W/certs/fullchain.pem \
+		-https_server_addr "10.0.2.2:443" \
+		>> $W/upstream-dnss.log 2>&1 &
+	UPSTREAM_DNSS_PID=$!
+}
+
+# Start the dnss we are testing, and the prober.
+function start_client() {
+	local dir=$1
+
+	local url="https://upstream.test/dns-query"
+	if [ "$UPSTREAM" == "ip" ]; then
+		url="https://10.0.2.2/dns-query"
+	fi
+
+	# shellcheck disable=SC2086
+	env $DNSS_ENV "$DNSS_BIN" -v 3 \
+		-enable_dns_to_https \
+		-dns_listen_addr "127.0.0.1:1053" \
+		-monitoring_listen_addr "127.0.0.1:1900" \
+		-fallback_upstream "10.0.2.2:53" \
+		-https_client_cafile $W/certs/fullchain.pem \
+		-https_upstream "$url" \
+		$DNSS_FLAGS \
+		> "$dir/dnss.log" 2>&1 &
+	CLIENT_PID=$!
+
+	$W/dnsprobe -server "127.0.0.1:1053" -interval "$PROBE_INTERVAL" \
+		> "$dir/probe.log" 2>&1 &
+	PROBE_PID=$!
+}
+
+function stop_client() {
+	kill $PROBE_PID $CLIENT_PID
+	wait $PROBE_PID $CLIENT_PID 2> /dev/null || true
+}
+
+# Number of goroutines in the dnss being tested.
+function goroutines() {
+	curl -s "http://127.0.0.1:1900/debug/pprof/goroutine?debug=1" \
+		| head -n 1 | grep -o '[0-9]*$' || echo "?"
+}
+
+# Number of TCP connections to the upstream DoH server (in any state).
+function upstream_conns() {
+	ss -Htan dst 10.0.2.2:443 | wc -l
+}
+
+# Analyze the probe log, looking at probes sent after the given time (in
+# unix ms), and before the cutoff (so we don't use incomplete results).
+# Prints how long it took (in seconds after the given time) to get the first
+# successful response, and to get $STABLE_COUNT successful responses in a
+# row. If they didn't happen, prints "-" instead.
+function analyze() {
+	local log=$1 from=$2 cutoff=$3
+	sort -n -k1,1 "$log" | awk -v from="$from" -v cutoff="$cutoff" \
+		-v stable_count=$STABLE_COUNT '
+		$1 < from || $1 > cutoff { next }
+		{ n++; t[n] = $1; ok[n] = ($3 == "ok") }
+		END {
+			first = "-"; stable = "-"; run = 0
+			for (i = 1; i <= n; i++) {
+				if (!ok[i]) { run = 0; continue }
+				if (first == "-")
+					first = sprintf("%.1f", (t[i] - from) / 1000)
+				run++
+				if (run == stable_count) {
+					stable = sprintf("%.1f",
+						(t[i - stable_count + 1] - from) / 1000)
+					break
+				}
+			}
+			print first, stable
+		}'
+}
+
+# Count the failed probes sent in the given time range.
+function count_failures() {
+	local log=$1 from=$2 to=$3
+	awk -v from="$from" -v to="$to" \
+		'$1 >= from && $1 < to && $3 != "ok" { n++ } END { print n + 0 }' \
+		"$log"
+}
+
+# Wait until dnss has recovered (or we give up), after the given time.
+function wait_for_recovery() {
+	local log=$1 from=$2
+	local deadline=$(( from + (RECOVERY_LIMIT + 30) * 1000 ))
+	while [ "$(now)" -lt $deadline ]; do
+		sleep 1
+		# Leave time for in-flight probes to complete.
+		local stable
+		stable=$(analyze "$log" "$from" $(( $(now) - 2500 )) | cut -d ' ' -f 2)
+		if [ "$stable" != "-" ]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
+#####################################
+# Outages. Each one has a function to start it and one to end it.
+
+function linkdown_start() {
+	ip link set c0 down
+}
+function linkdown_end() {
+	ip link set c0 up
+	ip route replace default via 10.0.1.1
+}
+
+function blackhole_start() {
+	inr ip route add blackhole 10.0.2.2/32
+}
+function blackhole_end() {
+	inr ip route del blackhole 10.0.2.2/32
+}
+
+function ipchange_start() {
+	ip addr del $CLIENT_IP/24 dev c0
+}
+function ipchange_end() {
+	if [ "$CLIENT_IP" == "10.0.1.2" ]; then
+		CLIENT_IP=10.0.1.3
+	else
+		CLIENT_IP=10.0.1.2
+	fi
+	ip addr add $CLIENT_IP/24 dev c0
+	ip route replace default via 10.0.1.1
+}
+
+function upstream-restart_start() {
+	kill $UPSTREAM_DNSS_PID
+	wait $UPSTREAM_DNSS_PID 2> /dev/null || true
+}
+function upstream-restart_end() {
+	start_upstream_dnss
+}
+
+# start-offline is like linkdown, but the outage starts before dnss does.
+function start-offline_start() {
+	linkdown_start
+}
+function start-offline_end() {
+	linkdown_end
+}
+
+# powercut starts like start-offline, and half way through the link comes up
+# but the upstream is blackholed.
+function powercut_start() {
+	linkdown_start
+}
+function powercut_middle() {
+	linkdown_end
+	blackhole_start
+}
+function powercut_end() {
+	blackhole_end
+}
+
+#####################################
+# Run a single scenario, with the given outage duration.
+
+RESULTS=$W/results.txt
+
+function run() {
+	local scenario=$1 duration=$2
+	local dir=$W/runs/$scenario-$duration
+	mkdir -p "$dir"
+
+	log "## $scenario, ${duration}s outage"
+
+	local pre_failures="-" gr_before="-"
+	local outage_start
+	if [ "$scenario" == "start-offline" ] || [ "$scenario" == "powercut" ]
+	then
+		outage_start=$(now)
+		"${scenario}_start"
+		start_client "$dir"
+	else
+		start_client "$dir"
+
+		# Warm up, so there is an established connection to the upstream
+		# before the outage starts.
+		local warmup_start
+		warmup_start=$(now)
+		sleep 5
+		gr_before=$(goroutines)
+		outage_start=$(now)
+		pre_failures=$(count_failures "$dir/probe.log" \
+			$(( warmup_start + 1000 )) "$outage_start")
+
+		log "outage start (conns: $(upstream_conns))"
+		"${scenario}_start"
+	fi
+
+	# Some scenarios have a middle step, half way through the outage.
+	if declare -F "${scenario}_middle" > /dev/null; then
+		sleep $(( duration / 2 ))
+		"${scenario}_middle"
+		log "outage middle step"
+		sleep $(( duration - duration / 2 ))
+	else
+		sleep "$duration"
+	fi
+
+	local outage_end
+	outage_end=$(now)
+	"${scenario}_end"
+	log "outage end (conns: $(upstream_conns))"
+
+	local status="PASS"
+	if ! wait_for_recovery "$dir/probe.log" "$outage_end"; then
+		status="FAIL"
+	fi
+
+	# Leave some time for in-flight queries and connections to finish, so
+	# the goroutine and connection counts are more meaningful.
+	sleep 5
+	local gr_after conns
+	gr_after=$(goroutines)
+	conns=$(upstream_conns)
+
+	local first stable outage_failures
+	read -r first stable < <(analyze "$dir/probe.log" "$outage_end" \
+		$(( $(now) - 2500 )))
+	outage_failures=$(count_failures "$dir/probe.log" \
+		"$outage_start" "$outage_end")
+
+	if [ "$stable" == "-" ] || \
+		[ "$(echo "$stable > $RECOVERY_LIMIT" | bc)" == "1" ]
+	then
+		status="FAIL"
+	fi
+
+	stop_client
+
+	{
+		echo "$scenario $duration $status $first $stable" \
+			"$pre_failures $outage_failures $gr_before $gr_after $conns"
+	} >> $RESULTS
+	log "$status: first ok after ${first}s, stable after ${stable}s"
+}
+
+#####################################
+# Main.
+
+log "## Setting up (testing $DNSS_BIN)"
+setup_network
+start_minidns
+start_upstream_dnss
+sleep 0.5
+
+for scenario in $SCENARIOS; do
+	for duration in $DURATIONS; do
+		run "$scenario" "$duration"
+	done
+done
+
+echo
+echo "## Results (testing $DNSS_BIN)"
+echo "# first/stable: seconds after the outage ended until the first"
+echo "#   successful query, and until $STABLE_COUNT successful in a row."
+echo "# fail pre/during: failed queries before and during the outage."
+echo "# goroutines/conns: before the outage, and at the end."
+echo
+{
+	echo "SCENARIO DURATION RESULT FIRST STABLE FAIL_PRE FAIL_DURING" \
+		"GOROUTINES_PRE GOROUTINES_POST CONNS_POST"
+	cat $RESULTS
+} | column -t
+
+if grep -q FAIL $RESULTS; then
+	exit 1
+fi