git » chasquid » commit b0011f5

diff --git a/Makefile b/Makefile
index 17463aa..aa779ba 100644
--- a/Makefile
+++ b/Makefile
@@ -39,6 +39,7 @@ dovecot-auth-cli:
 test:
 	go test ${GOFLAGS} ./...
 	setsid -w ./test/run.sh
+	setsid -w ./test/stress.sh
 	setsid -w ./cmd/chasquid-util/test.sh
 	setsid -w ./cmd/mda-lmtp/test.sh
 	setsid -w ./cmd/dovecot-auth-cli/test.sh
diff --git a/test/.gitignore b/test/.gitignore
index 8495b0c..3f21709 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -2,4 +2,6 @@
 # Ignore the user databases - we create them each time.
 t-*/config/**/users
 t-*/?/**/users
+stress-*/config/**/users
+stress-*/?/**/users
 
diff --git a/test/stress-01-load/config/chasquid.conf b/test/stress-01-load/config/chasquid.conf
new file mode 100644
index 0000000..9a35641
--- /dev/null
+++ b/test/stress-01-load/config/chasquid.conf
@@ -0,0 +1,15 @@
+hostname: "testserver"
+
+smtp_address: ":1025"
+submission_address: ":1587"
+submission_over_tls_address: ":1465"
+monitoring_address: ":1099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "%to%"
+
+data_dir: "../.data"
+mail_log_path: "../.logs/mail_log"
+
+suffix_separators: "+-"
+drop_characters: "._"
diff --git a/test/stress-01-load/config/domains/testserver/aliases b/test/stress-01-load/config/domains/testserver/aliases
new file mode 100644
index 0000000..c24415f
--- /dev/null
+++ b/test/stress-01-load/config/domains/testserver/aliases
@@ -0,0 +1,4 @@
+
+null: | true
+fail: | false
+
diff --git a/test/stress-01-load/hosts b/test/stress-01-load/hosts
new file mode 100644
index 0000000..2b9b623
--- /dev/null
+++ b/test/stress-01-load/hosts
@@ -0,0 +1 @@
+testserver localhost
diff --git a/test/stress-01-load/run.sh b/test/stress-01-load/run.sh
new file mode 100755
index 0000000..1b2600e
--- /dev/null
+++ b/test/stress-01-load/run.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/../util/lib.sh
+
+init
+
+generate_certs_for testserver
+add_user user@testserver secretpassword
+
+# Note we run the server with minimal logging, to avoid generating very large
+# log files, which are not very useful anyway.
+mkdir -p .logs
+chasquid -v=-1 --logfile=.logs/chasquid.log --config_dir=config &
+wait_until_ready 1025
+
+echo Peak RAM: `chasquid_ram_peak`
+
+if ! loadgen -logtime -addr=localhost:1025 -run_for=3s -noop; then
+	fail
+fi
+
+echo Peak RAM: `chasquid_ram_peak`
+
+if ! loadgen -logtime -addr=localhost:1025 -run_for=3s; then
+	fail
+fi
+
+echo Peak RAM: `chasquid_ram_peak`
+
+success
diff --git a/test/stress-02-connections/config/chasquid.conf b/test/stress-02-connections/config/chasquid.conf
new file mode 100644
index 0000000..9a35641
--- /dev/null
+++ b/test/stress-02-connections/config/chasquid.conf
@@ -0,0 +1,15 @@
+hostname: "testserver"
+
+smtp_address: ":1025"
+submission_address: ":1587"
+submission_over_tls_address: ":1465"
+monitoring_address: ":1099"
+
+mail_delivery_agent_bin: "test-mda"
+mail_delivery_agent_args: "%to%"
+
+data_dir: "../.data"
+mail_log_path: "../.logs/mail_log"
+
+suffix_separators: "+-"
+drop_characters: "._"
diff --git a/test/stress-02-connections/hosts b/test/stress-02-connections/hosts
new file mode 100644
index 0000000..2b9b623
--- /dev/null
+++ b/test/stress-02-connections/hosts
@@ -0,0 +1 @@
+testserver localhost
diff --git a/test/stress-02-connections/run.sh b/test/stress-02-connections/run.sh
new file mode 100755
index 0000000..1fb4bbd
--- /dev/null
+++ b/test/stress-02-connections/run.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/../util/lib.sh
+
+init
+
+generate_certs_for testserver
+add_user user@testserver secretpassword
+
+# Note we run the server with minimal logging, to avoid generating very large
+# log files, which are not very useful anyway.
+mkdir -p .logs
+chasquid -v=-1 --logfile=.logs/chasquid.log --config_dir=config &
+wait_until_ready 1025
+
+echo Peak RAM: `chasquid_ram_peak`
+
+# Set connection count to (max open files) - (leeway).
+# We set the leeway to account for file descriptors opened by the runtime and
+# listeners; 20 should be enough for now.
+# Cap it to 2000, as otherwise it can be problematic due to port availability.
+COUNT=$(( `ulimit -n` - 20 ))
+if [ $COUNT -gt 2000 ]; then
+	COUNT=2000
+fi
+
+if ! conngen -logtime -addr=localhost:1025 -count=$COUNT; then
+	tail -n 1 .logs/chasquid.log
+	fail
+fi
+
+echo Peak RAM: `chasquid_ram_peak`
+
+success
diff --git a/test/stress.sh b/test/stress.sh
new file mode 100755
index 0000000..1a8016f
--- /dev/null
+++ b/test/stress.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+. $(dirname ${0})/util/lib.sh
+
+init
+
+FAILED=0
+
+for i in stress-*; do
+	echo $i ...
+	setsid -w $i/run.sh
+	FAILED=$(( $FAILED + $? ))
+	echo
+done
+
+exit $FAILED
diff --git a/test/util/conngen.go b/test/util/conngen.go
new file mode 100644
index 0000000..8a98979
--- /dev/null
+++ b/test/util/conngen.go
@@ -0,0 +1,109 @@
+// +build ignore
+
+// SMTP connection generator, for testing purposes.
+package main
+
+import (
+	"flag"
+	"net"
+	"net/http"
+	"net/smtp"
+	"time"
+
+	"golang.org/x/net/trace"
+
+	_ "net/http/pprof"
+
+	"blitiri.com.ar/go/log"
+)
+
+var (
+	addr = flag.String("addr", "",
+		"server address")
+	httpAddr = flag.String("http_addr", "localhost:8011",
+		"monitoring HTTP server listening address")
+	wait = flag.Bool("wait", false,
+		"don't exit after --run_for has lapsed")
+	count = flag.Int("count", 1000,
+		"how many connections to open")
+)
+
+var (
+	host string
+	exit bool
+)
+
+func main() {
+	var err error
+
+	flag.Parse()
+	log.Init()
+
+	host, _, err = net.SplitHostPort(*addr)
+	if err != nil {
+		log.Fatalf("failed to split --addr=%q: %v", *addr, err)
+	}
+
+	if *wait {
+		go http.ListenAndServe(*httpAddr, nil)
+		log.Infof("monitoring address: http://%v/debug/requests?fam=one&b=11",
+			*httpAddr)
+	}
+
+	log.Infof("creating %d simultaneous connections", *count)
+	conns := []*C{}
+	for i := 0; i < *count; i++ {
+		c, err := newC()
+		if err != nil {
+			log.Fatalf("failed to connect #%d: %v", i, err)
+		}
+
+		conns = append(conns, c)
+
+		if i%200 == 0 {
+			log.Infof("  ... %d connections", i)
+
+		}
+	}
+	log.Infof("done, created %d simultaneous connections", *count)
+
+	if *wait {
+		for {
+			time.Sleep(24 * time.Hour)
+		}
+	}
+}
+
+type C struct {
+	tr trace.Trace
+	n  net.Conn
+	s  *smtp.Client
+}
+
+func newC() (*C, error) {
+	tr := trace.New("conn", *addr)
+
+	conn, err := net.Dial("tcp", *addr)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := smtp.NewClient(conn, host)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	err = client.Hello(host)
+	if err != nil {
+		return nil, err
+	}
+
+	return &C{tr: tr, n: conn, s: client}, nil
+}
+
+func (c *C) close() {
+	c.tr.Finish()
+	c.s.Close()
+	c.n.Close()
+}
diff --git a/test/util/lib.sh b/test/util/lib.sh
index 563424b..4fe2ece 100644
--- a/test/util/lib.sh
+++ b/test/util/lib.sh
@@ -24,23 +24,21 @@ function init() {
 	trap "kill 0" EXIT
 }
 
-function generate_cert() {
-	go run ${UTILDIR}/generate_cert.go "$@"
-}
-
 function chasquid() {
 	if [ "${COVER_DIR}" != "" ]; then
 		chasquid_cover "$@"
 		return
 	fi
 
+	( cd ${TBASE}/../../; go build ${RACE} . )
+
 	# HOSTALIASES: so we "fake" hostnames.
 	# PATH: so chasquid can call test-mda without path issues.
 	# MDA_DIR: so our test-mda knows where to deliver emails.
 	HOSTALIASES=${TBASE}/hosts \
 	PATH=${UTILDIR}:${PATH} \
 	MDA_DIR=${TBASE}/.mail \
-		go run ${RACE} ${TBASE}/../../chasquid.go "$@"
+		${TBASE}/../../chasquid "$@"
 }
 
 function chasquid_cover() {
@@ -96,6 +94,18 @@ function chamuyero() {
 	${UTILDIR}/chamuyero "$@"
 }
 
+function generate_cert() {
+	go run ${UTILDIR}/generate_cert.go "$@"
+}
+
+function loadgen() {
+	go run ${UTILDIR}/loadgen.go "$@"
+}
+
+function conngen() {
+	go run ${UTILDIR}/conngen.go "$@"
+}
+
 function success() {
 	echo success
 }
@@ -145,3 +155,11 @@ function skip_if_python_is_too_old() {
 		skip "python3 >= 3.5 not available"
 	fi
 }
+
+function chasquid_ram_peak() {
+	# Find the pid of the daemon, which we expect is running on the
+	# background somewhere within our current session.
+	SERVER_PID=`pgrep -s 0 -x chasquid`
+
+	echo $( cat /proc/$SERVER_PID/status | grep VmHWM | cut -d ':' -f 2- )
+}
diff --git a/test/util/loadgen.go b/test/util/loadgen.go
new file mode 100644
index 0000000..ca5c0ec
--- /dev/null
+++ b/test/util/loadgen.go
@@ -0,0 +1,195 @@
+// +build ignore
+
+// SMTP load generator, for testing purposes.
+package main
+
+import (
+	"flag"
+	"net"
+	"net/http"
+	"runtime"
+	"sync"
+	"time"
+
+	_ "net/http/pprof"
+
+	"golang.org/x/net/trace"
+
+	"blitiri.com.ar/go/chasquid/internal/smtp"
+	"blitiri.com.ar/go/log"
+)
+
+var (
+	addr = flag.String("addr", "",
+		"server address")
+	httpAddr = flag.String("http_addr", "localhost:8011",
+		"monitoring HTTP server listening address")
+	parallel = flag.Int("parallel", 0,
+		"how many sending loops to run in parallel")
+	runFor = flag.Duration("run_for", 0,
+		"how long to run for (0 = forever)")
+	wait = flag.Bool("wait", false,
+		"don't exit after --run_for has lapsed")
+	noop = flag.Bool("noop", false,
+		"don't send an email, just connect and run a NOOP")
+)
+
+var (
+	host string
+	exit bool
+
+	globalCount   int64 = 0
+	globalRuntime time.Duration
+	globalMu      = &sync.Mutex{}
+)
+
+func main() {
+	var err error
+
+	flag.Parse()
+	log.Init()
+
+	host, _, err = net.SplitHostPort(*addr)
+	if err != nil {
+		log.Fatalf("failed to split --addr=%q: %v", *addr, err)
+	}
+
+	if *wait {
+		go http.ListenAndServe(*httpAddr, nil)
+		log.Infof("monitoring address: http://%v/debug/requests?fam=one&b=11",
+			*httpAddr)
+	}
+
+	if *parallel == 0 {
+		*parallel = runtime.GOMAXPROCS(0)
+	}
+
+	lt := "full"
+	if *noop {
+		lt = "noop"
+	}
+
+	log.Infof("launching %d %s sending loops in parallel", *parallel, lt)
+	for i := 0; i < *parallel; i++ {
+		go serial(i)
+	}
+
+	var totalCount int64
+	var totalRuntime time.Duration
+	start := time.Now()
+	for range time.Tick(1 * time.Second) {
+		globalMu.Lock()
+		totalCount += globalCount
+		totalRuntime += globalRuntime
+		count := globalCount
+		runtime := globalRuntime
+		globalCount = 0
+		globalRuntime = 0
+		globalMu.Unlock()
+
+		if count == 0 {
+			log.Infof("0 ops")
+		} else {
+			log.Infof("%d ops, %v /op", count,
+				time.Duration(runtime.Nanoseconds()/count).Truncate(time.Microsecond))
+		}
+
+		if *runFor > 0 && time.Since(start) > *runFor {
+			exit = true
+			break
+		}
+	}
+
+	end := time.Now()
+	window := end.Sub(start)
+	log.Infof("total: %d ops, %v wall, %v run",
+		totalCount,
+		window.Truncate(time.Millisecond),
+		totalRuntime.Truncate(time.Millisecond))
+
+	avgLat := time.Duration(totalRuntime.Nanoseconds() / totalCount)
+	log.Infof("avg: %v /op, %.0f ops/s",
+		avgLat.Truncate(time.Microsecond),
+		float64(totalCount)/window.Seconds(),
+	)
+
+	if *wait {
+		for {
+			time.Sleep(24 * time.Hour)
+		}
+	}
+}
+
+func serial(id int) {
+	var count int64
+	start := time.Now()
+	for {
+		count += 1
+		err := one()
+		if err != nil {
+			log.Fatalf("%v", err)
+		}
+
+		if count == 5 {
+			globalMu.Lock()
+			globalCount += count
+			globalRuntime += time.Since(start)
+			globalMu.Unlock()
+			count = 0
+			start = time.Now()
+
+			if exit {
+				return
+			}
+		}
+	}
+}
+
+func one() error {
+	tr := trace.New("one", *addr)
+	defer tr.Finish()
+
+	conn, err := net.Dial("tcp", *addr)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client, err := smtp.NewClient(conn, host)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	if *noop {
+		err = client.Noop()
+		if err != nil {
+			return err
+		}
+	} else {
+		err = client.MailAndRcpt("test@test", "null@testserver")
+		if err != nil {
+			return err
+		}
+
+		w, err := client.Data()
+		if err != nil {
+			return err
+		}
+		_, err = w.Write(body)
+		if err != nil {
+			return err
+		}
+		err = w.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+var body = []byte(`Subject: Load test
+
+This is the body of the load test email.
+`)