git » chasquid » commit 1ecc957

queue: Internationalized Delivery Status Notifications (DSN)

author Alberto Bertogli
2019-01-11 16:49:30 UTC
committer Alberto Bertogli
2019-01-18 23:27:10 UTC
parent abf91eac8d375e03e78e275a30cbbf13374ef3a5
queue: Internationalized Delivery Status Notifications (DSN)

Our non-delivery status notifications are quite simple today, but that
makes it much more difficult to support internationalization and
cross-language reporting.

There is a standard for internationalized DSNs, RFC 6533 (which builds
on top of the structured DSNs from RFC 3464).

This patch changes our DSN messages to be based on those standards, so
it is easier for MUAs to display reports according to the users'
languages preferences.

Note we still use message/rfc822 + 8bit to transmit the message, instead
of message/global, for compatibility reasons. This seems to be more
universally compatible, but the decision might be revisited in the
future. See RFC 5335 (section 4.6 in particular).
internal/queue/dsn.go +72 -11
internal/queue/dsn_test.go +42 -9
test/t-05-null_address/content +3 -2
test/t-05-null_address/expected_dsr +37 -6
test/t-05-null_address/sendmail.cmy +3 -2 
diff --git a/internal/queue/dsn.go b/internal/queue/dsn.go
index 7ee2243..96783be 100644
--- a/internal/queue/dsn.go
+++ b/internal/queue/dsn.go
@@ -2,19 +2,23 @@ package queue
 
 import (
 	"bytes"
+	"net/mail"
 	"text/template"
 	"time"
 )
 
 // Maximum length of the original message to include in the DSN.
+// The receiver of the DSN might have a smaller message size than what we
+// accepted, so we truncate to a value that should be large enough to be
+// useful, but not problematic for modern deployments.
 const maxOrigMsgLen = 256 * 1024
 
 // deliveryStatusNotification creates a delivery status notification (DSN) for
 // the given item, and puts it in the queue.
 //
-// There is a standard, https://tools.ietf.org/html/rfc3464, although most
-// MTAs seem to use a plain email and include an X-Failed-Recipients header.
-// We're going with the latter for now, may extend it to the former later.
+// References:
+// - https://tools.ietf.org/html/rfc3464 (DSN)
+// - https://tools.ietf.org/html/rfc6533 (Internationalized DSN)
 func deliveryStatusNotification(domainFrom string, item *Item) ([]byte, error) {
 	info := dsnInfo{
 		OurDomain:   domainFrom,
@@ -44,11 +48,23 @@ func deliveryStatusNotification(domainFrom string, item *Item) ([]byte, error) {
 		info.OriginalMessage = string(item.Data)
 	}
 
+	info.OriginalMessageID = getMessageID(item.Data)
+
+	info.Boundary = <-newID
+
 	buf := &bytes.Buffer{}
 	err := dsnTemplate.Execute(buf, info)
 	return buf.Bytes(), err
 }
 
+func getMessageID(data []byte) string {
+	msg, err := mail.ReadMessage(bytes.NewBuffer(data))
+	if err != nil {
+		return ""
+	}
+	return msg.Header.Get("Message-ID")
+}
+
 type dsnInfo struct {
 	OurDomain         string
 	Destination       string
@@ -60,35 +76,80 @@ type dsnInfo struct {
 	FailedRecipients  []*Recipient
 	PendingRecipients []*Recipient
 	OriginalMessage   string
+
+	// Message-ID of the original message.
+	OriginalMessageID string
+
+	// MIME boundary to use to form the message.
+	Boundary string
 }
 
-var dsnTemplate = template.Must(template.New("dsn").Parse(
-	`From: Mail Delivery System <postmaster-dsn@{{.OurDomain}}>
+var dsnTemplate = template.Must(
+	template.New("dsn").Parse(
+		`From: Mail Delivery System <postmaster-dsn@{{.OurDomain}}>
 To: <{{.Destination}}>
 Subject: Mail delivery failed: returning message to sender
 Message-ID: <{{.MessageID}}>
 Date: {{.Date}}
+In-Reply-To: {{.OriginalMessageID}}
+References: {{.OriginalMessageID}}
 X-Failed-Recipients: {{range .FailedTo}}{{.}}, {{end}}
 Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+    boundary="{{.Boundary}}"
+
 
-Delivery to the following recipient(s) failed permanently:
+--{{.Boundary}}
+Content-Type: text/plain; charset="utf-8"
+Content-Disposition: inline
+Content-Description: Notification
+Content-Transfer-Encoding: 8bit
+
+Delivery of your message to the following recipient(s) failed permanently:
 
   {{range .FailedTo -}} - {{.}}
   {{- end}}
 
-
------ Technical details -----
-{{range .FailedRecipients}}
+Technical details:
+{{- range .FailedRecipients}}
 - "{{.Address}}" ({{.Type}}) failed permanently with error:
     {{.LastFailureMessage}}
-{{end}}
+{{- end}}
 {{- range .PendingRecipients}}
 - "{{.Address}}" ({{.Type}}) failed repeatedly and timed out, last error:
     {{.LastFailureMessage}}
+{{- end}}
+
+
+--{{.Boundary}}
+Content-Type: message/global-delivery-status
+Content-Description: Delivery Report
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; {{.OurDomain}}
+
+{{range .FailedRecipients -}}
+Original-Recipient: utf-8; {{.OriginalAddress}}
+Final-Recipient: utf-8; {{.Address}}
+Action: failed
+Status: 5.0.0
+Diagnostic-Code: smtp; {{.LastFailureMessage}}
+{{end}}
+{{range .PendingRecipients -}}
+Original-Recipient: utf-8; {{.OriginalAddress}}
+Final-Recipient: utf-8; {{.Address}}
+Action: failed
+Status: 4.0.0
+Diagnostic-Code: smtp; {{.LastFailureMessage}}
 {{end}}
 
------ Original message -----
+--{{.Boundary}}
+Content-Type: message/rfc822
+Content-Description: Undelivered Message
+Content-Transfer-Encoding: 8bit
 
 {{.OriginalMessage}}
 
+--{{.Boundary}}--
 `))
diff --git a/internal/queue/dsn_test.go b/internal/queue/dsn_test.go
index d4429fd..8e1cff4 100644
--- a/internal/queue/dsn_test.go
+++ b/internal/queue/dsn_test.go
@@ -12,12 +12,12 @@ func TestDSN(t *testing.T) {
 		Message: Message{
 			ID:   <-newID,
 			From: "from@from.org",
-			To:   []string{"toto@africa.org", "negra@sosa.org"},
+			To:   []string{"ñaca@africa.org", "negra@sosa.org"},
 			Rcpt: []*Recipient{
 				{"poe@rcpt", Recipient_EMAIL, Recipient_FAILED,
-					"oh! horror!", "toto@africa.org"},
+					"oh! horror!", "ñaca@africa.org"},
 				{"newman@rcpt", Recipient_EMAIL, Recipient_PENDING,
-					"oh! the humanity!", "toto@africa.org"},
+					"oh! the humanity!", "ñaca@africa.org"},
 				{"ant@rcpt", Recipient_EMAIL, Recipient_SENT,
 					"", "negra@sosa.org"},
 			},
@@ -42,27 +42,60 @@ To: <from@from.org>
 Subject: Mail delivery failed: returning message to sender
 Message-ID: <chasquid-dsn-???????????@dsnDomain>
 Date: *
-X-Failed-Recipients: toto@africa.org, 
+In-Reply-To: *
+References: *
+X-Failed-Recipients: ñaca@africa.org, 
 Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+    boundary="???????????"
 
-Delivery to the following recipient(s) failed permanently:
 
-  - toto@africa.org
+--???????????
+Content-Type: text/plain; charset="utf-8"
+Content-Disposition: inline
+Content-Description: Notification
+Content-Transfer-Encoding: 8bit
 
+Delivery of your message to the following recipient(s) failed permanently:
 
------ Technical details -----
+  - ñaca@africa.org
 
+Technical details:
 - "poe@rcpt" (EMAIL) failed permanently with error:
     oh! horror!
-
 - "newman@rcpt" (EMAIL) failed repeatedly and timed out, last error:
     oh! the humanity!
 
 
------ Original message -----
+--???????????
+Content-Type: message/global-delivery-status
+Content-Description: Delivery Report
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; dsnDomain
+
+Original-Recipient: utf-8; ñaca@africa.org
+Final-Recipient: utf-8; poe@rcpt
+Action: failed
+Status: 5.0.0
+Diagnostic-Code: smtp; oh! horror!
+
+Original-Recipient: utf-8; ñaca@africa.org
+Final-Recipient: utf-8; newman@rcpt
+Action: failed
+Status: 4.0.0
+Diagnostic-Code: smtp; oh! the humanity!
+
+
+--???????????
+Content-Type: message/rfc822
+Content-Description: Undelivered Message
+Content-Transfer-Encoding: 8bit
 
 data ñaca
 
+--???????????--
 `
 
 // flexibleEq compares two strings, supporting wildcards.
diff --git a/test/t-05-null_address/content b/test/t-05-null_address/content
index 911f0dc..d4e37cc 100644
--- a/test/t-05-null_address/content
+++ b/test/t-05-null_address/content
@@ -1,5 +1,6 @@
-From: Mailer daemon <somewhere@horns.com>
+From: Mailer daemon <somewhere@báratro>
 Subject: I've come to haunt you
+Message-ID: <booooo>
 
-Muahahahaha
+Ñañañañaña!
 
diff --git a/test/t-05-null_address/expected_dsr b/test/t-05-null_address/expected_dsr
index c5e83c2..b95611e 100644
--- a/test/t-05-null_address/expected_dsr
+++ b/test/t-05-null_address/expected_dsr
@@ -4,21 +4,49 @@ To: <user@testserver>
 Subject: Mail delivery failed: returning message to sender
 Message-ID: *
 Date: *
+In-Reply-To: <booooo>
+References: <booooo>
 X-Failed-Recipients: fail@testserver, 
 Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+    boundary="???????????"
 
-Delivery to the following recipient(s) failed permanently:
 
-  - fail@testserver
+--???????????
+Content-Type: text/plain; charset="utf-8"
+Content-Disposition: inline
+Content-Description: Notification
+Content-Transfer-Encoding: 8bit
 
+Delivery of your message to the following recipient(s) failed permanently:
 
------ Technical details -----
+  - fail@testserver
 
+Technical details:
 - "false" (PIPE) failed permanently with error:
     exit status 1
 
 
------ Original message -----
+--???????????
+Content-Type: message/global-delivery-status
+Content-Description: Delivery Report
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; testserver
+
+Original-Recipient: utf-8; fail@testserver
+Final-Recipient: utf-8; false
+Action: failed
+Status: 5.0.0
+Diagnostic-Code: smtp; exit status 1
+
+
+
+--???????????
+Content-Type: message/rfc822
+Content-Description: Undelivered Message
+Content-Transfer-Encoding: 8bit
 
 Received: from localhost
 	by testserver (chasquid) with ESMTPSA
@@ -26,9 +54,12 @@ Received: from localhost
 	(over *
 	; *
 Date: *
-From: Mailer daemon <somewhere@horns.com>
+From: Mailer daemon <somewhere@báratro>
 Subject: I've come to haunt you
+Message-Id: <booooo>
+
+Ñañañañaña!
 
-Muahahahaha
 
 
+--???????????--
diff --git a/test/t-05-null_address/sendmail.cmy b/test/t-05-null_address/sendmail.cmy
index b413e8d..d3f33d8 100644
--- a/test/t-05-null_address/sendmail.cmy
+++ b/test/t-05-null_address/sendmail.cmy
@@ -10,10 +10,11 @@ c -> RCPT TO: user@testserver
 c <~ 250
 c -> DATA
 c <~ 354
-c -> From: Mailer daemon <somewhere@horns.com>
+c -> From: Mailer daemon <somewhere@báratro>
 c -> Subject: I've come to haunt you
+c -> Message-ID: <booooo>
 c -> 
-c -> Muahahahaha
+c -> Ñañañañaña!
 c -> 
 c -> 
 c -> .