Merge branch 'master' of ssh://github.com/majestrate/nntpchan

contrib/backends/srndv2/src/srnd/frontend_http.go

@@ -743,7 +743,7 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 	}
 
 	ref := pr.Reference
-	if len(ref) > 0 {
+	if ref != "" {
 		if ValidMessageID(ref) {
 			if self.daemon.database.HasArticleLocal(ref) {
 				nntp.headers.Set("References", ref)
@@ -784,10 +784,10 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		return
 	}
 
-	subject := pr.Subject
+	subject := strings.TrimSpace(pr.Subject)
 
 	// set subject
-	if len(subject) == 0 {
+	if subject == "" {
 		subject = "None"
 	} else if len(subject) > 256 {
 		// subject too big
@@ -795,28 +795,20 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		return
 	}
 
-	nntp.headers.Set("Subject", subject)
-	if isSage(subject) {
+	nntp.headers.Set("Subject", safeHeader(subject))
+	if isSage(subject) && ref != "" {
 		nntp.headers.Set("X-Sage", "1")
 	}
 
-	name := pr.Name
-
+	name := strings.TrimSpace(pr.Name)
 	var tripcode_privkey []byte
-
-	// set name
-	if len(name) == 0 {
+	// tripcode
+	if idx := strings.IndexByte(name, '#'); idx >= 0 {
+		tripcode_privkey = parseTripcodeSecret(name[idx+1:])
+		name = strings.TrimSpace(name[:idx])
+	}
+	if name == "" {
 		name = "Anonymous"
-	} else {
-		idx := strings.Index(name, "#")
-		// tripcode
-		if idx >= 0 {
-			tripcode_privkey = parseTripcodeSecret(name[idx+1:])
-			name = strings.Trim(name[:idx], "\t ")
-			if name == "" {
-				name = "Anonymous"
-			}
-		}
 	}
 	if len(name) > 128 {
 		// name too long
@@ -829,7 +821,7 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		msgid = genMessageID(pr.Frontend)
 	}
 
-	nntp.headers.Set("From", nntpSanitize(fmt.Sprintf("%s <poster@%s>", name, pr.Frontend)))
+	nntp.headers.Set("From", formatAddress(safeHeader(name), "poster@"+pr.Frontend))
 	nntp.headers.Set("Message-ID", msgid)
 
 	// set message
@@ -842,7 +834,21 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 	}
 
 	if len(cites) > 0 {
-		nntp.headers.Set("Reply-To", strings.Join(cites, " "))
+		if ref == "" && len(cites) == 1 {
+			/*
+				this is workaround for:
+
+				{RFC 5322}
+				If the parent message does not contain
+				a "References:" field but does have an "In-Reply-To:" field
+				containing a single message identifier, then the "References:" field
+				will contain the contents of the parent's "In-Reply-To:" field
+				followed by the contents of the parent's "Message-ID:" field (if
+				any).
+			*/
+			cites = append(cites, "<0>")
+		}
+		nntp.headers.Set("In-Reply-To", strings.Join(cites, " "))
 	}
 
 	// set date

contrib/backends/srndv2/src/srnd/message.go

@@ -135,7 +135,7 @@ func (self *nntpArticle) Reset() {
 	self.boundary = ""
 	self.message = ""
 	if self.attachments != nil {
-		for idx, _ := range self.attachments {
+		for idx := range self.attachments {
 			self.attachments[idx].Reset()
 			self.attachments[idx] = nil
 		}
@@ -156,7 +156,7 @@ func newPlaintextArticle(message, email, subject, name, instance, message_id, ne
 	nntp := &nntpArticle{
 		headers: make(ArticleHeaders),
 	}
-	nntp.headers.Set("From", fmt.Sprintf("%s <%s>", name, email))
+	nntp.headers.Set("From", formatAddress(name, email))
 	nntp.headers.Set("Subject", subject)
 	if isSage(subject) {
 		nntp.headers.Set("X-Sage", "1")
@@ -296,18 +296,30 @@ func (self *nntpArticle) Newsgroup() string {
 
 func (self *nntpArticle) Name() string {
 	const defname = "Anonymous"
+
 	from := strings.TrimSpace(self.headers.Get("From", ""))
 	if from == "" {
 		return defname
 	}
+
 	a, e := mail.ParseAddress(from)
+	var name string
 	if e != nil {
-		return fmt.Sprintf("[Invalid From header: %v]", e)
+		// try older method - some nodes generate non-compliant stuff
+		if i := strings.IndexByte(from, '<'); i > 1 {
+			name = from[:i]
+		} else {
+			return "[Invalid From header]"
+		}
+	} else {
+		name = a.Name
 	}
-	name := strings.TrimSpace(a.Name)
+
+	name = safeHeader(name)
 	if name == "" {
 		return defname
 	}
+
 	return name
 }
 

contrib/backends/srndv2/src/srnd/nntp.go

@@ -914,7 +914,25 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 						for _, model := range models {
 							if model != nil {
 								if err == nil {
-									io.WriteString(dw, fmt.Sprintf("%.6d\t%s\t\"%s\" <%s@%s>\t%s\t%s\t%s\r\n", model.NNTPID(), model.Subject(), model.Name(), model.Name(), model.Frontend(), model.Date(), model.MessageID(), model.Reference()))
+									/*
+										The first 8 fields MUST be the following, in order:
+										     "0" or article number (see below)
+										     Subject header content
+										     From header content
+										     Date header content
+										     Message-ID header content
+										     References header content
+										     :bytes metadata item
+										     :lines metadata item
+									*/
+									fmt.Fprintf(dw,
+										"%.6d\t%s\t\"%s\" <%s@%s>\t%s\t%s\t%s\r\n",
+										model.NNTPID(),
+										safeHeader(model.Subject()),
+										safeHeader(model.Name()), safeHeader(model.Name()), safeHeader(model.Frontend()),
+										safeHeader(model.Date()),
+										safeHeader(model.MessageID()),
+										safeHeader(model.Reference()))
 								}
 							}
 						}

contrib/backends/srndv2/src/srnd/store.go

@@ -547,8 +547,8 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
 			if e != nil {
 				log.Println("failed to read entire message", e)
 			}
-			pw_in.Close()
-			pr_in.Close()
+			pw_in.CloseWithError(e)
+			pr_in.CloseWithError(e)
 		}()
 		r := bufio.NewReader(pr_out)
 		m, e := readMIMEHeader(r)
@@ -579,7 +579,7 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
 			return
 		}
 		writeMIMEHeader(wr, m.Header)
-		read_message_body(m.Body, m.Header, self, wr, false, process)
+		err = read_message_body(m.Body, m.Header, self, wr, false, process)
 	}
 	return
 }
@@ -632,7 +632,7 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 		body = io.TeeReader(body, wr)
 	}
 	boundary, ok := params["boundary"]
-	if ok || content_type == "multipart/mixed" {
+	if strings.HasPrefix(media_type, "multipart/") && ok {
 		partReader := multipart.NewReader(body, boundary)
 		for {
 			part, err := partReader.NextPart()

contrib/backends/srndv2/src/srnd/util.go

@@ -15,6 +15,7 @@ import (
 	"golang.org/x/crypto/ed25519"
 	"io"
 	"log"
+	"mime"
 	"net"
 	"net/http"
 	"net/mail"
@@ -26,6 +27,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 )
 
 func DelFile(fname string) {
@@ -90,6 +92,10 @@ func ValidMessageID(id string) bool {
 		strings.IndexAny(id[1:len(id)-1], "/\\") < 0
 }
 
+func ReservedMessageID(id string) bool {
+	return id == "<0>" || id == "<keepalive@dummy.tld>"
+}
+
 // message id hash
 func HashMessageID(msgid string) string {
 	return fmt.Sprintf("%x", sha1.Sum([]byte(msgid)))
@@ -161,6 +167,133 @@ func nntpSanitize(data string) (ret string) {
 	return ret
 }
 
+var safeHeaderReplacer = strings.NewReplacer(
+	"\t", " ",
+	"\n", string(unicode.ReplacementChar),
+	"\r", string(unicode.ReplacementChar),
+	"\000", string(unicode.ReplacementChar))
+
+// safeHeader replaces dangerous stuff from header,
+// also replaces space with tab for XOVER/OVER output
+func safeHeader(s string) string {
+	return strings.TrimSpace(safeHeaderReplacer.Replace(s))
+}
+
+func isVchar(r rune) bool {
+	// RFC 5234 B.1: VCHAR =  %x21-7E ; visible (printing) characters
+	// RFC 6532 3.2: VCHAR =/ UTF8-non-ascii
+	return (r >= 0x21 && r <= 0x7E) || r >= 0x80
+}
+
+func isAtext(r rune) bool {
+	// RFC 5322: Printable US-ASCII characters not including specials.  Used for atoms.
+	switch r {
+	case '(', ')', '<', '>', '[', ']', ':', ';', '@', '\\', ',', '.', '"':
+		return false
+	}
+	return isVchar(r)
+}
+
+func isWSP(r rune) bool { return r == ' ' || r == '\t' }
+
+func isQtext(r rune) bool {
+	if r == '\\' || r == '"' {
+		return false
+	}
+	return isVchar(r)
+}
+
+func writeQuoted(b *strings.Builder, s string) {
+	b.WriteByte('"')
+	for _, r := range s {
+		if isQtext(r) || isWSP(r) {
+			b.WriteRune(r)
+		} else {
+			b.WriteByte('\\')
+			b.WriteRune(r)
+		}
+	}
+	b.WriteByte('"')
+}
+
+func formatAddress(name, email string) string {
+	// somewhat based on stdlib' mail.Address.String()
+
+	b := &strings.Builder{}
+
+	if name != "" {
+		needsEncoding := false
+		needsQuoting := false
+		for i, r := range name {
+			if r >= 0x80 || (!isWSP(r) && !isVchar(r)) {
+				needsEncoding = true
+				break
+			}
+			if isAtext(r) {
+				continue
+			}
+			if r == ' ' && i > 0 && name[i-1] != ' ' && i < len(name)-1 {
+				// allow spaces but only surrounded by non-spaces
+				// otherwise they will be removed by receiver
+				continue
+			}
+			needsQuoting = true
+		}
+
+		if needsEncoding {
+			// Text in an encoded-word in a display-name must not contain certain
+			// characters like quotes or parentheses (see RFC 2047 section 5.3).
+			// When this is the case encode the name using base64 encoding.
+			if strings.ContainsAny(name, "\"#$%&'(),.:;<>@[]^`{|}~") {
+				b.WriteString(mime.BEncoding.Encode("utf-8", name))
+			} else {
+				b.WriteString(mime.QEncoding.Encode("utf-8", name))
+			}
+		} else if needsQuoting {
+			writeQuoted(b, name)
+		} else {
+			b.WriteString(name)
+		}
+
+		b.WriteByte(' ')
+	}
+
+	at := strings.LastIndex(email, "@")
+	var local, domain string
+	if at >= 0 {
+		local, domain = email[:at], email[at+1:]
+	} else {
+		local = email
+	}
+
+	quoteLocal := false
+	for i, r := range local {
+		if isAtext(r) {
+			// if atom then okay
+			continue
+		}
+		if r == '.' && r > 0 && local[i-1] != '.' && i < len(local)-1 {
+			// dots are okay but only if surrounded by non-dots
+			continue
+		}
+		quoteLocal = true
+		break
+	}
+
+	b.WriteByte('<')
+	if !quoteLocal {
+		b.WriteString(local)
+	} else {
+		writeQuoted(b, local)
+	}
+	b.WriteByte('@')
+	b.WriteString(domain)
+	b.WriteByte('>')
+
+	return b.String()
+}
+
+
 type int64Sorter []int64
 
 func (self int64Sorter) Len() int {
@@ -769,7 +902,7 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 		log.Println("dropping message with invalid mime header, no message-id")
 		_, err = io.Copy(Discard, body)
 		return
-	} else if ValidMessageID(msgid) {
+	} else if ValidMessageID(msgid) && !ReservedMessageID(msgid) {
 		f = daemon.store.CreateFile(msgid)
 	} else {
 		// invalid message-id
@@ -785,9 +918,9 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 	}
 
 	// ask for replies
-	replyTos := strings.Split(hdr.Get("Reply-To"), " ")
+	replyTos := strings.Split(hdr.Get("In-Reply-To"), " ")
 	for _, reply := range replyTos {
-		if ValidMessageID(reply) {
+		if ValidMessageID(reply) && !ReservedMessageID(reply) {
 			if !daemon.store.HasArticle(reply) {
 				go daemon.askForArticle(reply)
 			}
@@ -801,8 +934,8 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 	go func() {
 		var buff [65536]byte
 		writeMIMEHeader(pw, hdr)
-		io.CopyBuffer(pw, body, buff[:])
-		pw.Close()
+		_, e := io.CopyBuffer(pw, body, buff[:])
+		pw.CloseWithError(e)
 	}()
 	err = daemon.store.ProcessMessage(f, pr, daemon.CheckText, hdr.Get("Newsgroups"))
 	pr.Close()