add configurable max message sizes

contrib/backends/srndv2/src/srnd/attachment.go

@@ -309,6 +309,8 @@ func readAttachmentFromMimePartAndStore(part *multipart.Part, store ArticleStore
 		// attachment isn't there
 		// move it into it
 		err = os.Rename(fpath, att_fpath)
+	} else {
+		DelFile(fpath)
 	}
 	if err == nil {
 		// now thumbnail

contrib/backends/srndv2/src/srnd/daemon.go

@@ -362,6 +362,11 @@ func (self *NNTPDaemon) activeFeeds() (feeds []*feedStatus) {
 	return
 }
 
+func (self *NNTPDaemon) messageSizeLimitFor(newsgroup string) int64 {
+	// TODO: per newsgroup
+	return mapGetInt64(self.conf.store, "max_message_size", DefaultMaxMessageSize)
+}
+
 func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
 	log.Println(conf.Name, "persisting in", mode, "mode")
 	backoff := time.Second

contrib/backends/srndv2/src/srnd/frontend_http.go

@@ -220,9 +220,7 @@ type httpFrontend struct {
 
 // do we allow this newsgroup?
 func (self httpFrontend) AllowNewsgroup(group string) bool {
-	// XXX: hardcoded nntp prefix
+	return newsgroupValidFormat(group) || group == "ctl" && !strings.HasSuffix(group, ".")
-	// TODO: make configurable nntp prefix
-	return strings.HasPrefix(group, "overchan.") && newsgroupValidFormat(group) || group == "ctl" && group != "overchan."
 }
 
 func (self httpFrontend) PostsChan() chan frontendPost {
@@ -391,7 +389,11 @@ func (self *httpFrontend) poll() {
 					if err == nil {
 						err = writeMIMEHeader(f, msg.Header)
 						if err == nil {
-							err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), msg.Body)
+							body := &io.LimitedReader{
+								R: msg.Body,
+								N: self.daemon.messageSizeLimitFor(nntp.Newsgroup()),
+							}
+							err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body)
 						}
 					}
 				}

contrib/backends/srndv2/src/srnd/message.go

@@ -421,7 +421,7 @@ func (self *nntpArticle) WriteBody(wr io.Writer) (err error) {
 // verify a signed message's body
 // innerHandler must close reader when done
 // returns error if one happens while verifying article
-func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string][]string, io.Reader)) (err error) {
+func verifyMessage(pk, sig string, body *io.LimitedReader, innerHandler func(map[string][]string, io.Reader)) (err error) {
 	log.Println("unwrapping signed message from", pk)
 	pk_bytes := unhex(pk)
 	sig_bytes := unhex(sig)
@@ -437,7 +437,10 @@ func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string]
 		}
 		hdr_reader.Close()
 	}(pr)
-	body = io.TeeReader(body, pw)
+	body = &io.LimitedReader{
+		R: io.TeeReader(body, pw),
+		N: body.N,
+	}
 	// copy body 128 bytes at a time
 	var buff [128]byte
 	_, err = io.CopyBuffer(h, body, buff[:])

contrib/backends/srndv2/src/srnd/nntp.go

@@ -526,7 +526,7 @@ func (self *nntpConnection) checkMIMEHeaderNoAuth(daemon *NNTPDaemon, hdr textpr
 
 // store message, unpack attachments, register with daemon, send to daemon for federation
 // in that order
-func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader) (err error) {
+func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
 	var f io.WriteCloser
 	msgid := getMessageID(hdr)
 	if msgid == "" {
@@ -721,7 +721,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 							go daemon.askForArticle(ArticleEntry{reference, newsgroup})
 						}
 						// store message
-						err = self.storeMessage(daemon, hdr, msg.Body)
+						r := &io.LimitedReader{
+							R: msg.Body,
+							N: daemon.messageSizeLimitFor(newsgroup),
+						}
+						err = self.storeMessage(daemon, hdr, r)
 						if err == nil {
 							code = 239
 							reason = "gotten"
@@ -807,7 +811,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 									log.Println(self.name, "got reply to", reference, "but we don't have it")
 									go daemon.askForArticle(ArticleEntry{reference, newsgroup})
 								}
-								err = self.storeMessage(daemon, hdr, r)
+								body := &io.LimitedReader{
+									R: r,
+									N: daemon.messageSizeLimitFor(newsgroup),
+								}
+								err = self.storeMessage(daemon, hdr, body)
 								if err == nil {
 									conn.PrintfLine("235 We got it")
 								} else {
@@ -1162,7 +1170,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 								}
 							}
 							if success && daemon.database.HasNewsgroup(newsgroup) {
-								err = self.storeMessage(daemon, hdr, msg.Body)
+								body := &io.LimitedReader{
+									R: msg.Body,
+									N: daemon.messageSizeLimitFor(newsgroup),
+								}
+								err = self.storeMessage(daemon, hdr, body)
 							}
 						}
 					}
@@ -1173,6 +1185,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
 						// failed posting
 						if err != nil {
 							log.Println(self.name, "failed nntp POST", err)
+							reason = err.Error()
 						}
 						conn.PrintfLine("441 Posting Failed %s", reason)
 					}
@@ -1412,10 +1425,13 @@ func (self *nntpConnection) requestArticle(daemon *NNTPDaemon, conn *textproto.C
 					}
 				} else {
 					// yeh we want it open up a file to store it in
-					err = self.storeMessage(daemon, hdr, msg.Body)
+					body := &io.LimitedReader{
+						R: msg.Body,
+						N: daemon.messageSizeLimitFor(hdr.Get("Newsgroups")),
+					}
+					err = self.storeMessage(daemon, hdr, body)
 					if err != nil {
 						log.Println(self.name, "failed to obtain article", err)
-						// probably an invalid signature or format
 						daemon.database.BanArticle(msgid, err.Error())
 					}
 				}

contrib/backends/srndv2/src/srnd/store.go

@@ -22,6 +22,14 @@ import (
 	"strings"
 )
 
+var ErrOversizedMessage = errors.New("oversized message")
+
+// ~ 10 MB unbased64'd
+const DefaultMaxMessageSize = 1024 * 1024 * 10 * 6
+
+// HARD max message size
+const MaxMessageSize = 1024 * 1024 * 1024
+
 type ArticleStore interface {
 
 	// full filepath to attachment directory
@@ -57,7 +65,7 @@ type ArticleStore interface {
 	// process body of nntp message, register attachments and the article
 	// write the body into writer as we go through the body
 	// does NOT write mime header
-	ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) error
+	ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) error
 	// register this post with the daemon
 	RegisterPost(nntp NNTPMessage) error
 	// register signed message
@@ -429,7 +437,7 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
 	return hdr
 }
 
-func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) (err error) {
+func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
 	err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) {
 		err = self.RegisterPost(nntp)
 		if err == nil {
@@ -457,14 +465,23 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
 		if err == nil {
 			chnl := make(chan NNTPMessage)
 			hdr := textproto.MIMEHeader(msg.Header)
-			err = read_message_body(msg.Body, hdr, nil, nil, true, func(nntp NNTPMessage) {
+			body := &io.LimitedReader{
+				R: msg.Body,
+				N: MaxMessageSize,
+			}
+			err = read_message_body(body, hdr, nil, nil, true, func(nntp NNTPMessage) {
 				c := chnl
 				// inject pubkey for mod
 				nntp.Headers().Set("X-PubKey-Ed25519", hdr.Get("X-PubKey-Ed25519"))
 				c <- nntp
 				close(c)
 			})
-			nntp = <-chnl
+			if err == nil {
+				nntp = <-chnl
+			} else {
+				log.Println("GetMessage() failed to load", msgid, err)
+				close(chnl)
+			}
 		}
 	}
 	return
@@ -477,7 +494,7 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
 // if writer is nil and discardAttachmentBody is true the body is discarded entirely
 // if writer is nil and discardAttachmentBody is false the body is loaded into the nntp message
 // if the body contains a signed message it unrwarps 1 layer of signing
-func read_message_body(body io.Reader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error {
+func read_message_body(body *io.LimitedReader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error {
 	nntp := new(nntpArticle)
 	nntp.headers = ArticleHeaders(hdr)
 	content_type := nntp.ContentType()
@@ -488,7 +505,10 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 		return err
 	}
 	if wr != nil && !discardAttachmentBody {
-		body = io.TeeReader(body, wr)
+		body = &io.LimitedReader{
+			R: io.TeeReader(body, wr),
+			N: body.N,
+		}
 	}
 	boundary, ok := params["boundary"]
 	if ok || content_type == "multipart/mixed" {
@@ -496,7 +516,13 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 		for {
 			part, err := partReader.NextPart()
 			if err == io.EOF {
-				callback(nntp)
+				if body.N >= 0 {
+					callback(nntp)
+				} else {
+					log.Println("dropping oversized message")
+					nntp.Reset()
+					return ErrOversizedMessage
+				}
 				return nil
 			} else if err == nil {
 				hdr := part.Header
@@ -552,7 +578,11 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 		// verify message
 		err = verifyMessage(pk, sig, body, func(h map[string][]string, innerBody io.Reader) {
 			// handle inner message
-			err := read_message_body(innerBody, h, store, nil, true, callback)
+			ir := &io.LimitedReader{
+				R: innerBody,
+				N: body.N,
+			}
+			err := read_message_body(ir, h, store, nil, true, callback)
 			if err != nil {
 				log.Println("error reading inner signed message", err)
 			}

contrib/backends/srndv2/src/srnd/util.go

@@ -351,6 +351,18 @@ func mapGetInt(m map[string]string, key string, fallback int) int {
 	return fallback
 }
 
+// get from a map an uint64 given a key or fall back to a default value
+func mapGetInt64(m map[string]string, key string, fallback int64) int64 {
+	val, ok := m[key]
+	if ok {
+		i, err := strconv.ParseInt(val, 10, 64)
+		if err == nil {
+			return i
+		}
+	}
+	return fallback
+}
+
 func isSage(str string) bool {
 	str = strings.ToLower(str)
 	return str == "sage" || strings.HasPrefix(str, "sage ")