Archived
1
0

add configurable max message sizes

This commit is contained in:
Jeff Becker 2017-04-04 10:31:41 -04:00
parent fa66537f9e
commit 8b5952f66b
7 changed files with 90 additions and 20 deletions

View File

@ -309,6 +309,8 @@ func readAttachmentFromMimePartAndStore(part *multipart.Part, store ArticleStore
// attachment isn't there // attachment isn't there
// move it into it // move it into it
err = os.Rename(fpath, att_fpath) err = os.Rename(fpath, att_fpath)
} else {
DelFile(fpath)
} }
if err == nil { if err == nil {
// now thumbnail // now thumbnail

View File

@ -362,6 +362,11 @@ func (self *NNTPDaemon) activeFeeds() (feeds []*feedStatus) {
return return
} }
func (self *NNTPDaemon) messageSizeLimitFor(newsgroup string) int64 {
// TODO: per newsgroup
return mapGetInt64(self.conf.store, "max_message_size", DefaultMaxMessageSize)
}
func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) { func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
log.Println(conf.Name, "persisting in", mode, "mode") log.Println(conf.Name, "persisting in", mode, "mode")
backoff := time.Second backoff := time.Second

View File

@ -220,9 +220,7 @@ type httpFrontend struct {
// do we allow this newsgroup? // do we allow this newsgroup?
func (self httpFrontend) AllowNewsgroup(group string) bool { func (self httpFrontend) AllowNewsgroup(group string) bool {
// XXX: hardcoded nntp prefix return newsgroupValidFormat(group) || group == "ctl" && !strings.HasSuffix(group, ".")
// TODO: make configurable nntp prefix
return strings.HasPrefix(group, "overchan.") && newsgroupValidFormat(group) || group == "ctl" && group != "overchan."
} }
func (self httpFrontend) PostsChan() chan frontendPost { func (self httpFrontend) PostsChan() chan frontendPost {
@ -391,7 +389,11 @@ func (self *httpFrontend) poll() {
if err == nil { if err == nil {
err = writeMIMEHeader(f, msg.Header) err = writeMIMEHeader(f, msg.Header)
if err == nil { if err == nil {
err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), msg.Body) body := &io.LimitedReader{
R: msg.Body,
N: self.daemon.messageSizeLimitFor(nntp.Newsgroup()),
}
err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body)
} }
} }
} }

View File

@ -421,7 +421,7 @@ func (self *nntpArticle) WriteBody(wr io.Writer) (err error) {
// verify a signed message's body // verify a signed message's body
// innerHandler must close reader when done // innerHandler must close reader when done
// returns error if one happens while verifying article // returns error if one happens while verifying article
func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string][]string, io.Reader)) (err error) { func verifyMessage(pk, sig string, body *io.LimitedReader, innerHandler func(map[string][]string, io.Reader)) (err error) {
log.Println("unwrapping signed message from", pk) log.Println("unwrapping signed message from", pk)
pk_bytes := unhex(pk) pk_bytes := unhex(pk)
sig_bytes := unhex(sig) sig_bytes := unhex(sig)
@ -437,7 +437,10 @@ func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string]
} }
hdr_reader.Close() hdr_reader.Close()
}(pr) }(pr)
body = io.TeeReader(body, pw) body = &io.LimitedReader{
R: io.TeeReader(body, pw),
N: body.N,
}
// copy body 128 bytes at a time // copy body 128 bytes at a time
var buff [128]byte var buff [128]byte
_, err = io.CopyBuffer(h, body, buff[:]) _, err = io.CopyBuffer(h, body, buff[:])

View File

@ -526,7 +526,7 @@ func (self *nntpConnection) checkMIMEHeaderNoAuth(daemon *NNTPDaemon, hdr textpr
// store message, unpack attachments, register with daemon, send to daemon for federation // store message, unpack attachments, register with daemon, send to daemon for federation
// in that order // in that order
func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader) (err error) { func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
var f io.WriteCloser var f io.WriteCloser
msgid := getMessageID(hdr) msgid := getMessageID(hdr)
if msgid == "" { if msgid == "" {
@ -721,7 +721,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
go daemon.askForArticle(ArticleEntry{reference, newsgroup}) go daemon.askForArticle(ArticleEntry{reference, newsgroup})
} }
// store message // store message
err = self.storeMessage(daemon, hdr, msg.Body) r := &io.LimitedReader{
R: msg.Body,
N: daemon.messageSizeLimitFor(newsgroup),
}
err = self.storeMessage(daemon, hdr, r)
if err == nil { if err == nil {
code = 239 code = 239
reason = "gotten" reason = "gotten"
@ -807,7 +811,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
log.Println(self.name, "got reply to", reference, "but we don't have it") log.Println(self.name, "got reply to", reference, "but we don't have it")
go daemon.askForArticle(ArticleEntry{reference, newsgroup}) go daemon.askForArticle(ArticleEntry{reference, newsgroup})
} }
err = self.storeMessage(daemon, hdr, r) body := &io.LimitedReader{
R: r,
N: daemon.messageSizeLimitFor(newsgroup),
}
err = self.storeMessage(daemon, hdr, body)
if err == nil { if err == nil {
conn.PrintfLine("235 We got it") conn.PrintfLine("235 We got it")
} else { } else {
@ -1162,7 +1170,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
} }
} }
if success && daemon.database.HasNewsgroup(newsgroup) { if success && daemon.database.HasNewsgroup(newsgroup) {
err = self.storeMessage(daemon, hdr, msg.Body) body := &io.LimitedReader{
R: msg.Body,
N: daemon.messageSizeLimitFor(newsgroup),
}
err = self.storeMessage(daemon, hdr, body)
} }
} }
} }
@ -1173,6 +1185,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
// failed posting // failed posting
if err != nil { if err != nil {
log.Println(self.name, "failed nntp POST", err) log.Println(self.name, "failed nntp POST", err)
reason = err.Error()
} }
conn.PrintfLine("441 Posting Failed %s", reason) conn.PrintfLine("441 Posting Failed %s", reason)
} }
@ -1412,10 +1425,13 @@ func (self *nntpConnection) requestArticle(daemon *NNTPDaemon, conn *textproto.C
} }
} else { } else {
// yeh we want it open up a file to store it in // yeh we want it open up a file to store it in
err = self.storeMessage(daemon, hdr, msg.Body) body := &io.LimitedReader{
R: msg.Body,
N: daemon.messageSizeLimitFor(hdr.Get("Newsgroups")),
}
err = self.storeMessage(daemon, hdr, body)
if err != nil { if err != nil {
log.Println(self.name, "failed to obtain article", err) log.Println(self.name, "failed to obtain article", err)
// probably an invalid signature or format
daemon.database.BanArticle(msgid, err.Error()) daemon.database.BanArticle(msgid, err.Error())
} }
} }

View File

@ -22,6 +22,14 @@ import (
"strings" "strings"
) )
var ErrOversizedMessage = errors.New("oversized message")
// ~ 10 MB unbased64'd
const DefaultMaxMessageSize = 1024 * 1024 * 10 * 6
// HARD max message size
const MaxMessageSize = 1024 * 1024 * 1024
type ArticleStore interface { type ArticleStore interface {
// full filepath to attachment directory // full filepath to attachment directory
@ -57,7 +65,7 @@ type ArticleStore interface {
// process body of nntp message, register attachments and the article // process body of nntp message, register attachments and the article
// write the body into writer as we go through the body // write the body into writer as we go through the body
// does NOT write mime header // does NOT write mime header
ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) error ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) error
// register this post with the daemon // register this post with the daemon
RegisterPost(nntp NNTPMessage) error RegisterPost(nntp NNTPMessage) error
// register signed message // register signed message
@ -429,7 +437,7 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
return hdr return hdr
} }
func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) (err error) { func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) { err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) {
err = self.RegisterPost(nntp) err = self.RegisterPost(nntp)
if err == nil { if err == nil {
@ -457,14 +465,23 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
if err == nil { if err == nil {
chnl := make(chan NNTPMessage) chnl := make(chan NNTPMessage)
hdr := textproto.MIMEHeader(msg.Header) hdr := textproto.MIMEHeader(msg.Header)
err = read_message_body(msg.Body, hdr, nil, nil, true, func(nntp NNTPMessage) { body := &io.LimitedReader{
R: msg.Body,
N: MaxMessageSize,
}
err = read_message_body(body, hdr, nil, nil, true, func(nntp NNTPMessage) {
c := chnl c := chnl
// inject pubkey for mod // inject pubkey for mod
nntp.Headers().Set("X-PubKey-Ed25519", hdr.Get("X-PubKey-Ed25519")) nntp.Headers().Set("X-PubKey-Ed25519", hdr.Get("X-PubKey-Ed25519"))
c <- nntp c <- nntp
close(c) close(c)
}) })
if err == nil {
nntp = <-chnl nntp = <-chnl
} else {
log.Println("GetMessage() failed to load", msgid, err)
close(chnl)
}
} }
} }
return return
@ -477,7 +494,7 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
// if writer is nil and discardAttachmentBody is true the body is discarded entirely // if writer is nil and discardAttachmentBody is true the body is discarded entirely
// if writer is nil and discardAttachmentBody is false the body is loaded into the nntp message // if writer is nil and discardAttachmentBody is false the body is loaded into the nntp message
// if the body contains a signed message it unrwarps 1 layer of signing // if the body contains a signed message it unrwarps 1 layer of signing
func read_message_body(body io.Reader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error { func read_message_body(body *io.LimitedReader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error {
nntp := new(nntpArticle) nntp := new(nntpArticle)
nntp.headers = ArticleHeaders(hdr) nntp.headers = ArticleHeaders(hdr)
content_type := nntp.ContentType() content_type := nntp.ContentType()
@ -488,7 +505,10 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
return err return err
} }
if wr != nil && !discardAttachmentBody { if wr != nil && !discardAttachmentBody {
body = io.TeeReader(body, wr) body = &io.LimitedReader{
R: io.TeeReader(body, wr),
N: body.N,
}
} }
boundary, ok := params["boundary"] boundary, ok := params["boundary"]
if ok || content_type == "multipart/mixed" { if ok || content_type == "multipart/mixed" {
@ -496,7 +516,13 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
for { for {
part, err := partReader.NextPart() part, err := partReader.NextPart()
if err == io.EOF { if err == io.EOF {
if body.N >= 0 {
callback(nntp) callback(nntp)
} else {
log.Println("dropping oversized message")
nntp.Reset()
return ErrOversizedMessage
}
return nil return nil
} else if err == nil { } else if err == nil {
hdr := part.Header hdr := part.Header
@ -552,7 +578,11 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
// verify message // verify message
err = verifyMessage(pk, sig, body, func(h map[string][]string, innerBody io.Reader) { err = verifyMessage(pk, sig, body, func(h map[string][]string, innerBody io.Reader) {
// handle inner message // handle inner message
err := read_message_body(innerBody, h, store, nil, true, callback) ir := &io.LimitedReader{
R: innerBody,
N: body.N,
}
err := read_message_body(ir, h, store, nil, true, callback)
if err != nil { if err != nil {
log.Println("error reading inner signed message", err) log.Println("error reading inner signed message", err)
} }

View File

@ -351,6 +351,18 @@ func mapGetInt(m map[string]string, key string, fallback int) int {
return fallback return fallback
} }
// get from a map an uint64 given a key or fall back to a default value
func mapGetInt64(m map[string]string, key string, fallback int64) int64 {
val, ok := m[key]
if ok {
i, err := strconv.ParseInt(val, 10, 64)
if err == nil {
return i
}
}
return fallback
}
func isSage(str string) bool { func isSage(str string) bool {
str = strings.ToLower(str) str = strings.ToLower(str)
return str == "sage" || strings.HasPrefix(str, "sage ") return str == "sage" || strings.HasPrefix(str, "sage ")