archive/nntpchan
Archived
1
0
Fork 0
Code

add configurable max message sizes

Browse Source
This commit is contained in:
Jeff Becker 2017-04-04 10:31:41 -04:00
parent fa66537f9e
commit 8b5952f66b
7 changed files with 90 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/backends/srndv2/src/srnd/attachment.go
View File

@ -309,6 +309,8 @@ func readAttachmentFromMimePartAndStore(part *multipart.Part, store ArticleStore
// attachment isn't there
// move it into it
err = os.Rename(fpath, att_fpath)
} else {
DelFile(fpath)
}
if err == nil {
// now thumbnail

5
contrib/backends/srndv2/src/srnd/daemon.go
View File

@ -362,6 +362,11 @@ func (self *NNTPDaemon) activeFeeds() (feeds []*feedStatus) {
return
}
func (self *NNTPDaemon) messageSizeLimitFor(newsgroup string) int64 {
// TODO: per newsgroup
return mapGetInt64(self.conf.store, "max_message_size", DefaultMaxMessageSize)
}
func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
log.Println(conf.Name, "persisting in", mode, "mode")
backoff := time.Second

10
contrib/backends/srndv2/src/srnd/frontend_http.go
View File

@ -220,9 +220,7 @@ type httpFrontend struct {
// do we allow this newsgroup?
func (self httpFrontend) AllowNewsgroup(group string) bool {
// XXX: hardcoded nntp prefix
// TODO: make configurable nntp prefix
return strings.HasPrefix(group, "overchan.") && newsgroupValidFormat(group) || group == "ctl" && group != "overchan."
return newsgroupValidFormat(group) || group == "ctl" && !strings.HasSuffix(group, ".")
}
func (self httpFrontend) PostsChan() chan frontendPost {
@ -391,7 +389,11 @@ func (self *httpFrontend) poll() {
if err == nil {
err = writeMIMEHeader(f, msg.Header)
if err == nil {
err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), msg.Body)
body := &io.LimitedReader{
R: msg.Body,
N: self.daemon.messageSizeLimitFor(nntp.Newsgroup()),
}
err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body)
}
}
}

7
contrib/backends/srndv2/src/srnd/message.go
View File

@ -421,7 +421,7 @@ func (self *nntpArticle) WriteBody(wr io.Writer) (err error) {
// verify a signed message's body
// innerHandler must close reader when done
// returns error if one happens while verifying article
func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string][]string, io.Reader)) (err error) {
func verifyMessage(pk, sig string, body *io.LimitedReader, innerHandler func(map[string][]string, io.Reader)) (err error) {
log.Println("unwrapping signed message from", pk)
pk_bytes := unhex(pk)
sig_bytes := unhex(sig)
@ -437,7 +437,10 @@ func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string]
}
hdr_reader.Close()
}(pr)
body = io.TeeReader(body, pw)
body = &io.LimitedReader{
R: io.TeeReader(body, pw),
N: body.N,
}
// copy body 128 bytes at a time
var buff [128]byte
_, err = io.CopyBuffer(h, body, buff[:])

28
contrib/backends/srndv2/src/srnd/nntp.go
View File

@ -526,7 +526,7 @@ func (self *nntpConnection) checkMIMEHeaderNoAuth(daemon *NNTPDaemon, hdr textpr
// store message, unpack attachments, register with daemon, send to daemon for federation
// in that order
func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader) (err error) {
func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
var f io.WriteCloser
msgid := getMessageID(hdr)
if msgid == "" {
@ -721,7 +721,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
}
// store message
err = self.storeMessage(daemon, hdr, msg.Body)
r := &io.LimitedReader{
R: msg.Body,
N: daemon.messageSizeLimitFor(newsgroup),
}
err = self.storeMessage(daemon, hdr, r)
if err == nil {
code = 239
reason = "gotten"
@ -807,7 +811,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
log.Println(self.name, "got reply to", reference, "but we don't have it")
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
}
err = self.storeMessage(daemon, hdr, r)
body := &io.LimitedReader{
R: r,
N: daemon.messageSizeLimitFor(newsgroup),
}
err = self.storeMessage(daemon, hdr, body)
if err == nil {
conn.PrintfLine("235 We got it")
} else {
@ -1162,7 +1170,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
}
}
if success && daemon.database.HasNewsgroup(newsgroup) {
err = self.storeMessage(daemon, hdr, msg.Body)
body := &io.LimitedReader{
R: msg.Body,
N: daemon.messageSizeLimitFor(newsgroup),
}
err = self.storeMessage(daemon, hdr, body)
}
}
}
@ -1173,6 +1185,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
// failed posting
if err != nil {
log.Println(self.name, "failed nntp POST", err)
reason = err.Error()
}
conn.PrintfLine("441 Posting Failed %s", reason)
}
@ -1412,10 +1425,13 @@ func (self *nntpConnection) requestArticle(daemon *NNTPDaemon, conn *textproto.C
}
} else {
// yeh we want it open up a file to store it in
err = self.storeMessage(daemon, hdr, msg.Body)
body := &io.LimitedReader{
R: msg.Body,
N: daemon.messageSizeLimitFor(hdr.Get("Newsgroups")),
}
err = self.storeMessage(daemon, hdr, body)
if err != nil {
log.Println(self.name, "failed to obtain article", err)
// probably an invalid signature or format
daemon.database.BanArticle(msgid, err.Error())
}
}

42
contrib/backends/srndv2/src/srnd/store.go
View File

@ -22,6 +22,14 @@ import (
"strings"
)
var ErrOversizedMessage = errors.New("oversized message")
// ~ 10 MB unbased64'd
const DefaultMaxMessageSize = 1024 * 1024 * 10 * 6
// HARD max message size
const MaxMessageSize = 1024 * 1024 * 1024
type ArticleStore interface {
// full filepath to attachment directory
@ -57,7 +65,7 @@ type ArticleStore interface {
// process body of nntp message, register attachments and the article
// write the body into writer as we go through the body
// does NOT write mime header
ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) error
ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) error
// register this post with the daemon
RegisterPost(nntp NNTPMessage) error
// register signed message
@ -429,7 +437,7 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
return hdr
}
func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) (err error) {
func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) {
err = self.RegisterPost(nntp)
if err == nil {
@ -457,14 +465,23 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
if err == nil {
chnl := make(chan NNTPMessage)
hdr := textproto.MIMEHeader(msg.Header)
err = read_message_body(msg.Body, hdr, nil, nil, true, func(nntp NNTPMessage) {
body := &io.LimitedReader{
R: msg.Body,
N: MaxMessageSize,
}
err = read_message_body(body, hdr, nil, nil, true, func(nntp NNTPMessage) {
c := chnl
// inject pubkey for mod
nntp.Headers().Set("X-PubKey-Ed25519", hdr.Get("X-PubKey-Ed25519"))
c <- nntp
close(c)
})
if err == nil {
nntp = <-chnl
} else {
log.Println("GetMessage() failed to load", msgid, err)
close(chnl)
}
}
}
return
@ -477,7 +494,7 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
// if writer is nil and discardAttachmentBody is true the body is discarded entirely
// if writer is nil and discardAttachmentBody is false the body is loaded into the nntp message
// if the body contains a signed message it unrwarps 1 layer of signing
func read_message_body(body io.Reader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error {
func read_message_body(body *io.LimitedReader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error {
nntp := new(nntpArticle)
nntp.headers = ArticleHeaders(hdr)
content_type := nntp.ContentType()
@ -488,7 +505,10 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
return err
}
if wr != nil && !discardAttachmentBody {
body = io.TeeReader(body, wr)
body = &io.LimitedReader{
R: io.TeeReader(body, wr),
N: body.N,
}
}
boundary, ok := params["boundary"]
if ok || content_type == "multipart/mixed" {
@ -496,7 +516,13 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
for {
part, err := partReader.NextPart()
if err == io.EOF {
if body.N >= 0 {
callback(nntp)
} else {
log.Println("dropping oversized message")
nntp.Reset()
return ErrOversizedMessage
}
return nil
} else if err == nil {
hdr := part.Header
@ -552,7 +578,11 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
// verify message
err = verifyMessage(pk, sig, body, func(h map[string][]string, innerBody io.Reader) {
// handle inner message
err := read_message_body(innerBody, h, store, nil, true, callback)
ir := &io.LimitedReader{
R: innerBody,
N: body.N,
}
err := read_message_body(ir, h, store, nil, true, callback)
if err != nil {
log.Println("error reading inner signed message", err)
}

12
contrib/backends/srndv2/src/srnd/util.go
View File

@ -351,6 +351,18 @@ func mapGetInt(m map[string]string, key string, fallback int) int {
return fallback
}
// get from a map an uint64 given a key or fall back to a default value
func mapGetInt64(m map[string]string, key string, fallback int64) int64 {
val, ok := m[key]
if ok {
i, err := strconv.ParseInt(val, 10, 64)
if err == nil {
return i
}
}
return fallback
}
func isSage(str string) bool {
str = strings.ToLower(str)
return str == "sage" || strings.HasPrefix(str, "sage ")