add configurable max message sizes
This commit is contained in:
parent
fa66537f9e
commit
8b5952f66b
@ -309,6 +309,8 @@ func readAttachmentFromMimePartAndStore(part *multipart.Part, store ArticleStore
|
|||||||
// attachment isn't there
|
// attachment isn't there
|
||||||
// move it into it
|
// move it into it
|
||||||
err = os.Rename(fpath, att_fpath)
|
err = os.Rename(fpath, att_fpath)
|
||||||
|
} else {
|
||||||
|
DelFile(fpath)
|
||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// now thumbnail
|
// now thumbnail
|
||||||
|
@ -362,6 +362,11 @@ func (self *NNTPDaemon) activeFeeds() (feeds []*feedStatus) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (self *NNTPDaemon) messageSizeLimitFor(newsgroup string) int64 {
|
||||||
|
// TODO: per newsgroup
|
||||||
|
return mapGetInt64(self.conf.store, "max_message_size", DefaultMaxMessageSize)
|
||||||
|
}
|
||||||
|
|
||||||
func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
|
func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
|
||||||
log.Println(conf.Name, "persisting in", mode, "mode")
|
log.Println(conf.Name, "persisting in", mode, "mode")
|
||||||
backoff := time.Second
|
backoff := time.Second
|
||||||
|
@ -220,9 +220,7 @@ type httpFrontend struct {
|
|||||||
|
|
||||||
// do we allow this newsgroup?
|
// do we allow this newsgroup?
|
||||||
func (self httpFrontend) AllowNewsgroup(group string) bool {
|
func (self httpFrontend) AllowNewsgroup(group string) bool {
|
||||||
// XXX: hardcoded nntp prefix
|
return newsgroupValidFormat(group) || group == "ctl" && !strings.HasSuffix(group, ".")
|
||||||
// TODO: make configurable nntp prefix
|
|
||||||
return strings.HasPrefix(group, "overchan.") && newsgroupValidFormat(group) || group == "ctl" && group != "overchan."
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self httpFrontend) PostsChan() chan frontendPost {
|
func (self httpFrontend) PostsChan() chan frontendPost {
|
||||||
@ -391,7 +389,11 @@ func (self *httpFrontend) poll() {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
err = writeMIMEHeader(f, msg.Header)
|
err = writeMIMEHeader(f, msg.Header)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), msg.Body)
|
body := &io.LimitedReader{
|
||||||
|
R: msg.Body,
|
||||||
|
N: self.daemon.messageSizeLimitFor(nntp.Newsgroup()),
|
||||||
|
}
|
||||||
|
err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -421,7 +421,7 @@ func (self *nntpArticle) WriteBody(wr io.Writer) (err error) {
|
|||||||
// verify a signed message's body
|
// verify a signed message's body
|
||||||
// innerHandler must close reader when done
|
// innerHandler must close reader when done
|
||||||
// returns error if one happens while verifying article
|
// returns error if one happens while verifying article
|
||||||
func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string][]string, io.Reader)) (err error) {
|
func verifyMessage(pk, sig string, body *io.LimitedReader, innerHandler func(map[string][]string, io.Reader)) (err error) {
|
||||||
log.Println("unwrapping signed message from", pk)
|
log.Println("unwrapping signed message from", pk)
|
||||||
pk_bytes := unhex(pk)
|
pk_bytes := unhex(pk)
|
||||||
sig_bytes := unhex(sig)
|
sig_bytes := unhex(sig)
|
||||||
@ -437,7 +437,10 @@ func verifyMessage(pk, sig string, body io.Reader, innerHandler func(map[string]
|
|||||||
}
|
}
|
||||||
hdr_reader.Close()
|
hdr_reader.Close()
|
||||||
}(pr)
|
}(pr)
|
||||||
body = io.TeeReader(body, pw)
|
body = &io.LimitedReader{
|
||||||
|
R: io.TeeReader(body, pw),
|
||||||
|
N: body.N,
|
||||||
|
}
|
||||||
// copy body 128 bytes at a time
|
// copy body 128 bytes at a time
|
||||||
var buff [128]byte
|
var buff [128]byte
|
||||||
_, err = io.CopyBuffer(h, body, buff[:])
|
_, err = io.CopyBuffer(h, body, buff[:])
|
||||||
|
@ -526,7 +526,7 @@ func (self *nntpConnection) checkMIMEHeaderNoAuth(daemon *NNTPDaemon, hdr textpr
|
|||||||
|
|
||||||
// store message, unpack attachments, register with daemon, send to daemon for federation
|
// store message, unpack attachments, register with daemon, send to daemon for federation
|
||||||
// in that order
|
// in that order
|
||||||
func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader) (err error) {
|
func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
|
||||||
var f io.WriteCloser
|
var f io.WriteCloser
|
||||||
msgid := getMessageID(hdr)
|
msgid := getMessageID(hdr)
|
||||||
if msgid == "" {
|
if msgid == "" {
|
||||||
@ -721,7 +721,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
|
|||||||
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
|
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
|
||||||
}
|
}
|
||||||
// store message
|
// store message
|
||||||
err = self.storeMessage(daemon, hdr, msg.Body)
|
r := &io.LimitedReader{
|
||||||
|
R: msg.Body,
|
||||||
|
N: daemon.messageSizeLimitFor(newsgroup),
|
||||||
|
}
|
||||||
|
err = self.storeMessage(daemon, hdr, r)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
code = 239
|
code = 239
|
||||||
reason = "gotten"
|
reason = "gotten"
|
||||||
@ -807,7 +811,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
|
|||||||
log.Println(self.name, "got reply to", reference, "but we don't have it")
|
log.Println(self.name, "got reply to", reference, "but we don't have it")
|
||||||
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
|
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
|
||||||
}
|
}
|
||||||
err = self.storeMessage(daemon, hdr, r)
|
body := &io.LimitedReader{
|
||||||
|
R: r,
|
||||||
|
N: daemon.messageSizeLimitFor(newsgroup),
|
||||||
|
}
|
||||||
|
err = self.storeMessage(daemon, hdr, body)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
conn.PrintfLine("235 We got it")
|
conn.PrintfLine("235 We got it")
|
||||||
} else {
|
} else {
|
||||||
@ -1162,7 +1170,11 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if success && daemon.database.HasNewsgroup(newsgroup) {
|
if success && daemon.database.HasNewsgroup(newsgroup) {
|
||||||
err = self.storeMessage(daemon, hdr, msg.Body)
|
body := &io.LimitedReader{
|
||||||
|
R: msg.Body,
|
||||||
|
N: daemon.messageSizeLimitFor(newsgroup),
|
||||||
|
}
|
||||||
|
err = self.storeMessage(daemon, hdr, body)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1173,6 +1185,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string
|
|||||||
// failed posting
|
// failed posting
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println(self.name, "failed nntp POST", err)
|
log.Println(self.name, "failed nntp POST", err)
|
||||||
|
reason = err.Error()
|
||||||
}
|
}
|
||||||
conn.PrintfLine("441 Posting Failed %s", reason)
|
conn.PrintfLine("441 Posting Failed %s", reason)
|
||||||
}
|
}
|
||||||
@ -1412,10 +1425,13 @@ func (self *nntpConnection) requestArticle(daemon *NNTPDaemon, conn *textproto.C
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// yeh we want it open up a file to store it in
|
// yeh we want it open up a file to store it in
|
||||||
err = self.storeMessage(daemon, hdr, msg.Body)
|
body := &io.LimitedReader{
|
||||||
|
R: msg.Body,
|
||||||
|
N: daemon.messageSizeLimitFor(hdr.Get("Newsgroups")),
|
||||||
|
}
|
||||||
|
err = self.storeMessage(daemon, hdr, body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println(self.name, "failed to obtain article", err)
|
log.Println(self.name, "failed to obtain article", err)
|
||||||
// probably an invalid signature or format
|
|
||||||
daemon.database.BanArticle(msgid, err.Error())
|
daemon.database.BanArticle(msgid, err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -22,6 +22,14 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var ErrOversizedMessage = errors.New("oversized message")
|
||||||
|
|
||||||
|
// ~ 10 MB unbased64'd
|
||||||
|
const DefaultMaxMessageSize = 1024 * 1024 * 10 * 6
|
||||||
|
|
||||||
|
// HARD max message size
|
||||||
|
const MaxMessageSize = 1024 * 1024 * 1024
|
||||||
|
|
||||||
type ArticleStore interface {
|
type ArticleStore interface {
|
||||||
|
|
||||||
// full filepath to attachment directory
|
// full filepath to attachment directory
|
||||||
@ -57,7 +65,7 @@ type ArticleStore interface {
|
|||||||
// process body of nntp message, register attachments and the article
|
// process body of nntp message, register attachments and the article
|
||||||
// write the body into writer as we go through the body
|
// write the body into writer as we go through the body
|
||||||
// does NOT write mime header
|
// does NOT write mime header
|
||||||
ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) error
|
ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) error
|
||||||
// register this post with the daemon
|
// register this post with the daemon
|
||||||
RegisterPost(nntp NNTPMessage) error
|
RegisterPost(nntp NNTPMessage) error
|
||||||
// register signed message
|
// register signed message
|
||||||
@ -429,7 +437,7 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
|
|||||||
return hdr
|
return hdr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body io.Reader) (err error) {
|
func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
|
||||||
err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) {
|
err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) {
|
||||||
err = self.RegisterPost(nntp)
|
err = self.RegisterPost(nntp)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@ -457,14 +465,23 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
chnl := make(chan NNTPMessage)
|
chnl := make(chan NNTPMessage)
|
||||||
hdr := textproto.MIMEHeader(msg.Header)
|
hdr := textproto.MIMEHeader(msg.Header)
|
||||||
err = read_message_body(msg.Body, hdr, nil, nil, true, func(nntp NNTPMessage) {
|
body := &io.LimitedReader{
|
||||||
|
R: msg.Body,
|
||||||
|
N: MaxMessageSize,
|
||||||
|
}
|
||||||
|
err = read_message_body(body, hdr, nil, nil, true, func(nntp NNTPMessage) {
|
||||||
c := chnl
|
c := chnl
|
||||||
// inject pubkey for mod
|
// inject pubkey for mod
|
||||||
nntp.Headers().Set("X-PubKey-Ed25519", hdr.Get("X-PubKey-Ed25519"))
|
nntp.Headers().Set("X-PubKey-Ed25519", hdr.Get("X-PubKey-Ed25519"))
|
||||||
c <- nntp
|
c <- nntp
|
||||||
close(c)
|
close(c)
|
||||||
})
|
})
|
||||||
|
if err == nil {
|
||||||
nntp = <-chnl
|
nntp = <-chnl
|
||||||
|
} else {
|
||||||
|
log.Println("GetMessage() failed to load", msgid, err)
|
||||||
|
close(chnl)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@ -477,7 +494,7 @@ func (self *articleStore) GetMessage(msgid string) (nntp NNTPMessage) {
|
|||||||
// if writer is nil and discardAttachmentBody is true the body is discarded entirely
|
// if writer is nil and discardAttachmentBody is true the body is discarded entirely
|
||||||
// if writer is nil and discardAttachmentBody is false the body is loaded into the nntp message
|
// if writer is nil and discardAttachmentBody is false the body is loaded into the nntp message
|
||||||
// if the body contains a signed message it unrwarps 1 layer of signing
|
// if the body contains a signed message it unrwarps 1 layer of signing
|
||||||
func read_message_body(body io.Reader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error {
|
func read_message_body(body *io.LimitedReader, hdr map[string][]string, store ArticleStore, wr io.Writer, discardAttachmentBody bool, callback func(NNTPMessage)) error {
|
||||||
nntp := new(nntpArticle)
|
nntp := new(nntpArticle)
|
||||||
nntp.headers = ArticleHeaders(hdr)
|
nntp.headers = ArticleHeaders(hdr)
|
||||||
content_type := nntp.ContentType()
|
content_type := nntp.ContentType()
|
||||||
@ -488,7 +505,10 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if wr != nil && !discardAttachmentBody {
|
if wr != nil && !discardAttachmentBody {
|
||||||
body = io.TeeReader(body, wr)
|
body = &io.LimitedReader{
|
||||||
|
R: io.TeeReader(body, wr),
|
||||||
|
N: body.N,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
boundary, ok := params["boundary"]
|
boundary, ok := params["boundary"]
|
||||||
if ok || content_type == "multipart/mixed" {
|
if ok || content_type == "multipart/mixed" {
|
||||||
@ -496,7 +516,13 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
|
|||||||
for {
|
for {
|
||||||
part, err := partReader.NextPart()
|
part, err := partReader.NextPart()
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
|
if body.N >= 0 {
|
||||||
callback(nntp)
|
callback(nntp)
|
||||||
|
} else {
|
||||||
|
log.Println("dropping oversized message")
|
||||||
|
nntp.Reset()
|
||||||
|
return ErrOversizedMessage
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
} else if err == nil {
|
} else if err == nil {
|
||||||
hdr := part.Header
|
hdr := part.Header
|
||||||
@ -552,7 +578,11 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
|
|||||||
// verify message
|
// verify message
|
||||||
err = verifyMessage(pk, sig, body, func(h map[string][]string, innerBody io.Reader) {
|
err = verifyMessage(pk, sig, body, func(h map[string][]string, innerBody io.Reader) {
|
||||||
// handle inner message
|
// handle inner message
|
||||||
err := read_message_body(innerBody, h, store, nil, true, callback)
|
ir := &io.LimitedReader{
|
||||||
|
R: innerBody,
|
||||||
|
N: body.N,
|
||||||
|
}
|
||||||
|
err := read_message_body(ir, h, store, nil, true, callback)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("error reading inner signed message", err)
|
log.Println("error reading inner signed message", err)
|
||||||
}
|
}
|
||||||
|
@ -351,6 +351,18 @@ func mapGetInt(m map[string]string, key string, fallback int) int {
|
|||||||
return fallback
|
return fallback
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// get from a map an uint64 given a key or fall back to a default value
|
||||||
|
func mapGetInt64(m map[string]string, key string, fallback int64) int64 {
|
||||||
|
val, ok := m[key]
|
||||||
|
if ok {
|
||||||
|
i, err := strconv.ParseInt(val, 10, 64)
|
||||||
|
if err == nil {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
func isSage(str string) bool {
|
func isSage(str string) bool {
|
||||||
str = strings.ToLower(str)
|
str = strings.ToLower(str)
|
||||||
return str == "sage" || strings.HasPrefix(str, "sage ")
|
return str == "sage" || strings.HasPrefix(str, "sage ")
|
||||||
|
Reference in New Issue
Block a user