Merge branch 'master' of ssh://github.com/majestrate/nntpchan

contrib/backends/srndv2/src/srnd/config.go

@@ -5,18 +5,45 @@
 package srnd
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/base32"
 	"fmt"
 	"github.com/majestrate/configparser"
 	"github.com/majestrate/nacl"
+	"io/ioutil"
 	"log"
 	"net"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 )
 
+type FilterConfig struct {
+	globalFilters []*regexp.Regexp
+}
+
+func (fc *FilterConfig) LoadFile(fname string) (err error) {
+	var data []byte
+	data, err = ioutil.ReadFile(fname)
+	if err == nil {
+		r := bytes.NewReader(data)
+		sc := bufio.NewScanner(r)
+		for sc.Scan() {
+			txt := sc.Text()
+			idx := strings.Index(txt, "#")
+			if idx >= 0 {
+				txt = txt[:idx]
+			}
+			fc.globalFilters = append(fc.globalFilters, regexp.MustCompile(txt))
+		}
+
+	}
+	return
+}
+
 type FeedConfig struct {
 	policy           FeedPolicy
 	quarks           map[string]string
@@ -70,6 +97,7 @@ type SRNdConfig struct {
 	pprof         *ProfilingConfig
 	hooks         []*HookConfig
 	inboundPolicy *FeedPolicy
+	filter        FilterConfig
 }
 
 // check for config files
@@ -417,6 +445,15 @@ func ReadConfig() *SRNdConfig {
 		}
 	}
 
+	filterFile := "filters.txt"
+
+	if CheckFile(filterFile) {
+		err = sconf.filter.LoadFile(filterFile)
+		if err != nil {
+			log.Fatalf("failed to load %s: %s", filterFile, err)
+		}
+		log.Printf("loaded %d filters", len(sconf.filter.globalFilters))
+	}
 	return &sconf
 }
 

contrib/backends/srndv2/src/srnd/daemon.go

@@ -131,6 +131,16 @@ type NNTPDaemon struct {
 	article_lifetime  time.Duration
 }
 
+// return true if text passes all checks and is okay for posting
+func (self *NNTPDaemon) CheckText(text string) bool {
+	for _, re := range self.conf.filter.globalFilters {
+		if re.MatchString(text) {
+			return false
+		}
+	}
+	return true
+}
+
 func (self NNTPDaemon) End() {
 	if self.listener != nil {
 		self.listener.Close()
@@ -478,6 +488,12 @@ func (self *NNTPDaemon) syncPull(proxy_type, proxy_addr, remote_addr string) {
 	}
 }
 
+func (self *NNTPDaemon) ExpireAll() {
+	log.Println("expiring all orphans")
+	self.expire = createExpirationCore(self.database, self.store, self.informHooks)
+	self.expire.ExpireOrphans()
+}
+
 // run daemon
 func (self *NNTPDaemon) Run() {
 

contrib/backends/srndv2/src/srnd/expiration.go

@@ -77,6 +77,7 @@ func (self expire) ExpireThread(group, rootMsgid string) {
 		}
 	}
 	self.database.DeleteThread(rootMsgid)
+	self.database.DeleteArticle(rootMsgid)
 	self.expireCache(group, rootMsgid, rootMsgid)
 }
 

contrib/backends/srndv2/src/srnd/frontend_http.go

@@ -393,7 +393,7 @@ func (self *httpFrontend) poll() {
 								R: msg.Body,
 								N: self.daemon.messageSizeLimitFor(nntp.Newsgroup()),
 							}
-							err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body)
+							err = self.daemon.store.ProcessMessageBody(f, textproto.MIMEHeader(msg.Header), body, self.daemon.CheckText)
 						}
 					}
 				}
@@ -803,6 +803,11 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er
 		return
 	}
 
+	if !self.daemon.CheckText(pr.Message) {
+		e(errors.New("spam"))
+		return
+	}
+
 	if len(pr.Frontend) == 0 {
 		// :-DDD
 		pr.Frontend = "mongo.db.is.web.scale"

contrib/backends/srndv2/src/srnd/nntp.go

@@ -565,7 +565,7 @@ func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEH
 	// now store attachments and article
 	err = writeMIMEHeader(f, hdr)
 	if err == nil {
-		err = daemon.store.ProcessMessageBody(f, hdr, body)
+		err = daemon.store.ProcessMessageBody(f, hdr, body, daemon.CheckText)
 		if err == nil {
 			// tell daemon
 			daemon.loadFromInfeed(msgid)

contrib/backends/srndv2/src/srnd/store.go

@@ -65,7 +65,7 @@ type ArticleStore interface {
 	// process body of nntp message, register attachments and the article
 	// write the body into writer as we go through the body
 	// does NOT write mime header
-	ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) error
+	ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader, spamfilter func(string) bool) error
 	// register this post with the daemon
 	RegisterPost(nntp NNTPMessage) error
 	// register signed message
@@ -428,7 +428,9 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
 			var msg *mail.Message
 			msg, err = readMIMEHeader(r)
 			f.Close()
-			hdr = textproto.MIMEHeader(msg.Header)
+			if msg != nil {
+				hdr = textproto.MIMEHeader(msg.Header)
+			}
 		}
 		if err != nil {
 			log.Println("failed to load article headers for", messageID, err)
@@ -437,8 +439,12 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
 	return hdr
 }
 
-func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader) (err error) {
+func (self *articleStore) ProcessMessageBody(wr io.Writer, hdr textproto.MIMEHeader, body *io.LimitedReader, spamfilter func(string) bool) (err error) {
 	err = read_message_body(body, hdr, self, wr, false, func(nntp NNTPMessage) {
+		if !spamfilter(nntp.Message()) {
+			err = errors.New("spam message")
+			return
+		}
 		err = self.RegisterPost(nntp)
 		if err == nil {
 			pk := hdr.Get("X-PubKey-Ed25519")

contrib/backends/srndv2/src/srnd/varnish_cache.go

@@ -66,6 +66,8 @@ func (self *VarnishCache) invalidateUkko() {
 	self.invalidate(fmt.Sprintf("%s%so/", self.varnish_url, self.prefix))
 	// TODO: this is lazy af
 	self.RegenFrontPage()
+	// TODO: this is also lazy af
+	self.invalidate(fmt.Sprintf("%s%shistory.html", self.varnish_url, self.prefix))
 }
 
 func (self *VarnishCache) pollRegen() {

contrib/backends/srndv2/srnd.go

@@ -94,6 +94,9 @@ func main() {
 					} else {
 						fmt.Fprintf(os.Stdout, "usage: %s tool mod [[add|del] pubkey]|[do modactiongoeshere]\n", os.Args[0])
 					}
+				} else if tool == "expire" {
+					daemon.Setup()
+					daemon.ExpireAll()
 				} else if tool == "rethumb" {
 					if len(os.Args) >= 4 {
 						threads := runtime.NumCPU()

contrib/templates/placebo/graph_history.mustache

@@ -8,7 +8,8 @@ template parameters:
 <html>
   <head>
     <meta charset="utf-8"></meta>
-    <link rel="stylesheet" href="{{prefix}}static/site.css" />
+<!--  <link rel="stylesheet" href="{{prefix}}static/site.css" /> -->
+    <link rel="stylesheet" href="{{prefix}}static/krane.css" />
     <link rel="stylesheet" href="{{prefix}}static/user.css" />
     <title>{{#i18n.Translations}}{{post_history_title}}{{/i18n.Translations}}</title>
   </head>