Merge branch 'master' of ssh://github.com/majestrate/nntpchan

contrib/backends/srndv2/src/srnd/daemon.go

@@ -535,9 +535,17 @@ func (self *NNTPDaemon) ExpireAll() {
 	self.expire.ExpireOrphans()
 }
 
+func (self *NNTPDaemon) MarkSpam(msgid string) {
+	if ValidMessageID(msgid) {
+		err := self.mod.MarkSpam(msgid)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+}
+
 // run daemon
 func (self *NNTPDaemon) Run() {
-	self.spamFilter.Configure(self.conf.spamconf)
 	self.bind_addr = self.conf.daemon["bind"]
 
 	listener, err := net.Listen("tcp", self.bind_addr)
@@ -1144,7 +1152,10 @@ func (self *NNTPDaemon) Setup() {
 		self.frontend = NewHTTPFrontend(self, self.cache, self.conf.frontend, self.conf.worker["url"])
 	}
 	
+	self.spamFilter.Configure(self.conf.spamconf)
+
 	self.mod = &modEngine{
+		//spam:     &self.spamFilter,
 		store:    self.store,
 		database: self.database,
 		regen:    self.frontend.RegenOnModEvent,

contrib/backends/srndv2/src/srnd/frontend_http.go

@@ -1405,6 +1405,7 @@ func (self *httpFrontend) Mainloop() {
 	m.Path("/mod/feeds").HandlerFunc(self.modui.ServeModPage).Methods("GET")
 	m.Path("/mod/keygen").HandlerFunc(self.modui.HandleKeyGen).Methods("GET")
 	m.Path("/mod/login").HandlerFunc(self.modui.HandleLogin).Methods("POST")
+	m.Path("/mod/spam").HandlerFunc(self.modui.HandlePostSpam).Methods("POST")
 	m.Path("/mod/del/{article_hash}").HandlerFunc(self.modui.HandleDeletePost).Methods("GET")
 	m.Path("/mod/ban/{address}").HandlerFunc(self.modui.HandleBanAddress).Methods("GET")
 	m.Path("/mod/unban/{address}").HandlerFunc(self.modui.HandleUnbanAddress).Methods("GET")

contrib/backends/srndv2/src/srnd/mod.go

@@ -44,6 +44,10 @@ type ModUI interface {
 	HandleKeyGen(wr http.ResponseWriter, r *http.Request)
 	// handle admin command
 	HandleAdminCommand(wr http.ResponseWriter, r *http.Request)
+	// handle mark a post as spam
+	HandlePostSpam(wr http.ResponseWriter, r *http.Request)
+
+	
 	// get outbound message channel
 	MessageChan() chan NNTPMessage
 }
@@ -57,6 +61,8 @@ const ModStick = ModAction("overchan-stick")
 const ModLock = ModAction("overchan-lock")
 const ModHide = ModAction("overchan-hide")
 const ModSage = ModAction("overchan-sage")
+const ModSpam = ModAction("spam")
+const ModHam = ModAction("ham")
 const ModDeleteAlt = ModAction("delete")
 
 type ModEvent interface {
@@ -81,11 +87,15 @@ func (self simpleModEvent) String() string {
 }
 
 func (self simpleModEvent) Action() ModAction {
-	switch strings.Split(string(self), " ")[0] {
+	switch strings.ToLower(strings.Split(string(self), " ")[0]) {
 	case "delete":
 		return ModDelete
 	case "overchan-inet-ban":
 		return ModInetBan
+	case "spam":
+		return ModSpam
+	case "ham":
+		return ModHam
 	}
 	return ""
 }
@@ -122,6 +132,11 @@ func overchanInetBan(encAddr, key string, expire int64) ModEvent {
 	return simpleModEvent(fmt.Sprintf("overchan-inet-ban %s:%s:%d", encAddr, key, expire))
 }
 
+// create a mark as spam event
+func modMarkSpam(msgid string) ModEvent {
+	return simpleModEvent(fmt.Sprintf("spam %s", msgid))
+}
+
 // moderation message
 // wraps multiple mod events
 // is turned into an NNTPMessage later
@@ -171,6 +186,8 @@ type ModEngine interface {
 	HandleMessage(msgid string)
 	// delete post of a poster
 	DeletePost(msgid string) error
+	// mark message as spam
+	MarkSpam(msgid string) error
 	// ban a cidr
 	BanAddress(cidr string) error
 	// do we allow this public key to delete this message-id ?
@@ -190,9 +207,24 @@ type ModEngine interface {
 type modEngine struct {
 	database Database
 	store    ArticleStore
+	spam     *SpamFilter
 	regen    RegenFunc
 }
 
+func (self *modEngine) MarkSpam(msgid string) (err error) {
+	if self.spam == nil {
+		err = self.store.MarkSpam(msgid)
+	} else {
+		var f io.ReadCloser
+		f, err = self.store.OpenMessage(msgid)
+		if err == nil {
+			err = self.spam.MarkSpam(f)
+			f.Close()
+		}
+	}
+	return
+}
+
 func (self *modEngine) LoadMessage(msgid string) NNTPMessage {
 	return self.store.GetMessage(msgid)
 }
@@ -394,6 +426,10 @@ func (mod *modEngine) Do(ev ModEvent) {
 		} else {
 			log.Printf("invalid overchan-inet-ban: target=%s", target)
 		}
+	} else if action == ModSpam {
+		if ValidMessageID(target) {
+			mod.MarkSpam(target)
+		}
 	} else if action == ModHide {
 		// TODO: implement
 	} else if action == ModLock {
@@ -434,6 +470,11 @@ func (mod *modEngine) Execute(ev ModEvent, pubkey string) {
 			mod.Do(ev)
 		}
 		return
+	case ModSpam:
+		if mod.AllowJanitor(pubkey) {
+			mod.Do(ev)
+		}
+		return
 	case ModHide:
 	case ModLock:
 	case ModSage:
@@ -442,6 +483,7 @@ func (mod *modEngine) Execute(ev ModEvent, pubkey string) {
 		if mod.AllowJanitor(pubkey) {
 			mod.Do(ev)
 		}
+		return
 	default:
 		// invalid action
 	}

contrib/backends/srndv2/src/srnd/mod_http.go

@@ -35,8 +35,8 @@ type httpModUI struct {
 	cache            CacheHandler
 }
 
-func createHttpModUI(frontend *httpFrontend) httpModUI {
+func createHttpModUI(frontend *httpFrontend) *httpModUI {
-	return httpModUI{frontend.regenAll, frontend.Regen, frontend.RegenerateBoard, frontend.deleteThreadMarkup, frontend.deleteBoardMarkup, make(chan NNTPMessage), frontend.daemon, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/", frontend.GetCacheHandler()}
+	return &httpModUI{frontend.regenAll, frontend.Regen, frontend.RegenerateBoard, frontend.deleteThreadMarkup, frontend.deleteBoardMarkup, make(chan NNTPMessage), frontend.daemon, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/", frontend.GetCacheHandler()}
 
 }
 
@@ -44,7 +44,7 @@ func extractGroup(param map[string]interface{}) string {
 	return extractParam(param, "newsgroup")
 }
 
-func (self httpModUI) getAdminFunc(funcname string) AdminFunc {
+func (self *httpModUI) getAdminFunc(funcname string) AdminFunc {
 	if funcname == "template.reload" {
 		return func(param map[string]interface{}) (interface{}, error) {
 			tname, ok := param["template"]
@@ -390,7 +390,7 @@ func (self httpModUI) HandleAdminCommand(wr http.ResponseWriter, r *http.Request
 	}, wr, r)
 }
 
-func (self httpModUI) CheckPubkey(pubkey, scope string) (bool, error) {
+func (self *httpModUI) CheckPubkey(pubkey, scope string) (bool, error) {
 	is_admin, err := self.daemon.database.CheckAdminPubkey(pubkey)
 	if is_admin {
 		// admin can do what they want
@@ -413,7 +413,7 @@ func (self httpModUI) CheckPubkey(pubkey, scope string) (bool, error) {
 	return false, err
 }
 
-func (self httpModUI) CheckKey(privkey, scope string) (bool, error) {
+func (self *httpModUI) CheckKey(privkey, scope string) (bool, error) {
 	privkey_bytes, err := hex.DecodeString(privkey)
 	if err == nil {
 		pk, _ := naclSeedToKeyPair(privkey_bytes)
@@ -424,17 +424,17 @@ func (self httpModUI) CheckKey(privkey, scope string) (bool, error) {
 	return false, err
 }
 
-func (self httpModUI) MessageChan() chan NNTPMessage {
+func (self *httpModUI) MessageChan() chan NNTPMessage {
 	return self.modMessageChan
 }
 
-func (self httpModUI) getSession(r *http.Request) *sessions.Session {
+func (self *httpModUI) getSession(r *http.Request) *sessions.Session {
 	s, _ := self.store.Get(r, "nntpchan-mod")
 	return s
 }
 
 // get the session's private key as bytes or nil if we don't have it
-func (self httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
+func (self *httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
 	s := self.getSession(r)
 	k, ok := s.Values["privkey"]
 	if ok {
@@ -451,7 +451,7 @@ func (self httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
 
 // returns true if the session is okay for a scope
 // otherwise redirect to login page
-func (self httpModUI) checkSession(r *http.Request, scope string) bool {
+func (self *httpModUI) checkSession(r *http.Request, scope string) bool {
 	s := self.getSession(r)
 	k, ok := s.Values["privkey"]
 	if ok {
@@ -464,11 +464,11 @@ func (self httpModUI) checkSession(r *http.Request, scope string) bool {
 	return false
 }
 
-func (self httpModUI) writeTemplate(wr http.ResponseWriter, r *http.Request, name string) {
+func (self *httpModUI) writeTemplate(wr http.ResponseWriter, r *http.Request, name string) {
 	self.writeTemplateParam(wr, r, name, nil)
 }
 
-func (self httpModUI) writeTemplateParam(wr http.ResponseWriter, r *http.Request, name string, param map[string]interface{}) {
+func (self *httpModUI) writeTemplateParam(wr http.ResponseWriter, r *http.Request, name string, param map[string]interface{}) {
 	if param == nil {
 		param = make(map[string]interface{})
 	}
@@ -481,7 +481,7 @@ func (self httpModUI) writeTemplateParam(wr http.ResponseWriter, r *http.Request
 
 // do a function as authenticated
 // pass in the request path to the handler
-func (self httpModUI) asAuthed(scope string, handler func(string), wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) asAuthed(scope string, handler func(string), wr http.ResponseWriter, r *http.Request) {
 	if self.checkSession(r, scope) {
 		handler(r.URL.Path)
 	} else {
@@ -490,7 +490,7 @@ func (self httpModUI) asAuthed(scope string, handler func(string), wr http.Respo
 }
 
 // do stuff to a certain message if with have it and are authed
-func (self httpModUI) asAuthedWithMessage(scope string, handler func(ArticleEntry, *http.Request) map[string]interface{}, wr http.ResponseWriter, req *http.Request) {
+func (self *httpModUI) asAuthedWithMessage(scope string, handler func(ArticleEntry, *http.Request) map[string]interface{}, wr http.ResponseWriter, req *http.Request) {
 	self.asAuthed(scope, func(path string) {
 		// get the long hash
 		if strings.Count(path, "/") > 2 {
@@ -523,13 +523,47 @@ func (self httpModUI) asAuthedWithMessage(scope string, handler func(ArticleEntr
 	}, wr, req)
 }
 
-func (self httpModUI) HandleAddPubkey(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandlePostSpam(wr http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		wr.WriteHeader(405)
+		return
+	}
+	resp := make(map[string]interface{})
+	self.asAuthed("spam", func(path string) {
+		var mm ModMessage
+		var err error
+		keys := strings.Split(r.FormValue("spam"), ",")
+		for _, k := range keys {
+			k =  strings.TrimSpace(k)
+			go self.daemon.MarkSpam(k)
+			mm = append(mm, modMarkSpam(k)) 
+		}
+		privkey_bytes := self.getSessionPrivkeyBytes(r)
+		if privkey_bytes == nil {
+			// this should not happen
+			log.Println("failed to get privkey bytes from session")
+			resp["error"] = "failed to get private key from session. wtf?"
+		} else {
+			// wrap and sign
+			nntp := wrapModMessage(mm)
+			nntp, err = signArticle(nntp, privkey_bytes)
+			if err == nil {
+				// federate
+				self.modMessageChan <- nntp
+			}
+			resp["error"] = err
+		}
+	}, wr, r)
+	json.NewEncoder(wr).Encode(resp)
 }
 
-func (self httpModUI) HandleDelPubkey(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleAddPubkey(wr http.ResponseWriter, r *http.Request) {
 }
 
-func (self httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleDelPubkey(wr http.ResponseWriter, r *http.Request) {
+}
+
+func (self *httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request) {
 	self.asAuthed("ban", func(path string) {
 		// extract the ip address
 		// TODO: ip ranges and prefix detection
@@ -559,7 +593,7 @@ func (self httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request
 }
 
 // handle ban logic
-func (self httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[string]interface{} {
+func (self *httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[string]interface{} {
 	// get the article headers
 	resp := make(map[string]interface{})
 	msgid := msg.MessageID()
@@ -623,7 +657,7 @@ func (self httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[st
 	return resp
 }
 
-func (self httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[string]interface{} {
+func (self *httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[string]interface{} {
 	var mm ModMessage
 	resp := make(map[string]interface{})
 	msgid := msg.MessageID()
@@ -672,16 +706,16 @@ func (self httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[st
 }
 
 // ban the address of a poster
-func (self httpModUI) HandleBanAddress(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleBanAddress(wr http.ResponseWriter, r *http.Request) {
 	self.asAuthedWithMessage("ban", self.handleBanAddress, wr, r)
 }
 
 // delete a post
-func (self httpModUI) HandleDeletePost(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleDeletePost(wr http.ResponseWriter, r *http.Request) {
 	self.asAuthedWithMessage("login", self.handleDeletePost, wr, r)
 }
 
-func (self httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
 	privkey := r.FormValue("privkey")
 	msg := "failed login: "
 	if len(privkey) == 0 {
@@ -702,13 +736,13 @@ func (self httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
 	self.writeTemplateParam(wr, r, "modlogin_result", map[string]interface{}{"message": msg, csrf.TemplateTag: csrf.TemplateField(r)})
 }
 
-func (self httpModUI) HandleKeyGen(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) HandleKeyGen(wr http.ResponseWriter, r *http.Request) {
 	pk, sk := newNaclSignKeypair()
 	tripcode := makeTripcode(pk)
 	self.writeTemplateParam(wr, r, "keygen", map[string]interface{}{"public": pk, "secret": sk, "tripcode": tripcode})
 }
 
-func (self httpModUI) ServeModPage(wr http.ResponseWriter, r *http.Request) {
+func (self *httpModUI) ServeModPage(wr http.ResponseWriter, r *http.Request) {
 	if self.checkSession(r, "login") {
 		wr.Header().Set("X-CSRF-Token", csrf.Token(r))
 		// we are logged in

contrib/backends/srndv2/src/srnd/spam.go

@@ -34,20 +34,46 @@ type SpamResult struct {
 	IsSpam bool
 }
 
+// feed spam subsystem a spam post
+func (sp *SpamFilter) MarkSpam(msg io.Reader) (err error) {
+	var buf [65636]byte
+
+	var u *user.User
+	u, err = user.Current()
+	if err != nil {
+		return
+	}
+	var conn *net.TCPConn
+	conn, err = sp.openConn()
+	if err != nil {
+		return
+	}
+	defer conn.Close()
+	fmt.Fprintf(conn, "TELL SPAMC/1.5\r\nUser: %s\r\nMessage-class: spam\r\nSet: local\r\n\r\n", u.Username)
+	io.CopyBuffer(conn, msg, buf[:])
+	conn.CloseWrite()
+	r := bufio.NewReader(conn)
+	io.Copy(Discard, r)
+	return
+}
+
+func (sp *SpamFilter) openConn() (*net.TCPConn, error) {
+	addr, err := net.ResolveTCPAddr("tcp", sp.addr)
+	if err != nil {
+		return nil, err
+	}
+	return net.DialTCP("tcp", nil, addr)
+}
+
 func (sp *SpamFilter) Rewrite(msg io.Reader, out io.WriteCloser, group string) (result SpamResult) {
 	var buff [65636]byte
 	if !sp.Enabled(group) {
 		result.Err = ErrSpamFilterNotEnabled
 		return
 	}
-	var addr *net.TCPAddr
-	var c *net.TCPConn
 	var u *user.User
-	addr, result.Err = net.ResolveTCPAddr("tcp", sp.addr)
+	var c *net.TCPConn
-	if result.Err != nil {
+	c, result.Err = sp.openConn()
-		return
-	}
-	c, result.Err = net.DialTCP("tcp", nil, addr)
 	if result.Err != nil {
 		return
 	}

contrib/backends/srndv2/src/srnd/store.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -115,6 +116,7 @@ type articleStore struct {
 	identify_path string
 	placeholder   string
 	spamdir       string
+	hamdir        string
 	compression   bool
 	compWriter    *gzip.Writer
 	spamd         *SpamFilter
@@ -136,6 +138,8 @@ func createArticleStore(config map[string]string, thumbConfig *ThumbnailConfig,
 		compression:   config["compression"] == "1",
 		spamd:         spamd,
 		spamdir:       filepath.Join(config["store_dir"], "spam"),
+		hamdir: filepath.Join(config["store_dir"], "ham"),
+			
 		thumbnails:    thumbConfig,
 	}
 	store.Init()
@@ -781,7 +785,11 @@ func (self *articleStore) AcceptTempArticle(msgid string) (err error) {
 			} else {
 				err = os.Rename(temp, store)
 			}
+		} else {
+			err = fmt.Errorf("no such inbound article %s", temp)
 		}
+	} else {
+		err = fmt.Errorf("invalid message id %s", msgid)
 	}
 	return
 }

contrib/backends/srndv2/src/srnd/util.go

@@ -789,15 +789,12 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
 	} else {
 		log.Println("error processing message body", err)
 	}
-	if err != nil {
+
 	// clean up
 	if ValidMessageID(msgid) {
 		fname := daemon.store.GetFilenameTemp(msgid)
-			log.Println("clean up", fname)
 		DelFile(fname)
 	}
-		log.Println("error processing message", err)
-	}
 	return
 }
 

contrib/static/overchan.js

@@ -9,6 +9,69 @@ var ready = function() {
   for(var idx = 0; idx < _onreadyfuncs.length; idx++) _onreadyfuncs[idx]();
 };
 
+var nntpchan_mod_mark_spam = function(longhash) {
+  var elem = document.getElementById(longhash);
+  if(!elem) return;
+  elem.dataset.spam = "yes";
+  elem.innerText = "spam";
+};
+
+var nntpchan_mod_commit_spam = function(elem) {
+  var formdata = new FormData();
+  var posts = document.getElementsByClassName("post");
+  var spams = [];
+  for (var idx = 0; idx < posts.length; idx ++)
+  {
+    if(posts[idx].dataset.spam == "yes")
+    {
+      spams.push(posts[idx].dataset.msgid);
+    }
+  }
+  formdata.set("spam", spams.join(","));
+  var jax = new XMLHttpRequest();
+  jax.onreadystatechange = function() {
+    if(jax.readyState == 4)
+    {
+      if(jax.status == 200)
+      {
+        
+        var ajax = new XMLHttpRequest();
+        ajax.onreadystatechange = function() {
+        if(ajax.readyState == 4)
+          {
+            if(ajax.status == 200)
+            {
+              // success (?)
+              var j = JSON.parse(ajax.responseText);
+              if(j.error)
+              {
+                elem.innerText = "could not mark as spam: " + j.error;
+              }
+              else
+              {
+                elem.innerText = "OK: marked as spam";
+              }
+            }
+            else 
+            {
+              elem.innerText = "post not marked as spam on server: "+ ajax.statusText;
+            }
+          }
+        };
+        ajax.open("POST", "/mod/spam")
+        ajax.setRequestHeader("X-CSRF-Token", jax.getResponseHeader("X-CSRF-Token"));
+        ajax.send(formdata);
+      }
+      else
+      {
+        elem.innerText = "failed to moderate, not logged in";
+      }
+    }
+  };
+  jax.open("GET", "/mod/");
+  jax.send();
+};
+
 var nntpchan_mod_delete = function(longhash) {
   var elem = document.getElementById(longhash);
   var ajax = new XMLHttpRequest();

contrib/templates/placebo/board.mustache

@@ -38,6 +38,7 @@
             <b>Most of the rest of the wild west.</b>
         </div>
         <center><b>{{board.Name}}</b></center>
+        <center><button onclick="nntpchan_mod_commit_spam(this)">Moderate</button></center> 
         <br />
         {{{form}}}
         {{#board.Threads}}

contrib/templates/placebo/post.mustache

@@ -31,6 +31,7 @@
     <a href="{{Source}}" class="image_link" target="_blank" title="{{Filename}}"><img src="{{Thumbnail}}" class="image" /></a>
     {{/post.Attachments}}
     <a href="#" onclick="nntpchan_mod_delete('{{post.PostHash}}');">[Delete]</a>
+    <a href="#" onclick="nntpchan_mod_mark_spam('{{post.PostHash}}');">[Spam]</a>
     <a name="{{post.PostHash}}"></a><span class="topicline"><b data-subject="{{post.Subject}}" class="subject">{{post.Subject}}</b> {{post.Name}} <span class="published">{{post.Frontend}} || {{post.Date}}</span> {{{post.Pubkey}}} <a href="{{post.PostURL}}">[Reply]</a>
     <a href="#" onclick="return quickreply('{{post.ShortHash}}', '{{post.PostHash}}', '{{post.PostURL}}');"> {{post.ShortHash}}</a>
     </span>

contrib/templates/placebo/thread.mustache

@@ -48,6 +48,7 @@
             <b>Most of the rest of the wild west.</b>
         </div>
         <center><b><a href="{{thread.BoardURL}}">{{thread.Board}}</a></b></center>
+        <center><button onclick="nntpchan_mod_commit_spam(this)">Moderate</button></center>
         <br />
         {{{form}}}
             {{#thread.BumpLock}}

contrib/templates/placebo/ukko.mustache

@@ -31,6 +31,7 @@
         <div class="sitetitle">
             <h2><a href="#">CHANGOLIA</a></h2>
             <b>Most of the rest of the wild west.</b>
+            <center><button onclick="nntpchan_mod_commit_spam(this)">Moderate</button></center> 
         </div>
         <div id="paginator">
             {{#prev}}