* configurable thumbnails

* start working on spam folder

contrib/backends/srndv2/src/srnd/config.go

@@ -77,18 +77,17 @@ type CryptoConfig struct {
 	cert_dir     string
 }
 
+type ThumbnailConfig struct {
+	rules       []ThumbnailRule
+	placeholder string
+}
+
 // pprof settings
 type ProfilingConfig struct {
 	bind   string
 	enable bool
 }
 
-type HookConfig struct {
-	name   string
-	exec   string
-	enable bool
-}
-
 type SRNdConfig struct {
 	daemon        map[string]string
 	crypto        *CryptoConfig
@@ -104,6 +103,7 @@ type SRNdConfig struct {
 	inboundPolicy *FeedPolicy
 	filter        FilterConfig
 	spamconf      SpamConfig
+	thumbnails    *ThumbnailConfig
 }
 
 // check for config files
@@ -227,6 +227,14 @@ func GenSRNdConfig() *configparser.Configuration {
 	sect.Add("placeholder_thumbnail", "contrib/static/placeholder.png")
 	sect.Add("compression", "0")
 
+	// thumbnailing section
+	sect = conf.NewSection("thumbnails")
+	sect.Add("image/*", "{{convert}} -thumbnail 200 {{infile}} {{outfile}}")
+	sect.Add("image/gif", "{{convert}} -thumbnail 200 {{infile}}[0] {{outfile}}")
+	sect.Add("audio/*", "{{ffmpeg}} -i {{infile}} -an -vcodec copy {{outfile}}")
+	sect.Add("video/*", "{{ffmpeg}} -i {{infile}} -vf scale=300:200 -vframes 1 {{outfile}}")
+	sect.Add("*", "cp {{placeholder}} {{outfile}}")
+
 	// database backend config
 	sect = conf.NewSection("database")
 
@@ -251,6 +259,7 @@ func GenSRNdConfig() *configparser.Configuration {
 	sect.Add("allow_files", "1")
 	sect.Add("regen_on_start", "0")
 	sect.Add("regen_threads", "2")
+	sect.Add("board_creation", "1")
 	sect.Add("bind", "[::]:18000")
 	sect.Add("name", "web.srndv2.test")
 	sect.Add("webroot", "webroot")
@@ -449,6 +458,14 @@ func ReadConfig() *SRNdConfig {
 		}
 	}
 
+	s, err = conf.Section("thumbnails")
+	if err == nil {
+		log.Println("thumbnails section found")
+		sconf.thumbnails = new(ThumbnailConfig)
+		sconf.thumbnails.Load(s.Options())
+		sconf.thumbnails.placeholder = sconf.store["placeholder_thumbnail"]
+	}
+
 	// begin load feeds.ini
 
 	fname = "feeds.ini"

contrib/backends/srndv2/src/srnd/daemon.go

@@ -815,9 +815,7 @@ func (self *NNTPDaemon) pollfeeds() {
 func (self *NNTPDaemon) informHooks(group, msgid, ref string) {
 	if ValidMessageID(msgid) && ValidMessageID(ref) && ValidNewsgroup(group) {
 		for _, conf := range self.conf.hooks {
-			if conf.enable {
+			conf.Exec(group, msgid, ref)
-				ExecHook(conf, group, msgid, ref)
-			}
 		}
 	}
 }
@@ -1081,7 +1079,7 @@ func (self *NNTPDaemon) Setup() {
 
 	// set up store
 	log.Println("set up article store...")
-	self.store = createArticleStore(self.conf.store, self.database, &self.spamFilter)
+	self.store = createArticleStore(self.conf.store, self.conf.thumbnails, self.database, &self.spamFilter)
 
 	// do we enable the frontend?
 	if self.conf.frontend["enable"] == "1" {

contrib/backends/srndv2/src/srnd/hook.go

@@ -5,7 +5,14 @@ import (
 	"os/exec"
 )
 
-func ExecHook(config *HookConfig, group, msgid, ref string) {
+type HookConfig struct {
+	name   string
+	exec   string
+	enable bool
+}
+
+func (config *HookConfig) Exec(group, msgid, ref string) {
+	if config.enable {
 		cmd := exec.Command(config.exec, group, msgid, ref)
 		err := cmd.Run()
 		if err != nil {
@@ -13,4 +20,5 @@ func ExecHook(config *HookConfig, group, msgid, ref string) {
 			log.Println("calling hook", config.name, "failed")
 			log.Println(string(b))
 		}
+	}
 }

contrib/backends/srndv2/src/srnd/spam.go

@@ -29,43 +29,54 @@ func (sp *SpamFilter) Enabled(newsgroup string) bool {
 	return sp.enabled && newsgroup != "ctl"
 }
 
-func (sp *SpamFilter) Rewrite(msg io.Reader, out io.WriteCloser, group string) error {
+type SpamResult struct {
+	Err    error
+	IsSpam bool
+}
+
+func (sp *SpamFilter) Rewrite(msg io.Reader, out io.WriteCloser, group string) (result SpamResult) {
 	var buff [65636]byte
 	if !sp.Enabled(group) {
-		return ErrSpamFilterNotEnabled
+		result.Err = ErrSpamFilterNotEnabled
+		return
 	}
-	addr, err := net.ResolveTCPAddr("tcp", sp.addr)
+	var addr *net.TCPAddr
-	if err != nil {
+	var c *net.TCPConn
-		return err
+	var u *user.User
+	addr, result.Err = net.ResolveTCPAddr("tcp", sp.addr)
+	if result.Err != nil {
+		return
 	}
-	c, err := net.DialTCP("tcp", nil, addr)
+	c, result.Err = net.DialTCP("tcp", nil, addr)
-	if err != nil {
+	if result.Err != nil {
-		return err
+		return
 	}
-	u, err := user.Current()
+	u, result.Err = user.Current()
-	if err != nil {
+	if result.Err != nil {
-		return err
+		return
 	}
 	fmt.Fprintf(c, "PROCESS SPAMC/1.5\r\nUser: %s\r\n\r\n", u.Username)
 	io.CopyBuffer(c, msg, buff[:])
 	c.CloseWrite()
 	r := bufio.NewReader(c)
 	for {
-		l, err := r.ReadString(10)
+		var l string
-		if err != nil {
+		l, result.Err = r.ReadString(10)
-			return err
+		if result.Err != nil {
+			return
 		}
 		l = strings.TrimSpace(l)
 		if strings.HasPrefix(l, "Spam: True ") {
-			return ErrMessageIsSpam
+			result.IsSpam = true
 		}
 		log.Println("SpamFilter:", l)
 		if l == "" {
-			_, err = io.CopyBuffer(out, r, buff[:])
+			_, result.Err = io.CopyBuffer(out, r, buff[:])
 			c.Close()
 			out.Close()
-			return err
+			return
 		}
 	}
-	return ErrSpamFilterFailed
+	result.Err = ErrSpamFilterFailed
+	return
 }

contrib/backends/srndv2/src/srnd/store.go

@@ -10,6 +10,7 @@ import (
 	"compress/gzip"
 	"errors"
 	"io"
+	"io/ioutil"
 	"log"
 	"mime"
 	"mime/multipart"
@@ -81,6 +82,21 @@ type ArticleStore interface {
 
 	// delete message by message-id
 	Remove(msgid string) error
+
+	// move message to spam dir
+	MarkSpam(msgid string) error
+
+	// move message out of spam dir
+	UnmarkSpam(msgid string) error
+
+	// get filepath for spam file via msgid
+	SpamFile(msgid string) string
+
+	// iterate over all spam messages
+	IterSpam(func(string) error) error
+
+	// iterate over all spam message headers
+	IterSpamHeaders(func(map[string][]string) error) error
 }
 type articleStore struct {
 	directory     string
@@ -93,12 +109,14 @@ type articleStore struct {
 	sox_path      string
 	identify_path string
 	placeholder   string
+	spamdir       string
 	compression   bool
 	compWriter    *gzip.Writer
 	spamd         *SpamFilter
+	thumbnails    *ThumbnailConfig
 }
 
-func createArticleStore(config map[string]string, database Database, spamd *SpamFilter) ArticleStore {
+func createArticleStore(config map[string]string, thumbConfig *ThumbnailConfig, database Database, spamd *SpamFilter) ArticleStore {
 	store := &articleStore{
 		directory:     config["store_dir"],
 		temp:          config["incoming_dir"],
@@ -112,6 +130,8 @@ func createArticleStore(config map[string]string, database Database, spamd *Spam
 		database:      database,
 		compression:   config["compression"] == "1",
 		spamd:         spamd,
+		spamdir:       filepath.Join(config["store_dir"], "spam"),
+		thumbnails:    thumbConfig,
 	}
 	store.Init()
 	return store
@@ -135,6 +155,7 @@ func (self *articleStore) Init() {
 	EnsureDir(self.temp)
 	EnsureDir(self.attachments)
 	EnsureDir(self.thumbs)
+	EnsureDir(self.spamdir)
 	if !CheckFile(self.convert_path) {
 		log.Fatal("cannot find executable for convert: ", self.convert_path, " not found")
 	}
@@ -213,7 +234,6 @@ func (self *articleStore) isVideo(fname string) bool {
 
 func (self *articleStore) ThumbInfo(fpath string) (ThumbInfo, error) {
 	var info ThumbInfo
-	log.Println("made thumbnail for", fpath)
 	cmd := exec.Command(self.identify_path, "-format", "%[fx:w] %[fx:h]", fpath)
 	output, err := cmd.Output()
 	if err == nil {
@@ -225,12 +245,38 @@ func (self *articleStore) ThumbInfo(fpath string) (ThumbInfo, error) {
 			}
 		}
 	} else {
-		log.Println("failed to determine size of thumbnail")
+		log.Println("failed to determine size of thumbnail", err)
 	}
 	return info, err
 }
 
 func (self *articleStore) GenerateThumbnail(fname string) (info ThumbInfo, err error) {
+	outfname := self.ThumbnailFilepath(fname)
+	if self.thumbnails == nil {
+		err = self.generateThumbnailFallback(fname)
+		if err == nil {
+			info, err = self.ThumbInfo(outfname)
+		}
+		return
+	}
+	infname := self.AttachmentFilepath(fname)
+	err = self.thumbnails.GenerateThumbnail(infname, outfname, map[string]string{
+		"ffmpeg":   self.ffmpeg_path,
+		"convert":  self.convert_path,
+		"sox":      self.sox_path,
+		"identify": self.identify_path,
+	})
+	if err != nil {
+		log.Println(err.Error(), "so we'll use fallback thumbnailing")
+		err = self.generateThumbnailFallback(fname)
+	}
+	if err == nil {
+		info, err = self.ThumbInfo(outfname)
+	}
+	return
+}
+
+func (self *articleStore) generateThumbnailFallback(fname string) (err error) {
 	outfname := self.ThumbnailFilepath(fname)
 	infname := self.AttachmentFilepath(fname)
 	tmpfname := ""
@@ -270,7 +316,7 @@ func (self *articleStore) GenerateThumbnail(fname string) (info ThumbInfo, err e
 	if len(tmpfname) > 0 {
 		DelFile(tmpfname)
 	}
-	return info, err
+	return
 }
 
 func (self *articleStore) GetAllAttachments() (names []string, err error) {
@@ -391,7 +437,7 @@ func (self *articleStore) CreateFile(messageID string) io.WriteCloser {
 
 // return true if we have an article
 func (self *articleStore) HasArticle(messageID string) bool {
-	return CheckFile(self.GetFilename(messageID))
+	return CheckFile(self.GetFilename(messageID)) || CheckFile(self.SpamFile(messageID))
 }
 
 // get the filename for this article
@@ -420,10 +466,14 @@ func (self *articleStore) GetMIMEHeader(messageID string) textproto.MIMEHeader {
 	return self.getMIMEHeader(messageID)
 }
 
-// get article with headers only
+func (self *articleStore) getMIMEHeader(msgid string) (hdr textproto.MIMEHeader) {
-func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHeader) {
+	if ValidMessageID(msgid) {
-	if ValidMessageID(messageID) {
+		hdr = self.getMIMEHeaderByFile(self.GetFilename(msgid))
-		fname := self.GetFilename(messageID)
+	}
+	return
+}
+
+func (self *articleStore) getMIMEHeaderByFile(fname string) (hdr map[string][]string) {
 	f, err := os.Open(fname)
 	if f != nil {
 		r := bufio.NewReader(f)
@@ -431,14 +481,13 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
 		msg, err = readMIMEHeader(r)
 		f.Close()
 		if msg != nil {
-				hdr = textproto.MIMEHeader(msg.Header)
+			hdr = msg.Header
 		}
 	}
 	if err != nil {
-			log.Println("failed to load article headers for", messageID, err)
+		log.Println("failed to load article headers from", fname, err)
 	}
-	}
+	return
-	return hdr
 }
 
 func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter func(string) bool, group string) (err error) {
@@ -465,10 +514,10 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
 	if self.spamd.Enabled(group) {
 		pr_in, pw_in := io.Pipe()
 		pr_out, pw_out := io.Pipe()
-		ec := make(chan error)
+		resc := make(chan SpamResult)
 		go func() {
-			e := self.spamd.Rewrite(pr_in, pw_out, group)
+			res := self.spamd.Rewrite(pr_in, pw_out, group)
-			ec <- e
+			resc <- res
 		}()
 		go func() {
 			var buff [65536]byte
@@ -488,12 +537,18 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
 		if err != nil {
 			return
 		}
+		msgid := getMessageID(m.Header)
 		writeMIMEHeader(wr, m.Header)
-		read_message_body(m.Body, m.Header, self, wr, false, process)
+		err = read_message_body(m.Body, m.Header, self, wr, false, process)
-		er := <-ec
+		spamRes := <-resc
-		if er != nil {
+		if spamRes.Err != nil {
-			return er
+			return spamRes.Err
 		}
+
+		if spamRes.IsSpam {
+			err = self.MarkSpam(msgid)
+		}
+
 	} else {
 		r := bufio.NewReader(msg)
 		m, e := readMIMEHeader(r)
@@ -645,3 +700,54 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
 	}
 	return err
 }
+
+func (self *articleStore) SpamFile(msgid string) string {
+	return filepath.Join(self.spamdir, msgid)
+}
+
+func (self *articleStore) MarkSpam(msgid string) (err error) {
+	if ValidMessageID(msgid) {
+		err = os.Rename(self.GetFilename(msgid), self.SpamFile(msgid))
+	}
+	return
+}
+
+func (self *articleStore) UnmarkSpam(msgid string) (err error) {
+	if ValidMessageID(msgid) {
+		err = os.Rename(self.SpamFile(msgid), self.GetFilename(msgid))
+	}
+	return
+}
+
+func (self *articleStore) iterSpamFiles(v func(os.FileInfo) error) error {
+	infos, err := ioutil.ReadDir(self.spamdir)
+	if err == nil {
+		for idx := range infos {
+			err = v(infos[idx])
+			if err != nil {
+				break
+			}
+		}
+	}
+	return err
+}
+
+func (self *articleStore) IterSpam(v func(string) error) error {
+	return self.iterSpamFiles(func(i os.FileInfo) error {
+		fname := i.Name()
+		if ValidMessageID(fname) {
+			return v(fname)
+		}
+		return nil
+	})
+}
+
+func (self *articleStore) IterSpamHeaders(v func(map[string][]string) error) error {
+	return self.IterSpam(func(msgid string) error {
+		hdr := self.getMIMEHeaderByFile(self.SpamFile(msgid))
+		if hdr != nil {
+			return v(hdr)
+		}
+		return nil
+	})
+}

contrib/backends/srndv2/src/srnd/tools.go

@@ -35,7 +35,7 @@ func ThumbnailTool(threads int, missing bool) {
 		log.Println("cannot load config, ReadConfig() returned nil")
 		return
 	}
-	store := createArticleStore(conf.store, nil, &SpamFilter{})
+	store := createArticleStore(conf.store, conf.thumbnails, nil, &SpamFilter{})
 	reThumbnail(threads, store, missing)
 }