archive/nntpchan
Archived
1
0
Fork 0
Code

* configurable thumbnails

Browse Source 
* start working on spam folder
This commit is contained in:
Jeff Becker 2017-11-12 07:58:30 -05:00
parent a7718e9a0a
commit e093864ee7
No known key found for this signature in database
GPG Key ID: F357B3B42F6F9B05
6 changed files with 205 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
contrib/backends/srndv2/src/srnd/config.go
View File

@ -77,18 +77,17 @@ type CryptoConfig struct {
cert_dir string
}
type ThumbnailConfig struct {
rules []ThumbnailRule
placeholder string
}
// pprof settings
type ProfilingConfig struct {
bind string
enable bool
}
type HookConfig struct {
name string
exec string
enable bool
}
type SRNdConfig struct {
daemon map[string]string
crypto *CryptoConfig
@ -104,6 +103,7 @@ type SRNdConfig struct {
inboundPolicy *FeedPolicy
filter FilterConfig
spamconf SpamConfig
thumbnails *ThumbnailConfig
}
// check for config files
@ -227,6 +227,14 @@ func GenSRNdConfig() *configparser.Configuration {
sect.Add("placeholder_thumbnail", "contrib/static/placeholder.png")
sect.Add("compression", "0")
// thumbnailing section
sect = conf.NewSection("thumbnails")
sect.Add("image/*", "{{convert}} -thumbnail 200 {{infile}} {{outfile}}")
sect.Add("image/gif", "{{convert}} -thumbnail 200 {{infile}}[0] {{outfile}}")
sect.Add("audio/*", "{{ffmpeg}} -i {{infile}} -an -vcodec copy {{outfile}}")
sect.Add("video/*", "{{ffmpeg}} -i {{infile}} -vf scale=300:200 -vframes 1 {{outfile}}")
sect.Add("*", "cp {{placeholder}} {{outfile}}")
// database backend config
sect = conf.NewSection("database")
@ -251,6 +259,7 @@ func GenSRNdConfig() *configparser.Configuration {
sect.Add("allow_files", "1")
sect.Add("regen_on_start", "0")
sect.Add("regen_threads", "2")
sect.Add("board_creation", "1")
sect.Add("bind", "[::]:18000")
sect.Add("name", "web.srndv2.test")
sect.Add("webroot", "webroot")
@ -449,6 +458,14 @@ func ReadConfig() *SRNdConfig {
}
}
s, err = conf.Section("thumbnails")
if err == nil {
log.Println("thumbnails section found")
sconf.thumbnails = new(ThumbnailConfig)
sconf.thumbnails.Load(s.Options())
sconf.thumbnails.placeholder = sconf.store["placeholder_thumbnail"]
}
// begin load feeds.ini
fname = "feeds.ini"

6
contrib/backends/srndv2/src/srnd/daemon.go
View File

@ -815,9 +815,7 @@ func (self *NNTPDaemon) pollfeeds() {
func (self *NNTPDaemon) informHooks(group, msgid, ref string) {
if ValidMessageID(msgid) && ValidMessageID(ref) && ValidNewsgroup(group) {
for _, conf := range self.conf.hooks {
if conf.enable {
ExecHook(conf, group, msgid, ref)
}
conf.Exec(group, msgid, ref)
}
}
}
@ -1081,7 +1079,7 @@ func (self *NNTPDaemon) Setup() {
// set up store
log.Println("set up article store...")
self.store = createArticleStore(self.conf.store, self.database, &self.spamFilter)
self.store = createArticleStore(self.conf.store, self.conf.thumbnails, self.database, &self.spamFilter)
// do we enable the frontend?
if self.conf.frontend["enable"] == "1" {

10
contrib/backends/srndv2/src/srnd/hook.go
View File

@ -5,7 +5,14 @@ import (
"os/exec"
)
func ExecHook(config *HookConfig, group, msgid, ref string) {
type HookConfig struct {
name string
exec string
enable bool
}
func (config *HookConfig) Exec(group, msgid, ref string) {
if config.enable {
cmd := exec.Command(config.exec, group, msgid, ref)
err := cmd.Run()
if err != nil {
@ -13,4 +20,5 @@ func ExecHook(config *HookConfig, group, msgid, ref string) {
log.Println("calling hook", config.name, "failed")
log.Println(string(b))
}
}
}

47
contrib/backends/srndv2/src/srnd/spam.go
View File

@ -29,43 +29,54 @@ func (sp *SpamFilter) Enabled(newsgroup string) bool {
return sp.enabled && newsgroup != "ctl"
}
func (sp *SpamFilter) Rewrite(msg io.Reader, out io.WriteCloser, group string) error {
type SpamResult struct {
Err error
IsSpam bool
}
func (sp *SpamFilter) Rewrite(msg io.Reader, out io.WriteCloser, group string) (result SpamResult) {
var buff [65636]byte
if !sp.Enabled(group) {
return ErrSpamFilterNotEnabled
result.Err = ErrSpamFilterNotEnabled
return
}
addr, err := net.ResolveTCPAddr("tcp", sp.addr)
if err != nil {
return err
var addr *net.TCPAddr
var c *net.TCPConn
var u *user.User
addr, result.Err = net.ResolveTCPAddr("tcp", sp.addr)
if result.Err != nil {
return
}
c, err := net.DialTCP("tcp", nil, addr)
if err != nil {
return err
c, result.Err = net.DialTCP("tcp", nil, addr)
if result.Err != nil {
return
}
u, err := user.Current()
if err != nil {
return err
u, result.Err = user.Current()
if result.Err != nil {
return
}
fmt.Fprintf(c, "PROCESS SPAMC/1.5\r\nUser: %s\r\n\r\n", u.Username)
io.CopyBuffer(c, msg, buff[:])
c.CloseWrite()
r := bufio.NewReader(c)
for {
l, err := r.ReadString(10)
if err != nil {
return err
var l string
l, result.Err = r.ReadString(10)
if result.Err != nil {
return
}
l = strings.TrimSpace(l)
if strings.HasPrefix(l, "Spam: True ") {
return ErrMessageIsSpam
result.IsSpam = true
}
log.Println("SpamFilter:", l)
if l == "" {
_, err = io.CopyBuffer(out, r, buff[:])
_, result.Err = io.CopyBuffer(out, r, buff[:])
c.Close()
out.Close()
return err
return
}
}
return ErrSpamFilterFailed
result.Err = ErrSpamFilterFailed
return
}

146
contrib/backends/srndv2/src/srnd/store.go
View File

@ -10,6 +10,7 @@ import (
"compress/gzip"
"errors"
"io"
"io/ioutil"
"log"
"mime"
"mime/multipart"
@ -81,6 +82,21 @@ type ArticleStore interface {
// delete message by message-id
Remove(msgid string) error
// move message to spam dir
MarkSpam(msgid string) error
// move message out of spam dir
UnmarkSpam(msgid string) error
// get filepath for spam file via msgid
SpamFile(msgid string) string
// iterate over all spam messages
IterSpam(func(string) error) error
// iterate over all spam message headers
IterSpamHeaders(func(map[string][]string) error) error
}
type articleStore struct {
directory string
@ -93,12 +109,14 @@ type articleStore struct {
sox_path string
identify_path string
placeholder string
spamdir string
compression bool
compWriter *gzip.Writer
spamd *SpamFilter
thumbnails *ThumbnailConfig
}
func createArticleStore(config map[string]string, database Database, spamd *SpamFilter) ArticleStore {
func createArticleStore(config map[string]string, thumbConfig *ThumbnailConfig, database Database, spamd *SpamFilter) ArticleStore {
store := &articleStore{
directory: config["store_dir"],
temp: config["incoming_dir"],
@ -112,6 +130,8 @@ func createArticleStore(config map[string]string, database Database, spamd *Spam
database: database,
compression: config["compression"] == "1",
spamd: spamd,
spamdir: filepath.Join(config["store_dir"], "spam"),
thumbnails: thumbConfig,
}
store.Init()
return store
@ -135,6 +155,7 @@ func (self *articleStore) Init() {
EnsureDir(self.temp)
EnsureDir(self.attachments)
EnsureDir(self.thumbs)
EnsureDir(self.spamdir)
if !CheckFile(self.convert_path) {
log.Fatal("cannot find executable for convert: ", self.convert_path, " not found")
}
@ -213,7 +234,6 @@ func (self *articleStore) isVideo(fname string) bool {
func (self *articleStore) ThumbInfo(fpath string) (ThumbInfo, error) {
var info ThumbInfo
log.Println("made thumbnail for", fpath)
cmd := exec.Command(self.identify_path, "-format", "%[fx:w] %[fx:h]", fpath)
output, err := cmd.Output()
if err == nil {
@ -225,12 +245,38 @@ func (self *articleStore) ThumbInfo(fpath string) (ThumbInfo, error) {
}
}
} else {
log.Println("failed to determine size of thumbnail")
log.Println("failed to determine size of thumbnail", err)
}
return info, err
}
func (self *articleStore) GenerateThumbnail(fname string) (info ThumbInfo, err error) {
outfname := self.ThumbnailFilepath(fname)
if self.thumbnails == nil {
err = self.generateThumbnailFallback(fname)
if err == nil {
info, err = self.ThumbInfo(outfname)
}
return
}
infname := self.AttachmentFilepath(fname)
err = self.thumbnails.GenerateThumbnail(infname, outfname, map[string]string{
"ffmpeg": self.ffmpeg_path,
"convert": self.convert_path,
"sox": self.sox_path,
"identify": self.identify_path,
})
if err != nil {
log.Println(err.Error(), "so we'll use fallback thumbnailing")
err = self.generateThumbnailFallback(fname)
}
if err == nil {
info, err = self.ThumbInfo(outfname)
}
return
}
func (self *articleStore) generateThumbnailFallback(fname string) (err error) {
outfname := self.ThumbnailFilepath(fname)
infname := self.AttachmentFilepath(fname)
tmpfname := ""
@ -270,7 +316,7 @@ func (self *articleStore) GenerateThumbnail(fname string) (info ThumbInfo, err e
if len(tmpfname) > 0 {
DelFile(tmpfname)
}
return info, err
return
}
func (self *articleStore) GetAllAttachments() (names []string, err error) {
@ -391,7 +437,7 @@ func (self *articleStore) CreateFile(messageID string) io.WriteCloser {
// return true if we have an article
func (self *articleStore) HasArticle(messageID string) bool {
return CheckFile(self.GetFilename(messageID))
return CheckFile(self.GetFilename(messageID)) || CheckFile(self.SpamFile(messageID))
}
// get the filename for this article
@ -420,10 +466,14 @@ func (self *articleStore) GetMIMEHeader(messageID string) textproto.MIMEHeader {
return self.getMIMEHeader(messageID)
}
// get article with headers only
func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHeader) {
if ValidMessageID(messageID) {
fname := self.GetFilename(messageID)
func (self *articleStore) getMIMEHeader(msgid string) (hdr textproto.MIMEHeader) {
if ValidMessageID(msgid) {
hdr = self.getMIMEHeaderByFile(self.GetFilename(msgid))
}
return
}
func (self *articleStore) getMIMEHeaderByFile(fname string) (hdr map[string][]string) {
f, err := os.Open(fname)
if f != nil {
r := bufio.NewReader(f)
@ -431,14 +481,13 @@ func (self *articleStore) getMIMEHeader(messageID string) (hdr textproto.MIMEHea
msg, err = readMIMEHeader(r)
f.Close()
if msg != nil {
hdr = textproto.MIMEHeader(msg.Header)
hdr = msg.Header
}
}
if err != nil {
log.Println("failed to load article headers for", messageID, err)
log.Println("failed to load article headers from", fname, err)
}
}
return hdr
return
}
func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter func(string) bool, group string) (err error) {
@ -465,10 +514,10 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
if self.spamd.Enabled(group) {
pr_in, pw_in := io.Pipe()
pr_out, pw_out := io.Pipe()
ec := make(chan error)
resc := make(chan SpamResult)
go func() {
e := self.spamd.Rewrite(pr_in, pw_out, group)
ec <- e
res := self.spamd.Rewrite(pr_in, pw_out, group)
resc <- res
}()
go func() {
var buff [65536]byte
@ -488,12 +537,18 @@ func (self *articleStore) ProcessMessage(wr io.Writer, msg io.Reader, spamfilter
if err != nil {
return
}
msgid := getMessageID(m.Header)
writeMIMEHeader(wr, m.Header)
read_message_body(m.Body, m.Header, self, wr, false, process)
er := <-ec
if er != nil {
return er
err = read_message_body(m.Body, m.Header, self, wr, false, process)
spamRes := <-resc
if spamRes.Err != nil {
return spamRes.Err
}
if spamRes.IsSpam {
err = self.MarkSpam(msgid)
}
} else {
r := bufio.NewReader(msg)
m, e := readMIMEHeader(r)
@ -645,3 +700,54 @@ func read_message_body(body io.Reader, hdr map[string][]string, store ArticleSto
}
return err
}
func (self *articleStore) SpamFile(msgid string) string {
return filepath.Join(self.spamdir, msgid)
}
func (self *articleStore) MarkSpam(msgid string) (err error) {
if ValidMessageID(msgid) {
err = os.Rename(self.GetFilename(msgid), self.SpamFile(msgid))
}
return
}
func (self *articleStore) UnmarkSpam(msgid string) (err error) {
if ValidMessageID(msgid) {
err = os.Rename(self.SpamFile(msgid), self.GetFilename(msgid))
}
return
}
func (self *articleStore) iterSpamFiles(v func(os.FileInfo) error) error {
infos, err := ioutil.ReadDir(self.spamdir)
if err == nil {
for idx := range infos {
err = v(infos[idx])
if err != nil {
break
}
}
}
return err
}
func (self *articleStore) IterSpam(v func(string) error) error {
return self.iterSpamFiles(func(i os.FileInfo) error {
fname := i.Name()
if ValidMessageID(fname) {
return v(fname)
}
return nil
})
}
func (self *articleStore) IterSpamHeaders(v func(map[string][]string) error) error {
return self.IterSpam(func(msgid string) error {
hdr := self.getMIMEHeaderByFile(self.SpamFile(msgid))
if hdr != nil {
return v(hdr)
}
return nil
})
}

2
contrib/backends/srndv2/src/srnd/tools.go
View File

@ -35,7 +35,7 @@ func ThumbnailTool(threads int, missing bool) {
log.Println("cannot load config, ReadConfig() returned nil")
return
}
store := createArticleStore(conf.store, nil, &SpamFilter{})
store := createArticleStore(conf.store, conf.thumbnails, nil, &SpamFilter{})
reThumbnail(threads, store, missing)
}