From 8b7b894eb32a8bd8f236ba7f4908eff870ada22b Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 5 Aug 2018 09:35:47 +1000 Subject: [PATCH] add sfw mode, first try --- contrib/backends/srndv2/src/srnd/daemon.go | 13 +++--- .../backends/srndv2/src/srnd/file_cache.go | 14 +++---- contrib/backends/srndv2/src/srnd/model.go | 4 ++ contrib/backends/srndv2/src/srnd/model_mem.go | 41 +++++++++++++++++++ contrib/backends/srndv2/src/srnd/nntp.go | 4 +- .../backends/srndv2/src/srnd/null_cache.go | 21 +++++----- contrib/backends/srndv2/src/srnd/postgres.go | 2 +- .../srndv2/src/srnd/templates_impl.go | 29 ++++++++----- contrib/backends/srndv2/src/srnd/util.go | 3 +- 9 files changed, 93 insertions(+), 38 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/daemon.go b/contrib/backends/srndv2/src/srnd/daemon.go index 8735e7a..5f1259f 100644 --- a/contrib/backends/srndv2/src/srnd/daemon.go +++ b/contrib/backends/srndv2/src/srnd/daemon.go @@ -443,7 +443,7 @@ func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) { if mode == "sync" { // yeh, do it - self.syncPull(conf.proxy_type, conf.proxy_addr, conf.Addr) + self.syncPull(conf) // sleep for the sleep interval and continue log.Println(conf.Name, "waiting for", conf.sync_interval, "before next sync") time.Sleep(conf.sync_interval) @@ -492,15 +492,16 @@ func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) { } // do a oneshot pull based sync with another server -func (self *NNTPDaemon) syncPull(proxy_type, proxy_addr, remote_addr string) { - c, err := self.dialOut(proxy_type, proxy_addr, remote_addr) +func (self *NNTPDaemon) syncPull(conf *FeedConfig) { + c, err := self.dialOut(conf.proxy_type, conf.proxy_addr, conf.Addr) if err == nil { conn := textproto.NewConn(c) // we connected - nntp := createNNTPConnection(remote_addr) - nntp.name = remote_addr + "-sync" + nntp := createNNTPConnection(conf.Addr) + nntp.name = conf.Addr + "-sync" + nntp.feedname = conf.Name // do handshake - _, reader, _, err := nntp.outboundHandshake(conn, nil) + _, reader, _, err := nntp.outboundHandshake(conn, conf) if err != nil { log.Println("failed to scrape server", err) diff --git a/contrib/backends/srndv2/src/srnd/file_cache.go b/contrib/backends/srndv2/src/srnd/file_cache.go index 6213ec2..be59f5d 100644 --- a/contrib/backends/srndv2/src/srnd/file_cache.go +++ b/contrib/backends/srndv2/src/srnd/file_cache.go @@ -197,7 +197,7 @@ func (self *FileCache) regenerateThread(root ArticleEntry, json bool) { log.Println("did not write", fname, err) return } - template.genThread(self.attachments, self.requireCaptcha, root, self.prefix, self.name, wr, self.database, json, nil) + template.genThread(self.attachments, self.requireCaptcha, root, self.prefix, self.name, wr, self.database, json, nil, false) } else { log.Println("don't have root post", msgid, "not regenerating thread") } @@ -212,7 +212,7 @@ func (self *FileCache) regenerateBoardPage(board string, pages, page int, json b log.Println("error generating board page", page, "for", board, err) return } - template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, pages, page, wr, self.database, json, nil, false) + template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, pages, page, wr, self.database, json, nil, false, false) } // regenerate the catalog for a board @@ -224,7 +224,7 @@ func (self *FileCache) regenerateCatalog(board string) { log.Println("error generating catalog for", board, err) return } - template.genCatalog(self.prefix, self.name, board, wr, self.database, nil) + template.genCatalog(self.prefix, self.name, board, wr, self.database, nil, false) } // regenerate the front page @@ -264,7 +264,7 @@ func (self *FileCache) regenUkko() { log.Println("error generating ukko markup", err) return } - template.genUkko(self.prefix, self.name, wr, self.database, false, nil, false) + template.genUkko(self.prefix, self.name, wr, self.database, false, nil, false, false) // json fname = filepath.Join(self.webroot_dir, "ukko.json") @@ -274,7 +274,7 @@ func (self *FileCache) regenUkko() { log.Println("error generating ukko json", err) return } - template.genUkko(self.prefix, self.name, wr, self.database, true, nil, false) + template.genUkko(self.prefix, self.name, wr, self.database, true, nil, false, false) i := 0 for i < 10 { fname := fmt.Sprintf("ukko-%d.html", i) @@ -285,14 +285,14 @@ func (self *FileCache) regenUkko() { return } defer f.Close() - template.genUkkoPaginated(self.prefix, self.name, f, self.database, 10, i, false, nil, false) + template.genUkkoPaginated(self.prefix, self.name, f, self.database, 10, i, false, nil, false, false) j, err := os.Create(jname) if err != nil { log.Printf("failed to create json ukko", i, err) return } defer j.Close() - template.genUkkoPaginated(self.prefix, self.name, j, self.database, 10, i, true, nil, false) + template.genUkkoPaginated(self.prefix, self.name, j, self.database, 10, i, true, nil, false, false) } } diff --git a/contrib/backends/srndv2/src/srnd/model.go b/contrib/backends/srndv2/src/srnd/model.go index d6dcaf8..30eee2f 100644 --- a/contrib/backends/srndv2/src/srnd/model.go +++ b/contrib/backends/srndv2/src/srnd/model.go @@ -11,6 +11,9 @@ import ( // base model type type BaseModel interface { + // set sfw flag + MarkSFW(sfw bool) + // site url prefix Prefix() string @@ -185,6 +188,7 @@ type CatalogItemModel interface { OP() PostModel ReplyCount() string Page() string + MarkSFW(sfw bool) } type LinkModel interface { diff --git a/contrib/backends/srndv2/src/srnd/model_mem.go b/contrib/backends/srndv2/src/srnd/model_mem.go index 3159302..3247404 100644 --- a/contrib/backends/srndv2/src/srnd/model_mem.go +++ b/contrib/backends/srndv2/src/srnd/model_mem.go @@ -46,6 +46,12 @@ func (self *thread) I18N(i *I18N) { } } +func (self *boardModel) MarkSFW(sfw bool) { + for idx := range self.threads { + self.threads[idx].MarkSFW(sfw) + } +} + func (self *boardModel) I18N(i *I18N) { self._i18n = i for idx := range self.threads { @@ -57,6 +63,12 @@ func (self *attachment) I18N(i *I18N) { self._i18n = i } +func (self *catalogModel) MarkSFW(sfw bool) { + for idx := range self.threads { + self.threads[idx].MarkSFW(sfw) + } +} + func (self *catalogModel) Navbar() string { param := make(map[string]interface{}) param["name"] = fmt.Sprintf("Catalog for %s", self.board) @@ -99,6 +111,10 @@ func (self *catalogItemModel) OP() PostModel { return self.op } +func (self *catalogItemModel) MarkSFW(sfw bool) { + self.op.MarkSFW(sfw) +} + func (self *catalogItemModel) Page() string { return strconv.Itoa(self.page) } @@ -239,6 +255,7 @@ func (self *boardModel) Update(db Database) { type post struct { _i18n *I18N + SFW bool truncated bool prefix string board string @@ -322,6 +339,7 @@ type attachment struct { Name string ThumbWidth int ThumbHeight int + SFW bool } func (self *attachment) MarshalJSON() (b []byte, err error) { @@ -341,6 +359,10 @@ func (self *attachment) Hash() string { return strings.Split(self.Path, ".")[0] } +func (self *attachment) MarkSFW(sfw bool) { + self.SFW = sfw +} + func (self *attachment) ThumbInfo() ThumbInfo { return ThumbInfo{ Width: self.ThumbWidth, @@ -353,6 +375,9 @@ func (self *attachment) Prefix() string { } func (self *attachment) Thumbnail() string { + if self.SFW { + return self.prefix + "static/placeholder.png" + } return self.prefix + "thm/" + self.Path + ".jpg" } @@ -409,6 +434,13 @@ func (self *post) ShortHash() string { return ShortHashMessageID(self.MessageID()) } +func (self *post) MarkSFW(sfw bool) { + self.SFW = sfw + for idx := range self.Files { + self.Files[idx].MarkSFW(sfw) + } +} + func (self *post) PubkeyHex() string { return self.Key } @@ -564,6 +596,7 @@ func (self *post) Truncate() PostModel { Parent: self.Parent, sage: self.sage, Key: self.Key, + SFW: self.SFW, // TODO: copy? Files: self.Files, FrontendPublicKey: self.FrontendPublicKey, @@ -592,11 +625,19 @@ type thread struct { prefix string links []LinkModel Posts []PostModel + SFW bool dirty bool truncatedPostCount int truncatedImageCount int } +func (self *thread) MarkSFW(sfw bool) { + for idx := range self.Posts { + self.Posts[idx].MarkSFW(sfw) + } + self.SFW = sfw +} + func (self *thread) MarshalJSON() (b []byte, err error) { posts := []PostModel{self.OP()} posts = append(posts, self.Replies()...) diff --git a/contrib/backends/srndv2/src/srnd/nntp.go b/contrib/backends/srndv2/src/srnd/nntp.go index e8466aa..8f90dbd 100644 --- a/contrib/backends/srndv2/src/srnd/nntp.go +++ b/contrib/backends/srndv2/src/srnd/nntp.go @@ -1484,10 +1484,10 @@ func (self *nntpConnection) scrapeServer(daemon *NNTPDaemon, conn *textproto.Con line := sc.Text() idx := strings.IndexAny(line, " \t") if idx > 0 { - //log.Println(self.name, "got newsgroup", line[:idx]) + log.Println(self.name, "got newsgroup", line[:idx]) groups = append(groups, line[:idx]) } else if idx < 0 { - //log.Println(self.name, "got newsgroup", line) + log.Println(self.name, "got newsgroup", line) groups = append(groups, line) } else { // can't have it starting with WS diff --git a/contrib/backends/srndv2/src/srnd/null_cache.go b/contrib/backends/srndv2/src/srnd/null_cache.go index cf5aea9..8011296 100644 --- a/contrib/backends/srndv2/src/srnd/null_cache.go +++ b/contrib/backends/srndv2/src/srnd/null_cache.go @@ -62,6 +62,7 @@ func (self *nullHandler) GetI18N(r *http.Request) *I18N { func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sfw := strings.Count(r.URL.RawQuery, "sfw=1") > 0 i18n := self.GetI18N(r) path := r.URL.Path @@ -80,7 +81,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { goto notfound } - template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n) + template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n, sfw) return } else { goto notfound @@ -88,7 +89,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if strings.Trim(path, "/") == "overboard" { // generate ukko aka overboard - template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination) + template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination, sfw) return } @@ -124,7 +125,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { goto notfound } - template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination) + template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination, sfw) return } @@ -144,7 +145,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { page = int(pages) } } - template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination) + template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination, sfw) return } @@ -173,18 +174,18 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if strings.HasPrefix(file, "ukko.html") { - template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination) + template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination, sfw) return } if strings.HasPrefix(file, "ukko.json") { - template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination) + template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination, sfw) return } if strings.HasPrefix(file, "ukko-") { page := getUkkoPage(file) pages, _ := self.database.GetUkkoPageCount(10) - template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination) + template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination, sfw) return } if strings.HasPrefix(file, "thread-") { @@ -202,7 +203,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { goto notfound } - template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n) + template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n, sfw) return } if strings.HasPrefix(file, "catalog-") { @@ -214,7 +215,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !hasgroup { goto notfound } - template.genCatalog(self.prefix, self.name, group, w, self.database, i18n) + template.genCatalog(self.prefix, self.name, group, w, self.database, i18n, sfw) return } else { group, page := getGroupAndPage(file) @@ -229,7 +230,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if page >= int(pages) { goto notfound } - template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination) + template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination, sfw) return } diff --git a/contrib/backends/srndv2/src/srnd/postgres.go b/contrib/backends/srndv2/src/srnd/postgres.go index 535497b..bc98833 100644 --- a/contrib/backends/srndv2/src/srnd/postgres.go +++ b/contrib/backends/srndv2/src/srnd/postgres.go @@ -620,7 +620,7 @@ func (self *PostgresDatabase) upgrade7to8() { } func (self *PostgresDatabase) upgrade8to9() { - cmds := []string { + cmds := []string{ "ALTER TABLE ArticlePosts ADD COLUMN IF NOT EXISTS frontendpubkey TEXT DEFAULT ''", "CREATE TABLE IF NOT EXISTS nntpchan_pubkeys(status VARCHAR(16) NOT NULL, pubkey VARCHAR(64) PRIMARY KEY)", } diff --git a/contrib/backends/srndv2/src/srnd/templates_impl.go b/contrib/backends/srndv2/src/srnd/templates_impl.go index b84edb0..ff9153f 100644 --- a/contrib/backends/srndv2/src/srnd/templates_impl.go +++ b/contrib/backends/srndv2/src/srnd/templates_impl.go @@ -222,24 +222,25 @@ func (self *templateEngine) obtainBoard(prefix, frontend, group string, db Datab return } -func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Writer, db Database, i18n *I18N) { +func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Writer, db Database, i18n *I18N, sfw bool) { board := self.obtainBoard(prefix, frontend, group, db) catalog := new(catalogModel) catalog.prefix = prefix catalog.frontend = frontend catalog.board = group catalog.I18N(i18n) + catalog.MarkSFW(sfw) for page, bm := range board { for _, th := range bm.Threads() { th.Update(db) catalog.threads = append(catalog.threads, &catalogItemModel{op: th.OP(), page: page, replycount: len(th.Replies())}) } } - self.writeTemplate("catalog", map[string]interface{}{"board": catalog}, wr, i18n) + self.writeTemplate("catalog", map[string]interface{}{"board": catalog, "sfw": sfw}, wr, i18n) } // generate a board page -func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, pages, page int, wr io.Writer, db Database, json bool, i18n *I18N, invertPagination bool) { +func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, pages, page int, wr io.Writer, db Database, json bool, i18n *I18N, invertPagination, sfw bool) { // get the board page model perpage, _ := db.GetThreadsPerPage(newsgroup) var boardPage BoardModel @@ -250,29 +251,30 @@ func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix } boardPage.Update(db) boardPage.I18N(i18n) + boardPage.MarkSFW(sfw) // render it if json { self.renderJSON(wr, boardPage) } else { form := renderPostForm(prefix, newsgroup, "", allowFiles, requireCaptcha, i18n) - self.writeTemplate("board", map[string]interface{}{"board": boardPage, "page": page, "form": form}, wr, i18n) + self.writeTemplate("board", map[string]interface{}{"board": boardPage, "page": page, "form": form, "sfw": sfw}, wr, i18n) } } -func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N, invertPagination bool) { +func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N, invertPagination, sfw bool) { var page int64 pages, err := database.GetUkkoPageCount(10) if invertPagination { page = pages } if err == nil { - self.genUkkoPaginated(prefix, frontend, wr, database, int(pages), int(page), json, i18n, invertPagination) + self.genUkkoPaginated(prefix, frontend, wr, database, int(pages), int(page), json, i18n, invertPagination, sfw) } else { log.Println("genUkko()", err.Error()) } } -func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, pages, page int, json bool, i18n *I18N, invertPagination bool) { +func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, pages, page int, json bool, i18n *I18N, invertPagination, sfw bool) { var threads []ThreadModel var articles []ArticleEntry if invertPagination { @@ -285,10 +287,11 @@ func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writ thread, err := database.GetThreadModel(prefix, root) if err == nil { thread.I18N(i18n) + thread.MarkSFW(sfw) threads = append(threads, thread) } } - obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page} + obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page, "sfw": sfw} if page > 0 { obj["prev"] = map[string]interface{}{"no": page - 1} } @@ -310,7 +313,7 @@ func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writ } } -func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root ArticleEntry, prefix, frontend string, wr io.Writer, db Database, json bool, i18n *I18N) { +func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root ArticleEntry, prefix, frontend string, wr io.Writer, db Database, json bool, i18n *I18N, sfw bool) { newsgroup := root.Newsgroup() msgid := root.MessageID() @@ -322,12 +325,13 @@ func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root Arti */ t, err := db.GetThreadModel(prefix, msgid) if err == nil { + t.MarkSFW(sfw) if json { self.renderJSON(wr, t) } else { t.I18N(i18n) form := renderPostForm(prefix, newsgroup, msgid, allowFiles, requireCaptcha, i18n) - self.writeTemplate("thread", map[string]interface{}{"thread": t, "board": map[string]interface{}{"Name": newsgroup, "Frontend": frontend, "AllowFiles": allowFiles}, "form": form, "prefix": prefix}, wr, i18n) + self.writeTemplate("thread", map[string]interface{}{"sfw": sfw, "thread": t, "board": map[string]interface{}{"Name": newsgroup, "Frontend": frontend, "AllowFiles": allowFiles}, "form": form, "prefix": prefix}, wr, i18n) } } else { log.Println("templates: error getting thread for ", msgid, err.Error()) @@ -476,6 +480,11 @@ func (self *templateEngine) genBoardList(prefix, name string, wr io.Writer, db D // for each group groups := db.GetAllNewsgroups() for _, group := range groups { + // exclude banned + banned, _ := db.NewsgroupBanned(group) + if banned { + continue + } // posts this hour hour := db.CountPostsInGroup(group, 3600) // posts today diff --git a/contrib/backends/srndv2/src/srnd/util.go b/contrib/backends/srndv2/src/srnd/util.go index e7cf61a..fedd80a 100644 --- a/contrib/backends/srndv2/src/srnd/util.go +++ b/contrib/backends/srndv2/src/srnd/util.go @@ -801,8 +801,7 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader) return } - func hasAtLeastNWords(str string, n int) bool { parts := strings.Split(str, " ") return len(parts) > n -} \ No newline at end of file +}