archive/nntpchan
Archived
1
0
Fork 0
Code

add sfw mode, first try

Browse Source
This commit is contained in:
Jeff 2018-08-05 09:35:47 +10:00
parent c1fd82bab5
commit 8b7b894eb3
9 changed files with 93 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
contrib/backends/srndv2/src/srnd/daemon.go
View File

@ -443,7 +443,7 @@ func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
if mode == "sync" {
// yeh, do it
self.syncPull(conf.proxy_type, conf.proxy_addr, conf.Addr)
self.syncPull(conf)
// sleep for the sleep interval and continue
log.Println(conf.Name, "waiting for", conf.sync_interval, "before next sync")
time.Sleep(conf.sync_interval)
@ -492,15 +492,16 @@ func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
}
// do a oneshot pull based sync with another server
func (self *NNTPDaemon) syncPull(proxy_type, proxy_addr, remote_addr string) {
c, err := self.dialOut(proxy_type, proxy_addr, remote_addr)
func (self *NNTPDaemon) syncPull(conf *FeedConfig) {
c, err := self.dialOut(conf.proxy_type, conf.proxy_addr, conf.Addr)
if err == nil {
conn := textproto.NewConn(c)
// we connected
nntp := createNNTPConnection(remote_addr)
nntp.name = remote_addr + "-sync"
nntp := createNNTPConnection(conf.Addr)
nntp.name = conf.Addr + "-sync"
nntp.feedname = conf.Name
// do handshake
_, reader, _, err := nntp.outboundHandshake(conn, nil)
_, reader, _, err := nntp.outboundHandshake(conn, conf)
if err != nil {
log.Println("failed to scrape server", err)

14
contrib/backends/srndv2/src/srnd/file_cache.go
View File

@ -197,7 +197,7 @@ func (self *FileCache) regenerateThread(root ArticleEntry, json bool) {
log.Println("did not write", fname, err)
return
}
template.genThread(self.attachments, self.requireCaptcha, root, self.prefix, self.name, wr, self.database, json, nil)
template.genThread(self.attachments, self.requireCaptcha, root, self.prefix, self.name, wr, self.database, json, nil, false)
} else {
log.Println("don't have root post", msgid, "not regenerating thread")
}
@ -212,7 +212,7 @@ func (self *FileCache) regenerateBoardPage(board string, pages, page int, json b
log.Println("error generating board page", page, "for", board, err)
return
}
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, pages, page, wr, self.database, json, nil, false)
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, pages, page, wr, self.database, json, nil, false, false)
}
// regenerate the catalog for a board
@ -224,7 +224,7 @@ func (self *FileCache) regenerateCatalog(board string) {
log.Println("error generating catalog for", board, err)
return
}
template.genCatalog(self.prefix, self.name, board, wr, self.database, nil)
template.genCatalog(self.prefix, self.name, board, wr, self.database, nil, false)
}
// regenerate the front page
@ -264,7 +264,7 @@ func (self *FileCache) regenUkko() {
log.Println("error generating ukko markup", err)
return
}
template.genUkko(self.prefix, self.name, wr, self.database, false, nil, false)
template.genUkko(self.prefix, self.name, wr, self.database, false, nil, false, false)
// json
fname = filepath.Join(self.webroot_dir, "ukko.json")
@ -274,7 +274,7 @@ func (self *FileCache) regenUkko() {
log.Println("error generating ukko json", err)
return
}
template.genUkko(self.prefix, self.name, wr, self.database, true, nil, false)
template.genUkko(self.prefix, self.name, wr, self.database, true, nil, false, false)
i := 0
for i < 10 {
fname := fmt.Sprintf("ukko-%d.html", i)
@ -285,14 +285,14 @@ func (self *FileCache) regenUkko() {
return
}
defer f.Close()
template.genUkkoPaginated(self.prefix, self.name, f, self.database, 10, i, false, nil, false)
template.genUkkoPaginated(self.prefix, self.name, f, self.database, 10, i, false, nil, false, false)
j, err := os.Create(jname)
if err != nil {
log.Printf("failed to create json ukko", i, err)
return
}
defer j.Close()
template.genUkkoPaginated(self.prefix, self.name, j, self.database, 10, i, true, nil, false)
template.genUkkoPaginated(self.prefix, self.name, j, self.database, 10, i, true, nil, false, false)
}
}

4
contrib/backends/srndv2/src/srnd/model.go
View File

@ -11,6 +11,9 @@ import (
// base model type
type BaseModel interface {
// set sfw flag
MarkSFW(sfw bool)
// site url prefix
Prefix() string
@ -185,6 +188,7 @@ type CatalogItemModel interface {
OP() PostModel
ReplyCount() string
Page() string
MarkSFW(sfw bool)
}
type LinkModel interface {

41
contrib/backends/srndv2/src/srnd/model_mem.go
View File

@ -46,6 +46,12 @@ func (self *thread) I18N(i *I18N) {
}
}
func (self *boardModel) MarkSFW(sfw bool) {
for idx := range self.threads {
self.threads[idx].MarkSFW(sfw)
}
}
func (self *boardModel) I18N(i *I18N) {
self._i18n = i
for idx := range self.threads {
@ -57,6 +63,12 @@ func (self *attachment) I18N(i *I18N) {
self._i18n = i
}
func (self *catalogModel) MarkSFW(sfw bool) {
for idx := range self.threads {
self.threads[idx].MarkSFW(sfw)
}
}
func (self *catalogModel) Navbar() string {
param := make(map[string]interface{})
param["name"] = fmt.Sprintf("Catalog for %s", self.board)
@ -99,6 +111,10 @@ func (self *catalogItemModel) OP() PostModel {
return self.op
}
func (self *catalogItemModel) MarkSFW(sfw bool) {
self.op.MarkSFW(sfw)
}
func (self *catalogItemModel) Page() string {
return strconv.Itoa(self.page)
}
@ -239,6 +255,7 @@ func (self *boardModel) Update(db Database) {
type post struct {
_i18n *I18N
SFW bool
truncated bool
prefix string
board string
@ -322,6 +339,7 @@ type attachment struct {
Name string
ThumbWidth int
ThumbHeight int
SFW bool
}
func (self *attachment) MarshalJSON() (b []byte, err error) {
@ -341,6 +359,10 @@ func (self *attachment) Hash() string {
return strings.Split(self.Path, ".")[0]
}
func (self *attachment) MarkSFW(sfw bool) {
self.SFW = sfw
}
func (self *attachment) ThumbInfo() ThumbInfo {
return ThumbInfo{
Width: self.ThumbWidth,
@ -353,6 +375,9 @@ func (self *attachment) Prefix() string {
}
func (self *attachment) Thumbnail() string {
if self.SFW {
return self.prefix + "static/placeholder.png"
}
return self.prefix + "thm/" + self.Path + ".jpg"
}
@ -409,6 +434,13 @@ func (self *post) ShortHash() string {
return ShortHashMessageID(self.MessageID())
}
func (self *post) MarkSFW(sfw bool) {
self.SFW = sfw
for idx := range self.Files {
self.Files[idx].MarkSFW(sfw)
}
}
func (self *post) PubkeyHex() string {
return self.Key
}
@ -564,6 +596,7 @@ func (self *post) Truncate() PostModel {
Parent: self.Parent,
sage: self.sage,
Key: self.Key,
SFW: self.SFW,
// TODO: copy?
Files: self.Files,
FrontendPublicKey: self.FrontendPublicKey,
@ -592,11 +625,19 @@ type thread struct {
prefix string
links []LinkModel
Posts []PostModel
SFW bool
dirty bool
truncatedPostCount int
truncatedImageCount int
}
func (self *thread) MarkSFW(sfw bool) {
for idx := range self.Posts {
self.Posts[idx].MarkSFW(sfw)
}
self.SFW = sfw
}
func (self *thread) MarshalJSON() (b []byte, err error) {
posts := []PostModel{self.OP()}
posts = append(posts, self.Replies()...)

4
contrib/backends/srndv2/src/srnd/nntp.go
View File

@ -1484,10 +1484,10 @@ func (self *nntpConnection) scrapeServer(daemon *NNTPDaemon, conn *textproto.Con
line := sc.Text()
idx := strings.IndexAny(line, " \t")
if idx > 0 {
//log.Println(self.name, "got newsgroup", line[:idx])
log.Println(self.name, "got newsgroup", line[:idx])
groups = append(groups, line[:idx])
} else if idx < 0 {
//log.Println(self.name, "got newsgroup", line)
log.Println(self.name, "got newsgroup", line)
groups = append(groups, line)
} else {
// can't have it starting with WS

21
contrib/backends/srndv2/src/srnd/null_cache.go
View File

@ -62,6 +62,7 @@ func (self *nullHandler) GetI18N(r *http.Request) *I18N {
func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
sfw := strings.Count(r.URL.RawQuery, "sfw=1") > 0
i18n := self.GetI18N(r)
path := r.URL.Path
@ -80,7 +81,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
goto notfound
}
template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n, sfw)
return
} else {
goto notfound
@ -88,7 +89,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if strings.Trim(path, "/") == "overboard" {
// generate ukko aka overboard
template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination)
template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination, sfw)
return
}
@ -124,7 +125,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
goto notfound
}
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination, sfw)
return
}
@ -144,7 +145,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
page = int(pages)
}
}
template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination)
template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination, sfw)
return
}
@ -173,18 +174,18 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if strings.HasPrefix(file, "ukko.html") {
template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination)
template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination, sfw)
return
}
if strings.HasPrefix(file, "ukko.json") {
template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination)
template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination, sfw)
return
}
if strings.HasPrefix(file, "ukko-") {
page := getUkkoPage(file)
pages, _ := self.database.GetUkkoPageCount(10)
template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination)
template.genUkkoPaginated(self.prefix, self.name, w, self.database, int(pages), page, isjson, i18n, self.invertPagination, sfw)
return
}
if strings.HasPrefix(file, "thread-") {
@ -202,7 +203,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
goto notfound
}
template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n)
template.genThread(self.attachments, self.requireCaptcha, msg, self.prefix, self.name, w, self.database, isjson, i18n, sfw)
return
}
if strings.HasPrefix(file, "catalog-") {
@ -214,7 +215,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if !hasgroup {
goto notfound
}
template.genCatalog(self.prefix, self.name, group, w, self.database, i18n)
template.genCatalog(self.prefix, self.name, group, w, self.database, i18n, sfw)
return
} else {
group, page := getGroupAndPage(file)
@ -229,7 +230,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if page >= int(pages) {
goto notfound
}
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination, sfw)
return
}

2
contrib/backends/srndv2/src/srnd/postgres.go
View File

@ -620,7 +620,7 @@ func (self *PostgresDatabase) upgrade7to8() {
}
func (self *PostgresDatabase) upgrade8to9() {
cmds := []string {
cmds := []string{
"ALTER TABLE ArticlePosts ADD COLUMN IF NOT EXISTS frontendpubkey TEXT DEFAULT ''",
"CREATE TABLE IF NOT EXISTS nntpchan_pubkeys(status VARCHAR(16) NOT NULL, pubkey VARCHAR(64) PRIMARY KEY)",
}

29
contrib/backends/srndv2/src/srnd/templates_impl.go
View File

@ -222,24 +222,25 @@ func (self *templateEngine) obtainBoard(prefix, frontend, group string, db Datab
return
}
func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Writer, db Database, i18n *I18N) {
func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Writer, db Database, i18n *I18N, sfw bool) {
board := self.obtainBoard(prefix, frontend, group, db)
catalog := new(catalogModel)
catalog.prefix = prefix
catalog.frontend = frontend
catalog.board = group
catalog.I18N(i18n)
catalog.MarkSFW(sfw)
for page, bm := range board {
for _, th := range bm.Threads() {
th.Update(db)
catalog.threads = append(catalog.threads, &catalogItemModel{op: th.OP(), page: page, replycount: len(th.Replies())})
}
}
self.writeTemplate("catalog", map[string]interface{}{"board": catalog}, wr, i18n)
self.writeTemplate("catalog", map[string]interface{}{"board": catalog, "sfw": sfw}, wr, i18n)
}
// generate a board page
func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, pages, page int, wr io.Writer, db Database, json bool, i18n *I18N, invertPagination bool) {
func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, pages, page int, wr io.Writer, db Database, json bool, i18n *I18N, invertPagination, sfw bool) {
// get the board page model
perpage, _ := db.GetThreadsPerPage(newsgroup)
var boardPage BoardModel
@ -250,29 +251,30 @@ func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix
}
boardPage.Update(db)
boardPage.I18N(i18n)
boardPage.MarkSFW(sfw)
// render it
if json {
self.renderJSON(wr, boardPage)
} else {
form := renderPostForm(prefix, newsgroup, "", allowFiles, requireCaptcha, i18n)
self.writeTemplate("board", map[string]interface{}{"board": boardPage, "page": page, "form": form}, wr, i18n)
self.writeTemplate("board", map[string]interface{}{"board": boardPage, "page": page, "form": form, "sfw": sfw}, wr, i18n)
}
}
func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N, invertPagination bool) {
func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N, invertPagination, sfw bool) {
var page int64
pages, err := database.GetUkkoPageCount(10)
if invertPagination {
page = pages
}
if err == nil {
self.genUkkoPaginated(prefix, frontend, wr, database, int(pages), int(page), json, i18n, invertPagination)
self.genUkkoPaginated(prefix, frontend, wr, database, int(pages), int(page), json, i18n, invertPagination, sfw)
} else {
log.Println("genUkko()", err.Error())
}
}
func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, pages, page int, json bool, i18n *I18N, invertPagination bool) {
func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, pages, page int, json bool, i18n *I18N, invertPagination, sfw bool) {
var threads []ThreadModel
var articles []ArticleEntry
if invertPagination {
@ -285,10 +287,11 @@ func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writ
thread, err := database.GetThreadModel(prefix, root)
if err == nil {
thread.I18N(i18n)
thread.MarkSFW(sfw)
threads = append(threads, thread)
}
}
obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page}
obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page, "sfw": sfw}
if page > 0 {
obj["prev"] = map[string]interface{}{"no": page - 1}
}
@ -310,7 +313,7 @@ func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writ
}
}
func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root ArticleEntry, prefix, frontend string, wr io.Writer, db Database, json bool, i18n *I18N) {
func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root ArticleEntry, prefix, frontend string, wr io.Writer, db Database, json bool, i18n *I18N, sfw bool) {
newsgroup := root.Newsgroup()
msgid := root.MessageID()
@ -322,12 +325,13 @@ func (self *templateEngine) genThread(allowFiles, requireCaptcha bool, root Arti
*/
t, err := db.GetThreadModel(prefix, msgid)
if err == nil {
t.MarkSFW(sfw)
if json {
self.renderJSON(wr, t)
} else {
t.I18N(i18n)
form := renderPostForm(prefix, newsgroup, msgid, allowFiles, requireCaptcha, i18n)
self.writeTemplate("thread", map[string]interface{}{"thread": t, "board": map[string]interface{}{"Name": newsgroup, "Frontend": frontend, "AllowFiles": allowFiles}, "form": form, "prefix": prefix}, wr, i18n)
self.writeTemplate("thread", map[string]interface{}{"sfw": sfw, "thread": t, "board": map[string]interface{}{"Name": newsgroup, "Frontend": frontend, "AllowFiles": allowFiles}, "form": form, "prefix": prefix}, wr, i18n)
}
} else {
log.Println("templates: error getting thread for ", msgid, err.Error())
@ -476,6 +480,11 @@ func (self *templateEngine) genBoardList(prefix, name string, wr io.Writer, db D
// for each group
groups := db.GetAllNewsgroups()
for _, group := range groups {
// exclude banned
banned, _ := db.NewsgroupBanned(group)
if banned {
continue
}
// posts this hour
hour := db.CountPostsInGroup(group, 3600)
// posts today

1
contrib/backends/srndv2/src/srnd/util.go
View File

@ -801,7 +801,6 @@ func storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader)
return
}
func hasAtLeastNWords(str string, n int) bool {
parts := strings.Split(str, " ")
return len(parts) > n