archive/nntpchan
Archived
1
0
Fork 0
Code

add inverted pagination for archive mode

Browse Source
This commit is contained in:
Jeff Becker 2018-03-09 09:48:29 -05:00
parent 55ba1e6c7c
commit 2adcc73d92
No known key found for this signature in database
GPG Key ID: F357B3B42F6F9B05
11 changed files with 105 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contrib/backends/srndv2/src/srnd/cache_interface.go
View File

@ -26,6 +26,7 @@ type CacheInterface interface {
GetHandler() CacheHandler GetHandler() CacheHandler
SetRequireCaptcha(required bool) SetRequireCaptcha(required bool)
InvertPagination()
} }
//TODO only pass needed config //TODO only pass needed config

1
contrib/backends/srndv2/src/srnd/daemon.go
View File

@ -586,6 +586,7 @@ func (self *NNTPDaemon) Run() {
if self.conf.daemon["archive"] == "1" { if self.conf.daemon["archive"] == "1" {
log.Println("running in archive mode") log.Println("running in archive mode")
self.expire = nil self.expire = nil
self.frontend.ArchiveMode()
} else { } else {
self.expire = createExpirationCore(self.database, self.store, self.informHooks) self.expire = createExpirationCore(self.database, self.store, self.informHooks)
} }

3
contrib/backends/srndv2/src/srnd/database.go
View File

@ -339,6 +339,9 @@ type Database interface {
// find headers in group with lo/hi watermark and list of patterns // find headers in group with lo/hi watermark and list of patterns
FindHeaders(group, headername string, lo, hi int64) (ArticleHeaders, error) FindHeaders(group, headername string, lo, hi int64) (ArticleHeaders, error)
// count ukko pages
GetUkkoPageCount() (int64, error)
} }
func NewDatabase(db_type, schema, host, port, user, password string) Database { func NewDatabase(db_type, schema, host, port, user, password string) Database {

24
contrib/backends/srndv2/src/srnd/file_cache.go
View File

@ -157,8 +157,9 @@ func (self *FileCache) pollRegen() {
case _ = <-self.regenBoardTicker.C: case _ = <-self.regenBoardTicker.C:
self.regenBoardLock.Lock() self.regenBoardLock.Lock()
for _, v := range self.regenBoardMap { for _, v := range self.regenBoardMap {
self.regenerateBoardPage(v.group, v.page, false) pages := self.database.GetGroupPageCount(v.group)
self.regenerateBoardPage(v.group, v.page, true) self.regenerateBoardPage(v.group, int(pages), v.page, false)
self.regenerateBoardPage(v.group, int(pages), v.page, true)
} }
self.regenBoardMap = make(map[string]groupRegenRequest) self.regenBoardMap = make(map[string]groupRegenRequest)
self.regenBoardLock.Unlock() self.regenBoardLock.Unlock()
@ -173,12 +174,15 @@ func (self *FileCache) pollRegen() {
} }
} }
func (self *FileCache) InvertPagination() {
}
// regen every page of the board // regen every page of the board
func (self *FileCache) RegenerateBoard(group string) { func (self *FileCache) RegenerateBoard(group string) {
pages, _ := self.database.GetPagesPerBoard(group) pages, _ := self.database.GetPagesPerBoard(group)
for page := 0; page < pages; page++ { for page := 0; page < pages; page++ {
self.regenerateBoardPage(group, page, false) self.regenerateBoardPage(group, int(pages), page, false)
self.regenerateBoardPage(group, page, true) self.regenerateBoardPage(group, int(pages), page, true)
} }
} }
@ -200,7 +204,7 @@ func (self *FileCache) regenerateThread(root ArticleEntry, json bool) {
} }
// regenerate just a page on a board // regenerate just a page on a board
func (self *FileCache) regenerateBoardPage(board string, page int, json bool) { func (self *FileCache) regenerateBoardPage(board string, pages, page int, json bool) {
fname := self.getFilenameForBoardPage(board, page, json) fname := self.getFilenameForBoardPage(board, page, json)
wr, err := os.Create(fname) wr, err := os.Create(fname)
defer wr.Close() defer wr.Close()
@ -208,7 +212,7 @@ func (self *FileCache) regenerateBoardPage(board string, page int, json bool) {
log.Println("error generating board page", page, "for", board, err) log.Println("error generating board page", page, "for", board, err)
return return
} }
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, page, wr, self.database, json, nil) template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, pages, page, wr, self.database, json, nil, false)
} }
// regenerate the catalog for a board // regenerate the catalog for a board
@ -260,7 +264,7 @@ func (self *FileCache) regenUkko() {
log.Println("error generating ukko markup", err) log.Println("error generating ukko markup", err)
return return
} }
template.genUkko(self.prefix, self.name, wr, self.database, false, nil) template.genUkko(self.prefix, self.name, wr, self.database, false, nil, false)
// json // json
fname = filepath.Join(self.webroot_dir, "ukko.json") fname = filepath.Join(self.webroot_dir, "ukko.json")
@ -270,7 +274,7 @@ func (self *FileCache) regenUkko() {
log.Println("error generating ukko json", err) log.Println("error generating ukko json", err)
return return
} }
template.genUkko(self.prefix, self.name, wr, self.database, true, nil) template.genUkko(self.prefix, self.name, wr, self.database, true, nil, false)
i := 0 i := 0
for i < 10 { for i < 10 {
fname := fmt.Sprintf("ukko-%d.html", i) fname := fmt.Sprintf("ukko-%d.html", i)
@ -281,14 +285,14 @@ func (self *FileCache) regenUkko() {
return return
} }
defer f.Close() defer f.Close()
template.genUkkoPaginated(self.prefix, self.name, f, self.database, i, false, nil) template.genUkkoPaginated(self.prefix, self.name, f, self.database, i, false, nil, false)
j, err := os.Create(jname) j, err := os.Create(jname)
if err != nil { if err != nil {
log.Printf("failed to create json ukko", i, err) log.Printf("failed to create json ukko", i, err)
return return
} }
defer j.Close() defer j.Close()
template.genUkkoPaginated(self.prefix, self.name, j, self.database, i, true, nil) template.genUkkoPaginated(self.prefix, self.name, j, self.database, i, true, nil, false)
} }
} }

3
contrib/backends/srndv2/src/srnd/frontend.go
View File

@ -41,4 +41,7 @@ type Frontend interface {
RegenOnModEvent(newsgroup, msgid, root string, page int) RegenOnModEvent(newsgroup, msgid, root string, page int)
GetCacheHandler() CacheHandler GetCacheHandler() CacheHandler
// set archive mode
ArchiveMode()
} }

17
contrib/backends/srndv2/src/srnd/frontend_http.go
View File

@ -209,6 +209,9 @@ type httpFrontend struct {
// this is a very important thing by the way // this is a very important thing by the way
requireCaptcha bool requireCaptcha bool
// are we in archive mode?
archive bool
} }
// do we allow this newsgroup? // do we allow this newsgroup?
@ -240,6 +243,10 @@ func (self httpFrontend) deleteBoardMarkup(group string) {
self.cache.DeleteBoardMarkup(group) self.cache.DeleteBoardMarkup(group)
} }
func (self *httpFrontend) ArchiveMode() {
self.archive = true
}
// load post model and inform live ui // load post model and inform live ui
func (self *httpFrontend) informLiveUI(msgid, ref, group string) { func (self *httpFrontend) informLiveUI(msgid, ref, group string) {
// root post // root post
@ -387,8 +394,10 @@ func (self *httpFrontend) HandleNewPost(nntp frontendPost) {
entry := ArticleEntry{msgid, group} entry := ArticleEntry{msgid, group}
// regnerate thread // regnerate thread
self.Regen(entry) self.Regen(entry)
// regenerate all board pages // regenerate all board pages if not archiving
self.RegenerateBoard(group) if !self.archive {
self.RegenerateBoard(group)
}
// regen front page // regen front page
self.RegenFrontPage() self.RegenFrontPage()
} }
@ -1439,6 +1448,10 @@ func (self *httpFrontend) Mainloop() {
// run daemon's mod engine with our frontend // run daemon's mod engine with our frontend
// go RunModEngine(self.daemon.mod, self.cache.RegenOnModEvent) // go RunModEngine(self.daemon.mod, self.cache.RegenOnModEvent)
if self.archive {
self.cache.InvertPagination()
}
// start cache // start cache
self.cache.Start() self.cache.Start()

6
contrib/backends/srndv2/src/srnd/frontend_multi.go
View File

@ -15,6 +15,12 @@ func (self multiFrontend) GetCacheHandler() CacheHandler {
return nil return nil
} }
func (self multiFrontend) ArchiveMode() {
for _, f := range self.frontends {
f.ArchiveMode()
}
}
func (self multiFrontend) AllowNewsgroup(newsgroup string) bool { func (self multiFrontend) AllowNewsgroup(newsgroup string) bool {
return true return true
} }

36
contrib/backends/srndv2/src/srnd/null_cache.go
View File

@ -15,15 +15,20 @@ type NullCache struct {
handler *nullHandler handler *nullHandler
} }
func (self *NullCache) InvertPagination() {
self.handler.invertPagination = true
}
type nullHandler struct { type nullHandler struct {
database Database database Database
attachments bool attachments bool
requireCaptcha bool requireCaptcha bool
name string name string
prefix string prefix string
translations string translations string
i18n map[string]*I18N i18n map[string]*I18N
access sync.Mutex access sync.Mutex
invertPagination bool
} }
func (self *nullHandler) ForEachI18N(v func(string)) { func (self *nullHandler) ForEachI18N(v func(string)) {
@ -83,7 +88,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
if strings.Trim(path, "/") == "overboard" { if strings.Trim(path, "/") == "overboard" {
// generate ukko aka overboard // generate ukko aka overboard
template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n) template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination)
return return
} }
@ -116,7 +121,8 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if page >= int(pages) { if page >= int(pages) {
goto notfound goto notfound
} }
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, page, w, self.database, isjson, i18n)
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
return return
} }
@ -130,7 +136,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
goto notfound goto notfound
} }
} }
template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n) template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination)
return return
} }
@ -159,17 +165,17 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
if strings.HasPrefix(file, "ukko.html") { if strings.HasPrefix(file, "ukko.html") {
template.genUkko(self.prefix, self.name, w, self.database, false, i18n) template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination)
return return
} }
if strings.HasPrefix(file, "ukko.json") { if strings.HasPrefix(file, "ukko.json") {
template.genUkko(self.prefix, self.name, w, self.database, true, i18n) template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination)
return return
} }
if strings.HasPrefix(file, "ukko-") { if strings.HasPrefix(file, "ukko-") {
page := getUkkoPage(file) page := getUkkoPage(file)
template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n) template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination)
return return
} }
if strings.HasPrefix(file, "thread-") { if strings.HasPrefix(file, "thread-") {
@ -214,7 +220,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if page >= int(pages) { if page >= int(pages) {
goto notfound goto notfound
} }
template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, page, w, self.database, isjson, i18n) template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination)
return return
} }

7
contrib/backends/srndv2/src/srnd/postgres.go
View File

@ -150,6 +150,7 @@ const GetNNTPPostsInGroup = "GetNNTPPostsInGroup"
const GetCitesByPostHashLike = "GetCitesByPostHashLike" const GetCitesByPostHashLike = "GetCitesByPostHashLike"
const GetYearlyPostHistory = "GetYearlyPostHistory" const GetYearlyPostHistory = "GetYearlyPostHistory"
const GetNewsgroupList = "GetNewsgroupList" const GetNewsgroupList = "GetNewsgroupList"
const CountUkko = "CountUkko"
func (self *PostgresDatabase) prepareStatements() { func (self *PostgresDatabase) prepareStatements() {
self.stmt = map[string]string{ self.stmt = map[string]string{
@ -220,6 +221,7 @@ func (self *PostgresDatabase) prepareStatements() {
GetNNTPPostsInGroup: "SELECT message_no, ArticlePosts.message_id, subject, time_posted, ref_id, name, path FROM ArticleNumbers INNER JOIN ArticlePosts ON ArticleNumbers.message_id = ArticlePosts.message_id WHERE ArticlePosts.newsgroup = $1 ORDER BY message_no", GetNNTPPostsInGroup: "SELECT message_no, ArticlePosts.message_id, subject, time_posted, ref_id, name, path FROM ArticleNumbers INNER JOIN ArticlePosts ON ArticleNumbers.message_id = ArticlePosts.message_id WHERE ArticlePosts.newsgroup = $1 ORDER BY message_no",
GetCitesByPostHashLike: "SELECT message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1", GetCitesByPostHashLike: "SELECT message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1",
GetYearlyPostHistory: "WITH times(endtime, begintime) AS ( SELECT CAST(EXTRACT(epoch from i) AS BIGINT) AS endtime, CAST(EXTRACT(epoch from i - interval '1 month') AS BIGINT) AS begintime FROM generate_series(now() - interval '10 year', now(), '1 month'::interval) i ) SELECT begintime, endtime, ( SELECT count(*) FROM ArticlePosts WHERE time_posted > begintime AND time_posted < endtime) FROM times", GetYearlyPostHistory: "WITH times(endtime, begintime) AS ( SELECT CAST(EXTRACT(epoch from i) AS BIGINT) AS endtime, CAST(EXTRACT(epoch from i - interval '1 month') AS BIGINT) AS begintime FROM generate_series(now() - interval '10 year', now(), '1 month'::interval) i ) SELECT begintime, endtime, ( SELECT count(*) FROM ArticlePosts WHERE time_posted > begintime AND time_posted < endtime) FROM times",
CountUkko: "SELECT COUNT(message_id) FROM ArticlePosts WHERE newsgroup != 'ctl' AND ref_id = '' OR ref_id = message_id",
} }
} }
@ -1999,6 +2001,11 @@ func (self *PostgresDatabase) FindCitesInText(text string) (msgids []string, err
return return
} }
func (self *PostgresDatabase) GetUkkoPageCount() (count int64, err error) {
err = self.conn.QueryRow(self.stmt[CountUkko]).Scan(&count)
return
}
func (self *PostgresDatabase) FindHeaders(group, headername string, lo, hi int64) (hdr ArticleHeaders, err error) { func (self *PostgresDatabase) FindHeaders(group, headername string, lo, hi int64) (hdr ArticleHeaders, err error) {
hdr = make(ArticleHeaders) hdr = make(ArticleHeaders)
q := "SELECT header_value FROM nntpheaders WHERE header_name = $1 AND header_article_message_id IN ( SELECT message_id FROM articleposts WHERE newsgroup = $2 )" q := "SELECT header_value FROM nntpheaders WHERE header_name = $1 AND header_article_message_id IN ( SELECT message_id FROM articleposts WHERE newsgroup = $2 )"

38
contrib/backends/srndv2/src/srnd/templates_impl.go
View File

@ -239,10 +239,15 @@ func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Wri
} }
// generate a board page // generate a board page
func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, page int, wr io.Writer, db Database, json bool, i18n *I18N) { func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, pages, page int, wr io.Writer, db Database, json bool, i18n *I18N, invertPagination bool) {
// get the board page model // get the board page model
perpage, _ := db.GetThreadsPerPage(newsgroup) perpage, _ := db.GetThreadsPerPage(newsgroup)
boardPage := db.GetGroupForPage(prefix, frontend, newsgroup, page, int(perpage)) var boardPage BoardModel
if invertPagination {
boardPage = db.GetGroupForPage(prefix, frontend, newsgroup, int(pages)-page, int(perpage))
} else {
boardPage = db.GetGroupForPage(prefix, frontend, newsgroup, page, int(perpage))
}
boardPage.Update(db) boardPage.Update(db)
boardPage.I18N(i18n) boardPage.I18N(i18n)
// render it // render it
@ -254,11 +259,20 @@ func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix
} }
} }
func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N) { func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N, invertPagination bool) {
self.genUkkoPaginated(prefix, frontend, wr, database, 0, json, i18n) var page int64
var err error
if invertPagination {
page, err = database.GetUkkoPageCount()
}
if err == nil {
self.genUkkoPaginated(prefix, frontend, wr, database, int(page), json, i18n, invertPagination)
} else {
log.Println("genUkko()", err.Error())
}
} }
func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, page int, json bool, i18n *I18N) { func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, page int, json bool, i18n *I18N, invertPagination bool) {
var threads []ThreadModel var threads []ThreadModel
for _, article := range database.GetLastBumpedThreadsPaginated("", 10, page*10) { for _, article := range database.GetLastBumpedThreadsPaginated("", 10, page*10) {
root := article[0] root := article[0]
@ -269,11 +283,19 @@ func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writ
} }
} }
obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page} obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page}
if page > 0 { if invertPagination {
obj["prev"] = map[string]interface{}{"no": page + 1}
} else if page > 0 {
obj["prev"] = map[string]interface{}{"no": page - 1} obj["prev"] = map[string]interface{}{"no": page - 1}
} }
if page < 10 { if invertPagination {
obj["next"] = map[string]interface{}{"no": page + 1} if page > 0 {
obj["next"] = map[string]interface{}{"no": page - 1}
}
} else {
if page < 10 {
obj["next"] = map[string]interface{}{"no": page + 1}
}
} }
if json { if json {
self.renderJSON(wr, obj) self.renderJSON(wr, obj)

4
contrib/backends/srndv2/src/srnd/varnish_cache.go
View File

@ -18,6 +18,10 @@ type VarnishCache struct {
invalidateChan chan *url.URL invalidateChan chan *url.URL
} }
func (self *VarnishCache) InvertPagination() {
self.handler.invertPagination = true
}
func (self *VarnishCache) invalidate(r string) { func (self *VarnishCache) invalidate(r string) {
var langs []string var langs []string
langs = append(langs, "") langs = append(langs, "")