From 2adcc73d92ef350793f3533a7cad257d55ea3615 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Fri, 9 Mar 2018 09:48:29 -0500 Subject: [PATCH] add inverted pagination for archive mode --- .../srndv2/src/srnd/cache_interface.go | 1 + contrib/backends/srndv2/src/srnd/daemon.go | 1 + contrib/backends/srndv2/src/srnd/database.go | 3 ++ .../backends/srndv2/src/srnd/file_cache.go | 24 +++++++----- contrib/backends/srndv2/src/srnd/frontend.go | 3 ++ .../backends/srndv2/src/srnd/frontend_http.go | 17 ++++++++- .../srndv2/src/srnd/frontend_multi.go | 6 +++ .../backends/srndv2/src/srnd/null_cache.go | 36 ++++++++++-------- contrib/backends/srndv2/src/srnd/postgres.go | 7 ++++ .../srndv2/src/srnd/templates_impl.go | 38 +++++++++++++++---- .../backends/srndv2/src/srnd/varnish_cache.go | 4 ++ 11 files changed, 105 insertions(+), 35 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/cache_interface.go b/contrib/backends/srndv2/src/srnd/cache_interface.go index 947f991..48107f5 100644 --- a/contrib/backends/srndv2/src/srnd/cache_interface.go +++ b/contrib/backends/srndv2/src/srnd/cache_interface.go @@ -26,6 +26,7 @@ type CacheInterface interface { GetHandler() CacheHandler SetRequireCaptcha(required bool) + InvertPagination() } //TODO only pass needed config diff --git a/contrib/backends/srndv2/src/srnd/daemon.go b/contrib/backends/srndv2/src/srnd/daemon.go index 72ef72e..8735e7a 100644 --- a/contrib/backends/srndv2/src/srnd/daemon.go +++ b/contrib/backends/srndv2/src/srnd/daemon.go @@ -586,6 +586,7 @@ func (self *NNTPDaemon) Run() { if self.conf.daemon["archive"] == "1" { log.Println("running in archive mode") self.expire = nil + self.frontend.ArchiveMode() } else { self.expire = createExpirationCore(self.database, self.store, self.informHooks) } diff --git a/contrib/backends/srndv2/src/srnd/database.go b/contrib/backends/srndv2/src/srnd/database.go index ed03830..b456487 100644 --- a/contrib/backends/srndv2/src/srnd/database.go +++ b/contrib/backends/srndv2/src/srnd/database.go @@ -339,6 +339,9 @@ type Database interface { // find headers in group with lo/hi watermark and list of patterns FindHeaders(group, headername string, lo, hi int64) (ArticleHeaders, error) + + // count ukko pages + GetUkkoPageCount() (int64, error) } func NewDatabase(db_type, schema, host, port, user, password string) Database { diff --git a/contrib/backends/srndv2/src/srnd/file_cache.go b/contrib/backends/srndv2/src/srnd/file_cache.go index 2d33616..ee3e3db 100644 --- a/contrib/backends/srndv2/src/srnd/file_cache.go +++ b/contrib/backends/srndv2/src/srnd/file_cache.go @@ -157,8 +157,9 @@ func (self *FileCache) pollRegen() { case _ = <-self.regenBoardTicker.C: self.regenBoardLock.Lock() for _, v := range self.regenBoardMap { - self.regenerateBoardPage(v.group, v.page, false) - self.regenerateBoardPage(v.group, v.page, true) + pages := self.database.GetGroupPageCount(v.group) + self.regenerateBoardPage(v.group, int(pages), v.page, false) + self.regenerateBoardPage(v.group, int(pages), v.page, true) } self.regenBoardMap = make(map[string]groupRegenRequest) self.regenBoardLock.Unlock() @@ -173,12 +174,15 @@ func (self *FileCache) pollRegen() { } } +func (self *FileCache) InvertPagination() { +} + // regen every page of the board func (self *FileCache) RegenerateBoard(group string) { pages, _ := self.database.GetPagesPerBoard(group) for page := 0; page < pages; page++ { - self.regenerateBoardPage(group, page, false) - self.regenerateBoardPage(group, page, true) + self.regenerateBoardPage(group, int(pages), page, false) + self.regenerateBoardPage(group, int(pages), page, true) } } @@ -200,7 +204,7 @@ func (self *FileCache) regenerateThread(root ArticleEntry, json bool) { } // regenerate just a page on a board -func (self *FileCache) regenerateBoardPage(board string, page int, json bool) { +func (self *FileCache) regenerateBoardPage(board string, pages, page int, json bool) { fname := self.getFilenameForBoardPage(board, page, json) wr, err := os.Create(fname) defer wr.Close() @@ -208,7 +212,7 @@ func (self *FileCache) regenerateBoardPage(board string, page int, json bool) { log.Println("error generating board page", page, "for", board, err) return } - template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, page, wr, self.database, json, nil) + template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, pages, page, wr, self.database, json, nil, false) } // regenerate the catalog for a board @@ -260,7 +264,7 @@ func (self *FileCache) regenUkko() { log.Println("error generating ukko markup", err) return } - template.genUkko(self.prefix, self.name, wr, self.database, false, nil) + template.genUkko(self.prefix, self.name, wr, self.database, false, nil, false) // json fname = filepath.Join(self.webroot_dir, "ukko.json") @@ -270,7 +274,7 @@ func (self *FileCache) regenUkko() { log.Println("error generating ukko json", err) return } - template.genUkko(self.prefix, self.name, wr, self.database, true, nil) + template.genUkko(self.prefix, self.name, wr, self.database, true, nil, false) i := 0 for i < 10 { fname := fmt.Sprintf("ukko-%d.html", i) @@ -281,14 +285,14 @@ func (self *FileCache) regenUkko() { return } defer f.Close() - template.genUkkoPaginated(self.prefix, self.name, f, self.database, i, false, nil) + template.genUkkoPaginated(self.prefix, self.name, f, self.database, i, false, nil, false) j, err := os.Create(jname) if err != nil { log.Printf("failed to create json ukko", i, err) return } defer j.Close() - template.genUkkoPaginated(self.prefix, self.name, j, self.database, i, true, nil) + template.genUkkoPaginated(self.prefix, self.name, j, self.database, i, true, nil, false) } } diff --git a/contrib/backends/srndv2/src/srnd/frontend.go b/contrib/backends/srndv2/src/srnd/frontend.go index af8dfc8..5937ad8 100644 --- a/contrib/backends/srndv2/src/srnd/frontend.go +++ b/contrib/backends/srndv2/src/srnd/frontend.go @@ -41,4 +41,7 @@ type Frontend interface { RegenOnModEvent(newsgroup, msgid, root string, page int) GetCacheHandler() CacheHandler + + // set archive mode + ArchiveMode() } diff --git a/contrib/backends/srndv2/src/srnd/frontend_http.go b/contrib/backends/srndv2/src/srnd/frontend_http.go index e9b5e28..6431cd2 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_http.go +++ b/contrib/backends/srndv2/src/srnd/frontend_http.go @@ -209,6 +209,9 @@ type httpFrontend struct { // this is a very important thing by the way requireCaptcha bool + + // are we in archive mode? + archive bool } // do we allow this newsgroup? @@ -240,6 +243,10 @@ func (self httpFrontend) deleteBoardMarkup(group string) { self.cache.DeleteBoardMarkup(group) } +func (self *httpFrontend) ArchiveMode() { + self.archive = true +} + // load post model and inform live ui func (self *httpFrontend) informLiveUI(msgid, ref, group string) { // root post @@ -387,8 +394,10 @@ func (self *httpFrontend) HandleNewPost(nntp frontendPost) { entry := ArticleEntry{msgid, group} // regnerate thread self.Regen(entry) - // regenerate all board pages - self.RegenerateBoard(group) + // regenerate all board pages if not archiving + if !self.archive { + self.RegenerateBoard(group) + } // regen front page self.RegenFrontPage() } @@ -1439,6 +1448,10 @@ func (self *httpFrontend) Mainloop() { // run daemon's mod engine with our frontend // go RunModEngine(self.daemon.mod, self.cache.RegenOnModEvent) + if self.archive { + self.cache.InvertPagination() + } + // start cache self.cache.Start() diff --git a/contrib/backends/srndv2/src/srnd/frontend_multi.go b/contrib/backends/srndv2/src/srnd/frontend_multi.go index 4272b31..2c18458 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_multi.go +++ b/contrib/backends/srndv2/src/srnd/frontend_multi.go @@ -15,6 +15,12 @@ func (self multiFrontend) GetCacheHandler() CacheHandler { return nil } +func (self multiFrontend) ArchiveMode() { + for _, f := range self.frontends { + f.ArchiveMode() + } +} + func (self multiFrontend) AllowNewsgroup(newsgroup string) bool { return true } diff --git a/contrib/backends/srndv2/src/srnd/null_cache.go b/contrib/backends/srndv2/src/srnd/null_cache.go index a396734..45dc137 100644 --- a/contrib/backends/srndv2/src/srnd/null_cache.go +++ b/contrib/backends/srndv2/src/srnd/null_cache.go @@ -15,15 +15,20 @@ type NullCache struct { handler *nullHandler } +func (self *NullCache) InvertPagination() { + self.handler.invertPagination = true +} + type nullHandler struct { - database Database - attachments bool - requireCaptcha bool - name string - prefix string - translations string - i18n map[string]*I18N - access sync.Mutex + database Database + attachments bool + requireCaptcha bool + name string + prefix string + translations string + i18n map[string]*I18N + access sync.Mutex + invertPagination bool } func (self *nullHandler) ForEachI18N(v func(string)) { @@ -83,7 +88,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if strings.Trim(path, "/") == "overboard" { // generate ukko aka overboard - template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n) + template.genUkko(self.prefix, self.name, w, self.database, isjson, i18n, self.invertPagination) return } @@ -116,7 +121,8 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if page >= int(pages) { goto notfound } - template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, page, w, self.database, isjson, i18n) + + template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination) return } @@ -130,7 +136,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { goto notfound } } - template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n) + template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination) return } @@ -159,17 +165,17 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if strings.HasPrefix(file, "ukko.html") { - template.genUkko(self.prefix, self.name, w, self.database, false, i18n) + template.genUkko(self.prefix, self.name, w, self.database, false, i18n, self.invertPagination) return } if strings.HasPrefix(file, "ukko.json") { - template.genUkko(self.prefix, self.name, w, self.database, true, i18n) + template.genUkko(self.prefix, self.name, w, self.database, true, i18n, self.invertPagination) return } if strings.HasPrefix(file, "ukko-") { page := getUkkoPage(file) - template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n) + template.genUkkoPaginated(self.prefix, self.name, w, self.database, page, isjson, i18n, self.invertPagination) return } if strings.HasPrefix(file, "thread-") { @@ -214,7 +220,7 @@ func (self *nullHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if page >= int(pages) { goto notfound } - template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, page, w, self.database, isjson, i18n) + template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, group, int(pages), page, w, self.database, isjson, i18n, self.invertPagination) return } diff --git a/contrib/backends/srndv2/src/srnd/postgres.go b/contrib/backends/srndv2/src/srnd/postgres.go index ad05897..2bd7b3b 100644 --- a/contrib/backends/srndv2/src/srnd/postgres.go +++ b/contrib/backends/srndv2/src/srnd/postgres.go @@ -150,6 +150,7 @@ const GetNNTPPostsInGroup = "GetNNTPPostsInGroup" const GetCitesByPostHashLike = "GetCitesByPostHashLike" const GetYearlyPostHistory = "GetYearlyPostHistory" const GetNewsgroupList = "GetNewsgroupList" +const CountUkko = "CountUkko" func (self *PostgresDatabase) prepareStatements() { self.stmt = map[string]string{ @@ -220,6 +221,7 @@ func (self *PostgresDatabase) prepareStatements() { GetNNTPPostsInGroup: "SELECT message_no, ArticlePosts.message_id, subject, time_posted, ref_id, name, path FROM ArticleNumbers INNER JOIN ArticlePosts ON ArticleNumbers.message_id = ArticlePosts.message_id WHERE ArticlePosts.newsgroup = $1 ORDER BY message_no", GetCitesByPostHashLike: "SELECT message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1", GetYearlyPostHistory: "WITH times(endtime, begintime) AS ( SELECT CAST(EXTRACT(epoch from i) AS BIGINT) AS endtime, CAST(EXTRACT(epoch from i - interval '1 month') AS BIGINT) AS begintime FROM generate_series(now() - interval '10 year', now(), '1 month'::interval) i ) SELECT begintime, endtime, ( SELECT count(*) FROM ArticlePosts WHERE time_posted > begintime AND time_posted < endtime) FROM times", + CountUkko: "SELECT COUNT(message_id) FROM ArticlePosts WHERE newsgroup != 'ctl' AND ref_id = '' OR ref_id = message_id", } } @@ -1999,6 +2001,11 @@ func (self *PostgresDatabase) FindCitesInText(text string) (msgids []string, err return } +func (self *PostgresDatabase) GetUkkoPageCount() (count int64, err error) { + err = self.conn.QueryRow(self.stmt[CountUkko]).Scan(&count) + return +} + func (self *PostgresDatabase) FindHeaders(group, headername string, lo, hi int64) (hdr ArticleHeaders, err error) { hdr = make(ArticleHeaders) q := "SELECT header_value FROM nntpheaders WHERE header_name = $1 AND header_article_message_id IN ( SELECT message_id FROM articleposts WHERE newsgroup = $2 )" diff --git a/contrib/backends/srndv2/src/srnd/templates_impl.go b/contrib/backends/srndv2/src/srnd/templates_impl.go index 13074ed..f2047b1 100644 --- a/contrib/backends/srndv2/src/srnd/templates_impl.go +++ b/contrib/backends/srndv2/src/srnd/templates_impl.go @@ -239,10 +239,15 @@ func (self *templateEngine) genCatalog(prefix, frontend, group string, wr io.Wri } // generate a board page -func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, page int, wr io.Writer, db Database, json bool, i18n *I18N) { +func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix, frontend, newsgroup string, pages, page int, wr io.Writer, db Database, json bool, i18n *I18N, invertPagination bool) { // get the board page model perpage, _ := db.GetThreadsPerPage(newsgroup) - boardPage := db.GetGroupForPage(prefix, frontend, newsgroup, page, int(perpage)) + var boardPage BoardModel + if invertPagination { + boardPage = db.GetGroupForPage(prefix, frontend, newsgroup, int(pages)-page, int(perpage)) + } else { + boardPage = db.GetGroupForPage(prefix, frontend, newsgroup, page, int(perpage)) + } boardPage.Update(db) boardPage.I18N(i18n) // render it @@ -254,11 +259,20 @@ func (self *templateEngine) genBoardPage(allowFiles, requireCaptcha bool, prefix } } -func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N) { - self.genUkkoPaginated(prefix, frontend, wr, database, 0, json, i18n) +func (self *templateEngine) genUkko(prefix, frontend string, wr io.Writer, database Database, json bool, i18n *I18N, invertPagination bool) { + var page int64 + var err error + if invertPagination { + page, err = database.GetUkkoPageCount() + } + if err == nil { + self.genUkkoPaginated(prefix, frontend, wr, database, int(page), json, i18n, invertPagination) + } else { + log.Println("genUkko()", err.Error()) + } } -func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, page int, json bool, i18n *I18N) { +func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writer, database Database, page int, json bool, i18n *I18N, invertPagination bool) { var threads []ThreadModel for _, article := range database.GetLastBumpedThreadsPaginated("", 10, page*10) { root := article[0] @@ -269,11 +283,19 @@ func (self *templateEngine) genUkkoPaginated(prefix, frontend string, wr io.Writ } } obj := map[string]interface{}{"prefix": prefix, "threads": threads, "page": page} - if page > 0 { + if invertPagination { + obj["prev"] = map[string]interface{}{"no": page + 1} + } else if page > 0 { obj["prev"] = map[string]interface{}{"no": page - 1} } - if page < 10 { - obj["next"] = map[string]interface{}{"no": page + 1} + if invertPagination { + if page > 0 { + obj["next"] = map[string]interface{}{"no": page - 1} + } + } else { + if page < 10 { + obj["next"] = map[string]interface{}{"no": page + 1} + } } if json { self.renderJSON(wr, obj) diff --git a/contrib/backends/srndv2/src/srnd/varnish_cache.go b/contrib/backends/srndv2/src/srnd/varnish_cache.go index 4bf1869..da2118b 100644 --- a/contrib/backends/srndv2/src/srnd/varnish_cache.go +++ b/contrib/backends/srndv2/src/srnd/varnish_cache.go @@ -18,6 +18,10 @@ type VarnishCache struct { invalidateChan chan *url.URL } +func (self *VarnishCache) InvertPagination() { + self.handler.invertPagination = true +} + func (self *VarnishCache) invalidate(r string) { var langs []string langs = append(langs, "")