From 42cc7f26c417070ac9f8b0ab22a0b56910fa544c Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Sat, 30 Sep 2017 06:57:17 -0400 Subject: [PATCH] remove more channels to prevent deadlocks --- .../srndv2/src/srnd/cache_interface.go | 3 - .../backends/srndv2/src/srnd/frontend_http.go | 30 ++----- contrib/backends/srndv2/src/srnd/mod_http.go | 2 +- .../backends/srndv2/src/srnd/null_cache.go | 29 +------ .../backends/srndv2/src/srnd/varnish_cache.go | 85 ++++++++----------- 5 files changed, 43 insertions(+), 106 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/cache_interface.go b/contrib/backends/srndv2/src/srnd/cache_interface.go index 60f457d..dd10af8 100644 --- a/contrib/backends/srndv2/src/srnd/cache_interface.go +++ b/contrib/backends/srndv2/src/srnd/cache_interface.go @@ -17,9 +17,6 @@ type CacheInterface interface { Start() Close() - - GetThreadChan() chan ArticleEntry - GetGroupChan() chan groupRegenRequest GetHandler() http.Handler SetRequireCaptcha(required bool) diff --git a/contrib/backends/srndv2/src/srnd/frontend_http.go b/contrib/backends/srndv2/src/srnd/frontend_http.go index 299a417..4ae779b 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_http.go +++ b/contrib/backends/srndv2/src/srnd/frontend_http.go @@ -188,9 +188,7 @@ type httpFrontend struct { regen_on_start bool attachments bool - prefix string - regenThreadChan chan ArticleEntry - regenGroupChan chan groupRegenRequest + prefix string store *sessions.CookieStore @@ -224,15 +222,14 @@ func (self httpFrontend) AllowNewsgroup(group string) bool { func (self *httpFrontend) Regen(msg ArticleEntry) { self.cache.Regen(msg) } +func (self *httpFrontend) RegenerateBoard(board string) { + self.cache.RegenerateBoard(board) +} func (self httpFrontend) regenAll() { self.cache.RegenAll() } -func (self *httpFrontend) regenerateBoard(group string) { - self.cache.RegenerateBoard(group) -} - func (self httpFrontend) deleteThreadMarkup(root_post_id string) { self.cache.DeleteThreadMarkup(root_post_id) } @@ -414,19 +411,9 @@ func (self *httpFrontend) HandleNewPost(nntp frontendPost) { } entry := ArticleEntry{msgid, group} // regnerate thread - self.regenThreadChan <- entry - // regen the newsgroup we're in - // TODO: regen only what we need to - pages := self.daemon.database.GetGroupPageCount(group) - // regen all pages - var page int64 - for ; page < pages; page++ { - req := groupRegenRequest{ - group: group, - page: int(page), - } - self.regenGroupChan <- req - } + self.Regen(entry) + // regenerate all board pages + self.RegenerateBoard(group) } // create a new captcha, return as json object @@ -1565,9 +1552,6 @@ func NewHTTPFrontend(daemon *NNTPDaemon, cache CacheInterface, config map[string MaxAge: 600, } - front.regenThreadChan = front.cache.GetThreadChan() - front.regenGroupChan = front.cache.GetGroupChan() - // liveui related members front.liveui_chnl = make(chan PostModel, 128) front.liveui_register = make(chan *liveChan) diff --git a/contrib/backends/srndv2/src/srnd/mod_http.go b/contrib/backends/srndv2/src/srnd/mod_http.go index 0408213..4fca5ce 100644 --- a/contrib/backends/srndv2/src/srnd/mod_http.go +++ b/contrib/backends/srndv2/src/srnd/mod_http.go @@ -35,7 +35,7 @@ type httpModUI struct { } func createHttpModUI(frontend *httpFrontend) httpModUI { - return httpModUI{frontend.regenAll, frontend.Regen, frontend.regenerateBoard, frontend.deleteThreadMarkup, frontend.deleteBoardMarkup, make(chan NNTPMessage), frontend.daemon, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/"} + return httpModUI{frontend.regenAll, frontend.Regen, frontend.RegenerateBoard, frontend.deleteThreadMarkup, frontend.deleteBoardMarkup, make(chan NNTPMessage), frontend.daemon, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/"} } diff --git a/contrib/backends/srndv2/src/srnd/null_cache.go b/contrib/backends/srndv2/src/srnd/null_cache.go index 97f3954..a0ab710 100644 --- a/contrib/backends/srndv2/src/srnd/null_cache.go +++ b/contrib/backends/srndv2/src/srnd/null_cache.go @@ -10,9 +10,7 @@ import ( ) type NullCache struct { - regenThreadChan chan ArticleEntry - regenGroupChan chan groupRegenRequest - handler *nullHandler + handler *nullHandler } type nullHandler struct { @@ -202,20 +200,6 @@ func (self *NullCache) SetRequireCaptcha(required bool) { self.handler.requireCaptcha = required } -func (self *NullCache) pollRegen() { - for { - select { - // consume regen requests - case _ = <-self.regenGroupChan: - { - } - case _ = <-self.regenThreadChan: - { - } - } - } -} - // regen every page of the board func (self *NullCache) RegenerateBoard(group string) { } @@ -225,20 +209,11 @@ func (self *NullCache) RegenOnModEvent(newsgroup, msgid, root string, page int) } func (self *NullCache) Start() { - go self.pollRegen() } func (self *NullCache) Regen(msg ArticleEntry) { } -func (self *NullCache) GetThreadChan() chan ArticleEntry { - return self.regenThreadChan -} - -func (self *NullCache) GetGroupChan() chan groupRegenRequest { - return self.regenGroupChan -} - func (self *NullCache) GetHandler() http.Handler { return self.handler } @@ -249,8 +224,6 @@ func (self *NullCache) Close() { func NewNullCache(prefix, webroot, name string, attachments bool, db Database, store ArticleStore) CacheInterface { cache := new(NullCache) - cache.regenThreadChan = make(chan ArticleEntry, 16) - cache.regenGroupChan = make(chan groupRegenRequest, 8) cache.handler = &nullHandler{ prefix: prefix, name: name, diff --git a/contrib/backends/srndv2/src/srnd/varnish_cache.go b/contrib/backends/srndv2/src/srnd/varnish_cache.go index 05ccf9b..cdd4521 100644 --- a/contrib/backends/srndv2/src/srnd/varnish_cache.go +++ b/contrib/backends/srndv2/src/srnd/varnish_cache.go @@ -9,12 +9,11 @@ import ( ) type VarnishCache struct { - varnish_url string - prefix string - handler *nullHandler - client *http.Client - regenThreadChan chan ArticleEntry - regenGroupChan chan groupRegenRequest + varnish_url string + prefix string + handler *nullHandler + client *http.Client + threadsRegenChan chan ArticleEntry } func (self *VarnishCache) invalidate(r string) { @@ -33,8 +32,8 @@ func (self *VarnishCache) invalidate(r string) { func (self *VarnishCache) DeleteBoardMarkup(group string) { n, _ := self.handler.database.GetPagesPerBoard(group) for n > 0 { - go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n)) - go self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, group, n)) + self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n)) + self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, group, n)) n-- } self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, group)) @@ -51,7 +50,7 @@ func (self *VarnishCache) RegenAll() { // we will do this as it's used by rengen on start for frontend groups := self.handler.database.GetAllNewsgroups() for _, group := range groups { - self.handler.database.GetGroupThreads(group, self.regenThreadChan) + self.handler.database.GetGroupThreads(group, self.threadsRegenChan) } } @@ -59,34 +58,18 @@ func (self *VarnishCache) RegenFrontPage() { self.invalidate(fmt.Sprintf("%s%s", self.varnish_url, self.prefix)) // TODO: this is also lazy af self.invalidate(fmt.Sprintf("%s%shistory.html", self.varnish_url, self.prefix)) + self.invalidateUkko(10) } -func (self *VarnishCache) invalidateUkko() { +func (self *VarnishCache) invalidateUkko(pages int) { // TODO: invalidate paginated ukko self.invalidate(fmt.Sprintf("%s%sukko.html", self.varnish_url, self.prefix)) self.invalidate(fmt.Sprintf("%s%soverboard/", self.varnish_url, self.prefix)) self.invalidate(fmt.Sprintf("%s%so/", self.varnish_url, self.prefix)) - // TODO: this is lazy af - self.RegenFrontPage() -} - -func (self *VarnishCache) pollRegen() { - for { - select { - // consume regen requests - case ev := <-self.regenGroupChan: - { - self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, ev.group, ev.page)) - self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, ev.group, ev.page)) - if ev.page == 0 { - self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, ev.group)) - } - } - case ev := <-self.regenThreadChan: - { - self.Regen(ev) - } - } + n := 0 + for n < pages { + self.invalidate(fmt.Sprintf("%s%so/%d/", self.varnish_url, self.prefix, n)) + n++ } } @@ -94,8 +77,8 @@ func (self *VarnishCache) pollRegen() { func (self *VarnishCache) RegenerateBoard(group string) { n, _ := self.handler.database.GetPagesPerBoard(group) for n > 0 { - go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n)) - go self.invalidate(fmt.Sprintf("%s%s%s/%d/", self.varnish_url, self.prefix, group, n)) + self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n)) + self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, group, n)) n-- } self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, group)) @@ -103,28 +86,29 @@ func (self *VarnishCache) RegenerateBoard(group string) { // regenerate pages after a mod event func (self *VarnishCache) RegenOnModEvent(newsgroup, msgid, root string, page int) { - self.regenGroupChan <- groupRegenRequest{newsgroup, page} - self.regenThreadChan <- ArticleEntry{newsgroup, root} + self.Regen(ArticleEntry{newsgroup, root}) + if page == 0 { + self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, newsgroup)) + } + self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, newsgroup, page)) +} + +func (self *VarnishCache) poll() { + for { + ent := <-self.threadsRegenChan + self.Regen(ent) + } } func (self *VarnishCache) Start() { - go self.pollRegen() + go self.poll() } func (self *VarnishCache) Regen(msg ArticleEntry) { - go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, msg.Newsgroup(), 0)) - go self.invalidate(fmt.Sprintf("%s%s%s/%d/", self.varnish_url, self.prefix, msg.Newsgroup(), 0)) - go self.invalidate(fmt.Sprintf("%s%sthread-%s.html", self.varnish_url, self.prefix, HashMessageID(msg.MessageID()))) - go self.invalidate(fmt.Sprintf("%s%st/%s/", self.varnish_url, self.prefix, HashMessageID(msg.MessageID()))) - self.invalidateUkko() -} - -func (self *VarnishCache) GetThreadChan() chan ArticleEntry { - return self.regenThreadChan -} - -func (self *VarnishCache) GetGroupChan() chan groupRegenRequest { - return self.regenGroupChan + self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, msg.Newsgroup(), 0)) + self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, msg.Newsgroup(), 0)) + self.invalidate(fmt.Sprintf("%s%sthread-%s.html", self.varnish_url, self.prefix, HashMessageID(msg.MessageID()))) + self.invalidate(fmt.Sprintf("%s%st/%s/", self.varnish_url, self.prefix, HashMessageID(msg.MessageID()))) } func (self *VarnishCache) GetHandler() http.Handler { @@ -141,8 +125,7 @@ func (self *VarnishCache) SetRequireCaptcha(required bool) { func NewVarnishCache(varnish_url, bind_addr, prefix, webroot, name string, attachments bool, db Database, store ArticleStore) CacheInterface { cache := new(VarnishCache) - cache.regenThreadChan = make(chan ArticleEntry, 16) - cache.regenGroupChan = make(chan groupRegenRequest, 8) + cache.threadsRegenChan = make(chan ArticleEntry) local_addr, err := net.ResolveTCPAddr("tcp", bind_addr) if err != nil { log.Fatalf("failed to resolve %s for varnish cache: %s", bind_addr, err)