package srnd import ( "encoding/json" "fmt" "log" "net/http" "os" "path/filepath" "sync" "time" ) type FileCache struct { database Database store ArticleStore handler http.Handler webroot_dir string name string regen_threads int attachments bool requireCaptcha bool prefix string regenThreadChan chan ArticleEntry regenGroupChan chan groupRegenRequest regenBoardMap map[string]groupRegenRequest regenThreadMap map[string]ArticleEntry regenCatalogMap map[string]bool regenBoardTicker *time.Ticker ukkoTicker *time.Ticker longTermTicker *time.Ticker regenThreadTicker *time.Ticker regenCatalogTicker *time.Ticker regenThreadLock sync.RWMutex regenBoardLock sync.RWMutex regenCatalogLock sync.RWMutex } func (self *FileCache) DeleteBoardMarkup(group string) { pages64 := self.database.GetGroupPageCount(group) pages := int(pages64) for page := 0; page < pages; page++ { fname := self.getFilenameForBoardPage(group, page, false) os.Remove(fname) fname = self.getFilenameForBoardPage(group, page, true) os.Remove(fname) } } // try to delete root post's page func (self *FileCache) DeleteThreadMarkup(root_post_id string) { fname := self.getFilenameForThread(root_post_id, false) os.Remove(fname) fname = self.getFilenameForThread(root_post_id, true) os.Remove(fname) } func (self *FileCache) getFilenameForThread(root_post_id string, json bool) string { var ext string if json { ext = "json" } else { ext = "html" } fname := fmt.Sprintf("thread-%s.%s", HashMessageID(root_post_id), ext) return filepath.Join(self.webroot_dir, fname) } func (self *FileCache) getFilenameForBoardPage(boardname string, pageno int, json bool) string { var ext string if json { ext = "json" } else { ext = "html" } fname := fmt.Sprintf("%s-%d.%s", boardname, pageno, ext) return filepath.Join(self.webroot_dir, fname) } func (self *FileCache) getFilenameForCatalog(boardname string) string { fname := fmt.Sprintf("catalog-%s.html", boardname) return filepath.Join(self.webroot_dir, fname) } // regen every newsgroup func (self *FileCache) RegenAll() { log.Println("regen all on http frontend") // get all groups groups := self.database.GetAllNewsgroups() if groups != nil { for _, group := range groups { // send every thread for this group down the regen thread channel go self.database.GetGroupThreads(group, self.regenThreadChan) pages := self.database.GetGroupPageCount(group) var pg int64 for pg = 0; pg < pages; pg++ { self.regenGroupChan <- groupRegenRequest{group, int(pg)} } } } } func (self *FileCache) regenLongTerm() { wr, err := os.Create(filepath.Join(self.webroot_dir, "history.html")) defer wr.Close() if err != nil { log.Println("cannot render history graph", err) return } template.genGraphs(self.prefix, wr, self.database, nil) } func (self *FileCache) pollLongTerm() { for { <-self.longTermTicker.C // regenerate long term stuff self.regenLongTerm() } } func (self *FileCache) pollRegen() { for { select { // listen for regen board requests case req := <-self.regenGroupChan: self.regenBoardLock.Lock() self.regenBoardMap[fmt.Sprintf("%s|%s", req.group, req.page)] = req self.regenBoardLock.Unlock() self.regenCatalogLock.Lock() self.regenCatalogMap[req.group] = true self.regenCatalogLock.Unlock() // listen for regen thread requests case entry := <-self.regenThreadChan: self.regenThreadLock.Lock() self.regenThreadMap[fmt.Sprintf("%s|%s", entry[0], entry[1])] = entry self.regenThreadLock.Unlock() // regen ukko case _ = <-self.ukkoTicker.C: self.regenUkko() self.RegenFrontPage() case _ = <-self.regenThreadTicker.C: self.regenThreadLock.Lock() for _, entry := range self.regenThreadMap { self.regenerateThread(entry, false) self.regenerateThread(entry, true) } self.regenThreadMap = make(map[string]ArticleEntry) self.regenThreadLock.Unlock() case _ = <-self.regenBoardTicker.C: self.regenBoardLock.Lock() for _, v := range self.regenBoardMap { self.regenerateBoardPage(v.group, v.page, false) self.regenerateBoardPage(v.group, v.page, true) } self.regenBoardMap = make(map[string]groupRegenRequest) self.regenBoardLock.Unlock() case _ = <-self.regenCatalogTicker.C: self.regenCatalogLock.Lock() for board, _ := range self.regenCatalogMap { self.regenerateCatalog(board) } self.regenCatalogMap = make(map[string]bool) self.regenCatalogLock.Unlock() } } } // regen every page of the board func (self *FileCache) RegenerateBoard(group string) { pages, _ := self.database.GetPagesPerBoard(group) for page := 0; page < pages; page++ { self.regenerateBoardPage(group, page, false) self.regenerateBoardPage(group, page, true) } } // regenerate just a thread page func (self *FileCache) regenerateThread(root ArticleEntry, json bool) { msgid := root.MessageID() if self.store.HasArticle(msgid) { fname := self.getFilenameForThread(msgid, json) wr, err := os.Create(fname) defer wr.Close() if err != nil { log.Println("did not write", fname, err) return } template.genThread(self.attachments, self.requireCaptcha, root, self.prefix, self.name, wr, self.database, json, nil) } else { log.Println("don't have root post", msgid, "not regenerating thread") } } // regenerate just a page on a board func (self *FileCache) regenerateBoardPage(board string, page int, json bool) { fname := self.getFilenameForBoardPage(board, page, json) wr, err := os.Create(fname) defer wr.Close() if err != nil { log.Println("error generating board page", page, "for", board, err) return } template.genBoardPage(self.attachments, self.requireCaptcha, self.prefix, self.name, board, page, wr, self.database, json, nil) } // regenerate the catalog for a board func (self *FileCache) regenerateCatalog(board string) { fname := self.getFilenameForCatalog(board) wr, err := os.Create(fname) defer wr.Close() if err != nil { log.Println("error generating catalog for", board, err) return } template.genCatalog(self.prefix, self.name, board, wr, self.database, nil) } // regenerate the front page func (self *FileCache) RegenFrontPage() { indexwr, err1 := os.Create(filepath.Join(self.webroot_dir, "index.html")) defer indexwr.Close() if err1 != nil { log.Println("cannot render front page", err1) return } boardswr, err2 := os.Create(filepath.Join(self.webroot_dir, "boards.html")) defer boardswr.Close() if err2 != nil { log.Println("cannot render board list page", err2) return } template.genFrontPage(10, self.prefix, self.name, indexwr, boardswr, self.database, nil) j_boardswr, err2 := os.Create(filepath.Join(self.webroot_dir, "boards.json")) g := self.database.GetAllNewsgroups() err := json.NewEncoder(j_boardswr).Encode(g) if err != nil { log.Println("cannot render boards.json", err) } } // regenerate the overboard func (self *FileCache) regenUkko() { // markup fname := filepath.Join(self.webroot_dir, "ukko.html") wr, err := os.Create(fname) defer wr.Close() if err != nil { log.Println("error generating ukko markup", err) return } template.genUkko(self.prefix, self.name, wr, self.database, false, nil) // json fname = filepath.Join(self.webroot_dir, "ukko.json") wr, err = os.Create(fname) defer wr.Close() if err != nil { log.Println("error generating ukko json", err) return } template.genUkko(self.prefix, self.name, wr, self.database, true, nil) i := 0 for i < 10 { fname := fmt.Sprintf("ukko-%d.html", i) jname := fmt.Sprintf("ukko-%d.json", i) f, err := os.Create(fname) if err != nil { log.Println("Failed to create html ukko", i, err) return } defer f.Close() template.genUkkoPaginated(self.prefix, self.name, f, self.database, i, false, nil) j, err := os.Create(jname) if err != nil { log.Printf("failed to create json ukko", i, err) return } defer j.Close() template.genUkkoPaginated(self.prefix, self.name, j, self.database, i, true, nil) } } // regenerate pages after a mod event func (self *FileCache) RegenOnModEvent(newsgroup, msgid, root string, page int) { if root == msgid { fname := self.getFilenameForThread(root, false) os.Remove(fname) fname = self.getFilenameForThread(root, true) os.Remove(fname) } else { self.regenThreadChan <- ArticleEntry{root, newsgroup} } self.regenGroupChan <- groupRegenRequest{newsgroup, int(page)} self.regenUkko() } func (self *FileCache) Start() { threads := self.regen_threads // check for invalid number of threads if threads <= 0 { threads = 1 } // use N threads for regeneration for threads > 0 { go self.pollRegen() threads-- } // run long term regen jobs go self.pollLongTerm() } func (self *FileCache) Regen(msg ArticleEntry) { self.regenThreadChan <- msg self.RegenerateBoard(msg.Newsgroup()) } func (self *FileCache) GetThreadChan() chan ArticleEntry { return self.regenThreadChan } func (self *FileCache) GetGroupChan() chan groupRegenRequest { return self.regenGroupChan } func (self *FileCache) GetI18N(r *http.Request) *I18N { return nil } func (self *FileCache) GetHandler() CacheHandler { return self } func (self *FileCache) ServeHTTP(w http.ResponseWriter, r *http.Request) { self.handler.ServeHTTP(w, r) } func (self *FileCache) Close() { //nothig to do } func (self *FileCache) SetRequireCaptcha(require bool) { self.requireCaptcha = require } func NewFileCache(prefix, webroot, name string, threads int, attachments bool, db Database, store ArticleStore) CacheInterface { cache := new(FileCache) cache.handler = http.FileServer(http.Dir(webroot)) cache.regenBoardTicker = time.NewTicker(time.Second * 10) cache.longTermTicker = time.NewTicker(time.Hour) cache.ukkoTicker = time.NewTicker(time.Second * 30) cache.regenThreadTicker = time.NewTicker(time.Second) cache.regenCatalogTicker = time.NewTicker(time.Second * 20) cache.regenBoardMap = make(map[string]groupRegenRequest) cache.regenThreadMap = make(map[string]ArticleEntry) cache.regenCatalogMap = make(map[string]bool) cache.regenThreadChan = make(chan ArticleEntry, 16) cache.regenGroupChan = make(chan groupRegenRequest, 8) cache.prefix = prefix cache.webroot_dir = webroot cache.name = name cache.regen_threads = threads cache.attachments = attachments cache.database = db cache.store = store return cache }