From e224ee7aabd89b5fcc71cdc166c468d102f6ee7a Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Sat, 23 Sep 2017 09:28:25 -0400 Subject: [PATCH] set in-reply-to and fetch missing cites --- contrib/backends/srndv2/src/srnd/daemon.go | 22 +++++++-------- contrib/backends/srndv2/src/srnd/database.go | 3 +++ .../backends/srndv2/src/srnd/frontend_http.go | 11 ++++++++ contrib/backends/srndv2/src/srnd/nntp.go | 17 +++++++++--- contrib/backends/srndv2/src/srnd/postgres.go | 27 +++++++++++++++++++ 5 files changed, 65 insertions(+), 15 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/daemon.go b/contrib/backends/srndv2/src/srnd/daemon.go index 27b5466..5ce07e4 100644 --- a/contrib/backends/srndv2/src/srnd/daemon.go +++ b/contrib/backends/srndv2/src/srnd/daemon.go @@ -114,7 +114,7 @@ type NNTPDaemon struct { // channel for broadcasting a message to all feeds given their newsgroup, message_id send_all_feeds chan ArticleEntry // channel for broadcasting an ARTICLE command to all feeds in reader mode - ask_for_article chan ArticleEntry + ask_for_article chan string // operation of daemon done after sending bool down this channel done chan bool @@ -123,7 +123,7 @@ type NNTPDaemon struct { send_articles_mtx sync.RWMutex send_articles []ArticleEntry ask_articles_mtx sync.RWMutex - ask_articles []ArticleEntry + ask_articles []string pump_ticker *time.Ticker expiration_ticker *time.Ticker @@ -535,7 +535,7 @@ func (self *NNTPDaemon) Run() { self.get_feeds = make(chan chan []*feedStatus) self.get_feed = make(chan *feedStatusQuery) self.modify_feed_policy = make(chan *modifyFeedPolicyEvent) - self.ask_for_article = make(chan ArticleEntry) + self.ask_for_article = make(chan string) self.pump_ticker = time.NewTicker(time.Millisecond * 100) if self.conf.daemon["archive"] == "1" { @@ -823,10 +823,10 @@ func (self *NNTPDaemon) pump_article_requests() { } articles = nil self.ask_articles_mtx.Lock() - articles = append(articles, self.ask_articles...) + msgids := self.ask_articles self.ask_articles = nil self.ask_articles_mtx.Unlock() - for _, entry := range articles { + for _, entry := range msgids { self.ask_for_article <- entry } articles = nil @@ -898,15 +898,13 @@ func (self *NNTPDaemon) poll(worker int) { for _, f := range feeds { var send []*nntpConnection for _, feed := range f.Conns { - if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) { - if strings.HasSuffix(feed.name, "-reader") { - send = append(send, feed) - } + if strings.HasSuffix(feed.name, "-reader") { + send = append(send, feed) } } minconn := lowestBacklogConnection(send) if minconn != nil { - minconn.askForArticle(nntp.MessageID()) + go minconn.askForArticle(nntp) } } } @@ -928,9 +926,9 @@ func lowestBacklogConnection(conns []*nntpConnection) (minconn *nntpConnection) return } -func (self *NNTPDaemon) askForArticle(e ArticleEntry) { +func (self *NNTPDaemon) askForArticle(msgid string) { self.ask_articles_mtx.Lock() - self.ask_articles = append(self.ask_articles, e) + self.ask_articles = append(self.ask_articles, msgid) self.ask_articles_mtx.Unlock() } diff --git a/contrib/backends/srndv2/src/srnd/database.go b/contrib/backends/srndv2/src/srnd/database.go index 403b542..7432d7f 100644 --- a/contrib/backends/srndv2/src/srnd/database.go +++ b/contrib/backends/srndv2/src/srnd/database.go @@ -330,6 +330,9 @@ type Database interface { // get newsgroup list with watermarks GetNewsgroupList() (NewsgroupList, error) + + // find cites in text + FindCitesInText(msg string) ([]string, error) } func NewDatabase(db_type, schema, host, port, user, password string) Database { diff --git a/contrib/backends/srndv2/src/srnd/frontend_http.go b/contrib/backends/srndv2/src/srnd/frontend_http.go index e5d4958..4795e59 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_http.go +++ b/contrib/backends/srndv2/src/srnd/frontend_http.go @@ -865,6 +865,17 @@ func (self *httpFrontend) handle_postRequest(pr *postRequest, b bannedFunc, e er // set message nntp.message = nntpSanitize(pr.Message) + + cites, err := self.daemon.database.FindCitesInText(nntp.message) + if err != nil { + e(err) + return + } + + if len(cites) > 0 { + nntp.headers.Set("In-Reply-To", strings.Join(cites, " ")) + } + // set date nntp.headers.Set("Date", timeNowStr()) // append path from frontend diff --git a/contrib/backends/srndv2/src/srnd/nntp.go b/contrib/backends/srndv2/src/srnd/nntp.go index ac37ae4..d87cd84 100644 --- a/contrib/backends/srndv2/src/srnd/nntp.go +++ b/contrib/backends/srndv2/src/srnd/nntp.go @@ -560,6 +560,17 @@ func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEH _, err = io.Copy(Discard, body) return } + + // ask for replies + replyTos := strings.Split(hdr.Get("In-Reply-To"), " ") + for _, reply := range replyTos { + if ValidMessageID(reply) { + if !daemon.store.HasArticle(reply) { + go daemon.askForArticle(reply) + } + } + } + path := hdr.Get("Path") hdr.Set("Path", daemon.instance_name+"!"+path) // now store attachments and article @@ -741,7 +752,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string newsgroup := hdr.Get("Newsgroups") if reference != "" && ValidMessageID(reference) && !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) { log.Println(self.name, "got reply to", reference, "but we don't have it") - go daemon.askForArticle(ArticleEntry{reference, newsgroup}) + go daemon.askForArticle(reference) } // store message r := &io.LimitedReader{ @@ -832,7 +843,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string newsgroup := hdr.Get("Newsgroups") if reference != "" && ValidMessageID(reference) && !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) { log.Println(self.name, "got reply to", reference, "but we don't have it") - go daemon.askForArticle(ArticleEntry{reference, newsgroup}) + go daemon.askForArticle(reference) } body := &io.LimitedReader{ R: r, @@ -1179,7 +1190,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string if reference != "" && ValidMessageID(reference) { if !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) { log.Println(self.name, "got reply to", reference, "but we don't have it") - go daemon.askForArticle(ArticleEntry{reference, newsgroup}) + go daemon.askForArticle(reference) } else { h := daemon.store.GetMIMEHeader(reference) if strings.Trim(h.Get("References"), " ") == "" { diff --git a/contrib/backends/srndv2/src/srnd/postgres.go b/contrib/backends/srndv2/src/srnd/postgres.go index 6240ae0..88c6655 100644 --- a/contrib/backends/srndv2/src/srnd/postgres.go +++ b/contrib/backends/srndv2/src/srnd/postgres.go @@ -1917,3 +1917,30 @@ func (self *PostgresDatabase) GetNewsgroupList() (list NewsgroupList, err error) } return } + +func (self *PostgresDatabase) FindCitesInText(text string) (msgids []string, err error) { + hashes := findBacklinks(text) + if len(hashes) > 0 { + q := "SELECT message_id FROM ArticlePosts WHERE message_id_hash IN ( " + var params []string + var qparams []interface{} + for idx := range hashes { + params = append(params, fmt.Sprintf("$%d", idx+1)) + qparams = append(qparams, hashes[idx]) + } + q += strings.Join(params, ", ") + q += " )" + var rows *sql.Rows + rows, err = self.conn.Query(q, qparams...) + if err == sql.ErrNoRows { + err = nil + } else if err == nil { + for rows.Next() { + var msgid string + rows.Scan(&msgid) + msgids = append(msgids, msgid) + } + } + } + return +}