From 2e1b934705ef9d00a627e8a683060adde9b18613 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 12 Sep 2017 09:27:48 -0400 Subject: [PATCH] optimize queries more --- contrib/backends/srndv2/src/srnd/database.go | 8 ++++ contrib/backends/srndv2/src/srnd/nntp.go | 39 +++++++++----------- contrib/backends/srndv2/src/srnd/postgres.go | 18 +++++++++ 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/database.go b/contrib/backends/srndv2/src/srnd/database.go index 9fc98b8..403b542 100644 --- a/contrib/backends/srndv2/src/srnd/database.go +++ b/contrib/backends/srndv2/src/srnd/database.go @@ -65,6 +65,11 @@ type PostingStats struct { History []PostingStatsEntry } +// newsgroup, first, last +type NewsgroupListEntry [3]string + +type NewsgroupList []NewsgroupListEntry + type Database interface { Close() CreateTables() @@ -322,6 +327,9 @@ type Database interface { // get post message-id where hash is similar to string GetCitesByPostHashLike(like string) ([]MessageIDTuple, error) + + // get newsgroup list with watermarks + GetNewsgroupList() (NewsgroupList, error) } func NewDatabase(db_type, schema, host, port, user, password string) Database { diff --git a/contrib/backends/srndv2/src/srnd/nntp.go b/contrib/backends/srndv2/src/srnd/nntp.go index 0954dcf..ca19de0 100644 --- a/contrib/backends/srndv2/src/srnd/nntp.go +++ b/contrib/backends/srndv2/src/srnd/nntp.go @@ -1030,19 +1030,18 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string conn.PrintfLine("411 No Such Newsgroup") } } else if cmd == "LIST" && parts[1] == "NEWSGROUPS" { - conn.PrintfLine("215 list of newsgroups follows") // handle list command - groups := daemon.database.GetAllNewsgroups() - dw := conn.DotWriter() - for _, group := range groups { - last, first, err := daemon.database.GetLastAndFirstForGroup(group) - if err == nil { - io.WriteString(dw, fmt.Sprintf("%s %d %d y\r\n", group, first, last)) - } else { - log.Println("cannot get last/first ids for group", group, err) + list, err := daemon.database.GetNewsgroupList() + if err == nil { + conn.PrintfLine("215 list of newsgroups follows") + dw := conn.DotWriter() + for _, entry := range list { + io.WriteString(dw, fmt.Sprintf("%s %s %s y\r\n", entry[0], entry[1], entry[2])) } + dw.Close() + } else { + conn.PrintfLine("500 failed to get list: %s", err.Error()) } - dw.Close() } else if cmd == "STAT" { if len(self.group) == 0 { if len(parts) == 2 { @@ -1135,19 +1134,17 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string } } else { if line == "LIST" { - conn.PrintfLine("215 list of newsgroups follows") - // handle list command - groups := daemon.database.GetAllNewsgroups() - dw := conn.DotWriter() - for _, group := range groups { - last, first, err := daemon.database.GetLastAndFirstForGroup(group) - if err == nil { - io.WriteString(dw, fmt.Sprintf("%s %d %d y\r\n", group, first, last)) - } else { - log.Println("cannot get last/first ids for group", group, err) + list, err := daemon.database.GetNewsgroupList() + if err == nil { + conn.PrintfLine("215 list of newsgroups follows") + dw := conn.DotWriter() + for _, entry := range list { + io.WriteString(dw, fmt.Sprintf("%s %s %s y\r\n", entry[0], entry[1], entry[2])) } + dw.Close() + } else { + conn.PrintfLine("500 failed to get list: %s", err.Error()) } - dw.Close() } else if line == "POST" { if !self.authenticated { // needs tls to work if not logged in diff --git a/contrib/backends/srndv2/src/srnd/postgres.go b/contrib/backends/srndv2/src/srnd/postgres.go index 0b75385..5605be3 100644 --- a/contrib/backends/srndv2/src/srnd/postgres.go +++ b/contrib/backends/srndv2/src/srnd/postgres.go @@ -146,6 +146,7 @@ const SearchByHash_2 = "SearchByHash_2" const GetNNTPPostsInGroup = "GetNNTPPostsInGroup" const GetCitesByPostHashLike = "GetCitesByPostHashLike" const GetYearlyPostHistory = "GetYearlyPostHistory" +const GetNewsgroupList = "GetNewsgroupList" func (self *PostgresDatabase) prepareStatements() { self.stmt = map[string]string{ @@ -191,6 +192,7 @@ func (self *PostgresDatabase) prepareStatements() { GetMessageIDByHash: "SELECT message_id, message_newsgroup FROM Articles WHERE message_id_hash = $1 LIMIT 1", CheckEncIPBanned: "SELECT 1 FROM EncIPBans WHERE encaddr = $1", GetFirstAndLastForGroup: "WITH x(min_no, max_no) AS ( SELECT MIN(message_no) AS min_no, MAX(message_no) AS max_no FROM ArticleNumbers WHERE newsgroup = $1) SELECT CASE WHEN min_no IS NULL THEN 0 ELSE min_no END AS min_no FROM x UNION SELECT CASE WHEN max_no IS NULL THEN 1 ELSE max_no END AS max_no FROM x", + GetNewsgroupList: "SELECT newsgroup, min(message_no), max(message_no) FROM ArticlePosts GROUP BY newsgroup", GetMessageIDForNNTPID: "SELECT message_id FROM ArticleNumbers WHERE newsgroup = $1 AND message_no = $2 LIMIT 1", GetNNTPIDForMessageID: "SELECT message_no FROM ArticleNumbers WHERE newsgroup = $1 AND message_id = $2 LIMIT 1", IsExpired: "WITH x(msgid) AS ( SELECT message_id FROM Articles WHERE message_id = $1 INTERSECT ( SELECT message_id FROM ArticlePosts WHERE message_id = $1 ) ) SELECT COUNT(*) FROM x", @@ -1893,3 +1895,19 @@ func (self *PostgresDatabase) SearchByHash(prefix, group, text string, chnl chan close(chnl) return } + +func (self *PostgresDatabase) GetNewsgroupList() (list NewsgroupList, err error) { + var rows *sql.Rows + rows, err = self.conn.Query(self.stmt[GetNewsgroupList]) + if err == nil { + for rows.Next() { + var l NewsgroupListEntry + var lo, hi int64 + rows.Scan(&l[0], &lo, &hi) + l[1] = fmt.Sprintf("%d", lo) + l[2] = fmt.Sprintf("%d", hi) + } + rows.Close() + } + return +}