From b4d2de6ec86c37aa9358655aa66253e1493819f9 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Sun, 3 Mar 2019 12:51:25 -0500 Subject: [PATCH 1/6] add limit to search --- contrib/backends/srndv2/src/srnd/database.go | 4 ++-- .../backends/srndv2/src/srnd/frontend_http.go | 5 +++-- contrib/backends/srndv2/src/srnd/postgres.go | 20 +++++++++---------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/database.go b/contrib/backends/srndv2/src/srnd/database.go index 76ec26e..4c4feb8 100644 --- a/contrib/backends/srndv2/src/srnd/database.go +++ b/contrib/backends/srndv2/src/srnd/database.go @@ -323,10 +323,10 @@ type Database interface { GetPostingStats(granularity, begin, end int64) (PostingStats, error) // peform search query - SearchQuery(prefix, group, text string, chnl chan PostModel) error + SearchQuery(prefix, group, text string, chnl chan PostModel, limit int) error // find posts with similar hash - SearchByHash(prefix, group, posthash string, chnl chan PostModel) error + SearchByHash(prefix, group, posthash string, chnl chan PostModel, limit int) error // get full thread model GetThreadModel(prefix, root_msgid string) (ThreadModel, error) diff --git a/contrib/backends/srndv2/src/srnd/frontend_http.go b/contrib/backends/srndv2/src/srnd/frontend_http.go index 5ef20f7..ab8702a 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_http.go +++ b/contrib/backends/srndv2/src/srnd/frontend_http.go @@ -1112,10 +1112,11 @@ func (self *httpFrontend) handle_api_find(wr http.ResponseWriter, r *http.Reques } donechnl <- 0 }(wr) + limit := 50 if len(h) > 0 { - self.daemon.database.SearchByHash(self.prefix, g, h, chnl) + self.daemon.database.SearchByHash(self.prefix, g, h, chnl, limit) } else { - self.daemon.database.SearchQuery(self.prefix, g, s, chnl) + self.daemon.database.SearchQuery(self.prefix, g, s, chnl, limit) } chnl <- nil <-donechnl diff --git a/contrib/backends/srndv2/src/srnd/postgres.go b/contrib/backends/srndv2/src/srnd/postgres.go index a719128..64c7ffd 100644 --- a/contrib/backends/srndv2/src/srnd/postgres.go +++ b/contrib/backends/srndv2/src/srnd/postgres.go @@ -219,10 +219,10 @@ func (self *PostgresDatabase) prepareStatements() { GetMessageIDByCIDR: "SELECT message_id FROM ArticlePosts WHERE addr IN ( SELECT encaddr FROM EncryptedAddrs WHERE addr_cidr <<= cidr($1) )", GetMessageIDByEncryptedIP: "SELECT message_id FROM ArticlePosts WHERE addr = $1", GetPostsBefore: "SELECT message_id FROM ArticlePosts WHERE time_posted < $1", - SearchQuery_1: "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE message LIKE $1 ORDER BY time_posted DESC", - SearchQuery_2: "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE newsgroup = $1 AND message LIKE $2 ORDER BY time_posted DESC", - SearchByHash_1: "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1 ORDER BY time_obtained DESC", - SearchByHash_2: "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_newsgroup = $2 AND message_id_hash LIKE $1 ORDER BY time_obtained DESC", + SearchQuery_1: "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE message LIKE $1 ORDER BY time_posted DESC LIMIT $2", + SearchQuery_2: "SELECT newsgroup, message_id, ref_id FROM ArticlePosts WHERE newsgroup = $1 AND message LIKE $2 ORDER BY time_posted DESC LIMIT $3", + SearchByHash_1: "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1 ORDER BY time_obtained DESC LIMIT $2", + SearchByHash_2: "SELECT message_newsgroup, message_id, message_ref_id FROM Articles WHERE message_newsgroup = $2 AND message_id_hash LIKE $1 ORDER BY time_obtained DESC LIMIT $3", GetNNTPPostsInGroup: "SELECT message_no, ArticlePosts.message_id, subject, time_posted, ref_id, name, path FROM ArticleNumbers INNER JOIN ArticlePosts ON ArticleNumbers.message_id = ArticlePosts.message_id WHERE ArticlePosts.newsgroup = $1 ORDER BY message_no", GetCitesByPostHashLike: "SELECT message_id, message_ref_id FROM Articles WHERE message_id_hash LIKE $1", GetYearlyPostHistory: "WITH times(endtime, begintime) AS ( SELECT CAST(EXTRACT(epoch from i) AS BIGINT) AS endtime, CAST(EXTRACT(epoch from i - interval '1 month') AS BIGINT) AS begintime FROM generate_series(now() - interval '10 year', now(), '1 month'::interval) i ) SELECT begintime, endtime, ( SELECT count(*) FROM ArticlePosts WHERE time_posted > begintime AND time_posted < endtime) FROM times", @@ -1970,14 +1970,14 @@ func (self *PostgresDatabase) GetPostingStats(gran, begin, end int64) (st Postin return } -func (self *PostgresDatabase) SearchQuery(prefix, group string, text string, chnl chan PostModel) (err error) { +func (self *PostgresDatabase) SearchQuery(prefix, group string, text string, chnl chan PostModel, limit int) (err error) { if text != "" && strings.Count(text, "%") == 0 { text = "%" + text + "%" var rows *sql.Rows if group == "" { - rows, err = self.conn.Query(self.stmt[SearchQuery_1], text) + rows, err = self.conn.Query(self.stmt[SearchQuery_1], text, limit) } else { - rows, err = self.conn.Query(self.stmt[SearchQuery_2], group, text) + rows, err = self.conn.Query(self.stmt[SearchQuery_2], group, text, limit) } if err == nil { for rows.Next() { @@ -1991,15 +1991,15 @@ func (self *PostgresDatabase) SearchQuery(prefix, group string, text string, chn close(chnl) return } -func (self *PostgresDatabase) SearchByHash(prefix, group, text string, chnl chan PostModel) (err error) { +func (self *PostgresDatabase) SearchByHash(prefix, group, text string, chnl chan PostModel, limit int) (err error) { if text != "" && strings.Count(text, "%") == 0 { text = "%" + text + "%" var rows *sql.Rows if group == "" { - rows, err = self.conn.Query(self.stmt[SearchByHash_1], text) + rows, err = self.conn.Query(self.stmt[SearchByHash_1], text, limit) } else { - rows, err = self.conn.Query(self.stmt[SearchByHash_2], text, group) + rows, err = self.conn.Query(self.stmt[SearchByHash_2], text, group, limit) } if err == nil { for rows.Next() { From 43ce4490edd9d1439549a07d7cd7d843757609d7 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Sun, 3 Mar 2019 12:55:54 -0500 Subject: [PATCH 2/6] fix crash --- contrib/backends/srndv2/src/srnd/frontend_http.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/frontend_http.go b/contrib/backends/srndv2/src/srnd/frontend_http.go index ab8702a..186dc60 100644 --- a/contrib/backends/srndv2/src/srnd/frontend_http.go +++ b/contrib/backends/srndv2/src/srnd/frontend_http.go @@ -1114,13 +1114,12 @@ func (self *httpFrontend) handle_api_find(wr http.ResponseWriter, r *http.Reques }(wr) limit := 50 if len(h) > 0 { - self.daemon.database.SearchByHash(self.prefix, g, h, chnl, limit) + go self.daemon.database.SearchByHash(self.prefix, g, h, chnl, limit) } else { - self.daemon.database.SearchQuery(self.prefix, g, s, chnl, limit) + go self.daemon.database.SearchQuery(self.prefix, g, s, chnl, limit) } chnl <- nil <-donechnl - close(donechnl) io.WriteString(wr, " null ]") return } From c8e3faa4c6d0e58c9751a80eedb5f9fc3f2b06a8 Mon Sep 17 00:00:00 2001 From: cathugger Date: Fri, 29 Mar 2019 23:13:32 +0200 Subject: [PATCH 3/6] srnd: change unicode tripcode rune range --- contrib/backends/srndv2/src/srnd/util.go | 59 ++++++++++++++++-------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/util.go b/contrib/backends/srndv2/src/srnd/util.go index cd0f8eb..c816c23 100644 --- a/contrib/backends/srndv2/src/srnd/util.go +++ b/contrib/backends/srndv2/src/srnd/util.go @@ -477,28 +477,47 @@ func newNaclSignKeypair() (string, string) { return hex.EncodeToString(pk), hex.EncodeToString(sk) } +func makeTripcodeLen(pubkey string, length int) string { + var b strings.Builder + + data, err := hex.DecodeString(pubkey) + if err != nil { + return "[invalid]" + } + + if length <= 0 || length > len(data) { + length = len(data) + } + + // originally srnd (and srndv2) used 9600==0x2580 + // however, range shifted by 0x10 looks better to me (cathugger) + // (instead of `▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏` it'll use `⚀⚁⚂⚃⚄⚅⚆⚇⚈⚉⚊⚋⚌⚍⚎⚏`) + // and display equaly good both in torbrowser+DejaVuSans and phone + // since jeff ack'd it (he doesn't care probably), I'll just use it + const rstart = 0x2590 + // 0x2500 can display with TBB font whitelist, but looks too cryptic. + // startin from 0x2600 needs more than DejaVuSans so I'll avoid it + + // logic (same as in srnd): + // it first writes length/2 chars of begining + // and then length/2 chars of ending + // if length==len(data), that essentially means just using whole + i := 0 + for ; i < length/2; i++ { + b.WriteRune(rstart + rune(data[i])) + b.WriteRune(0xFE0E) // text style variant + } + for ; i < length; i++ { + b.WriteRune(rstart + rune(data[len(data)-length+i])) + b.WriteRune(0xFE0E) // text style variant + } + + return b.String() +} + // make a utf-8 tripcode func makeTripcode(pk string) string { - data, err := hex.DecodeString(pk) - if err == nil { - tripcode := "" - // here is the python code this is based off of - // i do something slightly different but this is the base - // - // for x in range(0, length / 2): - // pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[x*2:x*2+2], 16)) - // length -= length / 2 - // for x in range(0, length): - // pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[-(length*2):][x*2:x*2+2], 16)) - // - for _, c := range data { - ch := 9600 - ch += int(c) - tripcode += fmt.Sprintf("&#%04d;", ch) - } - return tripcode - } - return "[invalid]" + return makeTripcodeLen(pk, 0) } // generate a new message id with base name From 558dacac7905f1c9dfa52ac1ef3f391fc7d8e104 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 4 Apr 2019 07:25:38 -0400 Subject: [PATCH 4/6] off --- contrib/backends/srndv2/src/srnd/nntp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/backends/srndv2/src/srnd/nntp.go b/contrib/backends/srndv2/src/srnd/nntp.go index 2de8ca1..c016ad4 100644 --- a/contrib/backends/srndv2/src/srnd/nntp.go +++ b/contrib/backends/srndv2/src/srnd/nntp.go @@ -718,7 +718,7 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string if len(reason) > 0 { // discard, we do not want log.Println(self.name, "rejected", msgid, reason) - conn.PrintfLine("439 %s %s", code, msgid, reason) + conn.PrintfLine("439 %s %s", msgid, reason) _, err = io.Copy(ioutil.Discard, msg.Body) if ban { err = daemon.database.BanArticle(msgid, reason) From d8f888dffa2d7d234618a1849d88339bc36ce5e3 Mon Sep 17 00:00:00 2001 From: cathugger Date: Sat, 6 Apr 2019 20:36:54 +0300 Subject: [PATCH 5/6] srnd: saner default max message size --- contrib/backends/srndv2/src/srnd/store.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/contrib/backends/srndv2/src/srnd/store.go b/contrib/backends/srndv2/src/srnd/store.go index 930e885..f8c907c 100644 --- a/contrib/backends/srndv2/src/srnd/store.go +++ b/contrib/backends/srndv2/src/srnd/store.go @@ -26,8 +26,17 @@ import ( var ErrOversizedMessage = errors.New("oversized message") -// ~ 10 MB unbased64'd -const DefaultMaxMessageSize = 1024 * 1024 * 10 +// (cathugger) +// my test showed that 8MiB of attachments split in 5 parts +// plus some text produce something close to typhical big message +// resulted in 11483923 bytes. +// that's consistent with rough size calculation mentioned in +// +// ((origlen * 1.37) + 814) +// which resulted in 11493206 bytes for 8MiB of data. +// previous default of 10MiB (10485760) was too low in practice. +// use 11MiB (11534336) to leave some space for longer than usual texts. +const DefaultMaxMessageSize = 11 * 1024 * 1024 // HARD max message size const MaxMessageSize = 1024 * 1024 * 1024 From 8f39dec91b85d6210e587673ebe5cfe1a513f892 Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 7 Apr 2019 08:52:49 -0400 Subject: [PATCH 6/6] Update building.md --- doc/building.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/building.md b/doc/building.md index 9670fbf..88ae975 100644 --- a/doc/building.md +++ b/doc/building.md @@ -15,7 +15,7 @@ Dependancies: * imagemagick * ffmpeg * sox -* go 1.9 +* go * GNU make ## Debian instructions @@ -24,7 +24,7 @@ These are installation instructions for Debian. ### Install Go -Install the Go programming language version _1.9_ from the [Go website](https://golang.org/dl/). +Install the latest version of the Go programming language from the [Go website](https://golang.org/dl/). ### Install the dependancies