diff --git a/contrib/backends/nntpchan-daemon/include/nntpchan/base64.hpp b/contrib/backends/nntpchan-daemon/include/nntpchan/base64.hpp index c7a6161..8d42495 100644 --- a/contrib/backends/nntpchan-daemon/include/nntpchan/base64.hpp +++ b/contrib/backends/nntpchan-daemon/include/nntpchan/base64.hpp @@ -10,6 +10,13 @@ std::string B64Encode(const uint8_t *data, const std::size_t l); /** @brief returns true if decode was successful */ bool B64Decode(const std::string &data, std::vector &out); + +/** returns base32 encoded string */ +std::string B32Encode(const uint8_t *data, const std::size_t l); + +/** @brief returns true if decode was successful */ +bool B32Decode(const std::string &data, std::vector &out); + } #endif diff --git a/contrib/backends/nntpchan-daemon/include/nntpchan/crypto.hpp b/contrib/backends/nntpchan-daemon/include/nntpchan/crypto.hpp index 345a4fe..512f903 100644 --- a/contrib/backends/nntpchan-daemon/include/nntpchan/crypto.hpp +++ b/contrib/backends/nntpchan-daemon/include/nntpchan/crypto.hpp @@ -3,6 +3,7 @@ #include #include +#include namespace nntpchan { @@ -10,6 +11,12 @@ typedef std::array SHA512Digest; void SHA512(const uint8_t *d, std::size_t l, SHA512Digest &h); + typedef std::array Blake2BDigest; + void Blake2B(const uint8_t *d, std::size_t l, Blake2BDigest & h); + + std::string Blake2B_base32(const std::string & str); + + /** global crypto initializer */ struct Crypto { diff --git a/contrib/backends/nntpchan-daemon/include/nntpchan/storage.hpp b/contrib/backends/nntpchan-daemon/include/nntpchan/storage.hpp index 2acdc94..2162f7d 100644 --- a/contrib/backends/nntpchan-daemon/include/nntpchan/storage.hpp +++ b/contrib/backends/nntpchan-daemon/include/nntpchan/storage.hpp @@ -40,6 +40,7 @@ private: bool init_skiplist(const std::string &subdir) const; fs::path skiplist_root(const std::string &name) const; + fs::path skiplist_dir(const fs::path & root, const std::string & name) const; fs::path basedir; }; diff --git a/contrib/backends/nntpchan-daemon/libnntpchan/base64.cpp b/contrib/backends/nntpchan-daemon/libnntpchan/base64.cpp index 7da92cc..ac54c65 100644 --- a/contrib/backends/nntpchan-daemon/libnntpchan/base64.cpp +++ b/contrib/backends/nntpchan-daemon/libnntpchan/base64.cpp @@ -5,6 +5,14 @@ namespace i2p { namespace data { + +static const char T32[32] = { + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '2', '3', '4', '5', '6', '7', +}; + +const char *GetBase32SubstitutionTable() { return T32; } + static void iT64Build(void); /* @@ -18,7 +26,9 @@ static void iT64Build(void); static const char T64[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '~'}; + +const char *GetBase64SubstitutionTable() { return T64; } /* * Reverse Substitution Table (built in run time) @@ -41,7 +51,8 @@ static char P64 = '='; * Converts binary encoded data to BASE64 format. * */ -static size_t /* Number of bytes in the encoded buffer */ + +size_t /* Number of bytes in the encoded buffer */ ByteStreamToBase64(const uint8_t *InBuffer, /* Input buffer, binary data */ size_t InCount, /* Number of bytes in the input buffer */ char *OutBuffer, /* output buffer */ @@ -122,7 +133,8 @@ static size_t /* Number of bytes in the encode * not properly padded, buffer of negative length is returned * */ -static ssize_t /* Number of output bytes */ + +size_t /* Number of output bytes */ Base64ToByteStream(const char *InBuffer, /* BASE64 encoded buffer */ size_t InCount, /* Number of input bytes */ uint8_t *OutBuffer, /* output buffer length */ @@ -184,7 +196,7 @@ static ssize_t /* Number of output bytes */ return outCount; } -static size_t Base64EncodingBufferSize(const size_t input_size) +size_t Base64EncodingBufferSize(const size_t input_size) { auto d = div(input_size, 3); if (d.rem) @@ -192,6 +204,13 @@ static size_t Base64EncodingBufferSize(const size_t input_size) return 4 * d.quot; } +size_t Base32EncodingBufferSize(const size_t input_size) +{ + auto d = div(input_size, 5); + if (d.rem) + d.quot++; + return 8 * d.quot; +} /* * * iT64 @@ -211,6 +230,65 @@ static void iT64Build() iT64[(int)T64[i]] = i; iT64[(int)P64] = 0; } + +size_t Base32ToByteStream(const char *inBuf, size_t len, uint8_t *outBuf, size_t outLen) +{ + int tmp = 0, bits = 0; + size_t ret = 0; + for (size_t i = 0; i < len; i++) + { + char ch = inBuf[i]; + if (ch >= '2' && ch <= '7') // digit + ch = (ch - '2') + 26; // 26 means a-z + else if (ch >= 'a' && ch <= 'z') + ch = ch - 'a'; // a = 0 + else + return 0; // unexpected character + + tmp |= ch; + bits += 5; + if (bits >= 8) + { + if (ret >= outLen) + return ret; + outBuf[ret] = tmp >> (bits - 8); + bits -= 8; + ret++; + } + tmp <<= 5; + } + return ret; +} + +size_t ByteStreamToBase32(const uint8_t *inBuf, size_t len, char *outBuf, size_t outLen) +{ + size_t ret = 0, pos = 1; + int bits = 8, tmp = inBuf[0]; + while (ret < outLen && (bits > 0 || pos < len)) + { + if (bits < 5) + { + if (pos < len) + { + tmp <<= 8; + tmp |= inBuf[pos] & 0xFF; + pos++; + bits += 8; + } + else // last byte + { + tmp <<= (5 - bits); + bits = 5; + } + } + + bits -= 5; + int ind = (tmp >> bits) & 0x1F; + outBuf[ret] = (ind < 26) ? (ind + 'a') : ((ind - 26) + '2'); + ret++; + } + return ret; +} } } @@ -227,9 +305,30 @@ std::string B64Encode(const uint8_t *data, const std::size_t l) bool B64Decode(const std::string &data, std::vector &out) { out.resize(data.size()); - if (i2p::data::Base64ToByteStream(data.c_str(), data.size(), &out[0], out.size()) == -1) - return false; - out.shrink_to_fit(); - return true; + if (i2p::data::Base64ToByteStream(data.c_str(), data.size(), &out[0], out.size())) + { + out.shrink_to_fit(); + return true; + } + return false; +} + +std::string B32Encode(const uint8_t *data, const std::size_t l) +{ + std::string out; + out.resize(i2p::data::Base32EncodingBufferSize(l)); + i2p::data::ByteStreamToBase32(data, l, &out[0], out.size()); + return out; +} + +bool B32Decode(const std::string &data, std::vector &out) +{ + out.resize(data.size()); + if (i2p::data::Base32ToByteStream(data.c_str(), data.size(), &out[0], out.size())) + { + out.shrink_to_fit(); + return true; + } + return false; } } diff --git a/contrib/backends/nntpchan-daemon/libnntpchan/crypto.cpp b/contrib/backends/nntpchan-daemon/libnntpchan/crypto.cpp index 926e7ba..980e5d7 100644 --- a/contrib/backends/nntpchan-daemon/libnntpchan/crypto.cpp +++ b/contrib/backends/nntpchan-daemon/libnntpchan/crypto.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -6,6 +7,16 @@ namespace nntpchan { void SHA512(const uint8_t *d, const std::size_t l, SHA512Digest &h) { crypto_hash(h.data(), d, l); } + void Blake2B(const uint8_t *d, std::size_t l, Blake2BDigest & h) { crypto_generichash(h.data(), h.size(), d, l, nullptr, 0); } + + std::string Blake2B_base32(const std::string & str) + { + Blake2BDigest d; + Blake2B(reinterpret_cast(str.c_str()), str.size(), d); + return B32Encode(d.data(), d.size()); + } + + Crypto::Crypto() { assert(sodium_init() == 0); } Crypto::~Crypto() {} diff --git a/contrib/backends/nntpchan-daemon/libnntpchan/storage.cpp b/contrib/backends/nntpchan-daemon/libnntpchan/storage.cpp index 8a1c792..f389f61 100644 --- a/contrib/backends/nntpchan-daemon/libnntpchan/storage.cpp +++ b/contrib/backends/nntpchan-daemon/libnntpchan/storage.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,6 +7,9 @@ namespace nntpchan { + const fs::path posts_skiplist_dir = "posts"; + const fs::path threads_skiplist_dir = "threads"; + ArticleStorage::ArticleStorage(const fs::path &fpath) { SetPath(fpath); } ArticleStorage::~ArticleStorage() {} @@ -14,16 +18,21 @@ void ArticleStorage::SetPath(const fs::path &fpath) { basedir = fpath; fs::create_directories(basedir); - assert(init_skiplist("posts_skiplist")); + assert(init_skiplist(posts_skiplist_dir)); + assert(init_skiplist(threads_skiplist_dir)); } + bool ArticleStorage::init_skiplist(const std::string &subdir) const { - fs::path skiplist = basedir / fs::path(subdir); - fs::create_directories(skiplist); - const auto subdirs = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"}; + fs::path skiplist = skiplist_root(subdir); + const auto subdirs = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', + 'y', 'z', '2', '3', '4', '5', '6', '7', +}; for (const auto &s : subdirs) - fs::create_directories(skiplist / s); + fs::create_directories(skiplist / std::string(&s, 1)); return true; } @@ -67,5 +76,19 @@ bool ArticleStorage::LoadThread(Thread &thread, const std::string &rootmsgid) co } /** ensure symlinks are formed for this article by message id */ -void ArticleStorage::EnsureSymlinks(const std::string &msgid) const { (void)msgid; } +void ArticleStorage::EnsureSymlinks(const std::string &msgid) const +{ + std::string msgidhash = Blake2B_base32(msgid); + skiplist_dir(posts_skiplist_dir, msgidhash); +} + + + fs::path ArticleStorage::skiplist_root(const std::string & name ) const + { + return basedir / name; + } + fs::path ArticleStorage::skiplist_dir(const fs::path & root, const std::string & name ) const + { + return root / name.substr(0, 1) ; + } } diff --git a/contrib/backends/srndv2/src/srnd/database.go b/contrib/backends/srndv2/src/srnd/database.go index 7432d7f..e69c5b0 100644 --- a/contrib/backends/srndv2/src/srnd/database.go +++ b/contrib/backends/srndv2/src/srnd/database.go @@ -333,6 +333,9 @@ type Database interface { // find cites in text FindCitesInText(msg string) ([]string, error) + + // find headers in group with lo/hi watermark and list of patterns + FindHeaders(group, headername string, lo, hi int64) (ArticleHeaders, error) } func NewDatabase(db_type, schema, host, port, user, password string) Database { diff --git a/contrib/backends/srndv2/src/srnd/message.go b/contrib/backends/srndv2/src/srnd/message.go index ef2dd40..d169d6f 100644 --- a/contrib/backends/srndv2/src/srnd/message.go +++ b/contrib/backends/srndv2/src/srnd/message.go @@ -37,6 +37,13 @@ func (self ArticleHeaders) Add(key, val string) { } } +func (self ArticleHeaders) Len() (l int) { + for k := range self { + l += len(self[k]) + } + return +} + func (self ArticleHeaders) Get(key, fallback string) string { val, ok := self[key] if ok { diff --git a/contrib/backends/srndv2/src/srnd/nntp.go b/contrib/backends/srndv2/src/srnd/nntp.go index 6200959..def38f0 100644 --- a/contrib/backends/srndv2/src/srnd/nntp.go +++ b/contrib/backends/srndv2/src/srnd/nntp.go @@ -1108,6 +1108,50 @@ func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string } else { conn.PrintfLine("500 invalid daemon state, got STAT with group set but we don't have that group now?") } + } else if cmd == "XPAT" { + var hdr string + var msgid string + var lo, hi int64 + var pats []string + if len(parts) >= 3 { + hdr = parts[1] + if ValidMessageID(parts[2]) { + msgid = parts[2] + } else { + lo, hi = parseRange(parts[2]) + if !ValidNewsgroup(self.group) { + conn.PrintfLine("430 no such article") + return + } + } + pats = parts[3:] + var hdrs ArticleHeaders + + if len(msgid) > 0 { + hdrs, err = daemon.database.GetHeadersForMessage(msgid) + } else { + hdrs, err = daemon.database.FindHeaders(self.group, hdr, lo, hi) + } + if err == nil { + hdrs = headerFindPats(hdr, hdrs, pats) + if hdrs.Len() > 0 { + conn.PrintfLine("221 Header follows") + for _, vals := range hdrs { + for idx := range vals { + conn.PrintfLine("%s: %s", hdr, vals[idx]) + } + } + conn.PrintfLine(".") + } else { + conn.PrintfLine("430 no such article") + } + } else { + conn.PrintfLine("502 %s", err.Error()) + } + return + } + conn.PrintfLine("430 no such article") + return } else if cmd == "XHDR" { if len(self.group) > 0 { var msgid string @@ -1606,6 +1650,8 @@ func (self *nntpConnection) runConnection(daemon *NNTPDaemon, inbound, stream, r log.Println(self.name, "TLS initiated", self.authenticated) } else { log.Println("STARTTLS failed:", err) + nconn.Close() + return } } else if cmd == "CAPABILITIES" { // write capabilities diff --git a/contrib/backends/srndv2/src/srnd/postgres.go b/contrib/backends/srndv2/src/srnd/postgres.go index db3a403..42da2bf 100644 --- a/contrib/backends/srndv2/src/srnd/postgres.go +++ b/contrib/backends/srndv2/src/srnd/postgres.go @@ -1953,3 +1953,21 @@ func (self *PostgresDatabase) FindCitesInText(text string) (msgids []string, err } return } + +func (self *PostgresDatabase) FindHeaders(group, headername string, lo, hi int64) (hdr ArticleHeaders, err error) { + hdr = make(ArticleHeaders) + q := "SELECT header_value FROM nntpheaders WHERE header_name = $1 AND header_article_message_id IN ( SELECT message_id FROM articleposts WHERE newsgroup = $2 )" + var rows *sql.Rows + rows, err = self.conn.Query(q, strings.ToLower(headername), group) + if err == nil { + for rows.Next() { + var str string + rows.Scan(&str) + hdr.Add(headername, str) + } + rows.Close() + } else if err == sql.ErrNoRows { + err = nil + } + return +} diff --git a/contrib/backends/srndv2/src/srnd/util.go b/contrib/backends/srndv2/src/srnd/util.go index 57004bc..f44132e 100644 --- a/contrib/backends/srndv2/src/srnd/util.go +++ b/contrib/backends/srndv2/src/srnd/util.go @@ -682,3 +682,52 @@ func msgidFrontendSign(sk []byte, msgid string) string { h := sha512.Sum512([]byte(msgid)) return cryptoSignFucky(h[:], sk) } + +func patMatch(v, pat string) (found bool) { + parts := strings.Split(pat, ",") + for _, part := range parts { + var invert bool + if part[0] == '!' { + invert = true + if len(parts) == 0 { + return + } + part = part[1:] + } + found, _ = regexp.MatchString(v, part) + log.Println(v, part, found) + if invert { + found = !found + } + if found { + return + } + } + return +} + +func headerFindPats(header string, hdr ArticleHeaders, patterns []string) (found ArticleHeaders) { + found = make(ArticleHeaders) + if hdr.Has(header) && len(patterns) > 0 { + for _, v := range hdr[header] { + for _, pat := range patterns { + if patMatch(v, pat) { + found.Add(header, v) + } + } + } + } + return +} + +func parseRange(str string) (lo, hi int64) { + parts := strings.Split(str, "-") + if len(parts) == 1 { + i, _ := strconv.ParseInt(parts[0], 10, 64) + lo, hi = i, i + } else if len(parts) == 2 { + lo, _ = strconv.ParseInt(parts[0], 10, 64) + hi, _ = strconv.ParseInt(parts[1], 10, 64) + } + return +} diff --git a/contrib/backends/srndv2/src/srnd/vendor/wildmat/LICENSE b/contrib/backends/srndv2/src/srnd/vendor/wildmat/LICENSE new file mode 100644 index 0000000..2e598ad --- /dev/null +++ b/contrib/backends/srndv2/src/srnd/vendor/wildmat/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Sergey Demyanov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/contrib/backends/srndv2/src/srnd/vendor/wildmat/wildmat.go b/contrib/backends/srndv2/src/srnd/vendor/wildmat/wildmat.go new file mode 100644 index 0000000..8be0d4f --- /dev/null +++ b/contrib/backends/srndv2/src/srnd/vendor/wildmat/wildmat.go @@ -0,0 +1,95 @@ +package wildmat + +// taken from https://github.com/demon-xxi/wildmatch/blob/0d1569265aadb1eb16009dd7bad941b4bd2aca8d/wildmatch.go + +import ( + "strings" +) + +// IsSubsetOf verifies if `w` wildcard is a subset of `s`. +// I.e. checks if `s` is a superset of subset `w`. +// Wildcard A is subset of B if any possible path that matches A also matches B. +func IsSubsetOf(w string, s string) bool { + + // shortcut for identical sets + if s == w { + return true + } + + // only empty set is a subset of an empty set + if len(s) == 0 { + return len(w) == 0 + } + + // find nesting separators + sp := strings.Index(s, ",") + wp := strings.Index(w, ",") + + // check if this is a nested path + if sp >= 0 { + + // if set is nested then tested wildcard must be nested too + if wp < 0 { + return false + } + + // Special case for /**/ mask that matches any number of levels + if s[:sp] == "**" && + IsSubsetOf(w[wp+1:], s) || + IsSubsetOf(w, s[sp+1:]) { + return true + } + + // check that current level names are subsets + // and compare rest of the path to be subset also + return (IsSubsetOf(w[:wp], s[:sp]) && + IsSubsetOf(w[wp+1:], s[sp+1:])) + } + + // subset can't have more levels than set + if wp >= 0 { + return false + } + + // we are comparing names on the same nesting level here + // so let's do symbol by symbol comparison + switch s[0] { + case '?': + // ? matches non empty character. '*' can't be a subset of '?' + if len(w) == 0 || w[0] == '*' { + return false + } + // any onther symbol matches '?', so let's skip to next + return IsSubsetOf(w[1:], s[1:]) + case '*': + // '*' matches 0 and any other number of symbols + // so checking 0 and recursively subset without first letter + return IsSubsetOf(w, s[1:]) || + (len(w) > 0 && IsSubsetOf(w[1:], s)) + default: + // making sure next symbol in w exists and it's the same as in set + if len(w) == 0 || w[0] != s[0] { + return false + } + } + + // recursively check rest of the set and w + return IsSubsetOf(w[1:], s[1:]) +} + +// IsSubsetOfAny verifies if current wildcard `w` is a subset of any of the given sets. +// Wildcard A is subset of B if any possible path that matches A also matches B. +// If multiple subsets match then the smallest or first lexicographical set is returned +// Return -1 if not found or superset index. +func IsSubsetOfAny(w string, sets ...string) (found int) { + found = -1 // not found by default + for i, superset := range sets { + if !IsSubsetOf(w, superset) { + continue + } + if found < 0 || IsSubsetOf(superset, sets[found]) { + found = i + } + } + return +}