archive/nntpchan
Archived
1
0
Fork 0
Code

nntpchan-daeom code

Browse Source
This commit is contained in:
Jeff Becker 2017-10-13 07:58:41 -04:00
parent c583a03f81
commit e9c88ffd28
No known key found for this signature in database
GPG Key ID: F357B3B42F6F9B05
14 changed files with 328 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
contrib/backends/nntpchan-daemon/Makefile
View File

@ -10,7 +10,6 @@ MUSTACHE_PATH = $(REPO)/libmustache
MUSTACHE_SRC := $(wildcard $(MUSTACHE_PATH)/*.cpp)
MUSTACHE_SRC += $(wildcard $(MUSTACHE_PATH)/*/*.cpp)
MUSTACHE_HDR := $(wildcard $(MUSTACHE_PATH)/*.hpp)
MUSTACHE_OBJ := $(MUSTACHE_SRC:.cpp=.o)
@ -21,7 +20,10 @@ TOOL_PATH := $(REPO)/tools
TOOL_SRC := $(wildcard $(TOOL_PATH)/*.cpp)
TOOLS := $(TOOL_SRC:.cpp=)
OBJ := $(NNTPCHAN_OBJ) $(MUSTACHE_OBJ)
OBJ := $(NNTPCHAN_OBJ)
OBJ += $(MUSTACHE_OBJ)
TEST = $(REPO)/test
DAEMON_SRC = $(REPO)/daemon
@ -29,7 +31,9 @@ PKGS := libuv libsodium
LD_FLAGS := $(shell pkg-config --libs $(PKGS)) -lstdc++fs
INC_FLAGS := $(shell pkg-config --cflags $(PKGS)) -I$(HEADERS_PATH)
CXXFLAGS := -std=c++17 -Wall -Wextra -pedantic $(INC_FLAGS)
CXXFLAGS := -std=c++17 -Wall -Wextra -Werror -pedantic $(INC_FLAGS)
DEBUG = 1
ifeq ($(DEBUG),1)
CXXFLAGS += -g
@ -54,18 +58,16 @@ $(NNTPCHAN_LIB): $(NNTPCHAN_OBJ)
$(AR) -r $(NNTPCHAN_LIB) $(NNTPCHAN_OBJ)
$(EXE): $(LIBS)
$(CXX) $(CXXFLAGS) $(DAEMON_SRC)/main.cpp $(NNTPCHAN_LIB) $(LD_FLAGS) -o $(EXE)
$(CXX) $(CXXFLAGS) $(DAEMON_SRC)/main.cpp $(LIBS) $(LD_FLAGS) -o $(EXE)
$(TOOL_SRC): $(NNTPCHAN_LIB)
$(TOOLS): $(TOOL_SRC) $(LIBS)
$(CXX) $(CXXFLAGS) $< $(LIBS) $(LD_FLAGS) -o $@
$(TOOLS): $(TOOL_SRC)
$(CXX) $(CXXFLAGS) $< $(NNTPCHAN_LIB) $(LD_FLAGS) -o $@
build-test: $(LIB)
$(CXX) -o $(REPO)/test $(CXXFLAGS) test.cpp $(NNTPCHAN_LIB) $(LD_FLAGS)
build-test: $(LIBS)
$(CXX) -o $(TEST) $(CXXFLAGS) test.cpp $(LIBS) $(LD_FLAGS)
test: build-test
$(REPO)/test
$(TEST)
clean:
rm -f $(OBJ) $(NNTPCHAN_LIB) $(EXE) $(TOOLS)
$(RM) $(OBJ) $(LIBS) $(EXE) $(TOOLS) $(TEST)

2
contrib/backends/nntpchan-daemon/README.md
View File

@ -12,6 +12,8 @@ requirements:
* libuv 1.x
* boost variant (for now)
* GNU Make
building:

21
contrib/backends/nntpchan-daemon/include/nntpchan/message.hpp Normal file
View File

@ -0,0 +1,21 @@
#ifndef NNTPCHAN_MESSAGE_HPP
#define NNTPCHAN_MESSAGE_HPP
#include <memory>
#include <nntpchan/model.hpp>
namespace nntpchan
{
struct MessageDB
{
using BoardPage = nntpchan::model::BoardPage;
using Thread = nntpchan::model::Thread;
virtual bool LoadBoardPage(BoardPage & board, const std::string & newsgroup, uint32_t perpage, uint32_t page) const = 0;
virtual bool FindThreadByHash(const std::string & hashhex, std::string & msgid) const = 0;
virtual bool LoadThread(Thread & thread, const std::string & rootmsgid) const = 0;
};
typedef std::unique_ptr<MessageDB> MessageDB_ptr;
}
#endif

32
contrib/backends/nntpchan-daemon/include/nntpchan/model.hpp
View File

@ -1,17 +1,20 @@
#ifndef NNTPCHAN_MODEL_HPP
#define NNTPCHAN_MODEL_HPP
#include <algorithm>
#include <map>
#include <set>
#include <string>
#include <tuple>
#include <vector>
#include <variant>
#include <nntpchan/sanitize.hpp>
namespace nntpchan
{
namespace model
{
// MIME Header
typedef std::map<std::string, std::set<std::string> > PostHeader;
typedef std::map<std::string, std::vector<std::string> > PostHeader;
// text post contents
typedef std::string PostBody;
// single file attachment, (orig_filename, hexdigest, thumb_filename)
@ -22,37 +25,50 @@ namespace nntpchan
typedef std::tuple<PostHeader, PostBody, Attachments> Post;
// a thread (many posts in post order)
typedef std::vector<Post> Thread;
// a board page is many threads in bump order
typedef std::vector<Thread> BoardPage;
static inline std::string & GetFilename(PostAttachment & att)
static inline const std::string & GetFilename(const PostAttachment & att)
{
return std::get<0>(att);
}
static inline std::string & GetHexDigest(PostAttachment & att)
static inline const std::string & GetHexDigest(const PostAttachment & att)
{
return std::get<1>(att);
}
static inline std::string & GetThumbnail(PostAttachment & att)
static inline const std::string & GetThumbnail(const PostAttachment & att)
{
return std::get<2>(att);
}
static inline PostHeader & GetHeader(Post & post)
static inline const PostHeader & GetHeader(const Post & post)
{
return std::get<0>(post);
}
static inline PostBody & GetBody(Post & post)
static inline const PostBody & GetBody(const Post & post)
{
return std::get<1>(post);
}
static inline Attachments & GetAttachments(Post & post)
static inline const Attachments & GetAttachments(const Post & post)
{
return std::get<2>(post);
}
static inline const std::string & HeaderIFind(const PostHeader & header, const std::string & val, const std::string & fallback)
{
std::string ival = ToLower(val);
auto itr = std::find_if(header.begin(), header.end(), [ival](const auto & item) -> bool { return ToLower(item.first) == ival; });
if (itr == std::end(header))
return fallback;
else
return itr->second[0];
}
using Model = std::variant<Thread, BoardPage>;
}
}

4
contrib/backends/nntpchan-daemon/include/nntpchan/nntp_handler.hpp
View File

@ -11,7 +11,7 @@ namespace nntpchan
class NNTPServerHandler : public LineReader, public IConnHandler
{
public:
NNTPServerHandler(const std::string & storage);
NNTPServerHandler(const fs::path & storage);
~NNTPServerHandler();
virtual bool ShouldClose();
@ -51,7 +51,7 @@ namespace nntpchan
std::string m_articleName;
FileHandle_ptr m_article;
CredDB_ptr m_auth;
ArticleStorage m_store;
ArticleStorage_ptr m_store;
std::string m_mode;
bool m_authed;
State m_state;

1
contrib/backends/nntpchan-daemon/include/nntpchan/sanitize.hpp
View File

@ -6,6 +6,7 @@ namespace nntpchan
{
std::string NNTPSanitizeLine(const std::string & str);
std::string ToLower(const std::string & str);
std::string StripWhitespaces(const std::string & str);
bool IsValidMessageID(const std::string & msgid);
bool IsValidNewsgroup(const std::string & group);
}

13
contrib/backends/nntpchan-daemon/include/nntpchan/staticfile_frontend.hpp
View File

@ -1,6 +1,7 @@
#ifndef NNTPCHAN_STATICFILE_FRONTEND_HPP
#define NNTPCHAN_STATICFILE_FRONTEND_HPP
#include "frontend.hpp"
#include "message.hpp"
#include "template_engine.hpp"
#include "model.hpp"
#include <experimental/filesystem>
@ -23,17 +24,7 @@ namespace nntpchan
bool AcceptsMessage(const std::string & msgid);
private:
typedef nntpchan::model::Thread Thread_t;
typedef std::vector<Thread_t> Threads_t;
typedef std::vector<Threads_t> BoardPage_t;
BoardPage_t GetThreadsPaginated(const std::string & group, uint32_t perpage, uint32_t pages);
private:
MessageDB_ptr m_MessageDB;
TemplateEngine_ptr m_TemplateEngine;
fs::path m_TemplateDir;
fs::path m_OutDir;

27
contrib/backends/nntpchan-daemon/include/nntpchan/storage.hpp
View File

@ -4,36 +4,49 @@
#include <experimental/filesystem>
#include <string>
#include "file_handle.hpp"
#include "message.hpp"
namespace nntpchan
{
namespace fs = std::experimental::filesystem;
class ArticleStorage
class ArticleStorage : public MessageDB
{
public:
ArticleStorage();
ArticleStorage(const fs::path & fpath);
~ArticleStorage();
void SetPath(const fs::path & fpath);
FileHandle_ptr OpenWrite(const std::string & msgid);
FileHandle_ptr OpenRead(const std::string & msgid);
FileHandle_ptr OpenWrite(const std::string & msgid) const;
FileHandle_ptr OpenRead(const std::string & msgid) const;
/**
return true if we should accept a new message give its message id
*/
bool Accept(const std::string & msgid);
bool Accept(const std::string & msgid) const;
bool LoadBoardPage(BoardPage & board, const std::string & newsgroup, uint32_t perpage, uint32_t page) const;
bool FindThreadByHash(const std::string & hashhex, std::string & msgid) const;
bool LoadThread(Thread & thread, const std::string & rootmsgid) const;
/** ensure symlinks are formed for this article by message id */
void EnsureSymlinks(const std::string & msgid) const;
private:
void SetPath(const fs::path & fpath);
fs::path MessagePath(const std::string & msgid);
fs::path MessagePath(const std::string & msgid) const;
bool init_skiplist(const std::string & subdir) const;
fs::path skiplist_root(const std::string & name) const;
fs::path basedir;
};
typedef std::unique_ptr<ArticleStorage> ArticleStorage_ptr;
}

3
contrib/backends/nntpchan-daemon/include/nntpchan/template_engine.hpp
View File

@ -1,6 +1,7 @@
#ifndef NNTPCHAN_TEMPLATE_ENGINE_HPP
#define NNTPCHAN_TEMPLATE_ENGINE_HPP
#include "file_handle.hpp"
#include "model.hpp"
#include <any>
#include <map>
#include <memory>
@ -11,7 +12,7 @@ namespace nntpchan
struct TemplateEngine
{
using Args_t = std::map<std::string, std::any>;
typedef std::map<std::string, std::variant<nntpchan::model::Model, std::string>> Args_t;
virtual bool WriteTemplate(const fs::path & template_fpath, const Args_t & args, const FileHandle_ptr & out) = 0;
};

16
contrib/backends/nntpchan-daemon/libnntpchan/nntp_handler.cpp
View File

@ -9,11 +9,11 @@
namespace nntpchan
{
NNTPServerHandler::NNTPServerHandler(const std::string & storage) :
NNTPServerHandler::NNTPServerHandler(const fs::path & storage) :
LineReader(1024),
m_article(nullptr),
m_auth(nullptr),
m_store(storage),
m_store(std::make_unique<ArticleStorage>(storage)),
m_authed(false),
m_state(eStateReadCommand)
{
@ -115,22 +115,22 @@ namespace nntpchan
} else if (cmd == "CHECK") {
if(cmdlen >= 2) {
const std::string & msgid = command[1];
if(IsValidMessageID(msgid) && m_store.Accept(msgid))
if(IsValidMessageID(msgid) && m_store->Accept(msgid))
{
QueueLine("238 "+msgid);
return;
}
QueueLine("438 "+msgid);
else
QueueLine("438 "+msgid);
}
else
QueueLine("501 syntax error");
} else if (cmd == "TAKETHIS") {
if (cmdlen == 2)
if (cmdlen >= 2)
{
const std::string & msgid = command[1];
if(m_store.Accept(msgid))
if(m_store->Accept(msgid))
{
m_article = m_store.OpenWrite(msgid);
m_article = m_store->OpenWrite(msgid);
}
m_articleName = msgid;
EnterState(eStateStoreArticle);

10
contrib/backends/nntpchan-daemon/libnntpchan/sanitize.cpp
View File

@ -36,4 +36,14 @@ namespace nntpchan
{
return std::regex_search(msgid, re_ValidNewsgroup) == 1;
}
std::string StripWhitespaces(const std::string & str)
{
std::string stripped;
for(const auto & ch : str)
if(!(std::isspace(ch)||std::iscntrl(ch)))
stripped += ch;
return stripped;
}
}

105
contrib/backends/nntpchan-daemon/libnntpchan/staticfile_frontend.cpp
View File

@ -36,28 +36,79 @@ namespace nntpchan
// read body
std::map<std::string, std::any> thread_args;
auto findMsgidFunc = [](const std::pair<std::string, std::string> & item) -> bool {
auto lower = ToLower(item.first);
return (lower == "message-id") || (lower == "messageid");
};
auto msgid = std::find_if(header.begin(), header.end(), findMsgidFunc);
if(!IsValidMessageID(msgid->second))
auto msgid_itr = std::find_if(header.begin(), header.end(), findMsgidFunc);
if(msgid_itr == std::end(header))
{
std::clog << "invalid message-id: " << msgid->second << std::endl;
std::clog << "no message id for file " << fpath << std::endl;
return;
}
std::string msgid_hash = sha1_hex(msgid->second);
std::string msgid = StripWhitespaces(msgid_itr->second);
fs::path threadFilePath = m_OutDir / fs::path("thread-" + msgid_hash + ".html");
if(!IsValidMessageID(msgid))
{
std::clog << "invalid message-id: " << msgid << std::endl;
return;
}
std::string rootmsgid;
auto findReferences = [](const std::pair<std::string, std::string> & item) -> bool {
auto lower = ToLower(item.first);
return lower == "references";
};
auto references_itr = std::find_if(header.begin(), header.end(), findReferences);
if(references_itr == std::end(header) || StripWhitespaces(references_itr->second).size() == 0)
{
rootmsgid = msgid;
}
else
{
const auto & s = references_itr->second;
auto checkfunc = [] (unsigned char ch) -> bool { return std::isspace(ch) || std::iscntrl(ch); };
if(std::count_if(s.begin(), s.end(), checkfunc))
{
/** split off first element */
auto idx = std::find_if(s.begin(), s.end(), checkfunc);
rootmsgid = s.substr(0, s.find(*idx));
}
else
{
rootmsgid = references_itr->second;
}
}
std::string rootmsgid_hash = sha1_hex(rootmsgid);
fs::path threadFilePath = m_OutDir / fs::path("thread-" + rootmsgid_hash + ".html");
nntpchan::model::Thread thread;
if(!m_MessageDB)
{
std::clog << "no message database" << std::endl;
return;
}
if(!m_MessageDB->LoadThread(thread, rootmsgid))
{
std::clog << "cannot find thread with root " << rootmsgid << std::endl;
return;
}
TemplateEngine::Args_t thread_args;
thread_args["posts"] = thread;
if(m_TemplateEngine)
{
FileHandle_ptr out = OpenFile(threadFilePath, eWrite);
if(!m_TemplateEngine->WriteTemplate("thread.mustache", thread_args, out))
if(!out || !m_TemplateEngine->WriteTemplate("thread.mustache", thread_args, out))
{
std::clog << "failed to write " << threadFilePath << std::endl;
return;
@ -84,23 +135,35 @@ namespace nntpchan
if(IsValidNewsgroup(newsgroup))
newsgroups_list.insert(newsgroup);
}
nntpchan::model::BoardPage page;
for(const auto & name : newsgroups_list)
{
auto board = GetThreadsPaginated(name, 10, m_Pages);
uint32_t pageno = 0;
for(Threads_t threads : board)
while(pageno < m_Pages)
{
std::map<std::string, std::any> board_args;
board_args["group"] = std::make_any<std::string>(name);
board_args["pageno"] = std::make_any<uint32_t>(pageno);
board_args["threads"] = std::make_any<Threads_t>(threads);
fs::path boardPageFilename(newsgroup + "-" + std::to_string(pageno) + ".html");
page.clear();
if(!m_MessageDB->LoadBoardPage(page, name, 10, m_Pages))
{
std::clog << "cannot load board page "<< pageno << " for " << name << std::endl;
break;
}
TemplateEngine::Args_t page_args;
page_args["group"] = name;
page_args["threads"] = page;
page_args["pageno"] = std::to_string(pageno);
if(pageno)
page_args["prev_pageno"] = std::to_string(pageno-1);
if(pageno+1 < m_Pages)
page_args["next_pageno"] = std::to_string(pageno+1);
fs::path boardPageFilename(name + "-" + std::to_string(pageno) + ".html");
if(m_TemplateEngine)
{
FileHandle_ptr out = OpenFile(m_OutDir / boardPageFilename, eWrite);
m_TemplateEngine->WriteTemplate("board.mustache", board_args, out);
fs::path outfile = m_OutDir / boardPageFilename;
FileHandle_ptr out = OpenFile(outfile, eWrite);
if(out)
m_TemplateEngine->WriteTemplate("board.mustache", page_args, out);
else
std::clog << "failed to open board page " << outfile << std::endl;
}
++pageno;
@ -120,8 +183,4 @@ namespace nntpchan
return IsValidMessageID(msgid);
}
StaticFileFrontend::BoardPage_t StaticFileFrontend::GetThreadsPaginated(const std::string & group, uint32_t perpage, uint32_t pages)
{
return {};
}
}

20
contrib/backends/nntpchan-daemon/libnntpchan/storage.cpp
View File

@ -1,5 +1,6 @@
#include <nntpchan/storage.hpp>
#include <nntpchan/sanitize.hpp>
#include <cassert>
#include <sstream>
namespace nntpchan
@ -20,26 +21,37 @@ namespace nntpchan
{
basedir = fpath;
fs::create_directories(basedir);
assert(init_skiplist("posts_skiplist"));
}
bool ArticleStorage::Accept(const std::string& msgid)
bool ArticleStorage::init_skiplist(const std::string & subdir) const
{
fs::path skiplist = basedir / fs::path(subdir);
fs::create_directories(skiplist);
const auto subdirs = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"};
for (const auto & s : subdirs)
fs::create_directories(skiplist / s);
return true;
}
bool ArticleStorage::Accept(const std::string& msgid) const
{
if (!IsValidMessageID(msgid)) return false;
auto p = MessagePath(msgid);
return !fs::exists(p);
}
fs::path ArticleStorage::MessagePath(const std::string & msgid)
fs::path ArticleStorage::MessagePath(const std::string & msgid) const
{
return basedir / msgid;
}
FileHandle_ptr ArticleStorage::OpenRead(const std::string & msgid)
FileHandle_ptr ArticleStorage::OpenRead(const std::string & msgid) const
{
return OpenFile(MessagePath(msgid), eRead);
}
FileHandle_ptr ArticleStorage::OpenWrite(const std::string & msgid)
FileHandle_ptr ArticleStorage::OpenWrite(const std::string & msgid) const
{
return OpenFile(MessagePath(msgid), eWrite);
}

126
contrib/backends/nntpchan-daemon/libnntpchan/template_engine.cpp
View File

@ -1,25 +1,112 @@
#include <nntpchan/template_engine.hpp>
#include <nntpchan/sanitize.hpp>
#include <mstch/mstch.hpp>
#include <iostream>
#include <sstream>
namespace nntpchan
{
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
namespace mustache = mstch;
static mustache::map post_to_map(const nntpchan::model::Post & post)
{
mustache::map m;
mustache::array attachments;
mustache::map h;
for (const auto & att : nntpchan::model::GetAttachments(post))
{
mustache::map a;
a["filename"] = nntpchan::model::GetFilename(att);
a["hexdigest"] = nntpchan::model::GetHexDigest(att);
a["thumbnail"] = nntpchan::model::GetThumbnail(att);
attachments.push_back(a);
}
for (const auto & item : nntpchan::model::GetHeader(post))
{
mustache::array vals;
for(const auto & v : item.second)
vals.push_back(v);
h[item.first] = vals;
}
m["attachments"] = attachments;
m["message"] = nntpchan::model::GetBody(post);
m["header"] = h;
return m;
}
static mustache::map thread_to_map(const nntpchan::model::Thread & t)
{
mustache::map thread;
mustache::array posts;
for(const auto & post : t)
{
posts.push_back(post_to_map(post));
}
auto & opHeader = nntpchan::model::GetHeader(t[0]);
thread["title"] = nntpchan::model::HeaderIFind(opHeader, "subject", "None")[0];
thread["posts"] = posts;
return thread;
}
struct MustacheTemplateEngine : public TemplateEngine
{
struct Impl
{
Impl(const std::map<std::string, std::string> & partials) : m_partials(partials) {}
bool ParseTemplate(const FileHandle_ptr & in)
{
return true;
std::stringstream str;
std::string line;
while(std::getline(*in, line))
str << line << "\n";
m_tmplString = str.str();
return in->eof();
}
bool RenderFile(const Args_t & args, const FileHandle_ptr & out)
{
return true;
mustache::map obj;
for (const auto & item : args)
{
std::visit(overloaded {
[&obj, item](const nntpchan::model::Model & m) {
std::visit(overloaded {
[&obj, item](const nntpchan::model::BoardPage & p) {
mustache::array threads;
for (const auto & thread : p)
{
threads.push_back(thread_to_map(thread));
}
obj[item.first] = threads;
},
[&obj, item](const nntpchan::model::Thread & t) {
obj[item.first] = thread_to_map(t);
}
}, m);
}
,[&obj, item](const std::string & str) {
obj[item.first] = str;
}
}, item.second);
}
std::string str = mustache::render(m_tmplString, obj);
out->write(str.c_str(), str.size());
out->flush();
return !out->fail();
}
std::string m_tmplString;
const std::map<std::string, std::string> & m_partials;
};
virtual bool WriteTemplate(const fs::path & fpath, const Args_t & args, const FileHandle_ptr & out)
@ -30,13 +117,42 @@ namespace nntpchan
std::clog << "no such template at " << fpath << std::endl;
return false;
}
auto impl = std::make_unique<Impl>();
if(impl->ParseTemplate(templFile))
return impl->RenderFile(args, out);
std::map<std::string, std::string> partials;
if(!LoadPartials(fpath.parent_path(), partials))
{
std::clog << "failed to load partials" << std::endl;
return false;
}
Impl impl(partials);
if(impl.ParseTemplate(templFile))
{
return impl.RenderFile(args, out);
}
std::clog << "failed to parse template " << fpath << std::endl;
return false;
}
bool LoadPartials(fs::path dir, std::map<std::string, std::string> & partials)
{
const auto partial_files = { "header", "footer" };
for(const auto & fname : partial_files)
{
auto file = OpenFile(dir / fs::path(fname + std::string(".html")), eRead);
if(!file) {
std::clog << "no such partial: " << fname << std::endl;
return false;
}
std::string line;
std::stringstream input;
while(std::getline(*file, line))
input << line << "\n";
partials[fname] = input.str();
}
return true;
}
};
TemplateEngine * CreateTemplateEngine(const std::string & dialect)