archive/nntpchan
Archived
1
0
Fork 0
Code Activity

nntpchan-daeom code

Browse Source
This commit is contained in:
Jeff Becker
2017-10-13 07:58:41 -04:00
parent c583a03f81
commit e9c88ffd28
14 changed files with 328 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
contrib/backends/nntpchan-daemon/libnntpchan/nntp_handler.cpp
View File

@@ -9,11 +9,11 @@
namespace nntpchan
{
NNTPServerHandler::NNTPServerHandler(const std::string & storage) :
NNTPServerHandler::NNTPServerHandler(const fs::path & storage) :
LineReader(1024),
m_article(nullptr),
m_auth(nullptr),
m_store(storage),
m_store(std::make_unique<ArticleStorage>(storage)),
m_authed(false),
m_state(eStateReadCommand)
{
@@ -115,22 +115,22 @@ namespace nntpchan
} else if (cmd == "CHECK") {
if(cmdlen >= 2) {
const std::string & msgid = command[1];
if(IsValidMessageID(msgid) && m_store.Accept(msgid))
if(IsValidMessageID(msgid) && m_store->Accept(msgid))
{
QueueLine("238 "+msgid);
return;
}
QueueLine("438 "+msgid);
else
QueueLine("438 "+msgid);
}
else
QueueLine("501 syntax error");
} else if (cmd == "TAKETHIS") {
if (cmdlen == 2)
if (cmdlen >= 2)
{
const std::string & msgid = command[1];
if(m_store.Accept(msgid))
if(m_store->Accept(msgid))
{
m_article = m_store.OpenWrite(msgid);
m_article = m_store->OpenWrite(msgid);
}
m_articleName = msgid;
EnterState(eStateStoreArticle);

10
contrib/backends/nntpchan-daemon/libnntpchan/sanitize.cpp
View File

@@ -36,4 +36,14 @@ namespace nntpchan
{
return std::regex_search(msgid, re_ValidNewsgroup) == 1;
}
std::string StripWhitespaces(const std::string & str)
{
std::string stripped;
for(const auto & ch : str)
if(!(std::isspace(ch)||std::iscntrl(ch)))
stripped += ch;
return stripped;
}
}

109
contrib/backends/nntpchan-daemon/libnntpchan/staticfile_frontend.cpp
View File

@@ -36,28 +36,79 @@ namespace nntpchan
// read body
std::map<std::string, std::any> thread_args;
auto findMsgidFunc = [](const std::pair<std::string, std::string> & item) -> bool {
auto lower = ToLower(item.first);
return (lower == "message-id") || (lower == "messageid");
};
auto msgid = std::find_if(header.begin(), header.end(), findMsgidFunc);
if(!IsValidMessageID(msgid->second))
auto msgid_itr = std::find_if(header.begin(), header.end(), findMsgidFunc);
if(msgid_itr == std::end(header))
{
std::clog << "invalid message-id: " << msgid->second << std::endl;
std::clog << "no message id for file " << fpath << std::endl;
return;
}
std::string msgid = StripWhitespaces(msgid_itr->second);
if(!IsValidMessageID(msgid))
{
std::clog << "invalid message-id: " << msgid << std::endl;
return;
}
std::string rootmsgid;
auto findReferences = [](const std::pair<std::string, std::string> & item) -> bool {
auto lower = ToLower(item.first);
return lower == "references";
};
auto references_itr = std::find_if(header.begin(), header.end(), findReferences);
if(references_itr == std::end(header) || StripWhitespaces(references_itr->second).size() == 0)
{
rootmsgid = msgid;
}
else
{
const auto & s = references_itr->second;
auto checkfunc = [] (unsigned char ch) -> bool { return std::isspace(ch) || std::iscntrl(ch); };
if(std::count_if(s.begin(), s.end(), checkfunc))
{
/** split off first element */
auto idx = std::find_if(s.begin(), s.end(), checkfunc);
rootmsgid = s.substr(0, s.find(*idx));
}
else
{
rootmsgid = references_itr->second;
}
}
std::string rootmsgid_hash = sha1_hex(rootmsgid);
std::string msgid_hash = sha1_hex(msgid->second);
fs::path threadFilePath = m_OutDir / fs::path("thread-" + msgid_hash + ".html");
fs::path threadFilePath = m_OutDir / fs::path("thread-" + rootmsgid_hash + ".html");
nntpchan::model::Thread thread;
if(!m_MessageDB)
{
std::clog << "no message database" << std::endl;
return;
}
if(!m_MessageDB->LoadThread(thread, rootmsgid))
{
std::clog << "cannot find thread with root " << rootmsgid << std::endl;
return;
}
TemplateEngine::Args_t thread_args;
thread_args["posts"] = thread;
if(m_TemplateEngine)
{
FileHandle_ptr out = OpenFile(threadFilePath, eWrite);
if(!m_TemplateEngine->WriteTemplate("thread.mustache", thread_args, out))
if(!out || !m_TemplateEngine->WriteTemplate("thread.mustache", thread_args, out))
{
std::clog << "failed to write " << threadFilePath << std::endl;
return;
@@ -84,23 +135,35 @@ namespace nntpchan
if(IsValidNewsgroup(newsgroup))
newsgroups_list.insert(newsgroup);
}
nntpchan::model::BoardPage page;
for(const auto & name : newsgroups_list)
{
auto board = GetThreadsPaginated(name, 10, m_Pages);
uint32_t pageno = 0;
for(Threads_t threads : board)
while(pageno < m_Pages)
{
std::map<std::string, std::any> board_args;
board_args["group"] = std::make_any<std::string>(name);
board_args["pageno"] = std::make_any<uint32_t>(pageno);
board_args["threads"] = std::make_any<Threads_t>(threads);
fs::path boardPageFilename(newsgroup + "-" + std::to_string(pageno) + ".html");
page.clear();
if(!m_MessageDB->LoadBoardPage(page, name, 10, m_Pages))
{
std::clog << "cannot load board page "<< pageno << " for " << name << std::endl;
break;
}
TemplateEngine::Args_t page_args;
page_args["group"] = name;
page_args["threads"] = page;
page_args["pageno"] = std::to_string(pageno);
if(pageno)
page_args["prev_pageno"] = std::to_string(pageno-1);
if(pageno+1 < m_Pages)
page_args["next_pageno"] = std::to_string(pageno+1);
fs::path boardPageFilename(name + "-" + std::to_string(pageno) + ".html");
if(m_TemplateEngine)
{
FileHandle_ptr out = OpenFile(m_OutDir / boardPageFilename, eWrite);
m_TemplateEngine->WriteTemplate("board.mustache", board_args, out);
fs::path outfile = m_OutDir / boardPageFilename;
FileHandle_ptr out = OpenFile(outfile, eWrite);
if(out)
m_TemplateEngine->WriteTemplate("board.mustache", page_args, out);
else
std::clog << "failed to open board page " << outfile << std::endl;
}
++pageno;
@@ -119,9 +182,5 @@ namespace nntpchan
{
return IsValidMessageID(msgid);
}
StaticFileFrontend::BoardPage_t StaticFileFrontend::GetThreadsPaginated(const std::string & group, uint32_t perpage, uint32_t pages)
{
return {};
}
}

20
contrib/backends/nntpchan-daemon/libnntpchan/storage.cpp
View File

@@ -1,5 +1,6 @@
#include <nntpchan/storage.hpp>
#include <nntpchan/sanitize.hpp>
#include <cassert>
#include <sstream>
namespace nntpchan
@@ -20,26 +21,37 @@ namespace nntpchan
{
basedir = fpath;
fs::create_directories(basedir);
assert(init_skiplist("posts_skiplist"));
}
bool ArticleStorage::Accept(const std::string& msgid)
bool ArticleStorage::init_skiplist(const std::string & subdir) const
{
fs::path skiplist = basedir / fs::path(subdir);
fs::create_directories(skiplist);
const auto subdirs = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"};
for (const auto & s : subdirs)
fs::create_directories(skiplist / s);
return true;
}
bool ArticleStorage::Accept(const std::string& msgid) const
{
if (!IsValidMessageID(msgid)) return false;
auto p = MessagePath(msgid);
return !fs::exists(p);
}
fs::path ArticleStorage::MessagePath(const std::string & msgid)
fs::path ArticleStorage::MessagePath(const std::string & msgid) const
{
return basedir / msgid;
}
FileHandle_ptr ArticleStorage::OpenRead(const std::string & msgid)
FileHandle_ptr ArticleStorage::OpenRead(const std::string & msgid) const
{
return OpenFile(MessagePath(msgid), eRead);
}
FileHandle_ptr ArticleStorage::OpenWrite(const std::string & msgid)
FileHandle_ptr ArticleStorage::OpenWrite(const std::string & msgid) const
{
return OpenFile(MessagePath(msgid), eWrite);
}

126
contrib/backends/nntpchan-daemon/libnntpchan/template_engine.cpp
View File

@@ -1,25 +1,112 @@
#include <nntpchan/template_engine.hpp>
#include <nntpchan/sanitize.hpp>
#include <mstch/mstch.hpp>
#include <iostream>
#include <sstream>
namespace nntpchan
{
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
namespace mustache = mstch;
static mustache::map post_to_map(const nntpchan::model::Post & post)
{
mustache::map m;
mustache::array attachments;
mustache::map h;
for (const auto & att : nntpchan::model::GetAttachments(post))
{
mustache::map a;
a["filename"] = nntpchan::model::GetFilename(att);
a["hexdigest"] = nntpchan::model::GetHexDigest(att);
a["thumbnail"] = nntpchan::model::GetThumbnail(att);
attachments.push_back(a);
}
for (const auto & item : nntpchan::model::GetHeader(post))
{
mustache::array vals;
for(const auto & v : item.second)
vals.push_back(v);
h[item.first] = vals;
}
m["attachments"] = attachments;
m["message"] = nntpchan::model::GetBody(post);
m["header"] = h;
return m;
}
static mustache::map thread_to_map(const nntpchan::model::Thread & t)
{
mustache::map thread;
mustache::array posts;
for(const auto & post : t)
{
posts.push_back(post_to_map(post));
}
auto & opHeader = nntpchan::model::GetHeader(t[0]);
thread["title"] = nntpchan::model::HeaderIFind(opHeader, "subject", "None")[0];
thread["posts"] = posts;
return thread;
}
struct MustacheTemplateEngine : public TemplateEngine
{
struct Impl
{
Impl(const std::map<std::string, std::string> & partials) : m_partials(partials) {}
bool ParseTemplate(const FileHandle_ptr & in)
{
return true;
std::stringstream str;
std::string line;
while(std::getline(*in, line))
str << line << "\n";
m_tmplString = str.str();
return in->eof();
}
bool RenderFile(const Args_t & args, const FileHandle_ptr & out)
{
return true;
mustache::map obj;
for (const auto & item : args)
{
std::visit(overloaded {
[&obj, item](const nntpchan::model::Model & m) {
std::visit(overloaded {
[&obj, item](const nntpchan::model::BoardPage & p) {
mustache::array threads;
for (const auto & thread : p)
{
threads.push_back(thread_to_map(thread));
}
obj[item.first] = threads;
},
[&obj, item](const nntpchan::model::Thread & t) {
obj[item.first] = thread_to_map(t);
}
}, m);
}
,[&obj, item](const std::string & str) {
obj[item.first] = str;
}
}, item.second);
}
std::string str = mustache::render(m_tmplString, obj);
out->write(str.c_str(), str.size());
out->flush();
return !out->fail();
}
std::string m_tmplString;
const std::map<std::string, std::string> & m_partials;
};
virtual bool WriteTemplate(const fs::path & fpath, const Args_t & args, const FileHandle_ptr & out)
@@ -30,13 +117,42 @@ namespace nntpchan
std::clog << "no such template at " << fpath << std::endl;
return false;
}
auto impl = std::make_unique<Impl>();
if(impl->ParseTemplate(templFile))
return impl->RenderFile(args, out);
std::map<std::string, std::string> partials;
if(!LoadPartials(fpath.parent_path(), partials))
{
std::clog << "failed to load partials" << std::endl;
return false;
}
Impl impl(partials);
if(impl.ParseTemplate(templFile))
{
return impl.RenderFile(args, out);
}
std::clog << "failed to parse template " << fpath << std::endl;
return false;
}
bool LoadPartials(fs::path dir, std::map<std::string, std::string> & partials)
{
const auto partial_files = { "header", "footer" };
for(const auto & fname : partial_files)
{
auto file = OpenFile(dir / fs::path(fname + std::string(".html")), eRead);
if(!file) {
std::clog << "no such partial: " << fname << std::endl;
return false;
}
std::string line;
std::stringstream input;
while(std::getline(*file, line))
input << line << "\n";
partials[fname] = input.str();
}
return true;
}
};
TemplateEngine * CreateTemplateEngine(const std::string & dialect)