diff --git a/contrib/tools/migrate/lynxchan/lynx2nntp.js b/contrib/tools/migrate/lynxchan/lynx2nntp.js new file mode 100755 index 0000000..a3d2d5c --- /dev/null +++ b/contrib/tools/migrate/lynxchan/lynx2nntp.js @@ -0,0 +1,185 @@ +#!/usr/bin/env nodejs + +var memegod = require("mongodb"); +var memegodClient = memegod.MongoClient; +var assert = require("assert"); +var http = require("http"); + + +// memegod daemon url +var url = "mongodb://127.0.0.1:27017/lynxchan"; + +// nntp frontend name for article import +var frontendName = "memegod.import"; +// srndv2 http frontend server to do api requests against +var srndApiServ = "localhost"; +var srndApiPort = 18000; +// username password combo for using api +var srndApiLogin = "user:pass"; +// some memegod thing +var idPrefix="v1-"; + + +// create a newsgroup name given board name +var makeGroupName = function(board) { + return "overchan.test.endchan." + board; +} + +// call function for each op on a board +var foreachOpsOnBoard = function(db, board, callback) { + var cur = db.collection("threads").find({boardUri: board}); + cur.each(function(err, doc) { + if (doc) { + callback(doc); + } else { + console.log("error fetching op on "+board+" "+err); + } + }); +} + +// call a callback for each reply to op +// pass in the memegod post +var foreachReplyForOP = function(db, op, callback) { + getMessageIDForMemegodID(idPrefix+op._id, function(msgid) { + var cur = db.collection("posts").find({ threadId: op.threadId}); + cur.each(function (err, doc) { + callback(doc); + }); + }, function() { + // not found? + console.log("op not in database!? "+op._id); + }); +} + +// find all boards in memegod +// call callback for each board +var foreachBoard = function(db, callback) { + var cursor = db.collection('boards').find(); + cursor.each(function(err, doc) { + if (doc) { + callback(doc) + } + }); +}; + + +// convert a memegod post from board into an overchan article +// call a callback with the created post +var createArticle = function(post, board, callback) { + var article = { + ip: post.ip.join("."), + message: post.message || " ", + frontend: frontendName, + newsgroup: makeGroupName(board), + headers: { + } + }; + article.headers["X-Memegod-Post-Id"] = post.postId; + article.headers["X-Memegod-Thread-Id"] = post.threadId; + article.headers["X-Memegod-Id"] = idPrefix+post._id; + article.headers["X-Migrated-From"] = "MemeGod"; + article.name = post.name || "Stephen Lynx"; + article.subject = post.subject || "MongoDB is Web Scale"; + + callback(article); +} + +// get message id given a memegod id +// must have already been inserted +// call callback pass in message id +var getMessageIDForMemegodID = function(id, callback) { + checkViaHeader("X-Memegod-Id", id, function(msgid) { + callback(msgid); + }, function() { console.log("message is not there?! "+id); }); +} + +// post an overchan article via the api +// call callback passing in the message-id of the new post +var postArticle = function(article, callback) { + var req = http.request({ + port: srndApiPort, + method: "POST", + path: "/api/post", + auth: srndApiLogin, + headers: { + "Content-Type": "text/json", + } + }, function(res) { + var data = ""; + res.on("data", function (chunk) { + data += chunk; + }); + res.on("end", function() { + var j = JSON.parse(data) + if (j) { + var msgid = j[0]; + callback(msgid); + } + }) + }); + req.write(json.stringify(article)); + req.end(); +} + +// check if an article exists given header name and header value +var checkViaHeader = function(name, value, yesCb, noCb) { + var req = http.request({ + port: srndApiPort, + method: "GET", + path: "/api/header?name="+name+"&value="+value, + }, function (res) { + var data = ""; + res.on("data", function(chnk) { + data += chnk; + }); + res.on("end", function() { + var j = JSON.parse(data); + if ( j.length > 0 ) { + // it exists + yesCb(j[0]); + } else { + // does not exist + noCb(); + } + }); + }); +} + +// check if a post exists +var checkPostExists = function(post, yescb, nocb) { + checkViaHeader("X-Memegod-Id", idPrefix + post._id, + function(msgid) { yescb(post, msgid); }, + function() { nocb(post); }); +} + +var putBoard = function(db, board) { + // for each op + foreachOpsOnBoard(db, board.boardUri, function(op) { + // create OP + createArticle(op, board.boardUrl, function(opArticle) { + // post OP + postArticle(opArticle, function (opMsgId) { + console.log("posted op "+ opMsgId); + // for each reply for OP + foreachReplyForOP(db, op, function(post) { + // put create reply + createArticle(post, board.boardUri, function(article) { + // set references header + article.headers["References"] = opMsgId; + postArticle(article, function(replyMsgId) { + console.log("posted reply to "+opMsgId+" as "+replyMsgId); + }); + }); + }); + }); + }); + }); +} + +memegodClient.connect(url, function(err, db) { + console.log("connected to the meme god"); + foreachBoard(db, function(board) { + console.log("put board: "+board.boardUri); + putBoard(db, board); + }); +});