fixes to migration script

contrib/tools/migrate/lynxchan/lynx2nntp.js

@@ -22,32 +22,29 @@ var idPrefix="v1-";
 
 // create a newsgroup name given board name
 var makeGroupName = function(board) {
-  return "overchan.test.endchan." + board;
+  return "overchan.test.endchan." + board.boardUri;
 }
 
 // call function for each op on a board
 var foreachOpsOnBoard = function(db, board, callback) {
   var cur = db.collection("threads").find({boardUri: board});
   cur.each(function(err, doc) {
-    if (doc) {
+    callback(doc);
-      callback(doc);
-    } else {
-      console.log("error fetching op on "+board+" "+err);
-    }
   });
 }
 
 // call a callback for each reply to op
 // pass in the memegod post
 var foreachReplyForOP = function(db, op, callback) {
-  getMessageIDForMemegodID(idPrefix+op._id, function(msgid) {
+  checkViaHeader("X-Mememgod-Id", idPrefix+op._id, function(msgid) {
+    // we do already has got it
+    callback(null);
+  }, function(msgid) {
+    // we don't has got it
     var cur = db.collection("posts").find({ threadId: op.threadId});
     cur.each(function (err, doc) {
       callback(doc);
     });
-  }, function() {
-    // not found?
-    console.log("op not in database!? "+op._id);
   });
 }
 
@@ -66,6 +63,8 @@ var foreachBoard = function(db, callback) {
 // convert a memegod post from board into an overchan article
 // call a callback with the created post
 var createArticle = function(post, board, callback) {
+  if (post == null) return;
+      
   var article = {
     ip: post.ip.join("."),
     message: post.message || " ",
@@ -74,51 +73,53 @@ var createArticle = function(post, board, callback) {
     headers: {
     }
   };
-  article.headers["X-Memegod-Post-Id"] = post.postId;
+  if (post.postId) {
-  article.headers["X-Memegod-Thread-Id"] = post.threadId;
+    article.headers["X-Memegod-Post-Id"] = idPrefix+post.postId
+  }
+  if (post.threadId) {
+    article.headers["X-Memegod-Thread-Id"] = idPrefix+post.threadId;
+  }
   article.headers["X-Memegod-Id"] = idPrefix+post._id;
   article.headers["X-Migrated-From"] = "MemeGod";
   article.name = post.name || "Stephen Lynx";
   article.subject = post.subject || "MongoDB is Web Scale";
-  
   callback(article);
 }
 
-// get message id given a memegod id
-// must have already been inserted
-// call callback pass in message id
-var getMessageIDForMemegodID = function(id, callback) {
-  checkViaHeader("X-Memegod-Id", id, function(msgid) {
-    callback(msgid);
-  }, function() { console.log("message is not there?! "+id); });
-}
-
 // post an overchan article via the api
 // call callback passing in the message-id of the new post
 var postArticle = function(article, callback) {
-  var req = http.request({
+
-    port: srndApiPort,
+  checkViaHeader("X-Memegod-Id", article.headers["X-Memegod-Id"], function(msgid) {
-    method: "POST",
+    // we has got it already
-    path: "/api/post",
+    callback(msgid);
-    auth: srndApiLogin,
+  }, function(msgid) {
-    headers: {
+    // we don't has got it
-      "Content-Type": "text/json",
+    var req = http.request({
-    }
+      port: srndApiPort,
-  }, function(res) {
+      method: "POST",
-    var data = "";
+      path: "/api/post",
-    res.on("data", function (chunk) {
+      auth: srndApiLogin,
-      data += chunk;
+      headers: {
-    });
+        "Content-Type": "text/json",
-    res.on("end", function() {
-      var j = JSON.parse(data)
-      if (j) {
-        var msgid = j[0];
-        callback(msgid);
       }
-    })
+    }, function(res) {
+      var data = "";
+      res.on("data", function (chunk) {
+        data += chunk;
+      });
+      res.on("end", function() {
+        var j = JSON.parse(data)
+        var msgid = j.id;
+        callback(msgid);
+        
+      })
+    });
+    req.write(JSON.stringify(article));
+    req.end();
+  }, function(msgid) {
+    // not there
   });
-  req.write(JSON.stringify(article));
-  req.end();
 }
 
 // check if an article exists given header name and header value
@@ -143,6 +144,7 @@ var checkViaHeader = function(name, value, yesCb, noCb) {
       }
     });
   });
+  req.end();
 }
 
 // check if a post exists 
@@ -153,24 +155,24 @@ var checkPostExists = function(post, yescb, nocb) {
 }
 
 var putBoard = function(db, board) {
-  // for each op
   foreachOpsOnBoard(db, board.boardUri, function(op) {
     // create OP
-    createArticle(op, board.boardUrl, function(opArticle) {
+    createArticle(op, board, function(opArticle) {
       // post OP
       postArticle(opArticle, function (opMsgId) {
-        console.log("posted op "+ opMsgId);
         // for each reply for OP
         foreachReplyForOP(db, op, function(post) {
-          // put create reply
+          if (post) {
-          createArticle(post, board.boardUri, function(article) {
+            // put create reply
-            // set references header
+            createArticle(post, board, function(article) {
-            article.headers["References"] = opMsgId;
+              // set references header
-            postArticle(article, function(replyMsgId) {
+              article.headers["References"] = opMsgId;
-              console.log("posted reply to "+opMsgId+" as "+replyMsgId);
+              // post reply
+              postArticle(article, function() {});
             });
-          });
+          }
         });
+        
       });              
     });
   });
@@ -179,7 +181,9 @@ var putBoard = function(db, board) {
 memegodClient.connect(url, function(err, db) {
   console.log("connected to the meme god");
   foreachBoard(db, function(board) {
-    console.log("put board: "+board.boardUri);
+    console.log("updating "+board.boardUri);
     putBoard(db, board);
   });
+  console.log("the meme god has spoken");
+  db.close();
 });