add refactored rewrite

contrib/backends/nntpchand/Makefile

@@ -0,0 +1,11 @@
+REPO=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+
+all: clean build
+
+build: nntpchand
+
+nntpchand:
+	GOPATH=$(REPO) go build -v
+
+clean:
+	GOPATH=$(REPO) go clean -v

contrib/backends/nntpchand/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"nntpchan/cmd/nntpchan"
+)
+
+func main() {
+	nntpchan.Main()
+}

contrib/backends/nntpchand/src/nntpchan/cmd/nntpchan/main.go

@@ -0,0 +1,160 @@
+package nntpchan
+
+import (
+	log "github.com/Sirupsen/logrus"
+	"net"
+	_ "net/http/pprof"
+	"nntpchan/lib/config"
+	"nntpchan/lib/database"
+	"nntpchan/lib/frontend"
+	"nntpchan/lib/nntp"
+	"nntpchan/lib/store"
+	"nntpchan/lib/webhooks"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+type runStatus struct {
+	nntpListener net.Listener
+	run          bool
+	done         chan error
+}
+
+func (st *runStatus) Stop() {
+	st.run = false
+	if st.nntpListener != nil {
+		st.nntpListener.Close()
+	}
+	st.nntpListener = nil
+	log.Info("stopping daemon process")
+}
+
+func Main() {
+	st := &runStatus{
+		run:  true,
+		done: make(chan error),
+	}
+	log.Info("starting up nntpchan...")
+	cfgFname := "nntpchan.json"
+	conf, err := config.Ensure(cfgFname)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if conf.Log == "debug" {
+		log.SetLevel(log.DebugLevel)
+	}
+
+	sconfig := conf.Store
+
+	if sconfig == nil {
+		log.Fatal("no article storage configured")
+	}
+
+	nconfig := conf.NNTP
+
+	if nconfig == nil {
+		log.Fatal("no nntp server configured")
+	}
+
+	dconfig := conf.Database
+
+	if dconfig == nil {
+		log.Fatal("no database configured")
+	}
+
+	// create nntp server
+	nserv := nntp.NewServer()
+	nserv.Config = nconfig
+	nserv.Feeds = conf.Feeds
+
+	if nconfig.LoginsFile != "" {
+		nserv.Auth = nntp.FlatfileAuth(nconfig.LoginsFile)
+	}
+
+	// create article storage
+	nserv.Storage, err = store.NewFilesytemStorage(sconfig.Path, true)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if conf.WebHooks != nil && len(conf.WebHooks) > 0 {
+		// put webhooks into nntp server event hooks
+		nserv.Hooks = webhooks.NewWebhooks(conf.WebHooks, nserv.Storage)
+	}
+
+	if conf.NNTPHooks != nil && len(conf.NNTPHooks) > 0 {
+		var hooks nntp.MulitHook
+		if nserv.Hooks != nil {
+			hooks = append(hooks, nserv.Hooks)
+		}
+		for _, h := range conf.NNTPHooks {
+			hooks = append(hooks, nntp.NewHook(h))
+		}
+		nserv.Hooks = hooks
+	}
+
+	var db database.Database
+	for _, fconf := range conf.Frontends {
+		var f frontend.Frontend
+		f, err = frontend.NewHTTPFrontend(fconf, db)
+		if err == nil {
+			go f.Serve()
+		}
+	}
+
+	// start persisting feeds
+	go nserv.PersistFeeds()
+
+	// handle signals
+	sigchnl := make(chan os.Signal, 1)
+	signal.Notify(sigchnl, syscall.SIGHUP, os.Interrupt)
+	go func() {
+		for {
+			s := <-sigchnl
+			if s == syscall.SIGHUP {
+				// handle SIGHUP
+				conf, err := config.Ensure(cfgFname)
+				if err == nil {
+					log.Infof("reloading config: %s", cfgFname)
+					nserv.ReloadServer(conf.NNTP)
+					nserv.ReloadFeeds(conf.Feeds)
+				} else {
+					log.Errorf("failed to reload config: %s", err)
+				}
+			} else if s == os.Interrupt {
+				// handle interrupted, clean close
+				st.Stop()
+				return
+			}
+		}
+	}()
+	go func() {
+		var err error
+		for st.run {
+			var nl net.Listener
+			naddr := conf.NNTP.Bind
+			log.Infof("Bind nntp server to %s", naddr)
+			nl, err = net.Listen("tcp", naddr)
+			if err == nil {
+				st.nntpListener = nl
+				err = nserv.Serve(nl)
+				if err != nil {
+					nl.Close()
+					log.Errorf("nntpserver.serve() %s", err.Error())
+				}
+			} else {
+				log.Errorf("nntp server net.Listen failed: %s", err.Error())
+			}
+			time.Sleep(time.Second)
+		}
+		st.done <- err
+	}()
+	e := <-st.done
+	if e != nil {
+		log.Fatal(e)
+	}
+	log.Info("ended")
+}

contrib/backends/nntpchand/src/nntpchan/cmd/nntpserver/main.go

@@ -0,0 +1,42 @@
+package main
+
+// simple nntp server
+
+import (
+	log "github.com/Sirupsen/logrus"
+	"github.com/majestrate/srndv2/lib/config"
+	"github.com/majestrate/srndv2/lib/nntp"
+	"github.com/majestrate/srndv2/lib/store"
+	"net"
+)
+
+func main() {
+
+	log.Info("starting NNTP server...")
+	conf, err := config.Ensure("settings.json")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if conf.Log == "debug" {
+		log.SetLevel(log.DebugLevel)
+	}
+
+	serv := &nntp.Server{
+		Config: conf.NNTP,
+		Feeds:  conf.Feeds,
+	}
+	serv.Storage, err = store.NewFilesytemStorage(conf.Store.Path, false)
+	if err != nil {
+		log.Fatal(err)
+	}
+	l, err := net.Listen("tcp", conf.NNTP.Bind)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Info("listening on ", l.Addr())
+	err = serv.Serve(l)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/admin/doc.go

@@ -0,0 +1,4 @@
+//
+// server admin panel
+//
+package admin

contrib/backends/nntpchand/src/nntpchan/lib/admin/server.go

@@ -0,0 +1,16 @@
+package admin
+
+import (
+	"net/http"
+)
+
+type Server struct {
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+
+}
+
+func NewServer() *Server {
+	return &Server{}
+}

contrib/backends/nntpchand/src/nntpchan/lib/api/api.go

@@ -0,0 +1,10 @@
+package api
+
+import (
+	"nntpchan/lib/model"
+)
+
+// json api
+type API interface {
+	MakePost(p model.Post)
+}

contrib/backends/nntpchand/src/nntpchan/lib/api/doc.go

@@ -0,0 +1,2 @@
+// json api
+package api

contrib/backends/nntpchand/src/nntpchan/lib/api/server.go

@@ -0,0 +1,20 @@
+package api
+
+import (
+	"github.com/gorilla/mux"
+	"net/http"
+)
+
+// api server
+type Server struct {
+}
+
+func (s *Server) HandlePing(w http.ResponseWriter, r *http.Request) {
+
+}
+
+// inject api routes
+func (s *Server) SetupRoutes(r *mux.Router) {
+	// setup api pinger
+	r.Path("/ping").HandlerFunc(s.HandlePing)
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/article.go

@@ -0,0 +1,73 @@
+package config
+
+import "regexp"
+
+// configration for local article policies
+type ArticleConfig struct {
+	// explicitly allow these newsgroups (regexp)
+	AllowGroups []string `json:"whitelist"`
+	// explicitly disallow these newsgroups (regexp)
+	DisallowGroups []string `json:"blacklist"`
+	// only allow explicitly allowed groups
+	ForceWhitelist bool `json:"force-whitelist"`
+	// allow anonymous posts?
+	AllowAnon bool `json:"anon"`
+	// allow attachments?
+	AllowAttachments bool `json:"attachments"`
+	// allow anonymous attachments?
+	AllowAnonAttachments bool `json:"anon-attachments"`
+}
+
+func (c *ArticleConfig) AllowGroup(group string) bool {
+
+	for _, g := range c.DisallowGroups {
+		r := regexp.MustCompile(g)
+		if r.MatchString(group) && c.ForceWhitelist {
+			// disallowed
+			return false
+		}
+	}
+
+	// check allowed groups first
+	for _, g := range c.AllowGroups {
+		r := regexp.MustCompile(g)
+		if r.MatchString(g) {
+			return true
+		}
+	}
+
+	return !c.ForceWhitelist
+}
+
+// allow an article?
+func (c *ArticleConfig) Allow(msgid, group string, anon, attachment bool) bool {
+
+	// check attachment policy
+	if c.AllowGroup(group) {
+		allow := true
+		// no anon ?
+		if anon && !c.AllowAnon {
+			allow = false
+		}
+		// no attachments ?
+		if allow && attachment && !c.AllowAttachments {
+			allow = false
+		}
+		// no anon attachments ?
+		if allow && attachment && anon && !c.AllowAnonAttachments {
+			allow = false
+		}
+		return allow
+	} else {
+		return false
+	}
+}
+
+var DefaultArticlePolicy = ArticleConfig{
+	AllowGroups:          []string{"ctl", "overchan.test"},
+	DisallowGroups:       []string{"overchan.cp"},
+	ForceWhitelist:       false,
+	AllowAnon:            true,
+	AllowAttachments:     true,
+	AllowAnonAttachments: false,
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/cache.go

@@ -0,0 +1,13 @@
+package config
+
+// caching interface configuration
+type CacheConfig struct {
+	// backend cache driver name
+	Backend string `json:"backend"`
+	// address for cache
+	Addr string `json:"addr"`
+	// username for login
+	User string `json:"user"`
+	// password for login
+	Password string `json:"password"`
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/config.go

@@ -0,0 +1,87 @@
+package config
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+)
+
+// main configuration
+type Config struct {
+	// nntp server configuration
+	NNTP *NNTPServerConfig `json:"nntp"`
+	// log level
+	Log string `json:"log"`
+	// article storage config
+	Store *StoreConfig `json:"storage"`
+	// web hooks to call
+	WebHooks []*WebhookConfig `json:"webhooks"`
+	// external scripts to call
+	NNTPHooks []*NNTPHookConfig `json:"nntphooks"`
+	// database backend configuration
+	Database *DatabaseConfig `json:"db"`
+	// list of feeds to add on runtime
+	Feeds []*FeedConfig `json:"feeds"`
+	// frontend config
+	Frontends []*FrontendConfig `json:"frontends"`
+	// unexported fields ...
+
+	// absolute filepath to configuration
+	fpath string
+}
+
+// default configuration
+var DefaultConfig = Config{
+	Store:     &DefaultStoreConfig,
+	NNTP:      &DefaultNNTPConfig,
+	Database:  &DefaultDatabaseConfig,
+	WebHooks:  []*WebhookConfig{DefaultWebHookConfig},
+	NNTPHooks: []*NNTPHookConfig{DefaultNNTPHookConfig},
+	Feeds:     DefaultFeeds,
+	Frontends: []*FrontendConfig{&DefaultFrontendConfig},
+	Log:       "debug",
+}
+
+// reload configuration
+func (c *Config) Reload() (err error) {
+	var b []byte
+	b, err = ioutil.ReadFile(c.fpath)
+	if err == nil {
+		err = json.Unmarshal(b, c)
+	}
+	return
+}
+
+// ensure that a config file exists
+// creates one if it does not exist
+func Ensure(fname string) (cfg *Config, err error) {
+	_, err = os.Stat(fname)
+	if os.IsNotExist(err) {
+		err = nil
+		var d []byte
+		d, err = json.Marshal(&DefaultConfig)
+		if err == nil {
+			b := new(bytes.Buffer)
+			err = json.Indent(b, d, "", "  ")
+			if err == nil {
+				err = ioutil.WriteFile(fname, b.Bytes(), 0600)
+			}
+		}
+	}
+	if err == nil {
+		cfg, err = Load(fname)
+	}
+	return
+}
+
+// load configuration file
+func Load(fname string) (cfg *Config, err error) {
+	cfg = new(Config)
+	cfg.fpath = fname
+	err = cfg.Reload()
+	if err != nil {
+		cfg = nil
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/database.go

@@ -0,0 +1,18 @@
+package config
+
+type DatabaseConfig struct {
+	// url or address for database connector
+	Addr string `json:"addr"`
+	// password to use
+	Password string `json:"password"`
+	// username to use
+	Username string `json:"username"`
+	// type of database to use
+	Type string `json:"type"`
+}
+
+var DefaultDatabaseConfig = DatabaseConfig{
+	Type:     "postgres",
+	Addr:     "/var/run/postgresql",
+	Password: "",
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/doc.go

@@ -0,0 +1,4 @@
+//
+// package for parsing config files
+//
+package config

contrib/backends/nntpchand/src/nntpchan/lib/config/feed.go

@@ -0,0 +1,33 @@
+package config
+
+// configuration for 1 nntp feed
+type FeedConfig struct {
+	// feed's policy, filters articles
+	Policy *ArticleConfig `json:"policy"`
+	// remote server's address
+	Addr string `json:"addr"`
+	// proxy server config
+	Proxy *ProxyConfig `json:"proxy"`
+	// nntp username to log in with
+	Username string `json:"username"`
+	// nntp password to use when logging in
+	Password string `json:"password"`
+	// do we want to use tls?
+	TLS bool `json:"tls"`
+	// the name of this feed
+	Name string `json:"name"`
+	// how often to pull articles from the server in minutes
+	// 0 for never
+	PullInterval int `json:"pull"`
+}
+
+var DuummyFeed = FeedConfig{
+	Policy: &DefaultArticlePolicy,
+	Addr:   "nntp.dummy.tld:1119",
+	Proxy:  &DefaultTorProxy,
+	Name:   "dummy",
+}
+
+var DefaultFeeds = []*FeedConfig{
+	&DuummyFeed,
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/frontend.go

@@ -0,0 +1,21 @@
+package config
+
+type FrontendConfig struct {
+	// bind to address
+	BindAddr string `json:"bind"`
+	// frontend cache
+	Cache *CacheConfig `json:"cache"`
+	// frontend ssl settings
+	SSL *SSLSettings `json:"ssl"`
+	// static files directory
+	Static string `json:"static_dir"`
+	// http middleware configuration
+	Middleware *MiddlewareConfig `json:"middleware"`
+}
+
+// default Frontend Configuration
+var DefaultFrontendConfig = FrontendConfig{
+	BindAddr:   "127.0.0.1:18888",
+	Static:     "./files/static/",
+	Middleware: &DefaultMiddlewareConfig,
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/hook.go

@@ -0,0 +1,15 @@
+package config
+
+// config for external callback for nntp articles
+type NNTPHookConfig struct {
+	// name of hook
+	Name string `json:"name"`
+	// executable script path to be called with arguments: /path/to/article
+	Exec string `json:"exec"`
+}
+
+// default dummy hook
+var DefaultNNTPHookConfig = &NNTPHookConfig{
+	Name: "dummy",
+	Exec: "/bin/true",
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/middleware.go

@@ -0,0 +1,14 @@
+package config
+
+// configuration for http middleware
+type MiddlewareConfig struct {
+	// middleware type, currently just 1 is available: overchan
+	Type string `json:"type"`
+	// directory for our html templates
+	Templates string `json:"templates_dir"`
+}
+
+var DefaultMiddlewareConfig = MiddlewareConfig{
+	Type:      "overchan",
+	Templates: "./files/templates/overchan/",
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/nntp.go

@@ -0,0 +1,24 @@
+package config
+
+type NNTPServerConfig struct {
+	// address to bind to
+	Bind string `json:"bind"`
+	// name of the nntp server
+	Name string `json:"name"`
+	// default inbound article policy
+	Article *ArticleConfig `json:"policy"`
+	// do we allow anonymous NNTP sync?
+	AnonNNTP bool `json:"anon-nntp"`
+	// ssl settings for nntp
+	SSL *SSLSettings
+	// file with login credentials
+	LoginsFile string `json:"authfile"`
+}
+
+var DefaultNNTPConfig = NNTPServerConfig{
+	AnonNNTP:   false,
+	Bind:       "0.0.0.0:1119",
+	Name:       "nntp.server.tld",
+	Article:    &DefaultArticlePolicy,
+	LoginsFile: "",
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/proxy.go

@@ -0,0 +1,13 @@
+package config
+
+// proxy configuration
+type ProxyConfig struct {
+	Type string `json:"type"`
+	Addr string `json:"addr"`
+}
+
+// default tor proxy
+var DefaultTorProxy = ProxyConfig{
+	Type: "socks",
+	Addr: "127.0.0.1:9050",
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/ssl.go

@@ -0,0 +1,11 @@
+package config
+
+// settings for setting up ssl
+type SSLSettings struct {
+	// path to ssl private key
+	SSLKeyFile string `json:"key"`
+	// path to ssl certificate signed by CA
+	SSLCertFile string `json:"cert"`
+	// domain name to use for ssl
+	DomainName string `json:"fqdn"`
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/store.go

@@ -0,0 +1,10 @@
+package config
+
+type StoreConfig struct {
+	// path to article directory
+	Path string `json:"path"`
+}
+
+var DefaultStoreConfig = StoreConfig{
+	Path: "storage",
+}

contrib/backends/nntpchand/src/nntpchan/lib/config/webhooks.go

@@ -0,0 +1,17 @@
+package config
+
+// configuration for a single web hook
+type WebhookConfig struct {
+	// user provided name for this hook
+	Name string `json:"name"`
+	// callback URL for webhook
+	URL string `json:"url"`
+	// dialect to use when calling webhook
+	Dialect string `json:"dialect"`
+}
+
+var DefaultWebHookConfig = &WebhookConfig{
+	Name:    "vichan",
+	Dialect: "vichan",
+	URL:     "http://localhost/webhook.php",
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/doc.go

@@ -0,0 +1,5 @@
+//
+// nntpchan crypto package
+// wraps all external crypro libs
+//
+package crypto

contrib/backends/nntpchand/src/nntpchan/lib/crypto/hash.go

@@ -0,0 +1,8 @@
+package crypto
+
+import (
+	"github.com/dchest/blake256"
+)
+
+// common hash function is blake2
+var Hash = blake256.New

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl.go

@@ -0,0 +1,82 @@
+package crypto
+
+import (
+	"crypto/sha512"
+	"hash"
+	"nntpchan/lib/crypto/nacl"
+)
+
+type fuckyNacl struct {
+	k    []byte
+	hash hash.Hash
+}
+
+func (fucky *fuckyNacl) Write(d []byte) (int, error) {
+	return fucky.hash.Write(d)
+}
+
+func (fucky *fuckyNacl) Sign() (s Signature) {
+	h := fucky.hash.Sum(nil)
+	if h == nil {
+		panic("fuck.hash.Sum == nil")
+	}
+	kp := nacl.LoadSignKey(fucky.k)
+	defer kp.Free()
+	sk := kp.Secret()
+	sig := nacl.CryptoSignFucky(h, sk)
+	if sig == nil {
+		panic("fucky signer's call to nacl.CryptoSignFucky returned nil")
+	}
+	s = Signature(sig)
+	fucky.resetState()
+	return
+}
+
+// reset inner state so we can reuse this fuckyNacl for another operation
+func (fucky *fuckyNacl) resetState() {
+	fucky.hash = sha512.New()
+}
+
+func (fucky *fuckyNacl) Verify(sig Signature) (valid bool) {
+	h := fucky.hash.Sum(nil)
+	if h == nil {
+		panic("fuck.hash.Sum == nil")
+	}
+	valid = nacl.CryptoVerifyFucky(h, sig, fucky.k)
+	fucky.resetState()
+	return
+}
+
+func createFucky(k []byte) *fuckyNacl {
+	return &fuckyNacl{
+		k:    k,
+		hash: sha512.New(),
+	}
+}
+
+// create a standard signer given a secret key
+func CreateSigner(sk []byte) Signer {
+	return createFucky(sk)
+}
+
+// create a standard verifier given a public key
+func CreateVerifier(pk []byte) Verifer {
+	return createFucky(pk)
+}
+
+// get the public component given the secret key
+func ToPublic(sk []byte) (pk []byte) {
+	kp := nacl.LoadSignKey(sk)
+	defer kp.Free()
+	pk = kp.Public()
+	return
+}
+
+// create a standard keypair
+func GenKeypair() (pk, sk []byte) {
+	kp := nacl.GenSignKeypair()
+	defer kp.Free()
+	pk = kp.Public()
+	sk = kp.Seed()
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/box.go

@@ -0,0 +1,95 @@
+package nacl
+
+// #cgo freebsd CFLAGS: -I/usr/local/include
+// #cgo freebsd LDFLAGS: -L/usr/local/lib
+// #cgo LDFLAGS: -lsodium
+// #include <sodium.h>
+import "C"
+
+import (
+	"errors"
+)
+
+// encrypts a message to a user given their public key is known
+// returns an encrypted box
+func CryptoBox(msg, nounce, pk, sk []byte) ([]byte, error) {
+	msgbuff := NewBuffer(msg)
+	defer msgbuff.Free()
+
+	// check sizes
+	if len(pk) != int(C.crypto_box_publickeybytes()) {
+		err := errors.New("len(pk) != crypto_box_publickey_bytes")
+		return nil, err
+	}
+	if len(sk) != int(C.crypto_box_secretkeybytes()) {
+		err := errors.New("len(sk) != crypto_box_secretkey_bytes")
+		return nil, err
+	}
+	if len(nounce) != int(C.crypto_box_macbytes()) {
+		err := errors.New("len(nounce) != crypto_box_macbytes()")
+		return nil, err
+	}
+
+	pkbuff := NewBuffer(pk)
+	defer pkbuff.Free()
+	skbuff := NewBuffer(sk)
+	defer skbuff.Free()
+	nouncebuff := NewBuffer(nounce)
+	defer nouncebuff.Free()
+
+	resultbuff := malloc(msgbuff.size + nouncebuff.size)
+	defer resultbuff.Free()
+	res := C.crypto_box_easy(resultbuff.uchar(), msgbuff.uchar(), C.ulonglong(msgbuff.size), nouncebuff.uchar(), pkbuff.uchar(), skbuff.uchar())
+	if res != 0 {
+		err := errors.New("crypto_box_easy failed")
+		return nil, err
+	}
+	return resultbuff.Bytes(), nil
+}
+
+// open an encrypted box
+func CryptoBoxOpen(box, nounce, sk, pk []byte) ([]byte, error) {
+	boxbuff := NewBuffer(box)
+	defer boxbuff.Free()
+
+	// check sizes
+	if len(pk) != int(C.crypto_box_publickeybytes()) {
+		err := errors.New("len(pk) != crypto_box_publickey_bytes")
+		return nil, err
+	}
+	if len(sk) != int(C.crypto_box_secretkeybytes()) {
+		err := errors.New("len(sk) != crypto_box_secretkey_bytes")
+		return nil, err
+	}
+	if len(nounce) != int(C.crypto_box_macbytes()) {
+		err := errors.New("len(nounce) != crypto_box_macbytes()")
+		return nil, err
+	}
+
+	pkbuff := NewBuffer(pk)
+	defer pkbuff.Free()
+	skbuff := NewBuffer(sk)
+	defer skbuff.Free()
+	nouncebuff := NewBuffer(nounce)
+	defer nouncebuff.Free()
+	resultbuff := malloc(boxbuff.size - nouncebuff.size)
+	defer resultbuff.Free()
+
+	// decrypt
+	res := C.crypto_box_open_easy(resultbuff.uchar(), boxbuff.uchar(), C.ulonglong(boxbuff.size), nouncebuff.uchar(), pkbuff.uchar(), skbuff.uchar())
+	if res != 0 {
+		return nil, errors.New("crypto_box_open_easy failed")
+	}
+	// return result
+	return resultbuff.Bytes(), nil
+}
+
+// generate a new nounce
+func NewBoxNounce() []byte {
+	return RandBytes(NounceLen())
+}
+
+// length of a nounce
+func NounceLen() int {
+	return int(C.crypto_box_macbytes())
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/buffer.go

@@ -0,0 +1,86 @@
+package nacl
+
+// #cgo freebsd CFLAGS: -I/usr/local/include
+// #cgo freebsd LDFLAGS: -L/usr/local/lib
+// #cgo LDFLAGS: -lsodium
+// #include <sodium.h>
+//
+// unsigned char * deref_uchar(void * ptr) { return (unsigned char*) ptr; }
+//
+import "C"
+
+import (
+	"encoding/hex"
+	"reflect"
+	"unsafe"
+)
+
+// wrapper arround malloc/free
+type Buffer struct {
+	ptr    unsafe.Pointer
+	length C.int
+	size   C.size_t
+}
+
+// wrapper arround nacl.malloc
+func Malloc(size int) *Buffer {
+	if size > 0 {
+		return malloc(C.size_t(size))
+	}
+	return nil
+}
+
+// does not check for negatives
+func malloc(size C.size_t) *Buffer {
+	ptr := C.malloc(size)
+	C.sodium_memzero(ptr, size)
+	buffer := &Buffer{ptr: ptr, size: size, length: C.int(size)}
+	return buffer
+}
+
+// create a new buffer copying from a byteslice
+func NewBuffer(buff []byte) *Buffer {
+	buffer := Malloc(len(buff))
+	if buffer == nil {
+		return nil
+	}
+	if copy(buffer.Data(), buff) != len(buff) {
+		return nil
+	}
+	return buffer
+}
+
+func (self *Buffer) uchar() *C.uchar {
+	return C.deref_uchar(self.ptr)
+}
+
+func (self *Buffer) Length() int {
+	return int(self.length)
+}
+
+// get immutable byte slice
+func (self *Buffer) Bytes() []byte {
+	buff := make([]byte, self.Length())
+	copy(buff, self.Data())
+	return buff
+}
+
+// get underlying byte slice
+func (self *Buffer) Data() []byte {
+	hdr := reflect.SliceHeader{
+		Data: uintptr(self.ptr),
+		Len:  self.Length(),
+		Cap:  self.Length(),
+	}
+	return *(*[]byte)(unsafe.Pointer(&hdr))
+}
+
+func (self *Buffer) String() string {
+	return hex.EncodeToString(self.Data())
+}
+
+// zero out memory and then free
+func (self *Buffer) Free() {
+	C.sodium_memzero(self.ptr, self.size)
+	C.free(self.ptr)
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/key.go

@@ -0,0 +1,178 @@
+package nacl
+
+// #cgo freebsd CFLAGS: -I/usr/local/include
+// #cgo freebsd LDFLAGS: -L/usr/local/lib
+// #cgo LDFLAGS: -lsodium
+// #include <sodium.h>
+import "C"
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+)
+
+type KeyPair struct {
+	pk *Buffer
+	sk *Buffer
+}
+
+// free this keypair from memory
+func (self *KeyPair) Free() {
+	self.pk.Free()
+	self.sk.Free()
+}
+
+func (self *KeyPair) Secret() []byte {
+	return self.sk.Bytes()
+}
+
+func (self *KeyPair) Public() []byte {
+	return self.pk.Bytes()
+}
+
+func (self *KeyPair) Seed() []byte {
+	seed_len := C.crypto_sign_seedbytes()
+	return self.sk.Bytes()[:seed_len]
+}
+
+// generate a keypair
+func GenSignKeypair() *KeyPair {
+	sk_len := C.crypto_sign_secretkeybytes()
+	sk := malloc(sk_len)
+	pk_len := C.crypto_sign_publickeybytes()
+	pk := malloc(pk_len)
+	res := C.crypto_sign_keypair(pk.uchar(), sk.uchar())
+	if res == 0 {
+		return &KeyPair{pk, sk}
+	}
+	pk.Free()
+	sk.Free()
+	return nil
+}
+
+// get public key from secret key
+func GetSignPubkey(sk []byte) ([]byte, error) {
+	sk_len := C.crypto_sign_secretkeybytes()
+	if C.size_t(len(sk)) != sk_len {
+		return nil, errors.New(fmt.Sprintf("nacl.GetSignPubkey() invalid secret key size %d != %d", len(sk), sk_len))
+	}
+
+	pk_len := C.crypto_sign_publickeybytes()
+	pkbuff := malloc(pk_len)
+	defer pkbuff.Free()
+
+	skbuff := NewBuffer(sk)
+	defer skbuff.Free()
+	//XXX: hack
+	res := C.crypto_sign_seed_keypair(pkbuff.uchar(), skbuff.uchar(), skbuff.uchar())
+
+	if res != 0 {
+		return nil, errors.New(fmt.Sprintf("nacl.GetSignPubkey() failed to get public key from secret key: %d", res))
+	}
+
+	return pkbuff.Bytes(), nil
+}
+
+// make keypair from seed
+func LoadSignKey(seed []byte) *KeyPair {
+	seed_len := C.crypto_sign_seedbytes()
+	if C.size_t(len(seed)) != seed_len {
+		return nil
+	}
+	seedbuff := NewBuffer(seed)
+	defer seedbuff.Free()
+	pk_len := C.crypto_sign_publickeybytes()
+	sk_len := C.crypto_sign_secretkeybytes()
+	pkbuff := malloc(pk_len)
+	skbuff := malloc(sk_len)
+	res := C.crypto_sign_seed_keypair(pkbuff.uchar(), skbuff.uchar(), seedbuff.uchar())
+	if res != 0 {
+		pkbuff.Free()
+		skbuff.Free()
+		return nil
+	}
+	return &KeyPair{pkbuff, skbuff}
+}
+
+func GenBoxKeypair() *KeyPair {
+	sk_len := C.crypto_box_secretkeybytes()
+	sk := malloc(sk_len)
+	pk_len := C.crypto_box_publickeybytes()
+	pk := malloc(pk_len)
+	res := C.crypto_box_keypair(pk.uchar(), sk.uchar())
+	if res == 0 {
+		return &KeyPair{pk, sk}
+	}
+	pk.Free()
+	sk.Free()
+	return nil
+}
+
+// get public key from secret key
+func GetBoxPubkey(sk []byte) []byte {
+	sk_len := C.crypto_box_seedbytes()
+	if C.size_t(len(sk)) != sk_len {
+		return nil
+	}
+
+	pk_len := C.crypto_box_publickeybytes()
+	pkbuff := malloc(pk_len)
+	defer pkbuff.Free()
+
+	skbuff := NewBuffer(sk)
+	defer skbuff.Free()
+
+	// compute the public key
+	C.crypto_scalarmult_base(pkbuff.uchar(), skbuff.uchar())
+
+	return pkbuff.Bytes()
+}
+
+// load keypair from secret key
+func LoadBoxKey(sk []byte) *KeyPair {
+	pk := GetBoxPubkey(sk)
+	if pk == nil {
+		return nil
+	}
+	pkbuff := NewBuffer(pk)
+	skbuff := NewBuffer(sk)
+	return &KeyPair{pkbuff, skbuff}
+}
+
+// make keypair from seed
+func SeedBoxKey(seed []byte) *KeyPair {
+	seed_len := C.crypto_box_seedbytes()
+	if C.size_t(len(seed)) != seed_len {
+		return nil
+	}
+	seedbuff := NewBuffer(seed)
+	defer seedbuff.Free()
+	pk_len := C.crypto_box_publickeybytes()
+	sk_len := C.crypto_box_secretkeybytes()
+	pkbuff := malloc(pk_len)
+	skbuff := malloc(sk_len)
+	res := C.crypto_box_seed_keypair(pkbuff.uchar(), skbuff.uchar(), seedbuff.uchar())
+	if res != 0 {
+		pkbuff.Free()
+		skbuff.Free()
+		return nil
+	}
+	return &KeyPair{pkbuff, skbuff}
+}
+
+func (self *KeyPair) String() string {
+	return fmt.Sprintf("pk=%s sk=%s", hex.EncodeToString(self.pk.Data()), hex.EncodeToString(self.sk.Data()))
+}
+
+func CryptoSignPublicLen() int {
+	return int(C.crypto_sign_publickeybytes())
+}
+
+func CryptoSignSecretLen() int {
+	return int(C.crypto_sign_secretkeybytes())
+}
+
+func CryptoSignSeedLen() int {
+	return int(C.crypto_sign_seedbytes())
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/nacl.go

@@ -0,0 +1,44 @@
+package nacl
+
+// #cgo freebsd CFLAGS: -I/usr/local/include
+// #cgo freebsd LDFLAGS: -L/usr/local/lib
+// #cgo LDFLAGS: -lsodium
+// #include <sodium.h>
+import "C"
+
+import (
+	"log"
+)
+
+// return how many bytes overhead does CryptoBox have
+func CryptoBoxOverhead() int {
+	return int(C.crypto_box_macbytes())
+}
+
+// size of crypto_box public keys
+func CryptoBoxPubKeySize() int {
+	return int(C.crypto_box_publickeybytes())
+}
+
+// size of crypto_box private keys
+func CryptoBoxPrivKeySize() int {
+	return int(C.crypto_box_secretkeybytes())
+}
+
+// size of crypto_sign public keys
+func CryptoSignPubKeySize() int {
+	return int(C.crypto_sign_publickeybytes())
+}
+
+// size of crypto_sign private keys
+func CryptoSignPrivKeySize() int {
+	return int(C.crypto_sign_secretkeybytes())
+}
+
+// initialize sodium
+func init() {
+	status := C.sodium_init()
+	if status == -1 {
+		log.Fatalf("failed to initialize libsodium status=%d", status)
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/rand.go

@@ -0,0 +1,24 @@
+package nacl
+
+// #cgo freebsd CFLAGS: -I/usr/local/include
+// #cgo freebsd LDFLAGS: -L/usr/local/lib
+// #cgo LDFLAGS: -lsodium
+// #include <sodium.h>
+import "C"
+
+func randbytes(size C.size_t) *Buffer {
+
+	buff := malloc(size)
+	C.randombytes_buf(buff.ptr, size)
+	return buff
+
+}
+
+func RandBytes(size int) []byte {
+	if size > 0 {
+		buff := randbytes(C.size_t(size))
+		defer buff.Free()
+		return buff.Bytes()
+	}
+	return nil
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/sign.go

@@ -0,0 +1,58 @@
+package nacl
+
+// #cgo freebsd CFLAGS: -I/usr/local/include
+// #cgo freebsd LDFLAGS: -L/usr/local/lib
+// #cgo LDFLAGS: -lsodium
+// #include <sodium.h>
+import "C"
+
+// sign data detached with secret key sk
+func CryptoSignDetached(msg, sk []byte) []byte {
+	msgbuff := NewBuffer(msg)
+	defer msgbuff.Free()
+	skbuff := NewBuffer(sk)
+	defer skbuff.Free()
+	if skbuff.size != C.crypto_sign_bytes() {
+		return nil
+	}
+
+	// allocate the signature buffer
+	sig := malloc(C.crypto_sign_bytes())
+	defer sig.Free()
+	// compute signature
+	siglen := C.ulonglong(0)
+	res := C.crypto_sign_detached(sig.uchar(), &siglen, msgbuff.uchar(), C.ulonglong(msgbuff.size), skbuff.uchar())
+	if res == 0 && siglen == C.ulonglong(C.crypto_sign_bytes()) {
+		// return copy of signature buffer
+		return sig.Bytes()
+	}
+	// failure to sign
+	return nil
+}
+
+// sign data with secret key sk
+// return detached sig
+// this uses crypto_sign instead pf crypto_sign_detached
+func CryptoSignFucky(msg, sk []byte) []byte {
+	msgbuff := NewBuffer(msg)
+	defer msgbuff.Free()
+	skbuff := NewBuffer(sk)
+	defer skbuff.Free()
+	if skbuff.size != C.crypto_sign_bytes() {
+		return nil
+	}
+
+	// allocate the signed message buffer
+	sig := malloc(C.crypto_sign_bytes() + msgbuff.size)
+	defer sig.Free()
+	// compute signature
+	siglen := C.ulonglong(0)
+	res := C.crypto_sign(sig.uchar(), &siglen, msgbuff.uchar(), C.ulonglong(msgbuff.size), skbuff.uchar())
+	if res == 0 {
+		// return copy of signature inside the signed message
+		offset := int(C.crypto_sign_bytes())
+		return sig.Bytes()[:offset]
+	}
+	// failure to sign
+	return nil
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/stream.go

@@ -0,0 +1,342 @@
+package nacl
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"net"
+	"time"
+)
+
+// TOY encrypted authenticated stream protocol like tls
+
+var BadHandshake = errors.New("Bad handshake")
+var ShortWrite = errors.New("short write")
+var ShortRead = errors.New("short read")
+var Closed = errors.New("socket closed")
+
+// write boxes at 512 bytes at a time
+const DefaultMTU = 512
+
+// wrapper arround crypto_box
+// provides an authenticated encrypted stream
+// this is a TOY
+type CryptoStream struct {
+	// underlying stream to write on
+	stream io.ReadWriteCloser
+	// secret key seed
+	key *KeyPair
+	// public key of who we expect on the other end
+	remote_pk []byte
+	tx_nonce  []byte
+	rx_nonce  []byte
+	// box size
+	mtu int
+}
+
+func (cs *CryptoStream) Close() (err error) {
+	if cs.key != nil {
+		cs.key.Free()
+		cs.key = nil
+	}
+	return cs.stream.Close()
+}
+
+// implements io.Writer
+func (cs *CryptoStream) Write(data []byte) (n int, err error) {
+	// let's split it up
+	for n < len(data) && err == nil {
+		if n+cs.mtu < len(data) {
+			err = cs.writeSegment(data[n : n+cs.mtu])
+			n += cs.mtu
+		} else {
+			err = cs.writeSegment(data[n:])
+			if err == nil {
+				n = len(data)
+			}
+		}
+	}
+	return
+}
+
+func (cs *CryptoStream) public() (p []byte) {
+	p = cs.key.Public()
+	return
+}
+
+func (cs *CryptoStream) secret() (s []byte) {
+	s = cs.key.Secret()
+	return
+}
+
+// read 1 segment
+func (cs *CryptoStream) readSegment() (s []byte, err error) {
+	var stream_read int
+	var seg []byte
+	nl := NounceLen()
+	msg := make([]byte, cs.mtu+nl)
+	stream_read, err = cs.stream.Read(msg)
+	seg, err = CryptoBoxOpen(msg[:stream_read], cs.rx_nonce, cs.secret(), cs.remote_pk)
+	if err == nil {
+		copy(cs.rx_nonce, seg[:nl])
+		s = seg[nl:]
+	}
+	return
+}
+
+// write 1 segment encrypted
+// update nounce
+func (cs *CryptoStream) writeSegment(data []byte) (err error) {
+	var segment []byte
+	nl := NounceLen()
+	msg := make([]byte, len(data)+nl)
+	// generate next nounce
+	nextNounce := NewBoxNounce()
+	copy(msg, nextNounce)
+	copy(msg[nl:], data)
+	// encrypt segment with current nounce
+	segment, err = CryptoBox(data, cs.tx_nonce, cs.remote_pk, cs.secret())
+	var n int
+	n, err = cs.stream.Write(segment)
+	if n != len(segment) {
+		// short write?
+		err = ShortWrite
+		return
+	}
+	// update nounce
+	copy(cs.tx_nonce, nextNounce)
+	return
+}
+
+// implements io.Reader
+func (cs *CryptoStream) Read(data []byte) (n int, err error) {
+	var seg []byte
+	seg, err = cs.readSegment()
+	if err == nil {
+		if len(seg) <= len(data) {
+			copy(data, seg)
+			n = len(seg)
+		} else {
+			// too big?
+			err = ShortRead
+		}
+	}
+	return
+}
+
+// version 0 protocol magic
+var protocol_magic = []byte("BENIS|00")
+
+// verify that a handshake is signed right and is in the correct format etc
+func verifyHandshake(hs, pk []byte) (valid bool) {
+	ml := len(protocol_magic)
+	// valid handshake?
+	if bytes.Equal(hs[0:ml], protocol_magic) {
+		// check pk
+		pl := CryptoSignPublicLen()
+		nl := NounceLen()
+		if bytes.Equal(pk, hs[ml:ml+pl]) {
+			// check signature
+			msg := hs[0 : ml+pl+nl]
+			sig := hs[ml+pl+nl:]
+			valid = CryptoVerifyFucky(msg, sig, pk)
+		}
+	}
+	return
+}
+
+// get claimed public key from handshake
+func getPubkey(hs []byte) (pk []byte) {
+	ml := len(protocol_magic)
+	pl := CryptoSignPublicLen()
+	pk = hs[ml : ml+pl]
+	return
+}
+
+func (cs *CryptoStream) genHandshake() (d []byte) {
+	// protocol magic string version 00
+	// Benis Encrypted Network Information Stream
+	// :-DDDDD meme crypto
+	d = append(d, protocol_magic...)
+	// our public key
+	d = append(d, cs.public()...)
+	// nounce
+	cs.tx_nonce = NewBoxNounce()
+	d = append(d, cs.tx_nonce...)
+	// sign protocol magic string, nounce and pubkey
+	sig := CryptoSignFucky(d, cs.secret())
+	// if sig is nil we'll just die
+	d = append(d, sig...)
+	return
+}
+
+// extract nounce from handshake
+func getNounce(hs []byte) (n []byte) {
+	ml := len(protocol_magic)
+	pl := CryptoSignPublicLen()
+	nl := NounceLen()
+	n = hs[ml+pl : ml+pl+nl]
+	return
+}
+
+// initiate protocol handshake
+func (cs *CryptoStream) Handshake() (err error) {
+	// send them our info
+	hs := cs.genHandshake()
+	var n int
+	n, err = cs.stream.Write(hs)
+	if n != len(hs) {
+		err = ShortWrite
+		return
+	}
+	// read thier info
+	buff := make([]byte, len(hs))
+	_, err = io.ReadFull(cs.stream, buff)
+
+	if cs.remote_pk == nil {
+		// inbound
+		pk := getPubkey(buff)
+		cs.remote_pk = make([]byte, len(pk))
+		copy(cs.remote_pk, pk)
+	}
+
+	if !verifyHandshake(buff, cs.remote_pk) {
+		// verification failed
+		err = BadHandshake
+		return
+	}
+	cs.rx_nonce = make([]byte, NounceLen())
+	copy(cs.rx_nonce, getNounce(buff))
+	return
+}
+
+// create a client
+func Client(stream io.ReadWriteCloser, local_sk, remote_pk []byte) (c *CryptoStream) {
+	c = &CryptoStream{
+		stream: stream,
+		mtu:    DefaultMTU,
+	}
+	c.remote_pk = make([]byte, len(remote_pk))
+	copy(c.remote_pk, remote_pk)
+	c.key = LoadSignKey(local_sk)
+	if c.key == nil {
+		return nil
+	}
+	return c
+}
+
+type CryptoConn struct {
+	stream *CryptoStream
+	conn   net.Conn
+}
+
+func (cc *CryptoConn) Close() (err error) {
+	err = cc.stream.Close()
+	return
+}
+
+func (cc *CryptoConn) Write(d []byte) (n int, err error) {
+	return cc.stream.Write(d)
+}
+
+func (cc *CryptoConn) Read(d []byte) (n int, err error) {
+	return cc.stream.Read(d)
+}
+
+func (cc *CryptoConn) LocalAddr() net.Addr {
+	return cc.conn.LocalAddr()
+}
+
+func (cc *CryptoConn) RemoteAddr() net.Addr {
+	return cc.conn.RemoteAddr()
+}
+
+func (cc *CryptoConn) SetDeadline(t time.Time) (err error) {
+	return cc.conn.SetDeadline(t)
+}
+
+func (cc *CryptoConn) SetReadDeadline(t time.Time) (err error) {
+	return cc.conn.SetReadDeadline(t)
+}
+
+func (cc *CryptoConn) SetWriteDeadline(t time.Time) (err error) {
+	return cc.conn.SetWriteDeadline(t)
+}
+
+type CryptoListener struct {
+	l         net.Listener
+	handshake chan net.Conn
+	accepted  chan *CryptoConn
+	trust     func(pk []byte) bool
+	key       *KeyPair
+}
+
+func (cl *CryptoListener) Close() (err error) {
+	err = cl.l.Close()
+	close(cl.accepted)
+	close(cl.handshake)
+	cl.key.Free()
+	cl.key = nil
+	return
+}
+
+func (cl *CryptoListener) acceptInbound() {
+	for {
+		c, err := cl.l.Accept()
+		if err == nil {
+			cl.handshake <- c
+		} else {
+			return
+		}
+	}
+}
+
+func (cl *CryptoListener) runChans() {
+	for {
+		select {
+		case c := <-cl.handshake:
+			go func() {
+				s := &CryptoStream{
+					stream: c,
+					mtu:    DefaultMTU,
+					key:    cl.key,
+				}
+				err := s.Handshake()
+				if err == nil {
+					// we gud handshake was okay
+					if cl.trust(s.remote_pk) {
+						// the key is trusted okay
+						cl.accepted <- &CryptoConn{stream: s, conn: c}
+					} else {
+						// not trusted, close connection
+						s.Close()
+					}
+				}
+			}()
+		}
+	}
+}
+
+// accept inbound authenticated and trusted connections
+func (cl *CryptoListener) Accept() (c net.Conn, err error) {
+	var ok bool
+	c, ok = <-cl.accepted
+	if !ok {
+		err = Closed
+	}
+	return
+}
+
+// create a listener
+func Server(l net.Listener, local_sk []byte, trust func(pk []byte) bool) (s *CryptoListener) {
+	s = &CryptoListener{
+		l:         l,
+		trust:     trust,
+		handshake: make(chan net.Conn),
+		accepted:  make(chan *CryptoConn),
+	}
+	s.key = LoadSignKey(local_sk)
+	go s.runChans()
+	go s.acceptInbound()
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl/verfiy.go

@@ -0,0 +1,53 @@
+package nacl
+
+// #cgo freebsd CFLAGS: -I/usr/local/include
+// #cgo freebsd LDFLAGS: -L/usr/local/lib
+// #cgo LDFLAGS: -lsodium
+// #include <sodium.h>
+import "C"
+
+// verify a fucky detached sig
+func CryptoVerifyFucky(msg, sig, pk []byte) bool {
+	var smsg []byte
+	smsg = append(smsg, sig...)
+	smsg = append(smsg, msg...)
+	return CryptoVerify(smsg, pk)
+}
+
+// verify a signed message
+func CryptoVerify(smsg, pk []byte) bool {
+	smsg_buff := NewBuffer(smsg)
+	defer smsg_buff.Free()
+	pk_buff := NewBuffer(pk)
+	defer pk_buff.Free()
+
+	if pk_buff.size != C.crypto_sign_publickeybytes() {
+		return false
+	}
+	mlen := C.ulonglong(0)
+	msg := malloc(C.size_t(len(smsg)))
+	defer msg.Free()
+	smlen := C.ulonglong(smsg_buff.size)
+	return C.crypto_sign_open(msg.uchar(), &mlen, smsg_buff.uchar(), smlen, pk_buff.uchar()) != -1
+}
+
+// verfiy a detached signature
+// return true on valid otherwise false
+func CryptoVerifyDetached(msg, sig, pk []byte) bool {
+	msg_buff := NewBuffer(msg)
+	defer msg_buff.Free()
+	sig_buff := NewBuffer(sig)
+	defer sig_buff.Free()
+	pk_buff := NewBuffer(pk)
+	defer pk_buff.Free()
+
+	if pk_buff.size != C.crypto_sign_publickeybytes() {
+		return false
+	}
+
+	// invalid sig size
+	if sig_buff.size != C.crypto_sign_bytes() {
+		return false
+	}
+	return C.crypto_sign_verify_detached(sig_buff.uchar(), msg_buff.uchar(), C.ulonglong(len(msg)), pk_buff.uchar()) == 0
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/nacl_test.go

@@ -0,0 +1,34 @@
+package crypto
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io"
+	"testing"
+)
+
+func TestNaclToPublic(t *testing.T) {
+	pk, sk := GenKeypair()
+	t_pk := ToPublic(sk)
+	if !bytes.Equal(pk, t_pk) {
+		t.Logf("%q != %q", pk, t_pk)
+		t.Fail()
+	}
+}
+
+func TestNaclSignVerify(t *testing.T) {
+	var msg [1024]byte
+	pk, sk := GenKeypair()
+	io.ReadFull(rand.Reader, msg[:])
+
+	signer := CreateSigner(sk)
+	signer.Write(msg[:])
+	sig := signer.Sign()
+
+	verifier := CreateVerifier(pk)
+	verifier.Write(msg[:])
+	if !verifier.Verify(sig) {
+		t.Logf("%q is invalid signature and is %dB long", sig, len(sig))
+		t.Fail()
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/rand.go

@@ -0,0 +1,8 @@
+package crypto
+
+import (
+	"nntpchan/lib/crypto/nacl"
+)
+
+// generate random bytes
+var RandBytes = nacl.RandBytes

contrib/backends/nntpchand/src/nntpchan/lib/crypto/sig.go

@@ -0,0 +1,25 @@
+package crypto
+
+import "io"
+
+// a detached signature
+type Signature []byte
+
+type SigEncoder interface {
+	// encode a signature to an io.Writer
+	// return error if one occurrened while writing out signature
+	Encode(sig Signature, w io.Writer) error
+	// encode a signature to a string
+	EncodeString(sig Signature) string
+}
+
+// a decoder of signatures
+type SigDecoder interface {
+	// decode signature from io.Reader
+	// reads all data until io.EOF
+	// returns singaure or error if an error occured while reading
+	Decode(r io.Reader) (Signature, error)
+	// decode a signature from string
+	// returns signature or error if an error ocurred while decoding
+	DecodeString(str string) (Signature, error)
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/sign.go

@@ -0,0 +1,14 @@
+package crypto
+
+import "io"
+
+//
+// provides generic signing interface for producing detached signatures
+// call Write() to feed data to be signed, call Sign() to generate
+// a detached signature
+//
+type Signer interface {
+	io.Writer
+	// generate detached Signature from previously fed body via Write()
+	Sign() Signature
+}

contrib/backends/nntpchand/src/nntpchan/lib/crypto/verify.go

@@ -0,0 +1,14 @@
+package crypto
+
+import "io"
+
+// provides generic signature
+// call Write() to feed in message body
+// once the entire body has been fed in via Write() call Verify() with detached
+// signature to verify the detached signature against the previously fed body
+type Verifer interface {
+	io.Writer
+	// verify detached signature from body previously fed via Write()
+	// return true if the detached signature is valid given the body
+	Verify(sig Signature) bool
+}

contrib/backends/nntpchand/src/nntpchan/lib/database/database.go

@@ -0,0 +1,26 @@
+package database
+
+import (
+	"errors"
+	"nntpchan/lib/config"
+	"nntpchan/lib/model"
+	"strings"
+)
+
+//
+type Database interface {
+	ThreadByMessageID(msgid string) (*model.Thread, error)
+	ThreadByHash(hash string) (*model.Thread, error)
+	BoardPage(newsgroup string, pageno, perpage int) (*model.BoardPage, error)
+}
+
+// get new database connector from configuration
+func NewDBFromConfig(c *config.DatabaseConfig) (db Database, err error) {
+	dbtype := strings.ToLower(c.Type)
+	if dbtype == "postgres" {
+		db, err = createPostgresDatabase(c.Addr, c.Username, c.Password)
+	} else {
+		err = errors.New("no such database driver: " + c.Type)
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/database/doc.go

@@ -0,0 +1,4 @@
+//
+// database driver
+//
+package database

contrib/backends/nntpchand/src/nntpchan/lib/database/postgres.go

@@ -0,0 +1,28 @@
+package database
+
+import (
+	"nntpchan/lib/model"
+)
+
+type PostgresDB struct {
+}
+
+func (db *PostgresDB) ThreadByMessageID(msgid string) (thread *model.Thread, err error) {
+
+	return
+}
+
+func (db *PostgresDB) ThreadByHash(hash string) (thread *model.Thread, err error) {
+
+	return
+}
+
+func (db *PostgresDB) BoardPage(newsgroup string, pageno, perpage int) (page *model.BoardPage, err error) {
+
+	return
+}
+
+func createPostgresDatabase(addr, user, passwd string) (p *PostgresDB, err error) {
+
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/frontend/captcha.go

@@ -0,0 +1,123 @@
+package frontend
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/dchest/captcha"
+	"github.com/gorilla/mux"
+	"github.com/gorilla/sessions"
+	"net/http"
+	"nntpchan/lib/config"
+)
+
+// server of captchas
+// implements frontend.Middleware
+type CaptchaServer struct {
+	h           int
+	w           int
+	store       *sessions.CookieStore
+	prefix      string
+	sessionName string
+}
+
+// create new captcha server using existing session store
+func NewCaptchaServer(w, h int, prefix string, store *sessions.CookieStore) *CaptchaServer {
+	return &CaptchaServer{
+		h:           h,
+		w:           w,
+		prefix:      prefix,
+		store:       store,
+		sessionName: "captcha",
+	}
+}
+
+func (cs *CaptchaServer) Reload(c *config.MiddlewareConfig) {
+
+}
+
+func (cs *CaptchaServer) SetupRoutes(m *mux.Router) {
+	m.Path("/new").HandlerFunc(cs.NewCaptcha)
+	m.Path("/img/{f}").Handler(captcha.Server(cs.w, cs.h))
+	m.Path("/verify.json").HandlerFunc(cs.VerifyCaptcha)
+}
+
+// return true if this session has solved the last captcha given provided solution, otherwise false
+func (cs *CaptchaServer) CheckSession(w http.ResponseWriter, r *http.Request, solution string) (bool, error) {
+	s, err := cs.store.Get(r, cs.sessionName)
+	if err == nil {
+		id, ok := s.Values["captcha_id"]
+		if ok {
+			return captcha.VerifyString(id.(string), solution), nil
+		}
+	}
+	return false, err
+}
+
+// verify a captcha
+func (cs *CaptchaServer) VerifyCaptcha(w http.ResponseWriter, r *http.Request) {
+	dec := json.NewDecoder(r.Body)
+	defer r.Body.Close()
+	// request
+	req := make(map[string]string)
+	// response
+	resp := make(map[string]interface{})
+	resp["solved"] = false
+	// decode request
+	err := dec.Decode(req)
+	if err == nil {
+		// decode okay
+		id, ok := req["id"]
+		if ok {
+			// we have id
+			solution, ok := req["solution"]
+			if ok {
+				// we have solution and id
+				resp["solved"] = captcha.VerifyString(id, solution)
+			} else {
+				// we don't have solution
+				err = errors.New("no captcha solution provided")
+			}
+		} else {
+			// we don't have id
+			err = errors.New("no captcha id provided")
+		}
+	}
+	if err != nil {
+		// error happened
+		resp["error"] = err.Error()
+	}
+	// send reply
+	w.Header().Set("Content-Type", "text/json; encoding=UTF-8")
+	enc := json.NewEncoder(w)
+	enc.Encode(resp)
+}
+
+// generate a new captcha
+func (cs *CaptchaServer) NewCaptcha(w http.ResponseWriter, r *http.Request) {
+	// obtain session
+	sess, err := cs.store.Get(r, cs.sessionName)
+	if err != nil {
+		// failed to obtain session
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	// new captcha
+	id := captcha.New()
+	// do we want to interpret as json?
+	use_json := r.URL.Query().Get("t") == "json"
+	// image url
+	url := fmt.Sprintf("%simg/%s.png", cs.prefix, id)
+	if use_json {
+		// send json
+		enc := json.NewEncoder(w)
+		enc.Encode(map[string]string{"id": id, "url": url})
+	} else {
+		// set captcha id
+		sess.Values["captcha_id"] = id
+		// save session
+		sess.Save(r, w)
+		// rediect to image
+		http.Redirect(w, r, url, http.StatusFound)
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/frontend/doc.go

@@ -0,0 +1,5 @@
+//
+// nntpchan frontend
+// allows posting to nntpchan network via various implementations
+//
+package frontend

contrib/backends/nntpchand/src/nntpchan/lib/frontend/frontend.go

@@ -0,0 +1,46 @@
+package frontend
+
+import (
+	"nntpchan/lib/config"
+	"nntpchan/lib/database"
+	"nntpchan/lib/model"
+	"nntpchan/lib/nntp"
+)
+
+// a frontend that displays nntp posts and allows posting
+type Frontend interface {
+
+	// run mainloop
+	Serve()
+
+	// do we accept this inbound post?
+	AllowPost(p model.PostReference) bool
+
+	// trigger a manual regen of indexes for a root post
+	Regen(p model.PostReference)
+
+	// implements nntp.EventHooks
+	GotArticle(msgid nntp.MessageID, group nntp.Newsgroup)
+
+	// implements nntp.EventHooks
+	SentArticleVia(msgid nntp.MessageID, feedname string)
+
+	// reload config
+	Reload(c *config.FrontendConfig)
+}
+
+// create a new http frontend give frontend config
+func NewHTTPFrontend(c *config.FrontendConfig, db database.Database) (f Frontend, err error) {
+
+	var mid Middleware
+	if c.Middleware != nil {
+		// middleware configured
+		mid, err = OverchanMiddleware(c.Middleware, db)
+	}
+
+	if err == nil {
+		// create http frontend only if no previous errors
+		f, err = createHttpFrontend(c, mid, db)
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/frontend/http.go

@@ -0,0 +1,136 @@
+package frontend
+
+import (
+	"fmt"
+	log "github.com/Sirupsen/logrus"
+	"github.com/gorilla/mux"
+	"net/http"
+	"nntpchan/lib/admin"
+	"nntpchan/lib/api"
+	"nntpchan/lib/config"
+	"nntpchan/lib/database"
+	"nntpchan/lib/model"
+	"nntpchan/lib/nntp"
+	"time"
+)
+
+// http frontend server
+// provides glue layer between nntp and middleware
+type httpFrontend struct {
+	// bind address
+	addr string
+	// http mux
+	httpmux *mux.Router
+	// admin panel
+	adminPanel *admin.Server
+	// static files path
+	staticDir string
+	// http middleware
+	middleware Middleware
+	// api server
+	apiserve *api.Server
+	// database driver
+	db database.Database
+}
+
+// reload http frontend
+// reloads middleware
+func (f *httpFrontend) Reload(c *config.FrontendConfig) {
+	if f.middleware == nil {
+		if c.Middleware != nil {
+			var err error
+			// no middleware set, create middleware
+			f.middleware, err = OverchanMiddleware(c.Middleware, f.db)
+			if err != nil {
+				log.Errorf("overchan middleware reload failed: %s", err.Error())
+			}
+
+		}
+	} else {
+		// middleware exists
+		// do middleware reload
+		f.middleware.Reload(c.Middleware)
+	}
+
+}
+
+// serve http requests from net.Listener
+func (f *httpFrontend) Serve() {
+	// serve http
+	for {
+		err := http.ListenAndServe(f.addr, f.httpmux)
+		if err != nil {
+			log.Errorf("failed to listen and serve with frontend: %s", err)
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+// serve robots.txt page
+func (f *httpFrontend) serveRobots(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, "User-Agent: *\nDisallow: /\n")
+}
+
+func (f *httpFrontend) AllowPost(p model.PostReference) bool {
+	// TODO: implement
+	return true
+}
+
+func (f *httpFrontend) Regen(p model.PostReference) {
+	// TODO: implement
+}
+
+func (f *httpFrontend) GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) {
+	// TODO: implement
+}
+
+func (f *httpFrontend) SentArticleVia(msgid nntp.MessageID, feedname string) {
+	// TODO: implement
+}
+
+func createHttpFrontend(c *config.FrontendConfig, mid Middleware, db database.Database) (f *httpFrontend, err error) {
+	f = new(httpFrontend)
+	// set db
+	// db.Ensure() called elsewhere
+	f.db = db
+
+	// set bind address
+	f.addr = c.BindAddr
+
+	// set up mux
+	f.httpmux = mux.NewRouter()
+
+	// set up admin panel
+	f.adminPanel = admin.NewServer()
+
+	// set static files dir
+	f.staticDir = c.Static
+
+	// set middleware
+	f.middleware = mid
+
+	// set up routes
+
+	if f.adminPanel != nil {
+		// route up admin panel
+		f.httpmux.PathPrefix("/admin/").Handler(f.adminPanel)
+	}
+
+	if f.middleware != nil {
+		// route up middleware
+		f.middleware.SetupRoutes(f.httpmux)
+	}
+
+	if f.apiserve != nil {
+		// route up api
+		f.apiserve.SetupRoutes(f.httpmux.PathPrefix("/api/").Subrouter())
+	}
+
+	// route up robots.txt
+	f.httpmux.Path("/robots.txt").HandlerFunc(f.serveRobots)
+
+	// route up static files
+	f.httpmux.PathPrefix("/static/").Handler(http.FileServer(http.Dir(f.staticDir)))
+
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/frontend/middleware.go

@@ -0,0 +1,14 @@
+package frontend
+
+import (
+	"github.com/gorilla/mux"
+	"nntpchan/lib/config"
+)
+
+// http middleware
+type Middleware interface {
+	// set up routes
+	SetupRoutes(m *mux.Router)
+	// reload with new configuration
+	Reload(c *config.MiddlewareConfig)
+}

contrib/backends/nntpchand/src/nntpchan/lib/frontend/overchan.go

@@ -0,0 +1,115 @@
+package frontend
+
+import (
+	log "github.com/Sirupsen/logrus"
+	"github.com/gorilla/mux"
+	"github.com/gorilla/sessions"
+	"html/template"
+	"net/http"
+	"nntpchan/lib/config"
+	"nntpchan/lib/database"
+	"path/filepath"
+	"strconv"
+)
+
+// standard overchan imageboard middleware
+type overchanMiddleware struct {
+	templ   *template.Template
+	captcha *CaptchaServer
+	store   *sessions.CookieStore
+	db      database.Database
+}
+
+func (m *overchanMiddleware) SetupRoutes(mux *mux.Router) {
+	// setup front page handler
+	mux.Path("/").HandlerFunc(m.ServeIndex)
+	// setup thread handler
+	mux.Path("/t/{id}/").HandlerFunc(m.ServeThread)
+	// setup board page handler
+	mux.Path("/b/{name}/").HandlerFunc(m.ServeBoardPage)
+	// setup posting endpoint
+	mux.Path("/post")
+	// create captcha
+	captchaPrefix := "/captcha/"
+	m.captcha = NewCaptchaServer(200, 400, captchaPrefix, m.store)
+	// setup captcha endpoint
+	m.captcha.SetupRoutes(mux.PathPrefix(captchaPrefix).Subrouter())
+}
+
+// reload middleware
+func (m *overchanMiddleware) Reload(c *config.MiddlewareConfig) {
+	// reload templates
+	templ, err := template.ParseGlob(filepath.Join(c.Templates, "*.tmpl"))
+	if err == nil {
+		log.Infof("middleware reloaded templates")
+		m.templ = templ
+	} else {
+		log.Errorf("middleware reload failed: %s", err.Error())
+	}
+}
+
+func (m *overchanMiddleware) ServeBoardPage(w http.ResponseWriter, r *http.Request) {
+	param := mux.Vars(r)
+	board := param["name"]
+	page := r.URL.Query().Get("q")
+	pageno, err := strconv.Atoi(page)
+	if err == nil {
+		var obj interface{}
+		obj, err = m.db.BoardPage(board, pageno, 10)
+		if err == nil {
+			m.serveTemplate(w, r, "board.html.tmpl", obj)
+		} else {
+			m.serveTemplate(w, r, "error.html.tmpl", err)
+		}
+	} else {
+		// 404
+		http.NotFound(w, r)
+	}
+}
+
+// serve cached thread
+func (m *overchanMiddleware) ServeThread(w http.ResponseWriter, r *http.Request) {
+	param := mux.Vars(r)
+	obj, err := m.db.ThreadByHash(param["id"])
+	if err == nil {
+		m.serveTemplate(w, r, "thread.html.tmpl", obj)
+	} else {
+		m.serveTemplate(w, r, "error.html.tmpl", err)
+	}
+}
+
+// serve index page
+func (m *overchanMiddleware) ServeIndex(w http.ResponseWriter, r *http.Request) {
+	m.serveTemplate(w, r, "index.html.tmpl", nil)
+}
+
+// serve a template
+func (m *overchanMiddleware) serveTemplate(w http.ResponseWriter, r *http.Request, tname string, obj interface{}) {
+	t := m.templ.Lookup(tname)
+	if t == nil {
+		log.WithFields(log.Fields{
+			"template": tname,
+		}).Warning("template not found")
+		http.NotFound(w, r)
+	} else {
+		err := t.Execute(w, obj)
+		if err != nil {
+			// error getting model
+			log.WithFields(log.Fields{
+				"error":    err,
+				"template": tname,
+			}).Warning("failed to render template")
+		}
+	}
+}
+
+// create standard overchan middleware
+func OverchanMiddleware(c *config.MiddlewareConfig, db database.Database) (m Middleware, err error) {
+	om := new(overchanMiddleware)
+	om.templ, err = template.ParseGlob(filepath.Join(c.Templates, "*.tmpl"))
+	om.db = db
+	if err == nil {
+		m = om
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/frontend/post.go

@@ -0,0 +1 @@
+package frontend

contrib/backends/nntpchand/src/nntpchan/lib/frontend/webhooks.go

@@ -0,0 +1 @@
+package frontend

contrib/backends/nntpchand/src/nntpchan/lib/model/article.go

@@ -0,0 +1,15 @@
+package model
+
+type Article struct {
+	Subject     string
+	Name        string
+	Header      map[string][]string
+	Text        string
+	Attachments []Attachment
+	MessageID   string
+	Newsgroup   string
+	Reference   string
+	Path        string
+	Posted      int64
+	Addr        string
+}

contrib/backends/nntpchand/src/nntpchan/lib/model/attachment.go

@@ -0,0 +1,10 @@
+package model
+
+type Attachment struct {
+	Path string
+	Name string
+	Mime string
+	Hash string
+	// only filled for api
+	Body string
+}

contrib/backends/nntpchand/src/nntpchan/lib/model/board.go

@@ -0,0 +1,4 @@
+package model
+
+type Board struct {
+}

contrib/backends/nntpchand/src/nntpchan/lib/model/boardpage.go

@@ -0,0 +1,8 @@
+package model
+
+type BoardPage struct {
+	Name    string
+	Page    int
+	Pages   int
+	Threads []Thread
+}

contrib/backends/nntpchand/src/nntpchan/lib/model/doc.go

@@ -0,0 +1,2 @@
+// MVC models
+package model

contrib/backends/nntpchand/src/nntpchan/lib/model/misc.go

@@ -0,0 +1,29 @@
+package model
+
+import (
+	"time"
+)
+
+type ArticleHeader map[string][]string
+
+// a ( MessageID , newsgroup ) tuple
+type ArticleEntry [2]string
+
+func (self ArticleEntry) Newsgroup() string {
+	return self[1]
+}
+
+func (self ArticleEntry) MessageID() string {
+	return self[0]
+}
+
+// a ( time point, post count ) tuple
+type PostEntry [2]int64
+
+func (self PostEntry) Time() time.Time {
+	return time.Unix(self[0], 0)
+}
+
+func (self PostEntry) Count() int64 {
+	return self[1]
+}

contrib/backends/nntpchand/src/nntpchan/lib/model/post.go

@@ -0,0 +1,32 @@
+package model
+
+import (
+	"time"
+)
+
+type Tripcode string
+
+type Post struct {
+	MessageID   string
+	Newsgroup   string
+	Attachments []Attachment
+	Subject     string
+	Posted      time.Time
+	PostedAt    uint64
+	Name        string
+	Tripcode    Tripcode
+}
+
+// ( message-id, references, newsgroup )
+type PostReference [3]string
+
+func (r PostReference) MessageID() string {
+	return r[0]
+}
+
+func (r PostReference) References() string {
+	return r[1]
+}
+func (r PostReference) Newsgroup() string {
+	return r[2]
+}

contrib/backends/nntpchand/src/nntpchan/lib/model/thread.go

@@ -0,0 +1,6 @@
+package model
+
+type Thread struct {
+	Root    *Post
+	Replies []*Post
+}

contrib/backends/nntpchand/src/nntpchan/lib/network/dial.go

@@ -0,0 +1,37 @@
+package network
+
+import (
+	"errors"
+	"net"
+	"nntpchan/lib/config"
+	"strings"
+)
+
+// operation timed out
+var ErrTimeout = errors.New("timeout")
+
+// the operation was reset abruptly
+var ErrReset = errors.New("reset")
+
+// the operation was actively refused
+var ErrRefused = errors.New("refused")
+
+// generic dialer
+// dials out to a remote address
+// returns a net.Conn and nil on success
+// returns nil and error if an error happens while dialing
+type Dialer interface {
+	Dial(remote string) (net.Conn, error)
+}
+
+// create a new dialer from configuration
+func NewDialer(conf *config.ProxyConfig) (d Dialer) {
+	d = StdDialer
+	if conf != nil {
+		proxyType := strings.ToLower(conf.Type)
+		if proxyType == "socks" || proxyType == "socks4a" {
+			d = SocksDialer(conf.Addr)
+		}
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/network/doc.go

@@ -0,0 +1,4 @@
+//
+// network utilities
+//
+package network

contrib/backends/nntpchand/src/nntpchan/lib/network/i2p.go

@@ -0,0 +1 @@
+package network

contrib/backends/nntpchand/src/nntpchan/lib/network/socks.go

@@ -0,0 +1,140 @@
+package network
+
+import (
+	"errors"
+	log "github.com/Sirupsen/logrus"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+)
+
+type socksDialer struct {
+	socksAddr string
+	dialer    Dialer
+}
+
+// try dialing out via socks proxy
+func (sd *socksDialer) Dial(remote string) (c net.Conn, err error) {
+	log.WithFields(log.Fields{
+		"addr":  remote,
+		"socks": sd.socksAddr,
+	}).Debug("dailing out to socks proxy")
+	c, err = sd.dialer.Dial(sd.socksAddr)
+	if err == nil {
+		// dailed out to socks proxy good
+		remote_addr := remote
+		// generate request
+		idx := strings.LastIndex(remote_addr, ":")
+		if idx == -1 {
+			err = errors.New("invalid address: " + remote_addr)
+			return
+		}
+		var port uint64
+		addr := remote_addr[:idx]
+		port, err = strconv.ParseUint(remote_addr[idx+1:], 10, 16)
+		if port >= 25536 {
+			err = errors.New("bad proxy port")
+			c.Close()
+			c = nil
+			return
+		} else if err != nil {
+			c.Close()
+			return
+		}
+		var proxy_port uint16
+		proxy_port = uint16(port)
+		proxy_ident := "srndproxy"
+		req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8
+
+		req := make([]byte, req_len)
+		// pack request
+		req[0] = '\x04'
+		req[1] = '\x01'
+		req[2] = byte(proxy_port & 0xff00 >> 8)
+		req[3] = byte(proxy_port & 0x00ff)
+		req[7] = '\x01'
+		idx = 8
+
+		proxy_ident_b := []byte(proxy_ident)
+		addr_b := []byte(addr)
+
+		var bi int
+		for bi = range proxy_ident_b {
+			req[idx] = proxy_ident_b[bi]
+			idx += 1
+		}
+		idx += 1
+		for bi = range addr_b {
+			req[idx] = addr_b[bi]
+			idx += 1
+		}
+		log.WithFields(log.Fields{
+			"addr":  remote,
+			"socks": sd.socksAddr,
+			"req":   req,
+		}).Debug("write socks request")
+		n := 0
+		n, err = c.Write(req)
+		if err == nil && n == len(req) {
+			// wrote request okay
+			resp := make([]byte, 8)
+			_, err = io.ReadFull(c, resp)
+			if err == nil {
+				// got reply okay
+				if resp[1] == '\x5a' {
+					// successful socks connection
+					log.WithFields(log.Fields{
+						"addr":  remote,
+						"socks": sd.socksAddr,
+					}).Debug("socks proxy connection successful")
+				} else {
+					// unsucessful socks connect
+					log.WithFields(log.Fields{
+						"addr":  remote,
+						"socks": sd.socksAddr,
+						"code":  resp[1],
+					}).Warn("connect via socks proxy failed")
+					c.Close()
+					c = nil
+				}
+			} else {
+				// error reading reply
+				log.WithFields(log.Fields{
+					"addr":  remote,
+					"socks": sd.socksAddr,
+				}).Error("failed to read socks response ", err)
+				c.Close()
+				c = nil
+			}
+		} else {
+			if err == nil {
+				err = errors.New("short write")
+			}
+
+			// error writing request
+			log.WithFields(log.Fields{
+				"addr":  remote,
+				"socks": sd.socksAddr,
+			}).Error("failed to write socks request ", err)
+			c.Close()
+			c = nil
+
+		}
+	} else {
+		// dail fail
+		log.WithFields(log.Fields{
+			"addr":  remote,
+			"socks": sd.socksAddr,
+		}).Error("Cannot connect to socks proxy ", err)
+	}
+	return
+}
+
+// create a socks dialer that dials out via socks proxy at address
+func SocksDialer(addr string) Dialer {
+	return &socksDialer{
+		socksAddr: addr,
+		dialer:    StdDialer,
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/network/std.go

@@ -0,0 +1,14 @@
+package network
+
+import (
+	"net"
+)
+
+type stdDialer struct {
+}
+
+func (sd *stdDialer) Dial(addr string) (c net.Conn, err error) {
+	return net.Dial("tcp", addr)
+}
+
+var StdDialer = &stdDialer{}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/acceptor.go

@@ -0,0 +1,69 @@
+package nntp
+
+import (
+	"nntpchan/lib/nntp/message"
+)
+
+const (
+	// accepted article
+	ARTICLE_ACCEPT = iota
+	// reject article, don't send again
+	ARTICLE_REJECT
+	// defer article, send later
+	ARTICLE_DEFER
+	// reject + ban
+	ARTICLE_BAN
+)
+
+type PolicyStatus int
+
+const PolicyAccept = PolicyStatus(ARTICLE_ACCEPT)
+const PolicyReject = PolicyStatus(ARTICLE_REJECT)
+const PolicyDefer = PolicyStatus(ARTICLE_DEFER)
+const PolicyBan = PolicyStatus(ARTICLE_BAN)
+
+func (s PolicyStatus) String() string {
+	switch int(s) {
+	case ARTICLE_ACCEPT:
+		return "ACCEPTED"
+	case ARTICLE_REJECT:
+		return "REJECTED"
+	case ARTICLE_DEFER:
+		return "DEFERRED"
+	case ARTICLE_BAN:
+		return "BANNED"
+	default:
+		return "[invalid policy status]"
+	}
+}
+
+// is this an accept code?
+func (s PolicyStatus) Accept() bool {
+	return s == ARTICLE_ACCEPT
+}
+
+// is this a defer code?
+func (s PolicyStatus) Defer() bool {
+	return s == ARTICLE_DEFER
+}
+
+// is this a ban code
+func (s PolicyStatus) Ban() bool {
+	return s == ARTICLE_BAN
+}
+
+// is this a reject code?
+func (s PolicyStatus) Reject() bool {
+	return s == ARTICLE_BAN || s == ARTICLE_REJECT
+}
+
+// type defining a policy that determines if we want to accept/reject/defer an
+// incoming article
+type ArticleAcceptor interface {
+	// check article given an article header
+	CheckHeader(hdr message.Header) PolicyStatus
+	// check article given a message id
+	CheckMessageID(msgid MessageID) PolicyStatus
+	// get max article size in bytes
+	MaxArticleSize() int64
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/auth.go

@@ -0,0 +1,38 @@
+package nntp
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+// defines server side authentication mechanism
+type ServerAuth interface {
+	// check plaintext login
+	// returns nil on success otherwise error if one occurs during authentication
+	// returns true if authentication was successful and an error if a network io error happens
+	CheckLogin(username, passwd string) (bool, error)
+}
+
+type FlatfileAuth string
+
+func (fname FlatfileAuth) CheckLogin(username, passwd string) (found bool, err error) {
+	cred := fmt.Sprintf("%s:%s", username, passwd)
+	var f *os.File
+	f, err = os.Open(string(fname))
+	if err == nil {
+		defer f.Close()
+		r := bufio.NewReader(f)
+		for err == nil {
+			var line string
+			line, err = r.ReadString(10)
+			line = strings.Trim(line, "\r\n")
+			if line == cred {
+				found = true
+				break
+			}
+		}
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/codes.go

@@ -0,0 +1,206 @@
+package nntp
+
+// 1xx codes
+
+// help info follows
+const RPL_Help = "100"
+
+// capabilities info follows
+const RPL_Capabilities = "101"
+
+// server date time follows
+const RPL_Date = "111"
+
+// 2xx codes
+
+// posting is allowed
+const RPL_PostingAllowed = "200"
+
+// posting is not allowed
+const RPL_PostingNotAllowed = "201"
+
+// streaming mode enabled
+const RPL_PostingStreaming = "203"
+
+// reply to QUIT command, we will close the connection
+const RPL_Quit = "205"
+
+// reply for GROUP and LISTGROUP commands
+const RPL_Group = "211"
+
+// info list follows
+const RPL_List = "215"
+
+// index follows
+const RPL_Index = "218"
+
+// article follows
+const RPL_Article = "220"
+
+// article headers follows
+const RPL_ArticleHeaders = "221"
+
+// article body follows
+const RPL_ArticleBody = "222"
+
+// selected article exists
+const RPL_ArticleSelectedExists = "223"
+
+// overview info follows
+const RPL_Overview = "224"
+
+// list of article heards follows
+const RPL_HeadersList = "225"
+
+// list of new articles follows
+const RPL_NewArticles = "230"
+
+// list of newsgroups followes
+const RPL_NewsgroupList = "231"
+
+// article was transfered okay by IHAVE command
+const RPL_TransferOkay = "235"
+
+// article is not found by CHECK and we want it
+const RPL_StreamingAccept = "238"
+
+// article was transfered via TAKETHIS successfully
+const RPL_StreamingTransfered = "239"
+
+// article was transfered by POST command successfully
+const RPL_PostReceived = "240"
+
+// AUTHINFO SIMPLE accepted
+const RPL_AuthInfoAccepted = "250"
+
+// authentication creds have been accepted
+const RPL_AuthAccepted = "281"
+
+// binary content follows
+const RPL_Binary = "288"
+
+// line sent for posting allowed
+const Line_PostingAllowed = RPL_PostingAllowed + " Posting Allowed"
+
+// line sent for posting not allowed
+const Line_PostingNotAllowed = RPL_PostingNotAllowed + " Posting Not Allowed"
+
+// 3xx codes
+
+// article is accepted via IHAVE
+const RPL_TransferAccepted = "335"
+
+// article was accepted via POST
+const RPL_PostAccepted = "340"
+
+// continue with authorization
+const RPL_ContinueAuthorization = "350"
+
+// more authentication info required
+const RPL_MoreAuth = "381"
+
+// continue with tls handshake
+const RPL_TLSContinue = "382"
+
+// 4xx codes
+
+// server says servive is not avaiable on initial connection
+const RPL_NotAvaiable = "400"
+
+// server is in the wrong mode
+const RPL_WrongMode = "401"
+
+// generic fault prevent action from being taken
+const RPL_GenericError = "403"
+
+// newsgroup does not exist
+const RPL_NoSuchGroup = "411"
+
+// no newsgroup has been selected
+const RPL_NoGroupSelected = "412"
+
+// no tin style index available
+const RPL_NoIndex = "418"
+
+// current article number is invalid
+const RPL_NoArticleNum = "420"
+
+// no next article in this group (NEXT)
+const RPL_NoNextArticle = "421"
+
+// no previous article in this group (LAST)
+const RPL_NoPrevArticle = "422"
+
+// no article in specified range
+const RPL_NoArticleRange = "423"
+
+// no article with that message-id
+const RPL_NoArticleMsgID = "430"
+
+// defer article asked by CHECK comamnd
+const RPL_StreamingDefer = "431"
+
+// article is not wanted (1st stage of IHAVE)
+const RPL_TransferNotWanted = "435"
+
+// article was not sent defer sending (either stage of IHAVE)
+const RPL_TransferDefer = "436"
+
+// reject transfer do not retry (2nd stage IHAVE)
+const RPL_TransferReject = "437"
+
+// reject article and don't ask again (CHECK command)
+const RPL_StreamingReject = "438"
+
+// article transfer via streaming failed (TAKETHIS)
+const RPL_StreamingFailed = "439"
+
+// posting not permitted (1st stage of POST command)
+const RPL_PostingNotPermitted = "440"
+
+// posting failed (2nd stage of POST command)
+const RPL_PostingFailed = "441"
+
+// authorization required
+const RPL_AuthorizeRequired = "450"
+
+// authorization rejected
+const RPL_AuthorizeRejected = "452"
+
+// command unavaibale until client has authenticated
+const RPL_AuthenticateRequired = "480"
+
+// authentication creds rejected
+const RPL_AuthenticateRejected = "482"
+
+// command unavailable until connection is encrypted
+const RPL_EncryptionRequired = "483"
+
+// 5xx codes
+
+// got an unknown command
+const RPL_UnknownCommand = "500"
+
+// got a command with invalid syntax
+const RPL_SyntaxError = "501"
+
+// fatal error happened and connection will close
+const RPL_GenericFatal = "502"
+
+// feature is not supported
+const RPL_FeatureNotSupported = "503"
+
+// message encoding is bad
+const RPL_EncodingError = "504"
+
+// starttls can not be done
+const RPL_TLSRejected = "580"
+
+// line sent on invalid mode
+const Line_InvalidMode = RPL_SyntaxError + " Invalid Mode Selected"
+
+// line sent on successful streaming
+const Line_StreamingAllowed = RPL_PostingStreaming + " aw yeh streamit brah"
+
+// send this when we handle a QUIT command
+const Line_RPLQuit = RPL_Quit + " bai"

contrib/backends/nntpchand/src/nntpchan/lib/nntp/commands.go

@@ -0,0 +1,27 @@
+package nntp
+
+type Command string
+
+func (c Command) String() string {
+	return string(c)
+}
+
+// command to list newsgroups
+const CMD_Newsgroups = Command("NEWSGROUPS 0 0 GMT")
+
+// create group command for a newsgroup
+func CMD_Group(g Newsgroup) Command {
+	return Command("GROUP " + g.String())
+}
+
+const CMD_XOver = Command("XOVER 0")
+
+func CMD_Article(msgid MessageID) Command {
+	return Command("ARTICLE " + msgid.String())
+}
+
+func CMD_Head(msgid MessageID) Command {
+	return Command("HEAD " + msgid.String())
+}
+
+const CMD_Capabilities = Command("CAPABILITIES")

contrib/backends/nntpchand/src/nntpchan/lib/nntp/common.go

@@ -0,0 +1,75 @@
+package nntp
+
+import (
+	"crypto/sha1"
+	"fmt"
+	"io"
+	"nntpchan/lib/crypto"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var exp_valid_message_id = regexp.MustCompilePOSIX(`^<[a-zA-Z0-9$.]{2,128}@[a-zA-Z0-9\-.]{2,63}>$`)
+
+type MessageID string
+
+// return true if this message id is well formed, otherwise return false
+func (msgid MessageID) Valid() bool {
+	return exp_valid_message_id.Copy().MatchString(msgid.String())
+}
+
+// get message id as string
+func (msgid MessageID) String() string {
+	return string(msgid)
+}
+
+// compute long form hash of message id
+func (msgid MessageID) LongHash() string {
+	return fmt.Sprintf("%x", sha1.Sum([]byte(msgid)))
+}
+
+// compute truncated form of message id hash
+func (msgid MessageID) ShortHash() string {
+	return strings.ToLower(msgid.LongHash()[:18])
+}
+
+// compute blake2 hash of message id
+func (msgid MessageID) Blake2Hash() string {
+	h := crypto.Hash()
+	io.WriteString(h, msgid.String())
+	return strings.ToLower(fmt.Sprintf("%x", h.Sum(nil)))
+}
+
+// generate a new message id given name of server
+func GenMessageID(name string) MessageID {
+	r := crypto.RandBytes(4)
+	t := time.Now()
+	return MessageID(fmt.Sprintf("<%x$%d@%s>", r, t.Unix(), name))
+}
+
+var exp_valid_newsgroup = regexp.MustCompilePOSIX(`^[a-zA-Z0-9.]{1,128}$`)
+
+// an nntp newsgroup
+type Newsgroup string
+
+// return true if this newsgroup is well formed otherwise false
+func (g Newsgroup) Valid() bool {
+	return exp_valid_newsgroup.Copy().MatchString(g.String())
+}
+
+// get newsgroup as string
+func (g Newsgroup) String() string {
+	return string(g)
+}
+
+// (message-id, newsgroup) tuple
+type ArticleEntry [2]string
+
+func (e ArticleEntry) MessageID() MessageID {
+	return MessageID(e[0])
+}
+
+func (e ArticleEntry) Newsgroup() Newsgroup {
+	return Newsgroup(e[1])
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/common_test.go

@@ -0,0 +1,44 @@
+package nntp
+
+import (
+	"testing"
+)
+
+func TestGenMessageID(t *testing.T) {
+	msgid := GenMessageID("test.tld")
+	t.Logf("generated id %s", msgid)
+	if !msgid.Valid() {
+		t.Logf("invalid generated message-id %s", msgid)
+		t.Fail()
+	}
+	msgid = GenMessageID("<><><>")
+	t.Logf("generated id %s", msgid)
+	if msgid.Valid() {
+		t.Logf("generated valid message-id when it should've been invalid %s", msgid)
+		t.Fail()
+	}
+}
+
+func TestMessageIDHash(t *testing.T) {
+	msgid := GenMessageID("test.tld")
+	lh := msgid.LongHash()
+	sh := msgid.ShortHash()
+	bh := msgid.Blake2Hash()
+	t.Logf("long=%s short=%s blake2=%s", lh, sh, bh)
+}
+
+func TestValidNewsgroup(t *testing.T) {
+	g := Newsgroup("overchan.test")
+	if !g.Valid() {
+		t.Logf("%s is invalid?", g)
+		t.Fail()
+	}
+}
+
+func TestInvalidNewsgroup(t *testing.T) {
+	g := Newsgroup("asd.asd.asd.&&&")
+	if g.Valid() {
+		t.Logf("%s should be invalid", g)
+		t.Fail()
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/conn.go

@@ -0,0 +1,53 @@
+package nntp
+
+// an nntp connection
+type Conn interface {
+
+	// negotiate an nntp session on this connection
+	// returns nil if we negitated successfully
+	// returns ErrAuthRejected if the remote server rejected any authentication
+	// we sent or another error if one occured while negotiating
+	Negotiate(stream bool) error
+
+	// obtain connection state
+	GetState() *ConnState
+
+	// retutrn true if posting is allowed
+	// return false if posting is not allowed
+	PostingAllowed() bool
+
+	// handle inbound non-streaming connection
+	// call event hooks on event
+	ProcessInbound(hooks EventHooks)
+
+	// does this connection want to do nntp streaming?
+	WantsStreaming() bool
+
+	// what mode are we in?
+	// returns mode in all caps
+	Mode() Mode
+
+	// initiate nntp streaming
+	// after calling this the caller MUST call StreamAndQuit()
+	// returns a channel for message ids, true if caller sends on the channel or
+	// returns nil and ErrStreamingNotAllowed if streaming is not allowed on this
+	// connection or another error if one occurs while trying to start streaming
+	StartStreaming() (chan ArticleEntry, error)
+
+	// stream articles and quit when the channel obtained by StartStreaming() is
+	// closed, after which this nntp connection is no longer open
+	StreamAndQuit()
+
+	// is this nntp connection open?
+	IsOpen() bool
+
+	// send quit command and close connection
+	Quit()
+
+	// download all articles in a newsgroup
+	// returns error if a network error occurs
+	DownloadGroup(g Newsgroup) error
+
+	// get list of active newsgroups
+	ListNewsgroups() ([]Newsgroup, error)
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/dial.go

@@ -0,0 +1,15 @@
+package nntp
+
+import (
+	"crypto/tls"
+	"nntpchan/lib/network"
+)
+
+// establishes an outbound nntp connection to a remote server
+type Dialer interface {
+	// dial out with a dialer
+	// if cfg is not nil, try to establish a tls connection with STARTTLS
+	// returns a new nntp connection and nil on successful handshake and login
+	// returns nil and an error if an error happened
+	Dial(d network.Dialer, cfg *tls.Config) (*Conn, error)
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/doc.go

@@ -0,0 +1,4 @@
+//
+// nntp client/server
+//
+package nntp

contrib/backends/nntpchand/src/nntpchan/lib/nntp/event_test.go

@@ -0,0 +1,60 @@
+package nntp
+
+import (
+	"testing"
+)
+
+func TestTAKETHISParse(t *testing.T) {
+	msgid := GenMessageID("test.tld")
+	ev := stream_cmd_TAKETHIS(msgid)
+	t.Logf("event: %s", ev)
+	if ev.MessageID() != msgid {
+		t.Logf("%s != %s, event was %s", msgid, ev.MessageID(), ev)
+		t.Fail()
+	}
+	if ev.Command() != "TAKETHIS" {
+		t.Logf("%s != TAKETHIS, event was %s", ev.Command(), ev)
+		t.Fail()
+	}
+	if !ev.Valid() {
+		t.Logf("%s is invalid stream event", ev)
+		t.Fail()
+	}
+}
+
+func TestCHECKParse(t *testing.T) {
+	msgid := GenMessageID("test.tld")
+	ev := stream_cmd_CHECK(msgid)
+	t.Logf("event: %s", ev)
+	if ev.MessageID() != msgid {
+		t.Logf("%s != %s, event was %s", msgid, ev.MessageID(), ev)
+		t.Fail()
+	}
+	if ev.Command() != "CHECK" {
+		t.Logf("%s != CHECK, event was %s", ev.Command(), ev)
+		t.Fail()
+	}
+	if !ev.Valid() {
+		t.Logf("%s is invalid stream event", ev)
+		t.Fail()
+	}
+}
+
+func TestInvalidStremEvent(t *testing.T) {
+	str := "asd"
+	ev := StreamEvent(str)
+	t.Logf("invalid str=%s ev=%s", str, ev)
+	if ev.Valid() {
+		t.Logf("invalid CHECK command is valid? %s", ev)
+		t.Fail()
+	}
+
+	str = "asd asd"
+	ev = StreamEvent(str)
+	t.Logf("invalid str=%s ev=%s", str, ev)
+
+	if ev.Valid() {
+		t.Logf("invalid CHECK command is valid? %s", ev)
+		t.Fail()
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/filter.go

@@ -0,0 +1,19 @@
+package nntp
+
+import (
+	"io"
+	"nntpchan/lib/nntp/message"
+)
+
+// defines interface for filtering an nntp article
+// filters can (and does) modify the article it operates on
+type ArticleFilter interface {
+	// filter the article header
+	// returns the modified Header and an error if one occurs
+	FilterHeader(hdr message.Header) (message.Header, error)
+
+	// reads the article's body and write the filtered version to an io.Writer
+	// returns the number of bytes written to the io.Writer, true if the body was
+	// modifed (or false if body is unchanged) and an error if one occurs
+	FilterAndWriteBody(body io.Reader, wr io.Writer) (int64, bool, error)
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/hook.go

@@ -0,0 +1,30 @@
+package nntp
+
+import (
+	log "github.com/Sirupsen/logrus"
+	"nntpchan/lib/config"
+	"os/exec"
+)
+
+type Hook struct {
+	cfg *config.NNTPHookConfig
+}
+
+func NewHook(cfg *config.NNTPHookConfig) *Hook {
+	return &Hook{
+		cfg: cfg,
+	}
+}
+
+func (h *Hook) GotArticle(msgid MessageID, group Newsgroup) {
+	c := exec.Command(h.cfg.Exec, group.String(), msgid.String())
+	log.Infof("calling hook %s", h.cfg.Name)
+	err := c.Run()
+	if err != nil {
+		log.Errorf("error in nntp hook %s: %s", h.cfg.Name, err.Error())
+	}
+}
+
+func (*Hook) SentArticleVia(msgid MessageID, feedname string) {
+
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/hooks.go

@@ -0,0 +1,9 @@
+package nntp
+
+// callback hooks fired on certain events
+type EventHooks interface {
+	// called when we have obtained an article given its message-id
+	GotArticle(msgid MessageID, group Newsgroup)
+	// called when we have sent an article to a single remote feed
+	SentArticleVia(msgid MessageID, feedname string)
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/message/article.go

@@ -0,0 +1,11 @@
+package message
+
+// an nntp article
+type Article struct {
+
+	// the article's mime header
+	Header Header
+
+	// unexported fields ...
+
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/message/attachment.go

@@ -0,0 +1,16 @@
+package message
+
+import (
+	"io"
+)
+
+// attachment in an nntp article
+type Attachment struct {
+	// mimetype
+	Mime string
+	// the filename
+	FileName string
+	// the fully decoded attachment body
+	// must close when done
+	Body io.ReadCloser
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/message/doc.go

@@ -0,0 +1,2 @@
+// package for parsing, packing, signing, verifying nntp articles
+package message

contrib/backends/nntpchand/src/nntpchan/lib/nntp/message/header.go

@@ -0,0 +1,174 @@
+package message
+
+import (
+	"io"
+	"mime"
+	"strings"
+)
+
+// an nntp message header
+type Header map[string][]string
+
+// get message-id header
+func (self Header) MessageID() (v string) {
+	for _, hdr := range []string{"MessageID", "Message-ID", "Message-Id", "message-id"} {
+		v = self.Get(hdr, "")
+		if v != "" {
+			break
+		}
+	}
+	return
+}
+
+func (self Header) Reference() (ref string) {
+	return self.Get("Reference", self.MessageID())
+}
+
+// extract media type from content-type header
+func (self Header) GetMediaType() (mediatype string, params map[string]string, err error) {
+	return mime.ParseMediaType(self.Get("Content-Type", "text/plain"))
+}
+
+// is this header for a multipart message?
+func (self Header) IsMultipart() bool {
+	return strings.HasPrefix(self.Get("Content-Type", "text/plain"), "multipart/mixed")
+}
+
+func (self Header) IsSigned() bool {
+	return self.Get("X-Pubkey-Ed25519", "") != ""
+}
+
+func (self Header) Newsgroup() string {
+	return self.Get("Newsgroups", "overchan.discard")
+}
+
+// do we have a key in this header?
+func (self Header) Has(key string) bool {
+	_, ok := self[key]
+	return ok
+}
+
+// set key value
+func (self Header) Set(key, val string) {
+	self[key] = []string{val}
+}
+
+func (self Header) AppendPath(name string) {
+	p := self.Get("Path", name)
+	if p != name {
+		p = name + "!" + p
+	}
+	self.Set("Path", p)
+}
+
+// append value to key
+func (self Header) Add(key, val string) {
+	if self.Has(key) {
+		self[key] = append(self[key], val)
+	} else {
+		self.Set(key, val)
+	}
+}
+
+// get via key or return fallback value
+func (self Header) Get(key, fallback string) string {
+	val, ok := self[key]
+	if ok {
+		str := ""
+		for _, k := range val {
+			str += k + ", "
+		}
+		return str[:len(str)-2]
+	} else {
+		return fallback
+	}
+}
+
+// interface for types that can read an nntp header
+type HeaderReader interface {
+	// blocking read an nntp header from an io.Reader
+	// return the read header and nil on success
+	// return nil and an error if an error occurred while reading
+	ReadHeader(r io.Reader) (Header, error)
+}
+
+// interface for types that can write an nntp header
+type HeaderWriter interface {
+	// blocking write an nntp header to an io.Writer
+	// returns an error if one occurs otherwise nil
+	WriteHeader(hdr Header, w io.Writer) error
+}
+
+// implements HeaderReader and HeaderWriter
+type HeaderIO struct {
+	delim byte
+}
+
+// read header
+func (s *HeaderIO) ReadHeader(r io.Reader) (hdr Header, err error) {
+	hdr = make(Header)
+	var k, v string
+	var buf [1]byte
+	for err == nil {
+		// read key
+		for err == nil {
+			_, err = r.Read(buf[:])
+			if err != nil {
+				return
+			}
+			if buf[0] == 58 { // colin
+				// consume space
+				_, err = r.Read(buf[:])
+				for err == nil {
+					_, err = r.Read(buf[:])
+					if buf[0] == s.delim {
+						// got delimiter
+						hdr.Add(k, v)
+						k = ""
+						v = ""
+						break
+					} else {
+						v += string(buf[:])
+					}
+				}
+				break
+			} else if buf[0] == s.delim {
+				// done
+				return
+			} else {
+				k += string(buf[:])
+			}
+		}
+	}
+	return
+}
+
+// write header
+func (s *HeaderIO) WriteHeader(hdr Header, wr io.Writer) (err error) {
+	for k, vs := range hdr {
+		for _, v := range vs {
+			var line []byte
+			// key
+			line = append(line, []byte(k)...)
+			// ": "
+			line = append(line, 58, 32)
+			// value
+			line = append(line, []byte(v)...)
+			// delimiter
+			line = append(line, s.delim)
+			// write line
+			_, err = wr.Write(line)
+			if err != nil {
+				return
+			}
+		}
+	}
+	_, err = wr.Write([]byte{s.delim})
+	return
+}
+
+func NewHeaderIO() *HeaderIO {
+	return &HeaderIO{
+		delim: 10,
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/mode.go

@@ -0,0 +1,69 @@
+package nntp
+
+import (
+	"errors"
+	"strings"
+)
+
+var ErrInvalidMode = errors.New("invalid mode set")
+
+// a mode set by an nntp client
+type Mode string
+
+// reader mode
+const MODE_READER = Mode("reader")
+
+// streaming mode
+const MODE_STREAM = Mode("stream")
+
+// mode is not set
+const MODE_UNSET = Mode("")
+
+// get as string
+func (m Mode) String() string {
+	return strings.ToUpper(string(m))
+}
+
+// is this a valid mode of operation?
+func (m Mode) Valid() bool {
+	return m.Is(MODE_READER) || m.Is(MODE_STREAM)
+}
+
+// is this mode equal to another mode
+func (m Mode) Is(other Mode) bool {
+	return m.String() == other.String()
+}
+
+// a switch mode command
+type ModeCommand string
+
+// get as string
+func (m ModeCommand) String() string {
+	return strings.ToUpper(string(m))
+}
+
+// is this mode command well formed?
+// does not check the actual mode sent.
+func (m ModeCommand) Valid() bool {
+	s := m.String()
+	return strings.Count(s, " ") == 1 && strings.HasPrefix(s, "MODE ")
+}
+
+// get the mode selected in this mode command
+func (m ModeCommand) Mode() Mode {
+	return Mode(strings.Split(m.String(), " ")[1])
+}
+
+// check if this mode command is equal to an existing one
+func (m ModeCommand) Is(cmd ModeCommand) bool {
+	return m.String() == cmd.String()
+}
+
+// reader mode command
+const ModeReader = ModeCommand("mode reader")
+
+// streaming mode command
+const ModeStream = ModeCommand("mode stream")
+
+// line prefix for mode
+const LinePrefix_Mode = "MODE "

contrib/backends/nntpchand/src/nntpchan/lib/nntp/multi.go

@@ -0,0 +1,16 @@
+package nntp
+
+// multiplexed event hook
+type MulitHook []EventHooks
+
+func (m MulitHook) GotArticle(msgid MessageID, group Newsgroup) {
+	for _, h := range m {
+		h.GotArticle(msgid, group)
+	}
+}
+
+func (m MulitHook) SentArticleVia(msgid MessageID, feedname string) {
+	for _, h := range m {
+		h.SentArticleVia(msgid, feedname)
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/policy.go

@@ -0,0 +1,29 @@
+package nntp
+
+//
+// a policy that governs whether we federate an article via a feed
+//
+type FeedPolicy struct {
+	// list of whitelist regexps for newsgorups
+	Whitelist []string `json:"whitelist"`
+	// list of blacklist regexps for newsgroups
+	Blacklist []string `json:"blacklist"`
+	// are anon posts of any kind allowed?
+	AllowAnonPosts bool `json:"anon"`
+	// are anon posts with attachments allowed?
+	AllowAnonAttachments bool `json:"anon_attachments"`
+	// are any attachments allowed?
+	AllowAttachments bool `json:"attachments"`
+	// do we require Proof Of Work for untrusted connections?
+	UntrustedRequiresPoW bool `json:"pow"`
+}
+
+// default feed policy to be used if not configured explicitly
+var DefaultFeedPolicy = &FeedPolicy{
+	Whitelist:            []string{"ctl", "overchan.test"},
+	Blacklist:            []string{`!^overchan\.`},
+	AllowAnonPosts:       true,
+	AllowAnonAttachments: false,
+	UntrustedRequiresPoW: true,
+	AllowAttachments:     true,
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/server.go

@@ -0,0 +1,326 @@
+package nntp
+
+import (
+	log "github.com/Sirupsen/logrus"
+	"net"
+	"nntpchan/lib/config"
+	"nntpchan/lib/network"
+	"nntpchan/lib/store"
+	"time"
+)
+
+// nntp outfeed state
+type nntpFeed struct {
+	conn Conn
+	send chan ArticleEntry
+	conf *config.FeedConfig
+}
+
+// an nntp server
+type Server struct {
+	// user callback
+	Hooks EventHooks
+	// filters to apply
+	Filters []ArticleFilter
+	// global article acceptor
+	Acceptor ArticleAcceptor
+	// article storage
+	Storage store.Storage
+	// nntp config
+	Config *config.NNTPServerConfig
+	// outfeeds to connect to
+	Feeds []*config.FeedConfig
+	// inbound authentiaction mechanism
+	Auth ServerAuth
+	// send to outbound feed channel
+	send chan ArticleEntry
+	// register inbound feed channel
+	regis chan *nntpFeed
+	// deregister inbound feed channel
+	deregis chan *nntpFeed
+}
+
+func NewServer() *Server {
+	return &Server{
+		// XXX: buffered?
+		send:    make(chan ArticleEntry),
+		regis:   make(chan *nntpFeed),
+		deregis: make(chan *nntpFeed),
+	}
+}
+
+// reload server configuration
+func (s *Server) ReloadServer(c *config.NNTPServerConfig) {
+
+}
+
+// reload feeds
+func (s *Server) ReloadFeeds(feeds []*config.FeedConfig) {
+
+}
+
+func (s *Server) GotArticle(msgid MessageID, group Newsgroup) {
+	log.WithFields(log.Fields{
+		"pkg":   "nntp-server",
+		"msgid": msgid,
+		"group": group,
+	}).Info("obtained article")
+	if s.Hooks != nil {
+		s.Hooks.GotArticle(msgid, group)
+	}
+	// send to outbound feeds
+	s.send <- ArticleEntry{msgid.String(), group.String()}
+}
+
+func (s *Server) SentArticleVia(msgid MessageID, feedname string) {
+	log.WithFields(log.Fields{
+		"pkg":   "nntp-server",
+		"msgid": msgid,
+		"feed":  feedname,
+	}).Info("article sent")
+	if s.Hooks != nil {
+		s.Hooks.SentArticleVia(msgid, feedname)
+	}
+}
+
+func (s *Server) Name() string {
+	if s.Config == nil || s.Config.Name == "" {
+		return "nntp.anon.tld"
+	}
+	return s.Config.Name
+}
+
+// persist 1 feed forever
+func (s *Server) persist(cfg *config.FeedConfig) {
+	delay := time.Second
+
+	log.WithFields(log.Fields{
+		"name": cfg.Name,
+	}).Debug("Persist Feed")
+	for {
+		dialer := network.NewDialer(cfg.Proxy)
+		c, err := dialer.Dial(cfg.Addr)
+		if err == nil {
+			// successful connect
+			delay = time.Second
+			conn := newOutboundConn(c, s, cfg)
+			err = conn.Negotiate(true)
+			if err == nil {
+				// negotiation good
+				log.WithFields(log.Fields{
+					"name": cfg.Name,
+				}).Debug("Negotitation good")
+				// start streaming
+				var chnl chan ArticleEntry
+				chnl, err = conn.StartStreaming()
+				if err == nil {
+					// register new connection
+					f := &nntpFeed{
+						conn: conn,
+						send: chnl,
+						conf: cfg,
+					}
+					s.regis <- f
+					// start streaming
+					conn.StreamAndQuit()
+					// deregister
+					s.deregis <- f
+					continue
+				}
+			} else {
+				log.WithFields(log.Fields{
+					"name": cfg.Name,
+				}).Info("outbound nntp connection failed to negotiate ", err)
+			}
+			conn.Quit()
+		} else {
+			// failed dial, do exponential backoff up to 1 hour
+			if delay <= time.Hour {
+				delay *= 2
+			}
+			log.WithFields(log.Fields{
+				"name": cfg.Name,
+			}).Info("feed backoff for ", delay)
+			time.Sleep(delay)
+		}
+	}
+}
+
+// download all new posts from a remote server
+func (s *Server) downloadPosts(cfg *config.FeedConfig) error {
+	dialer := network.NewDialer(cfg.Proxy)
+	c, err := dialer.Dial(cfg.Addr)
+	if err != nil {
+		return err
+	}
+	conn := newOutboundConn(c, s, cfg)
+	err = conn.Negotiate(false)
+	if err != nil {
+		conn.Quit()
+		return err
+	}
+	groups, err := conn.ListNewsgroups()
+	if err != nil {
+		conn.Quit()
+		return err
+	}
+	for _, g := range groups {
+		if cfg.Policy != nil && cfg.Policy.AllowGroup(g.String()) {
+			log.WithFields(log.Fields{
+				"group": g,
+				"pkg":   "nntp-server",
+			}).Debug("downloading group")
+			err = conn.DownloadGroup(g)
+			if err != nil {
+				conn.Quit()
+				return err
+			}
+		}
+	}
+	conn.Quit()
+	return nil
+}
+
+func (s *Server) periodicDownload(cfg *config.FeedConfig) {
+	for cfg.PullInterval > 0 {
+		err := s.downloadPosts(cfg)
+		if err != nil {
+			// report error
+			log.WithFields(log.Fields{
+				"feed":  cfg.Name,
+				"pkg":   "nntp-server",
+				"error": err,
+			}).Error("periodic download failed")
+		}
+		time.Sleep(time.Minute * time.Duration(cfg.PullInterval))
+	}
+}
+
+// persist all outbound feeds
+func (s *Server) PersistFeeds() {
+	for _, f := range s.Feeds {
+		go s.persist(f)
+		go s.periodicDownload(f)
+	}
+
+	feeds := make(map[string]*nntpFeed)
+
+	for {
+		select {
+		case e, ok := <-s.send:
+			if !ok {
+				break
+			}
+			msgid := e.MessageID().String()
+			group := e.Newsgroup().String()
+			// TODO: determine anon
+			anon := false
+			// TODO: determine attachments
+			attachments := false
+
+			for _, f := range feeds {
+				if f.conf.Policy != nil && !f.conf.Policy.Allow(msgid, group, anon, attachments) {
+					// not allowed in this feed
+					continue
+				}
+				log.WithFields(log.Fields{
+					"name":  f.conf.Name,
+					"msgid": msgid,
+					"group": group,
+				}).Debug("sending article")
+				f.send <- e
+			}
+			break
+		case f, ok := <-s.regis:
+			if ok {
+				log.WithFields(log.Fields{
+					"name": f.conf.Name,
+				}).Debug("register feed")
+				feeds[f.conf.Name] = f
+			}
+			break
+		case f, ok := <-s.deregis:
+			if ok {
+				log.WithFields(log.Fields{
+					"name": f.conf.Name,
+				}).Debug("deregister feed")
+				delete(feeds, f.conf.Name)
+			}
+			break
+		}
+	}
+}
+
+// serve connections from listener
+func (s *Server) Serve(l net.Listener) (err error) {
+	log.WithFields(log.Fields{
+		"pkg":  "nntp-server",
+		"addr": l.Addr(),
+	}).Debug("Serving")
+	for err == nil {
+		var c net.Conn
+		c, err = l.Accept()
+		if err == nil {
+			// we got a new connection
+			go s.handleInboundConnection(c)
+		} else {
+			log.WithFields(log.Fields{
+				"pkg": "nntp-server",
+			}).Error("failed to accept inbound connection", err)
+		}
+	}
+	return
+}
+
+// get the article policy for a connection given its state
+func (s *Server) getPolicyFor(state *ConnState) ArticleAcceptor {
+	return s.Acceptor
+}
+
+// recv inbound streaming messages
+func (s *Server) recvInboundStream(chnl chan ArticleEntry) {
+	for {
+		e, ok := <-chnl
+		if ok {
+			s.GotArticle(e.MessageID(), e.Newsgroup())
+		} else {
+			return
+		}
+	}
+}
+
+// process an inbound connection
+func (s *Server) handleInboundConnection(c net.Conn) {
+	log.WithFields(log.Fields{
+		"pkg":  "nntp-server",
+		"addr": c.RemoteAddr(),
+	}).Debug("handling inbound connection")
+	var nc Conn
+	nc = newInboundConn(s, c)
+	err := nc.Negotiate(true)
+	if err == nil {
+		// do they want to stream?
+		if nc.WantsStreaming() {
+			// yeeeeeh let's stream
+			var chnl chan ArticleEntry
+			chnl, err = nc.StartStreaming()
+			// for inbound we will recv messages
+			go s.recvInboundStream(chnl)
+			nc.StreamAndQuit()
+			log.WithFields(log.Fields{
+				"pkg":  "nntp-server",
+				"addr": c.RemoteAddr(),
+			}).Info("streaming finished")
+			return
+		} else {
+			// handle non streaming commands
+			nc.ProcessInbound(s)
+		}
+	} else {
+		log.WithFields(log.Fields{
+			"pkg":  "nntp-server",
+			"addr": c.RemoteAddr(),
+		}).Warn("failed to negotiate with inbound connection", err)
+		c.Close()
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/state.go

@@ -0,0 +1,21 @@
+package nntp
+
+// state of an nntp connection
+type ConnState struct {
+	// name of parent feed
+	FeedName string `json:"feedname"`
+	// name of the connection
+	ConnName string `json:"connname"`
+	// hostname of remote connection
+	HostName string `json:"hostname"`
+	// current nntp mode
+	Mode Mode `json:"mode"`
+	// current selected nntp newsgroup
+	Group Newsgroup `json:"newsgroup"`
+	// current selected nntp article
+	Article string `json:"article"`
+	// parent feed's policy
+	Policy *FeedPolicy `json:"feedpolicy"`
+	// is this connection open?
+	Open bool `json:"open"`
+}

contrib/backends/nntpchand/src/nntpchan/lib/nntp/streaming.go

@@ -0,0 +1,65 @@
+package nntp
+
+import (
+	"fmt"
+	"strings"
+)
+
+// an nntp stream event
+// these are pipelined between nntp servers
+type StreamEvent string
+
+func (ev StreamEvent) MessageID() MessageID {
+	parts := strings.Split(string(ev), " ")
+	if len(parts) > 1 {
+		return MessageID(parts[1])
+	}
+	return ""
+}
+
+func (ev StreamEvent) String() string {
+	return string(ev)
+}
+
+func (ev StreamEvent) Command() string {
+	return strings.Split(ev.String(), " ")[0]
+}
+
+func (ev StreamEvent) Valid() bool {
+	return strings.Count(ev.String(), " ") == 1 && ev.MessageID().Valid()
+}
+
+var stream_TAKETHIS = "TAKETHIS"
+var stream_CHECK = "CHECK"
+
+func createStreamEvent(cmd string, msgid MessageID) StreamEvent {
+	if msgid.Valid() {
+		return StreamEvent(fmt.Sprintf("%s %s", cmd, msgid))
+	} else {
+		return ""
+	}
+}
+
+func stream_rpl_Accept(msgid MessageID) StreamEvent {
+	return createStreamEvent(RPL_StreamingAccept, msgid)
+}
+
+func stream_rpl_Reject(msgid MessageID) StreamEvent {
+	return createStreamEvent(RPL_StreamingReject, msgid)
+}
+
+func stream_rpl_Defer(msgid MessageID) StreamEvent {
+	return createStreamEvent(RPL_StreamingDefer, msgid)
+}
+
+func stream_rpl_Failed(msgid MessageID) StreamEvent {
+	return createStreamEvent(RPL_StreamingFailed, msgid)
+}
+
+func stream_cmd_TAKETHIS(msgid MessageID) StreamEvent {
+	return createStreamEvent(stream_TAKETHIS, msgid)
+}
+
+func stream_cmd_CHECK(msgid MessageID) StreamEvent {
+	return createStreamEvent(stream_CHECK, msgid)
+}

contrib/backends/nntpchand/src/nntpchan/lib/srnd/doc.go

@@ -0,0 +1,5 @@
+//
+// main package for srndv2
+// called from main
+//
+package srnd

contrib/backends/nntpchand/src/nntpchan/lib/store/doc.go

@@ -0,0 +1,4 @@
+//
+// nntp article storage
+//
+package store

contrib/backends/nntpchand/src/nntpchan/lib/store/fs.go

@@ -0,0 +1,416 @@
+package store
+
+import (
+	"encoding/base32"
+	"fmt"
+	log "github.com/Sirupsen/logrus"
+	"io"
+	"io/ioutil"
+	"net/textproto"
+	"nntpchan/lib/crypto"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+)
+
+const HighWaterHeader = "X-High-Water"
+const LowWaterHeader = "X-Low-Water"
+
+// filesystem storage of nntp articles and attachments
+type FilesystemStorage struct {
+	root               string
+	discardAttachments bool
+}
+
+func (fs FilesystemStorage) String() string {
+	return fs.root
+}
+
+func (fs FilesystemStorage) NewsgroupsDir() string {
+	return filepath.Join(fs.root, "newsgroups")
+}
+
+func (fs FilesystemStorage) metadataFileForNewsgroup(newsgroup string) string {
+	return filepath.Join(fs.newsgroupDir(newsgroup), "metadata")
+}
+
+func (fs FilesystemStorage) GetWatermark(newsgroup string) (hi, lo uint64, err error) {
+	var hdr textproto.MIMEHeader
+	hdr, err = fs.getMetadataForNewsgroup(newsgroup)
+	if err == nil {
+		hi, err = strconv.ParseUint(hdr.Get(HighWaterHeader), 10, 64)
+		if err == nil {
+			lo, err = strconv.ParseUint(hdr.Get(LowWaterHeader), 10, 64)
+		}
+	}
+	return
+}
+
+func (fs FilesystemStorage) getMetadataForNewsgroup(newsgroup string) (hdr textproto.MIMEHeader, err error) {
+	var f *os.File
+	fp := fs.metadataFileForNewsgroup(newsgroup)
+	_, err = os.Stat(fp)
+	if os.IsNotExist(err) {
+		f, err = os.OpenFile(fp, os.O_RDWR|os.O_CREATE, 0600)
+		if err == nil {
+			h := make(textproto.MIMEHeader)
+			h.Set(HighWaterHeader, "0")
+			c := textproto.NewConn(f)
+			for k := range hdr {
+				for _, v := range h[k] {
+					err = c.PrintfLine("%s: %s", k, v)
+					if err != nil {
+						c.Close()
+						return
+					}
+				}
+			}
+			c.Close()
+		}
+	}
+	f, err = os.OpenFile(fp, os.O_RDWR, 0600)
+	if err == nil {
+		c := textproto.NewConn(f)
+		hdr, err = c.ReadMIMEHeader()
+		c.Close()
+	}
+	return
+}
+
+func (fs FilesystemStorage) nextIDForNewsgroup(newsgroup string) (id uint64, err error) {
+	id, _, err = fs.GetWatermark(newsgroup)
+
+	if err == nil {
+		id++
+		var hdr textproto.MIMEHeader
+		hdr, err = fs.getMetadataForNewsgroup(newsgroup)
+		if err == nil {
+			hdr.Set(HighWaterHeader, fmt.Sprintf("%d", id))
+			var f *os.File
+			f, err = os.OpenFile(fs.metadataFileForNewsgroup(newsgroup), os.O_WRONLY, 0600)
+			if err == nil {
+				c := textproto.NewConn(f)
+				for k := range hdr {
+					for _, v := range hdr[k] {
+						err = c.PrintfLine("%s: %s", k, v)
+						if err != nil {
+							c.Close()
+							return
+						}
+					}
+				}
+				c.Close()
+			}
+		}
+	}
+	return
+}
+
+func (fs FilesystemStorage) GetAllNewsgroups() (newsgroups []string, err error) {
+	err = filepath.Walk(fs.NewsgroupsDir(), filepath.WalkFunc(func(f string, info os.FileInfo, e error) (er error) {
+		if e != nil {
+			er = e
+			newsgroups = nil
+		}
+		if info.IsDir() && err == nil {
+			newsgroups = append(newsgroups, f)
+		}
+		return er
+	}))
+	return
+}
+
+func (fs FilesystemStorage) HasNewsgroup(newsgroup string) (has bool, err error) {
+	_, err = os.Stat(fs.newsgroupDir(newsgroup))
+	has = err == nil
+	return
+}
+
+// ensure the filesystem storage exists and is well formed and read/writable
+func (fs FilesystemStorage) Ensure() (err error) {
+	_, err = os.Stat(fs.String())
+	if os.IsNotExist(err) {
+		// directory does not exist, create it
+		err = os.Mkdir(fs.String(), 0755)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"pkg":      "fs-store",
+				"filepath": fs.String(),
+			}).Error("failed to ensure directory", err)
+			// failed to create initial directory
+			return
+		}
+	}
+
+	// ensure subdirectories
+	for _, subdir := range []string{"att", "thm", "articles", "tmp", "newsgroups"} {
+		fpath := filepath.Join(fs.String(), subdir)
+		_, err = os.Stat(fpath)
+		if os.IsNotExist(err) {
+			// make subdirectory
+			err = os.Mkdir(fpath, 0755)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"pkg":      "fs-store",
+					"filepath": fpath,
+				}).Error("failed to ensure sub-directory", err)
+				// failed to create subdirectory
+				return
+			}
+		}
+	}
+	return
+}
+
+// get the temp file directory
+func (fs FilesystemStorage) TempDir() string {
+	return filepath.Join(fs.String(), "tmp")
+}
+
+// get the directory path for attachments
+func (fs FilesystemStorage) AttachmentDir() string {
+	return filepath.Join(fs.String(), "att")
+}
+
+// get the directory path for articles
+func (fs FilesystemStorage) ArticleDir() string {
+	return filepath.Join(fs.String(), "articles")
+}
+
+// get a temporary file we can use for read/write that deletes itself on close
+func (fs FilesystemStorage) obtainTempFile() (f *os.File, err error) {
+	fname := fmt.Sprintf("tempfile-%x-%d", crypto.RandBytes(4), time.Now().Unix())
+	log.WithFields(log.Fields{
+		"pkg":      "fs-store",
+		"filepath": fname,
+	}).Debug("opening temp file")
+	f, err = os.OpenFile(filepath.Join(fs.TempDir(), fname), os.O_RDWR|os.O_CREATE, 0400)
+	return
+}
+
+// store an article from a reader to disk
+func (fs FilesystemStorage) StoreArticle(r io.Reader, msgid, newsgroup string) (fpath string, err error) {
+	err = fs.HasArticle(msgid)
+	if err == nil {
+		// discard the body as we have it stored already
+		_, err = io.Copy(ioutil.Discard, r)
+		log.WithFields(log.Fields{
+			"pkg":   "fs-store",
+			"msgid": msgid,
+		}).Debug("discard article")
+	} else if err == ErrNoSuchArticle {
+		log.WithFields(log.Fields{
+			"pkg":   "fs-store",
+			"msgid": msgid,
+		}).Debug("storing article")
+		// don't have an article with this message id, write it to disk
+		var f *os.File
+		fpath = filepath.Join(fs.ArticleDir(), msgid)
+		f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644)
+		if err == nil {
+			// file opened okay, defer the close
+			defer f.Close()
+			// write to disk
+			log.WithFields(log.Fields{
+				"pkg":   "fs-store",
+				"msgid": msgid,
+			}).Debug("writing to disk")
+			var n int64
+			n, err = io.Copy(f, r)
+			if err == nil {
+				log.WithFields(log.Fields{
+					"pkg":     "fs-store",
+					"msgid":   msgid,
+					"written": n,
+				}).Debug("wrote article to disk")
+				// symlink
+				g := fs.newsgroupDir(newsgroup)
+				_, e := os.Stat(g)
+				if os.IsNotExist(e) {
+					err = os.Mkdir(g, 0700)
+				}
+				var nntpid uint64
+				nntpid, err = fs.nextIDForNewsgroup(newsgroup)
+				if err == nil {
+					err = os.Symlink(filepath.Join("..", "..", "articles", msgid), filepath.Join(g, fmt.Sprintf("%d", nntpid)))
+				}
+				if err != nil {
+					log.WithFields(log.Fields{
+						"pkg":   "fs-store",
+						"msgid": msgid,
+						"group": newsgroup,
+					}).Debug("failed to link article")
+				}
+			} else {
+				log.WithFields(log.Fields{
+					"pkg":     "fs-store",
+					"msgid":   msgid,
+					"written": n,
+				}).Error("write to disk failed")
+			}
+		} else {
+			log.WithFields(log.Fields{
+				"pkg":      "fs-store",
+				"msgid":    msgid,
+				"filepath": fpath,
+			}).Error("did not open file for storage", err)
+		}
+	}
+	return
+}
+
+func (fs FilesystemStorage) newsgroupDir(group string) string {
+	return filepath.Join(fs.NewsgroupsDir(), group)
+}
+
+// check if we have the artilce with this message id
+func (fs FilesystemStorage) HasArticle(msgid string) (err error) {
+	fpath := fs.ArticleDir()
+	fpath = filepath.Join(fpath, msgid)
+	log.WithFields(log.Fields{
+		"pkg":      "fs-store",
+		"msgid":    msgid,
+		"filepath": fpath,
+	}).Debug("check for article")
+	_, err = os.Stat(fpath)
+	if os.IsNotExist(err) {
+		err = ErrNoSuchArticle
+	}
+	return
+}
+
+func (fs FilesystemStorage) DeleteArticle(msgid string) (err error) {
+	err = os.Remove(filepath.Join(fs.ArticleDir(), msgid))
+	return
+}
+
+// store attachment onto filesystem
+func (fs FilesystemStorage) StoreAttachment(r io.Reader, filename string) (fpath string, err error) {
+	if fs.discardAttachments {
+		_, err = io.Copy(ioutil.Discard, r)
+		return
+	}
+	// open temp file for storage
+	var tf *os.File
+	tf, err = fs.obtainTempFile()
+	if err == nil {
+		// we have the temp file
+
+		// close tempfile when done
+		defer func() {
+			n := tf.Name()
+			tf.Close()
+			os.Remove(n)
+		}()
+
+		// create hasher
+		h := crypto.Hash()
+		// create multiwriter
+		mw := io.MultiWriter(tf, h)
+
+		log.WithFields(log.Fields{
+			"pkg":      "fs-store",
+			"filename": filename,
+		}).Debug("writing to disk")
+		var n int64
+		// write all of the reader to the multiwriter
+		n, err = io.Copy(mw, r)
+
+		if err == nil {
+			// successful write
+
+			// get file checksum
+			d := h.Sum(nil)
+
+			// rename file to hash + extension from filename
+			fpath = base32.StdEncoding.EncodeToString(d) + filepath.Ext(filename)
+			fpath = filepath.Join(fs.AttachmentDir(), fpath)
+
+			_, err = os.Stat(fpath)
+			// is that file there?
+			if os.IsNotExist(err) {
+				// it's not there, let's write it
+				var f *os.File
+				f, err = os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE, 0644)
+				if err == nil {
+					// file opened
+					defer f.Close()
+					// seek to beginning of tempfile
+					tf.Seek(0, os.SEEK_SET)
+					// write all of the temp file to the storage file
+					n, err = io.Copy(f, tf)
+					// if err == nil by here it's all good
+					l := log.WithFields(log.Fields{
+						"pkg":      "fs-store",
+						"filename": filename,
+						"hash":     d,
+						"filepath": fpath,
+						"size":     n,
+					})
+					if err == nil {
+						l.Debug("wrote attachment to disk")
+					} else {
+						l.Error("failed to write attachment to disk", err)
+					}
+				} else {
+					log.WithFields(log.Fields{
+						"pkg":      "fs-store",
+						"filename": filename,
+						"hash":     d,
+						"filepath": fpath,
+					}).Error("failed to open file")
+				}
+			} else {
+				log.WithFields(log.Fields{
+					"pkg":      "fs-store",
+					"filename": filename,
+					"hash":     d,
+					"filepath": fpath,
+					"size":     n,
+				}).Debug("attachment exists on disk")
+			}
+		}
+	} else {
+		log.WithFields(log.Fields{
+			"pkg":      "fs-store",
+			"filename": filename,
+		}).Error("cannot open temp file for attachment", err)
+	}
+	return
+}
+
+// open article given message-id
+// does not check validity
+func (fs FilesystemStorage) OpenArticle(msgid string) (r *os.File, err error) {
+	r, err = os.Open(filepath.Join(fs.ArticleDir(), msgid))
+	return
+}
+
+func (fs FilesystemStorage) ForEachInGroup(group string, chnl chan string) {
+	g := fs.newsgroupDir(group)
+	filepath.Walk(g, func(path string, info os.FileInfo, err error) error {
+		if info != nil {
+			chnl <- info.Name()
+		}
+		return err
+	})
+}
+
+// create a new filesystem storage directory
+// ensure directory and subdirectories
+func NewFilesytemStorage(dirname string, unpackAttachments bool) (fs FilesystemStorage, err error) {
+	dirname, err = filepath.Abs(dirname)
+	if err == nil {
+		log.WithFields(log.Fields{
+			"pkg":      "fs-store",
+			"filepath": dirname,
+		}).Info("Creating New Filesystem Storage")
+		fs = FilesystemStorage{
+			root:               dirname,
+			discardAttachments: unpackAttachments,
+		}
+		err = fs.Ensure()
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/store/log.go

@@ -0,0 +1 @@
+package store

contrib/backends/nntpchand/src/nntpchan/lib/store/null.go

@@ -0,0 +1,62 @@
+package store
+
+import (
+	"io"
+	"nntpchan/lib/util"
+	"os"
+)
+
+type nullStore struct{}
+
+func (n *nullStore) discard(r io.Reader) (s string, err error) {
+	_, err = io.Copy(util.Discard, r)
+	s = "/dev/null"
+	return
+}
+
+func (n *nullStore) HasArticle(msgid string) error {
+	return ErrNoSuchArticle
+}
+
+func (n *nullStore) StoreAttachment(r io.Reader, filename string) (string, error) {
+	return n.discard(r)
+}
+
+func (n *nullStore) StoreArticle(r io.Reader, msgid, newsgroup string) (string, error) {
+	return n.discard(r)
+}
+
+func (n *nullStore) DeleteArticle(msgid string) (err error) {
+	return
+}
+
+func (n *nullStore) Ensure() (err error) {
+	return
+}
+
+func (n *nullStore) ForEachInGroup(newsgroup string, chnl chan string) {
+	return
+}
+
+func (n *nullStore) OpenArticle(msgid string) (r *os.File, err error) {
+	err = ErrNoSuchArticle
+	return
+}
+
+func (n *nullStore) HasNewsgroup(newsgroup string) (has bool, err error) {
+	has = true
+	return
+}
+
+func (n *nullStore) GetAllNewsgroups() (list []string, err error) {
+	return
+}
+
+func (n *nullStore) GetWatermark(newsgroup string) (hi, lo uint64, err error) {
+	return
+}
+
+// create a storage backend that does nothing
+func NewNullStorage() Storage {
+	return &nullStore{}
+}

contrib/backends/nntpchand/src/nntpchan/lib/store/store.go

@@ -0,0 +1,50 @@
+package store
+
+import (
+	"errors"
+	"io"
+	"os"
+)
+
+var ErrNoSuchArticle = errors.New("no such article")
+
+// storage for nntp articles and attachments
+type Storage interface {
+	// store an attachment that we read from an io.Reader
+	// filename is used to hint to store what extension to store it as
+	// returns absolute filepath where attachment was stored and nil on success
+	// returns emtpy string and error if an error ocurred while storing
+	StoreAttachment(r io.Reader, filename string) (string, error)
+
+	// store an article that we read from an io.Reader
+	// message id is used to hint where the article is stored as well as newsgroup
+	// returns absolute filepath to where the article was stored and nil on success
+	// returns empty string and error if an error ocurred while storing
+	StoreArticle(r io.Reader, msgid, newsgroup string) (string, error)
+
+	// return nil if the article with the given message id exists in this storage
+	// return ErrNoSuchArticle if it does not exist or an error if another error occured while checking
+	HasArticle(msgid string) error
+
+	// delete article from underlying storage
+	DeleteArticle(msgid string) error
+
+	// open article for reading
+	OpenArticle(msgid string) (*os.File, error)
+
+	// ensure the underlying storage backend is created
+	Ensure() error
+
+	// iterate over all messages in a newsgroup
+	// send results down a channel
+	ForEachInGroup(newsgroup string, cnhl chan string)
+
+	// get a list of all newsgroups
+	GetAllNewsgroups() ([]string, error)
+
+	// determine if we have a newsgroup
+	HasNewsgroup(newsgroup string) (bool, error)
+
+	// get hi/lo watermark for newsgroup
+	GetWatermark(newsgroup string) (uint64, uint64, error)
+}

contrib/backends/nntpchand/src/nntpchan/lib/thumbnail/doc.go

@@ -0,0 +1,4 @@
+//
+// attachment thumbnailing
+//
+package thumbnail

contrib/backends/nntpchand/src/nntpchan/lib/thumbnail/exec.go

@@ -0,0 +1,40 @@
+package thumbnail
+
+import (
+	"os/exec"
+	"regexp"
+)
+
+// thumbnail by executing an external program
+type ExecThumbnailer struct {
+	// path to executable
+	Exec string
+	// regular expression that checks for acceptable infiles
+	Accept *regexp.Regexp
+	// function to generate arguments to use with external program
+	// inf and outf are the filenames of the input and output files respectively
+	// if this is nil the command will be passed in 2 arguments, infile and outfile
+	GenArgs func(inf, outf string) []string
+}
+
+func (exe *ExecThumbnailer) CanThumbnail(infpath string) bool {
+	re := exe.Accept.Copy()
+	return re.MatchString(infpath)
+}
+
+func (exe *ExecThumbnailer) Generate(infpath, outfpath string) (err error) {
+	// do sanity check
+	if exe.CanThumbnail(infpath) {
+		var args []string
+		if exe.GenArgs == nil {
+			args = []string{infpath, outfpath}
+		} else {
+			args = exe.GenArgs(infpath, outfpath)
+		}
+		cmd := exec.Command(exe.Exec, args...)
+		_, err = cmd.CombinedOutput()
+	} else {
+		err = ErrCannotThumbanil
+	}
+	return
+}

contrib/backends/nntpchand/src/nntpchan/lib/thumbnail/multi.go

@@ -0,0 +1,47 @@
+package thumbnail
+
+import (
+	"errors"
+)
+
+var ErrNoThumbnailer = errors.New("no thumbnailer found")
+
+type multiThumbnailer struct {
+	impls []Thumbnailer
+}
+
+// get the frist matching thumbnailer that works with the given file
+// if we can't find one return nil
+func (mth *multiThumbnailer) getThumbnailer(fpath string) Thumbnailer {
+	for _, th := range mth.impls {
+		if th.CanThumbnail(fpath) {
+			return th
+		}
+	}
+	return nil
+}
+
+func (mth *multiThumbnailer) Generate(infpath, outfpath string) (err error) {
+	th := mth.getThumbnailer(infpath)
+	if th == nil {
+		err = ErrNoThumbnailer
+	} else {
+		err = th.Generate(infpath, outfpath)
+	}
+	return
+}
+
+func (mth *multiThumbnailer) CanThumbnail(infpath string) bool {
+	for _, th := range mth.impls {
+		if th.CanThumbnail(infpath) {
+			return true
+		}
+	}
+	return false
+}
+
+func MuxThumbnailers(th ...Thumbnailer) Thumbnailer {
+	return &multiThumbnailer{
+		impls: th,
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/thumbnail/thumb.go

@@ -0,0 +1,76 @@
+package thumbnail
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var ErrCannotThumbanil = errors.New("cannot thumbnail file")
+
+// a generator of thumbnails
+type Thumbnailer interface {
+	// generate thumbnail of attachment
+	//
+	// infpath: absolute filepath to attachment
+	//
+	// outfpath: absolute filepath to thumbnail
+	//
+	// return error if the thumbnailing fails
+	Generate(infpath, outfpath string) error
+
+	// can we generate a thumbnail for this file?
+	CanThumbnail(infpath string) bool
+}
+
+// thumbnail configuration
+type Config struct {
+	// width of thumbnails
+	ThumbW int
+	// height of thumbnails
+	ThumbH int
+	// only generate jpg thumbnails
+	JpegOnly bool
+}
+
+var defaultCfg = &Config{
+	ThumbW:   300,
+	ThumbH:   200,
+	JpegOnly: true,
+}
+
+// create an imagemagick thumbnailer
+func ImageMagickThumbnailer(convertPath string, conf *Config) Thumbnailer {
+	if conf == nil {
+		conf = defaultCfg
+	}
+	return &ExecThumbnailer{
+		Exec:   convertPath,
+		Accept: regexp.MustCompilePOSIX(`\.(png|jpg|jpeg|gif|webp)$`),
+		GenArgs: func(inf, outf string) []string {
+			if strings.HasSuffix(inf, ".gif") {
+				inf += "[0]"
+			}
+			if conf.JpegOnly {
+				outf += ".jpeg"
+			}
+			return []string{"-thumbnail", fmt.Sprintf("%d", conf.ThumbW), inf, outf}
+		},
+	}
+}
+
+// generate a thumbnailer that uses ffmpeg
+func FFMpegThumbnailer(ffmpegPath string, conf *Config) Thumbnailer {
+	if conf == nil {
+		conf = defaultCfg
+	}
+	return &ExecThumbnailer{
+		Exec:   ffmpegPath,
+		Accept: regexp.MustCompilePOSIX(`\.(mkv|mp4|avi|webm|ogv|mov|m4v|mpg)$`),
+		GenArgs: func(inf, outf string) []string {
+			outf += ".jpeg"
+			return []string{"-i", inf, "-vf", fmt.Sprintf("scale=%d:%d", conf.ThumbW, conf.ThumbH), "-vframes", "1", outf}
+		},
+	}
+}

contrib/backends/nntpchand/src/nntpchan/lib/thumbnail/thumb_test.go

@@ -0,0 +1,56 @@
+package thumbnail
+
+import (
+	"testing"
+)
+
+func doTestThumb(t *testing.T, th Thumbnailer, allowed, disallowed []string) {
+	for _, f := range allowed {
+		if !th.CanThumbnail(f) {
+			t.Logf("cannot thumbnail expected file: %s", f)
+			t.Fail()
+		}
+	}
+
+	for _, f := range disallowed {
+		if th.CanThumbnail(f) {
+			t.Logf("can thumbnail wrong file: %s", f)
+			t.Fail()
+		}
+	}
+
+}
+
+var _image = []string{"asd.gif", "asd.jpeg", "asd.jpg", "asd.png", "asd.webp"}
+var _video = []string{"asd.mkv", "asd.mov", "asd.mp4", "asd.m4v", "asd.ogv", "asd.avi", "asd.mpg", "asd.webm"}
+var _sound = []string{"asd.flac", "asd.mp3", "asd.mp2", "asd.wav", "asd.ogg", "asd.opus", "asd.m4a"}
+var _misc = []string{"asd.txt", "asd.swf"}
+var _garbage = []string{"asd", "asd.asd", "asd.asd.asd.asd", "asd.benis"}
+
+func TestCanThumbnailImage(t *testing.T) {
+	th := ImageMagickThumbnailer("", nil)
+	var allowed []string
+	var disallowed []string
+
+	allowed = append(allowed, _image...)
+	disallowed = append(disallowed, _video...)
+	disallowed = append(disallowed, _sound...)
+	disallowed = append(disallowed, _misc...)
+	disallowed = append(disallowed, _garbage...)
+
+	doTestThumb(t, th, allowed, disallowed)
+}
+
+func TestCanThumbnailVideo(t *testing.T) {
+	th := FFMpegThumbnailer("", nil)
+	var allowed []string
+	var disallowed []string
+
+	allowed = append(allowed, _video...)
+	disallowed = append(disallowed, _image...)
+	disallowed = append(disallowed, _sound...)
+	disallowed = append(disallowed, _misc...)
+	disallowed = append(disallowed, _garbage...)
+
+	doTestThumb(t, th, allowed, disallowed)
+}

contrib/backends/nntpchand/src/nntpchan/lib/util/cache.go

@@ -0,0 +1,87 @@
+package util
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strconv"
+)
+
+func GetThreadHashHTML(file string) (thread string) {
+	exp := regexp.MustCompilePOSIX(`thread-([0-9a-f]+)\.html`)
+	matches := exp.FindStringSubmatch(file)
+	if len(matches) != 2 {
+		return ""
+	}
+	thread = matches[1]
+	return
+}
+
+func GetGroupAndPageHTML(file string) (board string, page int) {
+	exp := regexp.MustCompilePOSIX(`(.*)-([0-9]+)\.html`)
+	matches := exp.FindStringSubmatch(file)
+	if len(matches) != 3 {
+		return "", -1
+	}
+	var err error
+	board = matches[1]
+	tmp := matches[2]
+	page, err = strconv.Atoi(tmp)
+	if err != nil {
+		page = -1
+	}
+	return
+}
+
+func GetGroupForCatalogHTML(file string) (group string) {
+	exp := regexp.MustCompilePOSIX(`catalog-(.+)\.html`)
+	matches := exp.FindStringSubmatch(file)
+	if len(matches) != 2 {
+		return ""
+	}
+	group = matches[1]
+	return
+}
+
+func GetFilenameForBoardPage(webroot_dir, boardname string, pageno int, json bool) string {
+	var ext string
+	if json {
+		ext = "json"
+	} else {
+		ext = "html"
+	}
+	fname := fmt.Sprintf("%s-%d.%s", boardname, pageno, ext)
+	return filepath.Join(webroot_dir, fname)
+}
+
+func GetFilenameForThread(webroot_dir, root_post_id string, json bool) string {
+	var ext string
+	if json {
+		ext = "json"
+	} else {
+		ext = "html"
+	}
+	fname := fmt.Sprintf("thread-%s.%s", HashMessageID(root_post_id), ext)
+	return filepath.Join(webroot_dir, fname)
+}
+
+func GetFilenameForCatalog(webroot_dir, boardname string) string {
+	fname := fmt.Sprintf("catalog-%s.html", boardname)
+	return filepath.Join(webroot_dir, fname)
+}
+
+func GetFilenameForIndex(webroot_dir string) string {
+	return filepath.Join(webroot_dir, "index.html")
+}
+
+func GetFilenameForBoards(webroot_dir string) string {
+	return filepath.Join(webroot_dir, "boards.html")
+}
+
+func GetFilenameForHistory(webroot_dir string) string {
+	return filepath.Join(webroot_dir, "history.html")
+}
+
+func GetFilenameForUkko(webroot_dir string) string {
+	return filepath.Join(webroot_dir, "ukko.html")
+}

contrib/backends/nntpchand/src/nntpchan/lib/util/discard.go

@@ -0,0 +1,14 @@
+package util
+
+type ioDiscard struct{}
+
+func (discard *ioDiscard) Write(d []byte) (n int, err error) {
+	n = len(d)
+	return
+}
+
+func (discard *ioDiscard) Close() (err error) {
+	return
+}
+
+var Discard = new(ioDiscard)