Archived
1
0

add refactored rewrite

This commit is contained in:
Jeff Becker 2017-04-16 14:11:11 -04:00
parent c965f0f7af
commit e9e46e1f59
211 changed files with 31441 additions and 0 deletions

View File

@ -0,0 +1,11 @@
REPO=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
all: clean build
build: nntpchand
nntpchand:
GOPATH=$(REPO) go build -v
clean:
GOPATH=$(REPO) go clean -v

View File

@ -0,0 +1,9 @@
package main
import (
"nntpchan/cmd/nntpchan"
)
func main() {
nntpchan.Main()
}

View File

@ -0,0 +1,160 @@
package nntpchan
import (
log "github.com/Sirupsen/logrus"
"net"
_ "net/http/pprof"
"nntpchan/lib/config"
"nntpchan/lib/database"
"nntpchan/lib/frontend"
"nntpchan/lib/nntp"
"nntpchan/lib/store"
"nntpchan/lib/webhooks"
"os"
"os/signal"
"syscall"
"time"
)
type runStatus struct {
nntpListener net.Listener
run bool
done chan error
}
func (st *runStatus) Stop() {
st.run = false
if st.nntpListener != nil {
st.nntpListener.Close()
}
st.nntpListener = nil
log.Info("stopping daemon process")
}
func Main() {
st := &runStatus{
run: true,
done: make(chan error),
}
log.Info("starting up nntpchan...")
cfgFname := "nntpchan.json"
conf, err := config.Ensure(cfgFname)
if err != nil {
log.Fatal(err)
}
if conf.Log == "debug" {
log.SetLevel(log.DebugLevel)
}
sconfig := conf.Store
if sconfig == nil {
log.Fatal("no article storage configured")
}
nconfig := conf.NNTP
if nconfig == nil {
log.Fatal("no nntp server configured")
}
dconfig := conf.Database
if dconfig == nil {
log.Fatal("no database configured")
}
// create nntp server
nserv := nntp.NewServer()
nserv.Config = nconfig
nserv.Feeds = conf.Feeds
if nconfig.LoginsFile != "" {
nserv.Auth = nntp.FlatfileAuth(nconfig.LoginsFile)
}
// create article storage
nserv.Storage, err = store.NewFilesytemStorage(sconfig.Path, true)
if err != nil {
log.Fatal(err)
}
if conf.WebHooks != nil && len(conf.WebHooks) > 0 {
// put webhooks into nntp server event hooks
nserv.Hooks = webhooks.NewWebhooks(conf.WebHooks, nserv.Storage)
}
if conf.NNTPHooks != nil && len(conf.NNTPHooks) > 0 {
var hooks nntp.MulitHook
if nserv.Hooks != nil {
hooks = append(hooks, nserv.Hooks)
}
for _, h := range conf.NNTPHooks {
hooks = append(hooks, nntp.NewHook(h))
}
nserv.Hooks = hooks
}
var db database.Database
for _, fconf := range conf.Frontends {
var f frontend.Frontend
f, err = frontend.NewHTTPFrontend(fconf, db)
if err == nil {
go f.Serve()
}
}
// start persisting feeds
go nserv.PersistFeeds()
// handle signals
sigchnl := make(chan os.Signal, 1)
signal.Notify(sigchnl, syscall.SIGHUP, os.Interrupt)
go func() {
for {
s := <-sigchnl
if s == syscall.SIGHUP {
// handle SIGHUP
conf, err := config.Ensure(cfgFname)
if err == nil {
log.Infof("reloading config: %s", cfgFname)
nserv.ReloadServer(conf.NNTP)
nserv.ReloadFeeds(conf.Feeds)
} else {
log.Errorf("failed to reload config: %s", err)
}
} else if s == os.Interrupt {
// handle interrupted, clean close
st.Stop()
return
}
}
}()
go func() {
var err error
for st.run {
var nl net.Listener
naddr := conf.NNTP.Bind
log.Infof("Bind nntp server to %s", naddr)
nl, err = net.Listen("tcp", naddr)
if err == nil {
st.nntpListener = nl
err = nserv.Serve(nl)
if err != nil {
nl.Close()
log.Errorf("nntpserver.serve() %s", err.Error())
}
} else {
log.Errorf("nntp server net.Listen failed: %s", err.Error())
}
time.Sleep(time.Second)
}
st.done <- err
}()
e := <-st.done
if e != nil {
log.Fatal(e)
}
log.Info("ended")
}

View File

@ -0,0 +1,42 @@
package main
// simple nntp server
import (
log "github.com/Sirupsen/logrus"
"github.com/majestrate/srndv2/lib/config"
"github.com/majestrate/srndv2/lib/nntp"
"github.com/majestrate/srndv2/lib/store"
"net"
)
func main() {
log.Info("starting NNTP server...")
conf, err := config.Ensure("settings.json")
if err != nil {
log.Fatal(err)
}
if conf.Log == "debug" {
log.SetLevel(log.DebugLevel)
}
serv := &nntp.Server{
Config: conf.NNTP,
Feeds: conf.Feeds,
}
serv.Storage, err = store.NewFilesytemStorage(conf.Store.Path, false)
if err != nil {
log.Fatal(err)
}
l, err := net.Listen("tcp", conf.NNTP.Bind)
if err != nil {
log.Fatal(err)
}
log.Info("listening on ", l.Addr())
err = serv.Serve(l)
if err != nil {
log.Fatal(err)
}
}

View File

@ -0,0 +1,4 @@
//
// server admin panel
//
package admin

View File

@ -0,0 +1,16 @@
package admin
import (
"net/http"
)
type Server struct {
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
func NewServer() *Server {
return &Server{}
}

View File

@ -0,0 +1,10 @@
package api
import (
"nntpchan/lib/model"
)
// json api
type API interface {
MakePost(p model.Post)
}

View File

@ -0,0 +1,2 @@
// json api
package api

View File

@ -0,0 +1,20 @@
package api
import (
"github.com/gorilla/mux"
"net/http"
)
// api server
type Server struct {
}
func (s *Server) HandlePing(w http.ResponseWriter, r *http.Request) {
}
// inject api routes
func (s *Server) SetupRoutes(r *mux.Router) {
// setup api pinger
r.Path("/ping").HandlerFunc(s.HandlePing)
}

View File

@ -0,0 +1,73 @@
package config
import "regexp"
// configration for local article policies
type ArticleConfig struct {
// explicitly allow these newsgroups (regexp)
AllowGroups []string `json:"whitelist"`
// explicitly disallow these newsgroups (regexp)
DisallowGroups []string `json:"blacklist"`
// only allow explicitly allowed groups
ForceWhitelist bool `json:"force-whitelist"`
// allow anonymous posts?
AllowAnon bool `json:"anon"`
// allow attachments?
AllowAttachments bool `json:"attachments"`
// allow anonymous attachments?
AllowAnonAttachments bool `json:"anon-attachments"`
}
func (c *ArticleConfig) AllowGroup(group string) bool {
for _, g := range c.DisallowGroups {
r := regexp.MustCompile(g)
if r.MatchString(group) && c.ForceWhitelist {
// disallowed
return false
}
}
// check allowed groups first
for _, g := range c.AllowGroups {
r := regexp.MustCompile(g)
if r.MatchString(g) {
return true
}
}
return !c.ForceWhitelist
}
// allow an article?
func (c *ArticleConfig) Allow(msgid, group string, anon, attachment bool) bool {
// check attachment policy
if c.AllowGroup(group) {
allow := true
// no anon ?
if anon && !c.AllowAnon {
allow = false
}
// no attachments ?
if allow && attachment && !c.AllowAttachments {
allow = false
}
// no anon attachments ?
if allow && attachment && anon && !c.AllowAnonAttachments {
allow = false
}
return allow
} else {
return false
}
}
var DefaultArticlePolicy = ArticleConfig{
AllowGroups: []string{"ctl", "overchan.test"},
DisallowGroups: []string{"overchan.cp"},
ForceWhitelist: false,
AllowAnon: true,
AllowAttachments: true,
AllowAnonAttachments: false,
}

View File

@ -0,0 +1,13 @@
package config
// caching interface configuration
type CacheConfig struct {
// backend cache driver name
Backend string `json:"backend"`
// address for cache
Addr string `json:"addr"`
// username for login
User string `json:"user"`
// password for login
Password string `json:"password"`
}

View File

@ -0,0 +1,87 @@
package config
import (
"bytes"
"encoding/json"
"io/ioutil"
"os"
)
// main configuration
type Config struct {
// nntp server configuration
NNTP *NNTPServerConfig `json:"nntp"`
// log level
Log string `json:"log"`
// article storage config
Store *StoreConfig `json:"storage"`
// web hooks to call
WebHooks []*WebhookConfig `json:"webhooks"`
// external scripts to call
NNTPHooks []*NNTPHookConfig `json:"nntphooks"`
// database backend configuration
Database *DatabaseConfig `json:"db"`
// list of feeds to add on runtime
Feeds []*FeedConfig `json:"feeds"`
// frontend config
Frontends []*FrontendConfig `json:"frontends"`
// unexported fields ...
// absolute filepath to configuration
fpath string
}
// default configuration
var DefaultConfig = Config{
Store: &DefaultStoreConfig,
NNTP: &DefaultNNTPConfig,
Database: &DefaultDatabaseConfig,
WebHooks: []*WebhookConfig{DefaultWebHookConfig},
NNTPHooks: []*NNTPHookConfig{DefaultNNTPHookConfig},
Feeds: DefaultFeeds,
Frontends: []*FrontendConfig{&DefaultFrontendConfig},
Log: "debug",
}
// reload configuration
func (c *Config) Reload() (err error) {
var b []byte
b, err = ioutil.ReadFile(c.fpath)
if err == nil {
err = json.Unmarshal(b, c)
}
return
}
// ensure that a config file exists
// creates one if it does not exist
func Ensure(fname string) (cfg *Config, err error) {
_, err = os.Stat(fname)
if os.IsNotExist(err) {
err = nil
var d []byte
d, err = json.Marshal(&DefaultConfig)
if err == nil {
b := new(bytes.Buffer)
err = json.Indent(b, d, "", " ")
if err == nil {
err = ioutil.WriteFile(fname, b.Bytes(), 0600)
}
}
}
if err == nil {
cfg, err = Load(fname)
}
return
}
// load configuration file
func Load(fname string) (cfg *Config, err error) {
cfg = new(Config)
cfg.fpath = fname
err = cfg.Reload()
if err != nil {
cfg = nil
}
return
}

View File

@ -0,0 +1,18 @@
package config
type DatabaseConfig struct {
// url or address for database connector
Addr string `json:"addr"`
// password to use
Password string `json:"password"`
// username to use
Username string `json:"username"`
// type of database to use
Type string `json:"type"`
}
var DefaultDatabaseConfig = DatabaseConfig{
Type: "postgres",
Addr: "/var/run/postgresql",
Password: "",
}

View File

@ -0,0 +1,4 @@
//
// package for parsing config files
//
package config

View File

@ -0,0 +1,33 @@
package config
// configuration for 1 nntp feed
type FeedConfig struct {
// feed's policy, filters articles
Policy *ArticleConfig `json:"policy"`
// remote server's address
Addr string `json:"addr"`
// proxy server config
Proxy *ProxyConfig `json:"proxy"`
// nntp username to log in with
Username string `json:"username"`
// nntp password to use when logging in
Password string `json:"password"`
// do we want to use tls?
TLS bool `json:"tls"`
// the name of this feed
Name string `json:"name"`
// how often to pull articles from the server in minutes
// 0 for never
PullInterval int `json:"pull"`
}
var DuummyFeed = FeedConfig{
Policy: &DefaultArticlePolicy,
Addr: "nntp.dummy.tld:1119",
Proxy: &DefaultTorProxy,
Name: "dummy",
}
var DefaultFeeds = []*FeedConfig{
&DuummyFeed,
}

View File

@ -0,0 +1,21 @@
package config
type FrontendConfig struct {
// bind to address
BindAddr string `json:"bind"`
// frontend cache
Cache *CacheConfig `json:"cache"`
// frontend ssl settings
SSL *SSLSettings `json:"ssl"`
// static files directory
Static string `json:"static_dir"`
// http middleware configuration
Middleware *MiddlewareConfig `json:"middleware"`
}
// default Frontend Configuration
var DefaultFrontendConfig = FrontendConfig{
BindAddr: "127.0.0.1:18888",
Static: "./files/static/",
Middleware: &DefaultMiddlewareConfig,
}

View File

@ -0,0 +1,15 @@
package config
// config for external callback for nntp articles
type NNTPHookConfig struct {
// name of hook
Name string `json:"name"`
// executable script path to be called with arguments: /path/to/article
Exec string `json:"exec"`
}
// default dummy hook
var DefaultNNTPHookConfig = &NNTPHookConfig{
Name: "dummy",
Exec: "/bin/true",
}

View File

@ -0,0 +1,14 @@
package config
// configuration for http middleware
type MiddlewareConfig struct {
// middleware type, currently just 1 is available: overchan
Type string `json:"type"`
// directory for our html templates
Templates string `json:"templates_dir"`
}
var DefaultMiddlewareConfig = MiddlewareConfig{
Type: "overchan",
Templates: "./files/templates/overchan/",
}

View File

@ -0,0 +1,24 @@
package config
type NNTPServerConfig struct {
// address to bind to
Bind string `json:"bind"`
// name of the nntp server
Name string `json:"name"`
// default inbound article policy
Article *ArticleConfig `json:"policy"`
// do we allow anonymous NNTP sync?
AnonNNTP bool `json:"anon-nntp"`
// ssl settings for nntp
SSL *SSLSettings
// file with login credentials
LoginsFile string `json:"authfile"`
}
var DefaultNNTPConfig = NNTPServerConfig{
AnonNNTP: false,
Bind: "0.0.0.0:1119",
Name: "nntp.server.tld",
Article: &DefaultArticlePolicy,
LoginsFile: "",
}

View File

@ -0,0 +1,13 @@
package config
// proxy configuration
type ProxyConfig struct {
Type string `json:"type"`
Addr string `json:"addr"`
}
// default tor proxy
var DefaultTorProxy = ProxyConfig{
Type: "socks",
Addr: "127.0.0.1:9050",
}

View File

@ -0,0 +1,11 @@
package config
// settings for setting up ssl
type SSLSettings struct {
// path to ssl private key
SSLKeyFile string `json:"key"`
// path to ssl certificate signed by CA
SSLCertFile string `json:"cert"`
// domain name to use for ssl
DomainName string `json:"fqdn"`
}

View File

@ -0,0 +1,10 @@
package config
type StoreConfig struct {
// path to article directory
Path string `json:"path"`
}
var DefaultStoreConfig = StoreConfig{
Path: "storage",
}

View File

@ -0,0 +1,17 @@
package config
// configuration for a single web hook
type WebhookConfig struct {
// user provided name for this hook
Name string `json:"name"`
// callback URL for webhook
URL string `json:"url"`
// dialect to use when calling webhook
Dialect string `json:"dialect"`
}
var DefaultWebHookConfig = &WebhookConfig{
Name: "vichan",
Dialect: "vichan",
URL: "http://localhost/webhook.php",
}

View File

@ -0,0 +1,5 @@
//
// nntpchan crypto package
// wraps all external crypro libs
//
package crypto

View File

@ -0,0 +1,8 @@
package crypto
import (
"github.com/dchest/blake256"
)
// common hash function is blake2
var Hash = blake256.New

View File

@ -0,0 +1,82 @@
package crypto
import (
"crypto/sha512"
"hash"
"nntpchan/lib/crypto/nacl"
)
type fuckyNacl struct {
k []byte
hash hash.Hash
}
func (fucky *fuckyNacl) Write(d []byte) (int, error) {
return fucky.hash.Write(d)
}
func (fucky *fuckyNacl) Sign() (s Signature) {
h := fucky.hash.Sum(nil)
if h == nil {
panic("fuck.hash.Sum == nil")
}
kp := nacl.LoadSignKey(fucky.k)
defer kp.Free()
sk := kp.Secret()
sig := nacl.CryptoSignFucky(h, sk)
if sig == nil {
panic("fucky signer's call to nacl.CryptoSignFucky returned nil")
}
s = Signature(sig)
fucky.resetState()
return
}
// reset inner state so we can reuse this fuckyNacl for another operation
func (fucky *fuckyNacl) resetState() {
fucky.hash = sha512.New()
}
func (fucky *fuckyNacl) Verify(sig Signature) (valid bool) {
h := fucky.hash.Sum(nil)
if h == nil {
panic("fuck.hash.Sum == nil")
}
valid = nacl.CryptoVerifyFucky(h, sig, fucky.k)
fucky.resetState()
return
}
func createFucky(k []byte) *fuckyNacl {
return &fuckyNacl{
k: k,
hash: sha512.New(),
}
}
// create a standard signer given a secret key
func CreateSigner(sk []byte) Signer {
return createFucky(sk)
}
// create a standard verifier given a public key
func CreateVerifier(pk []byte) Verifer {
return createFucky(pk)
}
// get the public component given the secret key
func ToPublic(sk []byte) (pk []byte) {
kp := nacl.LoadSignKey(sk)
defer kp.Free()
pk = kp.Public()
return
}
// create a standard keypair
func GenKeypair() (pk, sk []byte) {
kp := nacl.GenSignKeypair()
defer kp.Free()
pk = kp.Public()
sk = kp.Seed()
return
}

View File

@ -0,0 +1,95 @@
package nacl
// #cgo freebsd CFLAGS: -I/usr/local/include
// #cgo freebsd LDFLAGS: -L/usr/local/lib
// #cgo LDFLAGS: -lsodium
// #include <sodium.h>
import "C"
import (
"errors"
)
// encrypts a message to a user given their public key is known
// returns an encrypted box
func CryptoBox(msg, nounce, pk, sk []byte) ([]byte, error) {
msgbuff := NewBuffer(msg)
defer msgbuff.Free()
// check sizes
if len(pk) != int(C.crypto_box_publickeybytes()) {
err := errors.New("len(pk) != crypto_box_publickey_bytes")
return nil, err
}
if len(sk) != int(C.crypto_box_secretkeybytes()) {
err := errors.New("len(sk) != crypto_box_secretkey_bytes")
return nil, err
}
if len(nounce) != int(C.crypto_box_macbytes()) {
err := errors.New("len(nounce) != crypto_box_macbytes()")
return nil, err
}
pkbuff := NewBuffer(pk)
defer pkbuff.Free()
skbuff := NewBuffer(sk)
defer skbuff.Free()
nouncebuff := NewBuffer(nounce)
defer nouncebuff.Free()
resultbuff := malloc(msgbuff.size + nouncebuff.size)
defer resultbuff.Free()
res := C.crypto_box_easy(resultbuff.uchar(), msgbuff.uchar(), C.ulonglong(msgbuff.size), nouncebuff.uchar(), pkbuff.uchar(), skbuff.uchar())
if res != 0 {
err := errors.New("crypto_box_easy failed")
return nil, err
}
return resultbuff.Bytes(), nil
}
// open an encrypted box
func CryptoBoxOpen(box, nounce, sk, pk []byte) ([]byte, error) {
boxbuff := NewBuffer(box)
defer boxbuff.Free()
// check sizes
if len(pk) != int(C.crypto_box_publickeybytes()) {
err := errors.New("len(pk) != crypto_box_publickey_bytes")
return nil, err
}
if len(sk) != int(C.crypto_box_secretkeybytes()) {
err := errors.New("len(sk) != crypto_box_secretkey_bytes")
return nil, err
}
if len(nounce) != int(C.crypto_box_macbytes()) {
err := errors.New("len(nounce) != crypto_box_macbytes()")
return nil, err
}
pkbuff := NewBuffer(pk)
defer pkbuff.Free()
skbuff := NewBuffer(sk)
defer skbuff.Free()
nouncebuff := NewBuffer(nounce)
defer nouncebuff.Free()
resultbuff := malloc(boxbuff.size - nouncebuff.size)
defer resultbuff.Free()
// decrypt
res := C.crypto_box_open_easy(resultbuff.uchar(), boxbuff.uchar(), C.ulonglong(boxbuff.size), nouncebuff.uchar(), pkbuff.uchar(), skbuff.uchar())
if res != 0 {
return nil, errors.New("crypto_box_open_easy failed")
}
// return result
return resultbuff.Bytes(), nil
}
// generate a new nounce
func NewBoxNounce() []byte {
return RandBytes(NounceLen())
}
// length of a nounce
func NounceLen() int {
return int(C.crypto_box_macbytes())
}

View File

@ -0,0 +1,86 @@
package nacl
// #cgo freebsd CFLAGS: -I/usr/local/include
// #cgo freebsd LDFLAGS: -L/usr/local/lib
// #cgo LDFLAGS: -lsodium
// #include <sodium.h>
//
// unsigned char * deref_uchar(void * ptr) { return (unsigned char*) ptr; }
//
import "C"
import (
"encoding/hex"
"reflect"
"unsafe"
)
// wrapper arround malloc/free
type Buffer struct {
ptr unsafe.Pointer
length C.int
size C.size_t
}
// wrapper arround nacl.malloc
func Malloc(size int) *Buffer {
if size > 0 {
return malloc(C.size_t(size))
}
return nil
}
// does not check for negatives
func malloc(size C.size_t) *Buffer {
ptr := C.malloc(size)
C.sodium_memzero(ptr, size)
buffer := &Buffer{ptr: ptr, size: size, length: C.int(size)}
return buffer
}
// create a new buffer copying from a byteslice
func NewBuffer(buff []byte) *Buffer {
buffer := Malloc(len(buff))
if buffer == nil {
return nil
}
if copy(buffer.Data(), buff) != len(buff) {
return nil
}
return buffer
}
func (self *Buffer) uchar() *C.uchar {
return C.deref_uchar(self.ptr)
}
func (self *Buffer) Length() int {
return int(self.length)
}
// get immutable byte slice
func (self *Buffer) Bytes() []byte {
buff := make([]byte, self.Length())
copy(buff, self.Data())
return buff
}
// get underlying byte slice
func (self *Buffer) Data() []byte {
hdr := reflect.SliceHeader{
Data: uintptr(self.ptr),
Len: self.Length(),
Cap: self.Length(),
}
return *(*[]byte)(unsafe.Pointer(&hdr))
}
func (self *Buffer) String() string {
return hex.EncodeToString(self.Data())
}
// zero out memory and then free
func (self *Buffer) Free() {
C.sodium_memzero(self.ptr, self.size)
C.free(self.ptr)
}

View File

@ -0,0 +1,178 @@
package nacl
// #cgo freebsd CFLAGS: -I/usr/local/include
// #cgo freebsd LDFLAGS: -L/usr/local/lib
// #cgo LDFLAGS: -lsodium
// #include <sodium.h>
import "C"
import (
"encoding/hex"
"errors"
"fmt"
)
type KeyPair struct {
pk *Buffer
sk *Buffer
}
// free this keypair from memory
func (self *KeyPair) Free() {
self.pk.Free()
self.sk.Free()
}
func (self *KeyPair) Secret() []byte {
return self.sk.Bytes()
}
func (self *KeyPair) Public() []byte {
return self.pk.Bytes()
}
func (self *KeyPair) Seed() []byte {
seed_len := C.crypto_sign_seedbytes()
return self.sk.Bytes()[:seed_len]
}
// generate a keypair
func GenSignKeypair() *KeyPair {
sk_len := C.crypto_sign_secretkeybytes()
sk := malloc(sk_len)
pk_len := C.crypto_sign_publickeybytes()
pk := malloc(pk_len)
res := C.crypto_sign_keypair(pk.uchar(), sk.uchar())
if res == 0 {
return &KeyPair{pk, sk}
}
pk.Free()
sk.Free()
return nil
}
// get public key from secret key
func GetSignPubkey(sk []byte) ([]byte, error) {
sk_len := C.crypto_sign_secretkeybytes()
if C.size_t(len(sk)) != sk_len {
return nil, errors.New(fmt.Sprintf("nacl.GetSignPubkey() invalid secret key size %d != %d", len(sk), sk_len))
}
pk_len := C.crypto_sign_publickeybytes()
pkbuff := malloc(pk_len)
defer pkbuff.Free()
skbuff := NewBuffer(sk)
defer skbuff.Free()
//XXX: hack
res := C.crypto_sign_seed_keypair(pkbuff.uchar(), skbuff.uchar(), skbuff.uchar())
if res != 0 {
return nil, errors.New(fmt.Sprintf("nacl.GetSignPubkey() failed to get public key from secret key: %d", res))
}
return pkbuff.Bytes(), nil
}
// make keypair from seed
func LoadSignKey(seed []byte) *KeyPair {
seed_len := C.crypto_sign_seedbytes()
if C.size_t(len(seed)) != seed_len {
return nil
}
seedbuff := NewBuffer(seed)
defer seedbuff.Free()
pk_len := C.crypto_sign_publickeybytes()
sk_len := C.crypto_sign_secretkeybytes()
pkbuff := malloc(pk_len)
skbuff := malloc(sk_len)
res := C.crypto_sign_seed_keypair(pkbuff.uchar(), skbuff.uchar(), seedbuff.uchar())
if res != 0 {
pkbuff.Free()
skbuff.Free()
return nil
}
return &KeyPair{pkbuff, skbuff}
}
func GenBoxKeypair() *KeyPair {
sk_len := C.crypto_box_secretkeybytes()
sk := malloc(sk_len)
pk_len := C.crypto_box_publickeybytes()
pk := malloc(pk_len)
res := C.crypto_box_keypair(pk.uchar(), sk.uchar())
if res == 0 {
return &KeyPair{pk, sk}
}
pk.Free()
sk.Free()
return nil
}
// get public key from secret key
func GetBoxPubkey(sk []byte) []byte {
sk_len := C.crypto_box_seedbytes()
if C.size_t(len(sk)) != sk_len {
return nil
}
pk_len := C.crypto_box_publickeybytes()
pkbuff := malloc(pk_len)
defer pkbuff.Free()
skbuff := NewBuffer(sk)
defer skbuff.Free()
// compute the public key
C.crypto_scalarmult_base(pkbuff.uchar(), skbuff.uchar())
return pkbuff.Bytes()
}
// load keypair from secret key
func LoadBoxKey(sk []byte) *KeyPair {
pk := GetBoxPubkey(sk)
if pk == nil {
return nil
}
pkbuff := NewBuffer(pk)
skbuff := NewBuffer(sk)
return &KeyPair{pkbuff, skbuff}
}
// make keypair from seed
func SeedBoxKey(seed []byte) *KeyPair {
seed_len := C.crypto_box_seedbytes()
if C.size_t(len(seed)) != seed_len {
return nil
}
seedbuff := NewBuffer(seed)
defer seedbuff.Free()
pk_len := C.crypto_box_publickeybytes()
sk_len := C.crypto_box_secretkeybytes()
pkbuff := malloc(pk_len)
skbuff := malloc(sk_len)
res := C.crypto_box_seed_keypair(pkbuff.uchar(), skbuff.uchar(), seedbuff.uchar())
if res != 0 {
pkbuff.Free()
skbuff.Free()
return nil
}
return &KeyPair{pkbuff, skbuff}
}
func (self *KeyPair) String() string {
return fmt.Sprintf("pk=%s sk=%s", hex.EncodeToString(self.pk.Data()), hex.EncodeToString(self.sk.Data()))
}
func CryptoSignPublicLen() int {
return int(C.crypto_sign_publickeybytes())
}
func CryptoSignSecretLen() int {
return int(C.crypto_sign_secretkeybytes())
}
func CryptoSignSeedLen() int {
return int(C.crypto_sign_seedbytes())
}

View File

@ -0,0 +1,44 @@
package nacl
// #cgo freebsd CFLAGS: -I/usr/local/include
// #cgo freebsd LDFLAGS: -L/usr/local/lib
// #cgo LDFLAGS: -lsodium
// #include <sodium.h>
import "C"
import (
"log"
)
// return how many bytes overhead does CryptoBox have
func CryptoBoxOverhead() int {
return int(C.crypto_box_macbytes())
}
// size of crypto_box public keys
func CryptoBoxPubKeySize() int {
return int(C.crypto_box_publickeybytes())
}
// size of crypto_box private keys
func CryptoBoxPrivKeySize() int {
return int(C.crypto_box_secretkeybytes())
}
// size of crypto_sign public keys
func CryptoSignPubKeySize() int {
return int(C.crypto_sign_publickeybytes())
}
// size of crypto_sign private keys
func CryptoSignPrivKeySize() int {
return int(C.crypto_sign_secretkeybytes())
}
// initialize sodium
func init() {
status := C.sodium_init()
if status == -1 {
log.Fatalf("failed to initialize libsodium status=%d", status)
}
}

View File

@ -0,0 +1,24 @@
package nacl
// #cgo freebsd CFLAGS: -I/usr/local/include
// #cgo freebsd LDFLAGS: -L/usr/local/lib
// #cgo LDFLAGS: -lsodium
// #include <sodium.h>
import "C"
func randbytes(size C.size_t) *Buffer {
buff := malloc(size)
C.randombytes_buf(buff.ptr, size)
return buff
}
func RandBytes(size int) []byte {
if size > 0 {
buff := randbytes(C.size_t(size))
defer buff.Free()
return buff.Bytes()
}
return nil
}

View File

@ -0,0 +1,58 @@
package nacl
// #cgo freebsd CFLAGS: -I/usr/local/include
// #cgo freebsd LDFLAGS: -L/usr/local/lib
// #cgo LDFLAGS: -lsodium
// #include <sodium.h>
import "C"
// sign data detached with secret key sk
func CryptoSignDetached(msg, sk []byte) []byte {
msgbuff := NewBuffer(msg)
defer msgbuff.Free()
skbuff := NewBuffer(sk)
defer skbuff.Free()
if skbuff.size != C.crypto_sign_bytes() {
return nil
}
// allocate the signature buffer
sig := malloc(C.crypto_sign_bytes())
defer sig.Free()
// compute signature
siglen := C.ulonglong(0)
res := C.crypto_sign_detached(sig.uchar(), &siglen, msgbuff.uchar(), C.ulonglong(msgbuff.size), skbuff.uchar())
if res == 0 && siglen == C.ulonglong(C.crypto_sign_bytes()) {
// return copy of signature buffer
return sig.Bytes()
}
// failure to sign
return nil
}
// sign data with secret key sk
// return detached sig
// this uses crypto_sign instead pf crypto_sign_detached
func CryptoSignFucky(msg, sk []byte) []byte {
msgbuff := NewBuffer(msg)
defer msgbuff.Free()
skbuff := NewBuffer(sk)
defer skbuff.Free()
if skbuff.size != C.crypto_sign_bytes() {
return nil
}
// allocate the signed message buffer
sig := malloc(C.crypto_sign_bytes() + msgbuff.size)
defer sig.Free()
// compute signature
siglen := C.ulonglong(0)
res := C.crypto_sign(sig.uchar(), &siglen, msgbuff.uchar(), C.ulonglong(msgbuff.size), skbuff.uchar())
if res == 0 {
// return copy of signature inside the signed message
offset := int(C.crypto_sign_bytes())
return sig.Bytes()[:offset]
}
// failure to sign
return nil
}

View File

@ -0,0 +1,342 @@
package nacl
import (
"bytes"
"errors"
"io"
"net"
"time"
)
// TOY encrypted authenticated stream protocol like tls
var BadHandshake = errors.New("Bad handshake")
var ShortWrite = errors.New("short write")
var ShortRead = errors.New("short read")
var Closed = errors.New("socket closed")
// write boxes at 512 bytes at a time
const DefaultMTU = 512
// wrapper arround crypto_box
// provides an authenticated encrypted stream
// this is a TOY
type CryptoStream struct {
// underlying stream to write on
stream io.ReadWriteCloser
// secret key seed
key *KeyPair
// public key of who we expect on the other end
remote_pk []byte
tx_nonce []byte
rx_nonce []byte
// box size
mtu int
}
func (cs *CryptoStream) Close() (err error) {
if cs.key != nil {
cs.key.Free()
cs.key = nil
}
return cs.stream.Close()
}
// implements io.Writer
func (cs *CryptoStream) Write(data []byte) (n int, err error) {
// let's split it up
for n < len(data) && err == nil {
if n+cs.mtu < len(data) {
err = cs.writeSegment(data[n : n+cs.mtu])
n += cs.mtu
} else {
err = cs.writeSegment(data[n:])
if err == nil {
n = len(data)
}
}
}
return
}
func (cs *CryptoStream) public() (p []byte) {
p = cs.key.Public()
return
}
func (cs *CryptoStream) secret() (s []byte) {
s = cs.key.Secret()
return
}
// read 1 segment
func (cs *CryptoStream) readSegment() (s []byte, err error) {
var stream_read int
var seg []byte
nl := NounceLen()
msg := make([]byte, cs.mtu+nl)
stream_read, err = cs.stream.Read(msg)
seg, err = CryptoBoxOpen(msg[:stream_read], cs.rx_nonce, cs.secret(), cs.remote_pk)
if err == nil {
copy(cs.rx_nonce, seg[:nl])
s = seg[nl:]
}
return
}
// write 1 segment encrypted
// update nounce
func (cs *CryptoStream) writeSegment(data []byte) (err error) {
var segment []byte
nl := NounceLen()
msg := make([]byte, len(data)+nl)
// generate next nounce
nextNounce := NewBoxNounce()
copy(msg, nextNounce)
copy(msg[nl:], data)
// encrypt segment with current nounce
segment, err = CryptoBox(data, cs.tx_nonce, cs.remote_pk, cs.secret())
var n int
n, err = cs.stream.Write(segment)
if n != len(segment) {
// short write?
err = ShortWrite
return
}
// update nounce
copy(cs.tx_nonce, nextNounce)
return
}
// implements io.Reader
func (cs *CryptoStream) Read(data []byte) (n int, err error) {
var seg []byte
seg, err = cs.readSegment()
if err == nil {
if len(seg) <= len(data) {
copy(data, seg)
n = len(seg)
} else {
// too big?
err = ShortRead
}
}
return
}
// version 0 protocol magic
var protocol_magic = []byte("BENIS|00")
// verify that a handshake is signed right and is in the correct format etc
func verifyHandshake(hs, pk []byte) (valid bool) {
ml := len(protocol_magic)
// valid handshake?
if bytes.Equal(hs[0:ml], protocol_magic) {
// check pk
pl := CryptoSignPublicLen()
nl := NounceLen()
if bytes.Equal(pk, hs[ml:ml+pl]) {
// check signature
msg := hs[0 : ml+pl+nl]
sig := hs[ml+pl+nl:]
valid = CryptoVerifyFucky(msg, sig, pk)
}
}
return
}
// get claimed public key from handshake
func getPubkey(hs []byte) (pk []byte) {
ml := len(protocol_magic)
pl := CryptoSignPublicLen()
pk = hs[ml : ml+pl]
return
}
func (cs *CryptoStream) genHandshake() (d []byte) {
// protocol magic string version 00
// Benis Encrypted Network Information Stream
// :-DDDDD meme crypto
d = append(d, protocol_magic...)
// our public key
d = append(d, cs.public()...)
// nounce
cs.tx_nonce = NewBoxNounce()
d = append(d, cs.tx_nonce...)
// sign protocol magic string, nounce and pubkey
sig := CryptoSignFucky(d, cs.secret())
// if sig is nil we'll just die
d = append(d, sig...)
return
}
// extract nounce from handshake
func getNounce(hs []byte) (n []byte) {
ml := len(protocol_magic)
pl := CryptoSignPublicLen()
nl := NounceLen()
n = hs[ml+pl : ml+pl+nl]
return
}
// initiate protocol handshake
func (cs *CryptoStream) Handshake() (err error) {
// send them our info
hs := cs.genHandshake()
var n int
n, err = cs.stream.Write(hs)
if n != len(hs) {
err = ShortWrite
return
}
// read thier info
buff := make([]byte, len(hs))
_, err = io.ReadFull(cs.stream, buff)
if cs.remote_pk == nil {
// inbound
pk := getPubkey(buff)
cs.remote_pk = make([]byte, len(pk))
copy(cs.remote_pk, pk)
}
if !verifyHandshake(buff, cs.remote_pk) {
// verification failed
err = BadHandshake
return
}
cs.rx_nonce = make([]byte, NounceLen())
copy(cs.rx_nonce, getNounce(buff))
return
}
// create a client
func Client(stream io.ReadWriteCloser, local_sk, remote_pk []byte) (c *CryptoStream) {
c = &CryptoStream{
stream: stream,
mtu: DefaultMTU,
}
c.remote_pk = make([]byte, len(remote_pk))
copy(c.remote_pk, remote_pk)
c.key = LoadSignKey(local_sk)
if c.key == nil {
return nil
}
return c
}
type CryptoConn struct {
stream *CryptoStream
conn net.Conn
}
func (cc *CryptoConn) Close() (err error) {
err = cc.stream.Close()
return
}
func (cc *CryptoConn) Write(d []byte) (n int, err error) {
return cc.stream.Write(d)
}
func (cc *CryptoConn) Read(d []byte) (n int, err error) {
return cc.stream.Read(d)
}
func (cc *CryptoConn) LocalAddr() net.Addr {
return cc.conn.LocalAddr()
}
func (cc *CryptoConn) RemoteAddr() net.Addr {
return cc.conn.RemoteAddr()
}
func (cc *CryptoConn) SetDeadline(t time.Time) (err error) {
return cc.conn.SetDeadline(t)
}
func (cc *CryptoConn) SetReadDeadline(t time.Time) (err error) {
return cc.conn.SetReadDeadline(t)
}
func (cc *CryptoConn) SetWriteDeadline(t time.Time) (err error) {
return cc.conn.SetWriteDeadline(t)
}
type CryptoListener struct {
l net.Listener
handshake chan net.Conn
accepted chan *CryptoConn
trust func(pk []byte) bool
key *KeyPair
}
func (cl *CryptoListener) Close() (err error) {
err = cl.l.Close()
close(cl.accepted)
close(cl.handshake)
cl.key.Free()
cl.key = nil
return
}
func (cl *CryptoListener) acceptInbound() {
for {
c, err := cl.l.Accept()
if err == nil {
cl.handshake <- c
} else {
return
}
}
}
func (cl *CryptoListener) runChans() {
for {
select {
case c := <-cl.handshake:
go func() {
s := &CryptoStream{
stream: c,
mtu: DefaultMTU,
key: cl.key,
}
err := s.Handshake()
if err == nil {
// we gud handshake was okay
if cl.trust(s.remote_pk) {
// the key is trusted okay
cl.accepted <- &CryptoConn{stream: s, conn: c}
} else {
// not trusted, close connection
s.Close()
}
}
}()
}
}
}
// accept inbound authenticated and trusted connections
func (cl *CryptoListener) Accept() (c net.Conn, err error) {
var ok bool
c, ok = <-cl.accepted
if !ok {
err = Closed
}
return
}
// create a listener
func Server(l net.Listener, local_sk []byte, trust func(pk []byte) bool) (s *CryptoListener) {
s = &CryptoListener{
l: l,
trust: trust,
handshake: make(chan net.Conn),
accepted: make(chan *CryptoConn),
}
s.key = LoadSignKey(local_sk)
go s.runChans()
go s.acceptInbound()
return
}

View File

@ -0,0 +1,53 @@
package nacl
// #cgo freebsd CFLAGS: -I/usr/local/include
// #cgo freebsd LDFLAGS: -L/usr/local/lib
// #cgo LDFLAGS: -lsodium
// #include <sodium.h>
import "C"
// verify a fucky detached sig
func CryptoVerifyFucky(msg, sig, pk []byte) bool {
var smsg []byte
smsg = append(smsg, sig...)
smsg = append(smsg, msg...)
return CryptoVerify(smsg, pk)
}
// verify a signed message
func CryptoVerify(smsg, pk []byte) bool {
smsg_buff := NewBuffer(smsg)
defer smsg_buff.Free()
pk_buff := NewBuffer(pk)
defer pk_buff.Free()
if pk_buff.size != C.crypto_sign_publickeybytes() {
return false
}
mlen := C.ulonglong(0)
msg := malloc(C.size_t(len(smsg)))
defer msg.Free()
smlen := C.ulonglong(smsg_buff.size)
return C.crypto_sign_open(msg.uchar(), &mlen, smsg_buff.uchar(), smlen, pk_buff.uchar()) != -1
}
// verfiy a detached signature
// return true on valid otherwise false
func CryptoVerifyDetached(msg, sig, pk []byte) bool {
msg_buff := NewBuffer(msg)
defer msg_buff.Free()
sig_buff := NewBuffer(sig)
defer sig_buff.Free()
pk_buff := NewBuffer(pk)
defer pk_buff.Free()
if pk_buff.size != C.crypto_sign_publickeybytes() {
return false
}
// invalid sig size
if sig_buff.size != C.crypto_sign_bytes() {
return false
}
return C.crypto_sign_verify_detached(sig_buff.uchar(), msg_buff.uchar(), C.ulonglong(len(msg)), pk_buff.uchar()) == 0
}

View File

@ -0,0 +1,34 @@
package crypto
import (
"bytes"
"crypto/rand"
"io"
"testing"
)
func TestNaclToPublic(t *testing.T) {
pk, sk := GenKeypair()
t_pk := ToPublic(sk)
if !bytes.Equal(pk, t_pk) {
t.Logf("%q != %q", pk, t_pk)
t.Fail()
}
}
func TestNaclSignVerify(t *testing.T) {
var msg [1024]byte
pk, sk := GenKeypair()
io.ReadFull(rand.Reader, msg[:])
signer := CreateSigner(sk)
signer.Write(msg[:])
sig := signer.Sign()
verifier := CreateVerifier(pk)
verifier.Write(msg[:])
if !verifier.Verify(sig) {
t.Logf("%q is invalid signature and is %dB long", sig, len(sig))
t.Fail()
}
}

View File

@ -0,0 +1,8 @@
package crypto
import (
"nntpchan/lib/crypto/nacl"
)
// generate random bytes
var RandBytes = nacl.RandBytes

View File

@ -0,0 +1,25 @@
package crypto
import "io"
// a detached signature
type Signature []byte
type SigEncoder interface {
// encode a signature to an io.Writer
// return error if one occurrened while writing out signature
Encode(sig Signature, w io.Writer) error
// encode a signature to a string
EncodeString(sig Signature) string
}
// a decoder of signatures
type SigDecoder interface {
// decode signature from io.Reader
// reads all data until io.EOF
// returns singaure or error if an error occured while reading
Decode(r io.Reader) (Signature, error)
// decode a signature from string
// returns signature or error if an error ocurred while decoding
DecodeString(str string) (Signature, error)
}

View File

@ -0,0 +1,14 @@
package crypto
import "io"
//
// provides generic signing interface for producing detached signatures
// call Write() to feed data to be signed, call Sign() to generate
// a detached signature
//
type Signer interface {
io.Writer
// generate detached Signature from previously fed body via Write()
Sign() Signature
}

View File

@ -0,0 +1,14 @@
package crypto
import "io"
// provides generic signature
// call Write() to feed in message body
// once the entire body has been fed in via Write() call Verify() with detached
// signature to verify the detached signature against the previously fed body
type Verifer interface {
io.Writer
// verify detached signature from body previously fed via Write()
// return true if the detached signature is valid given the body
Verify(sig Signature) bool
}

View File

@ -0,0 +1,26 @@
package database
import (
"errors"
"nntpchan/lib/config"
"nntpchan/lib/model"
"strings"
)
//
type Database interface {
ThreadByMessageID(msgid string) (*model.Thread, error)
ThreadByHash(hash string) (*model.Thread, error)
BoardPage(newsgroup string, pageno, perpage int) (*model.BoardPage, error)
}
// get new database connector from configuration
func NewDBFromConfig(c *config.DatabaseConfig) (db Database, err error) {
dbtype := strings.ToLower(c.Type)
if dbtype == "postgres" {
db, err = createPostgresDatabase(c.Addr, c.Username, c.Password)
} else {
err = errors.New("no such database driver: " + c.Type)
}
return
}

View File

@ -0,0 +1,4 @@
//
// database driver
//
package database

View File

@ -0,0 +1,28 @@
package database
import (
"nntpchan/lib/model"
)
type PostgresDB struct {
}
func (db *PostgresDB) ThreadByMessageID(msgid string) (thread *model.Thread, err error) {
return
}
func (db *PostgresDB) ThreadByHash(hash string) (thread *model.Thread, err error) {
return
}
func (db *PostgresDB) BoardPage(newsgroup string, pageno, perpage int) (page *model.BoardPage, err error) {
return
}
func createPostgresDatabase(addr, user, passwd string) (p *PostgresDB, err error) {
return
}

View File

@ -0,0 +1,123 @@
package frontend
import (
"encoding/json"
"errors"
"fmt"
"github.com/dchest/captcha"
"github.com/gorilla/mux"
"github.com/gorilla/sessions"
"net/http"
"nntpchan/lib/config"
)
// server of captchas
// implements frontend.Middleware
type CaptchaServer struct {
h int
w int
store *sessions.CookieStore
prefix string
sessionName string
}
// create new captcha server using existing session store
func NewCaptchaServer(w, h int, prefix string, store *sessions.CookieStore) *CaptchaServer {
return &CaptchaServer{
h: h,
w: w,
prefix: prefix,
store: store,
sessionName: "captcha",
}
}
func (cs *CaptchaServer) Reload(c *config.MiddlewareConfig) {
}
func (cs *CaptchaServer) SetupRoutes(m *mux.Router) {
m.Path("/new").HandlerFunc(cs.NewCaptcha)
m.Path("/img/{f}").Handler(captcha.Server(cs.w, cs.h))
m.Path("/verify.json").HandlerFunc(cs.VerifyCaptcha)
}
// return true if this session has solved the last captcha given provided solution, otherwise false
func (cs *CaptchaServer) CheckSession(w http.ResponseWriter, r *http.Request, solution string) (bool, error) {
s, err := cs.store.Get(r, cs.sessionName)
if err == nil {
id, ok := s.Values["captcha_id"]
if ok {
return captcha.VerifyString(id.(string), solution), nil
}
}
return false, err
}
// verify a captcha
func (cs *CaptchaServer) VerifyCaptcha(w http.ResponseWriter, r *http.Request) {
dec := json.NewDecoder(r.Body)
defer r.Body.Close()
// request
req := make(map[string]string)
// response
resp := make(map[string]interface{})
resp["solved"] = false
// decode request
err := dec.Decode(req)
if err == nil {
// decode okay
id, ok := req["id"]
if ok {
// we have id
solution, ok := req["solution"]
if ok {
// we have solution and id
resp["solved"] = captcha.VerifyString(id, solution)
} else {
// we don't have solution
err = errors.New("no captcha solution provided")
}
} else {
// we don't have id
err = errors.New("no captcha id provided")
}
}
if err != nil {
// error happened
resp["error"] = err.Error()
}
// send reply
w.Header().Set("Content-Type", "text/json; encoding=UTF-8")
enc := json.NewEncoder(w)
enc.Encode(resp)
}
// generate a new captcha
func (cs *CaptchaServer) NewCaptcha(w http.ResponseWriter, r *http.Request) {
// obtain session
sess, err := cs.store.Get(r, cs.sessionName)
if err != nil {
// failed to obtain session
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// new captcha
id := captcha.New()
// do we want to interpret as json?
use_json := r.URL.Query().Get("t") == "json"
// image url
url := fmt.Sprintf("%simg/%s.png", cs.prefix, id)
if use_json {
// send json
enc := json.NewEncoder(w)
enc.Encode(map[string]string{"id": id, "url": url})
} else {
// set captcha id
sess.Values["captcha_id"] = id
// save session
sess.Save(r, w)
// rediect to image
http.Redirect(w, r, url, http.StatusFound)
}
}

View File

@ -0,0 +1,5 @@
//
// nntpchan frontend
// allows posting to nntpchan network via various implementations
//
package frontend

View File

@ -0,0 +1,46 @@
package frontend
import (
"nntpchan/lib/config"
"nntpchan/lib/database"
"nntpchan/lib/model"
"nntpchan/lib/nntp"
)
// a frontend that displays nntp posts and allows posting
type Frontend interface {
// run mainloop
Serve()
// do we accept this inbound post?
AllowPost(p model.PostReference) bool
// trigger a manual regen of indexes for a root post
Regen(p model.PostReference)
// implements nntp.EventHooks
GotArticle(msgid nntp.MessageID, group nntp.Newsgroup)
// implements nntp.EventHooks
SentArticleVia(msgid nntp.MessageID, feedname string)
// reload config
Reload(c *config.FrontendConfig)
}
// create a new http frontend give frontend config
func NewHTTPFrontend(c *config.FrontendConfig, db database.Database) (f Frontend, err error) {
var mid Middleware
if c.Middleware != nil {
// middleware configured
mid, err = OverchanMiddleware(c.Middleware, db)
}
if err == nil {
// create http frontend only if no previous errors
f, err = createHttpFrontend(c, mid, db)
}
return
}

View File

@ -0,0 +1,136 @@
package frontend
import (
"fmt"
log "github.com/Sirupsen/logrus"
"github.com/gorilla/mux"
"net/http"
"nntpchan/lib/admin"
"nntpchan/lib/api"
"nntpchan/lib/config"
"nntpchan/lib/database"
"nntpchan/lib/model"
"nntpchan/lib/nntp"
"time"
)
// http frontend server
// provides glue layer between nntp and middleware
type httpFrontend struct {
// bind address
addr string
// http mux
httpmux *mux.Router
// admin panel
adminPanel *admin.Server
// static files path
staticDir string
// http middleware
middleware Middleware
// api server
apiserve *api.Server
// database driver
db database.Database
}
// reload http frontend
// reloads middleware
func (f *httpFrontend) Reload(c *config.FrontendConfig) {
if f.middleware == nil {
if c.Middleware != nil {
var err error
// no middleware set, create middleware
f.middleware, err = OverchanMiddleware(c.Middleware, f.db)
if err != nil {
log.Errorf("overchan middleware reload failed: %s", err.Error())
}
}
} else {
// middleware exists
// do middleware reload
f.middleware.Reload(c.Middleware)
}
}
// serve http requests from net.Listener
func (f *httpFrontend) Serve() {
// serve http
for {
err := http.ListenAndServe(f.addr, f.httpmux)
if err != nil {
log.Errorf("failed to listen and serve with frontend: %s", err)
}
time.Sleep(time.Second)
}
}
// serve robots.txt page
func (f *httpFrontend) serveRobots(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "User-Agent: *\nDisallow: /\n")
}
func (f *httpFrontend) AllowPost(p model.PostReference) bool {
// TODO: implement
return true
}
func (f *httpFrontend) Regen(p model.PostReference) {
// TODO: implement
}
func (f *httpFrontend) GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) {
// TODO: implement
}
func (f *httpFrontend) SentArticleVia(msgid nntp.MessageID, feedname string) {
// TODO: implement
}
func createHttpFrontend(c *config.FrontendConfig, mid Middleware, db database.Database) (f *httpFrontend, err error) {
f = new(httpFrontend)
// set db
// db.Ensure() called elsewhere
f.db = db
// set bind address
f.addr = c.BindAddr
// set up mux
f.httpmux = mux.NewRouter()
// set up admin panel
f.adminPanel = admin.NewServer()
// set static files dir
f.staticDir = c.Static
// set middleware
f.middleware = mid
// set up routes
if f.adminPanel != nil {
// route up admin panel
f.httpmux.PathPrefix("/admin/").Handler(f.adminPanel)
}
if f.middleware != nil {
// route up middleware
f.middleware.SetupRoutes(f.httpmux)
}
if f.apiserve != nil {
// route up api
f.apiserve.SetupRoutes(f.httpmux.PathPrefix("/api/").Subrouter())
}
// route up robots.txt
f.httpmux.Path("/robots.txt").HandlerFunc(f.serveRobots)
// route up static files
f.httpmux.PathPrefix("/static/").Handler(http.FileServer(http.Dir(f.staticDir)))
return
}

View File

@ -0,0 +1,14 @@
package frontend
import (
"github.com/gorilla/mux"
"nntpchan/lib/config"
)
// http middleware
type Middleware interface {
// set up routes
SetupRoutes(m *mux.Router)
// reload with new configuration
Reload(c *config.MiddlewareConfig)
}

View File

@ -0,0 +1,115 @@
package frontend
import (
log "github.com/Sirupsen/logrus"
"github.com/gorilla/mux"
"github.com/gorilla/sessions"
"html/template"
"net/http"
"nntpchan/lib/config"
"nntpchan/lib/database"
"path/filepath"
"strconv"
)
// standard overchan imageboard middleware
type overchanMiddleware struct {
templ *template.Template
captcha *CaptchaServer
store *sessions.CookieStore
db database.Database
}
func (m *overchanMiddleware) SetupRoutes(mux *mux.Router) {
// setup front page handler
mux.Path("/").HandlerFunc(m.ServeIndex)
// setup thread handler
mux.Path("/t/{id}/").HandlerFunc(m.ServeThread)
// setup board page handler
mux.Path("/b/{name}/").HandlerFunc(m.ServeBoardPage)
// setup posting endpoint
mux.Path("/post")
// create captcha
captchaPrefix := "/captcha/"
m.captcha = NewCaptchaServer(200, 400, captchaPrefix, m.store)
// setup captcha endpoint
m.captcha.SetupRoutes(mux.PathPrefix(captchaPrefix).Subrouter())
}
// reload middleware
func (m *overchanMiddleware) Reload(c *config.MiddlewareConfig) {
// reload templates
templ, err := template.ParseGlob(filepath.Join(c.Templates, "*.tmpl"))
if err == nil {
log.Infof("middleware reloaded templates")
m.templ = templ
} else {
log.Errorf("middleware reload failed: %s", err.Error())
}
}
func (m *overchanMiddleware) ServeBoardPage(w http.ResponseWriter, r *http.Request) {
param := mux.Vars(r)
board := param["name"]
page := r.URL.Query().Get("q")
pageno, err := strconv.Atoi(page)
if err == nil {
var obj interface{}
obj, err = m.db.BoardPage(board, pageno, 10)
if err == nil {
m.serveTemplate(w, r, "board.html.tmpl", obj)
} else {
m.serveTemplate(w, r, "error.html.tmpl", err)
}
} else {
// 404
http.NotFound(w, r)
}
}
// serve cached thread
func (m *overchanMiddleware) ServeThread(w http.ResponseWriter, r *http.Request) {
param := mux.Vars(r)
obj, err := m.db.ThreadByHash(param["id"])
if err == nil {
m.serveTemplate(w, r, "thread.html.tmpl", obj)
} else {
m.serveTemplate(w, r, "error.html.tmpl", err)
}
}
// serve index page
func (m *overchanMiddleware) ServeIndex(w http.ResponseWriter, r *http.Request) {
m.serveTemplate(w, r, "index.html.tmpl", nil)
}
// serve a template
func (m *overchanMiddleware) serveTemplate(w http.ResponseWriter, r *http.Request, tname string, obj interface{}) {
t := m.templ.Lookup(tname)
if t == nil {
log.WithFields(log.Fields{
"template": tname,
}).Warning("template not found")
http.NotFound(w, r)
} else {
err := t.Execute(w, obj)
if err != nil {
// error getting model
log.WithFields(log.Fields{
"error": err,
"template": tname,
}).Warning("failed to render template")
}
}
}
// create standard overchan middleware
func OverchanMiddleware(c *config.MiddlewareConfig, db database.Database) (m Middleware, err error) {
om := new(overchanMiddleware)
om.templ, err = template.ParseGlob(filepath.Join(c.Templates, "*.tmpl"))
om.db = db
if err == nil {
m = om
}
return
}

View File

@ -0,0 +1 @@
package frontend

View File

@ -0,0 +1 @@
package frontend

View File

@ -0,0 +1,15 @@
package model
type Article struct {
Subject string
Name string
Header map[string][]string
Text string
Attachments []Attachment
MessageID string
Newsgroup string
Reference string
Path string
Posted int64
Addr string
}

View File

@ -0,0 +1,10 @@
package model
type Attachment struct {
Path string
Name string
Mime string
Hash string
// only filled for api
Body string
}

View File

@ -0,0 +1,4 @@
package model
type Board struct {
}

View File

@ -0,0 +1,8 @@
package model
type BoardPage struct {
Name string
Page int
Pages int
Threads []Thread
}

View File

@ -0,0 +1,2 @@
// MVC models
package model

View File

@ -0,0 +1,29 @@
package model
import (
"time"
)
type ArticleHeader map[string][]string
// a ( MessageID , newsgroup ) tuple
type ArticleEntry [2]string
func (self ArticleEntry) Newsgroup() string {
return self[1]
}
func (self ArticleEntry) MessageID() string {
return self[0]
}
// a ( time point, post count ) tuple
type PostEntry [2]int64
func (self PostEntry) Time() time.Time {
return time.Unix(self[0], 0)
}
func (self PostEntry) Count() int64 {
return self[1]
}

View File

@ -0,0 +1,32 @@
package model
import (
"time"
)
type Tripcode string
type Post struct {
MessageID string
Newsgroup string
Attachments []Attachment
Subject string
Posted time.Time
PostedAt uint64
Name string
Tripcode Tripcode
}
// ( message-id, references, newsgroup )
type PostReference [3]string
func (r PostReference) MessageID() string {
return r[0]
}
func (r PostReference) References() string {
return r[1]
}
func (r PostReference) Newsgroup() string {
return r[2]
}

View File

@ -0,0 +1,6 @@
package model
type Thread struct {
Root *Post
Replies []*Post
}

View File

@ -0,0 +1,37 @@
package network
import (
"errors"
"net"
"nntpchan/lib/config"
"strings"
)
// operation timed out
var ErrTimeout = errors.New("timeout")
// the operation was reset abruptly
var ErrReset = errors.New("reset")
// the operation was actively refused
var ErrRefused = errors.New("refused")
// generic dialer
// dials out to a remote address
// returns a net.Conn and nil on success
// returns nil and error if an error happens while dialing
type Dialer interface {
Dial(remote string) (net.Conn, error)
}
// create a new dialer from configuration
func NewDialer(conf *config.ProxyConfig) (d Dialer) {
d = StdDialer
if conf != nil {
proxyType := strings.ToLower(conf.Type)
if proxyType == "socks" || proxyType == "socks4a" {
d = SocksDialer(conf.Addr)
}
}
return
}

View File

@ -0,0 +1,4 @@
//
// network utilities
//
package network

View File

@ -0,0 +1 @@
package network

View File

@ -0,0 +1,140 @@
package network
import (
"errors"
log "github.com/Sirupsen/logrus"
"io"
"net"
"strconv"
"strings"
)
type socksDialer struct {
socksAddr string
dialer Dialer
}
// try dialing out via socks proxy
func (sd *socksDialer) Dial(remote string) (c net.Conn, err error) {
log.WithFields(log.Fields{
"addr": remote,
"socks": sd.socksAddr,
}).Debug("dailing out to socks proxy")
c, err = sd.dialer.Dial(sd.socksAddr)
if err == nil {
// dailed out to socks proxy good
remote_addr := remote
// generate request
idx := strings.LastIndex(remote_addr, ":")
if idx == -1 {
err = errors.New("invalid address: " + remote_addr)
return
}
var port uint64
addr := remote_addr[:idx]
port, err = strconv.ParseUint(remote_addr[idx+1:], 10, 16)
if port >= 25536 {
err = errors.New("bad proxy port")
c.Close()
c = nil
return
} else if err != nil {
c.Close()
return
}
var proxy_port uint16
proxy_port = uint16(port)
proxy_ident := "srndproxy"
req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8
req := make([]byte, req_len)
// pack request
req[0] = '\x04'
req[1] = '\x01'
req[2] = byte(proxy_port & 0xff00 >> 8)
req[3] = byte(proxy_port & 0x00ff)
req[7] = '\x01'
idx = 8
proxy_ident_b := []byte(proxy_ident)
addr_b := []byte(addr)
var bi int
for bi = range proxy_ident_b {
req[idx] = proxy_ident_b[bi]
idx += 1
}
idx += 1
for bi = range addr_b {
req[idx] = addr_b[bi]
idx += 1
}
log.WithFields(log.Fields{
"addr": remote,
"socks": sd.socksAddr,
"req": req,
}).Debug("write socks request")
n := 0
n, err = c.Write(req)
if err == nil && n == len(req) {
// wrote request okay
resp := make([]byte, 8)
_, err = io.ReadFull(c, resp)
if err == nil {
// got reply okay
if resp[1] == '\x5a' {
// successful socks connection
log.WithFields(log.Fields{
"addr": remote,
"socks": sd.socksAddr,
}).Debug("socks proxy connection successful")
} else {
// unsucessful socks connect
log.WithFields(log.Fields{
"addr": remote,
"socks": sd.socksAddr,
"code": resp[1],
}).Warn("connect via socks proxy failed")
c.Close()
c = nil
}
} else {
// error reading reply
log.WithFields(log.Fields{
"addr": remote,
"socks": sd.socksAddr,
}).Error("failed to read socks response ", err)
c.Close()
c = nil
}
} else {
if err == nil {
err = errors.New("short write")
}
// error writing request
log.WithFields(log.Fields{
"addr": remote,
"socks": sd.socksAddr,
}).Error("failed to write socks request ", err)
c.Close()
c = nil
}
} else {
// dail fail
log.WithFields(log.Fields{
"addr": remote,
"socks": sd.socksAddr,
}).Error("Cannot connect to socks proxy ", err)
}
return
}
// create a socks dialer that dials out via socks proxy at address
func SocksDialer(addr string) Dialer {
return &socksDialer{
socksAddr: addr,
dialer: StdDialer,
}
}

View File

@ -0,0 +1,14 @@
package network
import (
"net"
)
type stdDialer struct {
}
func (sd *stdDialer) Dial(addr string) (c net.Conn, err error) {
return net.Dial("tcp", addr)
}
var StdDialer = &stdDialer{}

View File

@ -0,0 +1,69 @@
package nntp
import (
"nntpchan/lib/nntp/message"
)
const (
// accepted article
ARTICLE_ACCEPT = iota
// reject article, don't send again
ARTICLE_REJECT
// defer article, send later
ARTICLE_DEFER
// reject + ban
ARTICLE_BAN
)
type PolicyStatus int
const PolicyAccept = PolicyStatus(ARTICLE_ACCEPT)
const PolicyReject = PolicyStatus(ARTICLE_REJECT)
const PolicyDefer = PolicyStatus(ARTICLE_DEFER)
const PolicyBan = PolicyStatus(ARTICLE_BAN)
func (s PolicyStatus) String() string {
switch int(s) {
case ARTICLE_ACCEPT:
return "ACCEPTED"
case ARTICLE_REJECT:
return "REJECTED"
case ARTICLE_DEFER:
return "DEFERRED"
case ARTICLE_BAN:
return "BANNED"
default:
return "[invalid policy status]"
}
}
// is this an accept code?
func (s PolicyStatus) Accept() bool {
return s == ARTICLE_ACCEPT
}
// is this a defer code?
func (s PolicyStatus) Defer() bool {
return s == ARTICLE_DEFER
}
// is this a ban code
func (s PolicyStatus) Ban() bool {
return s == ARTICLE_BAN
}
// is this a reject code?
func (s PolicyStatus) Reject() bool {
return s == ARTICLE_BAN || s == ARTICLE_REJECT
}
// type defining a policy that determines if we want to accept/reject/defer an
// incoming article
type ArticleAcceptor interface {
// check article given an article header
CheckHeader(hdr message.Header) PolicyStatus
// check article given a message id
CheckMessageID(msgid MessageID) PolicyStatus
// get max article size in bytes
MaxArticleSize() int64
}

View File

@ -0,0 +1,38 @@
package nntp
import (
"bufio"
"fmt"
"os"
"strings"
)
// defines server side authentication mechanism
type ServerAuth interface {
// check plaintext login
// returns nil on success otherwise error if one occurs during authentication
// returns true if authentication was successful and an error if a network io error happens
CheckLogin(username, passwd string) (bool, error)
}
type FlatfileAuth string
func (fname FlatfileAuth) CheckLogin(username, passwd string) (found bool, err error) {
cred := fmt.Sprintf("%s:%s", username, passwd)
var f *os.File
f, err = os.Open(string(fname))
if err == nil {
defer f.Close()
r := bufio.NewReader(f)
for err == nil {
var line string
line, err = r.ReadString(10)
line = strings.Trim(line, "\r\n")
if line == cred {
found = true
break
}
}
}
return
}

View File

@ -0,0 +1,206 @@
package nntp
// 1xx codes
// help info follows
const RPL_Help = "100"
// capabilities info follows
const RPL_Capabilities = "101"
// server date time follows
const RPL_Date = "111"
// 2xx codes
// posting is allowed
const RPL_PostingAllowed = "200"
// posting is not allowed
const RPL_PostingNotAllowed = "201"
// streaming mode enabled
const RPL_PostingStreaming = "203"
// reply to QUIT command, we will close the connection
const RPL_Quit = "205"
// reply for GROUP and LISTGROUP commands
const RPL_Group = "211"
// info list follows
const RPL_List = "215"
// index follows
const RPL_Index = "218"
// article follows
const RPL_Article = "220"
// article headers follows
const RPL_ArticleHeaders = "221"
// article body follows
const RPL_ArticleBody = "222"
// selected article exists
const RPL_ArticleSelectedExists = "223"
// overview info follows
const RPL_Overview = "224"
// list of article heards follows
const RPL_HeadersList = "225"
// list of new articles follows
const RPL_NewArticles = "230"
// list of newsgroups followes
const RPL_NewsgroupList = "231"
// article was transfered okay by IHAVE command
const RPL_TransferOkay = "235"
// article is not found by CHECK and we want it
const RPL_StreamingAccept = "238"
// article was transfered via TAKETHIS successfully
const RPL_StreamingTransfered = "239"
// article was transfered by POST command successfully
const RPL_PostReceived = "240"
// AUTHINFO SIMPLE accepted
const RPL_AuthInfoAccepted = "250"
// authentication creds have been accepted
const RPL_AuthAccepted = "281"
// binary content follows
const RPL_Binary = "288"
// line sent for posting allowed
const Line_PostingAllowed = RPL_PostingAllowed + " Posting Allowed"
// line sent for posting not allowed
const Line_PostingNotAllowed = RPL_PostingNotAllowed + " Posting Not Allowed"
// 3xx codes
// article is accepted via IHAVE
const RPL_TransferAccepted = "335"
// article was accepted via POST
const RPL_PostAccepted = "340"
// continue with authorization
const RPL_ContinueAuthorization = "350"
// more authentication info required
const RPL_MoreAuth = "381"
// continue with tls handshake
const RPL_TLSContinue = "382"
// 4xx codes
// server says servive is not avaiable on initial connection
const RPL_NotAvaiable = "400"
// server is in the wrong mode
const RPL_WrongMode = "401"
// generic fault prevent action from being taken
const RPL_GenericError = "403"
// newsgroup does not exist
const RPL_NoSuchGroup = "411"
// no newsgroup has been selected
const RPL_NoGroupSelected = "412"
// no tin style index available
const RPL_NoIndex = "418"
// current article number is invalid
const RPL_NoArticleNum = "420"
// no next article in this group (NEXT)
const RPL_NoNextArticle = "421"
// no previous article in this group (LAST)
const RPL_NoPrevArticle = "422"
// no article in specified range
const RPL_NoArticleRange = "423"
// no article with that message-id
const RPL_NoArticleMsgID = "430"
// defer article asked by CHECK comamnd
const RPL_StreamingDefer = "431"
// article is not wanted (1st stage of IHAVE)
const RPL_TransferNotWanted = "435"
// article was not sent defer sending (either stage of IHAVE)
const RPL_TransferDefer = "436"
// reject transfer do not retry (2nd stage IHAVE)
const RPL_TransferReject = "437"
// reject article and don't ask again (CHECK command)
const RPL_StreamingReject = "438"
// article transfer via streaming failed (TAKETHIS)
const RPL_StreamingFailed = "439"
// posting not permitted (1st stage of POST command)
const RPL_PostingNotPermitted = "440"
// posting failed (2nd stage of POST command)
const RPL_PostingFailed = "441"
// authorization required
const RPL_AuthorizeRequired = "450"
// authorization rejected
const RPL_AuthorizeRejected = "452"
// command unavaibale until client has authenticated
const RPL_AuthenticateRequired = "480"
// authentication creds rejected
const RPL_AuthenticateRejected = "482"
// command unavailable until connection is encrypted
const RPL_EncryptionRequired = "483"
// 5xx codes
// got an unknown command
const RPL_UnknownCommand = "500"
// got a command with invalid syntax
const RPL_SyntaxError = "501"
// fatal error happened and connection will close
const RPL_GenericFatal = "502"
// feature is not supported
const RPL_FeatureNotSupported = "503"
// message encoding is bad
const RPL_EncodingError = "504"
// starttls can not be done
const RPL_TLSRejected = "580"
// line sent on invalid mode
const Line_InvalidMode = RPL_SyntaxError + " Invalid Mode Selected"
// line sent on successful streaming
const Line_StreamingAllowed = RPL_PostingStreaming + " aw yeh streamit brah"
// send this when we handle a QUIT command
const Line_RPLQuit = RPL_Quit + " bai"

View File

@ -0,0 +1,27 @@
package nntp
type Command string
func (c Command) String() string {
return string(c)
}
// command to list newsgroups
const CMD_Newsgroups = Command("NEWSGROUPS 0 0 GMT")
// create group command for a newsgroup
func CMD_Group(g Newsgroup) Command {
return Command("GROUP " + g.String())
}
const CMD_XOver = Command("XOVER 0")
func CMD_Article(msgid MessageID) Command {
return Command("ARTICLE " + msgid.String())
}
func CMD_Head(msgid MessageID) Command {
return Command("HEAD " + msgid.String())
}
const CMD_Capabilities = Command("CAPABILITIES")

View File

@ -0,0 +1,75 @@
package nntp
import (
"crypto/sha1"
"fmt"
"io"
"nntpchan/lib/crypto"
"regexp"
"strings"
"time"
)
var exp_valid_message_id = regexp.MustCompilePOSIX(`^<[a-zA-Z0-9$.]{2,128}@[a-zA-Z0-9\-.]{2,63}>$`)
type MessageID string
// return true if this message id is well formed, otherwise return false
func (msgid MessageID) Valid() bool {
return exp_valid_message_id.Copy().MatchString(msgid.String())
}
// get message id as string
func (msgid MessageID) String() string {
return string(msgid)
}
// compute long form hash of message id
func (msgid MessageID) LongHash() string {
return fmt.Sprintf("%x", sha1.Sum([]byte(msgid)))
}
// compute truncated form of message id hash
func (msgid MessageID) ShortHash() string {
return strings.ToLower(msgid.LongHash()[:18])
}
// compute blake2 hash of message id
func (msgid MessageID) Blake2Hash() string {
h := crypto.Hash()
io.WriteString(h, msgid.String())
return strings.ToLower(fmt.Sprintf("%x", h.Sum(nil)))
}
// generate a new message id given name of server
func GenMessageID(name string) MessageID {
r := crypto.RandBytes(4)
t := time.Now()
return MessageID(fmt.Sprintf("<%x$%d@%s>", r, t.Unix(), name))
}
var exp_valid_newsgroup = regexp.MustCompilePOSIX(`^[a-zA-Z0-9.]{1,128}$`)
// an nntp newsgroup
type Newsgroup string
// return true if this newsgroup is well formed otherwise false
func (g Newsgroup) Valid() bool {
return exp_valid_newsgroup.Copy().MatchString(g.String())
}
// get newsgroup as string
func (g Newsgroup) String() string {
return string(g)
}
// (message-id, newsgroup) tuple
type ArticleEntry [2]string
func (e ArticleEntry) MessageID() MessageID {
return MessageID(e[0])
}
func (e ArticleEntry) Newsgroup() Newsgroup {
return Newsgroup(e[1])
}

View File

@ -0,0 +1,44 @@
package nntp
import (
"testing"
)
func TestGenMessageID(t *testing.T) {
msgid := GenMessageID("test.tld")
t.Logf("generated id %s", msgid)
if !msgid.Valid() {
t.Logf("invalid generated message-id %s", msgid)
t.Fail()
}
msgid = GenMessageID("<><><>")
t.Logf("generated id %s", msgid)
if msgid.Valid() {
t.Logf("generated valid message-id when it should've been invalid %s", msgid)
t.Fail()
}
}
func TestMessageIDHash(t *testing.T) {
msgid := GenMessageID("test.tld")
lh := msgid.LongHash()
sh := msgid.ShortHash()
bh := msgid.Blake2Hash()
t.Logf("long=%s short=%s blake2=%s", lh, sh, bh)
}
func TestValidNewsgroup(t *testing.T) {
g := Newsgroup("overchan.test")
if !g.Valid() {
t.Logf("%s is invalid?", g)
t.Fail()
}
}
func TestInvalidNewsgroup(t *testing.T) {
g := Newsgroup("asd.asd.asd.&&&")
if g.Valid() {
t.Logf("%s should be invalid", g)
t.Fail()
}
}

View File

@ -0,0 +1,53 @@
package nntp
// an nntp connection
type Conn interface {
// negotiate an nntp session on this connection
// returns nil if we negitated successfully
// returns ErrAuthRejected if the remote server rejected any authentication
// we sent or another error if one occured while negotiating
Negotiate(stream bool) error
// obtain connection state
GetState() *ConnState
// retutrn true if posting is allowed
// return false if posting is not allowed
PostingAllowed() bool
// handle inbound non-streaming connection
// call event hooks on event
ProcessInbound(hooks EventHooks)
// does this connection want to do nntp streaming?
WantsStreaming() bool
// what mode are we in?
// returns mode in all caps
Mode() Mode
// initiate nntp streaming
// after calling this the caller MUST call StreamAndQuit()
// returns a channel for message ids, true if caller sends on the channel or
// returns nil and ErrStreamingNotAllowed if streaming is not allowed on this
// connection or another error if one occurs while trying to start streaming
StartStreaming() (chan ArticleEntry, error)
// stream articles and quit when the channel obtained by StartStreaming() is
// closed, after which this nntp connection is no longer open
StreamAndQuit()
// is this nntp connection open?
IsOpen() bool
// send quit command and close connection
Quit()
// download all articles in a newsgroup
// returns error if a network error occurs
DownloadGroup(g Newsgroup) error
// get list of active newsgroups
ListNewsgroups() ([]Newsgroup, error)
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,15 @@
package nntp
import (
"crypto/tls"
"nntpchan/lib/network"
)
// establishes an outbound nntp connection to a remote server
type Dialer interface {
// dial out with a dialer
// if cfg is not nil, try to establish a tls connection with STARTTLS
// returns a new nntp connection and nil on successful handshake and login
// returns nil and an error if an error happened
Dial(d network.Dialer, cfg *tls.Config) (*Conn, error)
}

View File

@ -0,0 +1,4 @@
//
// nntp client/server
//
package nntp

View File

@ -0,0 +1,60 @@
package nntp
import (
"testing"
)
func TestTAKETHISParse(t *testing.T) {
msgid := GenMessageID("test.tld")
ev := stream_cmd_TAKETHIS(msgid)
t.Logf("event: %s", ev)
if ev.MessageID() != msgid {
t.Logf("%s != %s, event was %s", msgid, ev.MessageID(), ev)
t.Fail()
}
if ev.Command() != "TAKETHIS" {
t.Logf("%s != TAKETHIS, event was %s", ev.Command(), ev)
t.Fail()
}
if !ev.Valid() {
t.Logf("%s is invalid stream event", ev)
t.Fail()
}
}
func TestCHECKParse(t *testing.T) {
msgid := GenMessageID("test.tld")
ev := stream_cmd_CHECK(msgid)
t.Logf("event: %s", ev)
if ev.MessageID() != msgid {
t.Logf("%s != %s, event was %s", msgid, ev.MessageID(), ev)
t.Fail()
}
if ev.Command() != "CHECK" {
t.Logf("%s != CHECK, event was %s", ev.Command(), ev)
t.Fail()
}
if !ev.Valid() {
t.Logf("%s is invalid stream event", ev)
t.Fail()
}
}
func TestInvalidStremEvent(t *testing.T) {
str := "asd"
ev := StreamEvent(str)
t.Logf("invalid str=%s ev=%s", str, ev)
if ev.Valid() {
t.Logf("invalid CHECK command is valid? %s", ev)
t.Fail()
}
str = "asd asd"
ev = StreamEvent(str)
t.Logf("invalid str=%s ev=%s", str, ev)
if ev.Valid() {
t.Logf("invalid CHECK command is valid? %s", ev)
t.Fail()
}
}

View File

@ -0,0 +1,19 @@
package nntp
import (
"io"
"nntpchan/lib/nntp/message"
)
// defines interface for filtering an nntp article
// filters can (and does) modify the article it operates on
type ArticleFilter interface {
// filter the article header
// returns the modified Header and an error if one occurs
FilterHeader(hdr message.Header) (message.Header, error)
// reads the article's body and write the filtered version to an io.Writer
// returns the number of bytes written to the io.Writer, true if the body was
// modifed (or false if body is unchanged) and an error if one occurs
FilterAndWriteBody(body io.Reader, wr io.Writer) (int64, bool, error)
}

View File

@ -0,0 +1,30 @@
package nntp
import (
log "github.com/Sirupsen/logrus"
"nntpchan/lib/config"
"os/exec"
)
type Hook struct {
cfg *config.NNTPHookConfig
}
func NewHook(cfg *config.NNTPHookConfig) *Hook {
return &Hook{
cfg: cfg,
}
}
func (h *Hook) GotArticle(msgid MessageID, group Newsgroup) {
c := exec.Command(h.cfg.Exec, group.String(), msgid.String())
log.Infof("calling hook %s", h.cfg.Name)
err := c.Run()
if err != nil {
log.Errorf("error in nntp hook %s: %s", h.cfg.Name, err.Error())
}
}
func (*Hook) SentArticleVia(msgid MessageID, feedname string) {
}

View File

@ -0,0 +1,9 @@
package nntp
// callback hooks fired on certain events
type EventHooks interface {
// called when we have obtained an article given its message-id
GotArticle(msgid MessageID, group Newsgroup)
// called when we have sent an article to a single remote feed
SentArticleVia(msgid MessageID, feedname string)
}

View File

@ -0,0 +1,11 @@
package message
// an nntp article
type Article struct {
// the article's mime header
Header Header
// unexported fields ...
}

View File

@ -0,0 +1,16 @@
package message
import (
"io"
)
// attachment in an nntp article
type Attachment struct {
// mimetype
Mime string
// the filename
FileName string
// the fully decoded attachment body
// must close when done
Body io.ReadCloser
}

View File

@ -0,0 +1,2 @@
// package for parsing, packing, signing, verifying nntp articles
package message

View File

@ -0,0 +1,174 @@
package message
import (
"io"
"mime"
"strings"
)
// an nntp message header
type Header map[string][]string
// get message-id header
func (self Header) MessageID() (v string) {
for _, hdr := range []string{"MessageID", "Message-ID", "Message-Id", "message-id"} {
v = self.Get(hdr, "")
if v != "" {
break
}
}
return
}
func (self Header) Reference() (ref string) {
return self.Get("Reference", self.MessageID())
}
// extract media type from content-type header
func (self Header) GetMediaType() (mediatype string, params map[string]string, err error) {
return mime.ParseMediaType(self.Get("Content-Type", "text/plain"))
}
// is this header for a multipart message?
func (self Header) IsMultipart() bool {
return strings.HasPrefix(self.Get("Content-Type", "text/plain"), "multipart/mixed")
}
func (self Header) IsSigned() bool {
return self.Get("X-Pubkey-Ed25519", "") != ""
}
func (self Header) Newsgroup() string {
return self.Get("Newsgroups", "overchan.discard")
}
// do we have a key in this header?
func (self Header) Has(key string) bool {
_, ok := self[key]
return ok
}
// set key value
func (self Header) Set(key, val string) {
self[key] = []string{val}
}
func (self Header) AppendPath(name string) {
p := self.Get("Path", name)
if p != name {
p = name + "!" + p
}
self.Set("Path", p)
}
// append value to key
func (self Header) Add(key, val string) {
if self.Has(key) {
self[key] = append(self[key], val)
} else {
self.Set(key, val)
}
}
// get via key or return fallback value
func (self Header) Get(key, fallback string) string {
val, ok := self[key]
if ok {
str := ""
for _, k := range val {
str += k + ", "
}
return str[:len(str)-2]
} else {
return fallback
}
}
// interface for types that can read an nntp header
type HeaderReader interface {
// blocking read an nntp header from an io.Reader
// return the read header and nil on success
// return nil and an error if an error occurred while reading
ReadHeader(r io.Reader) (Header, error)
}
// interface for types that can write an nntp header
type HeaderWriter interface {
// blocking write an nntp header to an io.Writer
// returns an error if one occurs otherwise nil
WriteHeader(hdr Header, w io.Writer) error
}
// implements HeaderReader and HeaderWriter
type HeaderIO struct {
delim byte
}
// read header
func (s *HeaderIO) ReadHeader(r io.Reader) (hdr Header, err error) {
hdr = make(Header)
var k, v string
var buf [1]byte
for err == nil {
// read key
for err == nil {
_, err = r.Read(buf[:])
if err != nil {
return
}
if buf[0] == 58 { // colin
// consume space
_, err = r.Read(buf[:])
for err == nil {
_, err = r.Read(buf[:])
if buf[0] == s.delim {
// got delimiter
hdr.Add(k, v)
k = ""
v = ""
break
} else {
v += string(buf[:])
}
}
break
} else if buf[0] == s.delim {
// done
return
} else {
k += string(buf[:])
}
}
}
return
}
// write header
func (s *HeaderIO) WriteHeader(hdr Header, wr io.Writer) (err error) {
for k, vs := range hdr {
for _, v := range vs {
var line []byte
// key
line = append(line, []byte(k)...)
// ": "
line = append(line, 58, 32)
// value
line = append(line, []byte(v)...)
// delimiter
line = append(line, s.delim)
// write line
_, err = wr.Write(line)
if err != nil {
return
}
}
}
_, err = wr.Write([]byte{s.delim})
return
}
func NewHeaderIO() *HeaderIO {
return &HeaderIO{
delim: 10,
}
}

View File

@ -0,0 +1,69 @@
package nntp
import (
"errors"
"strings"
)
var ErrInvalidMode = errors.New("invalid mode set")
// a mode set by an nntp client
type Mode string
// reader mode
const MODE_READER = Mode("reader")
// streaming mode
const MODE_STREAM = Mode("stream")
// mode is not set
const MODE_UNSET = Mode("")
// get as string
func (m Mode) String() string {
return strings.ToUpper(string(m))
}
// is this a valid mode of operation?
func (m Mode) Valid() bool {
return m.Is(MODE_READER) || m.Is(MODE_STREAM)
}
// is this mode equal to another mode
func (m Mode) Is(other Mode) bool {
return m.String() == other.String()
}
// a switch mode command
type ModeCommand string
// get as string
func (m ModeCommand) String() string {
return strings.ToUpper(string(m))
}
// is this mode command well formed?
// does not check the actual mode sent.
func (m ModeCommand) Valid() bool {
s := m.String()
return strings.Count(s, " ") == 1 && strings.HasPrefix(s, "MODE ")
}
// get the mode selected in this mode command
func (m ModeCommand) Mode() Mode {
return Mode(strings.Split(m.String(), " ")[1])
}
// check if this mode command is equal to an existing one
func (m ModeCommand) Is(cmd ModeCommand) bool {
return m.String() == cmd.String()
}
// reader mode command
const ModeReader = ModeCommand("mode reader")
// streaming mode command
const ModeStream = ModeCommand("mode stream")
// line prefix for mode
const LinePrefix_Mode = "MODE "

View File

@ -0,0 +1,16 @@
package nntp
// multiplexed event hook
type MulitHook []EventHooks
func (m MulitHook) GotArticle(msgid MessageID, group Newsgroup) {
for _, h := range m {
h.GotArticle(msgid, group)
}
}
func (m MulitHook) SentArticleVia(msgid MessageID, feedname string) {
for _, h := range m {
h.SentArticleVia(msgid, feedname)
}
}

View File

@ -0,0 +1,29 @@
package nntp
//
// a policy that governs whether we federate an article via a feed
//
type FeedPolicy struct {
// list of whitelist regexps for newsgorups
Whitelist []string `json:"whitelist"`
// list of blacklist regexps for newsgroups
Blacklist []string `json:"blacklist"`
// are anon posts of any kind allowed?
AllowAnonPosts bool `json:"anon"`
// are anon posts with attachments allowed?
AllowAnonAttachments bool `json:"anon_attachments"`
// are any attachments allowed?
AllowAttachments bool `json:"attachments"`
// do we require Proof Of Work for untrusted connections?
UntrustedRequiresPoW bool `json:"pow"`
}
// default feed policy to be used if not configured explicitly
var DefaultFeedPolicy = &FeedPolicy{
Whitelist: []string{"ctl", "overchan.test"},
Blacklist: []string{`!^overchan\.`},
AllowAnonPosts: true,
AllowAnonAttachments: false,
UntrustedRequiresPoW: true,
AllowAttachments: true,
}

View File

@ -0,0 +1,326 @@
package nntp
import (
log "github.com/Sirupsen/logrus"
"net"
"nntpchan/lib/config"
"nntpchan/lib/network"
"nntpchan/lib/store"
"time"
)
// nntp outfeed state
type nntpFeed struct {
conn Conn
send chan ArticleEntry
conf *config.FeedConfig
}
// an nntp server
type Server struct {
// user callback
Hooks EventHooks
// filters to apply
Filters []ArticleFilter
// global article acceptor
Acceptor ArticleAcceptor
// article storage
Storage store.Storage
// nntp config
Config *config.NNTPServerConfig
// outfeeds to connect to
Feeds []*config.FeedConfig
// inbound authentiaction mechanism
Auth ServerAuth
// send to outbound feed channel
send chan ArticleEntry
// register inbound feed channel
regis chan *nntpFeed
// deregister inbound feed channel
deregis chan *nntpFeed
}
func NewServer() *Server {
return &Server{
// XXX: buffered?
send: make(chan ArticleEntry),
regis: make(chan *nntpFeed),
deregis: make(chan *nntpFeed),
}
}
// reload server configuration
func (s *Server) ReloadServer(c *config.NNTPServerConfig) {
}
// reload feeds
func (s *Server) ReloadFeeds(feeds []*config.FeedConfig) {
}
func (s *Server) GotArticle(msgid MessageID, group Newsgroup) {
log.WithFields(log.Fields{
"pkg": "nntp-server",
"msgid": msgid,
"group": group,
}).Info("obtained article")
if s.Hooks != nil {
s.Hooks.GotArticle(msgid, group)
}
// send to outbound feeds
s.send <- ArticleEntry{msgid.String(), group.String()}
}
func (s *Server) SentArticleVia(msgid MessageID, feedname string) {
log.WithFields(log.Fields{
"pkg": "nntp-server",
"msgid": msgid,
"feed": feedname,
}).Info("article sent")
if s.Hooks != nil {
s.Hooks.SentArticleVia(msgid, feedname)
}
}
func (s *Server) Name() string {
if s.Config == nil || s.Config.Name == "" {
return "nntp.anon.tld"
}
return s.Config.Name
}
// persist 1 feed forever
func (s *Server) persist(cfg *config.FeedConfig) {
delay := time.Second
log.WithFields(log.Fields{
"name": cfg.Name,
}).Debug("Persist Feed")
for {
dialer := network.NewDialer(cfg.Proxy)
c, err := dialer.Dial(cfg.Addr)
if err == nil {
// successful connect
delay = time.Second
conn := newOutboundConn(c, s, cfg)
err = conn.Negotiate(true)
if err == nil {
// negotiation good
log.WithFields(log.Fields{
"name": cfg.Name,
}).Debug("Negotitation good")
// start streaming
var chnl chan ArticleEntry
chnl, err = conn.StartStreaming()
if err == nil {
// register new connection
f := &nntpFeed{
conn: conn,
send: chnl,
conf: cfg,
}
s.regis <- f
// start streaming
conn.StreamAndQuit()
// deregister
s.deregis <- f
continue
}
} else {
log.WithFields(log.Fields{
"name": cfg.Name,
}).Info("outbound nntp connection failed to negotiate ", err)
}
conn.Quit()
} else {
// failed dial, do exponential backoff up to 1 hour
if delay <= time.Hour {
delay *= 2
}
log.WithFields(log.Fields{
"name": cfg.Name,
}).Info("feed backoff for ", delay)
time.Sleep(delay)
}
}
}
// download all new posts from a remote server
func (s *Server) downloadPosts(cfg *config.FeedConfig) error {
dialer := network.NewDialer(cfg.Proxy)
c, err := dialer.Dial(cfg.Addr)
if err != nil {
return err
}
conn := newOutboundConn(c, s, cfg)
err = conn.Negotiate(false)
if err != nil {
conn.Quit()
return err
}
groups, err := conn.ListNewsgroups()
if err != nil {
conn.Quit()
return err
}
for _, g := range groups {
if cfg.Policy != nil && cfg.Policy.AllowGroup(g.String()) {
log.WithFields(log.Fields{
"group": g,
"pkg": "nntp-server",
}).Debug("downloading group")
err = conn.DownloadGroup(g)
if err != nil {
conn.Quit()
return err
}
}
}
conn.Quit()
return nil
}
func (s *Server) periodicDownload(cfg *config.FeedConfig) {
for cfg.PullInterval > 0 {
err := s.downloadPosts(cfg)
if err != nil {
// report error
log.WithFields(log.Fields{
"feed": cfg.Name,
"pkg": "nntp-server",
"error": err,
}).Error("periodic download failed")
}
time.Sleep(time.Minute * time.Duration(cfg.PullInterval))
}
}
// persist all outbound feeds
func (s *Server) PersistFeeds() {
for _, f := range s.Feeds {
go s.persist(f)
go s.periodicDownload(f)
}
feeds := make(map[string]*nntpFeed)
for {
select {
case e, ok := <-s.send:
if !ok {
break
}
msgid := e.MessageID().String()
group := e.Newsgroup().String()
// TODO: determine anon
anon := false
// TODO: determine attachments
attachments := false
for _, f := range feeds {
if f.conf.Policy != nil && !f.conf.Policy.Allow(msgid, group, anon, attachments) {
// not allowed in this feed
continue
}
log.WithFields(log.Fields{
"name": f.conf.Name,
"msgid": msgid,
"group": group,
}).Debug("sending article")
f.send <- e
}
break
case f, ok := <-s.regis:
if ok {
log.WithFields(log.Fields{
"name": f.conf.Name,
}).Debug("register feed")
feeds[f.conf.Name] = f
}
break
case f, ok := <-s.deregis:
if ok {
log.WithFields(log.Fields{
"name": f.conf.Name,
}).Debug("deregister feed")
delete(feeds, f.conf.Name)
}
break
}
}
}
// serve connections from listener
func (s *Server) Serve(l net.Listener) (err error) {
log.WithFields(log.Fields{
"pkg": "nntp-server",
"addr": l.Addr(),
}).Debug("Serving")
for err == nil {
var c net.Conn
c, err = l.Accept()
if err == nil {
// we got a new connection
go s.handleInboundConnection(c)
} else {
log.WithFields(log.Fields{
"pkg": "nntp-server",
}).Error("failed to accept inbound connection", err)
}
}
return
}
// get the article policy for a connection given its state
func (s *Server) getPolicyFor(state *ConnState) ArticleAcceptor {
return s.Acceptor
}
// recv inbound streaming messages
func (s *Server) recvInboundStream(chnl chan ArticleEntry) {
for {
e, ok := <-chnl
if ok {
s.GotArticle(e.MessageID(), e.Newsgroup())
} else {
return
}
}
}
// process an inbound connection
func (s *Server) handleInboundConnection(c net.Conn) {
log.WithFields(log.Fields{
"pkg": "nntp-server",
"addr": c.RemoteAddr(),
}).Debug("handling inbound connection")
var nc Conn
nc = newInboundConn(s, c)
err := nc.Negotiate(true)
if err == nil {
// do they want to stream?
if nc.WantsStreaming() {
// yeeeeeh let's stream
var chnl chan ArticleEntry
chnl, err = nc.StartStreaming()
// for inbound we will recv messages
go s.recvInboundStream(chnl)
nc.StreamAndQuit()
log.WithFields(log.Fields{
"pkg": "nntp-server",
"addr": c.RemoteAddr(),
}).Info("streaming finished")
return
} else {
// handle non streaming commands
nc.ProcessInbound(s)
}
} else {
log.WithFields(log.Fields{
"pkg": "nntp-server",
"addr": c.RemoteAddr(),
}).Warn("failed to negotiate with inbound connection", err)
c.Close()
}
}

View File

@ -0,0 +1,21 @@
package nntp
// state of an nntp connection
type ConnState struct {
// name of parent feed
FeedName string `json:"feedname"`
// name of the connection
ConnName string `json:"connname"`
// hostname of remote connection
HostName string `json:"hostname"`
// current nntp mode
Mode Mode `json:"mode"`
// current selected nntp newsgroup
Group Newsgroup `json:"newsgroup"`
// current selected nntp article
Article string `json:"article"`
// parent feed's policy
Policy *FeedPolicy `json:"feedpolicy"`
// is this connection open?
Open bool `json:"open"`
}

View File

@ -0,0 +1,65 @@
package nntp
import (
"fmt"
"strings"
)
// an nntp stream event
// these are pipelined between nntp servers
type StreamEvent string
func (ev StreamEvent) MessageID() MessageID {
parts := strings.Split(string(ev), " ")
if len(parts) > 1 {
return MessageID(parts[1])
}
return ""
}
func (ev StreamEvent) String() string {
return string(ev)
}
func (ev StreamEvent) Command() string {
return strings.Split(ev.String(), " ")[0]
}
func (ev StreamEvent) Valid() bool {
return strings.Count(ev.String(), " ") == 1 && ev.MessageID().Valid()
}
var stream_TAKETHIS = "TAKETHIS"
var stream_CHECK = "CHECK"
func createStreamEvent(cmd string, msgid MessageID) StreamEvent {
if msgid.Valid() {
return StreamEvent(fmt.Sprintf("%s %s", cmd, msgid))
} else {
return ""
}
}
func stream_rpl_Accept(msgid MessageID) StreamEvent {
return createStreamEvent(RPL_StreamingAccept, msgid)
}
func stream_rpl_Reject(msgid MessageID) StreamEvent {
return createStreamEvent(RPL_StreamingReject, msgid)
}
func stream_rpl_Defer(msgid MessageID) StreamEvent {
return createStreamEvent(RPL_StreamingDefer, msgid)
}
func stream_rpl_Failed(msgid MessageID) StreamEvent {
return createStreamEvent(RPL_StreamingFailed, msgid)
}
func stream_cmd_TAKETHIS(msgid MessageID) StreamEvent {
return createStreamEvent(stream_TAKETHIS, msgid)
}
func stream_cmd_CHECK(msgid MessageID) StreamEvent {
return createStreamEvent(stream_CHECK, msgid)
}

View File

@ -0,0 +1,5 @@
//
// main package for srndv2
// called from main
//
package srnd

View File

@ -0,0 +1,4 @@
//
// nntp article storage
//
package store

View File

@ -0,0 +1,416 @@
package store
import (
"encoding/base32"
"fmt"
log "github.com/Sirupsen/logrus"
"io"
"io/ioutil"
"net/textproto"
"nntpchan/lib/crypto"
"os"
"path/filepath"
"strconv"
"time"
)
const HighWaterHeader = "X-High-Water"
const LowWaterHeader = "X-Low-Water"
// filesystem storage of nntp articles and attachments
type FilesystemStorage struct {
root string
discardAttachments bool
}
func (fs FilesystemStorage) String() string {
return fs.root
}
func (fs FilesystemStorage) NewsgroupsDir() string {
return filepath.Join(fs.root, "newsgroups")
}
func (fs FilesystemStorage) metadataFileForNewsgroup(newsgroup string) string {
return filepath.Join(fs.newsgroupDir(newsgroup), "metadata")
}
func (fs FilesystemStorage) GetWatermark(newsgroup string) (hi, lo uint64, err error) {
var hdr textproto.MIMEHeader
hdr, err = fs.getMetadataForNewsgroup(newsgroup)
if err == nil {
hi, err = strconv.ParseUint(hdr.Get(HighWaterHeader), 10, 64)
if err == nil {
lo, err = strconv.ParseUint(hdr.Get(LowWaterHeader), 10, 64)
}
}
return
}
func (fs FilesystemStorage) getMetadataForNewsgroup(newsgroup string) (hdr textproto.MIMEHeader, err error) {
var f *os.File
fp := fs.metadataFileForNewsgroup(newsgroup)
_, err = os.Stat(fp)
if os.IsNotExist(err) {
f, err = os.OpenFile(fp, os.O_RDWR|os.O_CREATE, 0600)
if err == nil {
h := make(textproto.MIMEHeader)
h.Set(HighWaterHeader, "0")
c := textproto.NewConn(f)
for k := range hdr {
for _, v := range h[k] {
err = c.PrintfLine("%s: %s", k, v)
if err != nil {
c.Close()
return
}
}
}
c.Close()
}
}
f, err = os.OpenFile(fp, os.O_RDWR, 0600)
if err == nil {
c := textproto.NewConn(f)
hdr, err = c.ReadMIMEHeader()
c.Close()
}
return
}
func (fs FilesystemStorage) nextIDForNewsgroup(newsgroup string) (id uint64, err error) {
id, _, err = fs.GetWatermark(newsgroup)
if err == nil {
id++
var hdr textproto.MIMEHeader
hdr, err = fs.getMetadataForNewsgroup(newsgroup)
if err == nil {
hdr.Set(HighWaterHeader, fmt.Sprintf("%d", id))
var f *os.File
f, err = os.OpenFile(fs.metadataFileForNewsgroup(newsgroup), os.O_WRONLY, 0600)
if err == nil {
c := textproto.NewConn(f)
for k := range hdr {
for _, v := range hdr[k] {
err = c.PrintfLine("%s: %s", k, v)
if err != nil {
c.Close()
return
}
}
}
c.Close()
}
}
}
return
}
func (fs FilesystemStorage) GetAllNewsgroups() (newsgroups []string, err error) {
err = filepath.Walk(fs.NewsgroupsDir(), filepath.WalkFunc(func(f string, info os.FileInfo, e error) (er error) {
if e != nil {
er = e
newsgroups = nil
}
if info.IsDir() && err == nil {
newsgroups = append(newsgroups, f)
}
return er
}))
return
}
func (fs FilesystemStorage) HasNewsgroup(newsgroup string) (has bool, err error) {
_, err = os.Stat(fs.newsgroupDir(newsgroup))
has = err == nil
return
}
// ensure the filesystem storage exists and is well formed and read/writable
func (fs FilesystemStorage) Ensure() (err error) {
_, err = os.Stat(fs.String())
if os.IsNotExist(err) {
// directory does not exist, create it
err = os.Mkdir(fs.String(), 0755)
if err != nil {
log.WithFields(log.Fields{
"pkg": "fs-store",
"filepath": fs.String(),
}).Error("failed to ensure directory", err)
// failed to create initial directory
return
}
}
// ensure subdirectories
for _, subdir := range []string{"att", "thm", "articles", "tmp", "newsgroups"} {
fpath := filepath.Join(fs.String(), subdir)
_, err = os.Stat(fpath)
if os.IsNotExist(err) {
// make subdirectory
err = os.Mkdir(fpath, 0755)
if err != nil {
log.WithFields(log.Fields{
"pkg": "fs-store",
"filepath": fpath,
}).Error("failed to ensure sub-directory", err)
// failed to create subdirectory
return
}
}
}
return
}
// get the temp file directory
func (fs FilesystemStorage) TempDir() string {
return filepath.Join(fs.String(), "tmp")
}
// get the directory path for attachments
func (fs FilesystemStorage) AttachmentDir() string {
return filepath.Join(fs.String(), "att")
}
// get the directory path for articles
func (fs FilesystemStorage) ArticleDir() string {
return filepath.Join(fs.String(), "articles")
}
// get a temporary file we can use for read/write that deletes itself on close
func (fs FilesystemStorage) obtainTempFile() (f *os.File, err error) {
fname := fmt.Sprintf("tempfile-%x-%d", crypto.RandBytes(4), time.Now().Unix())
log.WithFields(log.Fields{
"pkg": "fs-store",
"filepath": fname,
}).Debug("opening temp file")
f, err = os.OpenFile(filepath.Join(fs.TempDir(), fname), os.O_RDWR|os.O_CREATE, 0400)
return
}
// store an article from a reader to disk
func (fs FilesystemStorage) StoreArticle(r io.Reader, msgid, newsgroup string) (fpath string, err error) {
err = fs.HasArticle(msgid)
if err == nil {
// discard the body as we have it stored already
_, err = io.Copy(ioutil.Discard, r)
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
}).Debug("discard article")
} else if err == ErrNoSuchArticle {
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
}).Debug("storing article")
// don't have an article with this message id, write it to disk
var f *os.File
fpath = filepath.Join(fs.ArticleDir(), msgid)
f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644)
if err == nil {
// file opened okay, defer the close
defer f.Close()
// write to disk
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
}).Debug("writing to disk")
var n int64
n, err = io.Copy(f, r)
if err == nil {
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
"written": n,
}).Debug("wrote article to disk")
// symlink
g := fs.newsgroupDir(newsgroup)
_, e := os.Stat(g)
if os.IsNotExist(e) {
err = os.Mkdir(g, 0700)
}
var nntpid uint64
nntpid, err = fs.nextIDForNewsgroup(newsgroup)
if err == nil {
err = os.Symlink(filepath.Join("..", "..", "articles", msgid), filepath.Join(g, fmt.Sprintf("%d", nntpid)))
}
if err != nil {
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
"group": newsgroup,
}).Debug("failed to link article")
}
} else {
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
"written": n,
}).Error("write to disk failed")
}
} else {
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
"filepath": fpath,
}).Error("did not open file for storage", err)
}
}
return
}
func (fs FilesystemStorage) newsgroupDir(group string) string {
return filepath.Join(fs.NewsgroupsDir(), group)
}
// check if we have the artilce with this message id
func (fs FilesystemStorage) HasArticle(msgid string) (err error) {
fpath := fs.ArticleDir()
fpath = filepath.Join(fpath, msgid)
log.WithFields(log.Fields{
"pkg": "fs-store",
"msgid": msgid,
"filepath": fpath,
}).Debug("check for article")
_, err = os.Stat(fpath)
if os.IsNotExist(err) {
err = ErrNoSuchArticle
}
return
}
func (fs FilesystemStorage) DeleteArticle(msgid string) (err error) {
err = os.Remove(filepath.Join(fs.ArticleDir(), msgid))
return
}
// store attachment onto filesystem
func (fs FilesystemStorage) StoreAttachment(r io.Reader, filename string) (fpath string, err error) {
if fs.discardAttachments {
_, err = io.Copy(ioutil.Discard, r)
return
}
// open temp file for storage
var tf *os.File
tf, err = fs.obtainTempFile()
if err == nil {
// we have the temp file
// close tempfile when done
defer func() {
n := tf.Name()
tf.Close()
os.Remove(n)
}()
// create hasher
h := crypto.Hash()
// create multiwriter
mw := io.MultiWriter(tf, h)
log.WithFields(log.Fields{
"pkg": "fs-store",
"filename": filename,
}).Debug("writing to disk")
var n int64
// write all of the reader to the multiwriter
n, err = io.Copy(mw, r)
if err == nil {
// successful write
// get file checksum
d := h.Sum(nil)
// rename file to hash + extension from filename
fpath = base32.StdEncoding.EncodeToString(d) + filepath.Ext(filename)
fpath = filepath.Join(fs.AttachmentDir(), fpath)
_, err = os.Stat(fpath)
// is that file there?
if os.IsNotExist(err) {
// it's not there, let's write it
var f *os.File
f, err = os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE, 0644)
if err == nil {
// file opened
defer f.Close()
// seek to beginning of tempfile
tf.Seek(0, os.SEEK_SET)
// write all of the temp file to the storage file
n, err = io.Copy(f, tf)
// if err == nil by here it's all good
l := log.WithFields(log.Fields{
"pkg": "fs-store",
"filename": filename,
"hash": d,
"filepath": fpath,
"size": n,
})
if err == nil {
l.Debug("wrote attachment to disk")
} else {
l.Error("failed to write attachment to disk", err)
}
} else {
log.WithFields(log.Fields{
"pkg": "fs-store",
"filename": filename,
"hash": d,
"filepath": fpath,
}).Error("failed to open file")
}
} else {
log.WithFields(log.Fields{
"pkg": "fs-store",
"filename": filename,
"hash": d,
"filepath": fpath,
"size": n,
}).Debug("attachment exists on disk")
}
}
} else {
log.WithFields(log.Fields{
"pkg": "fs-store",
"filename": filename,
}).Error("cannot open temp file for attachment", err)
}
return
}
// open article given message-id
// does not check validity
func (fs FilesystemStorage) OpenArticle(msgid string) (r *os.File, err error) {
r, err = os.Open(filepath.Join(fs.ArticleDir(), msgid))
return
}
func (fs FilesystemStorage) ForEachInGroup(group string, chnl chan string) {
g := fs.newsgroupDir(group)
filepath.Walk(g, func(path string, info os.FileInfo, err error) error {
if info != nil {
chnl <- info.Name()
}
return err
})
}
// create a new filesystem storage directory
// ensure directory and subdirectories
func NewFilesytemStorage(dirname string, unpackAttachments bool) (fs FilesystemStorage, err error) {
dirname, err = filepath.Abs(dirname)
if err == nil {
log.WithFields(log.Fields{
"pkg": "fs-store",
"filepath": dirname,
}).Info("Creating New Filesystem Storage")
fs = FilesystemStorage{
root: dirname,
discardAttachments: unpackAttachments,
}
err = fs.Ensure()
}
return
}

View File

@ -0,0 +1 @@
package store

View File

@ -0,0 +1,62 @@
package store
import (
"io"
"nntpchan/lib/util"
"os"
)
type nullStore struct{}
func (n *nullStore) discard(r io.Reader) (s string, err error) {
_, err = io.Copy(util.Discard, r)
s = "/dev/null"
return
}
func (n *nullStore) HasArticle(msgid string) error {
return ErrNoSuchArticle
}
func (n *nullStore) StoreAttachment(r io.Reader, filename string) (string, error) {
return n.discard(r)
}
func (n *nullStore) StoreArticle(r io.Reader, msgid, newsgroup string) (string, error) {
return n.discard(r)
}
func (n *nullStore) DeleteArticle(msgid string) (err error) {
return
}
func (n *nullStore) Ensure() (err error) {
return
}
func (n *nullStore) ForEachInGroup(newsgroup string, chnl chan string) {
return
}
func (n *nullStore) OpenArticle(msgid string) (r *os.File, err error) {
err = ErrNoSuchArticle
return
}
func (n *nullStore) HasNewsgroup(newsgroup string) (has bool, err error) {
has = true
return
}
func (n *nullStore) GetAllNewsgroups() (list []string, err error) {
return
}
func (n *nullStore) GetWatermark(newsgroup string) (hi, lo uint64, err error) {
return
}
// create a storage backend that does nothing
func NewNullStorage() Storage {
return &nullStore{}
}

View File

@ -0,0 +1,50 @@
package store
import (
"errors"
"io"
"os"
)
var ErrNoSuchArticle = errors.New("no such article")
// storage for nntp articles and attachments
type Storage interface {
// store an attachment that we read from an io.Reader
// filename is used to hint to store what extension to store it as
// returns absolute filepath where attachment was stored and nil on success
// returns emtpy string and error if an error ocurred while storing
StoreAttachment(r io.Reader, filename string) (string, error)
// store an article that we read from an io.Reader
// message id is used to hint where the article is stored as well as newsgroup
// returns absolute filepath to where the article was stored and nil on success
// returns empty string and error if an error ocurred while storing
StoreArticle(r io.Reader, msgid, newsgroup string) (string, error)
// return nil if the article with the given message id exists in this storage
// return ErrNoSuchArticle if it does not exist or an error if another error occured while checking
HasArticle(msgid string) error
// delete article from underlying storage
DeleteArticle(msgid string) error
// open article for reading
OpenArticle(msgid string) (*os.File, error)
// ensure the underlying storage backend is created
Ensure() error
// iterate over all messages in a newsgroup
// send results down a channel
ForEachInGroup(newsgroup string, cnhl chan string)
// get a list of all newsgroups
GetAllNewsgroups() ([]string, error)
// determine if we have a newsgroup
HasNewsgroup(newsgroup string) (bool, error)
// get hi/lo watermark for newsgroup
GetWatermark(newsgroup string) (uint64, uint64, error)
}

View File

@ -0,0 +1,4 @@
//
// attachment thumbnailing
//
package thumbnail

View File

@ -0,0 +1,40 @@
package thumbnail
import (
"os/exec"
"regexp"
)
// thumbnail by executing an external program
type ExecThumbnailer struct {
// path to executable
Exec string
// regular expression that checks for acceptable infiles
Accept *regexp.Regexp
// function to generate arguments to use with external program
// inf and outf are the filenames of the input and output files respectively
// if this is nil the command will be passed in 2 arguments, infile and outfile
GenArgs func(inf, outf string) []string
}
func (exe *ExecThumbnailer) CanThumbnail(infpath string) bool {
re := exe.Accept.Copy()
return re.MatchString(infpath)
}
func (exe *ExecThumbnailer) Generate(infpath, outfpath string) (err error) {
// do sanity check
if exe.CanThumbnail(infpath) {
var args []string
if exe.GenArgs == nil {
args = []string{infpath, outfpath}
} else {
args = exe.GenArgs(infpath, outfpath)
}
cmd := exec.Command(exe.Exec, args...)
_, err = cmd.CombinedOutput()
} else {
err = ErrCannotThumbanil
}
return
}

View File

@ -0,0 +1,47 @@
package thumbnail
import (
"errors"
)
var ErrNoThumbnailer = errors.New("no thumbnailer found")
type multiThumbnailer struct {
impls []Thumbnailer
}
// get the frist matching thumbnailer that works with the given file
// if we can't find one return nil
func (mth *multiThumbnailer) getThumbnailer(fpath string) Thumbnailer {
for _, th := range mth.impls {
if th.CanThumbnail(fpath) {
return th
}
}
return nil
}
func (mth *multiThumbnailer) Generate(infpath, outfpath string) (err error) {
th := mth.getThumbnailer(infpath)
if th == nil {
err = ErrNoThumbnailer
} else {
err = th.Generate(infpath, outfpath)
}
return
}
func (mth *multiThumbnailer) CanThumbnail(infpath string) bool {
for _, th := range mth.impls {
if th.CanThumbnail(infpath) {
return true
}
}
return false
}
func MuxThumbnailers(th ...Thumbnailer) Thumbnailer {
return &multiThumbnailer{
impls: th,
}
}

View File

@ -0,0 +1,76 @@
package thumbnail
import (
"errors"
"fmt"
"regexp"
"strings"
)
var ErrCannotThumbanil = errors.New("cannot thumbnail file")
// a generator of thumbnails
type Thumbnailer interface {
// generate thumbnail of attachment
//
// infpath: absolute filepath to attachment
//
// outfpath: absolute filepath to thumbnail
//
// return error if the thumbnailing fails
Generate(infpath, outfpath string) error
// can we generate a thumbnail for this file?
CanThumbnail(infpath string) bool
}
// thumbnail configuration
type Config struct {
// width of thumbnails
ThumbW int
// height of thumbnails
ThumbH int
// only generate jpg thumbnails
JpegOnly bool
}
var defaultCfg = &Config{
ThumbW: 300,
ThumbH: 200,
JpegOnly: true,
}
// create an imagemagick thumbnailer
func ImageMagickThumbnailer(convertPath string, conf *Config) Thumbnailer {
if conf == nil {
conf = defaultCfg
}
return &ExecThumbnailer{
Exec: convertPath,
Accept: regexp.MustCompilePOSIX(`\.(png|jpg|jpeg|gif|webp)$`),
GenArgs: func(inf, outf string) []string {
if strings.HasSuffix(inf, ".gif") {
inf += "[0]"
}
if conf.JpegOnly {
outf += ".jpeg"
}
return []string{"-thumbnail", fmt.Sprintf("%d", conf.ThumbW), inf, outf}
},
}
}
// generate a thumbnailer that uses ffmpeg
func FFMpegThumbnailer(ffmpegPath string, conf *Config) Thumbnailer {
if conf == nil {
conf = defaultCfg
}
return &ExecThumbnailer{
Exec: ffmpegPath,
Accept: regexp.MustCompilePOSIX(`\.(mkv|mp4|avi|webm|ogv|mov|m4v|mpg)$`),
GenArgs: func(inf, outf string) []string {
outf += ".jpeg"
return []string{"-i", inf, "-vf", fmt.Sprintf("scale=%d:%d", conf.ThumbW, conf.ThumbH), "-vframes", "1", outf}
},
}
}

View File

@ -0,0 +1,56 @@
package thumbnail
import (
"testing"
)
func doTestThumb(t *testing.T, th Thumbnailer, allowed, disallowed []string) {
for _, f := range allowed {
if !th.CanThumbnail(f) {
t.Logf("cannot thumbnail expected file: %s", f)
t.Fail()
}
}
for _, f := range disallowed {
if th.CanThumbnail(f) {
t.Logf("can thumbnail wrong file: %s", f)
t.Fail()
}
}
}
var _image = []string{"asd.gif", "asd.jpeg", "asd.jpg", "asd.png", "asd.webp"}
var _video = []string{"asd.mkv", "asd.mov", "asd.mp4", "asd.m4v", "asd.ogv", "asd.avi", "asd.mpg", "asd.webm"}
var _sound = []string{"asd.flac", "asd.mp3", "asd.mp2", "asd.wav", "asd.ogg", "asd.opus", "asd.m4a"}
var _misc = []string{"asd.txt", "asd.swf"}
var _garbage = []string{"asd", "asd.asd", "asd.asd.asd.asd", "asd.benis"}
func TestCanThumbnailImage(t *testing.T) {
th := ImageMagickThumbnailer("", nil)
var allowed []string
var disallowed []string
allowed = append(allowed, _image...)
disallowed = append(disallowed, _video...)
disallowed = append(disallowed, _sound...)
disallowed = append(disallowed, _misc...)
disallowed = append(disallowed, _garbage...)
doTestThumb(t, th, allowed, disallowed)
}
func TestCanThumbnailVideo(t *testing.T) {
th := FFMpegThumbnailer("", nil)
var allowed []string
var disallowed []string
allowed = append(allowed, _video...)
disallowed = append(disallowed, _image...)
disallowed = append(disallowed, _sound...)
disallowed = append(disallowed, _misc...)
disallowed = append(disallowed, _garbage...)
doTestThumb(t, th, allowed, disallowed)
}

View File

@ -0,0 +1,87 @@
package util
import (
"fmt"
"path/filepath"
"regexp"
"strconv"
)
func GetThreadHashHTML(file string) (thread string) {
exp := regexp.MustCompilePOSIX(`thread-([0-9a-f]+)\.html`)
matches := exp.FindStringSubmatch(file)
if len(matches) != 2 {
return ""
}
thread = matches[1]
return
}
func GetGroupAndPageHTML(file string) (board string, page int) {
exp := regexp.MustCompilePOSIX(`(.*)-([0-9]+)\.html`)
matches := exp.FindStringSubmatch(file)
if len(matches) != 3 {
return "", -1
}
var err error
board = matches[1]
tmp := matches[2]
page, err = strconv.Atoi(tmp)
if err != nil {
page = -1
}
return
}
func GetGroupForCatalogHTML(file string) (group string) {
exp := regexp.MustCompilePOSIX(`catalog-(.+)\.html`)
matches := exp.FindStringSubmatch(file)
if len(matches) != 2 {
return ""
}
group = matches[1]
return
}
func GetFilenameForBoardPage(webroot_dir, boardname string, pageno int, json bool) string {
var ext string
if json {
ext = "json"
} else {
ext = "html"
}
fname := fmt.Sprintf("%s-%d.%s", boardname, pageno, ext)
return filepath.Join(webroot_dir, fname)
}
func GetFilenameForThread(webroot_dir, root_post_id string, json bool) string {
var ext string
if json {
ext = "json"
} else {
ext = "html"
}
fname := fmt.Sprintf("thread-%s.%s", HashMessageID(root_post_id), ext)
return filepath.Join(webroot_dir, fname)
}
func GetFilenameForCatalog(webroot_dir, boardname string) string {
fname := fmt.Sprintf("catalog-%s.html", boardname)
return filepath.Join(webroot_dir, fname)
}
func GetFilenameForIndex(webroot_dir string) string {
return filepath.Join(webroot_dir, "index.html")
}
func GetFilenameForBoards(webroot_dir string) string {
return filepath.Join(webroot_dir, "boards.html")
}
func GetFilenameForHistory(webroot_dir string) string {
return filepath.Join(webroot_dir, "history.html")
}
func GetFilenameForUkko(webroot_dir string) string {
return filepath.Join(webroot_dir, "ukko.html")
}

View File

@ -0,0 +1,14 @@
package util
type ioDiscard struct{}
func (discard *ioDiscard) Write(d []byte) (n int, err error) {
n = len(d)
return
}
func (discard *ioDiscard) Close() (err error) {
return
}
var Discard = new(ioDiscard)

Some files were not shown because too many files have changed in this diff Show More