* change feed defaults to be sane

* update docs

* add git revision to version

* bump to 2.5.1

Makefile

@@ -9,6 +9,9 @@ SRND_DIR=$(REPO)/contrib/backends/srndv2
 NNTPCHAND_DIR=$(REPO)/contrib/backends/nntpchand
 NNTPCHAN_DAEMON_DIR=$(REPO)/contrib/backends/nntpchan-daemon
 SRND=$(REPO)/srndv2
+NNTPCHAND=$(REPO)/nntpchand
+NNTPD=$(REPO)/nntpd
+
 GOROOT=$(shell go env GOROOT)
 GO=$(GOROOT)/bin/go
 
@@ -16,6 +19,10 @@ all: clean build
 
 build: js srnd
 
+full: clean full-build
+
+full-build: srnd beta native
+
 js: $(JS)
 
 srnd: $(SRND)
@@ -37,18 +44,48 @@ $(SRND):
 	GOROOT=$(GOROOT) $(MAKE) -C $(SRND_DIR)
 	cp $(SRND_DIR)/srndv2 $(SRND)
 
-test: test-go test-native
+beta: $(NNTPCHAND)
 
-test-go:
+$(NNTPCHAND):
+	GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAND_DIR)
+	cp $(NNTPCHAND_DIR)/nntpchand $(NNTPCHAND)
+
+native: $(NNTPD)
+
+$(NNTPD):
+	$(MAKE) -C $(NNTPCHAN_DAEMON_DIR)
+	cp $(NNTPCHAN_DAEMON_DIR)/nntpd $(NNTPD)
+
+test: test-srnd
+
+test-full: test-srnd test-beta test-native
+
+test-srnd:
 	GOROOT=$(GOROOT) $(MAKE) -C $(SRND_DIR) test
+
+test-beta:
 	GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAND_DIR) test
 
 test-native:
 	GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAN_DAEMON_DIR) test
 
-clean:
+
-	rm -f $(SRND) $(JS)
+clean: clean-js clean-srnd clean-beta clean-native
+
+clean-srnd:
+	rm -f $(SRND)
 	GOROOT=$(GOROOT) $(MAKE) -C $(SRND_DIR) clean
 
+clean-js:
+	rm -f $(JS)
+
+clean-beta:
+	rm -f $(NNTPCHAND)
+	GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAND_DIR) clean
+
+clean-native:
+	rm -f $(NNTPD)
+	$(MAKE) -C $(NNTPCHAN_DAEMON_DIR) clean
+
 distclean: clean
 	rm -rf $(REPO_GOPATH)

README.md

@@ -9,6 +9,12 @@
 
 [This](doc) is a step-by-step guide for getting up-and-running with NNTPChan as well as documentation for developers who want to either work on NNTPChan directly or use NNTPChan in their aplications with the API.
 
+TL;DR edition (after installing [dependancies](doc/):
+
+    $ git clone https://github.com/majestrate/nntpchan
+    $ make
+    $ ./srndv2 setup 
+
 ## Bugs and issues
 
 *PLEASE* report any bugs you find while building, setting-up or using NNTPChan on the [GitHub issue tracker](https://github.com/majestrate/nntpchan/issues), the [issue tracker on tor](http://git.psii2pdloxelodts.onion/psi/nntpchan/), the [issue tracker on i2p](http://git.psi.i2p/psi/nntpchan/) or on the [GitGud issue tracker](https://gitgud.io/jeff/nntpchan/issues) so that the probelms can be resolved or discussed.

contrib/backends/srndv2/src/srnd/config.go

@@ -57,6 +57,7 @@ type FeedConfig struct {
 	Name             string
 	sync_interval    time.Duration
 	connections      int
+	disable          bool
 }
 
 type APIConfig struct {
@@ -107,16 +108,16 @@ func CheckConfig() {
 		if !CheckFile(os.Getenv("SRND_INI_PATH")) {
 			log.Printf("No config file found at %s...", os.Getenv("SRND_INI_PATH"))
 			var conf *configparser.Configuration
-			if !InstallerEnabled() {
+			if InstallerEnabled() {
-				log.Println("Creating srnd.ini in working directory...")
-				conf = GenSRNdConfig()
-			} else {
 				res := make(chan *configparser.Configuration)
 				installer := NewInstaller(res)
 				go installer.Start()
 				conf = <-res
 				installer.Stop()
 				close(res)
+			} else {
+				log.Println("Creating srnd.ini in working directory...")
+				conf = GenSRNdConfig()
 			}
 			err := configparser.Save(conf, "srnd.ini")
 			if err != nil {
@@ -138,18 +139,26 @@ func CheckConfig() {
 // generate default feeds.ini
 func GenFeedsConfig() error {
 	conf := configparser.NewConfiguration()
-	sect := conf.NewSection("feed-dummy")
+
-	sect.Add("proxy-type", "socks4a")
+	sect := conf.NewSection("global")
+	sect.Add("overchan.overchan", "1")
+	sect.Add("ctl", "1")
+	sect.Add("*", "0")
+
+	sect = conf.NewSection("feed-2hu")
+	sect.Add("proxy-type", "None")
 	sect.Add("proxy-host", "127.0.0.1")
 	sect.Add("proxy-port", "9050")
-	sect.Add("host", "dummy")
+	sect.Add("host", "2hu-ch.org")
 	sect.Add("port", "119")
-	sect.Add("connections", "1")
+	sect.Add("connections", "0")
+	sect.Add("sync", "1")
+	sect.Add("disable", "1")
 
-	sect = conf.NewSection("dummy")
+	sect = conf.NewSection("2hu")
-	sect.Add("overchan.*", "1")
+	sect.Add("overchan.overchan", "1")
-	sect.Add("ano.paste", "0")
 	sect.Add("ctl", "1")
+	sect.Add("*", "0")
 
 	return configparser.Save(conf, "feeds.ini")
 }
@@ -171,6 +180,7 @@ func GenSRNdConfig() *configparser.Configuration {
 	sect.Add("feeds", filepath.Join(".", "feeds.d"))
 	sect.Add("archive", "0")
 	sect.Add("article_lifetime", "0")
+	sect.Add("filters_file", "filters.txt")
 
 	// profiling settings
 	sect = conf.NewSection("pprof")
@@ -182,10 +192,15 @@ func GenSRNdConfig() *configparser.Configuration {
 	sect.Add("enable", "0")
 	sect.Add("exec", "/bin/true")
 
+	hostname, _ := os.Hostname()
+	if hostname == "" {
+		hostname = "!!please-manually-set-this"
+	}
+
 	// crypto related section
 	sect = conf.NewSection("crypto")
 	sect.Add("tls-keyname", "overchan")
-	sect.Add("tls-hostname", "!!put-hostname-or-ip-of-server-here")
+	sect.Add("tls-hostname", hostname)
 	sect.Add("tls-trust-dir", "certs")
 
 	// article store section
@@ -242,7 +257,7 @@ func GenSRNdConfig() *configparser.Configuration {
 	secret_bytes := randbytes(8)
 	secret := base32.StdEncoding.EncodeToString(secret_bytes)
 	sect.Add("api-secret", secret)
-
+	sect.Add("rapeme", "no")
 	return conf
 }
 
@@ -309,7 +324,7 @@ func ReadConfig() *SRNdConfig {
 		log.Fatal("cannot read config file ", fname)
 		return nil
 	}
-	var sconf SRNdConfig
+	sconf := new(SRNdConfig)
 
 	s, err = conf.Section("pprof")
 	if err == nil {
@@ -357,6 +372,12 @@ func ReadConfig() *SRNdConfig {
 
 	sconf.daemon = s.Options()
 
+	filtersFile := s.ValueOf("filters_file")
+
+	if filtersFile == "" {
+		filtersFile = "filters.txt"
+	}
+
 	s, err = conf.Section("database")
 	if err != nil {
 		log.Println("no section 'database' in srnd.ini")
@@ -420,7 +441,7 @@ func ReadConfig() *SRNdConfig {
 	var confs []FeedConfig
 	confs, sconf.inboundPolicy, err = feedParse(fname)
 	if err != nil {
-		log.Fatal("failed to parse", fname, err)
+		log.Fatal("failed to load feeds: ", err)
 	}
 
 	sconf.feeds = append(sconf.feeds, confs...)
@@ -437,23 +458,22 @@ func ReadConfig() *SRNdConfig {
 				log.Println("load feed", f)
 				confs, _, err := feedParse(f)
 				if err != nil {
-					log.Fatal("failed to parse feed", f, err)
+					log.Fatal("failed to parse feed ", f, ": ", err)
 				}
 				sconf.feeds = append(sconf.feeds, confs...)
 			}
 		}
 	}
 
-	filterFile := "filters.txt"
+	if CheckFile(filtersFile) {
-
+		log.Println("loading content filter file", filtersFile)
-	if CheckFile(filterFile) {
+		err = sconf.filter.LoadFile(filtersFile)
-		err = sconf.filter.LoadFile(filterFile)
 		if err != nil {
-			log.Fatalf("failed to load %s: %s", filterFile, err)
+			log.Fatalf("failed to load %s: %s", filtersFile, err)
 		}
 		log.Printf("loaded %d filters", len(sconf.filter.globalFilters))
 	}
-	return &sconf
+	return sconf
 }
 
 func feedParse(fname string) (confs []FeedConfig, inboundPolicy *FeedPolicy, err error) {
@@ -464,7 +484,7 @@ func feedParse(fname string) (confs []FeedConfig, inboundPolicy *FeedPolicy, err
 		return
 	}
 
-	default_sect, err := conf.Section("")
+	default_sect, err := conf.Section("global")
 	if err == nil {
 		opts := default_sect.Options()
 		inboundPolicy = &FeedPolicy{
@@ -505,6 +525,9 @@ func feedParse(fname string) (confs []FeedConfig, inboundPolicy *FeedPolicy, err
 				fconf.sync_interval = time.Second * time.Duration(i)
 			}
 
+			// check for feed disabled
+			fconf.disable = sect.ValueOf("disable") == "1"
+
 			// concurrent connection count
 			fconf.connections = mapGetInt(sect.Options(), "connections", 1)
 

contrib/backends/srndv2/src/srnd/daemon.go

@@ -304,6 +304,10 @@ func (self *NNTPDaemon) storeFeedsConfig() (err error) {
 	return
 }
 
+func (self *NNTPDaemon) AllowsNewsgroup(group string) bool {
+	return self.conf.inboundPolicy == nil || self.conf.inboundPolicy.AllowsNewsgroup(group)
+}
+
 // change a feed's policy given the feed's name
 // return error if one occurs while modifying feed's policy
 func (self *NNTPDaemon) modifyFeedPolicy(feedname string, policy FeedPolicy) (err error) {
@@ -373,6 +377,10 @@ func (self *NNTPDaemon) messageSizeLimitFor(newsgroup string) int64 {
 }
 
 func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) {
+	if conf.disable {
+		log.Println(conf.Name, "is disabled not persisting")
+		return
+	}
 	log.Println(conf.Name, "persisting in", mode, "mode")
 	backoff := time.Second
 	for {

contrib/backends/srndv2/src/srnd/installer.go

@@ -326,7 +326,7 @@ func (self *Installer) HandleInstallerPost(wr http.ResponseWriter, r *http.Reque
 		next, newErr := self.currentNode.post(self.currentNode, r.PostForm, self.config)
 		if next == nil {
 			self.result <- self.config
-			//defer self.srv.Stop(10 * time.Second)
+			go self.srv.Stop(5 * time.Second)
 		}
 		self.currentNode = next
 		self.currentErr = newErr
@@ -490,7 +490,7 @@ func checkHost(host string) error {
 
 func (self *Installer) Start() {
 	log.Println("starting installer on", self.srv.Server.Addr)
-	log.Println("open up http://127.0.0.1:18000 to do initial configuration")
+	log.Println(fmt.Sprintf("open up http://youserver%s/ to do initial configuration", self.srv.Server.Addr))
 	self.srv.ListenAndServe()
 }
 
@@ -499,5 +499,5 @@ func (self *Installer) Stop() {
 }
 
 func InstallerEnabled() bool {
-	return os.Getenv("SRND_NO_INSTALLER") != "1"
+	return os.Getenv("SRND_INSTALLER") != "0"
 }

contrib/backends/srndv2/src/srnd/nntp.go

@@ -1393,11 +1393,13 @@ func (self *nntpConnection) scrapeServer(daemon *NNTPDaemon, conn *textproto.Con
 						if banned {
 							// we don't want it
 						} else if err == nil {
-							// scrape the group
+							if daemon.AllowsNewsgroup(group) {
-							err = self.scrapeGroup(daemon, conn, group)
+								// scrape the group
-							if err != nil {
+								err = self.scrapeGroup(daemon, conn, group)
-								log.Println(self.name, "did not scrape", group, err)
+								if err != nil {
-								break
+									log.Println(self.name, "did not scrape", group, err)
+									break
+								}
 							}
 						} else {
 							// error while checking for ban

contrib/backends/srndv2/src/srnd/postgres.go

@@ -808,6 +808,9 @@ func (self *PostgresDatabase) AddModPubkey(pubkey string) error {
 
 func (self *PostgresDatabase) GetGroupForMessage(message_id string) (group string, err error) {
 	err = self.conn.QueryRow("SELECT newsgroup FROM ArticlePosts WHERE message_id = $1", message_id).Scan(&group)
+	if err == sql.ErrNoRows {
+		err = nil
+	}
 	return
 }
 
@@ -830,6 +833,8 @@ func (self *PostgresDatabase) GetInfoForMessage(msgid string) (root string, news
 		perpage, _ := self.GetPagesPerBoard(newsgroup)
 		err = self.conn.QueryRow("WITH thread(bump) AS (SELECT last_bump FROM ArticleThreads WHERE root_message_id = $1 ) SELECT COUNT(*) FROM ( SELECT last_bump FROM ArticleThreads INNER JOIN thread ON (thread.bump <= ArticleThreads.last_bump AND newsgroup = $2 ) ) AS amount", root, newsgroup).Scan(&page)
 		page = page / int64(perpage)
+	} else if err == sql.ErrNoRows {
+		err = nil
 	}
 	return
 }
@@ -956,6 +961,9 @@ func (self *PostgresDatabase) UnmarkPubkeyAdmin(pubkey string) (err error) {
 func (self *PostgresDatabase) CheckAdminPubkey(pubkey string) (admin bool, err error) {
 	var count int64
 	err = self.conn.QueryRow("SELECT COUNT(pubkey) FROM ModPrivs WHERE pubkey = $1 AND permission = $2", pubkey, "admin").Scan(&count)
+	if err == sql.ErrNoRows {
+		err = nil
+	}
 	if err == nil {
 		admin = count > 0
 	}
@@ -1106,10 +1114,11 @@ func (self *PostgresDatabase) GetPostModel(prefix, messageID string) PostModel {
 		// quiet fail
 		self.conn.QueryRow(self.stmt[GetArticlePubkey], messageID).Scan(&model.Key)
 		return model
-	} else {
+	} else if err != sql.ErrNoRows {
 		log.Println("failed to prepare query for geting post model for", messageID, err)
 		return nil
 	}
+	return nil
 }
 
 func (self *PostgresDatabase) GetCitesByPostHashLike(like string) (cites []MessageIDTuple, err error) {

contrib/backends/srndv2/src/srnd/version.go

@@ -7,7 +7,7 @@ package srnd
 import "fmt"
 
 const major_version = 5
-const minor_version = 0
+const minor_version = 1
 const program_name = "srnd"
 
 var GitVersion string

contrib/templates/default/inst_db.mustache

@@ -25,6 +25,7 @@
              <input type="radio" name="db" value="postgres" checked> {{#i18n.Translations}}{{postgres_name}}{{/i18n.Translations}}
            </td>
          </tr>
+         <!--
          <tr>
            <th>
              
@@ -33,6 +34,7 @@
              <input type="radio" name="db" value="redis" > {{#i18n.Translations}}{{redis_name}}{{/i18n.Translations}}
            </td>
          </tr>
+         -->
        </tbody>
      </table>
    </div>

doc/README.md

@@ -17,3 +17,4 @@ Hey, welcome to the documentation. This will help you use and develop with NNTPC
 
 1. [Protocol](developer/protocol.md) - NNTPChan's protocol specification
 2. [JSON-RPC API](developer/api.md) - NNTPChan's JSON-RPC API
+3. [In Progress Sub Projects](developer/subprojects.md) - Ongoing subprojects that are not deployed on the mainline network yet

doc/building-debian8.5.md

@@ -1,41 +0,0 @@
-# How to install nntpchan on Debian stable
-
-Install the initial dependencies:
-
-```
-apt-get -y --no-install-recommends install imagemagick sox git ca-certificates \
-ffmpeg build-essential tcl8.5 postgresql postgresql-contrib
-```
-
-## Configure postgresql
-
-```
-su - postgres -c "createuser --pwprompt --encrypted srnd"
-su - postgres -c "createdb srnd"
-```
-Don't forget the password you make for the srnd user, you will need it for configuration.
-## Install golang
-
-Download the golang tarball, extract it to `/usr/local`, and add it to the global profile:
-
-```
-cd /opt
-wget https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz
-tar -C /usr/local/ -xvzf go1.9.linux-amd64.tar.gz
-echo 'export PATH="$PATH:/usr/local/go/bin"' >> /etc/profile
-```
-
-Your `PATH` is set at login, so log out and back in before proceeding.
-
-## Install nntpchan
-
-```
-cd /opt
-git clone https://github.com/majestrate/nntpchan.git
-cd nntpchan
-make
-```
-
-Now you can proceed with [setting up NNTPChan](setting-up.md). When you get to the "set paths to external programs" step.
-
-Run `./srndv2 setup` and follow the instructions [here](setting-up.md).

doc/building-ubuntu16.10.md

@@ -1,16 +0,0 @@
-Install the initial dependencies:
-
-    # apt-get -y --no-install-recommends install imagemagick sox git ca-certificates \
-    ffmpeg build-essential tcl8.5 postgresql postgresql-contrib golang-go
-
-Configure PostgreSQL:
-
-    # su - postgres -c "createuser --pwprompt --encrypted srnd"
-    # su - postgres -c "createdb srnd"
-
-Install nntpchan:
-
-    # cd /opt
-    # git clone https://github.com/majestrate/nntpchan.git
-    # cd nntpchan
-    # make

doc/building.md

@@ -47,3 +47,4 @@ Run `make`:
 
     make
 
+Next [set up the daemon](setting-up.md)

doc/developer/subprojects.md

@@ -0,0 +1,60 @@
+# subprojects
+
+This is a list of subprojects related to the main nntpchan daemon
+
+Mostly rewrite attempts
+
+## nntpchand
+
+Golang refactor of srndv2
+
+Build Requirements:
+
+* go 1.9
+
+* GNU Make
+
+To build:
+
+    $ make beta
+    
+To run tests:
+    
+    $ make test-beta
+    
+To clean:
+    
+    $ make clean-beta
+    
+## nntpd
+
+Native C++ rewrite of nntpchan daemon
+
+Build Requirements:
+
+* GNU Make
+
+* clang c++ compiler
+
+* pkg-config
+
+* libuv-1.x
+
+* libsodium-1.x
+
+To build:
+
+    $ make native
+    
+To run tests:
+
+    $ make native-test
+    
+To clean:
+
+    $ make clean-native
+    
+    
+## Tests
+
+    $ make test-full