diff --git a/Makefile b/Makefile index 418f64a..d3ca1f1 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ SRND_DIR=$(REPO)/contrib/backends/srndv2 NNTPCHAND_DIR=$(REPO)/contrib/backends/nntpchand NNTPCHAN_DAEMON_DIR=$(REPO)/contrib/backends/nntpchan-daemon SRND=$(REPO)/srndv2 +NNTPCHAND=$(REPO)/nntpchand +NNTPD=$(REPO)/nntpd + GOROOT=$(shell go env GOROOT) GO=$(GOROOT)/bin/go @@ -16,6 +19,10 @@ all: clean build build: js srnd +full: clean full-build + +full-build: srnd beta native + js: $(JS) srnd: $(SRND) @@ -37,18 +44,48 @@ $(SRND): GOROOT=$(GOROOT) $(MAKE) -C $(SRND_DIR) cp $(SRND_DIR)/srndv2 $(SRND) -test: test-go test-native +beta: $(NNTPCHAND) -test-go: +$(NNTPCHAND): + GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAND_DIR) + cp $(NNTPCHAND_DIR)/nntpchand $(NNTPCHAND) + +native: $(NNTPD) + +$(NNTPD): + $(MAKE) -C $(NNTPCHAN_DAEMON_DIR) + cp $(NNTPCHAN_DAEMON_DIR)/nntpd $(NNTPD) + +test: test-srnd + +test-full: test-srnd test-beta test-native + +test-srnd: GOROOT=$(GOROOT) $(MAKE) -C $(SRND_DIR) test + +test-beta: GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAND_DIR) test test-native: GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAN_DAEMON_DIR) test -clean: - rm -f $(SRND) $(JS) + +clean: clean-js clean-srnd clean-beta clean-native + +clean-srnd: + rm -f $(SRND) GOROOT=$(GOROOT) $(MAKE) -C $(SRND_DIR) clean +clean-js: + rm -f $(JS) + +clean-beta: + rm -f $(NNTPCHAND) + GOROOT=$(GOROOT) $(MAKE) -C $(NNTPCHAND_DIR) clean + +clean-native: + rm -f $(NNTPD) + $(MAKE) -C $(NNTPCHAN_DAEMON_DIR) clean + distclean: clean rm -rf $(REPO_GOPATH) diff --git a/README.md b/README.md index eb1b2ee..cfdbb29 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ [This](doc) is a step-by-step guide for getting up-and-running with NNTPChan as well as documentation for developers who want to either work on NNTPChan directly or use NNTPChan in their aplications with the API. +TL;DR edition (after installing [dependancies](doc/): + + $ git clone https://github.com/majestrate/nntpchan + $ make + $ ./srndv2 setup + ## Bugs and issues *PLEASE* report any bugs you find while building, setting-up or using NNTPChan on the [GitHub issue tracker](https://github.com/majestrate/nntpchan/issues), the [issue tracker on tor](http://git.psii2pdloxelodts.onion/psi/nntpchan/), the [issue tracker on i2p](http://git.psi.i2p/psi/nntpchan/) or on the [GitGud issue tracker](https://gitgud.io/jeff/nntpchan/issues) so that the probelms can be resolved or discussed. diff --git a/contrib/backends/srndv2/src/srnd/config.go b/contrib/backends/srndv2/src/srnd/config.go index bc368e0..8e5c359 100644 --- a/contrib/backends/srndv2/src/srnd/config.go +++ b/contrib/backends/srndv2/src/srnd/config.go @@ -57,6 +57,7 @@ type FeedConfig struct { Name string sync_interval time.Duration connections int + disable bool } type APIConfig struct { @@ -107,16 +108,16 @@ func CheckConfig() { if !CheckFile(os.Getenv("SRND_INI_PATH")) { log.Printf("No config file found at %s...", os.Getenv("SRND_INI_PATH")) var conf *configparser.Configuration - if !InstallerEnabled() { - log.Println("Creating srnd.ini in working directory...") - conf = GenSRNdConfig() - } else { + if InstallerEnabled() { res := make(chan *configparser.Configuration) installer := NewInstaller(res) go installer.Start() conf = <-res installer.Stop() close(res) + } else { + log.Println("Creating srnd.ini in working directory...") + conf = GenSRNdConfig() } err := configparser.Save(conf, "srnd.ini") if err != nil { @@ -138,18 +139,26 @@ func CheckConfig() { // generate default feeds.ini func GenFeedsConfig() error { conf := configparser.NewConfiguration() - sect := conf.NewSection("feed-dummy") - sect.Add("proxy-type", "socks4a") + + sect := conf.NewSection("global") + sect.Add("overchan.overchan", "1") + sect.Add("ctl", "1") + sect.Add("*", "0") + + sect = conf.NewSection("feed-2hu") + sect.Add("proxy-type", "None") sect.Add("proxy-host", "127.0.0.1") sect.Add("proxy-port", "9050") - sect.Add("host", "dummy") + sect.Add("host", "2hu-ch.org") sect.Add("port", "119") - sect.Add("connections", "1") + sect.Add("connections", "0") + sect.Add("sync", "1") + sect.Add("disable", "1") - sect = conf.NewSection("dummy") - sect.Add("overchan.*", "1") - sect.Add("ano.paste", "0") + sect = conf.NewSection("2hu") + sect.Add("overchan.overchan", "1") sect.Add("ctl", "1") + sect.Add("*", "0") return configparser.Save(conf, "feeds.ini") } @@ -171,6 +180,7 @@ func GenSRNdConfig() *configparser.Configuration { sect.Add("feeds", filepath.Join(".", "feeds.d")) sect.Add("archive", "0") sect.Add("article_lifetime", "0") + sect.Add("filters_file", "filters.txt") // profiling settings sect = conf.NewSection("pprof") @@ -182,10 +192,15 @@ func GenSRNdConfig() *configparser.Configuration { sect.Add("enable", "0") sect.Add("exec", "/bin/true") + hostname, _ := os.Hostname() + if hostname == "" { + hostname = "!!please-manually-set-this" + } + // crypto related section sect = conf.NewSection("crypto") sect.Add("tls-keyname", "overchan") - sect.Add("tls-hostname", "!!put-hostname-or-ip-of-server-here") + sect.Add("tls-hostname", hostname) sect.Add("tls-trust-dir", "certs") // article store section @@ -242,7 +257,7 @@ func GenSRNdConfig() *configparser.Configuration { secret_bytes := randbytes(8) secret := base32.StdEncoding.EncodeToString(secret_bytes) sect.Add("api-secret", secret) - + sect.Add("rapeme", "no") return conf } @@ -309,7 +324,7 @@ func ReadConfig() *SRNdConfig { log.Fatal("cannot read config file ", fname) return nil } - var sconf SRNdConfig + sconf := new(SRNdConfig) s, err = conf.Section("pprof") if err == nil { @@ -357,6 +372,12 @@ func ReadConfig() *SRNdConfig { sconf.daemon = s.Options() + filtersFile := s.ValueOf("filters_file") + + if filtersFile == "" { + filtersFile = "filters.txt" + } + s, err = conf.Section("database") if err != nil { log.Println("no section 'database' in srnd.ini") @@ -420,7 +441,7 @@ func ReadConfig() *SRNdConfig { var confs []FeedConfig confs, sconf.inboundPolicy, err = feedParse(fname) if err != nil { - log.Fatal("failed to parse", fname, err) + log.Fatal("failed to load feeds: ", err) } sconf.feeds = append(sconf.feeds, confs...) @@ -437,23 +458,22 @@ func ReadConfig() *SRNdConfig { log.Println("load feed", f) confs, _, err := feedParse(f) if err != nil { - log.Fatal("failed to parse feed", f, err) + log.Fatal("failed to parse feed ", f, ": ", err) } sconf.feeds = append(sconf.feeds, confs...) } } } - filterFile := "filters.txt" - - if CheckFile(filterFile) { - err = sconf.filter.LoadFile(filterFile) + if CheckFile(filtersFile) { + log.Println("loading content filter file", filtersFile) + err = sconf.filter.LoadFile(filtersFile) if err != nil { - log.Fatalf("failed to load %s: %s", filterFile, err) + log.Fatalf("failed to load %s: %s", filtersFile, err) } log.Printf("loaded %d filters", len(sconf.filter.globalFilters)) } - return &sconf + return sconf } func feedParse(fname string) (confs []FeedConfig, inboundPolicy *FeedPolicy, err error) { @@ -464,7 +484,7 @@ func feedParse(fname string) (confs []FeedConfig, inboundPolicy *FeedPolicy, err return } - default_sect, err := conf.Section("") + default_sect, err := conf.Section("global") if err == nil { opts := default_sect.Options() inboundPolicy = &FeedPolicy{ @@ -505,6 +525,9 @@ func feedParse(fname string) (confs []FeedConfig, inboundPolicy *FeedPolicy, err fconf.sync_interval = time.Second * time.Duration(i) } + // check for feed disabled + fconf.disable = sect.ValueOf("disable") == "1" + // concurrent connection count fconf.connections = mapGetInt(sect.Options(), "connections", 1) diff --git a/contrib/backends/srndv2/src/srnd/daemon.go b/contrib/backends/srndv2/src/srnd/daemon.go index 5ce07e4..28a3c8e 100644 --- a/contrib/backends/srndv2/src/srnd/daemon.go +++ b/contrib/backends/srndv2/src/srnd/daemon.go @@ -304,6 +304,10 @@ func (self *NNTPDaemon) storeFeedsConfig() (err error) { return } +func (self *NNTPDaemon) AllowsNewsgroup(group string) bool { + return self.conf.inboundPolicy == nil || self.conf.inboundPolicy.AllowsNewsgroup(group) +} + // change a feed's policy given the feed's name // return error if one occurs while modifying feed's policy func (self *NNTPDaemon) modifyFeedPolicy(feedname string, policy FeedPolicy) (err error) { @@ -373,6 +377,10 @@ func (self *NNTPDaemon) messageSizeLimitFor(newsgroup string) int64 { } func (self *NNTPDaemon) persistFeed(conf *FeedConfig, mode string, n int) { + if conf.disable { + log.Println(conf.Name, "is disabled not persisting") + return + } log.Println(conf.Name, "persisting in", mode, "mode") backoff := time.Second for { diff --git a/contrib/backends/srndv2/src/srnd/installer.go b/contrib/backends/srndv2/src/srnd/installer.go index f34504a..5512ec8 100644 --- a/contrib/backends/srndv2/src/srnd/installer.go +++ b/contrib/backends/srndv2/src/srnd/installer.go @@ -326,7 +326,7 @@ func (self *Installer) HandleInstallerPost(wr http.ResponseWriter, r *http.Reque next, newErr := self.currentNode.post(self.currentNode, r.PostForm, self.config) if next == nil { self.result <- self.config - //defer self.srv.Stop(10 * time.Second) + go self.srv.Stop(5 * time.Second) } self.currentNode = next self.currentErr = newErr @@ -490,7 +490,7 @@ func checkHost(host string) error { func (self *Installer) Start() { log.Println("starting installer on", self.srv.Server.Addr) - log.Println("open up http://127.0.0.1:18000 to do initial configuration") + log.Println(fmt.Sprintf("open up http://youserver%s/ to do initial configuration", self.srv.Server.Addr)) self.srv.ListenAndServe() } @@ -499,5 +499,5 @@ func (self *Installer) Stop() { } func InstallerEnabled() bool { - return os.Getenv("SRND_NO_INSTALLER") != "1" + return os.Getenv("SRND_INSTALLER") != "0" } diff --git a/contrib/backends/srndv2/src/srnd/nntp.go b/contrib/backends/srndv2/src/srnd/nntp.go index d87cd84..3e1bcc1 100644 --- a/contrib/backends/srndv2/src/srnd/nntp.go +++ b/contrib/backends/srndv2/src/srnd/nntp.go @@ -1393,11 +1393,13 @@ func (self *nntpConnection) scrapeServer(daemon *NNTPDaemon, conn *textproto.Con if banned { // we don't want it } else if err == nil { - // scrape the group - err = self.scrapeGroup(daemon, conn, group) - if err != nil { - log.Println(self.name, "did not scrape", group, err) - break + if daemon.AllowsNewsgroup(group) { + // scrape the group + err = self.scrapeGroup(daemon, conn, group) + if err != nil { + log.Println(self.name, "did not scrape", group, err) + break + } } } else { // error while checking for ban diff --git a/contrib/backends/srndv2/src/srnd/postgres.go b/contrib/backends/srndv2/src/srnd/postgres.go index c83b78f..2b8970a 100644 --- a/contrib/backends/srndv2/src/srnd/postgres.go +++ b/contrib/backends/srndv2/src/srnd/postgres.go @@ -808,6 +808,9 @@ func (self *PostgresDatabase) AddModPubkey(pubkey string) error { func (self *PostgresDatabase) GetGroupForMessage(message_id string) (group string, err error) { err = self.conn.QueryRow("SELECT newsgroup FROM ArticlePosts WHERE message_id = $1", message_id).Scan(&group) + if err == sql.ErrNoRows { + err = nil + } return } @@ -830,6 +833,8 @@ func (self *PostgresDatabase) GetInfoForMessage(msgid string) (root string, news perpage, _ := self.GetPagesPerBoard(newsgroup) err = self.conn.QueryRow("WITH thread(bump) AS (SELECT last_bump FROM ArticleThreads WHERE root_message_id = $1 ) SELECT COUNT(*) FROM ( SELECT last_bump FROM ArticleThreads INNER JOIN thread ON (thread.bump <= ArticleThreads.last_bump AND newsgroup = $2 ) ) AS amount", root, newsgroup).Scan(&page) page = page / int64(perpage) + } else if err == sql.ErrNoRows { + err = nil } return } @@ -956,6 +961,9 @@ func (self *PostgresDatabase) UnmarkPubkeyAdmin(pubkey string) (err error) { func (self *PostgresDatabase) CheckAdminPubkey(pubkey string) (admin bool, err error) { var count int64 err = self.conn.QueryRow("SELECT COUNT(pubkey) FROM ModPrivs WHERE pubkey = $1 AND permission = $2", pubkey, "admin").Scan(&count) + if err == sql.ErrNoRows { + err = nil + } if err == nil { admin = count > 0 } @@ -1106,10 +1114,11 @@ func (self *PostgresDatabase) GetPostModel(prefix, messageID string) PostModel { // quiet fail self.conn.QueryRow(self.stmt[GetArticlePubkey], messageID).Scan(&model.Key) return model - } else { + } else if err != sql.ErrNoRows { log.Println("failed to prepare query for geting post model for", messageID, err) return nil } + return nil } func (self *PostgresDatabase) GetCitesByPostHashLike(like string) (cites []MessageIDTuple, err error) { diff --git a/contrib/backends/srndv2/src/srnd/version.go b/contrib/backends/srndv2/src/srnd/version.go index b660e70..1413fcd 100644 --- a/contrib/backends/srndv2/src/srnd/version.go +++ b/contrib/backends/srndv2/src/srnd/version.go @@ -7,7 +7,7 @@ package srnd import "fmt" const major_version = 5 -const minor_version = 0 +const minor_version = 1 const program_name = "srnd" var GitVersion string diff --git a/contrib/templates/default/inst_db.mustache b/contrib/templates/default/inst_db.mustache index 1af7884..54f00ba 100644 --- a/contrib/templates/default/inst_db.mustache +++ b/contrib/templates/default/inst_db.mustache @@ -25,6 +25,7 @@ {{#i18n.Translations}}{{postgres_name}}{{/i18n.Translations}} + diff --git a/doc/README.md b/doc/README.md index 63b3618..b5e7b49 100644 --- a/doc/README.md +++ b/doc/README.md @@ -17,3 +17,4 @@ Hey, welcome to the documentation. This will help you use and develop with NNTPC 1. [Protocol](developer/protocol.md) - NNTPChan's protocol specification 2. [JSON-RPC API](developer/api.md) - NNTPChan's JSON-RPC API +3. [In Progress Sub Projects](developer/subprojects.md) - Ongoing subprojects that are not deployed on the mainline network yet diff --git a/doc/building-debian8.5.md b/doc/building-debian8.5.md deleted file mode 100644 index 41a35d3..0000000 --- a/doc/building-debian8.5.md +++ /dev/null @@ -1,41 +0,0 @@ -# How to install nntpchan on Debian stable - -Install the initial dependencies: - -``` -apt-get -y --no-install-recommends install imagemagick sox git ca-certificates \ -ffmpeg build-essential tcl8.5 postgresql postgresql-contrib -``` - -## Configure postgresql - -``` -su - postgres -c "createuser --pwprompt --encrypted srnd" -su - postgres -c "createdb srnd" -``` -Don't forget the password you make for the srnd user, you will need it for configuration. -## Install golang - -Download the golang tarball, extract it to `/usr/local`, and add it to the global profile: - -``` -cd /opt -wget https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz -tar -C /usr/local/ -xvzf go1.9.linux-amd64.tar.gz -echo 'export PATH="$PATH:/usr/local/go/bin"' >> /etc/profile -``` - -Your `PATH` is set at login, so log out and back in before proceeding. - -## Install nntpchan - -``` -cd /opt -git clone https://github.com/majestrate/nntpchan.git -cd nntpchan -make -``` - -Now you can proceed with [setting up NNTPChan](setting-up.md). When you get to the "set paths to external programs" step. - -Run `./srndv2 setup` and follow the instructions [here](setting-up.md). diff --git a/doc/building-ubuntu16.10.md b/doc/building-ubuntu16.10.md deleted file mode 100644 index 696f29e..0000000 --- a/doc/building-ubuntu16.10.md +++ /dev/null @@ -1,16 +0,0 @@ -Install the initial dependencies: - - # apt-get -y --no-install-recommends install imagemagick sox git ca-certificates \ - ffmpeg build-essential tcl8.5 postgresql postgresql-contrib golang-go - -Configure PostgreSQL: - - # su - postgres -c "createuser --pwprompt --encrypted srnd" - # su - postgres -c "createdb srnd" - -Install nntpchan: - - # cd /opt - # git clone https://github.com/majestrate/nntpchan.git - # cd nntpchan - # make diff --git a/doc/building.md b/doc/building.md index d3db0d4..9670fbf 100644 --- a/doc/building.md +++ b/doc/building.md @@ -47,3 +47,4 @@ Run `make`: make +Next [set up the daemon](setting-up.md) diff --git a/doc/developer/subprojects.md b/doc/developer/subprojects.md new file mode 100644 index 0000000..c5eb7e4 --- /dev/null +++ b/doc/developer/subprojects.md @@ -0,0 +1,60 @@ +# subprojects + +This is a list of subprojects related to the main nntpchan daemon + +Mostly rewrite attempts + +## nntpchand + +Golang refactor of srndv2 + +Build Requirements: + +* go 1.9 + +* GNU Make + +To build: + + $ make beta + +To run tests: + + $ make test-beta + +To clean: + + $ make clean-beta + +## nntpd + +Native C++ rewrite of nntpchan daemon + +Build Requirements: + +* GNU Make + +* clang c++ compiler + +* pkg-config + +* libuv-1.x + +* libsodium-1.x + +To build: + + $ make native + +To run tests: + + $ make native-test + +To clean: + + $ make clean-native + + +## Tests + + $ make test-full