From f37a1422e2ccc2d7f375a591b66b1bf89dce9535 Mon Sep 17 00:00:00 2001 From: "Stanislav N. aka pztrn" Date: Wed, 5 Aug 2026 12:06:46 +0500 Subject: [PATCH] Initial commit. --- .gitignore | 8 ++ .golangci.yml | 128 +++++++++++++++++++ README.md | 13 ++ config.example.json | 23 ++++ configuration/configuration.go | 61 +++++++++ configuration/feed.go | 48 ++++++++ configuration/http_client.go | 22 ++++ configuration/nntp.go | 44 +++++++ configuration/outputs.go | 17 +++ configuration/rss.go | 29 +++++ fixers/fixers.go | 17 +++ fixers/opennet_ru.go | 17 +++ go.mod | 28 +++++ go.sum | 46 +++++++ main.go | 48 ++++++++ outputs/nntp/nntp.go | 219 +++++++++++++++++++++++++++++++++ outputs/output.go | 7 ++ outputs/stdout/stdout.go | 30 +++++ rss/parse.go | 102 +++++++++++++++ rss/rss.go | 55 +++++++++ rss/storage.go | 64 ++++++++++ 21 files changed, 1026 insertions(+) create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 README.md create mode 100644 config.example.json create mode 100644 configuration/configuration.go create mode 100644 configuration/feed.go create mode 100644 configuration/http_client.go create mode 100644 configuration/nntp.go create mode 100644 configuration/outputs.go create mode 100644 configuration/rss.go create mode 100644 fixers/fixers.go create mode 100644 fixers/opennet_ru.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 outputs/nntp/nntp.go create mode 100644 outputs/output.go create mode 100644 outputs/stdout/stdout.go create mode 100644 rss/parse.go create mode 100644 rss/rss.go create mode 100644 rss/storage.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2d0b7a0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*DS_Store* +fyne-cross +_build +*.apk +*.app +dist +.task +TASKS_NOTES.md diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..e6f8bd5 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,128 @@ +--- +version: "2" +linters: + default: all + disable: + - containedctx + - depguard + - exhaustruct + - funlen + - gochecknoglobals + - gocritic + - godot + - iface + - interfacebloat + - ireturn + - mnd + - paralleltest + - tagalign + - tagliatelle + - testpackage + - noinlineerr + - embeddedstructfieldcheck + - wsl # deprecated + - gomodguard # deprecated + - unqueryvet # we love SELECT * when working with databases + settings: + cyclop: + max-complexity: 25 + dupl: + threshold: 400 + forbidigo: + forbid: + - pattern: ^(fmt\.Print(|f|ln)|print|println)$ + - pattern: ^time\.Now\(\)($|\.F|\.A|\.B|\.L|\.UTC\(\)\.I|,|\))(# Calls of time\.Now() without \.UTC() is prohibited\.)? + funcorder: + constructor: true + struct-method: false + alphabetical: true + gocyclo: + min-complexity: 40 + govet: + enable-all: true + lll: + line-length: 120 + perfsprint: + concat-loop: false + revive: + enable-all-rules: true + rules: + # This linter creates more problems than gives profits. Probably should be configured? + - name: add-constant + disabled: true + # gocognit + - name: cognitive-complexity + disabled: true + - name: confusing-results + disabled: true + # gocyclo + - name: cyclomatic + disabled: true + - name: function-length + arguments: [200, 0] + - name: import-alias-naming + arguments: + - "^[a-z][a-zA-Z0-9]{0,}$" + - name: line-length-limit + disabled: true + - name: package-comments + disabled: true + - name: unused-receiver + disabled: true + - name: var-naming + arguments: + - ["ID"] + - ["VM"] + - - skipPackageNameChecks: true + - name: package-naming + disabled: true + wrapcheck: + ignore-sig-regexps: + - \.Write\( + - \.WriteString\( + - \.WriteJSON\( + - \.WriteHTML\( + - \.Redirect\( + - \.Error\( + wsl_v5: + allow-first-in-block: true + allow-whole-block: false + branch-max-lines: 2 + exclusions: + generated: lax + presets: + - common-false-positives + - legacy + - std-error-handling + rules: + - linters: + - godox + text: TODO + - linters: + - gosec + text: G101 + paths: + - billing + - cms + - monolit-apigateway + - third_party$ + - builtin$ + - examples$ +issues: + max-issues-per-linter: 0 + max-same-issues: 0 + uniq-by-line: false +formatters: + enable: + - gofmt + - gofumpt + settings: + gofumpt: + module-path: go.dev.pztrn.name/rsser + extra-rules: true + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/README.md b/README.md new file mode 100644 index 0000000..36ac24d --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# RSSer + +RSS parser with pluggable outputs. + +## Installation + +```shell +go install go.dev.pztrn.name/rsser +``` + +## Configuring + +See config.example.json for example configuration. diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..7bacd43 --- /dev/null +++ b/config.example.json @@ -0,0 +1,23 @@ +{ + "outputs": { + "nntp": { + "address": "news.host.tld:119", + "username": "user", + "password": "pass", + "connect_timeout": 5 + } + }, + "rss": { + "feeds": [ + { + "name": "OpenNET.ru: News", + "url": "https://www.opennet.ru/opennews/opennews_full.rss", + "output": "nntp", + "dest": "some.group" + } + ], + "http_client": { + "request_timeout_seconds": 10 + } + } +} diff --git a/configuration/configuration.go b/configuration/configuration.go new file mode 100644 index 0000000..791c04c --- /dev/null +++ b/configuration/configuration.go @@ -0,0 +1,61 @@ +package configuration + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +var errCfg = errors.New("configuration") + +// Config is a configuration controlling structure. +type Config struct { + RSS *RSS `json:"rss"` + Outputs *Outputs `json:"outputs"` +} + +// New creates new configuration controller. +func New(path string) (*Config, error) { + cfg := &Config{} + + // Get configuration file real path. + userHomeDir, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("%w: parse configuration file: get real path: get user home dir: %w", errCfg, err) + } + + path = strings.Replace(path, "~", userHomeDir, 1) + + absPath, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("%w: parse configuration file: get real path: get absolute path: %w", errCfg, err) + } + + cfgFileData, err := os.ReadFile(absPath) + if err != nil { + return nil, fmt.Errorf("%w: parse configuration file: read file: %w", errCfg, err) + } + + if err := json.Unmarshal(cfgFileData, cfg); err != nil { + return nil, fmt.Errorf("%w: parse configuration file: %w", errCfg, err) + } + + // Validate configuration. + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("%w: %w", errCfg, err) + } + + return cfg, nil +} + +// Validate validates app configuration. +func (c *Config) Validate() error { + if err := c.RSS.Validate(); err != nil { + return fmt.Errorf("validate configuration: %w", err) + } + + return nil +} diff --git a/configuration/feed.go b/configuration/feed.go new file mode 100644 index 0000000..b50d4f6 --- /dev/null +++ b/configuration/feed.go @@ -0,0 +1,48 @@ +package configuration + +import ( + "errors" + "fmt" + "net/url" +) + +var ( + errDestIsEmpty = errors.New("dest is empty") + errFeedValidate = errors.New("feed configuration validation") + errNameIsEmpty = errors.New("name is empty") + errOutputIsEmpty = errors.New("output is empty") + errURLIsEmpty = errors.New("URL is empty") +) + +// Feed is a single RSS feed configuration. +type Feed struct { + Name string `json:"name"` + URL string `json:"url"` + Output string `json:"output"` + Destination string `json:"dest"` +} + +// Validate validates RSS feed configuration. +func (f *Feed) Validate() error { + if f.Name == "" { + return fmt.Errorf("%w: %w", errFeedValidate, errNameIsEmpty) + } + + if f.Output == "" { + return fmt.Errorf("%w: %w", errFeedValidate, errOutputIsEmpty) + } + + if f.Destination == "" { + return fmt.Errorf("%w: %w", errFeedValidate, errDestIsEmpty) + } + + if f.URL == "" { + return fmt.Errorf("%w: %w", errFeedValidate, errURLIsEmpty) + } + + if _, err := url.Parse(f.URL); err != nil { + return fmt.Errorf("%w: parse feed URL: %w", errFeedValidate, err) + } + + return nil +} diff --git a/configuration/http_client.go b/configuration/http_client.go new file mode 100644 index 0000000..3fa005f --- /dev/null +++ b/configuration/http_client.go @@ -0,0 +1,22 @@ +package configuration + +import ( + "errors" + "fmt" +) + +var errRequestTimeoutTooLow = errors.New("request timeout is too low") + +// HTTPClient is an HTTP client configuration. +type HTTPClient struct { + RequestTimeoutSeconds int64 `json:"request_timeout_seconds"` +} + +// Validate validates configuration for HTTP clients. +func (h *HTTPClient) Validate() error { + if h.RequestTimeoutSeconds < 1 { + return fmt.Errorf("validate HTTP client configuration: %w", errRequestTimeoutTooLow) + } + + return nil +} diff --git a/configuration/nntp.go b/configuration/nntp.go new file mode 100644 index 0000000..f49a7bf --- /dev/null +++ b/configuration/nntp.go @@ -0,0 +1,44 @@ +package configuration + +import ( + "errors" + "fmt" + "net" +) + +var ( + errNNTPAddressIsEmpty = errors.New("address is empty") + errNNTPInvalidAddress = errors.New("address is invalid") + errNNTPInvalidConnTimeout = errors.New("connect timeout is invalid") + errNNTPUserOrPassIsEmpty = errors.New("username or password is empty") +) + +// NNTP is an NNTP output configuration. +type NNTP struct { + Address string `json:"address"` + Username string `json:"username"` + Password string `json:"password"` + ConnectTimeout int64 `json:"connect_timeout"` +} + +// Validate validates configuration. +func (n *NNTP) Validate() error { + if n.Address == "" { + return fmt.Errorf("validate NNTP output configuration: %w", errNNTPAddressIsEmpty) + } + + _, _, err := net.SplitHostPort(n.Address) + if err != nil { + return fmt.Errorf("validate NNTP output configuration: %w: %w", errNNTPInvalidAddress, err) + } + + if (n.Username != "" && n.Password == "") || (n.Username == "" && n.Password != "") { + return fmt.Errorf("validate NNTP output configuration: %w", errNNTPUserOrPassIsEmpty) + } + + if n.ConnectTimeout < 1 { + return fmt.Errorf("validate NNTP output configuration: %w", errNNTPInvalidConnTimeout) + } + + return nil +} diff --git a/configuration/outputs.go b/configuration/outputs.go new file mode 100644 index 0000000..149f00b --- /dev/null +++ b/configuration/outputs.go @@ -0,0 +1,17 @@ +package configuration + +import "fmt" + +// Outputs is an outputs configuration. +type Outputs struct { + NNTP *NNTP `json:"nntp"` +} + +// Validate validates configuration. +func (o *Outputs) Validate() error { + if err := o.NNTP.Validate(); err != nil { + return fmt.Errorf("validate outputs configuration: %w", err) + } + + return nil +} diff --git a/configuration/rss.go b/configuration/rss.go new file mode 100644 index 0000000..d0bbd2f --- /dev/null +++ b/configuration/rss.go @@ -0,0 +1,29 @@ +package configuration + +import ( + "errors" + "fmt" +) + +var errRSSValidate = errors.New("validate RSS configuration") + +// RSS is an RSS parser configuration. +type RSS struct { + HTTPClient *HTTPClient `json:"http_client"` + Feeds []*Feed `json:"feeds"` +} + +// Validate validates RSS parser configuration. +func (r *RSS) Validate() error { + for idx, feed := range r.Feeds { + if err := feed.Validate(); err != nil { + return fmt.Errorf("%w: feed %d: %w", errRSSValidate, idx, err) + } + } + + if err := r.HTTPClient.Validate(); err != nil { + return fmt.Errorf("%w: %w", errRSSValidate, err) + } + + return nil +} diff --git a/fixers/fixers.go b/fixers/fixers.go new file mode 100644 index 0000000..8174e8b --- /dev/null +++ b/fixers/fixers.go @@ -0,0 +1,17 @@ +package fixers + +type fixerFunc func(title, body string) (string, string) + +var fixers = map[string]fixerFunc{ + "www.opennet.ru": opennetFixer, +} + +// Fix fixes title and body depending on site parsed. +func Fix(domain, title, body string) (string, string) { + fixer, found := fixers[domain] + if !found { + return title, body + } + + return fixer(title, body) +} diff --git a/fixers/opennet_ru.go b/fixers/opennet_ru.go new file mode 100644 index 0000000..e22f32c --- /dev/null +++ b/fixers/opennet_ru.go @@ -0,0 +1,17 @@ +package fixers + +import "strings" + +func opennetFixer(title, body string) (string, string) { + newBody := &strings.Builder{} + + for line := range strings.Lines(body) { + if strings.HasPrefix(line, "Источник") { + continue + } + + _, _ = newBody.WriteString(line) + } + + return title, newBody.String() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5c02b9a --- /dev/null +++ b/go.mod @@ -0,0 +1,28 @@ +module go.dev.pztrn.name/rsser + +go 1.26.5 + +require ( + github.com/mmcdole/gofeed v1.4.0 + github.com/prodigeris/html2text v1.0.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.27 // indirect + github.com/mmcdole/goxpp/v2 v2.0.0 // indirect + github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect + github.com/olekukonko/errors v1.3.0 // indirect + github.com/olekukonko/ll v0.1.8 // indirect + github.com/olekukonko/tablewriter v1.1.4 // indirect + github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9e65b17 --- /dev/null +++ b/go.sum @@ -0,0 +1,46 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0= +github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= +github.com/mmcdole/gofeed v1.4.0 h1:+efDmI/yJXJgTfa8we5zg9GAKsU+2d7tnpt9QZwvjLQ= +github.com/mmcdole/gofeed v1.4.0/go.mod h1:ngV5MTB7UJko6fH3/fG5AkB/ABUGK1ZTePF9iRhzu/c= +github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= +github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= +github.com/olekukonko/errors v1.3.0 h1:teJvgLGUEqMzBUms+Dj3/3szNqCG/Jdw9iDbum8fR6U= +github.com/olekukonko/errors v1.3.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8= +github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw= +github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= +github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prodigeris/html2text v1.0.0 h1:5v5fJbyNQH+PfVFOegTnN4/MixsqujYBEZhzg5uJVDA= +github.com/prodigeris/html2text v1.0.0/go.mod h1:uJqKNfQm6IfD0gjAoiiquAe+RIuOwThzLOr7HiE4gyE= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..bc471ac --- /dev/null +++ b/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "flag" + "log/slog" + "os" + + "go.dev.pztrn.name/rsser/configuration" + "go.dev.pztrn.name/rsser/outputs" + "go.dev.pztrn.name/rsser/outputs/nntp" + "go.dev.pztrn.name/rsser/outputs/stdout" + "go.dev.pztrn.name/rsser/rss" +) + +var configPath string + +func main() { + slog.Info("Starting parsing RSS feeds...") + + flag.StringVar(&configPath, "config", "", "Full path to configuration file.") + + flag.Parse() + + cfg, err := configuration.New(configPath) + if err != nil { + slog.Error("Failed to create config controller!", "error", err.Error()) + os.Exit(1) + } + + outs := make([]outputs.Output, 0, 1) + outs = append(outs, stdout.New()) + + nntpClient, err := nntp.New(cfg) + if err != nil { + slog.Error("Failed to initialize NNTP outputter!", "error", err.Error()) + os.Exit(1) + } + + outs = append(outs, nntpClient) + + rsser, err := rss.New(cfg, outs) + if err != nil { + slog.Error("Failed to create RSS controller!", "error", err.Error()) + os.Exit(1) + } + + rsser.Parse() +} diff --git a/outputs/nntp/nntp.go b/outputs/nntp/nntp.go new file mode 100644 index 0000000..c45157e --- /dev/null +++ b/outputs/nntp/nntp.go @@ -0,0 +1,219 @@ +package nntp + +import ( + "bufio" + "encoding/base64" + "fmt" + "log/slog" + "net" + "strings" + "time" + + "go.dev.pztrn.name/rsser/configuration" + "go.dev.pztrn.name/rsser/outputs" +) + +type nntp struct { + cfg *configuration.Config + conn net.Conn + br *bufio.Reader + bw *bufio.Writer +} + +// New creates new simple NNTP client. Connection will be established only for sending posts. +func New(cfg *configuration.Config) (outputs.Output, error) { + c := &nntp{ + cfg: cfg, + } + + return c, nil +} + +func (c *nntp) establishConnection() error { + if c.conn != nil { + return nil + } + + //nolint:noctx + conn, err := net.DialTimeout( + "tcp", + c.cfg.Outputs.NNTP.Address, + time.Second*time.Duration(c.cfg.Outputs.NNTP.ConnectTimeout), + ) + if err != nil { + return fmt.Errorf("establishConnection: %w", err) + } + + c.conn = conn + c.br = bufio.NewReader(c.conn) + c.bw = bufio.NewWriter(c.conn) + + // Waiting for hello (200/201). + line, err := c.readLine() + if err != nil { + _ = conn.Close() + + return fmt.Errorf("no greeting from server: %w", err) + } + + if !strings.HasPrefix(line, "200") && !strings.HasPrefix(line, "201") { + _ = conn.Close() + + //nolint:err113 + return fmt.Errorf("unexpected greeting: %s", line) + } + + if err := c.authInfo(); err != nil { + //nolint:err113 + return fmt.Errorf("auth failed: %s", line) + } + + return nil +} + +func (c *nntp) authInfo() error { + resp, err := c.sendCommand("AUTHINFO USER " + c.cfg.Outputs.NNTP.Username) + if err != nil { + return err + } + + if !strings.HasPrefix(resp, "381") { // 381 = send PASS + // Some servers immediately giving 281 (success) or 480 (fail) + if strings.HasPrefix(resp, "281") { + return nil // уже авторизованы + } + + //nolint:err113 + return fmt.Errorf("AUTHINFO USER failed: %s", resp) + } + + resp, err = c.sendCommand("AUTHINFO PASS " + c.cfg.Outputs.NNTP.Password) + if err != nil { + return err + } + + if !strings.HasPrefix(resp, "281") { + //nolint:err113 + return fmt.Errorf("AUTHINFO PASS failed: %s", resp) + } + + return nil +} + +func (c *nntp) Do(dest, feedName, url, title, body string) error { + headers := map[string]string{ + "From": "\"" + feedName + "\" <" + c.cfg.Outputs.NNTP.Username + "@mail.invalid", + "Newsgroups": dest, + "Subject": c.encodeSubjectB(title), + "Date": time.Now().UTC().Format(time.RFC1123), + "Content-Type": "text/plain; charset=UTF-8; format=flowed", + "Content-Transfer-Encoding": "7bit", + } + + body = "URL: " + url + "\n\n" + body + + if err := c.postArticle(headers, body); err != nil { + return fmt.Errorf("Do: %w", err) + } + + return nil +} + +func (c *nntp) doubleDot(s string) string { + var result strings.Builder + + scanner := bufio.NewScanner(strings.NewReader(s)) + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 && line[0] == '.' { + result.WriteString("..") + result.WriteString(line[1:]) + } else { + result.WriteString(line) + } + + result.WriteByte('\n') + } + + return result.String() +} + +func (c *nntp) encodeSubjectB(s string) string { + b := base64.StdEncoding.EncodeToString([]byte(s)) + + return fmt.Sprintf("=?utf-8?B?%s?=", b) +} + +func (c *nntp) Name() string { + return "nntp" +} + +func (c *nntp) postArticle(headers map[string]string, body string) error { + if err := c.establishConnection(); err != nil { + return fmt.Errorf("PostArticle: %w", err) + } + + resp, err := c.sendCommand("POST") + if err != nil { + return err + } + + if !strings.HasPrefix(resp, "340") { // 340 = ready to accept article + //nolint:err113 + return fmt.Errorf("POST not accepted by server: %s", resp) + } + + // Заголовки + for k, v := range headers { + _, _ = fmt.Fprintf(c.bw, "%s: %s\r\n", k, v) + } + + _, _ = fmt.Fprint(c.bw, "\r\n") + + // Article body: double starting dot (RFC). + processedBody := c.doubleDot(body) + _, _ = fmt.Fprint(c.bw, processedBody) + _, _ = fmt.Fprintln(c.bw, ".") + + if flushErr := c.bw.Flush(); flushErr != nil { + return fmt.Errorf("PostArticle: %w", flushErr) + } + + finalResp, err := c.readLine() + if err != nil { + return err + } + + if !strings.HasPrefix(finalResp, "240") { + //nolint:err113 + return fmt.Errorf("article not accepted: %s", finalResp) + } + + slog.Info("Article posted successfully", "resp", finalResp) + + return nil +} + +func (c *nntp) readLine() (string, error) { + b, err := c.br.ReadBytes('\n') + if err != nil { + return "", fmt.Errorf("readLine: %w", err) + } + + s := strings.TrimRight(string(b), "\r\n") + + return s, nil +} + +func (c *nntp) sendCommand(cmd string) (string, error) { + _, err := fmt.Fprintf(c.bw, "%s\r\n", cmd) + if err != nil { + return "", fmt.Errorf("readLine: %w", err) + } + + if err := c.bw.Flush(); err != nil { + return "", fmt.Errorf("readLine: %w", err) + } + + return c.readLine() +} diff --git a/outputs/output.go b/outputs/output.go new file mode 100644 index 0000000..c3721a0 --- /dev/null +++ b/outputs/output.go @@ -0,0 +1,7 @@ +package outputs + +// Output is an interface for RSS outputters. +type Output interface { + Do(dest, feedName, url, title, body string) error + Name() string +} diff --git a/outputs/stdout/stdout.go b/outputs/stdout/stdout.go new file mode 100644 index 0000000..bfbf0a7 --- /dev/null +++ b/outputs/stdout/stdout.go @@ -0,0 +1,30 @@ +package stdout + +import ( + "fmt" + + "go.dev.pztrn.name/rsser/outputs" +) + +type stdout struct{} + +// New creates new STDOUT output handler. +func New() outputs.Output { + return &stdout{} +} + +//nolint:forbidigo +func (s *stdout) Do(_, _, url, title, body string) error { + fmt.Println(title) + fmt.Println("") + fmt.Println("URL:", url) + fmt.Println("") + fmt.Println(body) + fmt.Println("==========") + + return nil +} + +func (s *stdout) Name() string { + return "stdout" +} diff --git a/rss/parse.go b/rss/parse.go new file mode 100644 index 0000000..b6af33a --- /dev/null +++ b/rss/parse.go @@ -0,0 +1,102 @@ +package rss + +import ( + "context" + "log/slog" + "net/url" + "time" + + "github.com/mmcdole/gofeed" + "github.com/prodigeris/html2text" + "go.dev.pztrn.name/rsser/fixers" +) + +// Parse parses RSS feeds and produce output in outputs. +func (r *RSS) Parse() { + for _, feed := range r.cfg.RSS.Feeds { + ctx, cancelFunc := context.WithTimeout( + context.Background(), + time.Second*time.Duration(r.cfg.RSS.HTTPClient.RequestTimeoutSeconds), + ) + //revive:disable:defer + defer cancelFunc() + + parser := gofeed.NewParser() + + parsedFeedData, err := parser.ParseURLWithContext(feed.URL, ctx) + if err != nil { + slog.Error("Failed to parse feed!", "name", feed.Name, "error", err.Error()) + + return + } + + feedURL, _ := url.Parse(feed.URL) + + for _, item := range parsedFeedData.Items { + slog.Info( + "Processing feed item...", + "domain", feedURL.Host, + "guid", item.GUID, + ) + + itemWasProcessed, err := r.checkItemWasProcessed(feedURL.Host, item.GUID) + if err != nil { + slog.Error( + "Failed to check item processing status!", + "domain", feedURL.Host, + "guid", item.GUID, + "error", err.Error(), + ) + + return + } + + if itemWasProcessed { + slog.Info( + "Feed item already processed, skipping.", + "domain", feedURL.Host, + "guid", item.GUID, + ) + + continue + } + + description, err := html2text.FromString(item.Description, html2text.Options{ + PrettyTables: true, + }) + if err != nil { + description = item.Description + } + + title, fixedDescription := fixers.Fix(feedURL.Host, item.Title, description) + + for _, output := range r.outputs { + if feed.Output == output.Name() { + if err := output.Do(feed.Destination, feed.Name, item.Link, title, fixedDescription); err != nil { + slog.Error( + "Failed to process feed item with outputter!", + "feed", feed.Name, + "guid", item.GUID, + "error", err.Error(), + ) + + return + } + + break + } + } + + if err := r.addProcessedItem(feedURL.Host, item.GUID); err != nil { + slog.Error( + "Failed to save item processing status!", + "domain", feedURL.Host, + "guid", item.GUID, + "error", err.Error(), + ) + + return + } + } + } +} diff --git a/rss/rss.go b/rss/rss.go new file mode 100644 index 0000000..7929523 --- /dev/null +++ b/rss/rss.go @@ -0,0 +1,55 @@ +package rss + +import ( + "errors" + "fmt" + "log/slog" + "net/http" + "time" + + "go.dev.pztrn.name/rsser/configuration" + "go.dev.pztrn.name/rsser/outputs" +) + +var ( + errNoOutputs = errors.New("no outputs") + errRSS = errors.New("RSS controller") +) + +// RSS is an RSS feeds parsing controlling structure. +type RSS struct { + cfg *configuration.Config + httpClient *http.Client + outputs []outputs.Output +} + +// New creates new RSS parsing instance. +func New(cfg *configuration.Config, outs []outputs.Output) (*RSS, error) { + rss := &RSS{ + cfg: cfg, + outputs: outs, + } + + if err := rss.initialize(); err != nil { + return nil, fmt.Errorf("%w: %w", errRSS, err) + } + + return rss, nil +} + +func (r *RSS) initialize() error { + if len(r.outputs) == 0 { + return fmt.Errorf("initialize: %w", errNoOutputs) + } + + slog.Info( + "HTTP client configuration.", + "request_timeout_seconds", r.cfg.RSS.HTTPClient.RequestTimeoutSeconds, + ) + + r.httpClient = &http.Client{ + Timeout: time.Second * time.Duration(r.cfg.RSS.HTTPClient.RequestTimeoutSeconds), + } + + return nil +} diff --git a/rss/storage.go b/rss/storage.go new file mode 100644 index 0000000..3c6d42d --- /dev/null +++ b/rss/storage.go @@ -0,0 +1,64 @@ +package rss + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +var processedItemsDir string + +//nolint:gochecknoinits +func init() { + userHomeDir, err := os.UserHomeDir() + if err != nil { + panic("Failed to get user's home directory: " + err.Error()) + } + + processedItemsDir = strings.Replace("~/.rsser", "~", userHomeDir, 1) + + if err := os.MkdirAll(processedItemsDir, 0o755); err != nil { + panic("Failed to create ~/.rsser: " + err.Error()) + } +} + +func (r *RSS) addProcessedItem(domain, guid string) error { + file, err := os.OpenFile( + filepath.Join(processedItemsDir, domain), + os.O_APPEND|os.O_CREATE|os.O_WRONLY, + 0o644, + ) + if err != nil { + return fmt.Errorf("add processed item: %w", err) + } + defer file.Close() + + if _, err := file.WriteString(guid + "\n"); err != nil { + return fmt.Errorf("add processed item: %w", err) + } + + return nil +} + +func (r *RSS) checkItemWasProcessed(domain, guid string) (bool, error) { + fileDataBytes, err := os.ReadFile(filepath.Join(processedItemsDir, domain)) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + + return false, fmt.Errorf("check item processing: %w", err) + } + + fileData := string(fileDataBytes) + + for line := range strings.Lines(fileData) { + if line == guid+"\n" { + return true, nil + } + } + + return false, nil +}