Initial commit.
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
*DS_Store*
|
||||
fyne-cross
|
||||
_build
|
||||
*.apk
|
||||
*.app
|
||||
dist
|
||||
.task
|
||||
TASKS_NOTES.md
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
---
|
||||
version: "2"
|
||||
linters:
|
||||
default: all
|
||||
disable:
|
||||
- containedctx
|
||||
- depguard
|
||||
- exhaustruct
|
||||
- funlen
|
||||
- gochecknoglobals
|
||||
- gocritic
|
||||
- godot
|
||||
- iface
|
||||
- interfacebloat
|
||||
- ireturn
|
||||
- mnd
|
||||
- paralleltest
|
||||
- tagalign
|
||||
- tagliatelle
|
||||
- testpackage
|
||||
- noinlineerr
|
||||
- embeddedstructfieldcheck
|
||||
- wsl # deprecated
|
||||
- gomodguard # deprecated
|
||||
- unqueryvet # we love SELECT * when working with databases
|
||||
settings:
|
||||
cyclop:
|
||||
max-complexity: 25
|
||||
dupl:
|
||||
threshold: 400
|
||||
forbidigo:
|
||||
forbid:
|
||||
- pattern: ^(fmt\.Print(|f|ln)|print|println)$
|
||||
- pattern: ^time\.Now\(\)($|\.F|\.A|\.B|\.L|\.UTC\(\)\.I|,|\))(# Calls of time\.Now() without \.UTC() is prohibited\.)?
|
||||
funcorder:
|
||||
constructor: true
|
||||
struct-method: false
|
||||
alphabetical: true
|
||||
gocyclo:
|
||||
min-complexity: 40
|
||||
govet:
|
||||
enable-all: true
|
||||
lll:
|
||||
line-length: 120
|
||||
perfsprint:
|
||||
concat-loop: false
|
||||
revive:
|
||||
enable-all-rules: true
|
||||
rules:
|
||||
# This linter creates more problems than gives profits. Probably should be configured?
|
||||
- name: add-constant
|
||||
disabled: true
|
||||
# gocognit
|
||||
- name: cognitive-complexity
|
||||
disabled: true
|
||||
- name: confusing-results
|
||||
disabled: true
|
||||
# gocyclo
|
||||
- name: cyclomatic
|
||||
disabled: true
|
||||
- name: function-length
|
||||
arguments: [200, 0]
|
||||
- name: import-alias-naming
|
||||
arguments:
|
||||
- "^[a-z][a-zA-Z0-9]{0,}$"
|
||||
- name: line-length-limit
|
||||
disabled: true
|
||||
- name: package-comments
|
||||
disabled: true
|
||||
- name: unused-receiver
|
||||
disabled: true
|
||||
- name: var-naming
|
||||
arguments:
|
||||
- ["ID"]
|
||||
- ["VM"]
|
||||
- - skipPackageNameChecks: true
|
||||
- name: package-naming
|
||||
disabled: true
|
||||
wrapcheck:
|
||||
ignore-sig-regexps:
|
||||
- \.Write\(
|
||||
- \.WriteString\(
|
||||
- \.WriteJSON\(
|
||||
- \.WriteHTML\(
|
||||
- \.Redirect\(
|
||||
- \.Error\(
|
||||
wsl_v5:
|
||||
allow-first-in-block: true
|
||||
allow-whole-block: false
|
||||
branch-max-lines: 2
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
rules:
|
||||
- linters:
|
||||
- godox
|
||||
text: TODO
|
||||
- linters:
|
||||
- gosec
|
||||
text: G101
|
||||
paths:
|
||||
- billing
|
||||
- cms
|
||||
- monolit-apigateway
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
uniq-by-line: false
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
- gofumpt
|
||||
settings:
|
||||
gofumpt:
|
||||
module-path: go.dev.pztrn.name/rsser
|
||||
extra-rules: true
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
@@ -0,0 +1,13 @@
|
||||
# RSSer
|
||||
|
||||
RSS parser with pluggable outputs.
|
||||
|
||||
## Installation
|
||||
|
||||
```shell
|
||||
go install go.dev.pztrn.name/rsser
|
||||
```
|
||||
|
||||
## Configuring
|
||||
|
||||
See config.example.json for example configuration.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"outputs": {
|
||||
"nntp": {
|
||||
"address": "news.host.tld:119",
|
||||
"username": "user",
|
||||
"password": "pass",
|
||||
"connect_timeout": 5
|
||||
}
|
||||
},
|
||||
"rss": {
|
||||
"feeds": [
|
||||
{
|
||||
"name": "OpenNET.ru: News",
|
||||
"url": "https://www.opennet.ru/opennews/opennews_full.rss",
|
||||
"output": "nntp",
|
||||
"dest": "some.group"
|
||||
}
|
||||
],
|
||||
"http_client": {
|
||||
"request_timeout_seconds": 10
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var errCfg = errors.New("configuration")
|
||||
|
||||
// Config is a configuration controlling structure.
|
||||
type Config struct {
|
||||
RSS *RSS `json:"rss"`
|
||||
Outputs *Outputs `json:"outputs"`
|
||||
}
|
||||
|
||||
// New creates new configuration controller.
|
||||
func New(path string) (*Config, error) {
|
||||
cfg := &Config{}
|
||||
|
||||
// Get configuration file real path.
|
||||
userHomeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: parse configuration file: get real path: get user home dir: %w", errCfg, err)
|
||||
}
|
||||
|
||||
path = strings.Replace(path, "~", userHomeDir, 1)
|
||||
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: parse configuration file: get real path: get absolute path: %w", errCfg, err)
|
||||
}
|
||||
|
||||
cfgFileData, err := os.ReadFile(absPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: parse configuration file: read file: %w", errCfg, err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(cfgFileData, cfg); err != nil {
|
||||
return nil, fmt.Errorf("%w: parse configuration file: %w", errCfg, err)
|
||||
}
|
||||
|
||||
// Validate configuration.
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", errCfg, err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Validate validates app configuration.
|
||||
func (c *Config) Validate() error {
|
||||
if err := c.RSS.Validate(); err != nil {
|
||||
return fmt.Errorf("validate configuration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
var (
|
||||
errDestIsEmpty = errors.New("dest is empty")
|
||||
errFeedValidate = errors.New("feed configuration validation")
|
||||
errNameIsEmpty = errors.New("name is empty")
|
||||
errOutputIsEmpty = errors.New("output is empty")
|
||||
errURLIsEmpty = errors.New("URL is empty")
|
||||
)
|
||||
|
||||
// Feed is a single RSS feed configuration.
|
||||
type Feed struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Output string `json:"output"`
|
||||
Destination string `json:"dest"`
|
||||
}
|
||||
|
||||
// Validate validates RSS feed configuration.
|
||||
func (f *Feed) Validate() error {
|
||||
if f.Name == "" {
|
||||
return fmt.Errorf("%w: %w", errFeedValidate, errNameIsEmpty)
|
||||
}
|
||||
|
||||
if f.Output == "" {
|
||||
return fmt.Errorf("%w: %w", errFeedValidate, errOutputIsEmpty)
|
||||
}
|
||||
|
||||
if f.Destination == "" {
|
||||
return fmt.Errorf("%w: %w", errFeedValidate, errDestIsEmpty)
|
||||
}
|
||||
|
||||
if f.URL == "" {
|
||||
return fmt.Errorf("%w: %w", errFeedValidate, errURLIsEmpty)
|
||||
}
|
||||
|
||||
if _, err := url.Parse(f.URL); err != nil {
|
||||
return fmt.Errorf("%w: parse feed URL: %w", errFeedValidate, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var errRequestTimeoutTooLow = errors.New("request timeout is too low")
|
||||
|
||||
// HTTPClient is an HTTP client configuration.
|
||||
type HTTPClient struct {
|
||||
RequestTimeoutSeconds int64 `json:"request_timeout_seconds"`
|
||||
}
|
||||
|
||||
// Validate validates configuration for HTTP clients.
|
||||
func (h *HTTPClient) Validate() error {
|
||||
if h.RequestTimeoutSeconds < 1 {
|
||||
return fmt.Errorf("validate HTTP client configuration: %w", errRequestTimeoutTooLow)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
)
|
||||
|
||||
var (
|
||||
errNNTPAddressIsEmpty = errors.New("address is empty")
|
||||
errNNTPInvalidAddress = errors.New("address is invalid")
|
||||
errNNTPInvalidConnTimeout = errors.New("connect timeout is invalid")
|
||||
errNNTPUserOrPassIsEmpty = errors.New("username or password is empty")
|
||||
)
|
||||
|
||||
// NNTP is an NNTP output configuration.
|
||||
type NNTP struct {
|
||||
Address string `json:"address"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
ConnectTimeout int64 `json:"connect_timeout"`
|
||||
}
|
||||
|
||||
// Validate validates configuration.
|
||||
func (n *NNTP) Validate() error {
|
||||
if n.Address == "" {
|
||||
return fmt.Errorf("validate NNTP output configuration: %w", errNNTPAddressIsEmpty)
|
||||
}
|
||||
|
||||
_, _, err := net.SplitHostPort(n.Address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("validate NNTP output configuration: %w: %w", errNNTPInvalidAddress, err)
|
||||
}
|
||||
|
||||
if (n.Username != "" && n.Password == "") || (n.Username == "" && n.Password != "") {
|
||||
return fmt.Errorf("validate NNTP output configuration: %w", errNNTPUserOrPassIsEmpty)
|
||||
}
|
||||
|
||||
if n.ConnectTimeout < 1 {
|
||||
return fmt.Errorf("validate NNTP output configuration: %w", errNNTPInvalidConnTimeout)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package configuration
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Outputs is an outputs configuration.
|
||||
type Outputs struct {
|
||||
NNTP *NNTP `json:"nntp"`
|
||||
}
|
||||
|
||||
// Validate validates configuration.
|
||||
func (o *Outputs) Validate() error {
|
||||
if err := o.NNTP.Validate(); err != nil {
|
||||
return fmt.Errorf("validate outputs configuration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var errRSSValidate = errors.New("validate RSS configuration")
|
||||
|
||||
// RSS is an RSS parser configuration.
|
||||
type RSS struct {
|
||||
HTTPClient *HTTPClient `json:"http_client"`
|
||||
Feeds []*Feed `json:"feeds"`
|
||||
}
|
||||
|
||||
// Validate validates RSS parser configuration.
|
||||
func (r *RSS) Validate() error {
|
||||
for idx, feed := range r.Feeds {
|
||||
if err := feed.Validate(); err != nil {
|
||||
return fmt.Errorf("%w: feed %d: %w", errRSSValidate, idx, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.HTTPClient.Validate(); err != nil {
|
||||
return fmt.Errorf("%w: %w", errRSSValidate, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package fixers
|
||||
|
||||
type fixerFunc func(title, body string) (string, string)
|
||||
|
||||
var fixers = map[string]fixerFunc{
|
||||
"www.opennet.ru": opennetFixer,
|
||||
}
|
||||
|
||||
// Fix fixes title and body depending on site parsed.
|
||||
func Fix(domain, title, body string) (string, string) {
|
||||
fixer, found := fixers[domain]
|
||||
if !found {
|
||||
return title, body
|
||||
}
|
||||
|
||||
return fixer(title, body)
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package fixers
|
||||
|
||||
import "strings"
|
||||
|
||||
func opennetFixer(title, body string) (string, string) {
|
||||
newBody := &strings.Builder{}
|
||||
|
||||
for line := range strings.Lines(body) {
|
||||
if strings.HasPrefix(line, "Источник") {
|
||||
continue
|
||||
}
|
||||
|
||||
_, _ = newBody.WriteString(line)
|
||||
}
|
||||
|
||||
return title, newBody.String()
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
module go.dev.pztrn.name/rsser
|
||||
|
||||
go 1.26.5
|
||||
|
||||
require (
|
||||
github.com/mmcdole/gofeed v1.4.0
|
||||
github.com/prodigeris/html2text v1.0.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/fatih/color v1.19.0 // indirect
|
||||
github.com/goccy/go-json v0.10.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.27 // indirect
|
||||
github.com/mmcdole/goxpp/v2 v2.0.0 // indirect
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
||||
github.com/olekukonko/errors v1.3.0 // indirect
|
||||
github.com/olekukonko/ll v0.1.8 // indirect
|
||||
github.com/olekukonko/tablewriter v1.1.4 // indirect
|
||||
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8=
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
|
||||
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
|
||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0=
|
||||
github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8=
|
||||
github.com/mmcdole/gofeed v1.4.0 h1:+efDmI/yJXJgTfa8we5zg9GAKsU+2d7tnpt9QZwvjLQ=
|
||||
github.com/mmcdole/gofeed v1.4.0/go.mod h1:ngV5MTB7UJko6fH3/fG5AkB/ABUGK1ZTePF9iRhzu/c=
|
||||
github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ=
|
||||
github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os=
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
|
||||
github.com/olekukonko/errors v1.3.0 h1:teJvgLGUEqMzBUms+Dj3/3szNqCG/Jdw9iDbum8fR6U=
|
||||
github.com/olekukonko/errors v1.3.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
|
||||
github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8=
|
||||
github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw=
|
||||
github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I=
|
||||
github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prodigeris/html2text v1.0.0 h1:5v5fJbyNQH+PfVFOegTnN4/MixsqujYBEZhzg5uJVDA=
|
||||
github.com/prodigeris/html2text v1.0.0/go.mod h1:uJqKNfQm6IfD0gjAoiiquAe+RIuOwThzLOr7HiE4gyE=
|
||||
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
|
||||
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,48 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"go.dev.pztrn.name/rsser/configuration"
|
||||
"go.dev.pztrn.name/rsser/outputs"
|
||||
"go.dev.pztrn.name/rsser/outputs/nntp"
|
||||
"go.dev.pztrn.name/rsser/outputs/stdout"
|
||||
"go.dev.pztrn.name/rsser/rss"
|
||||
)
|
||||
|
||||
var configPath string
|
||||
|
||||
func main() {
|
||||
slog.Info("Starting parsing RSS feeds...")
|
||||
|
||||
flag.StringVar(&configPath, "config", "", "Full path to configuration file.")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := configuration.New(configPath)
|
||||
if err != nil {
|
||||
slog.Error("Failed to create config controller!", "error", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
outs := make([]outputs.Output, 0, 1)
|
||||
outs = append(outs, stdout.New())
|
||||
|
||||
nntpClient, err := nntp.New(cfg)
|
||||
if err != nil {
|
||||
slog.Error("Failed to initialize NNTP outputter!", "error", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
outs = append(outs, nntpClient)
|
||||
|
||||
rsser, err := rss.New(cfg, outs)
|
||||
if err != nil {
|
||||
slog.Error("Failed to create RSS controller!", "error", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
rsser.Parse()
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package nntp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.dev.pztrn.name/rsser/configuration"
|
||||
"go.dev.pztrn.name/rsser/outputs"
|
||||
)
|
||||
|
||||
type nntp struct {
|
||||
cfg *configuration.Config
|
||||
conn net.Conn
|
||||
br *bufio.Reader
|
||||
bw *bufio.Writer
|
||||
}
|
||||
|
||||
// New creates new simple NNTP client. Connection will be established only for sending posts.
|
||||
func New(cfg *configuration.Config) (outputs.Output, error) {
|
||||
c := &nntp{
|
||||
cfg: cfg,
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *nntp) establishConnection() error {
|
||||
if c.conn != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
//nolint:noctx
|
||||
conn, err := net.DialTimeout(
|
||||
"tcp",
|
||||
c.cfg.Outputs.NNTP.Address,
|
||||
time.Second*time.Duration(c.cfg.Outputs.NNTP.ConnectTimeout),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("establishConnection: %w", err)
|
||||
}
|
||||
|
||||
c.conn = conn
|
||||
c.br = bufio.NewReader(c.conn)
|
||||
c.bw = bufio.NewWriter(c.conn)
|
||||
|
||||
// Waiting for hello (200/201).
|
||||
line, err := c.readLine()
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
|
||||
return fmt.Errorf("no greeting from server: %w", err)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(line, "200") && !strings.HasPrefix(line, "201") {
|
||||
_ = conn.Close()
|
||||
|
||||
//nolint:err113
|
||||
return fmt.Errorf("unexpected greeting: %s", line)
|
||||
}
|
||||
|
||||
if err := c.authInfo(); err != nil {
|
||||
//nolint:err113
|
||||
return fmt.Errorf("auth failed: %s", line)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *nntp) authInfo() error {
|
||||
resp, err := c.sendCommand("AUTHINFO USER " + c.cfg.Outputs.NNTP.Username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(resp, "381") { // 381 = send PASS
|
||||
// Some servers immediately giving 281 (success) or 480 (fail)
|
||||
if strings.HasPrefix(resp, "281") {
|
||||
return nil // уже авторизованы
|
||||
}
|
||||
|
||||
//nolint:err113
|
||||
return fmt.Errorf("AUTHINFO USER failed: %s", resp)
|
||||
}
|
||||
|
||||
resp, err = c.sendCommand("AUTHINFO PASS " + c.cfg.Outputs.NNTP.Password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(resp, "281") {
|
||||
//nolint:err113
|
||||
return fmt.Errorf("AUTHINFO PASS failed: %s", resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *nntp) Do(dest, feedName, url, title, body string) error {
|
||||
headers := map[string]string{
|
||||
"From": "\"" + feedName + "\" <" + c.cfg.Outputs.NNTP.Username + "@mail.invalid",
|
||||
"Newsgroups": dest,
|
||||
"Subject": c.encodeSubjectB(title),
|
||||
"Date": time.Now().UTC().Format(time.RFC1123),
|
||||
"Content-Type": "text/plain; charset=UTF-8; format=flowed",
|
||||
"Content-Transfer-Encoding": "7bit",
|
||||
}
|
||||
|
||||
body = "URL: " + url + "\n\n" + body
|
||||
|
||||
if err := c.postArticle(headers, body); err != nil {
|
||||
return fmt.Errorf("Do: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *nntp) doubleDot(s string) string {
|
||||
var result strings.Builder
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(s))
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if len(line) > 0 && line[0] == '.' {
|
||||
result.WriteString("..")
|
||||
result.WriteString(line[1:])
|
||||
} else {
|
||||
result.WriteString(line)
|
||||
}
|
||||
|
||||
result.WriteByte('\n')
|
||||
}
|
||||
|
||||
return result.String()
|
||||
}
|
||||
|
||||
func (c *nntp) encodeSubjectB(s string) string {
|
||||
b := base64.StdEncoding.EncodeToString([]byte(s))
|
||||
|
||||
return fmt.Sprintf("=?utf-8?B?%s?=", b)
|
||||
}
|
||||
|
||||
func (c *nntp) Name() string {
|
||||
return "nntp"
|
||||
}
|
||||
|
||||
func (c *nntp) postArticle(headers map[string]string, body string) error {
|
||||
if err := c.establishConnection(); err != nil {
|
||||
return fmt.Errorf("PostArticle: %w", err)
|
||||
}
|
||||
|
||||
resp, err := c.sendCommand("POST")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(resp, "340") { // 340 = ready to accept article
|
||||
//nolint:err113
|
||||
return fmt.Errorf("POST not accepted by server: %s", resp)
|
||||
}
|
||||
|
||||
// Заголовки
|
||||
for k, v := range headers {
|
||||
_, _ = fmt.Fprintf(c.bw, "%s: %s\r\n", k, v)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprint(c.bw, "\r\n")
|
||||
|
||||
// Article body: double starting dot (RFC).
|
||||
processedBody := c.doubleDot(body)
|
||||
_, _ = fmt.Fprint(c.bw, processedBody)
|
||||
_, _ = fmt.Fprintln(c.bw, ".")
|
||||
|
||||
if flushErr := c.bw.Flush(); flushErr != nil {
|
||||
return fmt.Errorf("PostArticle: %w", flushErr)
|
||||
}
|
||||
|
||||
finalResp, err := c.readLine()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(finalResp, "240") {
|
||||
//nolint:err113
|
||||
return fmt.Errorf("article not accepted: %s", finalResp)
|
||||
}
|
||||
|
||||
slog.Info("Article posted successfully", "resp", finalResp)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *nntp) readLine() (string, error) {
|
||||
b, err := c.br.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("readLine: %w", err)
|
||||
}
|
||||
|
||||
s := strings.TrimRight(string(b), "\r\n")
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (c *nntp) sendCommand(cmd string) (string, error) {
|
||||
_, err := fmt.Fprintf(c.bw, "%s\r\n", cmd)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("readLine: %w", err)
|
||||
}
|
||||
|
||||
if err := c.bw.Flush(); err != nil {
|
||||
return "", fmt.Errorf("readLine: %w", err)
|
||||
}
|
||||
|
||||
return c.readLine()
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package outputs
|
||||
|
||||
// Output is an interface for RSS outputters.
|
||||
type Output interface {
|
||||
Do(dest, feedName, url, title, body string) error
|
||||
Name() string
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package stdout
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"go.dev.pztrn.name/rsser/outputs"
|
||||
)
|
||||
|
||||
type stdout struct{}
|
||||
|
||||
// New creates new STDOUT output handler.
|
||||
func New() outputs.Output {
|
||||
return &stdout{}
|
||||
}
|
||||
|
||||
//nolint:forbidigo
|
||||
func (s *stdout) Do(_, _, url, title, body string) error {
|
||||
fmt.Println(title)
|
||||
fmt.Println("")
|
||||
fmt.Println("URL:", url)
|
||||
fmt.Println("")
|
||||
fmt.Println(body)
|
||||
fmt.Println("==========")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stdout) Name() string {
|
||||
return "stdout"
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
"github.com/prodigeris/html2text"
|
||||
"go.dev.pztrn.name/rsser/fixers"
|
||||
)
|
||||
|
||||
// Parse parses RSS feeds and produce output in outputs.
|
||||
func (r *RSS) Parse() {
|
||||
for _, feed := range r.cfg.RSS.Feeds {
|
||||
ctx, cancelFunc := context.WithTimeout(
|
||||
context.Background(),
|
||||
time.Second*time.Duration(r.cfg.RSS.HTTPClient.RequestTimeoutSeconds),
|
||||
)
|
||||
//revive:disable:defer
|
||||
defer cancelFunc()
|
||||
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
parsedFeedData, err := parser.ParseURLWithContext(feed.URL, ctx)
|
||||
if err != nil {
|
||||
slog.Error("Failed to parse feed!", "name", feed.Name, "error", err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
feedURL, _ := url.Parse(feed.URL)
|
||||
|
||||
for _, item := range parsedFeedData.Items {
|
||||
slog.Info(
|
||||
"Processing feed item...",
|
||||
"domain", feedURL.Host,
|
||||
"guid", item.GUID,
|
||||
)
|
||||
|
||||
itemWasProcessed, err := r.checkItemWasProcessed(feedURL.Host, item.GUID)
|
||||
if err != nil {
|
||||
slog.Error(
|
||||
"Failed to check item processing status!",
|
||||
"domain", feedURL.Host,
|
||||
"guid", item.GUID,
|
||||
"error", err.Error(),
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if itemWasProcessed {
|
||||
slog.Info(
|
||||
"Feed item already processed, skipping.",
|
||||
"domain", feedURL.Host,
|
||||
"guid", item.GUID,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
description, err := html2text.FromString(item.Description, html2text.Options{
|
||||
PrettyTables: true,
|
||||
})
|
||||
if err != nil {
|
||||
description = item.Description
|
||||
}
|
||||
|
||||
title, fixedDescription := fixers.Fix(feedURL.Host, item.Title, description)
|
||||
|
||||
for _, output := range r.outputs {
|
||||
if feed.Output == output.Name() {
|
||||
if err := output.Do(feed.Destination, feed.Name, item.Link, title, fixedDescription); err != nil {
|
||||
slog.Error(
|
||||
"Failed to process feed item with outputter!",
|
||||
"feed", feed.Name,
|
||||
"guid", item.GUID,
|
||||
"error", err.Error(),
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.addProcessedItem(feedURL.Host, item.GUID); err != nil {
|
||||
slog.Error(
|
||||
"Failed to save item processing status!",
|
||||
"domain", feedURL.Host,
|
||||
"guid", item.GUID,
|
||||
"error", err.Error(),
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.dev.pztrn.name/rsser/configuration"
|
||||
"go.dev.pztrn.name/rsser/outputs"
|
||||
)
|
||||
|
||||
var (
|
||||
errNoOutputs = errors.New("no outputs")
|
||||
errRSS = errors.New("RSS controller")
|
||||
)
|
||||
|
||||
// RSS is an RSS feeds parsing controlling structure.
|
||||
type RSS struct {
|
||||
cfg *configuration.Config
|
||||
httpClient *http.Client
|
||||
outputs []outputs.Output
|
||||
}
|
||||
|
||||
// New creates new RSS parsing instance.
|
||||
func New(cfg *configuration.Config, outs []outputs.Output) (*RSS, error) {
|
||||
rss := &RSS{
|
||||
cfg: cfg,
|
||||
outputs: outs,
|
||||
}
|
||||
|
||||
if err := rss.initialize(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", errRSS, err)
|
||||
}
|
||||
|
||||
return rss, nil
|
||||
}
|
||||
|
||||
func (r *RSS) initialize() error {
|
||||
if len(r.outputs) == 0 {
|
||||
return fmt.Errorf("initialize: %w", errNoOutputs)
|
||||
}
|
||||
|
||||
slog.Info(
|
||||
"HTTP client configuration.",
|
||||
"request_timeout_seconds", r.cfg.RSS.HTTPClient.RequestTimeoutSeconds,
|
||||
)
|
||||
|
||||
r.httpClient = &http.Client{
|
||||
Timeout: time.Second * time.Duration(r.cfg.RSS.HTTPClient.RequestTimeoutSeconds),
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var processedItemsDir string
|
||||
|
||||
//nolint:gochecknoinits
|
||||
func init() {
|
||||
userHomeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
panic("Failed to get user's home directory: " + err.Error())
|
||||
}
|
||||
|
||||
processedItemsDir = strings.Replace("~/.rsser", "~", userHomeDir, 1)
|
||||
|
||||
if err := os.MkdirAll(processedItemsDir, 0o755); err != nil {
|
||||
panic("Failed to create ~/.rsser: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RSS) addProcessedItem(domain, guid string) error {
|
||||
file, err := os.OpenFile(
|
||||
filepath.Join(processedItemsDir, domain),
|
||||
os.O_APPEND|os.O_CREATE|os.O_WRONLY,
|
||||
0o644,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("add processed item: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := file.WriteString(guid + "\n"); err != nil {
|
||||
return fmt.Errorf("add processed item: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RSS) checkItemWasProcessed(domain, guid string) (bool, error) {
|
||||
fileDataBytes, err := os.ReadFile(filepath.Join(processedItemsDir, domain))
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("check item processing: %w", err)
|
||||
}
|
||||
|
||||
fileData := string(fileDataBytes)
|
||||
|
||||
for line := range strings.Lines(fileData) {
|
||||
if line == guid+"\n" {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
Reference in New Issue
Block a user