Templating, improved Matrix formatter, Gitlab parser, improved other parsers.

From now OpenSAPS supports template formatting. For examples see
Gitea or Gitlab parser for creating template and data and
matrixconnection.go for how to use them.

Improved Matrix formatter - now we can do links and put repeatables
(like commits) in resulting message.

Added Gitlab parser. Incomplete, but can parse commits, issues,
merge requests, tags messages.

Refactored (a little) Gitea parser. No functional changes, still
only commits parsing.

Both Gitlab and Gitea parsers will put back a message if unknown
message type was passed.
This commit is contained in:
Stanislav Nikitin 2017-09-13 11:44:25 +05:00
parent ac0a99ce10
commit b58baa8135
9 changed files with 497 additions and 20 deletions

View File

@ -81,7 +81,7 @@ func (c *Context) RegisterSlackAPIServerInterface(sasi slackapiserverinterface.S
c.SlackAPIServer.Initialize()
}
func (c *Context) SendToParser(name string, message slackmessage.SlackMessage) string {
func (c *Context) SendToParser(name string, message slackmessage.SlackMessage) map[string]string {
parser, found := c.Parsers[strings.ToLower(name)]
if !found {
c.Log.Errorf("Parser '%s' not found, will use default one!", name)

View File

@ -28,6 +28,7 @@ import (
"lab.pztrn.name/pztrn/opensaps/context"
"lab.pztrn.name/pztrn/opensaps/parsers/default"
"lab.pztrn.name/pztrn/opensaps/parsers/gitea"
"lab.pztrn.name/pztrn/opensaps/parsers/gitlab"
"lab.pztrn.name/pztrn/opensaps/pushers/matrix"
"lab.pztrn.name/pztrn/opensaps/slack"
)
@ -49,6 +50,7 @@ func main() {
// Initialize parsers.
defaultparser.New(c)
giteaparser.New(c)
gitlabparser.New(c)
// Initialize pushers.
matrixpusher.New(c)

View File

@ -28,7 +28,10 @@ func (dp DefaultParser) Initialize() {
c.Log.Infoln("Initializing default parser...")
}
func (dp DefaultParser) ParseMessage(message slackmessage.SlackMessage) string {
func (dp DefaultParser) ParseMessage(message slackmessage.SlackMessage) map[string]string {
c.Log.Debugln("Parsing default message...")
return message.Text
data := make(map[string]string)
data["message"] = message.Text
return data
}

View File

@ -21,6 +21,7 @@ import (
// stdlib
"fmt"
"regexp"
"strconv"
"strings"
// local
@ -79,31 +80,49 @@ func (gp GiteaParser) cutHeaderLinks(data string) [][]string {
return links
}
func (gp GiteaParser) ParseMessage(message slackmessage.SlackMessage) string {
c.Log.Debugln("Parsing Gitea message...")
var msg string = ""
func (gp GiteaParser) parseCommitNew(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[Repo: {repo} | Branch: {branch}] {header_message}{newline}{repeatables}"
// Parse header.
// [0] is repo, [1] is branch.
header_data := gp.cutHeaderLinks(message.Text)
msg += fmt.Sprintf("[Repo: <a href='%s'>%s</a> | Branch: <a href='%s'>%s</a>] ", header_data[0][0], header_data[0][1], header_data[1][0], header_data[1][1])
data["repo"] = header_data[0][1]
data["repo_url"] = header_data[0][0]
data["branch"] = header_data[1][1]
data["branch_url"] = header_data[1][0]
header_msg := strings.Split(message.Text, "] ")[1]
msg += header_msg + "<br />"
data["header_message"] = header_msg
// Parse commits.
data["repeatable_message"] = "{commit}: {message}{newline}"
data["repeatables"] = "commit,message"
idx := 0
for i := range message.Attachments {
// Commit link.
attachment_link := gp.cutCommitLink(message.Attachments[i].Text)
msg += fmt.Sprintf("<a href='%s'>%s</a>: ", attachment_link[0][0], attachment_link[0][1])
// Commit author and message.
authormsg := strings.Split(message.Attachments[i].Text, ">: ")[1]
msg += authormsg + "<br />"
data["repeatable_item_commit" + strconv.Itoa(idx)] = attachment_link[0][1]
data["repeatable_item_commit" + strconv.Itoa(idx) + "_url"] = attachment_link[0][0]
data["repeatable_item_message" + strconv.Itoa(idx)] = strings.Split(message.Attachments[i].Text, ">: ")[1]
idx += 1
}
data["repeatables_count"] = strconv.Itoa(idx)
return data
}
c.Log.Debugln("Message:", msg)
func (gp GiteaParser) ParseMessage(message slackmessage.SlackMessage) map[string]string {
c.Log.Debugln("Parsing Gitea message...")
return msg
var data map[string]string
if strings.Contains(message.Text, "new commit pushed by ") {
data = gp.parseCommitNew(message)
} else {
return map[string]string{"message": "Unknown message type:<br />" + fmt.Sprintf("%+v", message)}
}
c.Log.Debugln("Message:", fmt.Sprintf("%+x", data))
return data
}

View File

@ -0,0 +1,34 @@
// OpenSAPS - Open Slack API server for everyone.
//
// Copyright (c) 2017, Stanislav N. aka pztrn.
// All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package gitlabparser
import (
// local
"lab.pztrn.name/pztrn/opensaps/context"
"lab.pztrn.name/pztrn/opensaps/parsers/interface"
)
var (
c *context.Context
)
func New(cc *context.Context) {
c = cc
gp := GitlabParser{}
c.RegisterParserInterface("gitlab", parserinterface.ParserInterface(gp))
}

View File

@ -0,0 +1,306 @@
// OpenSAPS - Open Slack API server for everyone.
//
// Copyright (c) 2017, Stanislav N. aka pztrn.
// All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package gitlabparser
import (
// stdlib
"fmt"
"regexp"
"strconv"
"strings"
// local
"lab.pztrn.name/pztrn/opensaps/slack/message"
)
type GitlabParser struct {}
func (gp GitlabParser) Initialize() {
c.Log.Infoln("Initializing Gitlab parser...")
}
func (gp GitlabParser) parseCommit(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{repo}] {user} pushed to {branch}, {compare_changes}. Commits:{newline}{repeatables}"
data["user"] = strings.TrimSpace(strings.Split(message.Text, "pushed to")[0])
// Parse links.
links_data := gp.parseIssueCommentLink(message.Text)
data["branch"] = links_data[0][1]
data["branch_url"] = links_data[0][0]
data["repo"] = links_data[1][1]
data["repo_url"] = links_data[1][0]
data["compare_changes"] = "compare changes"
data["compare_changes_url"] = links_data[2][0]
// Parse commits.
data["repeatable_message"] = "{commit}: {commit_text}"
data["repeatables"] = "commit,commit_text"
idx := 0
for i := range message.Attachments {
commit_data := gp.parseIssueCommentLink(message.Attachments[i].Text)
data["repeatable_item_commit" + strconv.Itoa(idx)] = commit_data[0][1]
data["repeatable_item_commit" + strconv.Itoa(idx) + "_url"] = commit_data[0][0]
data["repeatable_item_commit_text" + strconv.Itoa(idx)] = strings.Split(message.Attachments[i].Text, ">: ")[1]
idx += 1
}
data["repeatables_count"] = strconv.Itoa(idx)
return data
}
func (gp GitlabParser) parseCommitLinks(data string) [][]string {
r := regexp.MustCompile("((htt[?p|ps]://[a-zA-Z0-9./-]+)\\|([a-zA-Z0-9./ _-]+))")
found := r.FindAllStringSubmatch(data, -1)
// [i][0] - link
// [i][1] - string for link
var result [][]string
for i := range found {
res := make([]string, 0, 2)
res = append(res, found[i][2])
res = append(res, found[i][3])
result = append(result, res)
}
c.Log.Debugln("Links cutted:", result)
return result
}
func (gp GitlabParser) parseIssueClosed(text string) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} closed issue {issue}"
// User name comes after "closed by" words.
data["user"] = strings.Split(text, "closed by ")[1]
// Parse links.
// Same as for parseIssueComment because this regexp returns
// needed data.
links_data := gp.parseIssueCommentLink(text)
data["project"] = links_data[0][1]
data["project_url"] = links_data[0][0]
data["issue"] = links_data[1][1]
data["issue_url"] = links_data[1][0]
return data
}
func (gp GitlabParser) parseIssueComment(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} {commented_on_issue} ({issue_name}):{newline}{repeatables}"
data["user"] = strings.TrimSpace(strings.Split(message.Text, " <")[0])
// Parse links in main message.
links_data := gp.parseIssueCommentLink(message.Text)
data["commented_on_issue"] = links_data[0][1]
data["commented_on_issue_url"] = links_data[0][0]
data["project"] = links_data[1][1]
data["project_url"] = links_data[1][0]
data["issue_name"] = strings.Split(message.Text, links_data[1][1] + ">: ")[1]
// Parse attachments, which contains comments.
data["repeatable_message"] = "{comment}{newline}"
data["repeatables"] = "comment"
idx := 0
for i := range message.Attachments {
data["repeatable_item_comment" + strconv.Itoa(idx)] = message.Attachments[i].Text
idx += 1
}
data["repeatables_count"] = strconv.Itoa(idx)
return data
}
func (gp GitlabParser) parseIssueCommentLink(data string) [][]string {
r := regexp.MustCompile("((htt[?p|ps]://[a-zA-Z0-9.#!*/ _-]+)\\|([a-zA-Z0-9.#!*/ _-]+))")
found := r.FindAllStringSubmatch(data, -1)
// [i][0] - link
// [i][1] - string for link
var result [][]string
for i := range found {
res := make([]string, 0, 2)
res = append(res, found[i][2])
res = append(res, found[i][3])
result = append(result, res)
}
c.Log.Debugln("Links cutted:", result)
return result
}
func (gp GitlabParser) parseIssueOpened(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} opened an issue: {issue}{newline}{issue_text}"
links_data := gp.parseIssueCommentLink(message.Text)
data["project"] = links_data[0][1]
data["project_url"] = links_data[0][0]
data["user"] = strings.Split(message.Text, "Issue opened by ")[1]
if len(message.Attachments) > 0 {
data["issue"] = message.Attachments[0].Title
// Generate valid issue URL.
issue_number_raw := strings.Fields(message.Attachments[0].Title)[0]
// Remove "#" and compose URL.
issue_number := strings.Replace(issue_number_raw, "#", "", 1)
data["issue_url"] = links_data[0][0] + "/issues/" + issue_number
data["issue_text"] = message.Attachments[0].Text
} else {
// Issue was reopened.
data["message"] = strings.Replace(data["message"], ": {issue}{newline}{issue_text}", "", 1)
data["message"] = strings.Replace(data["message"], "opened", "reopened", 1)
}
return data
}
func (gp GitlabParser) parseMergeRequestClosed(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} closed merge request: {merge_request}"
data["user"] = strings.Split(message.Text, " closed <")[0]
links_data := gp.parseIssueCommentLink(message.Text)
data["project"] = links_data[1][1]
data["project_url"] = links_data[1][0]
data["merge_request"] = links_data[0][1]
data["merge_request_url"] = links_data[0][0]
return data
}
func (gp GitlabParser) parseMergeRequestComment(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} {commented_on_merge_request} ({merge_request_name}):{newline}{repeatables}"
data["user"] = strings.TrimSpace(strings.Split(message.Text, " <")[0])
// Parse links in main message.
links_data := gp.parseIssueCommentLink(message.Text)
data["commented_on_merge_request"] = links_data[0][1]
data["commented_on_merge_request_url"] = links_data[0][0]
data["project"] = links_data[1][1]
data["project_url"] = links_data[1][0]
data["merge_request_name"] = strings.Split(message.Text, links_data[1][1] + ">: ")[1]
// Parse attachments, which contains comments.
data["repeatable_message"] = "{comment}{newline}"
data["repeatables"] = "comment"
idx := 0
for i := range message.Attachments {
data["repeatable_item_comment" + strconv.Itoa(idx)] = message.Attachments[i].Text
idx += 1
}
data["repeatables_count"] = strconv.Itoa(idx)
return data
}
func (gp GitlabParser) parseMergeRequestMerged(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} merged {merge_request}"
data["user"] = strings.Split(message.Text, " merged <")[0]
links_data := gp.parseIssueCommentLink(message.Text)
data["project"] = links_data[1][1]
data["project_url"] = links_data[1][0]
data["merge_request"] = links_data[0][1]
data["merge_request_url"] = links_data[0][0]
return data
}
func (gp GitlabParser) parseMergeRequestOpened(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} opened new merge request: {merge_request}"
data["user"] = strings.Split(message.Text, " opened <")[0]
links_data := gp.parseIssueCommentLink(message.Text)
data["project"] = links_data[1][1]
data["project_url"] = links_data[1][0]
data["merge_request"] = links_data[0][1]
data["merge_request_url"] = links_data[0][0]
return data
}
func (gp GitlabParser) parsePushedNewBranch(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} pushed new branch: {branch}"
links_data := gp.parseIssueCommentLink(message.Text)
data["branch"] = links_data[0][1]
data["branch_url"] = links_data[0][0]
data["project"] = links_data[1][1]
data["project_url"] = links_data[1][0]
data["user"] = strings.Split(message.Text, " pushed new branch")[0]
return data
}
func (gp GitlabParser) parseTagPush(message slackmessage.SlackMessage) map[string]string {
data := make(map[string]string)
data["message"] = "[{project}] {user} pushed new tag: {tag}"
data["user"] = strings.Split(message.Text, " pushed new tag")[0]
links_data := gp.parseIssueCommentLink(message.Text)
data["tag"] = links_data[0][1]
data["tag_url"] = links_data[0][0]
data["project"] = links_data[1][1]
data["project_url"] = links_data[1][0]
return data
}
func (gp GitlabParser) ParseMessage(message slackmessage.SlackMessage) map[string]string {
c.Log.Debugln("Parsing Gitlab message...")
var data map[string]string
if strings.Contains(message.Text, "pushed to") {
data = gp.parseCommit(message)
} else if strings.Contains(message.Text, "commented on issue") {
data = gp.parseIssueComment(message)
} else if strings.Contains(message.Text, "closed by ") {
data = gp.parseIssueClosed(message.Text)
} else if strings.Contains(message.Text, "Issue opened by ") {
data = gp.parseIssueOpened(message)
} else if strings.Contains(message.Text, "merge_requests") && strings.Contains(message.Text, " closed <") {
data = gp.parseMergeRequestClosed(message)
} else if strings.Contains(message.Text, "commented on merge request") {
data = gp.parseMergeRequestComment(message)
} else if strings.Contains(message.Text, "merge_requests") && strings.Contains(message.Text, " merged <") {
data = gp.parseMergeRequestMerged(message)
} else if strings.Contains(message.Text, "merge_requests") && strings.Contains(message.Text, " opened <") {
data = gp.parseMergeRequestOpened(message)
} else if strings.Contains(message.Text, "pushed new branch") {
data = gp.parsePushedNewBranch(message)
} else if strings.Contains(message.Text, " pushed new tag ") {
data = gp.parseTagPush(message)
} else {
return map[string]string{"message": "Unknown message type:<br />" + fmt.Sprintf("%+v", message)}
}
c.Log.Debugln("Message:", fmt.Sprintf("%+v", data))
return data
}

View File

@ -24,5 +24,5 @@ import (
type ParserInterface interface {
Initialize()
ParseMessage(message slackmessage.SlackMessage) string
ParseMessage(message slackmessage.SlackMessage) map[string]string
}

View File

@ -26,6 +26,8 @@ import (
"fmt"
"io/ioutil"
"net/http"
"strconv"
"strings"
// local
"lab.pztrn.name/pztrn/opensaps/slack/message"
@ -184,10 +186,102 @@ func (mxc *MatrixConnection) Initialize(conn_name string, api_root string, user
// It will prepare a message which will be passed to mxc.SendMessage().
func (mxc *MatrixConnection) ProcessMessage(message slackmessage.SlackMessage) {
// Prepare message body.
message_body := c.SendToParser(message.Username, message)
message_data := c.SendToParser(message.Username, message)
// Get message template.
msg_tpl := message_data["message"]
delete(message_data, "message")
// Repeatables.
var repeatables []string
repeatables_raw, repeatables_found := message_data["repeatables"]
if repeatables_found {
repeatables = strings.Split(repeatables_raw, ",")
c.Log.Debugln("Repeatable keys:", repeatables, ", length:", len(repeatables))
}
// Process keys.
for key, value := range message_data {
// Do nothing for keys with "_url" appendix.
if strings.Contains(key, "_url") {
c.Log.Debugln("_url key found in pre-stage, skipping:", key)
continue
}
// Do nothing (yet) on repeatables.
if strings.Contains(key, "repeatable") {
c.Log.Debugln("Key containing 'repeatable' in pre-stage, skipping:", key)
continue
}
if len(repeatables) > 0 {
if strings.Contains(key, "repeatable_item_") {
c.Log.Debugln("Repeatable key in pre-stage, skipping:", key)
continue
}
}
c.Log.Debugln("Processing message data key:", key)
// Check if we have an item with "_url" appendix. This means
// that we should generate a link.
val_url, found := message_data[key + "_url"]
// Generate a link and put into message if key with "_url"
// was found.
var s string = ""
if found {
c.Log.Debugln("Found _url key, will create HTML link")
s = fmt.Sprintf("<a href='%s'>%s</a>", val_url, value)
} else {
c.Log.Debugln("Found no _url key, will use as-is")
s = value
}
msg_tpl = strings.Replace(msg_tpl, "{" + key + "}", s, -1)
}
// Process repeatables.
repeatable_tpl, repeatable_found := message_data["repeatable_message"]
if repeatable_found {
var repeatables_string string = ""
repeatables_count, _ := strconv.Atoi(message_data["repeatables_count"])
idx := 0
for {
if (idx == repeatables_count) {
c.Log.Debug("IDX goes above repeatables_count, breaking loop")
break
}
var repstring string = repeatable_tpl
for i := range repeatables {
c.Log.Debugln("Processing repeatable variable:", repeatables[i] + strconv.Itoa(idx))
var data string = ""
rdata := message_data["repeatable_item_" + repeatables[i] + strconv.Itoa(idx)]
rurl, rurl_found := message_data["repeatable_item_" + repeatables[i] + strconv.Itoa(idx) + "_url"]
if rurl_found {
c.Log.Debugln("Found _url key, will create HTML link")
data = fmt.Sprintf("<a href='%s'>%s</a>", rurl, rdata)
} else {
c.Log.Debugln("Found no _url key, will use as-is")
data = rdata
}
repstring = strings.Replace(repstring, "{" + repeatables[i] + "}", data, -1)
}
repeatables_string += repstring
c.Log.Debugln("Repeatable string:", repstring)
idx += 1
}
msg_tpl = strings.Replace(msg_tpl, "{repeatables}", repeatables_string, -1)
}
msg_tpl = strings.Replace(msg_tpl, "{newline}", "<br />", -1)
// Replace all "\n" with "<br />".
msg_tpl = strings.Replace(msg_tpl, "\n", "<br />", -1)
c.Log.Debugln("Crafted message:", msg_tpl)
// Send message.
mxc.SendMessage(message_body)
mxc.SendMessage(msg_tpl)
}
// This function sends already prepared message to room.

View File

@ -23,6 +23,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
// local
@ -57,12 +58,30 @@ func (sh SlackHandler) ServeHTTP(respwriter http.ResponseWriter, req *http.Reque
if strings.Contains(url_splitted[2], config.Slack.Random1) && strings.Contains(url_splitted[3], config.Slack.Random2) && strings.Contains(url_splitted[4], config.Slack.LongRandom) {
c.Log.Debugf("Passed data belongs to '%s' and should go to '%s' pusher, protocol '%s'", name, config.Remote.PushTo, config.Remote.Pusher)
// Parse message into SlackMessage structure.
if strings.Contains(string(body)[0:7], "payload") {
// We have HTTP form payload. It still should be a
// parseable JSON string, we just need to do some
// preparations.
// First - remove "payload=" from the beginning.
temp_body := string(body)
temp_body = strings.Replace(temp_body, "payload=", "", 1)
// Second - unescape data.
temp_body, err := url.QueryUnescape(temp_body)
if err != nil {
c.Log.Errorln("Failed to decode body into parseable string!")
return
}
// And finally - convert body back to bytes.
body = []byte(temp_body)
}
slackmsg := slackmessage.SlackMessage{}
err := json.Unmarshal(body, &slackmsg)
if err != nil {
c.Log.Error("Failed to decode JSON into SlackMessage struct: '%s'", err.Error())
return
}
c.Log.Debug("Received message:", fmt.Sprintf("%+v", slackmsg))
c.SendToPusher(config.Remote.Pusher, config.Remote.PushTo, slackmsg)
sent_to_pusher = true
}