openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
app/admin/main/apm/service/BUILD Normal file
View File

@@ -0,0 +1,130 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"app_test.go",
"canal_test.go",
"dapper_test.go",
"databus_test.go",
"members_test.go",
"monitor_test.go",
"need_test.go",
"pprof_test.go",
"prometheus_test.go",
"service_test.go",
"tree_test.go",
"upload_test.go",
"user_test.go",
"ut_dash_test.go",
"ut_rank_test.go",
"ut_report_test.go",
"ut_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/admin/main/apm/conf:go_default_library",
"//app/admin/main/apm/dao:go_default_library",
"//app/admin/main/apm/dao/mock:go_default_library",
"//app/admin/main/apm/model/canal:go_default_library",
"//app/admin/main/apm/model/monitor:go_default_library",
"//app/admin/main/apm/model/need:go_default_library",
"//app/admin/main/apm/model/pprof:go_default_library",
"//app/admin/main/apm/model/ut:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
"//vendor/github.com/bouk/monkey:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"alarm.go",
"app.go",
"canal.go",
"dapper.go",
"databus.go",
"discovery.go",
"ecode.go",
"log.go",
"members.go",
"monitor.go",
"need.go",
"open.go",
"platform.go",
"pprof.go",
"prometheus.go",
"qywechat.go",
"service.go",
"task.go",
"tree.go",
"upload.go",
"user.go",
"ut.go",
"ut_app.go",
"ut_dash.go",
"ut_rank.go",
"ut_report.go",
],
importpath = "go-common/app/admin/main/apm/service",
tags = ["automanaged"],
deps = [
"//app/admin/main/apm/conf:go_default_library",
"//app/admin/main/apm/dao:go_default_library",
"//app/admin/main/apm/model/app:go_default_library",
"//app/admin/main/apm/model/canal:go_default_library",
"//app/admin/main/apm/model/databus:go_default_library",
"//app/admin/main/apm/model/discovery:go_default_library",
"//app/admin/main/apm/model/ecode:go_default_library",
"//app/admin/main/apm/model/log:go_default_library",
"//app/admin/main/apm/model/monitor:go_default_library",
"//app/admin/main/apm/model/need:go_default_library",
"//app/admin/main/apm/model/pprof:go_default_library",
"//app/admin/main/apm/model/tree:go_default_library",
"//app/admin/main/apm/model/user:go_default_library",
"//app/admin/main/apm/model/ut:go_default_library",
"//app/admin/main/config/model:go_default_library",
"//app/tool/saga/service/gitlab:go_default_library",
"//library/conf/env:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/naming:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/queue/databus/report:go_default_library",
"//library/sync/errgroup.v2:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
"//vendor/github.com/Shopify/sarama:go_default_library",
"//vendor/github.com/bsm/sarama-cluster:go_default_library",
"//vendor/github.com/jinzhu/gorm:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/github.com/robfig/cron:go_default_library",
"//vendor/github.com/smartystreets/goconvey/web/server/contract:go_default_library",
"//vendor/github.com/smartystreets/goconvey/web/server/parser:go_default_library",
"@com_github_gogo_protobuf//sortkeys:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

188
app/admin/main/apm/service/alarm.go Normal file
View File

@@ -0,0 +1,188 @@
package service
import (
"bytes"
"context"
"encoding/json"
"fmt"
"go-common/app/admin/main/apm/conf"
"go-common/app/admin/main/apm/model/databus"
"go-common/library/ecode"
"go-common/library/log"
"net/http"
)
//Alarm ...
// func (s *Service) Alarm(c context.Context, group, action string) (result *databus.AlarmOpen, err error) {
// url := conf.Conf.Alarm.DatabusURL
// var jsonBytes []byte
// body := &struct {
// Action string `json:"Action"`
// PublicKey string `json:"PublicKey"`
// Signature int8 `json:"Signature"`
// Group string `json:"Group"`
// }{
// Action: action,
// PublicKey: conf.Conf.Alarm.DatabusKey,
// Signature: 1,
// Group: group,
// }
// if jsonBytes, err = json.Marshal(body); err != nil {
// log.Error("json.Marshal(body) error(%v)", err)
// return
// }
// req, err := http.NewRequest("POST", url, strings.NewReader(string(jsonBytes)))
// if err != nil {
// err = ecode.RequestErr
// return
// }
// req.Header.Set("Content-Type", "application/json")
// // req.Header.Set("Cookie", cookie)
// result = &databus.AlarmOpen{}
// if err = s.client.Do(c, req, result); err != nil {
// fmt.Printf("result=(%v) error=(%v)", result, err)
// log.Error("Alarm() error(%v)", err)
// err = ecode.RequestErr
// return
// }
// return
// }
//Opsmind ...
func (s *Service) Opsmind(c context.Context, project, group, action, Owners string, percentage, fortime int64, silence bool) (result *databus.Res, err error) {
var scopes []databus.Scope
scopes = append(scopes, databus.Scope{Type: 0, Key: "group", Val: []string{group}})
var owner databus.Owner
owner.Owner = Owners
owner.App = project
url := conf.Conf.Alarm.DatabusURL
body := &struct {
Action string `json:"Action"`
PublicKey string `json:"PublicKey"`
Signature int8 `json:"Signature"`
PolicyID string `json:"PolicyId"`
CateGory string `json:"CateGory"`
Silence bool `json:"Silence"`
Scope []databus.Scope `json:"Scope"`
TriggerName string `json:"TriggerName"`
TriggerOperator string `json:"TriggerOperator"`
TriggerLevel string `json:"TriggerLevel"`
TriggerThreshold int64 `json:"TriggerThreshold"`
TriggerFor int64 `json:"TriggerFor"`
TriggerNoDataFor int64 `json:"TriggerNoDataFor"`
TriggerNotes databus.Owner `json:"TriggerNotes"`
}{
Action: action,
PublicKey: conf.Conf.Alarm.DatabusKey,
Signature: 1,
PolicyID: "3mwipx2caggxc",
CateGory: "Databus",
Silence: silence,
Scope: scopes,
TriggerName: fmt.Sprintf("%s消费落后告警", group),
TriggerOperator: "<",
TriggerLevel: "P3",
TriggerThreshold: percentage,
TriggerFor: fortime,
TriggerNoDataFor: 300,
TriggerNotes: owner,
}
// json.NewEncoder(os.Stdout).Encode(body)
buf := &bytes.Buffer{}
err = json.NewEncoder(buf).Encode(body)
if err != nil {
log.Error("json.Marshal(body) error(%v)", err)
return
}
req, err := http.NewRequest("POST", url, buf)
if err != nil {
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/json")
// req.Header.Set("Cookie", cookie)
result = &databus.Res{}
if err = s.client.Do(c, req, result); err != nil {
log.Error("Alarm() error(%v)", err)
err = ecode.RequestErr
return
}
return
}
// OpsmindRemove ...
func (s *Service) OpsmindRemove(c context.Context, adjustid, action string) (result *databus.Res, err error) {
url := conf.Conf.Alarm.DatabusURL
body := &struct {
Action string `json:"Action"`
PublicKey string `json:"PublicKey"`
Signature int8 `json:"Signature"`
PolicyID string `json:"PolicyId"`
AdjustID string `json:"AdjustId"`
}{
Action: action,
PublicKey: conf.Conf.Alarm.DatabusKey,
Signature: 1,
PolicyID: "3mwipx2caggxc",
AdjustID: adjustid,
}
// json.NewEncoder(os.Stdout).Encode(body)
buf := &bytes.Buffer{}
err = json.NewEncoder(buf).Encode(body)
if err != nil {
log.Error("json.Marshal(body) error(%v)", err)
return
}
req, err := http.NewRequest("POST", url, buf)
if err != nil {
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/json")
// req.Header.Set("Cookie", cookie)
result = &databus.Res{}
if err = s.client.Do(c, req, result); err != nil {
log.Error("Alarm() error(%v)", err)
err = ecode.RequestErr
return
}
return
}
//OpsmindQuery ...
func (s *Service) OpsmindQuery(c context.Context, group, action string) (result *databus.ResQuery, err error) {
var query []databus.Query
query = append(query, databus.Query{Key: "group", Val: []string{group}})
url := conf.Conf.Alarm.DatabusURL
body := &struct {
Action string `json:"Action"`
PublicKey string `json:"PublicKey"`
Signature int8 `json:"Signature"`
Query []databus.Query `json:"Query"`
}{
Action: action,
PublicKey: conf.Conf.Alarm.DatabusKey,
Signature: 1,
Query: query,
}
// json.NewEncoder(os.Stdout).Encode(body)
buf := &bytes.Buffer{}
err = json.NewEncoder(buf).Encode(body)
if err != nil {
log.Error("json.Marshal(body) error(%v)", err)
return
}
req, err := http.NewRequest("POST", url, buf)
if err != nil {
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/json")
// req.Header.Set("Cookie", cookie)
result = &databus.ResQuery{}
if err = s.client.Do(c, req, result); err != nil {
log.Error("Alarm() error(%v)", err)
err = ecode.RequestErr
}
return
}

136
app/admin/main/apm/service/app.go Normal file
View File

@@ -0,0 +1,136 @@
package service
import (
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/app/admin/main/apm/model/app"
"github.com/jinzhu/gorm"
)
// AppAdd Appadd
func (s *Service) AppAdd(c *bm.Context, username string, AppTreeID int64, AppID string, Limit int64) (err error) {
a := &app.App{}
b := &app.Auth{}
tx := s.DB.Begin()
var sqlLogs []*map[string]interface{}
if err = s.DB.Where("app_tree_id = ?", AppTreeID).First(a).Error; err == gorm.ErrRecordNotFound {
//新加
a = &app.App{
AppTreeID: AppTreeID,
AppID: AppID,
Limit: Limit,
}
if err = tx.Create(a).Error; err != nil {
log.Error("s.appAdd create error(%v)", err)
tx.Rollback()
return
}
sqlLog := &map[string]interface{}{
"SQLType": "add",
"Content": a,
}
sqlLogs = append(sqlLogs, sqlLog)
aa := &app.App{}
if a.AppID != "main.common-arch.msm-service" {
if err = tx.Where("app_id=?", "main.common-arch.msm-service").First(aa).Error; err != nil {
log.Error("s.appAdd not find main.common-arch.msm-service error(%v)", err)
tx.Rollback()
return
}
//查询授权
if err = tx.Where("service_tree_id=? and app_tree_id=?", aa.AppTreeID, a.AppTreeID).First(b).Error; err != nil {
//创建msm授权
b = &app.Auth{
ServiceTreeID: aa.AppTreeID,
ServiceID: aa.AppID,
AppTreeID: a.AppTreeID,
AppID: a.AppID,
RPCMethod: "ALL",
HTTPMethod: "ALL",
Quota: 10000000,
}
if err = tx.Create(b).Error; err != nil {
log.Error("s.appAdd main.common-arch.msm-service create error(%v)", err)
tx.Rollback()
return
}
sqlLog := &map[string]interface{}{
"SQLType": "add",
"Content": b,
}
sqlLogs = append(sqlLogs, sqlLog)
}
}
} else if err != nil {
log.Error("s.appAdd app_tree_id first error(%v)", err)
tx.Rollback()
return
} else {
//更新
ups := map[string]interface{}{
"app_id": AppID,
}
if err = tx.Model(a).Where("app_tree_id = ?", AppTreeID).Updates(ups).Error; err != nil {
log.Error("s.appEdit updates error(%v)", err)
tx.Rollback()
return
}
sqlLog := &map[string]interface{}{
"SQLType": "update",
"Where": "app_tree_id = ?",
"Value1": AppTreeID,
"Update": ups,
"Old": "",
}
sqlLogs = append(sqlLogs, sqlLog)
var (
services []*app.Auth
)
if err = tx.Where("app_tree_id = ?", AppTreeID).Find(&services).Error; err == nil {
for _, v := range services {
ups = map[string]interface{}{
"app_id": AppID,
}
if err = tx.Model(&app.Auth{}).Where("id = ?", v.ID).Updates(ups).Error; err != nil {
log.Error("s.appEdit auth2 app_tree_id updates error(%v)", err)
tx.Rollback()
return
}
sqlLog := &map[string]interface{}{
"SQLType": "update",
"Where": "id = ?",
"Value1": v.ID,
"Update": ups,
"Old": v,
}
sqlLogs = append(sqlLogs, sqlLog)
}
}
if err = tx.Where("service_tree_id=?", AppTreeID).Find(&services).Error; err == nil {
for _, v := range services {
ups = map[string]interface{}{
"service_id": AppID,
}
if err = tx.Model(&app.Auth{}).Where("id=?", v.ID).Updates(ups).Error; err != nil {
log.Error("s.appEdit auth2 service_tree_id updates error(%v)", err)
tx.Rollback()
return
}
sqlLog := &map[string]interface{}{
"SQLType": "update",
"Where": "id = ?",
"Value1": v.ID,
"Update": ups,
"Old": v,
}
sqlLogs = append(sqlLogs, sqlLog)
}
}
}
tx.Commit()
s.SendLog(*c, username, 0, 5, 0, "apmSvc.appAdd", sqlLogs)
return
}

13
app/admin/main/apm/service/app_test.go Normal file
View File

@@ -0,0 +1,13 @@
package service
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestApp(t *testing.T) {
Convey("app", t, func() {
t.Log("app test")
})
}

980
app/admin/main/apm/service/canal.go Normal file
View File

@@ -0,0 +1,980 @@
package service
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"go-common/app/admin/main/apm/conf"
cml "go-common/app/admin/main/apm/model/canal"
"go-common/app/admin/main/apm/model/user"
cgm "go-common/app/admin/main/config/model"
"go-common/library/conf/env"
"go-common/library/ecode"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"github.com/BurntSushi/toml"
)
const (
_dialTimeout = "500ms"
_readTimeout = "1s"
_writeTimeout = "1s"
_idleTimeout = "60s"
_flavor = "mysql"
_heartbeatPeriod = 60
_canreadTimeout = 90
)
var (
getBuildIDAPI = "%s/x/admin/config/build/builds"
getConfigValueAPI = "%s/x/admin/config/config/value"
getConfigIDAPI = "%s/x/admin/config/config/configs"
getAllErrorsAPI = "%s/x/internal/canal/errors"
createConfigAPI = "%s/x/admin/config/canal/config/create"
configByNameAPI = "%s/x/admin/config/canal/name/configs"
updateConfigAPI = "%s/x/admin/config/home/config/update"
checkMasterAPI = "%s/x/internal/canal/master/check"
ok = 0
)
type result struct {
Data json.RawMessage `json:"data"`
Code int `json:"code"`
}
type list []struct {
ID int `json:"id"`
}
type configs struct {
BuildFiles fileList `json:"build_files"`
}
type fileList []struct {
ID int `json:"id"`
Name string `json:"name"`
}
type groupInfo struct {
Group string `json:"group"`
Topic string `json:"topic"`
AppID int `json:"app_id"`
}
type appInfo struct {
ID int `json:"id"`
AppKey string `json:"app_key"`
AppSecret string `json:"app_secret"`
}
// ConfigProxy config proxy
func (s *Service) ConfigProxy(c context.Context, method string, uri string, params url.Values, cookie string, args ...interface{}) (data json.RawMessage, err error) {
//common params
if params != nil && args == nil {
params.Set("app_name", "main.common-arch.canal")
params.Set("tree_id", "3766")
params.Set("zone", env.Zone)
params.Set("env", env.DeployEnv)
}
res := result{}
fmt.Println("ConfigProxy uri=", uri, "params=", params.Encode())
req, err := s.client.NewRequest(method, uri, "", params)
if err != nil {
log.Error("s.client.NewRequest() error(%v)", err)
}
if cookie != "" {
req.Header.Set("Cookie", cookie)
}
if err = s.client.Do(c, req, &res); err != nil {
log.Error("canal.request get url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return
}
if res.Code != ok {
log.Error("canal.request get url:"+uri+" params:(%v) returnCode:(%v)", params.Encode(), res.Code)
return
}
data = res.Data
return
}
func (s *Service) getBuildID(c context.Context, cookie string) (dat list, err error) {
var (
params = url.Values{}
data = json.RawMessage{}
)
uri := fmt.Sprintf(getBuildIDAPI, conf.Conf.Host.SVENCo)
if data, err = s.ConfigProxy(c, "GET", uri, params, cookie); err != nil {
log.Error("getBuildID() response errors: %d", err)
return
}
if data == nil {
return
}
if err = json.Unmarshal([]byte(data), &dat); err != nil {
log.Error("getBuildID() json.Unmarshal errors: %s", err)
return
}
return
}
func (s *Service) getConfigID(c context.Context, id string, cookie string) (dat fileList, err error) {
var (
params = url.Values{}
data = json.RawMessage{}
)
params.Set("build_id", id)
uri := fmt.Sprintf(getConfigIDAPI, conf.Conf.Host.SVENCo)
if data, err = s.ConfigProxy(c, "GET", uri, params, cookie); err != nil {
log.Error("getConfigID() response errors: %d", err)
return
}
if data == nil {
return
}
res := &configs{}
if err = json.Unmarshal([]byte(data), res); err != nil {
log.Error("getConfigID() json.Unmarshal errors: %s", err)
return
}
dat = res.BuildFiles
return
}
func (s *Service) getConfigValue(c context.Context, id string, cookie string) (res *cml.Conf, err error) {
var (
params = url.Values{}
data = json.RawMessage{}
)
params.Set("config_id", id)
uri := fmt.Sprintf(getConfigValueAPI, conf.Conf.Host.SVENCo)
if data, err = s.ConfigProxy(c, "GET", uri, params, cookie); err != nil {
log.Error("getConfigValue() response errors: %d", err)
return
}
if data == nil {
return
}
res = new(cml.Conf)
if err = json.Unmarshal([]byte(data), &res); err != nil {
log.Error("getConfigValue() json.Unmarshal errors: %s", err)
return
}
return
}
//ScanByAddrFromConfig 根据addr查询配置信息
func (s *Service) ScanByAddrFromConfig(c context.Context, addr string, cookie string) (res *cml.Conf, err error) {
var (
buildIDList list
configIDList fileList
)
//find build_id
if buildIDList, err = s.getBuildID(c, cookie); err != nil {
return
}
fmt.Println("BuildID=", buildIDList)
for _, v := range buildIDList {
if v.ID != 0 {
//find config_id
if configIDList, err = s.getConfigID(c, strconv.Itoa(v.ID), cookie); err != nil {
return
}
fmt.Println("configID=", configIDList)
for _, val := range configIDList {
if val.ID != 0 && val.Name == addr+".toml" {
//find config
res, err = s.getConfigValue(c, strconv.Itoa(val.ID), cookie)
return
}
continue
}
}
continue
}
return
}
//ApplyAdd canal apply
func (s *Service) ApplyAdd(c *bm.Context, v *cml.Canal, username string) (err error) {
cnt := 0
f := strings.Contains(v.Addr, ":")
if !f {
err = ecode.CanalAddrFmtErr
return
}
if err = s.DBCanal.Model(&cml.Canal{}).Where("addr=?", v.Addr).Count(&cnt).Error; err != nil {
log.Error("apmSvc.CanalAdd count error(%v)", err)
err = ecode.RequestErr
return
}
if cnt > 0 {
err = ecode.CanalAddrExist
return
}
canal := &cml.Canal{
Addr: v.Addr,
Cluster: v.Cluster,
Leader: v.Leader,
BinName: v.BinName,
BinPos: v.BinPos,
Remark: v.Remark,
}
if err = s.DBCanal.Create(canal).Error; err != nil {
log.Error("apmSvc.CanalAdd create error(%v)", err)
return
}
s.SendLog(*c, username, 0, 1, canal.ID, "apmSvc.CanalAdd", canal)
return
}
//ApplyDelete canal delete
func (s *Service) ApplyDelete(c *bm.Context, v *cml.ScanReq, username string) (err error) {
cc := &cml.Canal{}
if err = s.DBCanal.Model(&cml.Canal{}).Where("addr=?", v.Addr).Find(cc).Error; err != nil {
log.Error("apmSvc.ApplyDelete count error(%v)", err)
err = ecode.RequestErr
return
}
id := cc.ID
if err = s.DBCanal.Model(&cml.Canal{}).Where("id = ?", id).Update("is_delete", 1).Error; err != nil {
log.Error("apmSvc.canalDelete canalDelete error(%v)", err)
return
}
sqlLog := &map[string]interface{}{
"SQLType": "delete",
"Value": v.Addr,
}
s.SendLog(*c, username, 0, 3, id, "apmSvc.canalDelete", sqlLog)
return
}
//ApplyEdit canal edit
func (s *Service) ApplyEdit(c *bm.Context, v *cml.EditReq, username string) (err error) {
cc := &cml.Canal{}
if err = s.DBCanal.Where("id = ?", v.ID).Find(cc).Error; err != nil {
log.Error("apmSvc.CanalEdit find(%d) error(%v)", v.ID, err)
return
}
ups := map[string]interface{}{}
if _, ok := c.Request.Form["bin_name"]; ok {
ups["bin_name"] = v.BinName
}
if _, ok := c.Request.Form["bin_pos"]; ok {
ups["bin_pos"] = v.BinPos
}
if _, ok := c.Request.Form["remark"]; ok {
ups["remark"] = v.Remark
}
if _, ok := c.Request.Form["project"]; ok {
ups["cluster"] = v.Project
}
if _, ok := c.Request.Form["leader"]; ok {
ups["leader"] = v.Leader
}
if err = s.DBCanal.Model(&cml.Canal{}).Where("id = ?", v.ID).Updates(ups).Error; err != nil {
log.Error("apmSvc.CanalEdit updates error(%v)", err)
return
}
sqlLog := &map[string]interface{}{
"SQLType": "update",
"Where": "id = ?",
"Value1": v.ID,
"Update": ups,
"Old": cc,
}
s.SendLog(*c, username, 0, 2, v.ID, "apmSvc.CanalEdit", sqlLog)
return
}
//GetScanInfo is
func (s *Service) GetScanInfo(c context.Context, v *cml.ScanReq, username string, cookie string) (confData *cml.Results, err error) {
if confData, err = s.getDocFromConf(c, v.Addr, cookie); err != nil {
return
}
if confData == nil {
return
}
if err = s.Permit(c, username, user.CanalEdit); err != nil {
confData.Document.Instance.User = ""
confData.Document.Instance.Password = ""
}
return confData, nil
}
//GetAllErrors 调用x/internal/canal/errors 查询错误信息
func (s *Service) GetAllErrors(c context.Context) (errs map[string]string, err error) {
var (
data json.RawMessage
host string
)
type v struct {
Error string `json:"error"`
InstanceError map[string]string `json:"instance_error"`
}
if host, err = s.getCanalInstance(c); err != nil {
return
}
uri := fmt.Sprintf(getAllErrorsAPI, host)
if data, err = s.ConfigProxy(c, "GET", uri, nil, ""); err != nil {
return
}
res := new(v)
if err = json.Unmarshal([]byte(data), &res); err != nil {
return
}
errs = res.InstanceError
return
}
func (s *Service) getCanalInstance(c context.Context) (host string, err error) {
params := url.Values{}
params.Set("appid", "main.common-arch.canal")
params.Set("env", env.DeployEnv)
params.Set("hostname", env.Hostname)
params.Set("status", "3")
var ins struct {
ZoneInstances map[string][]struct {
Addrs []string `json:"addrs"`
} `json:"zone_instances"`
}
resp, err := s.DiscoveryProxy(c, "GET", "fetch", params)
if err != nil {
return
}
rb, err := json.Marshal(resp)
if err != nil {
return
}
json.Unmarshal(rb, &ins)
inss := ins.ZoneInstances[env.Zone]
for _, zone := range inss {
for _, addr := range zone.Addrs {
if strings.Contains(addr, "http://") {
host = addr
break
}
}
}
return
}
//GetConfigsByName obtain configs from configByNameAPI
func (s *Service) getConfigsByName(c context.Context, name string, cookie string) (configs *cgm.Config, err error) {
var (
params = url.Values{}
data = json.RawMessage{}
result []*cgm.Config
)
params.Set("token", conf.Conf.AppToken)
params.Set("name", name+".toml")
uri := fmt.Sprintf(configByNameAPI, conf.Conf.Canal.CANALSVENCo)
if data, err = s.ConfigProxy(c, "POST", uri, params, cookie); err != nil {
err = ecode.GetConfigByNameErr
return
}
if data == nil {
return
}
if err = json.Unmarshal([]byte(data), &result); err != nil {
log.Error("configByNameAPI() json.Unmarshal errors: %s", err)
return
}
if len(result) == 0 {
return
}
configs = result[0]
return
}
//ProcessCanalList get canal list
func (s *Service) ProcessCanalList(c context.Context, v *cml.ListReq) (listdata *cml.Paper, err error) {
type errorCanal struct {
cml.Canal
Error string `json:"error"`
}
var (
cc []*errorCanal
count int
errMap map[string]string
)
query := " is_delete = 0 "
if v.Addr != "" {
query += fmt.Sprintf("and addr = '%s' ", v.Addr)
}
if v.Project != "" {
query += fmt.Sprintf("and cluster = '%s' ", v.Project)
}
err = s.DBCanal.Where(query).Order("id DESC").Offset((v.Pn - 1) * v.Ps).Limit(v.Ps).Find(&cc).Error
if err != nil {
log.Error("apmSvc.CanalList error(%v)", err)
return
}
err = s.DBCanal.Model(&cml.Canal{}).Where(query).Count(&count).Error
if err != nil {
log.Error("apmSvc.CanalList count error(%v)", err)
return
}
// add error info
if count > 0 {
if errMap, err = s.GetAllErrors(c); err != nil {
log.Error("apmSvc.DBCanalApply GetAllErrors error(%v)", err)
}
for _, va := range cc {
va.Error = errMap[va.Addr]
}
}
listdata = &cml.Paper{
Pn: v.Pn,
Ps: v.Ps,
Items: cc,
Total: count,
}
return
}
//ProcessApplyList get apply list
func (s *Service) ProcessApplyList(c context.Context, v *cml.ListReq) (listdata *cml.Paper, err error) {
var (
cc []*cml.Apply
count int
)
query := " state !=3 "
if v.Addr != "" {
query += fmt.Sprintf("and addr = '%s' ", v.Addr)
}
if v.Project != "" {
query += fmt.Sprintf("and cluster = '%s' ", v.Project)
}
if v.Status > 0 {
query += fmt.Sprintf("and state = '%d' ", v.Status)
}
err = s.DBCanal.Model(&cml.Apply{}).Where(query).Count(&count).Error
if err != nil {
log.Error("apmSvc.ApplyList count error(%v)", err)
return
}
err = s.DBCanal.Where(query).Order("id DESC").Offset((v.Pn - 1) * v.Ps).Limit(v.Ps).Find(&cc).Error
if err != nil {
log.Error("apmSvc.ApplyList error(%v)", err)
return
}
listdata = &cml.Paper{
Pn: v.Pn,
Ps: v.Ps,
Items: cc,
Total: count,
}
return
}
//ProcessConfigInfo process info to config center
func (s *Service) ProcessConfigInfo(c context.Context, v *cml.ConfigReq, cookie string, username string) (err error) {
var (
confs *cgm.Config
comment string
query []map[string]string
data []byte
)
if comment, err = s.jointConfigInfo(c, v, cookie); err != nil {
return
}
if confs, err = s.getConfigsByName(c, v.Addr, cookie); err != nil {
return
}
if confs != nil {
query = []map[string]string{{
"name": v.Addr + ".toml",
"comment": comment,
"mark": v.Mark,
}}
data, err = json.Marshal(query)
if err != nil {
return
}
va := url.Values{}
va.Set("data", string(data))
va.Set("user", username)
if _, err = s.updateConfig(c, va, cookie); err != nil {
return
}
} else {
//never register config's
params := url.Values{}
params.Set("comment", comment)
params.Set("mark", v.Mark)
params.Set("state", "2")
params.Set("from", "0")
params.Set("name", v.Addr+".toml")
params.Set("user", username)
if _, err = s.createConfig(c, params, cookie); err != nil {
return
}
}
if confs, err = s.getConfigsByName(c, v.Addr, cookie); confs == nil {
return
}
if err = s.dao.SetConfigID(confs.ID, v.Addr); err != nil {
return
}
return
}
//CheckMaster canal check
func (s *Service) CheckMaster(c context.Context, v *cml.ConfigReq) (err error) {
res := result{}
host, _ := s.getCanalInstance(c)
params := url.Values{}
params.Set("user", v.User)
params.Set("password", v.Password)
params.Set("addr", v.Addr)
uri := fmt.Sprintf(checkMasterAPI, host)
req, err := s.client.NewRequest("POST", uri, "", params)
if err != nil {
err = ecode.CheckMasterErr
return
}
if err = s.client.Do(c, req, &res); err != nil {
err = ecode.CheckMasterErr
return
}
if res.Code != 0 {
err = ecode.CheckMasterErr
return
}
return
}
//ProcessCanalInfo is
func (s *Service) ProcessCanalInfo(c context.Context, v *cml.ConfigReq, username string) (err error) {
var (
cnt = 0
info = &cml.Canal{}
)
if err = s.DBCanal.Select("addr,cluster,leader").Where("addr=?", v.Addr).Find(info).Error; err == nil {
if v.Project == "" {
v.Project = info.Cluster
}
if v.Leader == "" {
v.Leader = info.Leader
}
}
if cnt, err = s.dao.CanalApplyCounts(v); err != nil {
return
}
if cnt > 0 {
if err = s.dao.CanalApplyEdit(v, username); err != nil {
return
}
} else {
if err = s.dao.CanalApplyCreate(v, username); err != nil {
return
}
}
return
}
//getCommentFromConf get document info from configbyname
func (s *Service) getDocFromConf(c context.Context, addr string, cookie string) (confData *cml.Results, err error) {
var conf *cgm.Config
if conf, err = s.getConfigsByName(c, addr, cookie); err != nil {
return
}
if conf == nil {
return
}
confData = new(cml.Results)
if _, err = toml.Decode(conf.Comment, &confData.Document); err != nil {
err = ecode.ConfigParseErr
log.Error("comment toml.decode error(%v)", err)
return
}
row := &cml.Apply{}
err = s.DBCanal.Model(&cml.Apply{}).Select("`cluster`,`leader`").Where("addr=?", addr).Scan(row).Error
if err == nil {
confData.Cluster = row.Cluster
confData.Leader = row.Leader
} else {
res := &cml.Canal{}
err = s.DBCanal.Model(&cml.Canal{}).Select("`cluster`,`leader`").Where("addr=?", addr).Scan(res).Error
if err != nil {
log.Error("canalinfo get error(%v)", err)
return
}
confData.Cluster = res.Cluster
confData.Leader = res.Leader
}
confData.ID = conf.ID
confData.Addr = addr
return
}
//ProcessConfigInfo is
func (s *Service) jointConfigInfo(c context.Context, v *cml.ConfigReq, cookie string) (comment string, err error) {
var (
buf *bytes.Buffer
cfg *cml.Config
dbs []*cml.DB
di *cml.Databus
confData *cml.Results
sid int64
schemas = make(map[string]bool)
)
//analysis request params
if err = json.Unmarshal([]byte(v.Databases), &dbs); err != nil {
log.Error("apmSvc.jointConfigInfo Unmarshal error(%v)", err)
err = ecode.DatabasesUnmarshalErr
return
}
for _, db := range dbs {
if db.Databus == nil {
continue
}
//Find duplicate
if schemas[db.Schema+db.Databus.Group] {
log.Error("jointConfigInfo find duplicate databus group (%v)", db.Databus.Group)
err = ecode.DatabusDuplErr
return
}
schemas[db.Schema+db.Databus.Group] = true
//get databusinfo
if di, err = s.databusInfo(db.Databus.Group, db.Databus.Addr, db.Schema, db.Table); err != nil {
return
}
db.Databus = di
}
if sid, err = s.getServerID(v.Addr); err != nil {
err = ecode.CanalAddrFmtErr
return
}
ist := &cml.Instance{
Caddr: v.Addr,
MonitorPeriod: v.MonitorPeriod,
ServerID: sid,
Flavor: _flavor,
HeartbeatPeriod: _heartbeatPeriod,
ReadTimeout: _canreadTimeout,
DB: dbs,
}
if confData, err = s.getDocFromConf(c, v.Addr, cookie); err != nil {
return
}
if confData != nil && v.User == "" {
ist.User = confData.Document.Instance.User
} else {
ist.User = v.User
}
if confData != nil && v.Password == "" {
ist.Password = confData.Document.Instance.Password
} else {
ist.Password = v.Password
}
cfg = &cml.Config{
Instance: ist,
}
buf = new(bytes.Buffer)
if err = toml.NewEncoder(buf).Encode(cfg); err != nil {
return
}
comment = buf.String()
return
}
//CreateConfig send requests to createConfigAPI
func (s *Service) createConfig(c context.Context, params url.Values, cookie string) (res map[string]interface{}, err error) {
var (
data = json.RawMessage{}
)
uri := fmt.Sprintf(createConfigAPI, conf.Conf.Canal.CANALSVENCo)
params.Set("token", conf.Conf.AppToken)
if data, err = s.ConfigProxy(c, "POST", uri, params, cookie); err != nil {
err = ecode.ConfigCreateErr
return
}
if data == nil {
return
}
if err = json.Unmarshal([]byte(data), &res); err != nil {
log.Error("updateConfigAPI() json.Unmarshal errors: %s", err)
return
}
return
}
//UpdateConfig send requests to updateConfigAPI
func (s *Service) updateConfig(c context.Context, params url.Values, cookie string) (res map[string]interface{}, err error) {
var (
data = json.RawMessage{}
)
uri := fmt.Sprintf(updateConfigAPI, conf.Conf.Canal.CANALSVENCo)
params.Set("token", conf.Conf.AppToken)
if data, err = s.ConfigProxy(c, "POST", uri, params, cookie); err != nil {
err = ecode.ConfigUpdateErr
return
}
if data == nil {
return
}
if err = json.Unmarshal([]byte(data), &res); err != nil {
log.Error("updateConfigAPI() json.Unmarshal errors: %s", err)
return
}
return
}
//DatabusInfo joint databusinfo
func (s *Service) databusInfo(group string, addr string, schema string, table []*cml.Table) (d *cml.Databus, err error) {
var (
ai appInfo
gi groupInfo
act string
)
if ai, err = s.getAppInfo(group); err != nil {
err = ecode.DatabusAppErr
return
}
if gi, _, err = s.getGroupInfo(group); err != nil {
err = ecode.DatabusGroupErr
return
}
act = s.getAction(group)
name := "canal/" + schema
d = &cml.Databus{
Key: ai.AppKey,
Secret: ai.AppSecret,
Group: group,
Topic: gi.Topic,
Action: act,
Name: name,
Proto: "tcp",
Addr: addr,
Idle: 1,
Active: len(table),
DialTimeout: _dialTimeout,
ReadTimeout: _readTimeout,
WriteTimeout: _writeTimeout,
IdleTimeout: _idleTimeout,
}
return
}
//getAppInfo according group get appinfo
func (s *Service) getAppInfo(group string) (ai appInfo, err error) {
var table string
g, new, _ := s.getGroupInfo(group)
if !new {
table = "app"
} else {
table = "app2"
}
err = s.DBDatabus.Table(table).Select("`id`,`app_key`,`app_secret`").Where("`id`= ?", g.AppID).Find(&ai).Error
if err != nil {
log.Error("apmSvc.getAppInfo error(%v)", err)
return
}
return
}
//getGroupInfo according group get groupinfo
func (s *Service) getGroupInfo(group string) (gi groupInfo, new bool, err error) {
err = s.DBDatabus.Table("auth2").Select("auth2.group,topic.topic,auth2.app_id").Joins("join topic on topic.id=auth2.topic_id").Where("auth2.group= ?", group).Scan(&gi).Error
if err == nil {
new = true
return
}
err = s.DBDatabus.Table("auth").Select("group_name as `group`,topic,app_id").Where("group_name = ?", group).Find(&gi).Error
if err != nil {
log.Error("apmSvc.getGroupInfo error(%v", err)
return
}
return
}
//getAction according group get action
func (s *Service) getAction(group string) (action string) {
if strings.HasSuffix(group, "-P") {
action = "pub"
} else if strings.HasSuffix(group, "-S") {
action = "sub"
} else {
action = "notify"
}
return
}
//TableInfo get array table info
func (s *Service) TableInfo(table string) (infos []*cml.Table, err error) {
info := strings.Split(table, ",")
tab := make([]*cml.Table, len(info))
for i := range info {
tab[i] = &cml.Table{Name: info[i]}
}
infos = tab
return
}
//getServerID get server id from addr
func (s *Service) getServerID(addr string) (sid int64, err error) {
ip := strings.Split(addr, ".")
last := ip[len(ip)-1]
port := strings.Split(last, ":")
joint := fmt.Sprintf("%s%s%s", port[len(port)-1], ip[len(ip)-2], port[len(port)-2])
sid, err = strconv.ParseInt(joint, 10, 64)
return
}
//SendWechatMessage send wechat message
func (s *Service) SendWechatMessage(c context.Context, addr, aType, result, sender, note string, receiver []string) (err error) {
var (
detail string
users = []string{sender}
)
users = append(users, receiver...)
if env.DeployEnv != "prod" {
detail = fmt.Sprintf("http://%s-%s", env.DeployEnv, "%s")
} else {
detail = "http://%s"
}
switch aType {
case cml.TypeMap[cml.TypeApply]:
detail = fmt.Sprintf(detail, "sven.bilibili.co/#/canal/apply")
case cml.TypeMap[cml.TypeReview]:
detail = fmt.Sprintf(detail, "ops-log.bilibili.co/app/kibana 确认canal订阅消息")
}
msg := fmt.Sprintf("[sven系统抄送提示]\n发送方:%s\n事件: %s环境 DB:%s canal%s%s\n接收方:%s\n备注:%s\n详情:%s（请复制到浏览器打开）\n", sender, env.DeployEnv, addr, aType, result, strings.Join(receiver, ","), note, detail)
if err = s.dao.SendWechatToUsers(c, users, msg); err != nil {
log.Error("apmSvc.SendWechatMessage error(%v)", err)
return
}
return
}
// UpdateProcessTag canal审核通过之后,调用/x/admin/config/canal/tag/update,同步到配置中心发版
func (s *Service) UpdateProcessTag(c context.Context, configID int, cookie string) (err error) {
client := &http.Client{}
tokenURL := fmt.Sprintf("%s/x/admin/config/app/envs", conf.Conf.Canal.CANALSVENCo)
tokenParams := url.Values{}
tokenParams.Set("app_name", "main.common-arch.canal")
tokenParams.Set("tree_id", "3766")
tokenParams.Set("zone", env.Zone)
tokenReq, err := http.NewRequest(http.MethodGet, tokenURL+"?"+tokenParams.Encode(), nil)
if err != nil {
err = ecode.RequestErr
return
}
tokenReq.Header.Set("Content-Type", "application/json;charset=UTF-8")
tokenReq.Header.Set("Cookie", cookie)
tokenResp, err := client.Do(tokenReq)
if err != nil {
return
}
body, err := ioutil.ReadAll(tokenResp.Body)
if err != nil {
return
}
defer tokenResp.Body.Close()
var tokenRespObj struct {
Code int `json:"code"`
Data []struct {
Name string `json:"name"`
NickName string `json:"nickname"`
Token string `json:"token"`
} `json:"data"`
}
err = json.Unmarshal(body, &tokenRespObj)
if err != nil {
return fmt.Errorf("json unmarshal error: %s, get body: %s", err, body)
}
if tokenRespObj.Code != 0 {
log.Error(" tokenRespObj.Code: %d", tokenRespObj.Code)
return fmt.Errorf("tokenRespObj.Code: %d", tokenRespObj.Code)
}
var token string
for _, v := range tokenRespObj.Data {
if v.Name == env.DeployEnv {
token = v.Token
break
}
}
log.Info("tokenURL(%s), env.DeployEnv(%v), token(%v)", tokenURL+"?"+tokenParams.Encode(), env.DeployEnv, token)
updateURL := fmt.Sprintf("%s/x/admin/config/canal/tag/update", conf.Conf.Canal.CANALSVENCo)
params := url.Values{}
params.Set("app_name", "main.common-arch.canal")
params.Set("env", env.DeployEnv)
params.Set("zone", env.Zone)
params.Set("config_ids", fmt.Sprintf("%d", configID))
params.Set("tree_id", "3766")
params.Set("mark", "canal发版")
params.Set("user", "canalApprovalProcess")
params.Set("force", "1")
if conf.Conf.Canal.BUILD != "" {
params.Set("build", conf.Conf.Canal.BUILD)
} else {
params.Set("build", "docker-1")
}
log.Info("env:(%v), zone:(%v), build:(%v)", params.Get("env"), params.Get("zone"), params.Get("build"))
params.Set("token", token)
req, err := http.NewRequest(http.MethodPost, updateURL, strings.NewReader(params.Encode()))
if err != nil {
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Cookie", cookie)
var res struct {
Code int `json:"code"`
Message string `json:"message"`
}
if err = s.client.Do(c, req, &res); err != nil || res.Code != 0 {
log.Error("ApplyApprovalUpdateTag url(%s) err(%v), code(%v), message(%s)", updateURL+params.Encode(), err, res.Code, res.Message)
err = ecode.RequestErr
return
}
return
}

256
app/admin/main/apm/service/canal_test.go Normal file
View File

@@ -0,0 +1,256 @@
package service
import (
"context"
"net/url"
"testing"
"go-common/app/admin/main/apm/model/canal"
"github.com/BurntSushi/toml"
. "github.com/smartystreets/goconvey/convey"
)
var (
cookie = "username=fengshanshan; _AJSESSIONID=ee7c557c29e9b3f405b21c49a9aaa72a; sven-apm=9fc2bd4165a90a21df5803b4abc5cf7a81b7564e5119a8a547fd763bda757ceb"
_jsonstring = `[{ "schema":"123","table":[ {"name":"abc","primarykey":["order_id","new_id"],"omitfield":["new","old"]} , {"name":"def","primarykey":["order_id","new_id"],"omitfield":["new","old"] } ,{"name":"sfg","primarykey":["order_id","new_id"],"omitfield":["new","old"]} ],"infoc":{"taskID":"000846","proto":"tcp","addr":"172.19.100.20:5401","reporterAddr":"172.19.40.195:6200"}},{ "schema":"456","table":[ {"name":"abc" ,"primarykey":["order_id","new_id"],"omitfield":["new","old"]} , {"name":"def" } ,{"name":"sfg"} ],"databus": { "group": "LiveTime-LiveLive-P","addr": "172.16.33.158:6205"}}]`
)
func TestService_GetAllErrors(t *testing.T) {
Convey("test getAll errors", t, func() {
//svr = New(conf.Conf)
errS, err := svr.GetAllErrors(context.Background())
So(err, ShouldBeNil)
t.Log("getAllErrors=", errS)
})
}
func TestGetCanalInstance(t *testing.T) {
Convey("TestGetCanalInstance", t, func() {
host, err := svr.getCanalInstance(context.Background())
So(err, ShouldBeNil)
t.Log("canalinstance=", host)
})
}
func TestCheckMaster(t *testing.T) {
Convey("TestCheckMaster", t, func() {
req := &canal.ConfigReq{
Addr: "172.16.33.205:3310",
User: "gosync",
Password: "xmJ4KlRuXzc9UfNerOGHP0LpaS26VqoM",
}
err := svr.CheckMaster(context.Background(), req)
So(err, ShouldNotBeNil)
t.Log("err=", err)
})
}
func TestScanByAddrFromConfig(t *testing.T) {
Convey("test ScanByAddrFromConfig", t, func() {
//svr = New(conf.Conf)
res, err := svr.ScanByAddrFromConfig(context.Background(), "172.16.33.205:3308", cookie)
var document canal.Document
data, _ := toml.Decode(res.Comment, &document)
t.Log("toml=", data)
So(res, ShouldNotBeNil)
So(err, ShouldBeNil)
})
}
func TestScanInfo(t *testing.T) {
Convey("TestScanInfo", t, func() {
v := &canal.ScanReq{
Addr: "172.16.33.223:3308",
}
res, err := svr.GetScanInfo(context.Background(), v, "fss", cookie)
t.Logf("toml=%+v", res.Document)
So(res, ShouldNotBeNil)
So(err, ShouldBeNil)
})
}
func TestGetBuildID(t *testing.T) {
Convey("test getBuildID", t, func() {
res, err := svr.getBuildID(context.Background(), cookie)
t.Log("buidID=", res)
So(res, ShouldNotBeNil)
So(err, ShouldBeNil)
})
}
func TestGetConfigID(t *testing.T) {
Convey("test getConfigID", t, func() {
res, err := svr.getConfigID(context.Background(), "13", cookie)
t.Log("ConfigID=", res)
So(res, ShouldNotBeNil)
So(err, ShouldBeNil)
})
}
func TestGetConfigValue(t *testing.T) {
Convey("test getConfigValue", t, func() {
res, err := svr.getConfigValue(context.Background(), "112", cookie)
t.Log("ConfigValue=", res)
So(res, ShouldNotBeNil)
So(err, ShouldBeNil)
})
}
func TestConfigByName(t *testing.T) {
Convey("test configsByName", t, func() {
params := "172.16.33.205:3308"
res, err := svr.getConfigsByName(context.Background(), params, cookie)
t.Logf("configsByName:%+v,%+v", res, err)
So(res, ShouldNotBeEmpty)
})
}
func TestProcessCanalInfo(t *testing.T) {
Convey("TestProcessCanalInfo", t, func() {
v := &canal.ConfigReq{
Addr: "127.0.0.1:8001",
User: "admin",
Password: "admin",
// Project: "main.web-svr",
// Leader: "fss",
Databases: _jsonstring,
Mark: "sfs",
}
_ = svr.ProcessCanalInfo(context.Background(), v, "fengshanshan")
})
}
func TestJointConfigInfo(t *testing.T) {
Convey("TestJointConfigInfo", t, func() {
v := &canal.ConfigReq{
Addr: "172.16.33.999:3308",
User: "admin",
Password: "admin",
MonitorPeriod: "2h",
Project: "main.web-svr",
Leader: "fss",
Databases: _jsonstring,
Mark: "sfs",
}
comment, _ := svr.jointConfigInfo(context.Background(), v, cookie)
t.Logf("comment:%s", comment)
})
}
func TestCreateConfig(t *testing.T) {
Convey("test createConfig", t, func() {
params := url.Values{}
params.Set("comment", "23232ew")
params.Set("name", "demo1214s2.toml")
params.Set("state", "2")
params.Set("mark", "demo")
params.Set("from", "0")
params.Set("user", "sada")
res, err := svr.createConfig(context.Background(), params, cookie)
if err != nil {
So(err, ShouldContain, "643")
}
So(res, ShouldBeNil)
t.Log("createConfig=", res)
})
}
func TestUpdateConfig(t *testing.T) {
Convey("test updateConfig", t, func() {
params := url.Values{}
params.Set("config_id", "364")
params.Set("mtime", "1528975519")
params.Set("state", "2")
params.Set("mark", "update")
params.Set("comment", "dsdfasdsads")
res, err := svr.updateConfig(context.Background(), params, cookie)
if err != nil {
So(err, ShouldContain, "643")
}
So(res, ShouldBeNil)
t.Log("updateConfig=", res)
})
}
func TestGetGroupInfo(t *testing.T) {
Convey("test getGroupInfo", t, func() {
res, _, err := svr.getGroupInfo("test-group")
t.Log("GroupInfo=", res)
So(err, ShouldBeNil)
})
}
func TestGetAppInfo(t *testing.T) {
Convey("test getAppInfo", t, func() {
res, err := svr.getAppInfo("test-group")
t.Log("AppInfo=", res)
So(err, ShouldBeNil)
})
}
func TestGetAction(t *testing.T) {
Convey("test getAction", t, func() {
res := svr.getAction("LiveTimeS-LiveLivePÍ")
t.Log("Action=", res)
})
}
func TestGetTableInfo(t *testing.T) {
Convey("test getTableInfo", t, func() {
table := "172.16.33.12,172.16.33.48"
res, err := svr.TableInfo(table)
t.Log("table=", res)
So(err, ShouldBeNil)
})
}
func TestGetDatabusInfo(t *testing.T) {
Convey("test getDatabusInfo", t, func() {
tab1 := &canal.Table{
Name: "abc",
Primarykey: []string{"new", "old"},
Omitfield: []string{"new", "old"},
}
tables := []*canal.Table{tab1}
res, err := svr.databusInfo("LiveTime-LiveLive-P", "172.16.33.22", "relation", tables)
t.Logf("res:%+v", res)
So(err, ShouldBeNil)
})
}
func TestUpdateProcessTag(t *testing.T) {
Convey("test UpdateProcessTag", t, func() {
err := svr.UpdateProcessTag(context.Background(), 355, cookie)
So(err, ShouldBeNil)
})
}
func TestGetServerID(t *testing.T) {
Convey("test TestGetServerID", t, func() {
sid, err := svr.getServerID("172.16.448.789:9000")
t.Logf("sid:%v\n", sid)
So(sid, ShouldNotBeNil)
So(err, ShouldBeNil)
})
}
func TestServiceSendWechatMessage(t *testing.T) {
Convey("SendWechatMessage", t, func() {
var (
c = context.Background()
addr = "127.0.0.1:8000"
aType = canal.TypeMap[canal.TypeReview]
result = canal.TypeMap[canal.ReviewSuccess]
sender = "fengshanshan"
receiver = []string{"fengshanshan"}
note = "test"
)
err := svr.SendWechatMessage(c, addr, aType, result, sender, note, receiver)
So(err, ShouldBeNil)
})
}

42
app/admin/main/apm/service/dapper.go Normal file
View File

@@ -0,0 +1,42 @@
package service
import (
"net/http"
"net/http/httputil"
"net/url"
"strings"
"github.com/pkg/errors"
)
const (
_apmDapperPrefix = "/x/admin/apm/dapper"
_dapperPrxfix = "/x/internal/dapper"
)
type dapperProxy struct {
*httputil.ReverseProxy
}
func newDapperProxy(host string) (*dapperProxy, error) {
if !strings.HasPrefix(host, "http") {
host = "http://" + host
}
target, err := url.Parse(host)
if err != nil {
return nil, errors.Wrapf(err, "invalid dapper host %s", host)
}
proxy := httputil.NewSingleHostReverseProxy(target)
proxy.Director = func(r *http.Request) {
r.Host = target.Host
r.URL.Host = target.Host
r.URL.Scheme = target.Scheme
r.URL.Path = strings.Replace(r.URL.Path, _apmDapperPrefix, _dapperPrxfix, 1)
}
return &dapperProxy{ReverseProxy: proxy}, nil
}
// DapperProxy dapper proxy.
func (s *Service) DapperProxy(rw http.ResponseWriter, r *http.Request) {
s.dapperProxy.ServeHTTP(rw, r)
}

23
app/admin/main/apm/service/dapper_test.go Normal file
View File

@@ -0,0 +1,23 @@
package service
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestDapperProxy(t *testing.T) {
dp, err := newDapperProxy("http://172.16.38.143:6193")
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, "http://example.com/x/admin/apm/dapper/familys", nil)
if err != nil {
t.Fatal(err)
}
resp := httptest.NewRecorder()
dp.ServeHTTP(resp, req)
if resp.Code != http.StatusOK {
t.Errorf("proxy dapper error: %+v", resp)
}
}

414
app/admin/main/apm/service/databus.go Normal file
View File

@@ -0,0 +1,414 @@
package service
import (
"bytes"
"context"
"fmt"
"time"
"go-common/app/admin/main/apm/conf"
"go-common/app/admin/main/apm/model/databus"
"go-common/library/ecode"
"go-common/library/log"
"github.com/Shopify/sarama"
scluster "github.com/bsm/sarama-cluster"
)
var (
commitInterval = 20 * time.Millisecond
)
// Client kafka client
type Client struct {
sarama.Client
config *sarama.Config
addrs []string
topic string
group string
}
// NewClient returns a new kafka client instance
func NewClient(addrs []string, topic, group string) (c *Client, err error) {
config := sarama.NewConfig()
config.Version = sarama.V1_0_0_0
config.Consumer.Offsets.CommitInterval = commitInterval
config.Consumer.Offsets.Initial = sarama.OffsetOldest
config.Consumer.Return.Errors = true
c = &Client{
config: config,
addrs: addrs,
topic: topic,
group: group,
}
c.Client, err = sarama.NewClient(addrs, config)
return
}
// Close close kafka client
func (c *Client) Close() {
c.Client.Close()
}
// Topics return the list of topics for the given kafka cluster
func (c *Client) Topics() []string {
topics, err := c.Client.Topics()
if err != nil {
panic(err)
}
return topics
}
// Partitions returns the list of parititons for the given topic, as registered with SetMetadata
func (c *Client) Partitions() []int32 {
ps, err := c.Client.Partitions(c.topic)
if err != nil {
panic(err)
}
return ps
}
// OffsetNew return newest offset for given topic
func (c *Client) OffsetNew() (info map[int32]int64, err error) {
var (
offset int64
)
ps, err := c.Client.Partitions(c.topic)
if err != nil {
return
}
info = make(map[int32]int64)
for _, p := range ps {
offset, err = c.Client.GetOffset(c.topic, p, sarama.OffsetNewest)
if err != nil {
return
}
info[p] = offset
}
return
}
// OffsetOld return newest oldset for given topic
func (c *Client) OffsetOld() (info map[int32]int64, err error) {
var (
offset int64
)
ps, err := c.Client.Partitions(c.topic)
if err != nil {
return
}
info = make(map[int32]int64)
for _, p := range ps {
offset, err = c.Client.GetOffset(c.topic, p, sarama.OffsetOldest)
if err != nil {
return
}
info[p] = offset
}
return
}
// SetOffset commit offset of partition to kafka
func (c *Client) SetOffset(partition int32, offset int64) (err error) {
om, err := sarama.NewOffsetManagerFromClient(c.group, c.Client)
if err != nil {
return
}
defer om.Close()
pm, err := om.ManagePartition(c.topic, partition)
if err != nil {
return
}
pm.MarkOffset(offset, "")
if err = pm.Close(); err != nil {
return
}
time.Sleep(10 * commitInterval)
// verify
marked, err := c.OffsetMarked()
log.Info("partititon:%d, before:%d, after:%d\n", partition, offset, marked[partition])
return
}
// ResetOffset commit offset of partition to kafka
func (c *Client) ResetOffset(partition int32, offset int64) (err error) {
om, err := sarama.NewOffsetManagerFromClient(c.group, c.Client)
if err != nil {
return
}
defer om.Close()
pm, err := om.ManagePartition(c.topic, partition)
if err != nil {
return
}
pm.ResetOffset(offset, "")
if err = pm.Close(); err != nil {
return
}
time.Sleep(10 * commitInterval)
// verify
marked, err := c.OffsetMarked()
log.Info("partititon:%d, before:%d, after:%d\n", partition, offset, marked[partition])
return
}
// SeekBegin commit newest offset to kafka
func (c *Client) SeekBegin() (err error) {
offset, err := c.OffsetOld()
if err != nil {
return
}
for partition, offset := range offset {
err = c.ResetOffset(partition, offset)
if err != nil {
log.Info("partititon:%d, offset:%d, topic:%s, group:%s\n", partition, offset, c.topic, c.group)
return
}
}
return
}
// SeekEnd commit newest offset to kafka
func (c *Client) SeekEnd() (err error) {
offset, err := c.OffsetNew()
if err != nil {
return
}
for partition, offset := range offset {
err = c.SetOffset(partition, offset)
if err != nil {
log.Info("partititon:%d, offset:%d, topic:%s, group:%s\n", partition, offset, c.topic, c.group)
return
}
}
return
}
// OffsetMarked fetch marked offset
func (c *Client) OffsetMarked() (marked map[int32]int64, err error) {
req := &sarama.OffsetFetchRequest{
Version: 1,
ConsumerGroup: c.group,
}
partitions := c.Partitions()
marked = make(map[int32]int64)
broker, err := c.Client.Coordinator(c.group)
if err != nil {
return
}
defer broker.Close()
for _, partition := range partitions {
req.AddPartition(c.topic, partition)
}
resp, err := broker.FetchOffset(req)
if err != nil {
return
}
for _, p := range partitions {
block := resp.GetBlock(c.topic, p)
if block == nil {
err = sarama.ErrIncompleteResponse
return
}
if block.Err == sarama.ErrNoError {
marked[p] = block.Offset
} else {
err = block.Err
log.Info("block error(%v)", err)
return
}
}
return
}
// Diff offset
func Diff(cluster, topic, group string) (records []*databus.Record, err error) {
client, err := NewClient(conf.Conf.Kafka[cluster].Brokers, topic, group)
if err != nil {
log.Error("service.NewClient() error(%v)\n", err)
return
}
defer client.Close()
marked, err := client.OffsetMarked()
if err != nil {
log.Error("client.OffsetMarked() error(%v)\n", err)
return
}
new, err := client.OffsetNew()
if err != nil {
log.Error("client.OffsetNew() error(%v)\n", err)
return
}
records = make([]*databus.Record, 0, len(new))
for partition, offset := range new {
r := &databus.Record{
Partition: partition,
New: offset,
}
if tmp, ok := marked[partition]; ok {
r.Diff = offset - tmp
}
records = append(records, r)
}
return
}
// CreateTopic to kafka
func CreateTopic(addrs []string, topic string, partitions int32, factor int16) (err error) {
config := sarama.NewConfig()
config.Version = sarama.V1_0_0_0
config.Consumer.Return.Errors = true
cli, err := sarama.NewClient(addrs, config)
if err != nil {
return
}
defer cli.Close()
var topics []string
topics, err = cli.Topics()
if err != nil {
log.Error("CreateTopic get all topics on cluster err(%v)", err)
return
}
for _, t := range topics {
if t == topic {
return
}
}
broker, err := cli.Controller()
if err != nil {
return
}
// retention := "-1"
req := &sarama.CreateTopicsRequest{
TopicDetails: map[string]*sarama.TopicDetail{
topic: {
NumPartitions: partitions,
ReplicationFactor: factor,
// ConfigEntries: map[string]*string{
// "retention.ms": &retention,
// },
},
},
Timeout: time.Second,
}
var res *sarama.CreateTopicsResponse
if res, err = broker.CreateTopics(req); err != nil {
log.Info("CreateTopic CreateTopics error(%v) res(%v)", err, res)
return
}
if !(res.TopicErrors[topic].Err == sarama.ErrNoError || res.TopicErrors[topic].Err == sarama.ErrTopicAlreadyExists) {
log.Error("CreateTopic CreateTopics kfafka topic create error(%v) topic(%v)", res.TopicErrors[topic].Err, topic)
err = res.TopicErrors[topic].Err
}
log.Info("CreateTopic CreateTopics kfafka topic create info(%v) topic(%v)", res.TopicErrors[topic].Err, topic)
return
}
// FetchMessage fetch topic message by timestamp or offset.
func FetchMessage(c context.Context, cluster, topic, group, key string, start, end int64, limit int) (res []*databus.Message, err error) {
res = make([]*databus.Message, 0)
kc, ok := conf.Conf.Kafka[cluster]
if !ok {
err = ecode.NothingFound
return
}
cfg := scluster.NewConfig()
cfg.Version = sarama.V1_0_0_0
cfg.ClientID = fmt.Sprintf("%s-%s", group, topic)
cfg.Consumer.Offsets.Initial = sarama.OffsetOldest
cfg.Consumer.Return.Errors = true
consumer, err := scluster.NewConsumer(kc.Brokers, group, []string{topic}, cfg)
if err != nil {
log.Error("fetchMsg NewConsumer(%v,%s,%s) error(%v)", kc.Brokers, group, topic, err)
return
}
defer consumer.Close()
bkey := []byte(key)
for {
select {
case msg := <-consumer.Messages():
consumer.MarkPartitionOffset(topic, int32(msg.Partition), msg.Offset, "")
if key != "" && bytes.Equal(bkey, msg.Key) {
continue
}
if start > 0 && msg.Timestamp.UnixNano()/int64(time.Millisecond) < start {
continue
}
r := &databus.Message{}
r.Key = string(msg.Key)
r.Value = string(msg.Value)
r.Topic = topic
r.Partition = msg.Partition
r.Offset = msg.Offset
r.Timestamp = msg.Timestamp.Unix()
res = append(res, r)
if len(res) >= limit {
return
}
if end > 0 && msg.Timestamp.UnixNano()/int64(time.Millisecond) > end {
return
}
case err = <-consumer.Errors():
log.Error("fetchMsg error(%v)", err)
return
case <-time.After(time.Second * 10):
err = ecode.Deadline
return
case <-c.Done():
return
}
}
}
// NewOffset commit new offset to kafka
func (c *Client) NewOffset(partition int32, offset int64) (err error) {
err = c.ResetOffset(partition, offset)
if err != nil {
log.Info("partititon:%d, offset:%d, topic:%s, group:%s\n", partition, offset, c.topic, c.group)
return
}
return
}
// OffsetNewTime return newest offset for given topic
func (c *Client) OffsetNewTime(time int64) (info map[int32]int64, err error) {
var (
offset int64
message string
)
ps, err := c.Client.Partitions(c.topic)
if err != nil {
return
}
info = make(map[int32]int64)
for _, p := range ps {
offset, err = c.Client.GetOffset(c.topic, p, time)
message += fmt.Sprintf("partititon:%v, offset:%v, topic:%v, group:%v, time:%v, err:%v\n", p, offset, c.topic, c.group, time, err)
if err != nil {
log.Info(message)
return
}
info[p] = offset
}
log.Info(message)
return
}
// NewTime commit new time offset to kafka
func (c *Client) NewTime(time int64) (err error) {
offset, err := c.OffsetNewTime(time)
if err != nil {
return
}
for partition, offset := range offset {
err = c.ResetOffset(partition, offset)
if err != nil {
log.Info("partititon:%d, offset:%d, topic:%s, group:%s\n", partition, offset, c.topic, c.group)
return
}
}
return
}

111
app/admin/main/apm/service/databus_test.go Normal file
View File

@@ -0,0 +1,111 @@
package service
import (
"context"
"flag"
"os"
"path/filepath"
"testing"
"go-common/app/admin/main/apm/conf"
. "github.com/smartystreets/goconvey/convey"
)
var (
svr *Service
)
func TestMain(m *testing.M) {
var (
err error
)
dir, _ := filepath.Abs("../cmd/apm-admin-test.toml")
if err = flag.Set("conf", dir); err != nil {
panic(err)
}
if err = conf.Init(); err != nil {
panic(err)
}
svr = New(conf.Conf)
os.Exit(m.Run())
}
func TestFake(t *testing.T) {
Convey("fake", t, func() {
t.Log("fake test")
})
}
func TestService_NewClient(t *testing.T) {
Convey("should new client all", t, func() {
c, err := NewClient(conf.Conf.Kafka["test_kafka_9092-266"].Brokers, "Archive-T", "Archive-Live-S")
t.Log(err, c)
So(err, ShouldBeNil)
})
}
func TestService_OffsetNew(t *testing.T) {
Convey("should offset new all", t, func() {
c, err := NewClient(conf.Conf.Kafka["test_kafka_9092-266"].Brokers, "Archive-T", "Archive-Live-S")
So(err, ShouldBeNil)
info, err := c.OffsetNew()
t.Log(err, info)
So(err, ShouldBeNil)
})
}
func TestService_OffsetOld(t *testing.T) {
Convey("should offset old all", t, func() {
c, err := NewClient(conf.Conf.Kafka["test_kafka_9092-266"].Brokers, "Archive-T", "Archive-Live-S")
So(err, ShouldBeNil)
info, err := c.OffsetOld()
t.Log(err, info)
So(err, ShouldBeNil)
})
}
func TestService_SeekBegin(t *testing.T) {
Convey("should seek begin all", t, func() {
c, err := NewClient(conf.Conf.Kafka["test_kafka_9092-266"].Brokers, "Archive-T", "Archive-Live-S")
c.SeekBegin()
t.Log(err)
So(err, ShouldBeNil)
})
}
func TestService_SeekEnd(t *testing.T) {
Convey("should seek end all", t, func() {
c, err := NewClient(conf.Conf.Kafka["test_kafka_9092-266"].Brokers, "Archive-T", "Archive-Live-S")
So(err, ShouldBeNil)
err = c.SeekEnd()
t.Log(err)
So(err, ShouldBeNil)
})
}
func TestCreateTopic(t *testing.T) {
Convey("test create topic", t, func() {
err := CreateTopic([]string{"172.18.33.51:9098", "172.18.33.52:9098", "172.18.33.50:9098"}, "testcreate11", 1, 1)
So(err, ShouldBeNil)
})
}
func TestService_OffsetMarked(t *testing.T) {
Convey("should offset marked all", t, func() {
c, err := NewClient(conf.Conf.Kafka["test_kafka_9092-266"].Brokers, "Archive-T", "Archive-Live-S")
So(err, ShouldBeNil)
_, err = c.OffsetMarked()
t.Log(err)
So(err, ShouldBeNil)
})
}
func TestService_MsgFetch(t *testing.T) {
Convey("should msg fetch", t, func() {
res, err := FetchMessage(context.Background(), "test_kafka_9092-266", "Archive-T", "ArchiveAPM-MainCommonArch-S", "", 0, 0, 10)
So(err, ShouldBeNil)
for _, r := range res {
t.Logf("fetch key:%s value:%s partition:%d offset:%d timestamp:%d", r.Key, r.Value, r.Partition, r.Offset, r.Timestamp)
}
})
}

159
app/admin/main/apm/service/discovery.go Normal file
View File

@@ -0,0 +1,159 @@
package service
import (
"context"
"encoding/json"
"fmt"
"net/url"
"sort"
"sync"
"go-common/app/admin/main/apm/conf"
"go-common/app/admin/main/apm/model/discovery"
"go-common/library/conf/env"
"go-common/library/ecode"
"go-common/library/log"
"go-common/library/naming"
errgroup "go-common/library/sync/errgroup.v2"
)
// DiscoveryProxy discovery proxy.
func (s *Service) DiscoveryProxy(c context.Context, method, loc string, params url.Values) (data interface{}, err error) {
var bs json.RawMessage
uri := conf.Conf.Host.APICo + "/discovery/" + loc
if loc == "nodes" {
api := conf.Conf.Discovery.API
v1 := make(map[string]interface{})
for _, v := range api {
sData := make([]*discovery.Addr, 0, 3)
uri = "http://" + v + "/discovery/" + loc
bs, err = s.curl(c, method, uri, params)
if err != nil {
log.Error("apmSvc.DiscoveryProxy curl url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return nil, err
}
if err = json.Unmarshal(bs, &sData); err != nil {
return nil, err
}
Others := make([]*discovery.Addr, 0, len(sData))
tmpData := &discovery.Status{}
tmpData.Status = 0
for _, vv := range sData {
if vv.Addr == v {
tmpData.Status = vv.Status
continue
}
Others = append(Others, vv)
}
tmpData.Others = Others
v1[v] = tmpData
}
data = v1
} else if loc == "polling" {
api := conf.Conf.Discovery.API
v1 := make([]string, 0)
tmp := make(map[string]struct{})
for _, v := range api {
sData := make([]string, 0)
uri = "http://" + v + "/discovery/" + loc
bs, err = s.curl(c, method, uri, params)
if err != nil {
log.Error("apmSvc.DiscoveryProxy curl url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return nil, err
}
if err = json.Unmarshal(bs, &sData); err != nil {
return nil, err
}
for _, vv := range sData {
if _, ok := tmp[vv]; !ok {
v1 = append(v1, vv)
tmp[vv] = struct{}{}
}
}
}
sort.Strings(v1)
data = v1
} else {
data, err = s.curl(c, method, uri, params)
}
return
}
func (s *Service) curl(c context.Context, method, uri string, params url.Values) (data json.RawMessage, err error) {
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
Message string `json:"message"`
}
if method == "GET" {
if err = s.client.Get(c, uri, "", params, &res); err != nil {
log.Error("apmSvc.DiscoveryProxy get url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return
}
} else {
if err = s.client.Post(c, uri, "", params, &res); err != nil {
log.Error("apmSvc.DiscoveryProxy post url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return
}
}
if res.Code != 0 {
err = ecode.Int(res.Code)
log.Error("apmSvc.DiscoveryProxy url:"+uri+" params:(%v) ecode(%v)", params.Encode(), err)
return
}
data = res.Data
return
}
// DatabusConsumerAddrs databus consumer addrs.
func (s *Service) DatabusConsumerAddrs(c context.Context, group string) (addrs []string, err error) {
uri := conf.Conf.Host.APICo + "/discovery/fetch"
params := url.Values{}
params.Set("env", env.DeployEnv)
params.Set("appid", "middleware.databus")
params.Set("status", "1")
data, err := s.curl(c, "GET", uri, params)
if err != nil {
log.Error("curl discovery fetch error:%v", err)
return
}
var res struct {
ZoneInstances map[string][]*naming.Instance `json:"zone_instances"`
}
if err = json.Unmarshal(data, &res); err != nil {
log.Error("json unmarshal discovery data error:%v", err)
return
}
var (
wg errgroup.Group
lock sync.Mutex
)
for _, instances := range res.ZoneInstances {
for _, instance := range instances {
for _, addr := range instance.Addrs {
u, _ := url.Parse(addr)
if u.Scheme == "http" {
wg.Go(func(c context.Context) error {
p := url.Values{}
p.Set("group", group)
var as struct {
Code int `json:"code"`
Data []string `json:"data"`
}
if e := s.client.Get(context.Background(), fmt.Sprintf("http://%s/databus/consumer/addrs", u.Host), "", p, &as); e != nil {
log.Error("curl databus consumer addrs error:%v", e)
return nil
}
lock.Lock()
addrs = append(addrs, as.Data...)
lock.Unlock()
return nil
})
}
}
}
}
wg.Wait()
return
}

16
app/admin/main/apm/service/ecode.go Normal file
View File

@@ -0,0 +1,16 @@
package service
import (
"context"
"go-common/app/admin/main/apm/model/ecode"
"go-common/library/log"
)
// GetCodes ...
func (s *Service) GetCodes(c context.Context, Interval1, Interval2 string) (data []*codes.Codes, err error) {
data, err = s.dao.GetCodes(c, Interval1, Interval2)
if err != nil {
log.Error("service GetCodes error(%v)", err)
}
return
}

47
app/admin/main/apm/service/log.go Normal file
View File

@@ -0,0 +1,47 @@
package service
import (
"time"
mlog "go-common/app/admin/main/apm/model/log"
"go-common/library/log"
context "go-common/library/net/http/blademaster"
"go-common/library/queue/databus/report"
)
// SQLLog log
type SQLLog struct {
SQLType string
Content interface{}
}
// LogAdd add log
func (s *Service) LogAdd(c context.Context, lg *mlog.Log) (err error) {
l := &mlog.Log{
UserName: lg.UserName,
Business: lg.Business,
Info: lg.Info,
}
if err = s.dao.DB.Create(&l).Error; err != nil {
log.Error("s.LogAdd create error(%v)", err)
}
return
}
// SendLog log
func (s *Service) SendLog(c context.Context, username string, uid int64, tp int, oid int64, action string, context interface{}) (err error) {
report.Manager(&report.ManagerInfo{
Uname: username,
UID: uid,
Business: 71,
Type: tp, // 1 add 2 update 3 delete 4 soft delete 5 Transaction 6 kafka
Oid: oid,
Action: action,
Ctime: time.Now(),
// Index: []interface{}{0, 0},
Content: map[string]interface{}{
"content": context,
},
})
return
}

58
app/admin/main/apm/service/members.go Normal file
View File

@@ -0,0 +1,58 @@
package service
import (
"context"
"net/http"
"go-common/app/admin/main/apm/model/monitor"
"go-common/library/log"
)
const chatURL = "https://api.bilibili.com/x/web-interface/online"
// ChatResult .
type ChatResult struct {
Code int `json:"code"`
Data struct {
WebOnline int64 `json:"web_online"`
} `json:"data"`
}
// ChatProxy chat proxy
func (s *Service) ChatProxy(c context.Context) (ret *ChatResult, err error) {
var (
req *http.Request
)
if req, err = s.client.NewRequest(http.MethodGet, chatURL, "", nil); err != nil {
log.Error("s.ChatProxy.client.NewRequest err(%v)", err)
return
}
if err = s.client.Do(c, req, &ret); err != nil {
log.Error("s.ChatProxy.client.DO err(%v)", err)
return
}
if ret.Code != 0 {
log.Error("s.ChatProxy.client http_status(%d)", ret.Code)
return
}
return
}
// Members get current online members
func (s *Service) Members(c context.Context) (mt *monitor.Monitor, err error) {
var (
ret = &ChatResult{}
)
mt = &monitor.Monitor{}
if ret, err = s.ChatProxy(c); err != nil {
return nil, err
}
mt.AppID = "online"
mt.Interface = "online-count"
mt = &monitor.Monitor{
AppID: "online",
Interface: "online-count",
Count: ret.Data.WebOnline,
}
return
}

24
app/admin/main/apm/service/members_test.go Normal file
View File

@@ -0,0 +1,24 @@
package service
import (
"context"
"testing"
"go-common/app/admin/main/apm/model/monitor"
"github.com/smartystreets/goconvey/convey"
)
func TestService_Members(t *testing.T) {
var (
c = context.Background()
mt *monitor.Monitor
err error
)
convey.Convey("Members", t, func(ctx convey.C) {
mt, err = svr.Members(c)
ctx.So(mt, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
t.Logf("AppID=%s, Interface=%s, Count=%d", mt.AppID, mt.Interface, mt.Count)
})
}

235
app/admin/main/apm/service/monitor.go Normal file
View File

@@ -0,0 +1,235 @@
package service
import (
"context"
"fmt"
"strings"
"time"
"go-common/app/admin/main/apm/model/monitor"
"go-common/library/log"
xtime "go-common/library/time"
)
// AddMonitor get monitor data and insert
func (s *Service) AddMonitor(c context.Context) (err error) {
var (
rpc = make([]*monitor.Monitor, 0)
http = make([]*monitor.Monitor, 0)
tcs = make([]*monitor.Monitor, 0)
dabs = make([]*monitor.Monitor, 0)
key = make([]string, 0)
value = make([]interface{}, 0)
mt = &monitor.Monitor{}
insertSQL = "INSERT INTO `monitor`(`app_id`, `interface`, `count`, `cost`) VALUES %s"
)
if rpc, err = s.RPCMonitor(c); err != nil {
return
}
if http, err = s.HTTPMonitor(c); err != nil {
return
}
if mt, err = s.Members(c); err != nil {
return
}
if tcs, err = s.TenCent(c); err != nil {
return
}
if dabs, err = s.DataBus(c); err != nil {
return
}
rpc = append(rpc, http...)
rpc = append(rpc, dabs...)
rpc = append(rpc, tcs...)
rpc = append(rpc, mt)
if len(rpc) == 0 {
err = fmt.Errorf("monitor data empty")
log.Error("s.AddMonitor error(%v)", err)
return
}
for _, mt := range rpc {
key = append(key, "(?, ?, ?, ?)")
value = append(value, mt.AppID, mt.Interface, mt.Count, mt.Cost)
}
return s.DB.Model(&monitor.Monitor{}).Exec(fmt.Sprintf(insertSQL, strings.Join(key, ",")), value...).Error
}
// AppNameList return app list
func (s *Service) AppNameList(c context.Context) (apps []string) {
return s.c.Apps.Name
}
// PrometheusList return PrometheusList data
func (s *Service) PrometheusList(c context.Context, app, method, mType string) (ret *monitor.MoniRet, err error) {
var (
mt = &monitor.Monitor{}
mts = make([]*monitor.Monitor, 0)
)
if err = s.DB.Select("interface,count,cost,mtime").Where("app_id = ?", app+"-"+method).Group("mtime,interface").Order("interface,mtime").Find(&mts).Error; err != nil {
log.Error("s.PrometheusList query all error(%v)", err)
return
}
if len(mts) < 1 {
return
}
if err = s.DB.Where("app_id = ?", app+"-"+method).First(mt).Error; err != nil {
log.Error("s.Prometheus query first error(%v)", err)
return
}
return merge(s.packing(mts), s.times(mt.MTime), mType), err
}
// BroadCastList return BroadCastList data
func (s *Service) BroadCastList(c context.Context) (ret *monitor.MoniRet, err error) {
var (
mt = &monitor.Monitor{}
mts = make([]*monitor.Monitor, 0)
)
if err = s.DB.Select("substring_index(interface, '_', -1) as temp_name,sum(count) as count,mtime").Where("interface REGEXP 'ESTAB|InBound|OutBound|InPacket|OutPacket$'").Group("mtime,temp_name").Order("temp_name,mtime").Find(&mts).Error; err != nil {
log.Error("s.BroadCastList query error(%v)", err)
return
}
if err = s.DB.Where("interface REGEXP 'ESTAB|InBound|OutBound|InPacket|OutPacket$'").First(mt).Error; err != nil {
log.Error("s.BroadCastList query first error(%v)", err)
return
}
return merge(s.packing(mts), s.times(mt.MTime), "count"), err
}
// DataBusList return DataBusList data
func (s *Service) DataBusList(c context.Context) (ret *monitor.MoniRet, err error) {
var (
mts = make([]*monitor.Monitor, 0)
mt = &monitor.Monitor{}
)
if err = s.DB.Select("interface,count,mtime").Where("app_id=?", "kafka-databus").Group("mtime,interface").Order("interface,mtime").Find(&mts).Error; err != nil {
log.Error("s.MonitorList query error(%v)", err)
return
}
if len(mts) < 1 {
return
}
if err = s.DB.Where("app_id=?", "kafka-databus").First(mt).Error; err != nil {
log.Error("s.DataBusList query first error(%v)", err)
return
}
return merge(s.packing(mts), s.times(mt.MTime), "count"), err
}
// OnlineList return online data
func (s *Service) OnlineList(c context.Context) (ret *monitor.MoniRet, err error) {
var (
mts = make([]*monitor.Monitor, 0)
mt = &monitor.Monitor{}
)
if err = s.DB.Select("interface,count,mtime").Where("app_id=?", "online").Find(&mts).Error; err != nil {
log.Error("s.OnlineList query error(%v)", err)
}
if len(mts) < 1 {
return
}
if err = s.DB.Where("app_id=?", "online").First(mt).Error; err != nil {
log.Error("s.OnlineList query error(%v)", err)
}
return merge(s.packing(mts), s.times(mt.MTime), "count"), err
}
// merge .
func merge(dts []*monitor.Data, strs []string, mType string) (ret *monitor.MoniRet) {
items := make([]*monitor.Items, 0)
for _, dt := range dts {
var (
yAxis []int64
item = &monitor.Items{}
)
if mType == "count" {
yAxis = formatArray(strs, dt.Times, dt.Counts)
} else {
yAxis = formatArray(strs, dt.Times, dt.Costs)
}
item.Interface = dt.Interface
item.YAxis = yAxis
items = append(items, item)
}
if len(items) > 0 {
ret = &monitor.MoniRet{
XAxis: strs,
Items: items,
}
}
return
}
// formatArray formatArray missing data by time
func formatArray(strs, times []string, counts []int64) []int64 {
var newCounts []int64
for _, str := range strs {
if len(counts) < 1 {
break
}
if ok, index := inArray(times, str); ok {
newCounts = append(newCounts, counts[index])
} else {
newCounts = append(newCounts, 0)
}
}
return newCounts
}
// inArray check key in array or not and return position
func inArray(arrays []string, key string) (bool, int) {
for index, arr := range arrays {
if key == arr {
return true, index
}
}
return false, 0
}
// times return a standard string time array
func (s *Service) times(t xtime.Time) []string {
var (
nextDay string
tList = []string{t.Time().Format("2006-01-02")}
curDay = time.Now().Format("2006-01-02")
nextTime = t.Time().Add(24 * time.Hour)
)
for {
nextDay = nextTime.Format("2006-01-02")
tList = append(tList, nextDay)
nextTime = nextTime.Add(24 * time.Hour)
if nextDay == curDay {
break
}
}
return tList
}
// packing .
func (s *Service) packing(mts []*monitor.Monitor) (data []*monitor.Data) {
d := &monitor.Data{}
for k, mt := range mts {
if mt.Interface == "" {
mt.Interface = mt.TempName
}
if d.Interface != mt.Interface {
if d.Interface != "" {
data = append(data, d)
}
d = &monitor.Data{
Interface: mt.Interface,
Counts: []int64{mt.Count},
Costs: []int64{mt.Cost},
Times: []string{mt.MTime.Time().Format("2006-01-02")},
}
continue
}
d.Counts = append(d.Counts, mt.Count)
d.Costs = append(d.Costs, mt.Cost)
d.Times = append(d.Times, mt.MTime.Time().Format("2006-01-02"))
if k == len(mts)-1 {
data = append(data, d)
}
}
return
}

56
app/admin/main/apm/service/monitor_test.go Normal file
View File

@@ -0,0 +1,56 @@
package service
import (
"context"
"testing"
"go-common/app/admin/main/apm/model/monitor"
"go-common/library/log"
"github.com/smartystreets/goconvey/convey"
)
func TestService_AddMonitor(t *testing.T) {
var (
c = context.Background()
err error
)
convey.Convey("Test AddMonitor", t, func(ctx convey.C) {
err = svr.AddMonitor(c)
ctx.So(err, convey.ShouldBeNil)
})
}
func TestService_PrometheusList(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("Test PrometheusList", t, func(ctx convey.C) {
ret, err := svr.PrometheusList(c, "main.account.member-service", "xx", "count")
ctx.So(ret, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
})
}
func TestService_BroadCastList(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("Test BroadCastList", t, func(ctx convey.C) {
ret, err := svr.BroadCastList(c)
ctx.So(ret, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
})
}
func TestService_Times(t *testing.T) {
mt := &monitor.Monitor{}
if err := svr.DB.Where("app_id = ?", "main.app-svr.app-feed-http").First(mt).Error; err != nil {
log.Error("s.Prometheus query first error(%v)", err)
return
}
convey.Convey("Test Times", t, func(ctx convey.C) {
ret := svr.times(mt.MTime)
t.Logf("times=%v", ret)
})
}

195
app/admin/main/apm/service/need.go Normal file
View File

@@ -0,0 +1,195 @@
package service
import (
"context"
"fmt"
"go-common/app/admin/main/apm/model/need"
"go-common/library/ecode"
"go-common/library/log"
)
//NeedInfoAdd is
func (s *Service) NeedInfoAdd(c context.Context, req *need.NAddReq, username string) (err error) {
if err = s.dao.NeedInfoAdd(req, username); err != nil {
return
}
return
}
//NeedInfoList is
func (s *Service) NeedInfoList(c context.Context, arg *need.NListReq, username string) (res []*need.NInfo, count int64, err error) {
var (
like *need.UserLikes
)
if count, err = s.dao.NeedInfoCount(arg); err != nil {
return
}
if count == 0 {
return
}
if res, err = s.dao.NeedInfoList(arg); err != nil {
return
}
for _, r := range res {
lq := &need.Likereq{
ReqID: r.ID,
}
if like, err = s.dao.GetVoteInfo(lq, username); err == nil {
r.LikeState = like.LikeType
}
}
err = nil
return
}
//NeedInfoEdit is
func (s *Service) NeedInfoEdit(c context.Context, arg *need.NEditReq, username string) (err error) {
var (
ni *need.NInfo
)
if ni, err = s.dao.GetNeedInfo(arg.ID); err != nil {
err = ecode.NeedInfoErr
return
}
if ni.Reporter != username {
err = ecode.AccessDenied
return
}
if ni.Status != 5 && ni.Status != 1 {
err = ecode.NeedEditErr
return
}
if err = s.dao.NeedInfoEdit(arg); err != nil {
return
}
return
}
//NeedInfoVerify is
func (s *Service) NeedInfoVerify(c context.Context, arg *need.NVerifyReq) (ni *need.NInfo, err error) {
if ni, err = s.dao.GetNeedInfo(arg.ID); err != nil {
err = ecode.NeedInfoErr
return
}
if ni.Status != 1 {
err = ecode.NeedVerifyErr
return
}
if err = s.dao.NeedVerify(arg); err != nil {
return
}
return
}
//NeedInfoVote is
func (s *Service) NeedInfoVote(c context.Context, arg *need.Likereq, username string) (err error) {
var (
nu *need.UserLikes
like, dislike int
)
if _, err = s.dao.GetNeedInfo(arg.ReqID); err != nil {
err = ecode.NeedInfoErr
return
}
if nu, err = s.dao.GetVoteInfo(arg, username); err != nil {
if err = s.dao.AddVoteInfo(arg, username); err != nil {
return
}
switch arg.LikeType {
case need.TypeLike:
like = 1
dislike = 0
case need.TypeDislike:
like = 0
dislike = 1
}
} else {
if err = s.dao.UpdateVoteInfo(arg, username); err != nil {
log.Error("arg.liketype:%d,db.like%+v,like:%d,dislike:%d", arg.LikeType, nu, like, dislike)
return
}
if nu.LikeType != arg.LikeType {
switch {
case arg.LikeType == need.TypeCancel && nu.LikeType == need.TypeLike:
like = -1
case arg.LikeType == need.TypeCancel && nu.LikeType == need.TypeDislike:
dislike = -1
case arg.LikeType == need.TypeLike && nu.LikeType == need.TypeCancel:
like = 1
case arg.LikeType == need.TypeDislike && nu.LikeType == need.TypeCancel:
dislike = 1
case arg.LikeType == need.TypeLike && nu.LikeType == need.TypeDislike:
like = 1
dislike = -1
case arg.LikeType == need.TypeDislike && nu.LikeType == need.TypeLike:
like = -1
dislike = 1
}
} else {
return
}
}
if err = s.dao.LikeCountsStats(arg, like, dislike); err != nil {
if nu != nil {
log.Error("arg.liketype:%d,db.like:%+v,like:%d,dislike:%d", arg.LikeType, nu, like, dislike)
}
return
}
return
}
//NeedVoteList is show votelist
func (s *Service) NeedVoteList(c context.Context, arg *need.Likereq) (res []*need.UserLikes, count int64, err error) {
if count, err = s.dao.VoteInfoCounts(arg); err != nil {
return
}
if count == 0 {
return
}
if res, err = s.dao.VoteInfoList(arg); err != nil {
return
}
return
}
//SendWeMessage send wechat message
func (s *Service) SendWeMessage(c context.Context, title, task, result, sender string, receiver []string) (err error) {
var (
msg = "[sven需求与建议来信 📨 ]\n"
users []string
)
switch task {
case need.VerifyType[need.NeedApply]:
msg += fmt.Sprintf("%s童鞋认真地提了一份建议(%s)，快去看看吧~\n%s\n", sender, title, "http://sven.bilibili.co/#/suggestion/list")
users = append(users, receiver...)
case need.VerifyType[need.NeedVerify]:
msg += fmt.Sprintf("%s童鞋的建议(%s)反馈我们已经收到啦，先发一朵小红花感谢支持！🌺 \n", sender, title)
users = append(users, sender)
case need.VerifyType[need.NeedReview]:
msg += fmt.Sprintf("%s童鞋的建议(%s)审核结果是%s, %s", receiver, title, result, "%s")
users = append(users, receiver...)
}
switch result {
case need.VerifyType[need.VerifyAccept]:
msg = fmt.Sprintf(msg, "恭喜恭喜，喝杯快乐水开心一下~ 🍻 ")
case need.VerifyType[need.VerifyReject]:
msg = fmt.Sprintf(msg, "不要灰心，可能是需求描述或使用姿势不够准确，还请多多支持，欢迎再来！🙇‍ 🙆‍ ")
case need.VerifyType[need.VerifyObserved]:
msg = fmt.Sprintf(msg, "您的意见我们先保留啦，还请多多支持，欢迎再来补充！🙇‍ 🙆‍ ")
}
if err = s.dao.SendWechatToUsers(c, users, msg); err != nil {
log.Error("apmSvc.SendWechatMessage error(%v)", err)
return
}
return
}

178
app/admin/main/apm/service/need_test.go Normal file
View File

@@ -0,0 +1,178 @@
package service
import (
"context"
"fmt"
"testing"
"go-common/app/admin/main/apm/dao/mock"
"go-common/app/admin/main/apm/model/need"
"github.com/bouk/monkey"
"github.com/smartystreets/goconvey/convey"
)
//TestServiceNeedList is
func TestServiceNeedList(t *testing.T) {
convey.Convey("TestServiceNeedList", t, func() {
arg := &need.NListReq{
Status: 1,
Ps: 10,
Pn: 1,
}
guard := mock.MockDaoNeedInfoCount(svr.dao, 0, nil)
defer guard.Unpatch()
res, _, err := svr.NeedInfoList(context.Background(), arg, "fengshanshan")
for _, v := range res {
t.Logf("res:%+v", v)
}
convey.So(res, convey.ShouldNotBeNil)
convey.So(err, convey.ShouldBeNil)
})
}
//TestServiceNeedEdit is
func TestServiceNeedEdit(t *testing.T) {
convey.Convey("TestServiceNeedEdit", t, func() {
arg := &need.NEditReq{
ID: 147,
Title: "22222",
}
err := svr.NeedInfoEdit(context.Background(), arg, "fengshanshan")
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("TestServiceNeedEdit status err", t, func() {
arg := &need.NEditReq{
ID: 99,
Title: "22222",
}
err := svr.NeedInfoEdit(context.Background(), arg, "fengshanshan")
convey.So(err, convey.ShouldEqual, 70018)
})
convey.Convey("TestServiceNeedEdit no access", t, func() {
arg := &need.NEditReq{
ID: 147,
Title: "22222",
}
err := svr.NeedInfoEdit(context.Background(), arg, "fss")
convey.So(err, convey.ShouldEqual, -403)
})
}
//TestServiceNeedVerify is
func TestServiceNeedVerify(t *testing.T) {
convey.Convey("TestServiceNeedVerify status err", t, func() {
arg := &need.NVerifyReq{
ID: 117,
Status: 2,
}
_, err := svr.NeedInfoVerify(context.Background(), arg)
convey.So(err, convey.ShouldEqual, 70019)
})
convey.Convey("TestServiceNeedVerify not exist", t, func() {
arg := &need.NVerifyReq{
ID: 10000,
Status: 1,
}
_, err := svr.NeedInfoVerify(context.Background(), arg)
convey.So(err, convey.ShouldEqual, 70017)
})
}
func TestServiceNeedVote(t *testing.T) {
convey.Convey("TestServiceNeedVote", t, func() {
arg := &need.Likereq{
ReqID: 148,
LikeType: 1,
}
err := svr.NeedInfoVote(context.Background(), arg, "fengshanshan")
convey.So(err, convey.ShouldBeNil)
})
convey.Convey("TestServiceNeedVote not exist", t, func() {
arg := &need.Likereq{
ReqID: 789,
LikeType: 1,
}
err := svr.NeedInfoVote(context.Background(), arg, "fengshanshan")
convey.So(err, convey.ShouldEqual, 70017)
})
}
func TestServiceNeedVoteList(t *testing.T) {
convey.Convey("NeedVoteList", t, func(ctx convey.C) {
var (
c = context.Background()
arg = &need.Likereq{
ReqID: 11,
LikeType: 1,
}
resp = []*need.UserLikes{
{ID: 1, User: "fengshanshan", LikeType: 1},
{ID: 2, User: "fengshanshan", LikeType: 2},
}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
svr.dao.MockVoteInfoCounts(6, nil)
svr.dao.MockVoteInfoList(resp, nil)
res, count, err := svr.NeedVoteList(c, arg)
ctx.Convey("Then err should be nil.res,count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
ctx.Reset(func() {
monkey.UnpatchAll()
})
})
})
}
func TestServiceNeedInfoAdd(t *testing.T) {
convey.Convey("NeedInfoAdd", t, func(ctx convey.C) {
var (
c = context.Background()
req = &need.NAddReq{
Title: "32323",
Content: "ewewerw",
}
username = "fengshanshan"
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
guard := svr.dao.MockNeedInfoAdd(nil)
defer guard.Unpatch()
err := svr.NeedInfoAdd(c, req, username)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
ctx.Convey("When return err", func(ctx convey.C) {
guard := svr.dao.MockNeedInfoAdd(fmt.Errorf("aaa"))
defer guard.Unpatch()
err := svr.NeedInfoAdd(c, req, username)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceSendWeMessage(t *testing.T) {
convey.Convey("SendWeMessage", t, func(convCtx convey.C) {
var (
c = context.Background()
title = "有个xxxxxxx"
task = need.VerifyType[need.NeedReview]
result = need.VerifyType[need.VerifyAccept]
sender = "fengshanshan"
receiver = []string{"fengshanshan"}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := svr.SendWeMessage(c, title, task, result, sender, receiver)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}

44
app/admin/main/apm/service/open.go Normal file
View File

@@ -0,0 +1,44 @@
package service
import (
"context"
"encoding/json"
"net/url"
"go-common/app/admin/main/apm/conf"
"go-common/library/ecode"
"go-common/library/log"
)
// OpenProxy proxy.
func (s *Service) OpenProxy(c context.Context, method, loc string, params url.Values) (data interface{}, err error) {
uri := conf.Conf.Host.MANAGERCo + "/" + loc
data, err = s.ocurl(c, method, uri, loc, params)
return
}
func (s *Service) ocurl(c context.Context, method, uri, loc string, params url.Values) (data json.RawMessage, err error) {
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
Message string `json:"message"`
}
if method == "GET" {
if err = s.client.Get(c, uri, "", params, &res); err != nil {
log.Error("apmSvc.OpenProxy get url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return
}
} else {
if err = s.client.Post(c, uri, "", params, &res); err != nil {
log.Error("apmSvc.OpenProxy post url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return
}
}
if res.Code != 0 {
err = ecode.Int(res.Code)
log.Error("apmSvc.OpenProxy url:"+uri+" params:(%v) ecode(%v)", params.Encode(), err)
return
}
data = res.Data
return
}

44
app/admin/main/apm/service/platform.go Normal file
View File

@@ -0,0 +1,44 @@
package service
import (
"context"
"encoding/json"
"net/url"
"go-common/app/admin/main/apm/conf"
"go-common/library/ecode"
"go-common/library/log"
)
// PlatformProxy proxy.
func (s *Service) PlatformProxy(c context.Context, method, loc string, params url.Values) (data interface{}, err error) {
uri := conf.Conf.Host.MANAGERCo + "/" + loc
data, err = s.pcurl(c, method, uri, loc, params)
return
}
func (s *Service) pcurl(c context.Context, method, uri, loc string, params url.Values) (data json.RawMessage, err error) {
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
Message string `json:"message"`
}
if method == "GET" {
if err = s.client.Get(c, uri, "", params, &res); err != nil {
log.Error("apmSvc.PlatformProxy get url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return
}
} else {
if err = s.client.Post(c, uri, "", params, &res); err != nil {
log.Error("apmSvc.PlatformProxy post url:"+uri+" params:(%v) error(%v)", params.Encode(), err)
return
}
}
if res.Code != 0 {
err = ecode.Int(res.Code)
log.Error("apmSvc.PlatformProxy url:"+uri+" params:(%v) ecode(%v)", params.Encode(), err)
return
}
data = res.Data
return
}

249
app/admin/main/apm/service/pprof.go Normal file
View File

@@ -0,0 +1,249 @@
package service
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
"go-common/app/admin/main/apm/conf"
"go-common/app/admin/main/apm/model/pprof"
"go-common/library/log"
xtime "go-common/library/time"
)
var (
port = "2333"
cpuURL = "%s/x/admin/apm/pprof/svg?name=%s&uri=%s&hostname=%s"
msg = "<a href=\"%s\">点击查看详情</a>(注：生成时间有延迟，如未显示，稍后重试)"
)
// kind .
const (
CPUPerformace = 1 // CPU性能图
CPUFrame = 2 // CPU火焰图
HeapPerformance = 3 // 内存性能图
HeapFrame = 4 // 内存火焰图
)
// Pprof ...
func (s *Service) Pprof(url, uri, svgName, hostName string, time int64, sType int8) (err error) {
var (
out bytes.Buffer
errOut bytes.Buffer
)
goPath := "go"
if len(conf.Conf.Pprof.GoPath) > 0 {
goPath = conf.Conf.Pprof.GoPath
}
f, err := exec.LookPath(goPath)
if err != nil {
log.Error("pprof go error(%v) goPath=(%v)", err, goPath)
fmt.Printf("pprof=(%v)", err)
return
}
cmd := exec.Command(f, "tool", "pprof", "--seconds="+strconv.FormatInt(time, 10), "--svg", "--output="+conf.Conf.Pprof.Dir+"/"+svgName+"_"+hostName+"_"+uri+".svg", url+"/debug/pprof/"+uri)
cmd.Stdout = &out
cmd.Stderr = &errOut
// 执行命令
if sType == 1 { //串行
if err = cmd.Run(); err != nil {
log.Error("pprof Run stdout=(%s) stderr=(%s) error(%v)", out.String(), errOut.String(), err)
}
} else { //阻塞
if err = cmd.Start(); err != nil {
log.Error("pprof Start stdout=(%s) stderr=(%s) error(%v)", out.String(), errOut.String(), err)
}
if err = cmd.Wait(); err != nil {
log.Error("s.Pprof cmd.Wait() error(%v)", err)
}
}
return
}
// Torch ...
func (s *Service) Torch(c context.Context, url, uri, svgName, hostName string, time int64, sType int8) (err error) {
goPath := "go-torch"
// if len(conf.Conf.Pprof.GoPath) > 0 {
// goPath = conf.Conf.Pprof.GoPath
// }
f, err := exec.LookPath(goPath)
if err != nil {
log.Error("go-torch error(%v) goPath=(%v)", err, goPath)
fmt.Printf("go-torch=(%v)", err)
return
}
cmd := exec.Command(f, "--url="+url, "--suffix=/debug/pprof/"+uri, "--seconds="+strconv.FormatInt(time, 10), "-f="+conf.Conf.Pprof.Dir+"/"+svgName+"_"+hostName+"_"+uri+"_flame.svg")
var (
out bytes.Buffer
cmdErr bytes.Buffer
)
cmd.Stdout = &out
cmd.Stderr = &cmdErr
// 执行命令
if sType == 1 { //串行
if err = cmd.Run(); err != nil {
log.Error("go-torch Run stdout=(%s) stderr=(%s) error(%v)", out.String(), cmdErr.String(), err)
}
} else { //阻塞
if err = cmd.Start(); err != nil {
log.Error("go-torch Start stdout=(%s) stderr=(%s) error(%v)", out.String(), cmdErr.String(), err)
}
if err = cmd.Wait(); err != nil {
log.Error("cmd.Wait() error(%v)", err)
}
}
return
}
// ActiveWarning active
func (s *Service) ActiveWarning(c context.Context, text string) (err error) {
var (
ins *pprof.Ins
title = "【%s】性能告警抓取通知"
warn = &pprof.Warning{}
pws = make([]*pprof.Warn, 0)
times = time.Now().Unix()
curTime = xtime.Time(times)
reg = regexp.MustCompile(`[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}`)
)
if err = json.Unmarshal([]byte(text), warn); err != nil {
log.Error("s.ActiveWarning json.Unmarshal data error(%v)", err)
return
}
if warn.Tags.App == "" {
log.Info("s.ActiveWarning skipped(AppName is empty)")
return
}
title = fmt.Sprintf(title, warn.Tags.App)
if ins, err = s.dao.Instances(c, warn.Tags.App); err != nil {
log.Error("s.ActiveWaring get instances error(%v)", err)
return
}
for _, instance := range ins.Instances {
var (
ip string
addrs = instance.Addrs
strs = strings.Split(instance.Hostname, "-")
hostName = fmt.Sprintf("%s-%d", strs[len(strs)-1], times)
pprofWarn = &pprof.Warn{}
)
if len(addrs) < 1 {
log.Info("s.ActiveWarning not found adds")
continue
}
ip = reg.FindString(addrs[0])
host := fmt.Sprintf("http://%s:%s", ip, port)
pprofWarn.IP = ip
pprofWarn.AppID = warn.Tags.App
pprofWarn.SvgName = hostName
pprofWarn.Ctime = curTime
pprofWarn.Mtime = curTime
if err = s.Torch(c, host, "profile", warn.Tags.App, hostName, 30, 2); err == nil {
pprofWarn.Kind = CPUPerformace
pws = append(pws, packing(pprofWarn))
}
if err = s.Pprof(host, "profile", warn.Tags.App, hostName, 30, 2); err == nil {
pprofWarn.Kind = CPUFrame
pws = append(pws, packing(pprofWarn))
}
if err = s.Torch(c, host, "heap", warn.Tags.App, hostName, 30, 2); err == nil {
pprofWarn.Kind = HeapPerformance
pws = append(pws, packing(pprofWarn))
}
if err = s.Pprof(host, "heap", warn.Tags.App, hostName, 30, 2); err == nil {
pprofWarn.Kind = HeapFrame
pws = append(pws, packing(pprofWarn))
}
}
if len(pws) == 0 {
return
}
if err = s.AddPprofWarn(c, pws); err != nil {
return
}
return s.SendWeChat(c, title, fmt.Sprintf(msg, s.c.Host.SVENCo), warn.Tags.App, strings.Join(s.c.WeChat.Users, ","))
}
// packing .
func packing(pw *pprof.Warn) (pprofWarn *pprof.Warn) {
pprofWarn = &pprof.Warn{
AppID: pw.AppID,
SvgName: pw.SvgName,
IP: pw.IP,
Kind: pw.Kind,
Mtime: pw.Mtime,
Ctime: pw.Ctime,
}
return
}
// AddPprofWarn .
func (s *Service) AddPprofWarn(c context.Context, pws []*pprof.Warn) (err error) {
var (
sql = "INSERT INTO `pprof_warn`(`app_id`, `svg_name`, `ip`, `kind`, `ctime`, `mtime`) VALUES %s"
key = make([]string, 0)
value = make([]interface{}, 0)
)
for _, pw := range pws {
key = append(key, "(?,?,?,?,?,?)")
value = append(value, pw.AppID, pw.SvgName, pw.IP, pw.Kind, pw.Ctime, pw.Mtime)
}
if err = s.DB.Exec(fmt.Sprintf(sql, strings.Join(key, ",")), value...).Error; err != nil {
log.Error("s.AddPprofWarn error(%v)", err)
}
return
}
// PprofWarn .
func (s *Service) PprofWarn(c context.Context, req *pprof.Params) (pws []*pprof.Warn, err error) {
var (
query = s.DB.Where("1=1")
)
pws = make([]*pprof.Warn, 0)
if req.AppID == "" && req.IP == "" && req.SvgName == "" && req.Kind == 0 && req.StartTime == 0 && req.EndTime == 0 {
return
}
if req.AppID != "" {
query = query.Where("app_id=?", req.AppID)
}
if req.SvgName != "" {
query = query.Where("svg_name=?", req.SvgName)
}
if req.IP != "" {
query = query.Where("ip=?", req.IP)
}
if req.Kind != 0 {
query = query.Where("kind=?", req.Kind)
}
if req.StartTime != 0 && req.EndTime != 0 {
query = query.Where("mtime between ? and ?", req.StartTime, req.EndTime)
}
if err = query.Order("mtime desc").Find(&pws).Error; err != nil {
log.Error("s.PprofWarn query error(%v)", err)
}
s.setSvgURL(pws)
return
}
// setSvgURL .
func (s *Service) setSvgURL(pws []*pprof.Warn) {
for _, pw := range pws {
switch {
case pw.Kind == CPUPerformace:
pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "profile", pw.SvgName)
case pw.Kind == CPUFrame:
pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "profile_flame", pw.SvgName)
case pw.Kind == HeapPerformance:
pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "heap", pw.SvgName)
case pw.Kind == HeapFrame:
pw.URL = fmt.Sprintf(cpuURL, s.c.Host.SVENCo, pw.AppID, "heap_flame", pw.SvgName)
default:
}
}
}

50
app/admin/main/apm/service/pprof_test.go Normal file
View File

@@ -0,0 +1,50 @@
package service
import (
"context"
"testing"
"go-common/app/admin/main/apm/model/pprof"
"github.com/smartystreets/goconvey/convey"
)
var (
text = `
{
"title": "主站 HTTP_SERVER 错误率过高(主告警条件)",
"tags": {
"app": "account.service.member",
"code": "-404",
"exported_job": "caster_app_metrics",
"method": "x/v2/view"
}
}`
)
func TestService_ActiveWarning(t *testing.T) {
convey.Convey("ActiveWarning", t, func() {
err := svr.ActiveWarning(context.Background(), text)
convey.So(err, convey.ShouldBeNil)
})
}
func TestService_Pprof(t *testing.T) {
var (
err error
req = &pprof.Params{
AppID: "account.service.member",
Kind: 1,
SvgName: "4zf56-1539587841",
}
pws = make([]*pprof.Warn, 0)
)
convey.Convey("PprofWarn", t, func() {
pws, err = svr.PprofWarn(context.Background(), req)
convey.So(err, convey.ShouldBeNil)
convey.So(pws, convey.ShouldNotBeEmpty)
for _, pw := range pws {
t.Logf("pw.Kind=%d, pw.URL=%s", pw.Kind, pw.URL)
}
})
}

289
app/admin/main/apm/service/prometheus.go Normal file
View File

@@ -0,0 +1,289 @@
package service
import (
"context"
"crypto/sha1"
"encoding/hex"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"go-common/app/admin/main/apm/model/monitor"
"go-common/library/log"
"github.com/gogo/protobuf/sortkeys"
)
const (
countQuery = "sum(rate(go_%s_server_count{app='%s'}[2m])) by (method)"
costQuery = "avg(increase(go_%s_server_sum{app='%s'}[5m]) >0) by (method)"
inPacketQuery = "irate(node_network_receive_packets{instance_name=~'%s', device!~'^(lo|bond).*'}[5m]) or irate(node_network_receive_packets_total{instance_name='~%s', device!~'^(lo|bond).*'}[5m])"
outPacketQuery = "irate(node_network_transmit_packets{instance_name=~'%s', device!~\"^(lo|bond).*\"}[5m]) or irate(node_network_transmit_packets_total{instance_name=~'%s', device!~\"^(lo|bond).*\"}[5m])"
inBoundQuery = "irate(node_network_receive_bytes{instance_name=~'%s', device!~\"^(lo|bond).*\"}[5m]) * 8 or irate(node_network_receive_bytes_total{instance_name=~'%s', device!~\"^(lo|bond).*\"}[5m]) * 8"
outBoundQuery = "irate(node_network_transmit_bytes{instance_name=~'%s', device!~\"^(lo|bond).*\"}[5m]) * 8 or irate(node_network_transmit_bytes_total{instance_name=~'%s', device!~\"^(lo|bond).*\"}[5m]) * 8"
tcpStatQuery = "max(node_tcp_stat{instance_name=~'%s', stat='ESTAB'}) by (stat)"
producerQuery = "sum(increase(go_databus_counter{operation='producer_msg_speed'}[2m]))"
consumerQuery = "sum(increase(go_databus_counter{operation='consumer_msg_speed'}[2m]))"
)
// PrometheusRes http result
type PrometheusRes struct {
RetCode int `json:"RetCode"`
Data []*Prometheus `json:"data"`
}
// Prometheus .
type Prometheus struct {
Metric struct {
Method string `json:"method"`
} `json:"metric"`
Values [][]interface{} `json:"values"`
}
// Max max value
type Max struct {
K string
V int64
}
// CommonRes packet or bound result data
type CommonRes struct {
RetCode int `json:"RetCode"`
Data []*Common `json:"data"`
}
// Common .
type Common struct {
Metric struct {
InstanceName string `json:"instance_name"`
} `json:"metric"`
Value []interface{} `json:"value"`
}
// auth calc sign
func (s *Service) auth(params url.Values) string {
var (
sortKey = make([]string, 0)
hash = sha1.New()
signature string
str string
)
for key := range params {
sortKey = append(sortKey, key)
}
sortkeys.Strings(sortKey)
for _, key := range sortKey {
str += key + params.Get(key)
}
str += s.c.Prometheus.Secret
hash.Write([]byte(str))
signature = hex.EncodeToString(hash.Sum(nil))
return signature
}
// PrometheusProxy Get Prometheus Data
func (s *Service) PrometheusProxy(c context.Context, params url.Values, res interface{}) (err error) {
var (
req *http.Request
uri = s.c.Prometheus.URL
)
if req, err = s.client.NewRequest(http.MethodPost, uri, "", params); err != nil {
log.Error("s.PrometheusProxy.NewRequest error(%v) params(%s)", err, params.Encode())
return
}
if err = s.client.Do(c, req, res); err != nil {
log.Error("s.PrometheusProxy.Client.Do error(%v) params(%s)", err, params.Encode())
}
return
}
// prometheus get prometheus data
func (s *Service) prometheus(c context.Context, method string) (mts []*monitor.Monitor, err error) {
var (
sign, ins string
names []string
params = url.Values{}
)
ins, _ = s.GetInterfaces(c)
mts = make([]*monitor.Monitor, 0)
params.Add("Action", "GetPromDataRange")
params.Add("PublicKey", s.c.Prometheus.Key)
params.Add("DataSource", "app")
sign = s.auth(params)
params.Add("Signature", sign)
date := time.Now().Format("2006-01-02")
params.Set("Start", date+" 23:00:00")
params.Set("End", date+" 23:00:10")
params.Set("Step", "30")
names = s.c.Apps.Name
for _, name := range names {
var (
costRet = &PrometheusRes{}
countRet = &PrometheusRes{}
)
params.Set("Query", fmt.Sprintf(costQuery, method, name))
if err = s.PrometheusProxy(c, params, costRet); err != nil {
return
}
params.Set("Query", fmt.Sprintf(countQuery, method, name))
if err = s.PrometheusProxy(c, params, countRet); err != nil {
return
}
for _, val := range costRet.Data {
var (
count float64
api = val.Metric.Method
)
if api == "inner.Ping" || len(val.Values) < 1 || len(val.Values[0]) < 1 {
continue
}
cost, _ := strconv.ParseFloat(val.Values[0][1].(string), 64)
if int64(cost) < s.c.Apps.Max && !strings.Contains(ins, api) {
continue
}
for _, v := range countRet.Data {
if api == v.Metric.Method {
count, _ = strconv.ParseFloat(v.Values[0][1].(string), 64)
break
}
}
mt := &monitor.Monitor{
AppID: name + "-" + method,
Interface: api,
Count: int64(count),
Cost: int64(cost),
}
mts = append(mts, mt)
}
}
return
}
// GetInterfaces .
func (s *Service) GetInterfaces(c context.Context) (string, error) {
mt := &monitor.Monitor{}
if err := s.DB.Select("group_concat(interface) as interface").Find(mt).Error; err != nil {
log.Error("s.GetInterfaces query error(%v)", err)
return "", err
}
return mt.Interface, nil
}
// RPCMonitor get rpc monitor data
func (s *Service) RPCMonitor(c context.Context) ([]*monitor.Monitor, error) {
return s.prometheus(c, "rpc")
}
// HTTPMonitor get http monitor data
func (s *Service) HTTPMonitor(c context.Context) ([]*monitor.Monitor, error) {
return s.prometheus(c, "http")
}
// DataBus return DataBus monitor data
func (s *Service) DataBus(c context.Context) (mts []*monitor.Monitor, err error) {
var (
sign string
params = url.Values{}
proRet = &CommonRes{}
conRet = &CommonRes{}
)
mts = make([]*monitor.Monitor, 0)
sign = s.auth(params)
params.Add("Action", "GetCurrentPromData")
params.Add("PublicKey", s.c.Prometheus.Key)
params.Add("DataSource", "app")
params.Add("Signature", sign)
params.Set("Query", producerQuery)
if err = s.PrometheusProxy(c, params, proRet); err != nil {
return
}
mts = append(mts, pack(proRet.Data, "kafka-databus", "producer")...)
params.Set("Query", consumerQuery)
if err = s.PrometheusProxy(c, params, conRet); err != nil {
return
}
mts = append(mts, pack(conRet.Data, "kafka-databus", "consumer")...)
return
}
// TenCent return TenCent monitor data
func (s *Service) TenCent(c context.Context) (mts []*monitor.Monitor, err error) {
return s.BroadCast(c, s.c.BroadCast.TenCent, "tencent_main")
}
// KingSoft return KingSoft monitor data
func (s *Service) KingSoft(c context.Context) (mts []*monitor.Monitor, err error) {
return s.BroadCast(c, s.c.BroadCast.KingSoft, "kingsoft_main")
}
// BroadCast return monitor data
func (s *Service) BroadCast(c context.Context, appName []string, app string) (mts []*monitor.Monitor, err error) {
var (
sign string
names string
params = url.Values{}
inPacket = &CommonRes{}
outPacket = &CommonRes{}
inBound = &CommonRes{}
outBound = &CommonRes{}
tcpStat = &CommonRes{}
)
mts = make([]*monitor.Monitor, 0)
params.Add("Action", "GetCurrentPromData")
params.Add("PublicKey", s.c.Prometheus.Key)
sign = s.auth(params)
params.Add("Signature", sign)
names = strings.Join(appName, "|")
params.Set("Query", fmt.Sprintf(inPacketQuery, names, names))
params.Set("DataSource", app)
if err = s.PrometheusProxy(c, params, inPacket); err != nil {
return
}
mts = append(mts, pack(inPacket.Data, "", "InPacket")...)
params.Set("Query", fmt.Sprintf(outPacketQuery, names, names))
if err = s.PrometheusProxy(c, params, outPacket); err != nil {
return
}
mts = append(mts, pack(outPacket.Data, "", "OutPacket")...)
params.Set("Query", fmt.Sprintf(inBoundQuery, names, names))
if err = s.PrometheusProxy(c, params, inBound); err != nil {
return
}
mts = append(mts, pack(inBound.Data, "", "InBound")...)
params.Set("Query", fmt.Sprintf(outBoundQuery, names, names))
if err = s.PrometheusProxy(c, params, outBound); err != nil {
return
}
mts = append(mts, pack(outBound.Data, "", "OutBound")...)
for _, name := range appName {
params.Set("Query", fmt.Sprintf(tcpStatQuery, name))
if err = s.PrometheusProxy(c, params, tcpStat); err != nil {
return
}
mts = append(mts, pack(tcpStat.Data, name, "ESTAB")...)
}
return
}
// pack .
func pack(pts []*Common, K, V string) (mts []*monitor.Monitor) {
for _, pt := range pts {
if K != "" {
pt.Metric.InstanceName = K
}
count, _ := strconv.ParseFloat(pt.Value[1].(string), 64)
if count == 0 {
continue
}
mt := &monitor.Monitor{
AppID: pt.Metric.InstanceName,
Interface: pt.Metric.InstanceName + "_" + V,
Count: int64(count),
}
mts = append(mts, mt)
}
return mts
}

101
app/admin/main/apm/service/prometheus_test.go Normal file
View File

@@ -0,0 +1,101 @@
package service
import (
"context"
"testing"
"go-common/app/admin/main/apm/model/monitor"
"github.com/smartystreets/goconvey/convey"
)
func TestService_RPCMonitor(t *testing.T) {
var (
c = context.Background()
mts []*monitor.Monitor
err error
)
convey.Convey("Prometheus", t, func(ctx convey.C) {
mts, err = svr.RPCMonitor(c)
convey.Convey("Then pts should not be nil, err should be nil", func(ctx convey.C) {
ctx.So(mts, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
for _, mt := range mts {
t.Logf("mt.Interface=%s, mt.Cost=%d, mt.Count=%d", mt.Interface, mt.Cost, mt.Count)
}
})
})
}
func TestService_HttpMonitor(t *testing.T) {
var (
c = context.Background()
mts []*monitor.Monitor
err error
)
convey.Convey("Prometheus", t, func(ctx convey.C) {
mts, err = svr.HTTPMonitor(c)
convey.Convey("Then pts should not be nil, err should be nil", func(ctx convey.C) {
ctx.So(mts, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
for _, mt := range mts {
t.Logf("mt.Interface=%s, mt.Cost=%d, mt.Count=%d", mt.Interface, mt.Cost, mt.Count)
}
})
})
}
func TestService_TenCent(t *testing.T) {
var (
c = context.Background()
mts = make([]*monitor.Monitor, 0)
err error
)
convey.Convey("TenCent", t, func(ctx convey.C) {
mts, err = svr.TenCent(c)
convey.Convey("Then mts should not be nil, err should be nil", func(ctx convey.C) {
ctx.So(mts, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
for _, mt := range mts {
t.Logf("mt.AppID=%s, mt.Interface=%s, mt.Count=%d", mt.AppID, mt.Interface, mt.Count)
}
})
})
}
func TestService_KingSoft(t *testing.T) {
var (
c = context.Background()
mts = make([]*monitor.Monitor, 0)
err error
)
convey.Convey("KingSoft", t, func(ctx convey.C) {
mts, err = svr.KingSoft(c)
convey.Convey("Then mts should not be nil, err should be nil", func(ctx convey.C) {
ctx.So(mts, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
for _, mt := range mts {
t.Logf("mt.AppID=%s, mt.Interface=%s, mt.Count=%d", mt.AppID, mt.Interface, mt.Count)
}
})
})
}
func TestService_DataBus(t *testing.T) {
var (
c = context.Background()
mts = make([]*monitor.Monitor, 0)
err error
)
convey.Convey("DataBus", t, func(ctx convey.C) {
mts, err = svr.DataBus(c)
convey.Convey("Then mts should not be nil, err should be nil", func(ctx convey.C) {
ctx.So(mts, convey.ShouldNotBeEmpty)
ctx.So(err, convey.ShouldBeNil)
for _, mt := range mts {
t.Logf("mt.AppID=%s, mt.Interface=%s, mt.Count=%d", mt.AppID, mt.Interface, mt.Count)
}
})
})
}

39
app/admin/main/apm/service/qywechat.go Normal file
View File

@@ -0,0 +1,39 @@
package service
import (
"context"
"net/url"
"go-common/library/log"
)
// Ret .
type Ret struct {
ReqID string `json:"ReqId"`
Action string `json:"Action"`
RetCode int `json:"RetCode"`
Data []string `json:"Data"`
Response struct {
Status int `json:"status"`
} `json:"Response"`
}
// SendWeChat send message to WeChat
// users: zhangsan,lisi,wangwu
func (s *Service) SendWeChat(c context.Context, title, msg, treeID, users string) (err error) {
var (
params = url.Values{}
ret = &Ret{}
)
params.Add("Action", "CreateWechatMessage")
params.Add("PublicKey", s.c.Prometheus.Key)
params.Add("Signature", "1")
params.Add("UserName", users)
params.Add("Title", title)
params.Add("Content", title+"\n"+msg)
params.Add("TreeId", "bilibili."+treeID)
if err = s.PrometheusProxy(context.Background(), params, ret); err != nil {
log.Error("s.SendWeChat error(%v)", err)
}
return
}

91
app/admin/main/apm/service/service.go Normal file
View File

@@ -0,0 +1,91 @@
package service
import (
"context"
"go-common/app/admin/main/apm/conf"
"go-common/app/admin/main/apm/dao"
"go-common/app/admin/main/apm/model/tree"
"go-common/app/admin/main/apm/model/ut"
"go-common/app/tool/saga/service/gitlab"
bm "go-common/library/net/http/blademaster"
"sync"
"github.com/jinzhu/gorm"
"github.com/robfig/cron"
)
// Service is a service.
type Service struct {
c *conf.Config
dao *dao.Dao
DB *gorm.DB
DBDatabus *gorm.DB
DBCanal *gorm.DB
client *bm.Client
// tree cache
treeCache map[string][]*tree.Node
treeLock sync.RWMutex
// cron cron
cron *cron.Cron
// discoveryID cache
discoveryIDCache map[string]*tree.Resd
discoveryIDLock sync.RWMutex
ranksCache *ut.RanksCache
appsCache *ut.AppsCache
// dapper proxy
dapperProxy *dapperProxy
// gitlab api conf
gitlab *gitlab.Gitlab
}
// New new a service
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
client: bm.NewClient(c.HTTPClient),
// tree cache
treeCache: map[string][]*tree.Node{},
// discoveryID cache
discoveryIDCache: map[string]*tree.Resd{},
// ranks cache
ranksCache: &ut.RanksCache{},
appsCache: &ut.AppsCache{},
// cron cron
cron: cron.New(),
}
s.gitlab = gitlab.New(conf.Conf.Gitlab.API, conf.Conf.Gitlab.Token)
s.DB = s.dao.DB
s.DBDatabus = s.dao.DBDatabus
s.DBCanal = s.dao.DBCanal
if err := s.cron.AddFunc(s.c.Cron.Crontab, s.taskAddMonitor); err != nil {
panic(err)
}
if err := s.cron.AddFunc(s.c.Cron.Crontab, s.taskAddCache); err != nil {
panic(err)
}
if err := s.cron.AddFunc(s.c.Cron.CrontabRepo, s.taskRankWechatReport); err != nil {
panic(err)
}
if err := s.cron.AddFunc(s.c.Cron.CrontabRepo, s.taskWeeklyWechatReport); err != nil {
panic(err)
}
dp, err := newDapperProxy(c.Host.DapperCo)
if err != nil {
panic(err)
}
s.dapperProxy = dp
s.cron.Start()
go s.taskAddCache()
return
}
// Ping ping db,
func (s *Service) Ping(c context.Context) (err error) {
return
}
// Close close resource.
func (s *Service) Close() {
s.dao.Close()
}

21
app/admin/main/apm/service/service_test.go Normal file
View File

@@ -0,0 +1,21 @@
package service
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestService(t *testing.T) {
Convey("service", t, func() {
t.Log("service test")
})
}
func TestPing(t *testing.T) {
Convey("ping", t, func() {
var s = &Service{}
s.Ping(context.Background())
})
}

48
app/admin/main/apm/service/task.go Normal file
View File

@@ -0,0 +1,48 @@
package service
import (
"context"
"time"
"go-common/library/conf/env"
"go-common/library/log"
)
// taskAddMonitor add monitor task
func (s *Service) taskAddMonitor() {
if err := s.AddMonitor(context.TODO()); err != nil {
log.Error("task.taskAddMonitor error(%v)", err)
}
}
// taskAddCache add cache task
func (s *Service) taskAddCache() {
if err := s.RanksCache(context.Background()); err != nil {
log.Error("task.RanksCache error(%v)", err)
}
if err := s.AppsCache(context.Background()); err != nil {
log.Error("task.AppsCache error(%v)", err)
}
}
// taskRankWechatReport send rank report to wechat group task
func (s *Service) taskRankWechatReport() {
if env.DeployEnv != env.DeployEnvProd || time.Now().Weekday() == time.Sunday || time.Now().Weekday() == time.Saturday {
return
}
if err := s.RankWechatReport(context.TODO()); err != nil {
log.Error("task.taskRankWechatReport error(%v)", err)
}
}
// taskWeeklyWechatReport send Weekly report and reset redis every Friday 19:00
func (s *Service) taskWeeklyWechatReport() {
if env.DeployEnv == env.DeployEnvProd && time.Now().Weekday() == time.Friday {
if err := s.SummaryWechatReport(context.TODO()); err != nil {
log.Error("task.taskWeeklyWechatReport error(%v)", err)
}
if err := s.dao.SetAppCovCache(context.TODO()); err != nil {
log.Error("task.taskWeeklyWechatReport error(%v)", err)
}
}
}

287
app/admin/main/apm/service/tree.go Normal file
View File

@@ -0,0 +1,287 @@
package service
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
"go-common/app/admin/main/apm/conf"
"go-common/app/admin/main/apm/model/tree"
"go-common/library/ecode"
"go-common/library/log"
)
const (
_discoveryIDKey = "%s_discoveryIDKey"
)
// Appids get appid by username.
func (s *Service) Appids(c context.Context, username, cookie string) (appids []string, err error) {
s.treeLock.RLock()
nodem, ok := s.treeCache[username]
tmp := make(map[string]struct{})
if !ok {
s.treeLock.RUnlock()
s.TreeSync(c, username, cookie)
s.treeLock.RLock()
nodem = s.treeCache[username]
}
for _, v := range nodem {
nameArr := strings.Split(v.Path, ".")
newName := nameArr[1] + "." + nameArr[2] + "." + nameArr[3]
if _, ok := tmp[newName]; !ok {
appids = append(appids, newName)
tmp[newName] = struct{}{}
}
}
s.treeLock.RUnlock()
return
}
// Projects get projects by username.
func (s *Service) Projects(c context.Context, username, cookie string) (projects []string, err error) {
s.treeLock.RLock()
nodem, ok := s.treeCache[username]
mm := make(map[string]struct{})
if !ok {
s.treeLock.RUnlock()
s.TreeSync(c, username, cookie)
s.treeLock.RLock()
nodem = s.treeCache[username]
}
for _, v := range nodem {
nameArr := strings.Split(v.Path, ".")
newName := nameArr[1] + "." + nameArr[2]
if _, ok := mm[newName]; ok {
continue
}
mm[newName] = struct{}{}
projects = append(projects, newName)
}
s.treeLock.RUnlock()
return
}
// TreeSync sync tree cache by username.
func (s *Service) TreeSync(c context.Context, username, cookie string) {
nodem, err := s.roleTrees(c, username, cookie)
if err != nil {
log.Error("TreeSync(%s) error(%v)", username, err)
return
}
s.treeLock.Lock()
s.treeCache[username] = nodem
s.treeLock.Unlock()
}
// trees get service trees by username.
func (s *Service) roleTrees(c context.Context, username, cookie string) (trees []*tree.Node, err error) {
url := "http://easyst.bilibili.co/v1/auth"
req, err := http.NewRequest("GET", url, nil)
if err != nil {
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/json;charset=UTF-8")
req.Header.Set("Cookie", cookie)
var result = &tree.TokenResult{}
if err = s.client.Do(c, req, result); err != nil {
log.Error("TreeSync(%s) error(%v)", username, err)
err = ecode.RequestErr
return
}
url = "http://easyst.bilibili.co/v1/node/role/app"
if req, err = http.NewRequest("GET", url, nil); err != nil {
log.Error("TreeSync(%s) get token error(%v)", username, err)
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Authorization-Token", result.Data.Token)
var dat = &tree.Resp{}
if err = s.client.Do(c, req, dat); err != nil {
log.Error("TreeSync(%s) token(%s) error(%v)", username, result.Data.Token, err)
err = ecode.RequestErr
return
}
if len(dat.Data) == 0 {
log.Error("TreeSync(%s) no data", username)
return
}
trees = dat.Data
log.Info("TreeSync(%s) data(%v)", username, trees)
return
}
// Trees tree list
func (s *Service) Trees(c context.Context, username, cookie string) (nodem []*tree.Node, err error) {
s.treeLock.RLock()
nodem, ok := s.treeCache[username]
if !ok {
s.treeLock.RUnlock()
s.TreeSync(c, username, cookie)
s.treeLock.RLock()
nodem = s.treeCache[username]
}
s.treeLock.RUnlock()
return
}
// AllTrees AllTrees
// func (s *Service) AllTrees(c context.Context, username, cookie string) (trees []*tree.Info, err error) {
// var (
// jsonBytes []byte
// url = "http://easyst.bilibili.co/v1/token"
// )
// body := &struct {
// Username string `json:"user_name"`
// PlatformID string `json:"platform_id"`
// }{
// Username: "main",
// PlatformID: conf.Conf.Tree.PlatformID,
// }
// if jsonBytes, err = json.Marshal(body); err != nil {
// log.Error("json.Marshal(body) error(%v)", err)
// return
// }
// req, err := http.NewRequest("POST", url, strings.NewReader(string(jsonBytes)))
// if err != nil {
// log.Error("http.NewRequest failed", err)
// return
// }
// req.Header.Set("Content-Type", "application/json")
// result := &struct {
// Code int `json:"code"`
// Data *model.ServiceTreeToken `json:"data"`
// Message string `json:"message"`
// Status int `json:"status"`
// }{}
// if err = s.client.Do(c, req, result); err != nil {
// log.Error("TreesAll(%s) get token error(%v)", username, err)
// err = ecode.RequestErr
// return
// }
// url = "http://easyst.bilibili.co/v1/node/app/extendinfo"
// if req, err = http.NewRequest("GET", url, nil); err != nil {
// log.Error("TreesAll(%s) error(%v)", username, err)
// err = ecode.RequestErr
// return
// }
// req.Header.Set("Content-Type", "application/json")
// req.Header.Set("X-Authorization-Token", result.Data.Token)
// var dat = &tree.Rest{}
// if err = s.client.Do(c, req, dat); err != nil {
// log.Error("TreesAll(%s) token(%s) error(%v)", username, result.Data.Token, err)
// err = ecode.RequestErr
// return
// }
// if len(dat.Data) == 0 {
// log.Error("TreesAll(%s) no data", username)
// return
// }
// trees = dat.Data
// log.Info("TreesAll(%s) data(%v)", username, trees)
// return
// }
// DiscoveryID get appid by username.
func (s *Service) DiscoveryID(c context.Context, username, cookie string) (appids []string, err error) {
keyName := discoveryIDKey(username)
s.discoveryIDLock.RLock()
nodem, ok := s.discoveryIDCache[keyName]
if !ok || (time.Since(nodem.CTime) > 60*time.Second) {
s.discoveryIDLock.RUnlock()
s.DiscoveryTreeSync(c, username, cookie)
s.discoveryIDLock.RLock()
nodem = s.discoveryIDCache[keyName]
}
s.discoveryIDLock.RUnlock()
tmp := make(map[string]struct{})
for _, v := range nodem.Data {
var newName string
if v.DiscoveryID != "" {
newName = v.DiscoveryID
} else {
nameArr := strings.Split(v.AppID, ".")
newName = nameArr[0] + "." + nameArr[1] + "." + nameArr[2]
}
if _, ok := tmp[newName]; !ok {
appids = append(appids, newName)
tmp[newName] = struct{}{}
}
}
return
}
// discoveryAllTrees ...
func (s *Service) discoveryAllTrees(c context.Context, username, cookie string) (dat *tree.Resd, err error) {
url := "http://easyst.bilibili.co/v1/token"
var jsonBytes []byte
body := &struct {
Username string `json:"user_name"`
PlatformID string `json:"platform_id"`
}{
Username: "msm",
PlatformID: conf.Conf.Tree.MsmPlatformID,
}
if jsonBytes, err = json.Marshal(body); err != nil {
log.Error("json.Marshal(body) error(%v)", err)
return
}
req, err := http.NewRequest("POST", url, strings.NewReader(string(jsonBytes)))
if err != nil {
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/json;charset=UTF-8")
req.Header.Set("Cookie", cookie)
var result = &tree.TokenResult{}
if err = s.client.Do(c, req, result); err != nil {
log.Error("TreeSync(%s) error(%v)", username, err)
err = ecode.RequestErr
return
}
dat = &tree.Resd{}
url = "http://easyst.bilibili.co/v1/node/app/secretinfo/prod"
if req, err = http.NewRequest("GET", url, nil); err != nil {
log.Error("TreeSync(%s) get token error(%v)", username, err)
err = ecode.RequestErr
return
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Authorization-Token", result.Data.Token)
if err = s.client.Do(c, req, dat); err != nil {
log.Error("TreeSync(%s) token(%s) error(%v)", username, result.Data.Token, err)
err = ecode.RequestErr
return
}
if len(dat.Data) == 0 {
log.Error("TreeSync(%s) no data", username)
return
}
dat.CTime = time.Now()
log.Info("TreeSync(%s) data(%v)", username, dat)
return
}
func discoveryIDKey(username string) string {
return fmt.Sprintf(_discoveryIDKey, username)
}
// DiscoveryTreeSync sync tree cache by username discoverykey.
func (s *Service) DiscoveryTreeSync(c context.Context, username, cookie string) {
keyName := discoveryIDKey(username)
nodem, err := s.discoveryAllTrees(c, username, cookie)
if err != nil {
log.Error("DiscoveryTreeSync(%s) error(%v)", username, err)
return
}
s.discoveryIDLock.Lock()
s.discoveryIDCache[keyName] = nodem
s.discoveryIDLock.Unlock()
}

13
app/admin/main/apm/service/tree_test.go Normal file
View File

@@ -0,0 +1,13 @@
package service
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestTree(t *testing.T) {
convey.Convey("tree", t, func() {
t.Log("tree test")
})
}

139
app/admin/main/apm/service/upload.go Normal file
View File

@@ -0,0 +1,139 @@
package service
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strconv"
"strings"
"go-common/app/admin/main/apm/model/ut"
"go-common/library/log"
"github.com/smartystreets/goconvey/web/server/contract"
"github.com/smartystreets/goconvey/web/server/parser"
)
// Upload upload file to bfs with no filename
func (s *Service) Upload(c context.Context, fileType string, expire int64, body []byte) (url string, err error) {
if len(body) == 0 {
err = fmt.Errorf("s.Upload error(empty file)")
return
}
if url, err = s.dao.UploadProxy(c, fileType, expire, body); err != nil {
log.Error("s.Upload error(%v)", err)
}
return
}
// ParseContent parse go test output to go convey result json.
// If result not contains "=== RUN", that means execute ut err.
func (s *Service) ParseContent(c context.Context, body []byte) (content []byte, err error) {
var (
text = string(body)
index = strings.Index(text, "=== RUN")
)
if index == -1 {
err = fmt.Errorf(text)
return
}
var res = new(contract.PackageResult)
parser.ParsePackageResults(res, text[index:])
if content, err = json.Marshal(res); err != nil {
log.Error("service.Upload json.Marshal err (%v)", err)
}
return
}
// CalcCount calc count
func (s *Service) CalcCount(c context.Context, body []byte) (pkg *ut.PkgAnls, err error) {
res := new(contract.PackageResult)
pkg = new(ut.PkgAnls)
if err = json.Unmarshal(body, res); err != nil {
log.Error("service.CalcCount json.Unmarshal err (%v)", err)
return
}
pkg.Coverage = res.Coverage * 100
for _, v := range res.TestResults {
if len(v.Stories) == 0 {
pkg.Assertions++
if v.Error != "" {
pkg.Panics++
} else if !v.Passed {
pkg.Failures++
} else if v.Skipped {
pkg.Skipped++
} else {
pkg.Passed++
}
}
for _, story := range v.Stories {
for _, ass := range story.Assertions {
pkg.Assertions++
if ass.Skipped {
pkg.Skipped++
} else if ass.Failure != "" {
pkg.Failures++
} else if ass.Error != nil {
pkg.Panics++
} else {
pkg.Passed++
}
}
}
}
return
}
// CalcCountFiles calculating lines and statements in files
func (s *Service) CalcCountFiles(c context.Context, res *ut.UploadRes, body []byte) (utFiles []*ut.File, err error) {
var (
fblocks = make(map[string][]ut.Block)
data = strings.Split(string(body[:]), "\n")
reg = regexp.MustCompile(`^(.+):([0-9]+).([0-9]+),([0-9]+).([0-9]+) ([0-9]+) ([0-9]+)$`)
)
if !strings.HasPrefix(data[0], "mode:") {
return nil, fmt.Errorf("Wrong cover.dat/cover.out file format")
}
for i := 1; i < len(data); i++ {
if data[i] == "" {
continue
}
m := reg.FindStringSubmatch(data[i])
if m == nil {
return nil, fmt.Errorf("line %s doesn't match expected format: %v", data[i], reg)
}
b := ut.Block{
Start: toInt(m[2]),
End: toInt(m[4]),
Statements: toInt(m[6]),
Count: toInt(m[7]),
}
fblocks[m[1]] = append(fblocks[m[1]], b)
}
for name, blocks := range fblocks {
utFile := &ut.File{
Name: name,
CommitID: res.CommitID,
PKG: res.PKG,
}
for i := 0; i < len(blocks); i++ {
utFile.Statements += int64(blocks[i].Statements)
if blocks[i].Count > 0 {
utFile.CoveredStatements += int64(blocks[i].Statements)
}
}
utFiles = append(utFiles, utFile)
}
return
}
// Assist functions
func toInt(s string) int {
i, err := strconv.Atoi(s)
if err != nil {
log.Error("strconv.Atoi(%s) error(%v)", s, err)
}
return i
}

105
app/admin/main/apm/service/upload_test.go Normal file
View File

@@ -0,0 +1,105 @@
package service
import (
"context"
"go-common/app/admin/main/apm/model/ut"
"io/ioutil"
"testing"
"github.com/smartystreets/goconvey/convey"
)
// import (
// "context"
// "testing"
// "time"
// "github.com/smartystreets/goconvey/convey"
// )
// const content = `panic: runtime error: invalid memory address or nil pointer dereference
// [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x1565c97]
// goroutine 1 [running]:
// go-common/app/interface/main/answer/dao/account.New(0x1a972e0, 0xc42017f4d0)
// /Users/bilibili/go/src/go-common/app/interface/main/answer/dao/account/pendant.go:36 +0x37
// go-common/app/interface/main/answer/dao/account.init.0()
// /Users/bilibili/go/src/go-common/app/interface/main/answer/dao/account/pendant_test.go:21 +0x68
// FAIL go-common/app/interface/main/answer/dao/account 0.031s
// panic: runtime error: invalid memory address or nil pointer dereference
// [signal SIGSEGV: segmentation violation code=0x1 addr=0x50 pc=0x1439ed6]
// === RUN TestDao_MoralLog
// >->->OPEN-JSON->->->
// {
// "Title": "MoralLog",
// "File": "/Users/bilibili/go/src/go-common/app/service/main/member/dao/hbase_test.go",
// "Line": 11,
// "Depth": 1,
// "Assertions": [
// {
// "File": "/Users/bilibili/go/src/go-common/app/service/main/member/dao/hbase_test.go",
// "Line": 12,
// "Expected": "",
// "Actual": "",
// "Failure": "",
// "Error": "runtime error: invalid memory address or nil pointer dereference",",
// "Skipped": false
// }
// ],
// "Output": ""
// },
// <-<-<-CLOSE-JSON<-<-<
// --- FAIL: TestDao_MoralLog (0.00s)
// `
// func TestService_Upload(t *testing.T) {
// convey.Convey("ParserContent", t, func() {
// data, err := svr.ParseContent(context.Background(), []byte(content))
// convey.So(err, convey.ShouldBeNil)
// convey.So(data, convey.ShouldNotBeNil)
// t.Logf("after parsercontent: %s", string(data))
// convey.So(err, convey.ShouldBeNil)
// info, err := svr.CalcCount(context.Background(), data)
// convey.So(err, convey.ShouldBeNil)
// t.Logf("pass: %d", info.Passed)
// t.Logf("fail: %d", info.Failures)
// t.Logf("skip: %d", info.Skipped)
// t.Logf("panics: %d", info.Panics)
// t.Logf("total: %d", info.Assertions)
// t.Logf("coverage: %s", info.Coverage)
// convey.Convey("Upload", func() {
// var (
// body = data
// )
// url, err := svr.Upload(context.Background(), "json", time.Now().Unix(), body)
// convey.So(err, convey.ShouldBeNil)
// convey.So(url, convey.ShouldNotBeNil)
// t.Logf("Location: %s", url)
// })
// })
// }
func TestServiceCalcCountFiles(t *testing.T) {
convey.Convey("CalcCountFiles", t, func(ctx convey.C) {
var (
c = context.Background()
res = &ut.UploadRes{
CommitID: "somestringhasnothingtodo",
PKG: "go-common/app/admin/main/apm/dao",
}
filename = "/data/ut1/cover.out"
)
body, _ := ioutil.ReadFile(filename)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
utfiles, err := svr.CalcCountFiles(c, res, body)
t.Logf("\nutfiles:%#v\n", utfiles)
for i, utfile := range utfiles {
t.Logf("\nutfiles[%d]:%#v\n", i, utfile)
}
ctx.So(err, convey.ShouldBeNil)
})
})
}

78
app/admin/main/apm/service/user.go Normal file
View File

@@ -0,0 +1,78 @@
package service
import (
"context"
"go-common/app/admin/main/apm/model/user"
"go-common/library/ecode"
"go-common/library/log"
"github.com/jinzhu/gorm"
)
// Permit check user permission.
func (s *Service) Permit(c context.Context, username string, rule string) (err error) {
for _, man := range s.c.Superman {
if username == man {
return
}
}
usr := &user.User{}
if err = s.DB.Where("username=?", username).First(usr).Error; err != nil {
log.Error("s.DB.User(%s) user error(%v)", username, err)
err = ecode.AccessDenied
return
}
if user.Rules[rule].Permit == user.PermitDefault {
return
}
cnt := 0
if err = s.DB.Model(&user.Rule{}).Where("user_id=? AND rule=?", usr.ID, rule).Count(&cnt).Error; err != nil {
log.Error("s.DB.User(%s) count error(%v)", username, err)
err = ecode.AccessDenied
return
}
if cnt == 0 {
log.Warn("s.DB.User(%s) count=0", username)
err = ecode.AccessDenied
}
return
}
// GetDefaultPermission get the modules and rules which have default permission
func (s *Service) GetDefaultPermission(c context.Context) (modules []string, rules []string) {
for m, mp := range user.Modules {
if mp.Permit == user.PermitDefault {
modules = append(modules, m)
}
}
for r, rp := range user.Rules {
if rp.Permit == user.PermitDefault {
rules = append(rules, r)
}
}
return
}
// GetUser get user info by username if it exists, otherwise create the user info
func (s *Service) GetUser(c context.Context, username string) (usr *user.User, err error) {
usr = &user.User{}
err = s.DB.Where("username = ?", username).First(usr).Error
if err == gorm.ErrRecordNotFound {
usr.UserName = username
usr.NickName = username
err = s.DB.Create(usr).Error
}
if err != nil {
log.Error("apmSvc.GetUser error(%v)", err)
return
}
s.ranksCache.Lock()
if s.ranksCache.Map[username] != nil {
usr.AvatarURL = s.ranksCache.Map[username].AvatarURL
} else {
usr.AvatarURL, _ = s.dao.GitLabFace(c, username)
}
s.ranksCache.Unlock()
return
}

37
app/admin/main/apm/service/user_test.go Normal file
View File

@@ -0,0 +1,37 @@
package service
import (
"context"
"reflect"
"testing"
"go-common/app/admin/main/apm/dao"
"github.com/bouk/monkey"
"github.com/smartystreets/goconvey/convey"
)
func TestUser(t *testing.T) {
convey.Convey("user", t, func() {
t.Log("user test")
})
}
func TestServiceGetUser(t *testing.T) {
convey.Convey("GetUser", t, func(ctx convey.C) {
var (
c = context.Background()
username = "chenjianrong"
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(svr.dao), "GitLabFace", func(_ *dao.Dao, _ context.Context, _ string) (string, error) {
return "http://git.bilibili.co/some/pic.jpg", nil
})
defer guard.Unpatch()
usr, err := svr.GetUser(c, username)
ctx.Convey("Then err should be nil.usr should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(usr, convey.ShouldNotBeNil)
})
})
})
}

430
app/admin/main/apm/service/ut.go Normal file
View File

@@ -0,0 +1,430 @@
package service
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"go-common/app/admin/main/apm/conf"
"go-common/app/admin/main/apm/model/ut"
"go-common/library/log"
xtime "go-common/library/time"
"github.com/jinzhu/gorm"
)
const (
_utMerge = "ut_merge"
_utCommit = "ut_commit"
_utpkg = "ut_pkganls"
)
// UtList return ut_list by merge_id or username
func (s *Service) UtList(c context.Context, arg *ut.MergeReq) (mrInfs []*ut.Merge, count int, err error) {
var (
mtime = time.Now().Add(-time.Hour * 24 * 90)
hql = fmt.Sprintf("mtime>'%s'", mtime)
)
if arg.MergeID != 0 {
hql += fmt.Sprintf(" and merge_id=%d ", arg.MergeID)
}
if arg.UserName != "" {
hql += fmt.Sprintf(" and username='%s' ", arg.UserName)
}
if arg.IsMerged != 0 {
hql += fmt.Sprintf(" and is_merged='%d' ", arg.IsMerged)
}
if err = s.DB.Table(_utMerge).Where(hql).Count(&count).Error; err != nil {
log.Error("service.UtList error(%v)", err)
return
}
if err = s.DB.Where(hql).Offset((arg.Pn - 1) * arg.Ps).Limit(arg.Ps).Order("mtime desc").Find(&mrInfs).Error; err != nil {
log.Error("service.UtList error(%v)", err)
return
}
for _, mr := range mrInfs {
var commit = &ut.Commit{}
if err = s.DB.Where("merge_id=?", mr.MergeID).Order("id desc").First(commit).Error; err != nil {
log.Error("service.UtList mergeID(%d) error(%v)", mr.MergeID, err)
return
}
if err = s.DB.Select(`max(commit_id) as commit_id, substring_index(pkg,"/",5) as pkg, ROUND(AVG(coverage/100),2) as coverage, SUM(assertions) as assertions, SUM(panics) as panics, SUM(passed) as passed, ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate, SUM(failures) as failures, MAX(mtime) as mtime`).
Where(`commit_id=? and (pkg!=substring_index(pkg, "/", 5) or pkg like "go-common/library/%")`, commit.CommitID).
Group(`substring_index(pkg, "/", 5)`).Find(&commit.PkgAnls).Error; err != nil {
log.Error("service.UtList commitID(%s) error(%v)", commit.CommitID, err)
return
}
mr.Commit = commit
}
return
}
// UtDetailList get ut pkganls by commit_id&pkg
func (s *Service) UtDetailList(c context.Context, arg *ut.DetailReq) (utpkgs []*ut.PkgAnls, err error) {
var (
mtime = time.Now().Add(-time.Hour * 24 * 90)
hql = fmt.Sprintf("mtime>'%s'", mtime)
)
if arg.CommitID != "" {
hql += fmt.Sprintf(" and commit_id='%s'", arg.CommitID)
}
if arg.PKG != "" {
hql += fmt.Sprintf(" and substring_index(pkg,'/',5)='%s' and (pkg!=substring_index(pkg,'/',5) or pkg like 'go-common/library/%%')", arg.PKG)
}
if err = s.dao.DB.Select("id, commit_id, merge_id, pkg, ROUND(coverage/100,2) as coverage, ROUND(passed/assertions*100,2) as pass_rate,panics,failures,skipped,passed,assertions,html_url,report_url,mtime,ctime").Where(hql).Find(&utpkgs).Error; err != nil {
log.Error("service.UTDetilList commitID(%s) project(%s) error(%v)", arg.CommitID, arg.PKG, err)
return
}
return
}
// UtHistoryCommit get commits history list by merge_id
func (s *Service) UtHistoryCommit(c context.Context, arg *ut.HistoryCommitReq) (utcmts []*ut.Commit, count int, err error) {
var (
mtime = time.Now().Add(-time.Hour * 24 * 90)
hql = fmt.Sprintf("mtime>'%s'", mtime)
)
hql += fmt.Sprintf(" and merge_id=%d", arg.MergeID)
if err = s.DB.Table(_utCommit).Where(hql).Count(&count).Error; err != nil {
log.Error("service.UTHistoryCommit mergeID(%d) error(%v)", arg.MergeID, err)
return
}
if err = s.dao.DB.Where(hql).Offset((arg.Pn - 1) * arg.Ps).Limit(arg.Ps).Order("mtime desc").Find(&utcmts).Error; err != nil {
log.Error("service.UTHistoryCommit mergeID(%d) error(%v)", arg.MergeID, err)
return
}
for _, commit := range utcmts {
if err = s.dao.DB.Select(`max(commit_id) as commit_id, substring_index(pkg,"/",5) as pkg, ROUND(AVG(coverage/100),2) as coverage, SUM(assertions) as assertions, SUM(panics) as panics, SUM(passed) as passed, ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate, SUM(failures) as failures, MAX(mtime) as mtime`).
Where("commit_id=? and (pkg!=substring_index(pkg,'/',5) or pkg like 'go-common/library/%')", commit.CommitID).Group(`substring_index(pkg, "/", 5)`).
Order(`substring_index(pkg, "/", 5)`).Find(&commit.PkgAnls).Error; err != nil {
log.Error("service.UTHistoryCommit commitID(%s) error(%v)", commit.CommitID, err)
return
}
}
for i := 0; i < len(utcmts)-1; i++ {
for _, curPkg := range utcmts[i].PkgAnls {
if utcmts[i+1] == nil || len(utcmts[i+1].PkgAnls) == 0 {
continue
}
for _, prePkg := range utcmts[i+1].PkgAnls {
if prePkg.PKG == curPkg.PKG {
curPkg.CovChange, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", curPkg.Coverage-prePkg.Coverage), 64)
}
}
}
}
return
}
// AddUT store ut data with a transaction
func (s *Service) AddUT(c context.Context, pkg *ut.PkgAnls, files []*ut.File, request *ut.UploadRes, dataURL, reportURL, HTMLURL string) (err error) {
var (
tx = s.DB.Begin()
// pkgCoverage float64
)
if err = s.AddUTMerge(c, tx, request); err != nil {
tx.Rollback()
return
}
if err = s.AddUTCommit(c, tx, request); err != nil {
tx.Rollback()
return
}
if err = s.AddUTPKG(c, tx, pkg, request, reportURL, HTMLURL, dataURL); err != nil {
tx.Rollback()
return
}
if err = s.AddUTFiles(c, tx, files); err != nil {
tx.Rollback()
return
}
tx.Commit()
return
}
// AddUTMerge insert merge if not exist
func (s *Service) AddUTMerge(c context.Context, tx *gorm.DB, request *ut.UploadRes) (err error) {
var (
uts = make([]*ut.Merge, 0)
merge = &ut.Merge{
UserName: request.UserName,
MergeID: request.MergeID,
}
)
if err = tx.Where("merge_id=?", merge.MergeID).Order("mtime desc").Find(&uts).Error; err != nil {
log.Error("service.CreateUtList err(%v)", err)
return
}
if len(uts) > 0 {
return
}
if err = tx.Create(merge).Error; err != nil {
log.Error("service.CreateUtInfo err (%v)", err)
}
return
}
// AddUTPKG if not exists pkg.commit_id then insert,otherwise,update
func (s *Service) AddUTPKG(c context.Context, tx *gorm.DB, pkg *ut.PkgAnls, request *ut.UploadRes, reportURL, HTMLURL, dataURL string) (err error) {
var (
pkgs = make([]*ut.PkgAnls, 0)
in = map[string]interface{}{
"merge_id": request.MergeID,
"failures": pkg.Failures,
"passed": pkg.Passed,
"assertions": pkg.Assertions,
"panics": pkg.Panics,
"skipped": pkg.Skipped,
"coverage": pkg.Coverage,
"html_url": HTMLURL,
"report_url": reportURL,
"data_url": dataURL,
"mtime": xtime.Time(time.Now().Unix())}
)
pkg.MergeID = request.MergeID
pkg.PKG = request.PKG
pkg.CommitID = request.CommitID
pkg.HTMLURL = HTMLURL
pkg.ReportURL = reportURL
pkg.DataURL = dataURL
if err = tx.Where("commit_id=? and pkg=?", pkg.CommitID, pkg.PKG).Find(&pkgs).Error; err != nil {
log.Error("service.CreateUtPKG query error(%v)", err)
return
}
if len(pkgs) > 0 {
log.Info("s.CreateUtPKG commit_id(%s) pkg(%+v)", pkg.CommitID, pkg)
err = tx.Table(_utpkg).Where("commit_id=? and pkg=?", pkg.CommitID, pkg.PKG).Update(in).Error
} else {
err = tx.Create(pkg).Error
}
if err != nil {
log.Error("service.CreateUtPKG create error(%v)", err)
}
return
}
// AddUTFiles add files in table ut_file
func (s *Service) AddUTFiles(c context.Context, tx *gorm.DB, files []*ut.File) (err error) {
var (
recordFiles []*ut.File
)
if len(files) == 0 {
err = fmt.Errorf("length of files is 0")
log.Error("apmSvc.AddUTFiles Error(%v)", err)
return
}
if err = tx.Where("commit_id=? and pkg=?", files[0].CommitID, files[0].PKG).Find(&recordFiles).Error; err != nil {
log.Error("apmSvc.AddUTFiles Error(%v)", err)
return
}
if len(recordFiles) > 0 {
return
}
for _, file := range files {
if err = tx.Create(file).Error; err != nil {
log.Error("service.AddUTFiles create error(%v)", err)
return
}
}
return
}
// AddUTCommit insert commit if not exist
func (s *Service) AddUTCommit(c context.Context, tx *gorm.DB, request *ut.UploadRes) (err error) {
var (
cts = make([]*ut.Commit, 0)
ct = &ut.Commit{
CommitID: request.CommitID,
MergeID: request.MergeID,
}
)
if request.Author == "" || request.Author == "null" {
ct.UserName = request.UserName
} else {
ct.UserName = request.Author
}
if err = tx.Where("commit_id=?", ct.CommitID).Find(&cts).Error; err != nil {
log.Error("s.CreateUtCommit query error(%v)", err)
return
}
if len(cts) > 0 {
log.Info("s.AddUTCommit(%d) update merge request", ct.MergeID)
err = tx.Table(_utCommit).Where("commit_id=?", ct.CommitID).Update("merge_id", ct.MergeID).Error
} else {
err = tx.Create(ct).Error
}
if err != nil {
log.Error("s.CreateUtCommit(%+v) error(%v)", ct, err)
}
return
}
// SetMerged set is_merged = 1 in ut_merge
func (s *Service) SetMerged(c context.Context, mrid int64) (err error) {
if err = s.dao.DB.Table(_utMerge).Where("merge_id=?", mrid).Update("is_merged", 1).Error; err != nil {
log.Error("s.SetMerged error(%v)", err)
return
}
return
}
// CheckUT .coverage>=30% && pass_rate=100% && increase >=0
func (s *Service) CheckUT(c context.Context, cid string) (t *ut.Tyrant, err error) {
var (
pkgs = make([]*ut.PkgAnls, 0)
)
if err = s.DB.Where("commit_id=? and (pkg!=substring_index(pkg,'/',5) or pkg like 'go-common/library/%')", cid).Find(&pkgs).Error; err != nil {
log.Error("s.Tyrant query by commit_id error(%v)", err)
return
}
if len(pkgs) == 0 {
err = fmt.Errorf("pkgs is none")
log.Error("s.Tyrant query by commit_id error(%v)", err)
return
}
for _, pkg := range pkgs {
if t, err = s.tyrant(pkg); err != nil {
return
}
if t.Tyrant {
break
}
}
return
}
// tyrant determine whether the standard is up to standard
func (s *Service) tyrant(pkg *ut.PkgAnls) (t *ut.Tyrant, err error) {
var (
pkgs = make([]*ut.PkgAnls, 0)
curCover = pkg.Coverage / 100
preCover float64
)
if err = s.DB.Select("commit_id, coverage").Where("pkg=?", pkg.PKG).Order("coverage desc").Find(&pkgs).Error; err != nil {
log.Error("s.Tyrant query by pkg error(%v)", err)
return
}
if len(pkgs) == 0 {
return
} else if len(pkgs) == 1 || pkg.CommitID != pkgs[0].CommitID {
preCover = pkgs[0].Coverage / 100
} else if pkg.CommitID == pkgs[0].CommitID {
preCover = pkgs[1].Coverage / 100
}
t = &ut.Tyrant{
Package: pkg.PKG,
Coverage: curCover,
PassRate: float64(pkg.Passed) / float64(pkg.Assertions) * 100,
Tyrant: curCover < float64(s.c.UTBaseLine.Coverage) || pkg.Passed != pkg.Assertions,
Standard: s.c.UTBaseLine.Coverage,
Increase: curCover - preCover,
LastCID: pkgs[0].CommitID,
}
return
}
//QATrend give single user's coverage ,passrate and score data
func (s *Service) QATrend(c context.Context, arg *ut.QATrendReq) (trend *ut.QATrendResp, err error) {
var (
date, group, order string
details []*ut.PkgAnls
where = "1=1 and (ut_pkganls.pkg!=substring_index(ut_pkganls.pkg,'/',5) or ut_pkganls.pkg like 'go-common/library/%')"
)
if arg.User != "" {
where += fmt.Sprintf(" and ut_commit.username = '%s'", arg.User)
}
if arg.StartTime != 0 && arg.EndTime != 0 {
where += fmt.Sprintf(" and ut_pkganls.mtime >= '%s' and ut_pkganls.mtime <= '%s'", time.Unix(arg.StartTime, 0).Format("2006-01-02 15:04:05"), time.Unix(arg.EndTime, 0).Format("2006-01-02 15:04:05"))
} else {
where += fmt.Sprintf(" and ut_pkganls.mtime >= '%s'", time.Now().AddDate(0, 0, -arg.LastTime))
}
if arg.Period == "hour" {
order += "SUBSTRING(ut_pkganls.mtime,11,2) asc"
} else {
order += "mtime asc"
}
if arg.Period == "day" {
group += "LEFT(ut_pkganls.mtime,10)"
} else {
group += fmt.Sprintf("%s(ut_pkganls.mtime)", arg.Period)
}
if err = s.DB.Table(_utpkg).Joins("join ut_commit on ut_pkganls.commit_id = ut_commit.commit_id").
Select(`ROUND(AVG(coverage/100),2) AS coverage,ROUND(SUM(passed)/SUM(assertions)*100,2) AS pass_rate, ROUND((AVG(coverage)/100*(SUM(passed)/SUM(assertions))),2) AS score, MAX(ut_pkganls.mtime) AS mtime,GROUP_CONCAT(Distinct ut_commit.commit_id) AS cids`).
Where(where).Group(group).Order(order).Find(&details).Error; err != nil {
log.Error("service.QATrend find detail error(%v)", err)
return
}
trend = &ut.QATrendResp{BaseLine: conf.Conf.UTBaseLine.Coverage}
for _, detail := range details {
switch arg.Period {
case "hour":
date = fmt.Sprintf("%d时", detail.MTime.Time().Hour())
case "month":
date = detail.MTime.Time().Month().String()
default:
date = detail.MTime.Time().Format("01-02")
}
trend.Dates = append(trend.Dates, date)
trend.Coverages = append(trend.Coverages, detail.Coverage)
trend.PassRates = append(trend.PassRates, detail.PassRate)
trend.Scores = append(trend.Scores, detail.Score)
trend.CommitIDs = append(trend.CommitIDs, detail.Cids)
}
return
}
//UTGernalCommit get singe or all users' general commit infos
func (s *Service) UTGernalCommit(c context.Context, commits string) (cmInfos []*ut.CommitInfo, err error) {
if err = s.DB.Table(_utpkg).Joins("join ut_commit on ut_pkganls.commit_id = ut_commit.commit_id").
Select("ROUND(AVG(coverage/100),2) as coverage,ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate,ut_pkganls.merge_id,ut_pkganls.commit_id,ut_pkganls.mtime").
Where("ut_pkganls.commit_id in (?) and (pkg!=substring_index(ut_pkganls.pkg,'/',5)", strings.Split(commits, ",")).Group("commit_id").Order("mtime DESC").Find(&cmInfos).Error; err != nil {
log.Error("service.UTGernalCommit get cmInfos error(%v)", err)
return
}
for i, cmInfo := range cmInfos {
cmInfos[i].GitlabCommit, _ = s.dao.GitLabCommits(context.Background(), cmInfo.CommitID)
}
return
}
// CommentOnMR create or modify comment on relavent merge request
func (s *Service) CommentOnMR(c context.Context, projID, mrID int, msg string) (err error) {
var (
mr = &ut.Merge{}
noteID int
)
if err = s.DB.Where("merge_id=?", mrID).First(mr).Error; err != nil {
log.Error("apmSvc.GitReport error search ut_merge error(%v)", err)
return
}
if mr.NoteID == 0 {
if noteID, err = s.gitlab.CreateMRNote(projID, mrID, msg); err != nil {
log.Error("apmSvc.GitReport error CreateMRNote error(%v)", err)
return
}
if err = s.DB.Table(_utMerge).Where("merge_id=?", mrID).Update("note_id", noteID).Error; err != nil {
log.Error("apmSvc.GitReport error update ut_merge error(%v)", err)
return
}
} else {
if err = s.gitlab.UpdateMRNote(projID, mrID, mr.NoteID, msg); err != nil {
log.Error("apmSvc.GitReport error CreateMRNote error(%v)", err)
return
}
}
return
}
// CommitHistory .
func (s *Service) CommitHistory(c context.Context, username string, times int64) (pkgs []*ut.PkgAnls, err error) {
pkgs = make([]*ut.PkgAnls, 0)
if err = s.DB.Table("ut_pkganls p").
Select("p.merge_id, p.commit_id, group_concat( p.pkg ) AS pkg,group_concat( ROUND( p.coverage / 100, 2 ) ) AS coverages,group_concat( ROUND( p.passed / p.assertions * 100, 2 ) ) AS pass_rates,p.mtime").
Joins("left join ut_commit c on c.commit_id=p.commit_id").Where(" c.username=? AND (p.pkg!=substring_index(p.pkg,'/',5) or p.pkg like 'go-common/library/%')", username).
Group("p.commit_id").Order("p.mtime desc").Limit(times).Find(&pkgs).Error; err != nil {
log.Error("s.CommitHistory query error(%v)", err)
}
return
}

158
app/admin/main/apm/service/ut_app.go Normal file
View File

@@ -0,0 +1,158 @@
package service
import (
"context"
"fmt"
"strings"
"go-common/app/admin/main/apm/model/ut"
"go-common/library/log"
)
var (
_upsertUtApp = "INSERT INTO ut_app (path,owner) VALUES %s"
)
// AppsCache flush cache for apps.
func (s *Service) AppsCache(c context.Context) (err error) {
var (
appSlice []*ut.App
appMap = make(map[string]*ut.App)
ownerMap = make(map[string][]*ut.App)
deptMap = make(map[string]*ut.Department)
)
if err = s.DB.Table("ut_app").Find(&appSlice).Error; err != nil {
log.Error("s.AppsCache.Find() error(%v)", err)
return
}
for _, app := range appSlice {
appMap[app.Path] = app
owners := strings.Split(app.Owner, ",")
for _, owner := range owners {
ownerMap[owner] = append(ownerMap[owner], app)
}
pathSlice := strings.Split(app.Path, "/")
if len(pathSlice) < 2 {
continue
}
deptName := pathSlice[len(pathSlice)-2]
if deptMap[deptName] == nil {
deptMap[deptName] = &ut.Department{}
}
deptMap[deptName].Name = deptName
deptMap[deptName].Total++
if app.HasUt == 1 {
deptMap[deptName].Access++
deptMap[deptName].Coverage += app.Coverage
}
}
for _, v := range deptMap {
if v.Access > 0 {
v.Coverage = v.Coverage / float64(v.Access)
}
}
s.appsCache.Lock()
s.appsCache.Map = appMap
s.appsCache.Slice = appSlice
s.appsCache.Owner = ownerMap
s.appsCache.Dept = deptMap
s.appsCache.Unlock()
return
}
// AddUTApp add path to ut_app
func (s *Service) AddUTApp(c context.Context, apps []*ut.App) (err error) {
var (
valueStrings []string
valueArgs []interface{}
)
s.appsCache.Lock()
for _, app := range apps {
var cache, ok = s.appsCache.Map[app.Path]
if !ok || cache == nil {
valueStrings = append(valueStrings, "(?,?)")
valueArgs = append(valueArgs, app.Path)
valueArgs = append(valueArgs, app.Owner)
continue
}
if cache.Owner == app.Owner {
continue
}
if err = s.DB.Table("ut_app").Where("ID=?", cache.ID).
Update("owner", app.Owner).Error; err != nil {
log.Error("AddUTApp err (%v)", err)
return
}
cache.Owner = app.Owner
}
s.appsCache.Unlock()
if len(valueStrings) == 0 {
return
}
stmt := fmt.Sprintf(_upsertUtApp, strings.Join(valueStrings, ","))
if err = s.DB.Exec(stmt, valueArgs...).Error; err != nil {
return
}
// update AppsCache
if err = s.AppsCache(c); err != nil {
return
}
return
}
// UpdateUTApp update has_ut=1
func (s *Service) UpdateUTApp(c context.Context, pkg *ut.PkgAnls) (err error) {
s.appsCache.Lock()
defer s.appsCache.Unlock()
path := paserPkg(pkg.PKG)
app, ok := s.appsCache.Map[path]
if !ok || (app.HasUt != 0 && app.Coverage == pkg.Coverage) {
log.Info("s.UpdateUTApp(%s) skiped.", pkg.PKG)
return
}
app.HasUt = 1
app.Coverage = pkg.Coverage
if err = s.DB.Table("ut_app").Where("ID=?", app.ID).Updates(app).Error; err != nil {
log.Error("UpdateUTApp err (%v)", err)
return
}
return
}
func paserPkg(pkg string) (path string) {
temp := strings.Split(pkg, "/")
if len(temp) < int(5) {
path = pkg
return
}
path = strings.Join(temp[0:5], "/")
return
}
// UTApps .
func (s *Service) UTApps(c context.Context, v *ut.AppReq) (result []*ut.App, count int, err error) {
if v.Path != "" {
if err = s.DB.Table("ut_app").Where("path LIKE ?", "%"+v.Path+"%").
Count(&count).Find(&result).Error; err != nil {
log.Error("UtProject err (%v)", err)
return
}
} else {
if err = s.DB.Table("ut_app").Where("has_ut=?", v.HasUt).Count(&count).Offset((v.Pn - 1) * v.Ps).
Limit(v.Ps).Find(&result).Error; err != nil {
log.Error("UtProject err (%v)", err)
return
}
}
for _, v := range result {
v.Link = parsePath(v.Path)
}
return
}
// "go-common/app/service/main/share" to "go-common/tree/master/app/service/main/share"
func parsePath(path string) (link string) {
temp := strings.SplitN(path, "/", 2)
link = fmt.Sprintf("%s%s%s", temp[0], "/tree/master/", temp[1])
return
}

100
app/admin/main/apm/service/ut_dash.go Normal file
View File

@@ -0,0 +1,100 @@
package service
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"go-common/app/admin/main/apm/model/ut"
"go-common/library/log"
)
// DashCurveGraph is a curve graph for leader to show the team members' code coverage
func (s *Service) DashCurveGraph(c context.Context, username string, req *ut.PCurveReq) (res []*ut.PCurveResp, err error) {
s.appsCache.Lock()
paths := s.appsCache.PathsByOwner(username)
s.appsCache.Unlock()
if err = s.DB.Table("ut_pkganls").Select("SUBSTRING_INDEX(pkg, '/', 5) AS pkg, ROUND(AVG(coverage/100),2) as coverage, SUM(assertions) as assertions, SUM(panics) as panics, SUM(passed) as passed, ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate, SUM(failures) as failures, mtime").
Where("mtime BETWEEN ? AND ? AND SUBSTRING_INDEX(pkg, '/', 5) IN (?) AND (pkg!=SUBSTRING_INDEX(pkg, '/', 5) OR p.pkg like 'go-common/library/%')", time.Unix(req.StartTime, 0).Format("2006-01-02"), time.Unix(req.EndTime, 0).Format("2006-01-02"), paths).
Group("date(mtime),SUBSTRING_INDEX(pkg, '/', 5)").Order("mtime DESC").Find(&res).Error; err != nil {
log.Error("s.ProjectCurveGraph execute sql error(%v)", err)
return
}
return
}
// DashGraphDetail project graph detail for members.
func (s *Service) DashGraphDetail(c context.Context, username string, req *ut.PCurveReq) (res []*ut.PCurveDetailResp, err error) {
var (
paths []string
)
if req.Path == "" || req.Path == "All" {
s.appsCache.Lock()
paths = s.appsCache.PathsByOwner(username)
s.appsCache.Unlock()
} else {
paths = append(paths, req.Path)
}
if err = s.DB.Table("ut_pkganls").Select("ROUND(AVG(coverage/100),2) as coverage, SUM(assertions) as assertions, SUM(panics) as panics, SUM(passed) as passed, ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate, SUM(failures) as failures, ut_commit.mtime, ut_commit.username").
Joins("INNER JOIN ut_commit ON ut_commit.commit_id=ut_pkganls.commit_id").
Where("ut_commit.mtime BETWEEN ? AND ? AND SUBSTRING_INDEX(pkg, '/', 5) IN (?) AND pkg!=SUBSTRING_INDEX(pkg, '/', 5)", time.Unix(req.StartTime, 0).Format("2006-01-02"), time.Unix(req.EndTime, 0).Format("2006-01-02"), paths).
Group("ut_commit.username").Find(&res).Error; err != nil {
log.Error("s.ProjectGraphDetail execute sql error(%v)", err)
return
}
return
}
// DashGraphDetailSingle project graph detail for Single member.
func (s *Service) DashGraphDetailSingle(c context.Context, username string, req *ut.PCurveReq) (res []*ut.PCurveDetailResp, err error) {
s.appsCache.Lock()
paths := s.appsCache.PathsByOwner(username)
s.appsCache.Unlock()
if err = s.DB.Table("ut_pkganls").Select("SUBSTRING_INDEX(pkg, '/', 5) AS pkg, ROUND(AVG(coverage/100),2) as coverage, SUM(assertions) as assertions, SUM(panics) as panics, SUM(passed) as passed, ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate, SUM(failures) as failures, ut_commit.mtime, ut_commit.username").
Joins("INNER JOIN ut_commit ON ut_commit.commit_id=ut_pkganls.commit_id").
Where("ut_commit.mtime BETWEEN ? AND ? AND SUBSTRING_INDEX(pkg, '/', 5) IN (?) AND ut_commit.username=? AND (pkg!=SUBSTRING_INDEX(pkg, '/', 5) OR p.pkg like 'go-common/library/%')", time.Unix(req.StartTime, 0).Format("2006-01-02"), time.Unix(req.EndTime, 0).Format("2006-01-02"), paths, req.User).
Group("SUBSTRING_INDEX(pkg,'/',5)").Find(&res).Error; err != nil {
log.Error("s.ProjectGraphDetailSingle execute sql error(%v)", err)
return
}
return
}
// DashPkgsTree get all the lastest committed pkgs by username
func (s *Service) DashPkgsTree(c context.Context, path string, username string) (pkgs []*ut.PkgAnls, err error) {
var (
mtime = time.Now().Add(-time.Hour * 24 * 30) //two week
count = strconv.Itoa(strings.Count(path, "/") + 1)
hql = fmt.Sprintf("select id,commit_id,merge_id,concat(substring_index(pkg,'/',%s),'/') as pkg,ROUND(AVG(coverage/100),2) as coverage,ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate,sum(panics) as panics,sum(failures) as failures,sum(skipped) as skipped,sum(passed) as passed,sum(assertions) as assertions,html_url,report_url,ctime,mtime from `ut_pkganls` as tmp where tmp.mtime>'%s' and tmp.mtime = (select max(`ut_pkganls`.mtime) from `ut_pkganls` inner join `ut_commit` on ut_commit.`commit_id`=ut_pkganls.`commit_id` where pkg = tmp.pkg and username='%s') group by substring_index(pkg,'/',%s) having pkg like '%s%%'",
count, mtime, username, count, path)
)
if err = s.DB.Raw(hql).Find(&pkgs).Error; err != nil {
log.Error("apm.Svc DashboardPkgs error(%v)", err)
return
}
for _, pkg := range pkgs {
if strings.HasSuffix(pkg.PKG, ".go") {
continue
}
if path == pkg.PKG {
if strings.HasPrefix(path, "go-common/app") && strings.Count(path, "/") < 6 {
continue
}
var files []*ut.PkgAnls
if files, err = s.dao.ParseUTFiles(c, pkg.HTMLURL); err != nil {
log.Error("apm.Svc DashboardPkgs error(%v)", err)
return
}
if len(files) == 0 {
return nil, fmt.Errorf("Get .go files error. Please check your hosts")
}
*pkg = *files[0]
pkgs = append(pkgs, files[1:]...)
}
}
return
}

121
app/admin/main/apm/service/ut_dash_test.go Normal file
View File

@@ -0,0 +1,121 @@
package service
import (
"context"
"testing"
"go-common/app/admin/main/apm/model/ut"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceDashCurveGraph(t *testing.T) {
convey.Convey("ProjectCurveGraph", t, func(ctx convey.C) {
var (
c = context.Background()
req = &ut.PCurveReq{
StartTime: 1536508800,
EndTime: 1541779200,
}
username = "fengshanshan"
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := svr.DashCurveGraph(c, username, req)
for _, r := range res {
t.Logf("res:%+v", r)
}
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceDashGraphDetail(t *testing.T) {
convey.Convey("ProjectGraphDetail", t, func(ctx convey.C) {
var (
c = context.Background()
username = "haoguanwei"
req = &ut.PCurveReq{
StartTime: 1536508800,
EndTime: 1541779200,
Path: "go-common/app/admin/main/apm",
}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := svr.DashGraphDetail(c, username, req)
for _, r := range res {
t.Logf("res:%+v", r)
}
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceDashGraphDetailSingle(t *testing.T) {
convey.Convey("ProjectGraphDetailSingle", t, func(ctx convey.C) {
var (
c = context.Background()
username = "haoguanwei"
req = &ut.PCurveReq{
User: "fengshanshan",
StartTime: 1536508800,
EndTime: 1541779200,
}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := svr.DashGraphDetailSingle(c, username, req)
for _, r := range res {
t.Logf("res:%+v\n", r)
}
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceDashPkgs(t *testing.T) {
convey.Convey("DashboardPkgs", t, func(ctx convey.C) {
var (
c = context.Background()
path = "go-common/app/"
username = "zhaobingqing"
)
ctx.Convey("When path is none", func(ctx convey.C) {
val, err := svr.DashPkgsTree(c, path, username)
ctx.Convey("Error should be nil, pkgs should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(val, convey.ShouldNotBeNil)
t.Logf("the val are %+v", val)
})
})
// ctx.Convey("When path is not none", func(ctx convey.C) {
// path = "go-common/app/service/main/block"
// val, err := svr.GetPersonalPkgs(c, path, username)
// ctx.Convey("Error should be nil, pkgs should not be nil", func(ctx convey.C) {
// ctx.So(err, convey.ShouldBeNil)
// ctx.So(val, convey.ShouldNotBeNil)
// t.Logf("the val are %+v", val)
// })
// })
})
}
func TestServiceAppsCache(t *testing.T) {
convey.Convey("AppsCache", t, func(ctx convey.C) {
var (
c = context.Background()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := svr.AppsCache(c)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}

106
app/admin/main/apm/service/ut_rank.go Normal file
View File

@@ -0,0 +1,106 @@
package service
import (
"context"
"fmt"
"math"
"sort"
"strconv"
"time"
"go-common/app/admin/main/apm/model/ut"
"go-common/library/log"
xtime "go-common/library/time"
)
// RankTen find data and avg(coverage)
func (s *Service) RankTen(c context.Context, order string) (ranks []*ut.RankResp, err error) {
s.ranksCache.Lock()
allRanks := s.ranksCache.Slice
s.ranksCache.Unlock()
ranksLen := len(allRanks)
if ranksLen == 0 {
log.Info("s.RankTen order(%s) ranks is empty", order)
return
}
if ranksLen > 10 {
ranksLen = 10
}
if order == "desc" {
sort.Slice(allRanks, func(i, j int) bool { return allRanks[i].Score > allRanks[j].Score })
} else {
sort.Slice(allRanks, func(i, j int) bool { return allRanks[i].Score < allRanks[j].Score })
}
ranks = append(ranks, allRanks[0:ranksLen]...)
return
}
// UserRank return one's rank
func (s *Service) UserRank(c context.Context, username string) (rank *ut.RankResp, err error) {
s.ranksCache.Lock()
allRanks := s.ranksCache.Map
s.ranksCache.Unlock()
if len(allRanks) == 0 {
log.Info("s.RankTen(%s) ranks is empty", username)
return
}
rank = allRanks[username]
return
}
// RanksCache flush cache for ranks.
func (s *Service) RanksCache(c context.Context) (err error) {
s.ranksCache.Lock()
defer s.ranksCache.Unlock()
var (
rankSlice []*ut.RankResp
rankMap = make(map[string]*ut.RankResp)
endTime = time.Now().AddDate(0, -3, 0).Format("2006-01-02 15:04:05")
)
if err = s.DB.Table("ut_merge").Raw("select * from (select ut_commit.username,ROUND(avg(coverage)/100,2) AS coverage,ROUND(SUM(passed)/SUM(assertions),2)*100 AS pass_rate, SUM(assertions)AS assertions, SUM(passed) AS passed, MAX(ut_commit.mtime) as mtime from ut_merge,ut_commit,ut_pkganls where ut_merge.merge_id=ut_commit.merge_id and ut_commit.commit_id=ut_pkganls.commit_id and ut_merge.is_merged=1 and ut_merge.mtime >=? and (ut_pkganls.pkg!=substring_index(ut_pkganls.pkg,'/',5) or ut_pkganls.pkg like 'go-common/library/%') GROUP BY ut_commit.username) as t1", endTime).
Find(&rankSlice).Error; err != nil {
log.Error("RankResult error(%v)", err)
return
}
total := len(rankSlice)
for _, rank := range rankSlice {
rank.Total = total
rank.Newton = NewtonScore(rank.Mtime)
Score := rank.Coverage * WilsonScore(rank.Passed, rank.Assertions) * rank.Newton
rank.Score, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", Score), 64)
}
// desc
sort.Slice(rankSlice, func(i, j int) bool { return rankSlice[i].Score > rankSlice[j].Score })
for index, rank := range rankSlice {
rank.Rank = index + 1
rank.AvatarURL, _ = s.dao.GitLabFace(c, rank.UserName)
rankMap[rank.UserName] = rank
if history, ok := s.ranksCache.Map[rank.UserName]; ok {
rank.Change = history.Rank - rank.Rank
}
}
s.ranksCache.Map = rankMap
s.ranksCache.Slice = rankSlice
return
}
//WilsonScore Wilson-score-interval
func WilsonScore(pos int, total int) (score float64) {
z := float64(1.96)
posRat := float64(pos) / float64(total)
score = (posRat + math.Pow(z, 2)/(2*float64(total))) / (1 + math.Pow(z, 2)/float64(total))
return
}
// NewtonScore . Newton's Law of Cooling
// 冷却因子经我们需求计算 a = 0.02
// deltaDays<7 day -a*x^2+b >7 NewTonScore
func NewtonScore(maxMtime xtime.Time) (score float64) {
deltaDays := int(time.Since(maxMtime.Time()).Hours() / 24)
if deltaDays <= int(7) {
score = -(float64(0.0012))*math.Pow(float64(deltaDays), 2) + float64(1)
return
}
score = math.Exp(-(float64(0.02) * float64(deltaDays)))
return
}

24
app/admin/main/apm/service/ut_rank_test.go Normal file
View File

@@ -0,0 +1,24 @@
package service
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestService_RankTen(t *testing.T) {
convey.Convey("RankTen", t, func() {
_, err := svr.RankTen(context.Background(), "desc")
convey.So(err, convey.ShouldBeNil)
})
}
func TestService_UserRank(t *testing.T) {
username := "hedan"
convey.Convey("UserRank", t, func() {
_, err := svr.UserRank(context.Background(), username)
convey.So(err, convey.ShouldBeNil)
})
}

197
app/admin/main/apm/service/ut_report.go Normal file
View File

@@ -0,0 +1,197 @@
package service
import (
"context"
"fmt"
"go-common/app/admin/main/apm/model/ut"
"go-common/library/log"
"sort"
"strings"
"time"
"github.com/jinzhu/gorm"
)
// GitReport post simple report using SAGA account
func (s *Service) GitReport(c context.Context, projID int, mrID int, commitID string) (err error) {
var (
pkgs = make([]*ut.PkgAnls, 0)
row = `<tr><td colspan="2"><a href="http://sven.bilibili.co/#/ut/detail?commit_id=%s&pkg=%s">%s</a></td><td>%.2f</td><td>%.2f</td><td>%.2f</td><td style="text-align: center">%s</td></tr>`
msg = fmt.Sprintf(`<pre><h4>单元测试速报</h4>(Commit:<a href="http://sven.bilibili.co/#/ut?merge_id=%d&pn=1&ps=20">%s</a>)</pre>`, mrID, commitID) + `<table><thead><tr><th colspan="2">包名</th><th>覆盖率(%%)</th><th>通过率(%%)</th><th>覆盖率较历史最高(%%)</th><th>是否合格</th></tr></thead><tbody>%s</tbody>%s</table>`
rows string
root = ""
file = &ut.File{}
)
if err = s.DB.Where("commit_id=? AND (pkg!=substring_index(pkg, '/', 5) OR pkg like 'go-common/library/%')", commitID).Find(&pkgs).Error; err != nil {
log.Error("apmSvc.GitReport query error(%v)", err)
return
}
for _, pkg := range pkgs {
t, _ := s.tyrant(pkg)
app := pkg.PKG
if !strings.Contains(pkg.PKG, "/library") && len(pkg.PKG) >= 5 {
app = strings.Join(strings.Split(pkg.PKG, "/")[:5], "/")
}
rows += fmt.Sprintf(row, pkg.CommitID, app, pkg.PKG, pkg.Coverage/100, t.PassRate, t.Increase, "%s")
if t.Tyrant {
rows = fmt.Sprintf(rows, "❌")
} else {
rows = fmt.Sprintf(rows, "✔️")
}
}
if err = s.DB.Select(`count(id) as id, commit_id, sum(statements) as statements, sum(covered_statements) as covered_statements`).
Where(`pkg!=substring_index(pkg, "/", 5) OR ut_pkganls.pkg like 'go-common/library/%'`).Group(`commit_id`).Having("commit_id=?", commitID).First(file).Error; err != nil {
if err != gorm.ErrRecordNotFound {
log.Error("apmSvc.GitReport query error(%v)", err)
return
}
err = nil
} else {
root = fmt.Sprintf(`<tfoot><tr><td><b>总覆盖率: </b>%.2f%%<br><b>总检测文件数: </b>%d</br></td><td><b>Tracked语句数: </b>%d<br><b>覆盖的语句数: </b>%d</br></td><td colspan="4" align="center">总覆盖率 = 覆盖语句数 / Tracked语句数</td></tr></tfoot>`,
float64(file.CoveredStatements)/float64(file.Statements)*100, file.ID, file.Statements, file.CoveredStatements)
}
if err = s.CommentOnMR(c, projID, mrID, fmt.Sprintf(msg, rows, root)); err != nil {
log.Error("apmSvc.GitReport call CommentOnMR error(%v)", err)
return
}
return
}
// WechatReport send wechat msg to a group when mr is merged
func (s *Service) WechatReport(c context.Context, mrid int64, cid, src, des string) (err error) {
var (
pkgs []*ut.PkgAnls
mr = &ut.Merge{}
foot = fmt.Sprintf("\n*覆盖率增长:该包本次合并最后一次commit与过往已合并记录中最大覆盖率进行比较\nMR:http://git.bilibili.co/platform/go-common/merge_requests/%d\n单测报告:http://sven.bilibili.co/#/ut?merge_id=%d&pn=1&ps=20\n", mrid, mrid)
)
if err = s.dao.DB.Where("merge_id=? AND is_merged=?", mrid, 1).First(mr).Error; err != nil {
log.Error("apmSvc.WechatReport Error(%v)", err)
return
}
msg := "【测试姬】新鲜的单测速报出炉啦ᕕ( ᐛ )ᕗ\n\n" + fmt.Sprintf("由 %s 发起的 MR (%s->%s) 合并成功！\n\n", mr.UserName, src, des)
if err = s.dao.DB.Where("commit_id=? AND (pkg!=substring_index(pkg, '/', 5) OR ut_pkganls.pkg like 'go-common/library/%')", cid).Find(&pkgs).Error; err != nil || len(pkgs) == 0 {
log.Error("apmSvc.WechatReport Error(%v)", err)
return
}
for _, pkg := range pkgs {
var (
arrow = ""
maxPkgs = make([]ut.PkgAnls, 0)
increase = pkg.Coverage / 100
file = &ut.File{}
lastFile = &ut.File{}
)
if err = s.DB.Table("ut_pkganls").Joins("left join ut_merge ON ut_merge.merge_id=ut_pkganls.merge_id").Select("ut_pkganls.commit_id, ut_pkganls.coverage").Where("ut_pkganls.pkg=? AND ut_pkganls.merge_id!=? AND ut_merge.is_merged=1", pkg.PKG, mrid).Order("coverage desc").Find(&maxPkgs).Error; err != nil {
log.Error("apmSvc.WechatReport error(%v)", err)
return
}
if len(maxPkgs) != 0 {
increase = pkg.Coverage/100 - maxPkgs[0].Coverage/100
}
if increase < float64(0) {
arrow = "⬇️"
if !strings.Contains(foot, "本次合并后有包覆盖率下降了喔~还需要再加油鸭💪~") {
foot += "\n本次合并后有包覆盖率下降了喔~还需要再加油鸭💪~"
}
} else if increase > float64(0) {
arrow = "⬆️"
}
msg += fmt.Sprintf("*%s\n\t覆盖率：%.2f%%\t覆盖率增长：%.2f%%\t%s\n", pkg.PKG, pkg.Coverage/100, increase, arrow)
if err = s.DB.Select("count(id) as id,commit_id,pkg,sum(statements) as statements,sum(covered_statements) as covered_statements").Group("commit_id,pkg").Having("commit_id=? AND pkg=?", cid, pkg.PKG).First(file).Error; err != nil {
if err == gorm.ErrRecordNotFound {
err = nil
continue
}
log.Error("apmSvc.WechatReport error(%v)", err)
return
}
msg += fmt.Sprintf("\tTracked语句数：%d\t覆盖语句数：%d\n", file.Statements, file.CoveredStatements)
if err = s.DB.Table("ut_file").Joins("left join ut_commit on ut_commit.commit_id = ut_file.commit_id left join ut_merge on ut_merge.merge_id =ut_commit.merge_id").Select("ut_file.commit_id,ut_file.pkg,count(ut_file.id) as id,sum(statements) as statements, sum(covered_statements) as covered_statements,ut_file.mtime").Where("is_merged=1 and ut_merge.merge_id!=?", mrid).Group("commit_id,pkg").Having("pkg=?", pkg.PKG).Order("mtime desc").First(lastFile).Error; err != nil {
if err == gorm.ErrRecordNotFound {
err = nil
continue
}
log.Error("apmSvc.WechatReport error(%v)", err)
return
}
msg += fmt.Sprintf("\tTracked语句增长：%d\t覆盖语句增长：%d\n", file.Statements-lastFile.Statements, file.CoveredStatements-lastFile.CoveredStatements)
}
if err = s.dao.SendWechatToGroup(c, s.c.WeChat.ChatID, msg+foot); err != nil {
log.Error("apmSvc.WechatReport Error(%v)", err)
return
}
return
}
// RankWechatReport send rank to wechat group 19:00 everyday
func (s *Service) RankWechatReport(c context.Context) (err error) {
var (
topRanks []*ut.RankResp
msg = "【测试姬】今日份的单测榜单🎉\n\n"
root = "更多详情请戳：http://sven.bilibili.co/#/ut/leaderboard\n感谢辛勤工作的一天的你☕️一起快乐下班吧☕️\n"
)
if topRanks, err = s.RankTen(c, "desc"); err != nil {
log.Error("apmSvc.RankWechatReport Error(%v)", err)
return
}
msg += "🔘TOP 10\n"
for i, r := range topRanks {
msg += fmt.Sprintf("%d: %s\t分数: %.2f\n", i+1, r.UserName, r.Score)
}
msg += "恭喜以上各位~还请继续保持哟(๑•̀ㅂ•́)و✧\n\n"
if err = s.dao.SendWechatToGroup(c, s.c.WeChat.ChatID, msg+root); err != nil {
log.Error("apmSvc.RankWechatReport Error(%v)", err)
return
}
return
}
// SummaryWechatReport send depts' summary to ut wechat group every Friday(19:00)
func (s *Service) SummaryWechatReport(c context.Context) (err error) {
var (
depts = `"main","ep","openplatform","live","video","bbq"`
msg = fmt.Sprintf("【测试姬】Kratos大仓库单元测试周刊 ( %s )\n\n>> 接入情况: \n", time.Now().Format("2006-01-02"))
covMsg = ">> 覆盖情况: \n"
rankMsg = ">> 本周Top 3应用:\n"
root = "\n\n更多汇总信息请姥爷们查看: http://sven.bilibili.co/#/ut/dashboard?tab=project\n更多文档信息: http://info.bilibili.co/pages/viewpage.action?pageId=6947230\n吐槽建议: http://sven.bilibili.co/#/suggestion/list\n"
tpAcc = "\t* %s\t接入数: %d\t总数: %d\t接入率: %.2f%%\n"
tpCov = "\t* %s\t应用平均覆盖率: %.2f%%\t增长率：%.2f%%\n"
tpProjRank = "\t%d. %s \t包平均覆盖率: %.2f%%\n"
sumTotal, sumAccess int64
sumCov float64
)
s.appsCache.Lock()
defer s.appsCache.Unlock()
for _, dept := range s.appsCache.Dept {
if !strings.Contains(depts, "\""+dept.Name+"\"") {
continue
}
sumTotal += dept.Total
sumAccess += dept.Access
msg += fmt.Sprintf(tpAcc, dept.Name, dept.Access, dept.Total, float64(dept.Access)/float64(dept.Total)*100)
if dept.Coverage == float64(0) {
continue
}
var preCoverage float64
if preCoverage, err = s.dao.GetAppCovCache(c, dept.Name); err != nil {
log.Error("service.SummaryWechatReport GetAppCov Error(%v)", err)
return
}
covMsg += fmt.Sprintf(tpCov, dept.Name, dept.Coverage/100, (dept.Coverage-preCoverage)/100)
}
sort.Slice(s.appsCache.Slice, func(i, j int) bool { return s.appsCache.Slice[i].Coverage > s.appsCache.Slice[j].Coverage })
for k, slice := range s.appsCache.Slice {
if k <= 2 {
rankMsg += fmt.Sprintf(tpProjRank, k+1, slice.Path, slice.Coverage/100)
}
if slice.HasUt == 1 {
sumCov += slice.Coverage
}
}
msg += fmt.Sprintf("\t总接入情况 - 接入应用数: %d，应用总数: %d\n\n", sumAccess, sumTotal) + covMsg + fmt.Sprintf("\t总覆盖情况 - 应用平均覆盖率: %.2f%%\n\n", sumCov/float64(sumAccess)/100) + rankMsg
if err = s.dao.SendWechatToGroup(c, s.c.WeChat.ChatID, msg+root); err != nil {
log.Error("apmSvc.WechatReport Error(%v)", err)
return
}
return
}

67
app/admin/main/apm/service/ut_report_test.go Normal file
View File

@@ -0,0 +1,67 @@
package service
import (
"context"
"go-common/app/admin/main/apm/dao"
"reflect"
"testing"
"github.com/bouk/monkey"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceWechatReport(t *testing.T) {
convey.Convey("WechatReport", t, func(ctx convey.C) {
var (
c = context.Background()
mrid = int64(10760)
cid = "8d2f1b49661c7089e2b595eafff326033a138c23"
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(svr.dao), "SendWechatToGroup", func(_ *dao.Dao, _ context.Context, _ string, _ string) error {
return nil
})
defer guard.Unpatch()
err := svr.WechatReport(c, mrid, cid, "xxx", "master")
ctx.Convey("Than err should be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestServiceRankWechatReport(t *testing.T) {
convey.Convey("RankWechatReport", t, func(ctx convey.C) {
c := context.Background()
ctx.Convey("When everything goes positive", func(ctx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(svr.dao), "SendWechatToGroup", func(_ *dao.Dao, _ context.Context, _ string, _ string) error {
return nil
})
defer guard.Unpatch()
err := svr.RankWechatReport(c)
ctx.Convey("Then err should be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestServiceSummaryWechatReport(t *testing.T) {
convey.Convey("SummaryWechatReport", t, func(ctx convey.C) {
var (
c = context.Background()
)
ctx.Convey("When everything gose positive", func(ctx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(svr.dao), "SendWechatToGroup", func(_ *dao.Dao, _ context.Context, _ string, _ string) error {
return nil
})
defer guard.Unpatch()
svr.dao.SetAppCovCache(c)
err := svr.SummaryWechatReport(c)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
// t.Logf("\n%s", msg)
ctx.So(err, convey.ShouldBeNil)
})
})
})
}

152
app/admin/main/apm/service/ut_test.go Normal file
View File

@@ -0,0 +1,152 @@
package service
import (
"context"
"testing"
"go-common/app/admin/main/apm/model/ut"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceUtDetailList(t *testing.T) {
convey.Convey("UtDetailList", t, func(ctx convey.C) {
req := &ut.DetailReq{
CommitID: "4f5ca7122e023c605555ed4bddf274709635d019",
PKG: "go-common/app/admin/main/apm",
}
ctx.Convey("When everything goes positive", func(ctx convey.C) {
utpkgs, err := svr.UtDetailList(context.Background(), req)
ctx.Convey("Than utpkgs should not be nil. err should be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(utpkgs, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceUtHistoryCommit(t *testing.T) {
convey.Convey("UtHistoryCommit", t, func(ctx convey.C) {
dt := &ut.HistoryCommitReq{
MergeID: 1,
Ps: 20,
Pn: 1,
}
ctx.Convey("When everything goes possitive", func(ctx convey.C) {
utcmts, count, err := svr.UtHistoryCommit(context.Background(), dt)
ctx.Convey("Then err should be nil.count shoulde be greater than 0.utcmts should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldBeGreaterThan, 0)
ctx.So(utcmts, convey.ShouldNotBeNil)
t.Logf("history count: %+v", count)
for _, commit := range utcmts {
for _, pkg := range commit.PkgAnls {
t.Logf("%s %s cov:%.2f\n", pkg.CommitID, pkg.PKG, pkg.Coverage)
}
}
})
})
})
}
//
//func TestService_CreateUtMerge(t *testing.T) {
// convey.Convey("CreateUtMerge", t, func() {
// res := &ut.UploadRes{
// CommitID: "eaefwa",
// UserName: "ChengXing",
// MergeID: 1000,
// }
// err := svr.AddUTMerge(context.Background(), res)
// convey.So(err, convey.ShouldBeNil)
// })
//}
//
//func TestService_CreateUtCommit(t *testing.T) {
// convey.Convey("CreateUtCommit", t, func() {
// res := &ut.UploadRes{
// CommitID: "eaefwa",
// UserName: "ChengXing",
// MergeID: 1000,
// }
// err := svr.AddUTCommit(context.Background(), res)
// convey.So(err, convey.ShouldBeNil)
// })
//}
//
//func TestService_CreateUtPKG(t *testing.T) {
// convey.Convey("CreateUtPKG", t, func() {
// pkg := &ut.PkgAnls{
// CommitID: "eawfsdfeasdfe",
// PassRate: 64.12,
// PKG: "go-common/app/server/main/coin/dao",
// Assertions: 10,
// Passed: 8,
// }
// res := &ut.UploadRes{
// CommitID: "eaefwa",
// UserName: "ChengXing",
// MergeID: 1000,
// }
// err := svr.AddUTPKG(context.Background(), pkg, res, "", "")
// convey.So(err, convey.ShouldBeNil)
// })
//}
// func TestService_SAGAReport(t *testing.T) {
// convey.Convey("SAGAReport", t, func() {
// saga, err := svr.SAGAReport(context.Background(), "8c9d27c106caf6325c1155d127fcc952d1f4a441")
// convey.So(saga, convey.ShouldNotBeEmpty)
// convey.So(err, convey.ShouldBeNil)
// for _, s := range saga {
// t.Logf("Coverage:(%.02f) PKG:(%s)", s.Coverage, s.PKG)
// }
// })
// }
func TestServiceQATrend(t *testing.T) {
convey.Convey("QATrend", t, func(ctx convey.C) {
req := &ut.QATrendReq{
User: "fengshanshan",
LastTime: 30,
Period: "hour",
StartTime: 1540465325,
EndTime: 1540551725,
}
ctx.Convey("When everything goes positive", func(ctx convey.C) {
trend, err := svr.QATrend(context.Background(), req)
t.Logf("trend:%+v\n", trend)
ctx.Convey("Than trend should not be nil. err should be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(trend, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceUTGernalCommit(t *testing.T) {
convey.Convey("UTGernalCommit", t, func(ctx convey.C) {
req := "ae1377033a11ca85a19bca365af32a5b0ebea31c,324a7609f2be27ccbedf7540f970982317d6cd6b,a1e94b169c728dab26ff583f2619b35d40519752"
ctx.Convey("When everything goes positive", func(ctx convey.C) {
commits, err := svr.UTGernalCommit(context.Background(), req)
for _, commit := range commits {
t.Logf("commit:%+v\n", commit)
t.Logf("commit:%+v\n", commit.GitlabCommit)
}
ctx.Convey("Than commitInfo should not be nil. err should be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(commits, convey.ShouldNotBeNil)
})
})
})
}
func TestService_CommitHistory(t *testing.T) {
convey.Convey("CommitHistory", t, func() {
data, err := svr.CommitHistory(context.Background(), "zhaobingqing", 10)
convey.So(data, convey.ShouldNotBeEmpty)
convey.So(err, convey.ShouldBeNil)
for _, d := range data {
t.Logf("d.MergeID=%d, d.CommitID=%s, d.PKG=%s, d.Coverage=%0.2f, d.PassRate=%0.2f", d.MergeID, d.CommitID, d.PKG, d.Coverage, d.PassRate)
}
})
}