432 lines
10 KiB
Go
432 lines
10 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"regexp"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
|
||
|
|
"go-common/app/interface/openplatform/monitor-end/model"
|
||
|
|
"go-common/app/interface/openplatform/monitor-end/model/monitor"
|
||
|
|
"go-common/app/interface/openplatform/monitor-end/model/prom"
|
||
|
|
"go-common/library/log"
|
||
|
|
|
||
|
|
"github.com/json-iterator/go"
|
||
|
|
)
|
||
|
|
|
||
|
|
const _regex = `.*\d{2,}`
|
||
|
|
|
||
|
|
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||
|
|
|
||
|
|
const (
|
||
|
|
_typeAPP = "app"
|
||
|
|
_typeWeb = "web/h5"
|
||
|
|
_typeAPPH = "app_h"
|
||
|
|
_eventPage = "page"
|
||
|
|
_eventAPI = "api"
|
||
|
|
_eventResource = "resource"
|
||
|
|
_levelInfo = "info"
|
||
|
|
_levelWarning = "warning"
|
||
|
|
_levelError = "error"
|
||
|
|
_logTypeOne = "1"
|
||
|
|
_logTypeTwo = "2"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Report .
|
||
|
|
func (s *Service) Report(c context.Context, params *model.LogParams, mid int64, ip string, buvid string, userAgent string) (err error) {
|
||
|
|
var l *monitor.Log
|
||
|
|
log.Info("get log report(%+v)", params)
|
||
|
|
if params.IsAPP == 1 {
|
||
|
|
_, err = s.nativeLog(c, params.Log)
|
||
|
|
} else {
|
||
|
|
l, err = s.frontendLog(c, params.Source, params.Log, mid, ip, buvid, userAgent)
|
||
|
|
go s.promsFE(context.TODO(), l)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
func induceSubEvent(s string) string {
|
||
|
|
if strings.Contains(s, "?") {
|
||
|
|
s = s[:strings.Index(s, "?")]
|
||
|
|
}
|
||
|
|
if strings.Contains(s, "://") {
|
||
|
|
s = s[(strings.Index(s, "://") + 3):]
|
||
|
|
}
|
||
|
|
if len(s) > 512 {
|
||
|
|
s = s[:512]
|
||
|
|
}
|
||
|
|
var (
|
||
|
|
ok bool
|
||
|
|
err error
|
||
|
|
)
|
||
|
|
for {
|
||
|
|
if ok, err = regexp.Match(_regex, []byte(s)); err != nil {
|
||
|
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, s)
|
||
|
|
return s
|
||
|
|
}
|
||
|
|
if !ok {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if !strings.Contains(s, "/") {
|
||
|
|
return s
|
||
|
|
}
|
||
|
|
s = s[:strings.LastIndex(s, "/")]
|
||
|
|
}
|
||
|
|
return s
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) promsFE(c context.Context, l *monitor.Log) {
|
||
|
|
if !s.c.Prom.Promed || l == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
var (
|
||
|
|
ok bool
|
||
|
|
err error
|
||
|
|
)
|
||
|
|
l.SubEvent = induceSubEvent(l.SubEvent)
|
||
|
|
if l.Event == _eventResource {
|
||
|
|
regex := `.*i\d{1}.hdslb.com`
|
||
|
|
if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
|
||
|
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if ok {
|
||
|
|
l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
|
||
|
|
}
|
||
|
|
prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
prom.AddHTTPCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
|
||
|
|
prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
|
||
|
|
if l.Details == nil {
|
||
|
|
var cost int64
|
||
|
|
if l.Duration == "" {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
|
||
|
|
log.Warn("s.Info.ParseInt can not convert duration(%s) to int64", l.Duration)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
prom.AddCommonLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
|
||
|
|
} else {
|
||
|
|
prom.AddDetailedLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.Details)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) nativeLog(c context.Context, data string) (l *monitor.Log, err error) {
|
||
|
|
l = &monitor.Log{}
|
||
|
|
if err = json.Unmarshal([]byte(data), l); err != nil {
|
||
|
|
log.Error("s.nativeLog.unmarshal error(%+v), data(%s)", err, data)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
l.Type = _typeAPPH
|
||
|
|
s.handleLog(c, l)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// HandleMsg .
|
||
|
|
func (s *Service) HandleMsg(msg []byte) {
|
||
|
|
l := monitor.LogFromBytes(msg)
|
||
|
|
l.Type = _typeAPP
|
||
|
|
s.handleLog(context.TODO(), l)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) handleLog(c context.Context, l *monitor.Log) {
|
||
|
|
var (
|
||
|
|
appID string
|
||
|
|
kv []log.D
|
||
|
|
err error
|
||
|
|
logtype int
|
||
|
|
isInt bool
|
||
|
|
cost int64
|
||
|
|
ok bool
|
||
|
|
app = "android"
|
||
|
|
)
|
||
|
|
l.TraceidSvr = l.Traceid
|
||
|
|
l.Traceid = ""
|
||
|
|
l.CalCode()
|
||
|
|
if appID, _, kv, err = l.LogData(); err != nil {
|
||
|
|
log.Error("s.nativeLog.LogData error(%+v), data(%+v)", err, l)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if l.Result == "1" || l.Result == "" {
|
||
|
|
s.mh.Info(c, appID, kv...)
|
||
|
|
} else {
|
||
|
|
s.mh.Error(c, appID, kv...)
|
||
|
|
}
|
||
|
|
if l.SubProduct == "" {
|
||
|
|
l.SubProduct = l.Product
|
||
|
|
}
|
||
|
|
if strings.Contains(l.RequestURI, "ios") {
|
||
|
|
app = "ios"
|
||
|
|
isInt = true
|
||
|
|
}
|
||
|
|
if logtype, err = bStrToInt(l.LogType, isInt); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if s.c.Prom.IgnoreNA && !s.checkProduct(l) {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
// 丢弃老版本的network日志
|
||
|
|
if l.Event == "network" && l.Codes == "" {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
l.SubEvent = induceSubEvent(l.SubEvent)
|
||
|
|
regex := `.*i\d{1}.hdslb.com`
|
||
|
|
if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
|
||
|
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if ok {
|
||
|
|
l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
|
||
|
|
}
|
||
|
|
if (logtype & 1) == 1 {
|
||
|
|
// 性能日志
|
||
|
|
if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
|
||
|
|
log.Warn("s.handleLog.ParseInt can not convert duration(%s) to int64", l.Duration)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
prom.AddCommonLog(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
|
||
|
|
}
|
||
|
|
if (logtype >> 2 & 1) == 1 {
|
||
|
|
// 成功/失败日志
|
||
|
|
if l.Event == "network" {
|
||
|
|
// 网络类型 上报业务码和http状态
|
||
|
|
prom.AddHTTPCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
|
||
|
|
prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
|
||
|
|
} else {
|
||
|
|
// 其他类型 只上报失败成功 result=1 表示成功
|
||
|
|
res := "999"
|
||
|
|
if l.Result == "1" {
|
||
|
|
res = "0"
|
||
|
|
}
|
||
|
|
prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, res)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) frontendLog(c context.Context, source string, data string, mid int64, ip string, buvid string, userAgent string) (l *monitor.Log, err error) {
|
||
|
|
var (
|
||
|
|
ms []interface{}
|
||
|
|
kv []log.D
|
||
|
|
logtype, loglevel string
|
||
|
|
url, query string
|
||
|
|
)
|
||
|
|
if err = json.Unmarshal([]byte(data), &ms); err != nil {
|
||
|
|
log.Error("s.frontendLog.unmarshal error(%+v), data(%s)", err, data)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, m := range ms {
|
||
|
|
switch m := m.(type) {
|
||
|
|
case map[string]interface{}:
|
||
|
|
logtype = stringValueByKey(m, "1", "logtype")
|
||
|
|
loglevel = stringValueByKey(m, "info", "level")
|
||
|
|
tmp := stringValueByKey(m, "", "url")
|
||
|
|
if strings.Contains(tmp, "?") {
|
||
|
|
url = strings.Split(tmp, "?")[0]
|
||
|
|
query = strings.Split(tmp, "?")[1]
|
||
|
|
} else {
|
||
|
|
url = tmp
|
||
|
|
}
|
||
|
|
switch logtype {
|
||
|
|
case _logTypeOne:
|
||
|
|
l = &monitor.Log{
|
||
|
|
LogType: _logTypeOne,
|
||
|
|
Type: _typeWeb,
|
||
|
|
Product: source,
|
||
|
|
IP: ip,
|
||
|
|
Buvid: buvid,
|
||
|
|
UserAgent: userAgent,
|
||
|
|
Event: _eventAPI,
|
||
|
|
Mid: strconv.FormatInt(mid, 10),
|
||
|
|
SubEvent: url,
|
||
|
|
Query: query,
|
||
|
|
Duration: stringValueByKey(m, "", "cost"),
|
||
|
|
TraceidSvr: stringValueByKey(m, "", "traceid_svr"),
|
||
|
|
TraceidEnd: stringValueByKey(m, "", "traceid_end"),
|
||
|
|
HTTPCode: stringValueByKey(m, "200", "status"),
|
||
|
|
BusinessCode: stringValueByKey(m, "0", "errno", "code"),
|
||
|
|
SubProduct: stringValueByKey(m, source, "sub_product"),
|
||
|
|
Result: "0",
|
||
|
|
}
|
||
|
|
srcURL := stringValueByKey(m, "", "srcUrl")
|
||
|
|
if srcURL != "" {
|
||
|
|
l.Event = _eventResource
|
||
|
|
l.SubEvent = srcURL
|
||
|
|
l.BusinessCode = "-999"
|
||
|
|
}
|
||
|
|
if loglevel == _levelInfo {
|
||
|
|
l.Result = "1"
|
||
|
|
}
|
||
|
|
if _, _, kv, err = l.LogData(); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
kv = append(kv, extKV(m)...)
|
||
|
|
case _logTypeTwo:
|
||
|
|
var msg []byte
|
||
|
|
l = &monitor.Log{
|
||
|
|
LogType: _logTypeTwo,
|
||
|
|
Type: _typeWeb,
|
||
|
|
Product: source,
|
||
|
|
Event: _eventPage,
|
||
|
|
IP: ip,
|
||
|
|
Buvid: buvid,
|
||
|
|
UserAgent: userAgent,
|
||
|
|
Mid: strconv.FormatInt(mid, 10),
|
||
|
|
SubEvent: url,
|
||
|
|
Query: query,
|
||
|
|
HTTPCode: "200",
|
||
|
|
Details: details(m),
|
||
|
|
Result: "1",
|
||
|
|
BusinessCode: "0",
|
||
|
|
SubProduct: stringValueByKey(m, source, "sub_product"),
|
||
|
|
}
|
||
|
|
if msg, err = json.Marshal(m); err != nil {
|
||
|
|
log.Error("s.frontendLog.Marshal error(%v), data(%v)", err, m)
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
l.Message = string(msg)
|
||
|
|
if _, _, kv, err = l.LogData(); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
default:
|
||
|
|
log.Error("s.frontendLog error logype(%v)", logtype)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
switch loglevel {
|
||
|
|
case _levelInfo:
|
||
|
|
s.mh.Info(c, source, kv...)
|
||
|
|
case _levelWarning:
|
||
|
|
s.mh.Warn(c, source, kv...)
|
||
|
|
case _levelError:
|
||
|
|
s.collectFE(c, l)
|
||
|
|
s.mh.Error(c, source, kv...)
|
||
|
|
default:
|
||
|
|
s.mh.Info(c, source, kv...)
|
||
|
|
}
|
||
|
|
default:
|
||
|
|
log.Error("s.frontendLog log data is not json type: " + data)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) collectFE(c context.Context, l *monitor.Log) {
|
||
|
|
if s.c.CollectFE {
|
||
|
|
regex := `.*i\d{1}.hdslb.com`
|
||
|
|
if ok, err := regexp.Match(regex, []byte(l.SubEvent)); err != nil {
|
||
|
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
|
||
|
|
return
|
||
|
|
} else if ok {
|
||
|
|
l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
|
||
|
|
}
|
||
|
|
go s.Collect(context.TODO(), l)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func stringValueByKey(m map[string]interface{}, defValue string, keys ...string) (r string) {
|
||
|
|
r = defValue
|
||
|
|
if m == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
var (
|
||
|
|
res interface{}
|
||
|
|
ok bool
|
||
|
|
)
|
||
|
|
for _, key := range keys {
|
||
|
|
if res, ok = m[key]; ok {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if res == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
switch d := res.(type) {
|
||
|
|
case int:
|
||
|
|
r = strconv.Itoa(d)
|
||
|
|
case float64:
|
||
|
|
r = strconv.FormatFloat(d, 'f', -1, 64)
|
||
|
|
case int64:
|
||
|
|
r = strconv.FormatInt(d, 10)
|
||
|
|
case string:
|
||
|
|
r = d
|
||
|
|
default:
|
||
|
|
log.Warn("s.stringValueByKey unexcept type of value(%v)", d)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func extKV(m map[string]interface{}) (kv []log.D) {
|
||
|
|
if m == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
var keys = []string{"level", "logtype", "url", "cost", "traceid_svr", "traceid_end", "status", "code", "errno", "sub_product"}
|
||
|
|
for _, key := range keys {
|
||
|
|
delete(m, key)
|
||
|
|
}
|
||
|
|
for k, v := range m {
|
||
|
|
kv = append(kv, log.KV(k, v))
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func details(m map[string]interface{}) (res map[string]int64) {
|
||
|
|
res = make(map[string]int64)
|
||
|
|
if m == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
delete(m, "level")
|
||
|
|
delete(m, "logtype")
|
||
|
|
delete(m, "url")
|
||
|
|
for k, v := range m {
|
||
|
|
var d int64
|
||
|
|
if v == nil {
|
||
|
|
res[k] = 0
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
switch v := v.(type) {
|
||
|
|
case int64:
|
||
|
|
d = v
|
||
|
|
case float64:
|
||
|
|
d = int64(v)
|
||
|
|
case int:
|
||
|
|
d = int64(v)
|
||
|
|
case int32:
|
||
|
|
d = int64(v)
|
||
|
|
case string:
|
||
|
|
d, _ = strconv.ParseInt(v, 10, 64)
|
||
|
|
default:
|
||
|
|
log.Warn("s.details unexcept type of value(%v)", v)
|
||
|
|
}
|
||
|
|
res[k] = d
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func bStrToInt(str string, isInt bool) (r int, err error) {
|
||
|
|
if isInt {
|
||
|
|
if r, err = strconv.Atoi(str); err != nil {
|
||
|
|
log.Warn("s.bStrToInt error(%+v), string(%s)", err, str)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, i := range str {
|
||
|
|
if i == '1' {
|
||
|
|
r = r*2 + 1
|
||
|
|
} else {
|
||
|
|
r *= 2
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) checkProduct(l *monitor.Log) bool {
|
||
|
|
if s.naProducts == nil {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
if s.naProducts[l.Product] {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|