Create & Init Project...

app/job/main/aegis/dao/monitor/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "dao.go",
+        "db.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/aegis/dao/monitor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//app/job/main/aegis/model/monitor:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "api_test.go",
+        "dao_test.go",
+        "db_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//app/job/main/aegis/model/monitor:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)

app/job/main/aegis/dao/monitor/api.go

@@ -0,0 +1,40 @@
+package monitor
+
+import (
+	"context"
+	"go-common/app/job/main/aegis/model/monitor"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"net/url"
+	"strconv"
+)
+
+const (
+	_arcAdditURL = "/videoup/archive/addit"
+)
+
+// ArchiveAttr 获取稿件都附加属性
+func (d *Dao) ArchiveAttr(c context.Context, aid int64) (addit *monitor.ArchiveAddit, err error) {
+	params := url.Values{}
+	params.Set("aid", strconv.FormatInt(aid, 10))
+	var res struct {
+		Code int                   `json:"code"`
+		Data *monitor.ArchiveAddit `json:"data"`
+	}
+	if err = d.http.Get(c, d.URLArcAddit, "", params, &res); err != nil {
+		log.Error("d.ArchiveAttr(%s)  error(%v)", d.URLArcAddit+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 0 {
+		err = ecode.Int(res.Code)
+		log.Error("d.ArchiveAttr(%s) Code=(%d)", d.URLArcAddit+"?"+params.Encode(), res.Code)
+		return
+	}
+	if res.Data == nil {
+		err = ecode.NothingFound
+		log.Warn("d.ArchiveAttr(%s) Code=(%d) data nil", d.URLArcAddit+"?"+params.Encode(), res.Code)
+		return
+	}
+	addit = res.Data
+	return
+}

app/job/main/aegis/dao/monitor/api_test.go

@@ -0,0 +1,26 @@
+package monitor
+
+import (
+	"context"
+	"gopkg.in/h2non/gock.v1"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestMonitorArchiveAttr(t *testing.T) {
+	convey.Convey("ArchiveAttr", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			aid = int64(1212)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			defer gock.OffAll()
+			httpMock("GET", d.URLArcAddit).Reply(200).JSON(`{"code":0,"data":{}}`)
+			_, err := d.ArchiveAttr(c, aid)
+			convCtx.Convey("Then err should be nil.addit should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/monitor/dao.go

@@ -0,0 +1,41 @@
+package monitor
+
+import (
+	"context"
+	"go-common/app/job/main/aegis/conf"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+type Dao struct {
+	c           *conf.Config
+	redis       *redis.Pool
+	db          *xsql.DB
+	http        *bm.Client
+	URLArcAddit string
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:           c,
+		redis:       redis.NewPool(c.Redis),
+		db:          xsql.NewMySQL(c.MySQL.Fast),
+		http:        bm.NewClient(c.HTTP.Fast),
+		URLArcAddit: c.Host.Videoup + _arcAdditURL,
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return d.db.Ping(c)
+}

app/job/main/aegis/dao/monitor/dao_test.go

@@ -0,0 +1,43 @@
+package monitor
+
+import (
+	"flag"
+	"go-common/app/job/main/aegis/conf"
+	"gopkg.in/h2non/gock.v1"
+	"os"
+	"strings"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.archive.aegis-job")
+		flag.Set("conf_token", "aed3cc21ca345ffc284c6036da32352b")
+		flag.Set("tree_id", "61819")
+		flag.Set("conf_version", "1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/aegis-job.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	d.http.SetTransport(gock.DefaultTransport)
+	return r
+}

app/job/main/aegis/dao/monitor/db.go

@@ -0,0 +1,74 @@
+package monitor
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/pkg/errors"
+	"go-common/app/job/main/aegis/model/monitor"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	"time"
+)
+
+const (
+	_rulesByBid    = "SELECT id,type,bid,name,state,stime,etime,rule,uid,ctime,mtime FROM monitor_rule WHERE bid = ? AND state = 1 AND stime < ? AND etime > ?"
+	_allValidRules = "SELECT id,type,bid,name,state,stime,etime,rule,uid,ctime,mtime FROM monitor_rule WHERE state = 1 AND stime < ? AND etime > ?"
+)
+
+// RulesByBid 获取某业务的监控
+func (d *Dao) RulesByBid(c context.Context, bid int64) (rules []*monitor.Rule, err error) {
+	var (
+		rows *xsql.Rows
+		now  = time.Now()
+	)
+	if rows, err = d.db.Query(c, _rulesByBid, bid, now, now); err != nil {
+		log.Error("d.db.Exec error(%v)", errors.WithStack(err))
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		rule := &monitor.Rule{}
+		var confStr string
+		if err = rows.Scan(&rule.ID, &rule.Type, &rule.BID, &rule.Name, &rule.State, &rule.STime, &rule.ETime, &confStr, &rule.UID, &rule.CTime, &rule.MTime); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		conf := &monitor.RuleConf{}
+		if err = json.Unmarshal([]byte(confStr), conf); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", confStr, err)
+			return
+		}
+		rule.RuleConf = conf
+		rules = append(rules, rule)
+	}
+	return
+}
+
+// ValidRules 获取有效的监控
+func (d *Dao) ValidRules(c context.Context) (rules []*monitor.Rule, err error) {
+	var (
+		rows *xsql.Rows
+		now  = time.Now()
+	)
+	if rows, err = d.db.Query(c, _allValidRules, now, now); err != nil {
+		log.Error("d.db.Exec error(%v)", errors.WithStack(err))
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		rule := &monitor.Rule{}
+		var confStr string
+		if err = rows.Scan(&rule.ID, &rule.Type, &rule.BID, &rule.Name, &rule.State, &rule.STime, &rule.ETime, &confStr, &rule.UID, &rule.CTime, &rule.MTime); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		conf := &monitor.RuleConf{}
+		if err = json.Unmarshal([]byte(confStr), conf); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", confStr, err)
+			return
+		}
+		rule.RuleConf = conf
+		rules = append(rules, rule)
+	}
+	return
+}

app/job/main/aegis/dao/monitor/db_test.go

@@ -0,0 +1,37 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestMonitorRulesByBid(t *testing.T) {
+	convey.Convey("RulesByBid", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			bid = int64(2)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.RulesByBid(c, bid)
+			convCtx.Convey("Then err should be nil.rules should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorValidRules(t *testing.T) {
+	convey.Convey("ValidRules", t, func(convCtx convey.C) {
+		var (
+			c = context.Background()
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.ValidRules(c)
+			convCtx.Convey("Then err should be nil.rules should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/monitor/redis.go

@@ -0,0 +1,204 @@
+package monitor
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/main/aegis/model/monitor"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"strconv"
+	"time"
+)
+
+const (
+	// _maxAge Sorted
+	_maxAge = 604800 //7天
+)
+
+// AddToSet add monitor stats
+func (d *Dao) AddToSet(c context.Context, keys []string, oid int64) (logs []string, err error) {
+	if len(keys) == 0 {
+		return
+	}
+	var (
+		conn = d.redis.Get(c)
+		now  = time.Now().Unix()
+	)
+	defer conn.Close()
+	for _, key := range keys {
+		//先判断key是否存在，存在则忽略
+		if v, _ := redis.Int(conn.Do("ZSCORE", key, oid)); v != 0 {
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(ZSCORE, %s, %d) member exists success", key, oid))
+			continue
+		}
+		if _, err = conn.Do("ZADD", key, now, oid); err != nil {
+			log.Error("conn.Do(ZADD, %s, %d, %d) error(%v)", key, now, oid, err)
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(ZADD, %s, %d, %d) error(%v)", key, now, oid, err))
+		} else {
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(ZADD, %s, %d, %d) success", key, now, oid))
+		}
+		if _, err = conn.Do("EXPIRE", key, _maxAge); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", key, _maxAge, err)
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(EXPIRE, %s, %d) error(%v)", key, _maxAge, err))
+		} else {
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(EXPIRE, %s, %d) success", key, _maxAge))
+		}
+	}
+	return
+}
+
+// RemFromSet remove monitor stats
+func (d *Dao) RemFromSet(c context.Context, keys []string, oid int64) (logs []string, err error) {
+	if len(keys) == 0 {
+		return
+	}
+	var (
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, key := range keys {
+		if _, er := conn.Do("ZREM", key, oid); er != nil {
+			err = er
+			log.Error("conn.Do(ZREM, %s, %d) error(%v)", key, oid, err)
+			logs = append(logs, fmt.Sprintf("RemFromSet() conn.Do(ZREM, %s, %d) error(%v)", key, oid, err))
+			continue
+		}
+		logs = append(logs, fmt.Sprintf("RemFromSet() conn.Do(ZREM, %s, %d) success", key, oid))
+	}
+	return
+}
+
+// ClearExpireSet clear expire stats
+func (d *Dao) ClearExpireSet(c context.Context, keys []string) (logs []string, err error) {
+	if len(keys) == 0 {
+		return
+	}
+	var (
+		conn = d.redis.Get(c)
+		now  = time.Now().Unix()
+		min  int64
+		max  = now - _maxAge
+	)
+	defer conn.Close()
+	for _, key := range keys {
+		if _, er := conn.Do("ZREMRANGEBYSCORE", key, min, max); er != nil {
+			err = er
+			log.Error("conn.Do(ZREMRANGEBYSCORE, %s, %d, %d) error(%v)", key, min, max, err)
+			logs = append(logs, fmt.Sprintf("ClearExpireSet() key: %s min:%d max:%d error:%v", key, min, max, err))
+			continue
+		}
+		logs = append(logs, fmt.Sprintf("ClearExpireSet() key: %s min:%d max:%d success", key, min, max))
+	}
+	return
+}
+
+// AddToDelArc 添加稿件信息到
+func (d *Dao) AddToDelArc(c context.Context, a *monitor.BinlogArchive) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		bs   []byte
+	)
+	defer conn.Close()
+	info := &monitor.DelArcInfo{
+		AID:   a.ID,
+		MID:   a.MID,
+		Time:  a.MTime,
+		Title: a.Title,
+	}
+	if bs, err = json.Marshal(info); err != nil {
+		log.Error("json.Marshal(%+v) error:%v", info, err)
+		return
+	}
+	if _, err = conn.Do("HSET", monitor.RedisDelArcInfo, a.ID, string(bs)); err != nil {
+		log.Error("conn.Send(HSET,%s,%d,%s) error(%v)", monitor.RedisDelArcInfo, a.ID, bs, err)
+		return
+	}
+	return
+}
+
+// ArcDelInfos 获取被删除稿件的信息
+func (d *Dao) ArcDelInfos(c context.Context, aids []int64) (infos map[int64]*monitor.DelArcInfo, err error) {
+	var (
+		conn = d.redis.Get(c)
+		strs []string
+	)
+	defer conn.Close()
+	infos = make(map[int64]*monitor.DelArcInfo)
+	if len(aids) == 0 {
+		return
+	}
+	args := redis.Args{}
+	args = args.Add(monitor.RedisDelArcInfo)
+	for _, id := range aids {
+		args = args.Add(id)
+	}
+	log.Info("s.monitorNotify() ArcDelInfos. aids(%v) args(%+v)", aids, args)
+	if strs, err = redis.Strings(conn.Do("HMGET", args...)); err != nil {
+		log.Error("conn.Send(HMGET,%v) error(%v)", args, err)
+		return
+	}
+	log.Info("s.monitorNotify() ArcDelInfos. aids(%v) strs(%v)", aids, strs)
+	for _, v := range strs {
+		info := &monitor.DelArcInfo{}
+		if err = json.Unmarshal([]byte(v), info); err != nil {
+			log.Error("json.Unmarshal(%s) error:%v", v, err)
+			continue
+		}
+		infos[info.AID] = info
+	}
+	return
+}
+
+// MoniRuleStats 获取监控统计
+func (d *Dao) MoniRuleStats(c context.Context, id int64, min, max int64) (stats *monitor.Stats, err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = fmt.Sprintf(monitor.RedisPrefix, id)
+		now  = time.Now().Unix()
+	)
+	stats = &monitor.Stats{}
+	defer conn.Close()
+	if stats.TotalCount, err = redis.Int(conn.Do("ZCOUNT", key, 0, now)); err != nil {
+		log.Error("conn.Do(ZCOUNT,%s,0,%d) error(%v)", key, now, err)
+		return
+	}
+	if stats.MoniCount, err = redis.Int(conn.Do("ZCOUNT", key, min, max)); err != nil {
+		log.Error("conn.Do(ZCOUNT,%s,%d,%d) error(%v)", key, min, max, err)
+		return
+	}
+	var oldest map[string]string //进入列表最久的项
+	oldest, err = redis.StringMap(conn.Do("ZRANGE", key, 0, 0, "WITHSCORES"))
+	for _, t := range oldest {
+		var i int
+		if i, err = strconv.Atoi(t); err != nil {
+			return
+		}
+		stats.MaxTime = int(now) - i
+	}
+	return
+}
+
+// MoniRuleOids 获取监控的id
+func (d *Dao) MoniRuleOids(c context.Context, id int64, min, max int64) (oidMap map[int64]int, err error) {
+	var (
+		conn   = d.redis.Get(c)
+		key    = fmt.Sprintf(monitor.RedisPrefix, id)
+		intMap map[string]int
+	)
+	oidMap = make(map[int64]int)
+	intMap = make(map[string]int)
+	defer conn.Close()
+	if intMap, err = redis.IntMap(conn.Do("ZRANGEBYSCORE", key, min, max, "WITHSCORES")); err != nil {
+		log.Error("redis.IntMap(conn.Do(\"ZRANGEBYSCORE\", %s, %d, %d, \"WITHSCORES\")) error(%v)", key, min, max, err)
+		return
+	}
+	for k, v := range intMap {
+		oid := 0
+		if oid, err = strconv.Atoi(k); err != nil {
+			log.Error("strconv.Atoi(%s) error(%v)", k, err)
+		}
+		oidMap[int64(oid)] = v
+	}
+	return
+}

app/job/main/aegis/dao/monitor/redis_test.go

@@ -0,0 +1,122 @@
+package monitor
+
+import (
+	"context"
+	"go-common/app/job/main/aegis/model/monitor"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestMonitorAddToSet(t *testing.T) {
+	convey.Convey("AddToSet", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			keys = []string{"monitor_test"}
+			oid  = int64(123)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.AddToSet(c, keys, oid)
+			convCtx.Convey("Then err should be nil.logs should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorRemFromSet(t *testing.T) {
+	convey.Convey("RemFromSet", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			keys = []string{"monitor_test"}
+			oid  = int64(123)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.RemFromSet(c, keys, oid)
+			convCtx.Convey("Then err should be nil.logs should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorClearExpireSet(t *testing.T) {
+	convey.Convey("ClearExpireSet", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			keys = []string{"monitor_test"}
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.ClearExpireSet(c, keys)
+			convCtx.Convey("Then err should be nil.logs should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorAddToDelArc(t *testing.T) {
+	convey.Convey("AddToDelArc", t, func(convCtx convey.C) {
+		var (
+			c = context.Background()
+			a = &monitor.BinlogArchive{
+				ID: 123,
+			}
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			err := d.AddToDelArc(c, a)
+			convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorArcDelInfos(t *testing.T) {
+	convey.Convey("ArcDelInfos", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			aids = []int64{123}
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.ArcDelInfos(c, aids)
+			convCtx.Convey("Then err should be nil.infos should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorMoniRuleStats(t *testing.T) {
+	convey.Convey("MoniRuleStats", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			id  = int64(1)
+			min = int64(0)
+			max = int64(0)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.MoniRuleStats(c, id, min, max)
+			convCtx.Convey("Then err should be nil.stats should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorMoniRuleOids(t *testing.T) {
+	convey.Convey("MoniRuleOids", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			id  = int64(1)
+			min = int64(0)
+			max = int64(0)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.MoniRuleOids(c, id, min, max)
+			convCtx.Convey("Then err should be nil.oidMap should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}