Create & Init Project...

app/job/main/aegis/dao/BUILD

@@ -0,0 +1,80 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "task_db.go",
+        "task_http.go",
+        "task_mc.go",
+        "task_orm.go",
+        "task_redis.go",
+        "task_report.go",
+        "task_rpc.go",
+    ],
+    importpath = "go-common/app/job/main/aegis/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//app/job/main/aegis/model:go_default_library",
+        "//app/service/main/relation/model:go_default_library",
+        "//app/service/main/relation/rpc/client:go_default_library",
+        "//app/service/main/up/api/v1:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/jinzhu/gorm:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/aegis/dao/email:all-srcs",
+        "//app/job/main/aegis/dao/monitor:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "task_db_test.go",
+        "task_mc_test.go",
+        "task_orm_test.go",
+        "task_redis_test.go",
+        "task_rpc_test.go",
+        "task_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//app/job/main/aegis/model:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/aegis/dao/dao.go

@@ -0,0 +1,72 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/aegis/conf"
+	relrpc "go-common/app/service/main/relation/rpc/client"
+	uprpc "go-common/app/service/main/up/api/v1"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/orm"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/jinzhu/gorm"
+)
+
+// Dao dao
+type Dao struct {
+	c      *conf.Config
+	mc     *memcache.Pool
+	redis  *redis.Pool
+	slowdb *xsql.DB
+	fastdb *xsql.DB
+	orm    *gorm.DB
+	//gorpc
+	relRPC *relrpc.Service
+	//grpc
+	upRPC uprpc.UpClient
+
+	httpFast *bm.Client
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:      c,
+		mc:     memcache.NewPool(c.Memcache),
+		redis:  redis.NewPool(c.Redis),
+		fastdb: xsql.NewMySQL(c.MySQL.Fast),
+		slowdb: xsql.NewMySQL(c.MySQL.Slow),
+		orm:    orm.NewMySQL(c.ORM),
+
+		httpFast: bm.NewClient(c.HTTP.Fast),
+	}
+
+	// rpc
+	if !c.Debug {
+		dao.relRPC = relrpc.New(c.RPC.Rel)
+		var err error
+		if dao.upRPC, err = uprpc.NewClient(c.GRPC.Up); err != nil {
+			panic(err)
+		}
+	}
+
+	dao.orm.LogMode(true)
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.mc.Close()
+	d.redis.Close()
+	d.slowdb.Close()
+	d.fastdb.Close()
+	d.orm.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	return d.fastdb.Ping(c)
+}

app/job/main/aegis/dao/dao_test.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/aegis/conf"
+	"go-common/app/job/main/aegis/model"
+)
+
+var (
+	d     *Dao
+	task1 = &model.Task{ID: 1, BusinessID: 1, FlowID: 1, UID: 1, Weight: 3, Ctime: model.IntTime(time.Now().Unix())}
+	task2 = &model.Task{ID: 2, BusinessID: 1, FlowID: 1, UID: 1, Weight: 2, Ctime: model.IntTime(time.Now().Unix())}
+	task3 = &model.Task{ID: 3, BusinessID: 1, FlowID: 1, UID: 1, Weight: 1, Ctime: model.IntTime(time.Now().Unix())}
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.archive.aegis-job")
+		flag.Set("conf_token", "aed3cc21ca345ffc284c6036da32352b")
+		flag.Set("tree_id", "61819")
+		flag.Set("conf_version", "1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/aegis-job.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/aegis/dao/email/BUILD

@@ -0,0 +1,47 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dao.go"],
+    importpath = "go-common/app/job/main/aegis/dao/email",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//app/job/main/aegis/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["dao_test.go"],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/aegis/dao/email/dao.go

@@ -0,0 +1,98 @@
+package email
+
+import (
+	"context"
+	"encoding/json"
+	"go-common/app/job/main/aegis/conf"
+	"go-common/app/job/main/aegis/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"gopkg.in/gomail.v2"
+	"time"
+)
+
+type Dao struct {
+	c     *conf.Config
+	redis *redis.Pool
+	email *gomail.Dialer
+}
+
+const (
+	// MoniEmailKey 监控邮件队列key
+	MoniEmailKey = "monitor_stats_email"
+)
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:     c,
+		email: gomail.NewDialer(c.Mail.Host, c.Mail.Port, c.Mail.Username, c.Mail.Password),
+		redis: redis.NewPool(c.Redis),
+	}
+	return d
+}
+
+// MonitorEmailAsync 异步发送监控邮件
+func (d *Dao) MonitorEmailAsync(c context.Context, members []string, title, content string) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		bs   []byte
+	)
+	defer conn.Close()
+	temp := &model.MoniTemp{
+		From:    d.c.Mail.Username,
+		Members: members,
+		Subject: title,
+		Body:    content,
+	}
+	if bs, err = json.Marshal(temp); err != nil {
+		log.Error("d.MonitorEmailAsync() json.Marshal(%+v) error(%v) key(%s)", temp, err, MoniEmailKey)
+		return
+	}
+	if _, err = conn.Do("RPUSH", MoniEmailKey, bs); err != nil {
+		log.Error("d.MonitorEmailAsync() conn.Do(RPUSH, %s, %s) error(%v)", MoniEmailKey, bs, err)
+	}
+	return
+}
+
+// MonitorEmailProc 发送监控邮件
+func (d *Dao) MonitorEmailProc() (err error) {
+	var (
+		bs      []byte
+		temp    *model.MoniTemp
+		headers map[string][]string
+	)
+	headers = make(map[string][]string)
+	bs, err = d.PopRedis(context.TODO(), MoniEmailKey)
+	if err != nil || bs == nil {
+		log.Warn("d.MonitorEmailProc() warn:%v content:%s", err, bs)
+		time.Sleep(5 * time.Second)
+		return
+	}
+	if err = json.Unmarshal(bs, &temp); err != nil {
+		log.Error("d.MonitorEmailProc() json.unmarshal error(%v) content(%s)", err, bs)
+		return
+	}
+	msg := gomail.NewMessage()
+	headers["From"] = []string{d.c.Mail.Username}
+	headers["To"] = temp.Members
+	headers["Subject"] = []string{temp.Subject}
+	msg.SetHeaders(headers)
+	msg.SetBody("text/html", temp.Body)
+	if err = d.email.DialAndSend(msg); err != nil {
+		log.Error("d.email.DialAndSend(%+v) error:%v", msg, err)
+		return
+	}
+	return
+}
+
+// PopRedis lpop fail item from redis
+func (d *Dao) PopRedis(c context.Context, key string) (bs []byte, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+
+	if bs, err = redis.Bytes(conn.Do("LPOP", key)); err != nil && err != redis.ErrNil {
+		log.Error("d.PopRedis() redis.Bytes(conn.Do(LPOP, %s)) error(%v)", key, err)
+	}
+	return
+}

app/job/main/aegis/dao/email/dao_test.go

@@ -0,0 +1,60 @@
+package email
+
+import (
+	"context"
+	"flag"
+	"github.com/smartystreets/goconvey/convey"
+	"go-common/app/job/main/aegis/conf"
+	"os"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.archive.aegis-job")
+		flag.Set("conf_token", "aed3cc21ca345ffc284c6036da32352b")
+		flag.Set("tree_id", "61819")
+		flag.Set("conf_version", "1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/aegis-job.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}
+
+func TestDao_MonitorEmailAsync(t *testing.T) {
+	convey.Convey("MonitorEmailAsync", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.MonitorEmailAsync(c, []string{"abc@bilibili.com"}, "测试标题", "测试内容<a href=\"https://www.bilibili.com\">link</a>")
+			ctx.Convey("Then err should be nil.tasks,lastid should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+func TestDao_MonitorEmailProc(t *testing.T) {
+	convey.Convey("MonitorEmailProc", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.MonitorEmailProc()
+			ctx.Convey("Then err should be nil.tasks,lastid should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/monitor/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "dao.go",
+        "db.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/aegis/dao/monitor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//app/job/main/aegis/model/monitor:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "api_test.go",
+        "dao_test.go",
+        "db_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/aegis/conf:go_default_library",
+        "//app/job/main/aegis/model/monitor:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)

app/job/main/aegis/dao/monitor/api.go

@@ -0,0 +1,40 @@
+package monitor
+
+import (
+	"context"
+	"go-common/app/job/main/aegis/model/monitor"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"net/url"
+	"strconv"
+)
+
+const (
+	_arcAdditURL = "/videoup/archive/addit"
+)
+
+// ArchiveAttr 获取稿件都附加属性
+func (d *Dao) ArchiveAttr(c context.Context, aid int64) (addit *monitor.ArchiveAddit, err error) {
+	params := url.Values{}
+	params.Set("aid", strconv.FormatInt(aid, 10))
+	var res struct {
+		Code int                   `json:"code"`
+		Data *monitor.ArchiveAddit `json:"data"`
+	}
+	if err = d.http.Get(c, d.URLArcAddit, "", params, &res); err != nil {
+		log.Error("d.ArchiveAttr(%s)  error(%v)", d.URLArcAddit+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 0 {
+		err = ecode.Int(res.Code)
+		log.Error("d.ArchiveAttr(%s) Code=(%d)", d.URLArcAddit+"?"+params.Encode(), res.Code)
+		return
+	}
+	if res.Data == nil {
+		err = ecode.NothingFound
+		log.Warn("d.ArchiveAttr(%s) Code=(%d) data nil", d.URLArcAddit+"?"+params.Encode(), res.Code)
+		return
+	}
+	addit = res.Data
+	return
+}

app/job/main/aegis/dao/monitor/api_test.go

@@ -0,0 +1,26 @@
+package monitor
+
+import (
+	"context"
+	"gopkg.in/h2non/gock.v1"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestMonitorArchiveAttr(t *testing.T) {
+	convey.Convey("ArchiveAttr", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			aid = int64(1212)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			defer gock.OffAll()
+			httpMock("GET", d.URLArcAddit).Reply(200).JSON(`{"code":0,"data":{}}`)
+			_, err := d.ArchiveAttr(c, aid)
+			convCtx.Convey("Then err should be nil.addit should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/monitor/dao.go

@@ -0,0 +1,41 @@
+package monitor
+
+import (
+	"context"
+	"go-common/app/job/main/aegis/conf"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+type Dao struct {
+	c           *conf.Config
+	redis       *redis.Pool
+	db          *xsql.DB
+	http        *bm.Client
+	URLArcAddit string
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:           c,
+		redis:       redis.NewPool(c.Redis),
+		db:          xsql.NewMySQL(c.MySQL.Fast),
+		http:        bm.NewClient(c.HTTP.Fast),
+		URLArcAddit: c.Host.Videoup + _arcAdditURL,
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return d.db.Ping(c)
+}

app/job/main/aegis/dao/monitor/dao_test.go

@@ -0,0 +1,43 @@
+package monitor
+
+import (
+	"flag"
+	"go-common/app/job/main/aegis/conf"
+	"gopkg.in/h2non/gock.v1"
+	"os"
+	"strings"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.archive.aegis-job")
+		flag.Set("conf_token", "aed3cc21ca345ffc284c6036da32352b")
+		flag.Set("tree_id", "61819")
+		flag.Set("conf_version", "1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/aegis-job.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	d.http.SetTransport(gock.DefaultTransport)
+	return r
+}

app/job/main/aegis/dao/monitor/db.go

@@ -0,0 +1,74 @@
+package monitor
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/pkg/errors"
+	"go-common/app/job/main/aegis/model/monitor"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	"time"
+)
+
+const (
+	_rulesByBid    = "SELECT id,type,bid,name,state,stime,etime,rule,uid,ctime,mtime FROM monitor_rule WHERE bid = ? AND state = 1 AND stime < ? AND etime > ?"
+	_allValidRules = "SELECT id,type,bid,name,state,stime,etime,rule,uid,ctime,mtime FROM monitor_rule WHERE state = 1 AND stime < ? AND etime > ?"
+)
+
+// RulesByBid 获取某业务的监控
+func (d *Dao) RulesByBid(c context.Context, bid int64) (rules []*monitor.Rule, err error) {
+	var (
+		rows *xsql.Rows
+		now  = time.Now()
+	)
+	if rows, err = d.db.Query(c, _rulesByBid, bid, now, now); err != nil {
+		log.Error("d.db.Exec error(%v)", errors.WithStack(err))
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		rule := &monitor.Rule{}
+		var confStr string
+		if err = rows.Scan(&rule.ID, &rule.Type, &rule.BID, &rule.Name, &rule.State, &rule.STime, &rule.ETime, &confStr, &rule.UID, &rule.CTime, &rule.MTime); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		conf := &monitor.RuleConf{}
+		if err = json.Unmarshal([]byte(confStr), conf); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", confStr, err)
+			return
+		}
+		rule.RuleConf = conf
+		rules = append(rules, rule)
+	}
+	return
+}
+
+// ValidRules 获取有效的监控
+func (d *Dao) ValidRules(c context.Context) (rules []*monitor.Rule, err error) {
+	var (
+		rows *xsql.Rows
+		now  = time.Now()
+	)
+	if rows, err = d.db.Query(c, _allValidRules, now, now); err != nil {
+		log.Error("d.db.Exec error(%v)", errors.WithStack(err))
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		rule := &monitor.Rule{}
+		var confStr string
+		if err = rows.Scan(&rule.ID, &rule.Type, &rule.BID, &rule.Name, &rule.State, &rule.STime, &rule.ETime, &confStr, &rule.UID, &rule.CTime, &rule.MTime); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		conf := &monitor.RuleConf{}
+		if err = json.Unmarshal([]byte(confStr), conf); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", confStr, err)
+			return
+		}
+		rule.RuleConf = conf
+		rules = append(rules, rule)
+	}
+	return
+}

app/job/main/aegis/dao/monitor/db_test.go

@@ -0,0 +1,37 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestMonitorRulesByBid(t *testing.T) {
+	convey.Convey("RulesByBid", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			bid = int64(2)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.RulesByBid(c, bid)
+			convCtx.Convey("Then err should be nil.rules should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorValidRules(t *testing.T) {
+	convey.Convey("ValidRules", t, func(convCtx convey.C) {
+		var (
+			c = context.Background()
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.ValidRules(c)
+			convCtx.Convey("Then err should be nil.rules should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/monitor/redis.go

@@ -0,0 +1,204 @@
+package monitor
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/main/aegis/model/monitor"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"strconv"
+	"time"
+)
+
+const (
+	// _maxAge Sorted
+	_maxAge = 604800 //7天
+)
+
+// AddToSet add monitor stats
+func (d *Dao) AddToSet(c context.Context, keys []string, oid int64) (logs []string, err error) {
+	if len(keys) == 0 {
+		return
+	}
+	var (
+		conn = d.redis.Get(c)
+		now  = time.Now().Unix()
+	)
+	defer conn.Close()
+	for _, key := range keys {
+		//先判断key是否存在，存在则忽略
+		if v, _ := redis.Int(conn.Do("ZSCORE", key, oid)); v != 0 {
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(ZSCORE, %s, %d) member exists success", key, oid))
+			continue
+		}
+		if _, err = conn.Do("ZADD", key, now, oid); err != nil {
+			log.Error("conn.Do(ZADD, %s, %d, %d) error(%v)", key, now, oid, err)
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(ZADD, %s, %d, %d) error(%v)", key, now, oid, err))
+		} else {
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(ZADD, %s, %d, %d) success", key, now, oid))
+		}
+		if _, err = conn.Do("EXPIRE", key, _maxAge); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", key, _maxAge, err)
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(EXPIRE, %s, %d) error(%v)", key, _maxAge, err))
+		} else {
+			logs = append(logs, fmt.Sprintf("AddToSet() conn.Do(EXPIRE, %s, %d) success", key, _maxAge))
+		}
+	}
+	return
+}
+
+// RemFromSet remove monitor stats
+func (d *Dao) RemFromSet(c context.Context, keys []string, oid int64) (logs []string, err error) {
+	if len(keys) == 0 {
+		return
+	}
+	var (
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, key := range keys {
+		if _, er := conn.Do("ZREM", key, oid); er != nil {
+			err = er
+			log.Error("conn.Do(ZREM, %s, %d) error(%v)", key, oid, err)
+			logs = append(logs, fmt.Sprintf("RemFromSet() conn.Do(ZREM, %s, %d) error(%v)", key, oid, err))
+			continue
+		}
+		logs = append(logs, fmt.Sprintf("RemFromSet() conn.Do(ZREM, %s, %d) success", key, oid))
+	}
+	return
+}
+
+// ClearExpireSet clear expire stats
+func (d *Dao) ClearExpireSet(c context.Context, keys []string) (logs []string, err error) {
+	if len(keys) == 0 {
+		return
+	}
+	var (
+		conn = d.redis.Get(c)
+		now  = time.Now().Unix()
+		min  int64
+		max  = now - _maxAge
+	)
+	defer conn.Close()
+	for _, key := range keys {
+		if _, er := conn.Do("ZREMRANGEBYSCORE", key, min, max); er != nil {
+			err = er
+			log.Error("conn.Do(ZREMRANGEBYSCORE, %s, %d, %d) error(%v)", key, min, max, err)
+			logs = append(logs, fmt.Sprintf("ClearExpireSet() key: %s min:%d max:%d error:%v", key, min, max, err))
+			continue
+		}
+		logs = append(logs, fmt.Sprintf("ClearExpireSet() key: %s min:%d max:%d success", key, min, max))
+	}
+	return
+}
+
+// AddToDelArc 添加稿件信息到
+func (d *Dao) AddToDelArc(c context.Context, a *monitor.BinlogArchive) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		bs   []byte
+	)
+	defer conn.Close()
+	info := &monitor.DelArcInfo{
+		AID:   a.ID,
+		MID:   a.MID,
+		Time:  a.MTime,
+		Title: a.Title,
+	}
+	if bs, err = json.Marshal(info); err != nil {
+		log.Error("json.Marshal(%+v) error:%v", info, err)
+		return
+	}
+	if _, err = conn.Do("HSET", monitor.RedisDelArcInfo, a.ID, string(bs)); err != nil {
+		log.Error("conn.Send(HSET,%s,%d,%s) error(%v)", monitor.RedisDelArcInfo, a.ID, bs, err)
+		return
+	}
+	return
+}
+
+// ArcDelInfos 获取被删除稿件的信息
+func (d *Dao) ArcDelInfos(c context.Context, aids []int64) (infos map[int64]*monitor.DelArcInfo, err error) {
+	var (
+		conn = d.redis.Get(c)
+		strs []string
+	)
+	defer conn.Close()
+	infos = make(map[int64]*monitor.DelArcInfo)
+	if len(aids) == 0 {
+		return
+	}
+	args := redis.Args{}
+	args = args.Add(monitor.RedisDelArcInfo)
+	for _, id := range aids {
+		args = args.Add(id)
+	}
+	log.Info("s.monitorNotify() ArcDelInfos. aids(%v) args(%+v)", aids, args)
+	if strs, err = redis.Strings(conn.Do("HMGET", args...)); err != nil {
+		log.Error("conn.Send(HMGET,%v) error(%v)", args, err)
+		return
+	}
+	log.Info("s.monitorNotify() ArcDelInfos. aids(%v) strs(%v)", aids, strs)
+	for _, v := range strs {
+		info := &monitor.DelArcInfo{}
+		if err = json.Unmarshal([]byte(v), info); err != nil {
+			log.Error("json.Unmarshal(%s) error:%v", v, err)
+			continue
+		}
+		infos[info.AID] = info
+	}
+	return
+}
+
+// MoniRuleStats 获取监控统计
+func (d *Dao) MoniRuleStats(c context.Context, id int64, min, max int64) (stats *monitor.Stats, err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = fmt.Sprintf(monitor.RedisPrefix, id)
+		now  = time.Now().Unix()
+	)
+	stats = &monitor.Stats{}
+	defer conn.Close()
+	if stats.TotalCount, err = redis.Int(conn.Do("ZCOUNT", key, 0, now)); err != nil {
+		log.Error("conn.Do(ZCOUNT,%s,0,%d) error(%v)", key, now, err)
+		return
+	}
+	if stats.MoniCount, err = redis.Int(conn.Do("ZCOUNT", key, min, max)); err != nil {
+		log.Error("conn.Do(ZCOUNT,%s,%d,%d) error(%v)", key, min, max, err)
+		return
+	}
+	var oldest map[string]string //进入列表最久的项
+	oldest, err = redis.StringMap(conn.Do("ZRANGE", key, 0, 0, "WITHSCORES"))
+	for _, t := range oldest {
+		var i int
+		if i, err = strconv.Atoi(t); err != nil {
+			return
+		}
+		stats.MaxTime = int(now) - i
+	}
+	return
+}
+
+// MoniRuleOids 获取监控的id
+func (d *Dao) MoniRuleOids(c context.Context, id int64, min, max int64) (oidMap map[int64]int, err error) {
+	var (
+		conn   = d.redis.Get(c)
+		key    = fmt.Sprintf(monitor.RedisPrefix, id)
+		intMap map[string]int
+	)
+	oidMap = make(map[int64]int)
+	intMap = make(map[string]int)
+	defer conn.Close()
+	if intMap, err = redis.IntMap(conn.Do("ZRANGEBYSCORE", key, min, max, "WITHSCORES")); err != nil {
+		log.Error("redis.IntMap(conn.Do(\"ZRANGEBYSCORE\", %s, %d, %d, \"WITHSCORES\")) error(%v)", key, min, max, err)
+		return
+	}
+	for k, v := range intMap {
+		oid := 0
+		if oid, err = strconv.Atoi(k); err != nil {
+			log.Error("strconv.Atoi(%s) error(%v)", k, err)
+		}
+		oidMap[int64(oid)] = v
+	}
+	return
+}

app/job/main/aegis/dao/monitor/redis_test.go

@@ -0,0 +1,122 @@
+package monitor
+
+import (
+	"context"
+	"go-common/app/job/main/aegis/model/monitor"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestMonitorAddToSet(t *testing.T) {
+	convey.Convey("AddToSet", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			keys = []string{"monitor_test"}
+			oid  = int64(123)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.AddToSet(c, keys, oid)
+			convCtx.Convey("Then err should be nil.logs should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorRemFromSet(t *testing.T) {
+	convey.Convey("RemFromSet", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			keys = []string{"monitor_test"}
+			oid  = int64(123)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.RemFromSet(c, keys, oid)
+			convCtx.Convey("Then err should be nil.logs should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorClearExpireSet(t *testing.T) {
+	convey.Convey("ClearExpireSet", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			keys = []string{"monitor_test"}
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.ClearExpireSet(c, keys)
+			convCtx.Convey("Then err should be nil.logs should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorAddToDelArc(t *testing.T) {
+	convey.Convey("AddToDelArc", t, func(convCtx convey.C) {
+		var (
+			c = context.Background()
+			a = &monitor.BinlogArchive{
+				ID: 123,
+			}
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			err := d.AddToDelArc(c, a)
+			convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorArcDelInfos(t *testing.T) {
+	convey.Convey("ArcDelInfos", t, func(convCtx convey.C) {
+		var (
+			c    = context.Background()
+			aids = []int64{123}
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.ArcDelInfos(c, aids)
+			convCtx.Convey("Then err should be nil.infos should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorMoniRuleStats(t *testing.T) {
+	convey.Convey("MoniRuleStats", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			id  = int64(1)
+			min = int64(0)
+			max = int64(0)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.MoniRuleStats(c, id, min, max)
+			convCtx.Convey("Then err should be nil.stats should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestMonitorMoniRuleOids(t *testing.T) {
+	convey.Convey("MoniRuleOids", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			id  = int64(1)
+			min = int64(0)
+			max = int64(0)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			_, err := d.MoniRuleOids(c, id, min, max)
+			convCtx.Convey("Then err should be nil.oidMap should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/task_db.go

@@ -0,0 +1,79 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"go-common/app/job/main/aegis/model"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_queryTaskSQL = "SELECT id,business_id,flow_id,rid,admin_id,uid,state,weight,utime,gtime,mid,fans,`group`,reason,ctime,mtime FROM task WHERE state=? AND mtime<=? AND id>? ORDER BY id LIMIT ?"
+
+	_upSetWeightSQL = "UPDATE task SET weight=? WHERE id=?"
+
+	_assignTaskSQL = "UPDATE task SET admin_id=?,uid=? WHERE id=? AND state=?"
+
+	_checkTaskSQL = "SELECT id FROM task WHERE flow_id=? AND rid=? AND state<?"
+)
+
+// CheckTask 某资源存在未完成任务，不重复添加
+func (d *Dao) CheckTask(c context.Context, flowid, rid int64) (id int64) {
+	if err := d.fastdb.QueryRow(c, _checkTaskSQL, flowid, rid, model.TaskStateSubmit).Scan(&id); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("d.db.QueryRow error(%v)", err)
+		}
+	}
+	return
+}
+
+// AssignTask .
+func (d *Dao) AssignTask(c context.Context, task *model.Task) (rows int64, err error) {
+	var res sql.Result
+	if res, err = d.fastdb.Exec(c, _assignTaskSQL, task.AdminID, task.UID, task.ID, model.TaskStateInit); err != nil {
+		log.Error("d.db.Exec error(%v)", errors.WithStack(err))
+		return
+	}
+	return res.RowsAffected()
+}
+
+// QueryTask .
+func (d *Dao) QueryTask(c context.Context, state int8, mtime time.Time, id, limit int64) (tasks []*model.Task, lastid int64, err error) {
+	var rows *xsql.Rows
+	rows, err = d.slowdb.Query(c, _queryTaskSQL, state, mtime, id, limit)
+	if err != nil {
+		log.Error("db.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		task := &model.Task{}
+		if err = rows.Scan(&task.ID, &task.BusinessID, &task.FlowID, &task.RID, &task.AdminID, &task.UID, &task.State, &task.Weight,
+			&task.Utime, &task.Gtime, &task.MID, &task.Fans, &task.Group, &task.Reason, &task.Ctime, &task.Mtime); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+
+		tasks = append(tasks, task)
+		lastid = task.ID
+	}
+	return
+}
+
+// SetWeightDB .
+func (d *Dao) SetWeightDB(c context.Context, taskid, weight int64) (rows int64, err error) {
+	res, err := d.fastdb.Exec(c, _upSetWeightSQL, weight, taskid)
+	if err != nil {
+		log.Error("db.Exec error(%v)", err)
+		return
+	}
+	return res.LastInsertId()
+}

app/job/main/aegis/dao/task_db_test.go

@@ -0,0 +1,64 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoAssignTask(t *testing.T) {
+	convey.Convey("AssignTask", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			task = task1
+		)
+		task.AdminID = 1
+		task.UID = 1
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			rows, err := d.AssignTask(c, task)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoQueryTask(t *testing.T) {
+	convey.Convey("QueryTask", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			state = int8(0)
+			mtime = time.Now()
+			id    = int64(0)
+			limit = int64(1000)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			tasks, lastid, err := d.QueryTask(c, state, mtime, id, limit)
+			ctx.Convey("Then err should be nil.tasks,lastid should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(lastid, convey.ShouldNotBeNil)
+				ctx.So(tasks, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSetWeightDB(t *testing.T) {
+	convey.Convey("SetWeightDB", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			taskid = int64(1)
+			weight = int64(10)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			rows, err := d.SetWeightDB(c, taskid, weight)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/task_http.go

@@ -0,0 +1,51 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+
+	"go-common/app/job/main/aegis/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_addURL    = "/x/internal/aegis/add"
+	_updateURL = "/x/internal/aegis/update"
+	_cancelURL = "/x/internal/aegis/cancel"
+)
+
+//RscAdd resource add
+func (d *Dao) RscAdd(c context.Context, opt *model.AddOption) error {
+	uri := d.c.Host.API + _addURL
+	params := opt.ToQueryURI()
+	return d.commonPost(c, uri, params)
+}
+
+//RscUpdate resource update
+func (d *Dao) RscUpdate(c context.Context, opt *model.UpdateOption) error {
+	uri := d.c.Host.API + _updateURL
+	params := opt.ToQueryURI()
+	return d.commonPost(c, uri, params)
+}
+
+//RscCancel resource cancel
+func (d *Dao) RscCancel(c context.Context, opt *model.CancelOption) error {
+	uri := d.c.Host.API + _cancelURL
+	params := opt.ToQueryURI()
+	return d.commonPost(c, uri, params)
+}
+
+func (d *Dao) commonPost(c context.Context, uri string, params url.Values) error {
+	res := new(model.BaseResponse)
+	if err := d.httpFast.Post(c, uri, "", params, res); err != nil {
+		log.Error("d.httpFast.Post(%s) params(%s) error(%v)", uri, params.Encode(), err)
+		return err
+	}
+
+	if res.Code != 0 {
+		log.Error("d.httpFast.Post(%s) params(%s) res(%+v)", uri, params.Encode(), res)
+		return ecode.Code(res.Code)
+	}
+	return nil
+}

app/job/main/aegis/dao/task_mc.go

@@ -0,0 +1,30 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	gmc "go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+// IsConsumerOn .
+func (d *Dao) IsConsumerOn(c context.Context, bizid, flowid int, uid int64) (isOn bool, err error) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	key := mcKey(bizid, flowid, uid)
+	if _, err = conn.Get(key); err != nil {
+		if err == gmc.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("IsConsumerOn error(%v)", err)
+		}
+		return
+	}
+	isOn = true
+	return
+}
+
+func mcKey(bizid, flowid int, uid int64) string {
+	return fmt.Sprintf("aegis%d_%d_%d", bizid, flowid, uid)
+}

app/job/main/aegis/dao/task_mc_test.go

@@ -0,0 +1,28 @@
+package dao
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoIsConsumerOn(t *testing.T) {
+	convey.Convey("IsConsumerOn", t, func(ctx convey.C) {
+	})
+}
+
+func TestDaomcKey(t *testing.T) {
+	convey.Convey("mcKey", t, func(ctx convey.C) {
+		var (
+			bizid  = int(0)
+			flowid = int(0)
+			uid    = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := mcKey(bizid, flowid, uid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/task_orm.go

@@ -0,0 +1,135 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/aegis/model"
+	"go-common/library/log"
+
+	"github.com/jinzhu/gorm"
+)
+
+const (
+	_taskReleaseSQL = "update task SET admin_id=0,state=0,uid=0,gtime=0 where state=? AND mtime<=?"
+	_taskClearSQL   = "DELETE FROM task WHERE mtime<=? AND state>=3 LIMIT ?"
+)
+
+// TaskActiveConfigs list config
+func (d *Dao) TaskActiveConfigs(c context.Context) (configs []*model.TaskConfig, err error) {
+	db := d.orm.Model(&model.TaskConfig{}).Where("state=?", model.ConfigStateOn)
+
+	if err = db.Find(&configs).Error; err != nil {
+		log.Error("query error(%v)", err)
+		return
+	}
+	return
+}
+
+// TaskActiveConsumer list consumer
+func (d *Dao) TaskActiveConsumer(c context.Context) (consumerCache map[string]map[int64]struct{}, err error) {
+	rows, err := d.orm.Table("task_consumer").Select("business_id,flow_id,uid").Where("state=?", model.ConsumerStateOn).Rows()
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	consumerCache = make(map[string]map[int64]struct{})
+	for rows.Next() {
+		var bizID, flowID, UID int64
+		if err = rows.Scan(&bizID, &flowID, &UID); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			continue
+		}
+
+		key := fmt.Sprintf("%d-%d", bizID, flowID)
+
+		if _, ok := consumerCache[key]; ok {
+			consumerCache[key][UID] = struct{}{}
+		} else {
+			consumerCache[key] = map[int64]struct{}{UID: {}}
+		}
+	}
+	return
+}
+
+// KickOutConsumer 踢出用户
+func (d *Dao) KickOutConsumer(c context.Context, bizid, flowid, uid int64) (err error) {
+	return d.orm.Table("task_consumer").Where("business_id=? AND flow_id=? AND uid=?", bizid, flowid, uid).
+		Update("state", model.ConsumerStateOff).Error
+}
+
+// Resource .
+func (d *Dao) Resource(c context.Context, rid int64) (res *model.Resource, err error) {
+	res = &model.Resource{}
+	if err = d.orm.Where("id = ?", rid).First(res).Error; err == gorm.ErrRecordNotFound {
+		res = nil
+		err = nil
+	}
+	return
+}
+
+//RscState resource state
+func (d *Dao) RscState(c context.Context, rid int64) (state int64, err error) {
+	err = d.orm.Table("resource_result").Select("state").Where("rid=?", rid).Row().Scan(&state)
+	return
+}
+
+// TaskRelease .
+func (d *Dao) TaskRelease(c context.Context, mtime time.Time) (err error) {
+	return d.orm.Exec(_taskReleaseSQL, model.TaskStateDispatch, mtime).Error
+}
+
+// ReleaseByConsumer .
+func (d *Dao) ReleaseByConsumer(c context.Context, bizid, flowid, uid int64) (err error) {
+	return d.orm.Table("task").Where("business_id=? AND flow_id=? AND uid=? AND (state=1 or (admin_id>0 AND state=0))", bizid, flowid, uid).Update(
+		map[string]interface{}{
+			"uid":      0,
+			"state":    0,
+			"gtime":    0,
+			"admin_id": 0,
+		}).Error
+}
+
+//Report .
+func (d *Dao) Report(c context.Context, rt *model.Report) (err error) {
+	return d.orm.Create(rt).Error
+}
+
+//TaskClear 已完成任务最多保留3天
+func (d *Dao) TaskClear(c context.Context, mtime time.Time, limit int64) (rows int64, err error) {
+	db := d.orm.Exec(_taskClearSQL, mtime, limit)
+	rows, err = db.RowsAffected, db.Error
+	return
+}
+
+//CheckFlow 检查资源是否在对应流程上
+func (d *Dao) CheckFlow(c context.Context, rid, flowid int64) (ok bool, err error) {
+	var id int64
+	err = d.orm.Table("net_flow_resource").Select("id").
+		Where("rid=? AND flow_id=? AND state!=-1", rid, flowid).Row().Scan(&id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("CheckFlow(%d,%d) error(%v)", rid, flowid)
+		}
+		return
+	}
+	if id > 0 {
+		ok = true
+	}
+	return
+}
+
+// CreateTask .
+func (d *Dao) CreateTask(c context.Context, task *model.Task) error {
+	return d.orm.Table("task").Where("rid=? AND flow_id=? AND state<?", task.RID, task.FlowID, model.TaskStateSubmit).
+		Assign(map[string]interface{}{
+			"mid":   task.MID,
+			"fans":  task.Fans,
+			"group": task.Group,
+		}).FirstOrCreate(task).Error
+}

app/job/main/aegis/dao/task_orm_test.go

@@ -0,0 +1,145 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/aegis/model"
+	"go-common/library/sync/errgroup"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoTaskActiveConfigs(t *testing.T) {
+	convey.Convey("TaskActiveConfigs", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			configs, err := d.TaskActiveConfigs(c)
+			ctx.Convey("Then err should be nil.configs should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(configs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoTaskActiveConsumer(t *testing.T) {
+	convey.Convey("TaskActiveConsumer", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			consumerCache, err := d.TaskActiveConsumer(c)
+			ctx.Convey("Then err should be nil.consumerCache should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(consumerCache, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoResource(t *testing.T) {
+	convey.Convey("Resource", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			rid = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := d.Resource(c, rid)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoTaskRelease(t *testing.T) {
+	convey.Convey("TaskRelease", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			mtime = time.Now().Add(-10 * time.Second)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.TaskRelease(c, mtime)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoReport(t *testing.T) {
+	convey.Convey("Report", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			rt := &model.Report{
+				BusinessID: 1,
+				FlowID:     1,
+				UID:        1,
+				Content:    []byte("sguyiuo"),
+			}
+			err := d.Report(c, rt)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoTaskClear(t *testing.T) {
+	convey.Convey("TaskClear", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := d.TaskClear(c, time.Now().Add(-3*24*time.Hour), 1000)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoCheckFlow(t *testing.T) {
+	convey.Convey("CheckFlow", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := d.CheckFlow(c, 1, 1)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoCreateTask(t *testing.T) {
+	f := func() error {
+		for i := 1; i < 100; i++ {
+			task := &model.Task{
+				BusinessID: int64(i),
+				RID:        int64(i),
+				FlowID:     int64(i),
+			}
+			if err := d.CreateTask(context.Background(), task); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	wg := errgroup.Group{}
+
+	wg.Go(f)
+	wg.Go(f)
+	wg.Go(f)
+
+	if err := wg.Wait(); err != nil {
+		t.Fail()
+	}
+}

app/job/main/aegis/dao/task_redis.go

@@ -0,0 +1,289 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/pkg/errors"
+
+	"go-common/app/job/main/aegis/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_hashexpire = 24 * 60 * 60
+)
+
+func personalKey(businessID, flowID int64, uid int64) string {
+	return fmt.Sprintf("personal_%d_%d_%d", businessID, flowID, uid)
+}
+
+func publicKey(businessID, flowID int64) string {
+	return fmt.Sprintf("{%d-%d}public_%d_%d", businessID, flowID, businessID, flowID)
+}
+
+func publicBackKey(businessID, flowID int64) string {
+	return fmt.Sprintf("{%d-%d}publicBackup_%d_%d", businessID, flowID, businessID, flowID)
+}
+
+func delayKey(businessID, flowID int64, uid int64) string {
+	return fmt.Sprintf("delay_%d_%d_%d", businessID, flowID, uid)
+}
+
+func haskKey(taskid int64) string {
+	return fmt.Sprintf("task_%d", taskid)
+}
+
+func zsetKey(taskid int64) string {
+	return fmt.Sprintf("%.11d", taskid)
+}
+
+// SetTask .
+func (d *Dao) SetTask(c context.Context, task *model.Task) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	var bs []byte
+	key := haskKey(task.ID)
+	if bs, err = json.Marshal(task); err != nil {
+		log.Error("json.Marshal(%+v) error(%v)", task, err)
+		return
+	}
+
+	if err = conn.Send("SET", key, bs); err != nil {
+		log.Error("HSET error(%v)", err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, _hashexpire); err != nil {
+		log.Error("EXPIRE error(%v)", err)
+		return
+	}
+
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		return
+	}
+
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// GetTask .
+func (d *Dao) GetTask(c context.Context, id int64) (task *model.Task, err error) {
+	var bs []byte
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	key := haskKey(id)
+
+	if bs, err = redis.Bytes(conn.Do("GET", key)); err != nil {
+		log.Error("conn.Get(%s) error(%v)", key, err)
+		return
+	}
+
+	task = new(model.Task)
+	if err = json.Unmarshal(bs, task); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", string(bs), err)
+		return
+	}
+
+	return
+}
+
+// RemovePersonalTask 任务延迟或完成
+func (d *Dao) RemovePersonalTask(c context.Context, businessID, flowID int64, uid, taskid int64) (err error) {
+	key := personalKey(businessID, flowID, uid)
+	return d.removeList(c, key, taskid)
+}
+
+// RemoveDelayTask 延迟任务完成
+func (d *Dao) RemoveDelayTask(c context.Context, businessID, flowID int64, uid, taskid int64) (err error) {
+	key := delayKey(businessID, flowID, uid)
+	return d.removeList(c, key, taskid)
+}
+
+func (d *Dao) removeList(c context.Context, key string, id int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	if _, err = conn.Do("LREM", key, 0, id); err != nil {
+		log.Error("LREM error(%v)", errors.WithStack(err))
+	}
+	return
+}
+
+// PushPersonalTask 放入本人任务池
+func (d *Dao) PushPersonalTask(c context.Context, businessID, flowID int64, uid, taskid int64) (err error) {
+	key := personalKey(businessID, flowID, uid)
+	return d.pushList(c, key, taskid)
+}
+
+// PushDelayTask 延迟任务队列
+func (d *Dao) PushDelayTask(c context.Context, businessID, flowID int64, uid, taskid int64) (err error) {
+	key := delayKey(businessID, flowID, uid)
+	return d.pushList(c, key, taskid)
+}
+
+func (d *Dao) pushList(c context.Context, key string, values ...interface{}) (err error) {
+	var (
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+
+	args1 := []interface{}{key, 0}
+	args1 = append(args1, values...)
+	if _, err = conn.Do("LREM", args1...); err != nil {
+		log.Error("conn.Do(RPUSH, %v) error(%v)", args1, err)
+		return
+	}
+	args2 := []interface{}{key}
+	args2 = append(args2, values...)
+	if _, err = conn.Do("RPUSH", args2...); err != nil {
+		log.Error("conn.Do(RPUSH, %v) error(%v)", args2, err)
+	}
+	return
+}
+
+// RemovePublicTask 移除
+func (d *Dao) RemovePublicTask(c context.Context, businessID, flowID int64, taskid int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	key := publicKey(businessID, flowID)
+	args := []interface{}{key}
+	args = append(args, zsetKey(taskid))
+	if _, err = conn.Do("ZREM", args...); err != nil {
+		log.Error("(ZREM,%v) error(%v)", args, errors.WithStack(err))
+	}
+	return err
+}
+
+// PushPublicTask 放入实时任务池
+func (d *Dao) PushPublicTask(c context.Context, tasks ...*model.Task) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	for _, task := range tasks {
+		key := publicKey(task.BusinessID, task.FlowID)
+		if _, err = conn.Do("ZADD", key, -task.Weight, zsetKey(task.ID)); err != nil {
+			log.Error("conn.Do(ZADD,%s) error(%v)", key, errors.WithStack(err))
+		}
+	}
+
+	return
+}
+
+// SetWeight set weight
+func (d *Dao) SetWeight(c context.Context, businessID, flowID int64, id, weight int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	var (
+		ow  int64
+		key = publicKey(businessID, flowID)
+		zid = zsetKey(id)
+	)
+
+	if ow, err = redis.Int64(conn.Do("ZSCORE", key, zid)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("redis (ZSCORE,%s,%s) error(%v)", key, zid, err)
+		}
+		return
+	}
+
+	// 为了从大到小排序，weight取负值
+	nw := -(weight + ow)
+	if _, err = conn.Do("ZINCRBY", key, nw, zid); err != nil {
+		log.Error("redis (ZINCRBY,%s,%s,%d) error(%v)", key, nw, zid, err)
+		return
+	}
+
+	return
+}
+
+// GetWeight get Weight
+func (d *Dao) GetWeight(c context.Context, businessID, flowID int64, id int64) (weight int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	key := publicKey(businessID, flowID)
+
+	weight, err = redis.Int64(conn.Do("ZSCORE", key, zsetKey(id)))
+	if err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("conn.Do(ZSCORE %s %s) error(%v)", key, zsetKey(id), errors.WithStack(err))
+		}
+	}
+	weight = -weight
+	return
+}
+
+// TopWeights .
+func (d *Dao) TopWeights(c context.Context, businessID, flowID int64, toplen int64) (wis []*model.WeightItem, err error) {
+	key := publicKey(businessID, flowID)
+	return d.zrange(c, key, 0, toplen)
+}
+
+// CreateUnionSet  创建分身集合
+func (d *Dao) CreateUnionSet(c context.Context, businessID, flowID int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	key := publicKey(businessID, flowID)
+	backKey := publicBackKey(businessID, flowID)
+
+	if _, err = conn.Do("ZUNIONSTORE", backKey, 1, key); err != nil {
+		log.Error("conn.Do(ZUNIONSTORE,%s) error(%v)", key, errors.WithStack(err))
+	}
+
+	return
+}
+
+// RangeUinonSet  批次取出
+func (d *Dao) RangeUinonSet(c context.Context, businessID, flowID int64, start, stop int64) (wis []*model.WeightItem, err error) {
+	key := publicBackKey(businessID, flowID)
+	return d.zrange(c, key, start, stop)
+}
+
+// DeleteUinonSet  清空分身
+func (d *Dao) DeleteUinonSet(c context.Context, businessID, flowID int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	backKey := publicBackKey(businessID, flowID)
+
+	if _, err = conn.Do("DEL", backKey); err != nil {
+		log.Error("conn.Do(ZUNIODELNSTORE,%s) error(%v)", backKey, errors.WithStack(err))
+	}
+
+	return
+}
+
+func (d *Dao) zrange(c context.Context, key string, start, stop int64) (wis []*model.WeightItem, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	reply, err := redis.Int64s(conn.Do("ZRANGE", key, start, stop, "WITHSCORES"))
+	if err != nil {
+		log.Error("conn.Do(ZADD,%s) error(%v)", key, errors.WithStack(err))
+		return
+	}
+	// 单数是id,双数是weight
+	for i := 0; i < len(reply); i += 2 {
+		wi := &model.WeightItem{}
+		wi.ID = reply[i]
+		wi.Weight = -reply[i+1]
+		wis = append(wis, wi)
+	}
+
+	return
+}

app/job/main/aegis/dao/task_redis_test.go

@@ -0,0 +1,247 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/aegis/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoSetTask(t *testing.T) {
+	convey.Convey("SetTask", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err1 := d.SetTask(c, task1)
+			err2 := d.SetTask(c, task2)
+			err3 := d.SetTask(c, task3)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err1, convey.ShouldBeNil)
+				ctx.So(err2, convey.ShouldBeNil)
+				ctx.So(err3, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetTask(t *testing.T) {
+	convey.Convey("GetTask", t, func(ctx convey.C) {
+		var (
+			c  = context.Background()
+			id = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			task, err := d.GetTask(c, id)
+			ctx.Convey("Then err should be nil.task should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(task, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoPushPublicTask(t *testing.T) {
+	convey.Convey("PushPublicTask", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.PushPublicTask(c, task1, task2, task3)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRemovePublicTask(t *testing.T) {
+	convey.Convey("RemovePublicTask", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.RemovePublicTask(c, 1, 1, 1)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoPushPersonalTask(t *testing.T) {
+	convey.Convey("PushPersonalTask", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			businessID = int64(1)
+			flowID     = int64(1)
+			uid        = int64(1)
+			taskid     = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.PushPersonalTask(c, businessID, flowID, uid, taskid)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRemovePersonalTask(t *testing.T) {
+	convey.Convey("RemovePersonalTask", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			businessID = int64(1)
+			flowID     = int64(1)
+			uid        = int64(1)
+			taskid     = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.RemovePersonalTask(c, businessID, flowID, uid, taskid)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoPushDelayTask(t *testing.T) {
+	convey.Convey("PushDelayTask", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			businessID = int64(1)
+			flowID     = int64(1)
+			uid        = int64(1)
+			taskid     = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.PushDelayTask(c, businessID, flowID, uid, taskid)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRemoveDelayTask(t *testing.T) {
+	convey.Convey("RemoveDelayTask", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			businessID = int64(1)
+			flowID     = int64(1)
+			uid        = int64(1)
+			taskid     = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.RemoveDelayTask(c, businessID, flowID, uid, taskid)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSetWeight(t *testing.T) {
+	convey.Convey("SetWeight", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			businessID = int64(1)
+			flowID     = int64(1)
+			id         = int64(1)
+			weight     = int64(10)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.SetWeight(c, businessID, flowID, id, weight)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			})
+		})
+	})
+}
+
+func TestDaoGetWeight(t *testing.T) {
+	convey.Convey("GetWeight", t, func(ctx convey.C) {
+	})
+}
+
+func TestDaoTopWeights(t *testing.T) {
+	convey.Convey("TopWeights", t, func(ctx convey.C) {
+	})
+}
+
+func TestDaoCreateUnionSet(t *testing.T) {
+	convey.Convey("CreateUnionSet", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			businessID = int64(1)
+			flowID     = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.CreateUnionSet(c, businessID, flowID)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRangeUinonSet(t *testing.T) {
+	convey.Convey("RangeUinonSet", t, func(ctx convey.C) {
+	})
+}
+
+func TestDaoDeleteUinonSet(t *testing.T) {
+	convey.Convey("DeleteUinonSet", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			businessID = int64(1)
+			flowID     = int64(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.DeleteUinonSet(c, businessID, flowID)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoIncresByField(t *testing.T) {
+	convey.Convey("IncresByField", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.IncresByField(c, 1, 1, 1, model.Dispatch, 1)
+
+			err = d.IncresByField(c, 1, 1, 1, model.Release, 1)
+			err = d.IncresByField(c, 1, 1, 1, model.Submit, 1)
+			err = d.IncresByField(c, 1, 1, 1, model.Delay, 1)
+			err = d.IncresByField(c, 1, 1, 1, fmt.Sprintf(model.RscState, 1), 1)
+			err = d.IncresByField(c, 1, 1, 1, model.UseTime, 112)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoFlushReport(t *testing.T) {
+	convey.Convey("FlushReport", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			data, err := d.FlushReport(c)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				for key, val := range data {
+					fmt.Println("key:", key)
+					fmt.Println("val:", string(val))
+				}
+			})
+		})
+	})
+}

app/job/main/aegis/dao/task_report.go

@@ -0,0 +1,143 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+
+	"go-common/app/job/main/aegis/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+var rmux sync.Mutex
+
+//IncresByField .
+func (d *Dao) IncresByField(c context.Context, bizid, flowid, uid int64, field string, value int64) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		hk   = model.PersonalHashKey(bizid, flowid, uid)
+	)
+	rmux.Lock()
+	defer rmux.Unlock()
+	defer conn.Close()
+
+	if err = d.setSet(conn, hk); err != nil {
+		return
+	}
+	if err = d.setHash(conn, hk, "ds"); err != nil {
+		return
+	}
+	return d.setField(conn, hk, field, value)
+}
+
+//IncresTaskInOut 总进审量-出审量
+func (d *Dao) IncresTaskInOut(c context.Context, bizid, flowid int64, inOrOut string) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		hk   = model.TotalHashKey(bizid, flowid)
+	)
+	rmux.Lock()
+	defer rmux.Unlock()
+	defer conn.Close()
+
+	if err = d.setSet(conn, hk); err != nil {
+		return
+	}
+	if err = d.setHash(conn, hk, inOrOut); err != nil {
+		return
+	}
+
+	return d.setField(conn, hk, inOrOut, 1)
+}
+
+//FlushReport .
+func (d *Dao) FlushReport(c context.Context) (data map[string][]byte, err error) {
+	data = make(map[string][]byte)
+	rmux.Lock()
+	defer rmux.Unlock()
+
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	keys, err := redis.Strings(conn.Do("SMEMBERS", model.SetKey))
+	if err != nil {
+		log.Error("SMEMBERS %s error(%v)", model.SetKey, err)
+		return
+	}
+	if len(keys) == 0 {
+		log.Info("FlushReport empty")
+		return
+	}
+
+	for _, key := range keys {
+		if err = conn.Send("HGETALL", key); err != nil {
+			log.Error("HGETALL %s error(%v)", key, err)
+			return
+		}
+	}
+	conn.Flush()
+
+	for _, key := range keys {
+		var (
+			bs []byte
+			mp map[string]int64
+		)
+		if mp, err = redis.Int64Map(conn.Receive()); err != nil {
+			log.Error("Receive error(%v)", err)
+			return
+		}
+		if bs, err = json.Marshal(mp); err != nil {
+			log.Error("Marshal mp(%+v) error(%v)", mp, err)
+		}
+		data[key] = bs
+	}
+
+	for _, key := range keys {
+		conn.Do("DEL", key)
+	}
+	conn.Do("DEL", model.SetKey)
+	return
+}
+
+//记录key
+func (d *Dao) setSet(conn redis.Conn, hk string) (err error) {
+	if _, err := conn.Do("SADD", model.SetKey, hk); err != nil {
+		log.Error("setSet SADD(%s,%s) error(%v)", model.SetKey, hk, err)
+	}
+	return
+}
+
+//创建hash
+func (d *Dao) setHash(conn redis.Conn, key string, defaultfield string) (err error) {
+	var exist bool
+	if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
+		log.Error("setHash EXISTS(%s) error(%v)", key, err)
+		return
+	}
+	if !exist {
+		if _, err = conn.Do("HMSET", key, defaultfield, 0); err != nil {
+			log.Error("setHash HMSET(%s,%s,%d) error(%v)", key, defaultfield, 0, err)
+		}
+	}
+	return
+}
+
+//每个field赋值
+func (d *Dao) setField(conn redis.Conn, key string, field string, value int64) (err error) {
+	var exist bool
+	if exist, err = redis.Bool(conn.Do("HEXISTS", key, field)); err != nil {
+		log.Error("setField HEXISTS(%s,%s,%s) error(%v)", key, field, err)
+		return
+	}
+	if !exist {
+		if _, err = conn.Do("HMSET", key, field, 0); err != nil {
+			log.Error("setField HMSET(%s,%s,%d) error(%v)", key, field, 0, err)
+		}
+	}
+	if _, err = conn.Do("HINCRBY", key, field, value); err != nil {
+		log.Error("setField HINCRBY(%s,%s,%d) error(%v)", key, field, 1, err)
+	}
+
+	return nil
+}

app/job/main/aegis/dao/task_rpc.go

@@ -0,0 +1,43 @@
+package dao
+
+import (
+	"context"
+
+	relmod "go-common/app/service/main/relation/model"
+	uprpc "go-common/app/service/main/up/api/v1"
+	"go-common/library/log"
+
+	terrors "github.com/pkg/errors"
+)
+
+// FansCount 粉丝数
+func (d *Dao) FansCount(c context.Context, mid int64) (fans int64, err error) {
+	if d.c.Debug {
+		return 10086, nil
+	}
+	arg := &relmod.ArgMid{Mid: mid}
+	stat, err := d.relRPC.Stat(c, arg)
+	if err != nil || stat == nil {
+		log.Error("FansCount error(%v)", terrors.WithStack(err))
+		return
+	}
+	fans = stat.Follower
+	return
+}
+
+// UpSpecial 分组信息
+func (d *Dao) UpSpecial(c context.Context, mid int64) (groupids []int64, err error) {
+	if d.c.Debug {
+		return
+	}
+	req := &uprpc.UpSpecialReq{Mid: mid}
+	var reply *uprpc.UpSpecialReply
+	if reply, err = d.upRPC.UpSpecial(c, req); err != nil || reply == nil {
+		log.Error("UpSpecial(%d) error(%+v)", mid, err)
+		return
+	}
+	if reply.UpSpecial != nil {
+		groupids = reply.UpSpecial.GroupIDs
+	}
+	return
+}

app/job/main/aegis/dao/task_rpc_test.go

@@ -0,0 +1,24 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoFansCount(t *testing.T) {
+	convey.Convey("FansCount", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			fans, err := d.FansCount(c, mid)
+			ctx.Convey("Then err should be nil.fans should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(fans, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/aegis/dao/task_test.go

@@ -0,0 +1,22 @@
+package dao
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_TopWeights(t *testing.T) {
+	Convey("TopWeights", t, func() {
+	})
+}
+
+func Test_SetWeight(t *testing.T) {
+	Convey("TopWeights", t, func() {
+	})
+}
+
+func Test_TaskActiveConsumer(t *testing.T) {
+	Convey("TaskActiveConsumer", t, func() {
+	})
+}