Create & Init Project...

app/job/main/spy/dao/BUILD

@@ -0,0 +1,69 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "account_test.go",
+        "dao_test.go",
+        "memcache_test.go",
+        "mysql_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "account.go",
+        "dao.go",
+        "hbase.go",
+        "memcache.go",
+        "mysql.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/spy/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/dao/account.go

@@ -0,0 +1,33 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"go-common/library/log"
+)
+
+// BlockAccount block account
+func (d *Dao) BlockAccount(c context.Context, mid int64, reason string) (err error) {
+	params := url.Values{}
+	params.Set("mid", fmt.Sprintf("%d", mid))
+	params.Set("admin_reason", reason)
+	params.Set("blockType", "6")
+	params.Set("operator", "anticheat")
+	params.Set("type", "json")
+	var resp struct {
+		Code int64 `json:"code"`
+	}
+	// get
+	if err = d.httpClient.Get(c, d.c.Property.BlockAccountURL, "", params, &resp); err != nil {
+		log.Error("httpClient.Do() error(%v)", err)
+		return
+	}
+	if resp.Code != 0 {
+		err = fmt.Errorf("GET block account url resp(%v)", resp)
+		return
+	}
+	log.Info("account user(%d) block suc(%v)", mid, resp)
+	return
+}

app/job/main/spy/dao/account_test.go

@@ -0,0 +1,18 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+	gock "gopkg.in/h2non/gock.v1"
+)
+
+func TestDaoBlockAccount(t *testing.T) {
+	convey.Convey("BlockAccount", t, func() {
+		defer gock.OffAll()
+		httpMock("GET", d.c.Property.BlockAccountURL).Reply(200).JSON(`{"code":0}`)
+		err := d.BlockAccount(context.TODO(), 7593623, "1")
+		convey.So(err, convey.ShouldBeNil)
+	})
+}

app/job/main/spy/dao/dao.go

@@ -0,0 +1,71 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/hbase.v2"
+	"go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao event dao def.
+type Dao struct {
+	c             *conf.Config
+	db            *sql.DB
+	mc            *memcache.Pool
+	hbase         *hbase.Client
+	redis         *redis.Pool
+	httpClient    *bm.Client
+	expire        int
+	msgUUIDExpire int
+}
+
+// New create instance of dao and return.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:             c,
+		db:            sql.NewMySQL(c.DB),
+		mc:            memcache.NewPool(c.Memcache),
+		redis:         redis.NewPool(c.Redis.Config),
+		httpClient:    bm.NewClient(c.HTTPClient),
+		expire:        int(time.Duration(c.Redis.Expire) / time.Second),
+		msgUUIDExpire: int(time.Duration(c.Redis.MsgUUIDExpire) / time.Second),
+	}
+	if c.HBase != nil {
+		d.hbase = hbase.NewClient(c.HBase.Config)
+	}
+	return
+}
+
+// Ping check db health.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.pingMC(c); err != nil {
+		return
+	}
+	if err = d.PingRedis(c); err != nil {
+		return
+	}
+	if d.hbase != nil {
+		if err = d.hbase.Ping(c); err != nil {
+			return
+		}
+	}
+	return d.db.Ping(c)
+}
+
+// Close close all db connections.
+func (d *Dao) Close() {
+	if d.db != nil {
+		d.db.Close()
+	}
+	if d.redis != nil {
+		d.redis.Close()
+	}
+	if d.mc != nil {
+		d.mc.Close()
+	}
+}

app/job/main/spy/dao/dao_test.go

@@ -0,0 +1,58 @@
+package dao
+
+import (
+	"context"
+	"flag"
+	"os"
+	"strings"
+	"testing"
+
+	"go-common/app/job/main/spy/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+	gock "gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d *Dao
+	c = context.Background()
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.account-law.spy-job")
+		flag.Set("conf_appid", "main.account-law.spy-job")
+		flag.Set("conf_token", "1404609bda3db4cd20ad9823bfd20a83")
+		flag.Set("tree_id", "2856")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_env", "10")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/spy-job-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.httpClient.SetTransport(gock.DefaultTransport)
+	os.Exit(m.Run())
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}
+
+func TestPing(t *testing.T) {
+	Convey("TestPing", t, func() {
+		err := d.Ping(c)
+		So(err, ShouldBeNil)
+	})
+
+}

app/job/main/spy/dao/hbase.go

@@ -0,0 +1,55 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/spy/model"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+var (
+	// table
+	_hbaseTableActive = "active_data"
+	// family
+	_familyDays  = "activeDays"
+	_familyDaysB = []byte(_familyDays)
+)
+
+func strRowKey(mid int64) string {
+	return fmt.Sprintf("%d", mid)
+}
+
+// GetActiveData get user active days and watched bangumi video time
+func (dao *Dao) GetActiveData(c context.Context, mid int64) (active *model.Active, err error) {
+	var (
+		result      *hrpc.Result
+		key         = strRowKey(mid)
+		ctx, cancel = context.WithTimeout(c, time.Duration(dao.c.HBase.ReadTimeout))
+	)
+	defer cancel()
+	if result, err = dao.hbase.GetStr(ctx, _hbaseTableActive, key); err != nil {
+		err = errors.Wrapf(err, "hbase.GetStr(%s,%s)", _hbaseTableActive, key)
+		return
+	}
+	active = &model.Active{}
+	for _, c := range result.Cells {
+		h := &model.Active{}
+		if c != nil && bytes.Equal(c.Family, _familyDaysB) {
+			days, err := strconv.ParseInt(string(c.Qualifier), 10, 64)
+			if err != nil {
+				log.Error("strconv.ParseInt err(%v)", err)
+				continue
+			}
+			h.Active = days
+			active = h
+		}
+	}
+	return
+}

app/job/main/spy/dao/memcache.go

@@ -0,0 +1,63 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_partitionKey = "p_%s_%d"
+)
+
+// pingMC ping memcache.
+func (d *Dao) pingMC(ctx context.Context) (err error) {
+	conn := d.mc.Get(ctx)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: "ping", Value: []byte{1}, Expiration: 0}); err != nil {
+		log.Error("conn.Store(set,ping,1) error(%v)", err)
+	}
+	return
+}
+
+func partitionCacheKey(e string, p int32) string {
+	return fmt.Sprintf(_partitionKey, e, p)
+}
+
+// OffsetCache .
+func (d *Dao) OffsetCache(ctx context.Context, event string, p int32) (offset int64, err error) {
+	var (
+		key  = partitionCacheKey(event, p)
+		conn = d.mc.Get(ctx)
+	)
+	defer conn.Close()
+	reply, err := conn.Get(key)
+	if err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+			return
+		}
+		log.Error("conn.Get(get,%d) error(%v)", p, err)
+		return
+	}
+	if err = conn.Scan(reply, &offset); err != nil {
+		log.Error("reply.Scan(%s) error(%v)", string(reply.Value), err)
+	}
+	return
+}
+
+// SetOffsetCache .
+func (d *Dao) SetOffsetCache(ctx context.Context, event string, p int32, offset int64) (err error) {
+	var (
+		key  = partitionCacheKey(event, p)
+		conn = d.mc.Get(ctx)
+	)
+	defer conn.Close()
+
+	if err = conn.Set(&memcache.Item{Key: key, Object: offset, Expiration: 0, Flags: memcache.FlagJSON}); err != nil {
+		log.Error("conn.Set(%s,%d) error(%v)", key, offset, err)
+	}
+	return
+}

app/job/main/spy/dao/memcache_test.go

@@ -0,0 +1,22 @@
+package dao
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestMCInfo(t *testing.T) {
+	Convey("TestMCInfo", t, func() {
+		var (
+			partition int32 = -2233
+			offset1   int64 = 778899
+			offset2   int64
+			err       error
+		)
+		err = d.SetOffsetCache(c, "test", partition, offset1)
+		So(err, ShouldBeNil)
+		offset2, err = d.OffsetCache(c, "test", partition)
+		So(offset1, ShouldEqual, offset2)
+	})
+}

app/job/main/spy/dao/mysql.go

@@ -0,0 +1,295 @@
+package dao
+
+import (
+	"context"
+	xsql "database/sql"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_reBuildMidCountSQL    = "SELECT count(mid) FROM spy_user_info_%02d WHERE state=? AND score>=30 AND mtime BETWEEN ? AND ?;"
+	_reBuildMidSQL         = "SELECT mid FROM spy_user_info_%02d WHERE state=? AND score>=30 AND mtime BETWEEN ? AND ? LIMIT ?;"
+	_getAllConfigSQL       = "SELECT id, property,name,val,ctime FROM spy_system_config;"
+	_addEventHistorySQL    = "INSERT INTO spy_user_event_history_%02d (mid,event_id,score,base_score,event_score,remark,reason,factor_val,ctime) VALUES (?,?,?,?,?,?,?,?,?);"
+	_addPunishmentSQL      = "INSERT INTO spy_punishment (mid,type,reason,batch_no,ctime) VALUES (?,?,?,?,?);"
+	_updateUserStateSQL    = "UPDATE spy_user_info_%02d SET state=? WHERE mid=?"
+	_getLastHistorySQL     = "SELECT id,mid,event_id,score,base_score,event_score,remark,reason,factor_val,ctime FROM spy_user_event_history_%02d WHERE mid=? ORDER BY id DESC LIMIT 1;"
+	_getHistoryListSQL     = "SELECT remark,reason FROM spy_user_event_history_%02d WHERE mid= ? ORDER BY id DESC LIMIT ?;"
+	_updateEventScoreSQL   = "UPDATE spy_user_info_%02d SET event_score=?, score=? WHERE mid=?;"
+	_userInfoSQL           = "SELECT id,mid,score,base_score,event_score,state,ctime,mtime FROM spy_user_info_%02d WHERE mid=? LIMIT 1;"
+	_punishmentCountSQL    = "SELECT COUNT(1) FROM spy_punishment where mtime > ? and mtime < ?;"
+	_securityLoginCountSQL = "SELECT COUNT(1) FROM spy_user_event_history_%02d where reason = ? and mtime > ? and mtime < ?;"
+	_insertReportSQL       = "INSERT INTO `spy_report`(`name`,`date_version`,`val`,`ctime`)VALUES(?,?,?,?);"
+	_insertIncrStatSQL     = "INSERT INTO `spy_statistics`(`target_mid`,`target_id`,`event_id`,`state`,`type`,`quantity`,`ctime`)VALUES(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE quantity=quantity+?;"
+	_insertStatSQL         = "INSERT INTO `spy_statistics`(`target_mid`,`target_id`,`event_id`,`state`,`type`,`quantity`,`ctime`)VALUES(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE quantity=?;"
+	_allEventSQL           = "SELECT id,name,nick_name,service_id,status,ctime,mtime FROM spy_event WHERE status<>0"
+)
+
+func hitHistory(id int64) int64 {
+	return id % conf.Conf.Property.HistoryShard
+}
+
+func hitInfo(id int64) int64 {
+	return id % conf.Conf.Property.UserInfoShard
+}
+
+// BeginTran begin transaction.
+func (d *Dao) BeginTran(c context.Context) (*sql.Tx, error) {
+	return d.db.Begin(c)
+}
+
+// ReBuildMidCount count for need reBuild user.
+func (d *Dao) ReBuildMidCount(c context.Context, i int, state int8, start, end time.Time) (res int64, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_reBuildMidCountSQL, i), state, start, end)
+	if err = row.Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// ReBuildMidList query reBuild user mid list by page.
+func (d *Dao) ReBuildMidList(c context.Context, i int, t int8, start, end time.Time, ps int64) (res []int64, err error) {
+	var rows *sql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_reBuildMidSQL, i), t, start, end, ps); err != nil {
+		log.Error("d.reBuildMidSQL.Query(%d, %s, %s, %d) error(%v)", t, start, end, ps, err)
+		return
+	}
+	defer rows.Close()
+	res = []int64{}
+	for rows.Next() {
+		var r int64
+		if err = rows.Scan(&r); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			res = nil
+			return
+		}
+		res = append(res, r)
+	}
+	err = rows.Err()
+	return
+}
+
+// Configs spy system configs.
+func (d *Dao) Configs(c context.Context) (res map[string]string, err error) {
+	var rows *sql.Rows
+	if rows, err = d.db.Query(c, _getAllConfigSQL); err != nil {
+		log.Error("d.getAllConfigSQL.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	res = map[string]string{}
+	for rows.Next() {
+		var r model.Config
+		if err = rows.Scan(&r.ID, &r.Property, &r.Name, &r.Val, &r.Ctime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			res = nil
+			return
+		}
+		res[r.Property] = r.Val
+	}
+	err = rows.Err()
+	return
+}
+
+// TxAddEventHistory insert user_event_history.
+func (d *Dao) TxAddEventHistory(c context.Context, tx *sql.Tx, ueh *model.UserEventHistory) (err error) {
+	var (
+		now = time.Now()
+	)
+	if _, err = tx.Exec(fmt.Sprintf(_addEventHistorySQL, hitHistory(ueh.Mid)), ueh.Mid, ueh.EventID, ueh.Score, ueh.BaseScore, ueh.EventScore, ueh.Remark, ueh.Reason, ueh.FactorVal, now); err != nil {
+		log.Error("db.Exec(%v) error(%v)", ueh, err)
+		return
+	}
+	return
+}
+
+// TxAddPunishment insert punishment.
+func (d *Dao) TxAddPunishment(c context.Context, tx *sql.Tx, mid int64, t int8, reason string, blockNo int64) (err error) {
+	var (
+		now = time.Now()
+	)
+	if _, err = tx.Exec(_addPunishmentSQL, mid, t, reason, blockNo, now); err != nil {
+		log.Error("db.Exec(%d, %d, %s) error(%v)", mid, t, reason, err)
+		return
+	}
+	return
+}
+
+// History get last one user history.
+func (d *Dao) History(c context.Context, mid int64) (h *model.UserEventHistory, err error) {
+	var (
+		row *sql.Row
+	)
+	h = &model.UserEventHistory{}
+	row = d.db.QueryRow(c, fmt.Sprintf(_getLastHistorySQL, hitHistory(mid)), mid)
+	if err = row.Scan(&h.ID, &h.Mid, &h.EventID, &h.Score, &h.BaseScore, &h.EventScore, &h.Remark, &h.Reason, &h.FactorVal, &h.CTime); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			h = nil
+			return
+		}
+		log.Error("History row.Scan(%d) error(%v)", mid, err)
+	}
+	return
+}
+
+// TxUpdateUserState insert or update  user state by mid.
+func (d *Dao) TxUpdateUserState(c context.Context, tx *sql.Tx, info *model.UserInfo) (err error) {
+	if _, err = d.db.Exec(c, fmt.Sprintf(_updateUserStateSQL, hitInfo(info.Mid)), info.State, info.Mid); err != nil {
+		log.Error("TxUpdateUserState db.Exec(%d, %v) error(%v)", info.Mid, info, err)
+		return
+	}
+	return
+}
+
+// HistoryList query .
+func (d *Dao) HistoryList(c context.Context, mid int64, size int) (res []*model.UserEventHistory, err error) {
+	var rows *sql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_getHistoryListSQL, hitHistory(mid)), mid, size); err != nil {
+		log.Error("d.HistoryList.Query(%d, %d) error(%v)", mid, size, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		r := &model.UserEventHistory{}
+		if err = rows.Scan(&r.Remark, &r.Reason); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			res = nil
+			return
+		}
+		res = append(res, r)
+	}
+	err = rows.Err()
+	return
+}
+
+// UserInfo get info by mid.
+func (d *Dao) UserInfo(c context.Context, mid int64) (res *model.UserInfo, err error) {
+	var (
+		row *sql.Row
+	)
+	res = &model.UserInfo{}
+	row = d.db.QueryRow(c, fmt.Sprintf(_userInfoSQL, hitInfo(mid)), mid)
+	if err = row.Scan(&res.ID, &res.Mid, &res.Score, &res.BaseScore, &res.EventScore, &res.State, &res.CTime, &res.MTime); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			res = nil
+			return
+		}
+		log.Error("row.Scan() error(%v)", err)
+	}
+	return
+}
+
+// TxUpdateEventScore update event score.
+func (d *Dao) TxUpdateEventScore(c context.Context, tx *sql.Tx, mid int64, escore, score int8) (err error) {
+	if _, err = tx.Exec(fmt.Sprintf(_updateEventScoreSQL, hitInfo(mid)), escore, score, mid); err != nil {
+		log.Error("db.TxUpdateEventScore(%s, %d, %d, %d) error(%v)", _updateEventScoreSQL, escore, score, mid, err)
+		return
+	}
+	return
+}
+
+// AddReport add report info.
+func (d *Dao) AddReport(c context.Context, r *model.Report) (affected int64, err error) {
+	var res xsql.Result
+	if res, err = d.db.Exec(c, _insertReportSQL, r.Name, r.DateVersion, r.Val, r.Ctime); err != nil {
+		log.Error("AddReport: db.Exec(%v) error(%v)", r, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// PunishmentCount punishment count.
+func (d *Dao) PunishmentCount(c context.Context, start, end time.Time) (res int64, err error) {
+	row := d.db.QueryRow(c, _punishmentCountSQL, start, end)
+	if err = row.Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// SecurityLoginCount security login count.
+func (d *Dao) SecurityLoginCount(c context.Context, index int64, reason string, stime, etime time.Time) (res int64, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_securityLoginCountSQL, hitHistory(index)), reason, stime, etime)
+	if err = row.Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// AddStatistics add statistics info.
+func (d *Dao) AddStatistics(c context.Context, s *model.Statistics) (id int64, err error) {
+	var res xsql.Result
+	if res, err = d.db.Exec(c, _insertStatSQL, s.TargetMid, s.TargetID, s.EventID, s.State, s.Type, s.Quantity, s.Ctime, s.Quantity); err != nil {
+		return
+	}
+	if id, err = res.LastInsertId(); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}
+
+// AddIncrStatistics add increase statistics info.
+func (d *Dao) AddIncrStatistics(c context.Context, s *model.Statistics) (id int64, err error) {
+	var res xsql.Result
+	if res, err = d.db.Exec(c, _insertIncrStatSQL, s.TargetMid, s.TargetID, s.EventID, s.State, s.Type, s.Quantity, s.Ctime, s.Quantity); err != nil {
+		return
+	}
+	if id, err = res.LastInsertId(); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}
+
+//AllEvent all event.
+func (d *Dao) AllEvent(c context.Context) (list []*model.Event, err error) {
+	var (
+		rows *sql.Rows
+	)
+	list = make([]*model.Event, 0)
+	if rows, err = d.db.Query(c, _allEventSQL); err != nil {
+		log.Error("d.db.Query(%s) error(%v)", _allEventSQL, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var event = &model.Event{}
+		if err = rows.Scan(&event.ID, &event.Name, &event.NickName, &event.ServiceID, &event.Status, &event.Ctime, &event.Mtime); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		list = append(list, &model.Event{
+			ID:        event.ID,
+			Name:      event.Name,
+			NickName:  event.NickName,
+			ServiceID: event.ServiceID,
+			Status:    event.Status,
+			Ctime:     event.Ctime,
+			Mtime:     event.Mtime,
+		})
+	}
+	return
+}

app/job/main/spy/dao/mysql_test.go

@@ -0,0 +1,175 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/spy/model"
+	"go-common/library/database/sql"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testMysqlMid int64 = 15555180
+	size               = 3
+)
+
+func Test_Configs(t *testing.T) {
+	Convey("Test_Configs info data", t, func() {
+		res, err := d.Configs(context.TODO())
+		fmt.Println(res)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+		So(res, ShouldContainKey, model.LimitBlockCount)
+		So(res, ShouldContainKey, model.LessBlockScore)
+		So(res, ShouldContainKey, model.AutoBlock)
+	})
+}
+
+func Test_History(t *testing.T) {
+	Convey("Test_History get history data", t, func() {
+		res, err := d.History(context.TODO(), testMysqlMid)
+		fmt.Println(res)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	})
+}
+
+func Test_HistoryList(t *testing.T) {
+	Convey("Test_HistoryList get history list data", t, func() {
+		res, err := d.HistoryList(context.TODO(), testMysqlMid, size)
+		fmt.Println(res)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	})
+}
+
+func Test_TxUpdateUserState(t *testing.T) {
+	Convey("Test_TxUpdateUserState no err", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+			tx  *sql.Tx
+			ui  = &model.UserInfo{Mid: 15555180}
+		)
+		tx, err = d.db.Begin(c)
+		So(err, ShouldBeNil)
+		ui.State = model.StateNormal
+		err = d.TxUpdateUserState(context.TODO(), tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxAddPunishment(t *testing.T) {
+	Convey("Test_TxAddPunishment add data", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+			tx  *sql.Tx
+		)
+		tx, err = d.db.Begin(c)
+		So(err, ShouldBeNil)
+		err = d.TxAddPunishment(context.TODO(), tx, testMysqlMid, 0, "test封禁", 100000000)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+//  go test  -test.v -test.run Test_Stat
+func Test_Stat(t *testing.T) {
+	Convey("AddIncrStatistics", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+		)
+		stat := &model.Statistics{
+			TargetMid: 1,
+			TargetID:  1,
+			EventID:   1,
+			State:     0,
+			Type:      1,
+			Quantity:  1,
+			Ctime:     time.Now(),
+		}
+		_, err = d.AddIncrStatistics(c, stat)
+		So(err, ShouldBeNil)
+	})
+	Convey("AddStatistics", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+		)
+		stat := &model.Statistics{
+			TargetMid: 1,
+			TargetID:  1,
+			EventID:   2,
+			State:     0,
+			Type:      1,
+			Quantity:  1,
+			Ctime:     time.Now(),
+		}
+		stat.Quantity = 3
+		_, err = d.AddStatistics(c, stat)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxAddEventHistory(t *testing.T) {
+	Convey("test userinfo", t, func() {
+		res, err := d.UserInfo(c, 7593623)
+		So(err, ShouldBeNil)
+		if res == nil {
+			_, err = d.db.Exec(c, "insert into spy_user_info_23 (mid) values (7593623)")
+			So(err, ShouldBeNil)
+		}
+	})
+	var tx *sql.Tx
+	Convey("Test_TxAddEventHistory start", t, func() {
+		var err error
+		tx, err = d.BeginTran(c)
+		So(err, ShouldBeNil)
+	})
+	ueh := &model.UserEventHistory{Mid: 7593634, BaseScore: 100}
+	Convey("Test_TxAddEventHistory", t, func() {
+
+		err := d.TxAddEventHistory(c, tx, ueh)
+		So(err, ShouldBeNil)
+	})
+	Convey("TxUpdateEventScore", t, func() {
+		err := d.TxUpdateEventScore(c, tx, 7593623, 100, 100)
+		So(err, ShouldBeNil)
+	})
+	Convey("Test_TxAddEventHistory commit", t, func() {
+		err := tx.Commit()
+		So(err, ShouldBeNil)
+	})
+	Convey("AllEvent", t, func() {
+		res, err := d.AllEvent(c)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+	})
+	Convey("clean data", t, func() {
+		d.db.Exec(c, "delete from spy_user_event_history_23 where mid = 7593623")
+
+	})
+}
+
+func Test_SecurityLoginCount(t *testing.T) {
+	Convey("Test_SecurityLoginCount", t, func() {
+		_, err := d.SecurityLoginCount(c, 1, "test", time.Now(), time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_ReBuildMidList(t *testing.T) {
+	Convey("Test_ReBuildMidList", t, func() {
+		_, err := d.ReBuildMidList(c, 1, 1, time.Now(), time.Now(), 1)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/spy/dao/redis.go

@@ -0,0 +1,227 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_keyWaitBlock   = "wb_" // b_batch_no wait block
+	_keyBlock       = "bl_" // b_batch_no  block
+	_preLock        = "lk_"
+	_keyUniqueCheck = "uc:"
+	times           = 3
+)
+
+// keyWaitBlock return block cache key.
+func keyWaitBlock(batchNo int64) string {
+	return _keyWaitBlock + fmt.Sprintf("%d", batchNo)
+}
+
+// keyBlock return block cache key.
+func keyBlock() string {
+	return _keyBlock
+}
+
+func lockKey(key string) string {
+	return _preLock + key
+}
+
+func uniqueCheckKey(uuid string) string {
+	return _keyUniqueCheck + uuid
+}
+
+// BlockMidCache get wait block mids.
+func (d *Dao) BlockMidCache(c context.Context, batchNo int64, num int64) (res []int64, err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = keyWaitBlock(batchNo)
+	)
+	defer conn.Close()
+	if res, err = redis.Int64s(conn.Do("ZREVRANGEBYSCORE", key, "+inf", "-inf", "LIMIT", 0, num)); err != nil {
+		log.Error("redis(ZREVRANGEBYSCORE %s,%d) error(%v)", key, num, err)
+		return
+	}
+	return
+}
+
+// DelBlockCache delete the wait block redis.
+func (d *Dao) DelBlockCache(c context.Context, batchNo int64, mid int64) (err error) {
+	var (
+		key  = keyWaitBlock(batchNo)
+		args = []interface{}{key, mid}
+	)
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if err = conn.Send("ZREM", args...); err != nil {
+		log.Error("conn.Send(ZREM %s,%v) error(%v)", key, mid, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	if _, err = conn.Receive(); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+		return
+	}
+	return
+}
+
+//SetNXLockCache redis lock.
+func (d *Dao) SetNXLockCache(c context.Context, k string, times int64) (res bool, err error) {
+	var (
+		key  = lockKey(k)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if res, err = redis.Bool(conn.Do("SETNX", key, "1")); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("conn.Do(SETNX(%d)) error(%v)", key, err)
+			return
+		}
+	}
+	if res {
+		if _, err = redis.Bool(conn.Do("EXPIRE", key, times)); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", key, times, err)
+			return
+		}
+	}
+	return
+}
+
+//DelLockCache del lock cache.
+func (d *Dao) DelLockCache(c context.Context, k string) (err error) {
+	var (
+		key  = lockKey(k)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("DEL", key); err != nil {
+		log.Error("conn.Do(del,%v) err(%v)", key, err)
+	}
+	return
+}
+
+//AddBlockCache add block cache.
+func (d *Dao) AddBlockCache(c context.Context, mid int64, score int8, blockNo int64) (err error) {
+	var (
+		key = keyWaitBlock(blockNo)
+	)
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if err = conn.Send("ZADD", key, score, mid); err != nil {
+		log.Error("conn.Send(ZADD %s,%d,%d) error(%v)", key, score, mid, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.expire); err != nil {
+		log.Error("conn.Send(EXPIRE) error(%v)", err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// SetBlockCache block.
+func (d *Dao) SetBlockCache(c context.Context, mids []int64) (err error) {
+	var (
+		key  = keyBlock()
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, mid := range mids {
+		if err = conn.Send("SADD", key, mid); err != nil {
+			log.Error("SADD conn.Send error(%v)", err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.expire); err != nil {
+		log.Error("EXPIRE conn.Send error(%v)", err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < len(mids); i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("SetBlockCache Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+//SPOPBlockCache pop mid.
+func (d *Dao) SPOPBlockCache(c context.Context) (mid int64, err error) {
+	var (
+		key  = keyBlock()
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if mid, err = redis.Int64(conn.Do("SPOP", key)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("SPOP conn.Do(%s,%v) err(%v)", key, err)
+		}
+	}
+	return
+}
+
+// PingRedis check redis connection
+func (d *Dao) PingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	_, err = conn.Do("SET", "PING", "PONG")
+	conn.Close()
+	return
+}
+
+// PfaddCache SetNX.
+func (d *Dao) PfaddCache(c context.Context, uuid string) (ok bool, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	key := uniqueCheckKey(uuid)
+	if err = conn.Send("SETNX", key, 1); err != nil {
+		log.Error("SETNX conn.Send error(%v)", err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.msgUUIDExpire); err != nil {
+		log.Error("conn.Send(EXPIRE) error(%v)", err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("DelLock conn.Flush() error(%v)", err)
+		return
+	}
+	if ok, err = redis.Bool(conn.Receive()); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+		return
+	}
+	if _, err = conn.Receive(); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+	}
+	return
+}
+
+// TTL get redis cache ttl.
+func (d *Dao) TTL(c context.Context, key string) (ttl int64, err error) {
+	conn := d.redis.Get(c)
+	ttl, err = redis.Int64(conn.Do("TTL", key))
+	conn.Close()
+	return
+}

app/job/main/spy/dao/redis_test.go

@@ -0,0 +1,116 @@
+package dao
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"strings"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testAddMidRedis int64 = 15555180
+	testLowScore    int8  = 7
+	testBlockNo     int64 = 100000
+	testKey               = "test_lock"
+	testMidList           = []int64{15555180}
+)
+
+func Test_PingRedis(t *testing.T) {
+	Convey("ping redis", t, func() {
+		So(d.PingRedis(context.TODO()), ShouldBeNil)
+	})
+}
+
+func Test_AddBlockCache(t *testing.T) {
+	Convey("AddBlockCache redis ", t, func() {
+		c := context.TODO()
+		ret, err := d.BlockMidCache(c, testBlockNo, 10)
+		So(ret, ShouldBeEmpty)
+		So(err, ShouldBeNil)
+
+		err = d.AddBlockCache(c, testAddMidRedis, testLowScore, testBlockNo)
+		So(err, ShouldBeNil)
+
+		ret, err = d.BlockMidCache(c, testBlockNo, 10)
+		So(ret, ShouldContain, testAddMidRedis)
+		So(err, ShouldBeNil)
+
+		err = d.DelBlockCache(c, testBlockNo, testAddMidRedis)
+		So(err, ShouldBeNil)
+
+		ret, err = d.BlockMidCache(c, testBlockNo, 10)
+		So(ret, ShouldNotContain, testAddMidRedis)
+		So(err, ShouldBeNil)
+
+	})
+}
+
+func Test_SetNXLockCache(t *testing.T) {
+	Convey("SetNXLockCache", t, func() {
+		ret, err := d.SetNXLockCache(c, testKey, 2)
+		So(err, ShouldBeNil)
+		So(ret, ShouldBeTrue)
+
+		err = d.DelLockCache(c, testKey)
+		So(err, ShouldBeNil)
+
+		ret, err = d.SetNXLockCache(c, testKey, 2)
+		So(err, ShouldBeNil)
+		So(ret, ShouldBeTrue)
+
+		ret, err = d.SetNXLockCache(c, testKey, 2)
+		So(err, ShouldBeNil)
+		So(ret, ShouldBeFalse)
+	})
+
+}
+
+func Test_SetBlockCache(t *testing.T) {
+	Convey("ping SetBlockCache", t, func() {
+		err := d.SetBlockCache(context.TODO(), testMidList)
+		So(err, ShouldBeNil)
+		Convey("ping SetBlockCache", func() {
+			mid, err := d.SPOPBlockCache(context.TODO())
+			So(err, ShouldBeNil)
+			So(testMidList, ShouldContain, mid)
+			Convey("ping SetBlockCache 2", func() {
+				mid, err := d.SPOPBlockCache(context.TODO())
+				So(err, ShouldBeNil)
+				So(mid == 0, ShouldBeTrue)
+			})
+		})
+	})
+}
+
+// go test  -test.v -test.run TestPfaddCache
+func TestPfaddCache(t *testing.T) {
+	Convey("PfaddCache", t, func() {
+		idx := strings.Replace(randHex(), "-", "", -1)
+		ok, err := d.PfaddCache(context.TODO(), idx)
+		So(err, ShouldBeNil)
+		So(ok, ShouldBeTrue)
+		ok, err = d.PfaddCache(context.TODO(), idx)
+		So(err, ShouldBeNil)
+		So(ok, ShouldBeFalse)
+	})
+}
+
+// go test  -test.v -test.run TestBatchPfaddCache
+func TestBatchPfaddCache(t *testing.T) {
+	Convey("PfaddCache", t, func() {
+		for i := 0; i < 10; i++ {
+			idx := strings.Replace(randHex(), "-", "", -1)
+			_, err := d.PfaddCache(context.TODO(), idx)
+			So(err, ShouldBeNil)
+		}
+	})
+}
+
+func randHex() string {
+	bs := make([]byte, 16)
+	rand.Read(bs)
+	return hex.EncodeToString(bs)
+}