Create & Init Project...

app/job/main/spy/service/BUILD

@@ -0,0 +1,69 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "block_test.go",
+        "rebuild_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "block.go",
+        "rebuild.go",
+        "report.go",
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/spy/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/dao:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//app/service/main/spy/model:go_default_library",
+        "//app/service/main/spy/rpc/client:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/service/block.go

@@ -0,0 +1,173 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+func (s *Service) lastBlockNo(seconds int64) int64 {
+	return time.Now().Unix()/seconds - 1 //cron -1 get the last hours block mids
+}
+
+//BlockTask block task.
+func (s *Service) BlockTask(c context.Context) {
+	var (
+		blockNo = s.lastBlockNo(s.c.Property.Block.CycleTimes)
+	)
+	v, ok := s.Config(model.AutoBlock)
+	if !ok {
+		log.Error("Verfiy get config error(%s,%v)", model.AutoBlock, s.spyConfig)
+		return
+	}
+	if v.(int8) != model.AutoBlockOpen {
+		log.Info("autoBlock Close(%s)", blockNo)
+		return
+	}
+	mids, _ := s.blockUsers(c, blockNo)
+	if len(mids) == 0 {
+		log.Info("s.blockUsers len is zero")
+		return
+	}
+	if err := s.dao.SetBlockCache(c, mids); err != nil {
+		log.Error("s.dao.SetBlockCache(%v) error(%v)", mids, err)
+		return
+	}
+}
+
+func (s *Service) blockUsers(c context.Context, blockNo int64) (mids []int64, err error) {
+	v, ok := s.Config(model.LimitBlockCount)
+	if !ok {
+		log.Error("blockUsers get config error(%v)", s.spyConfig)
+		return
+	}
+	if mids, err = s.dao.BlockMidCache(c, blockNo, v.(int64)); err != nil {
+		log.Error("s.dao.BlockMidCache(%s, %d) error(%v)", blockNo, v.(int64), err)
+		return
+	}
+	return
+}
+
+func (s *Service) blockByMid(c context.Context, mid int64) (err error) {
+	ui, ok := s.canBlock(c, mid)
+	if !ok {
+		log.Info("s.canBlock user had block(%d)", mid)
+		return
+	}
+	reason, remake := s.blockReason(c, mid)
+	if err = s.block(c, mid, ui, reason, remake); err != nil {
+		log.Error("s.block(%d,%v,%s) err(%v)", mid, ui, reason, err)
+	}
+	return
+}
+
+func (s *Service) block(c context.Context, mid int64, ui *model.UserInfo, reason string, remake string) (err error) {
+	var (
+		tx *sql.Tx
+	)
+	ui.State = model.StateBlock
+	if tx, err = s.dao.BeginTran(c); err != nil {
+		log.Error("s.dao.BeginTran() err(%v)", err)
+		return
+	}
+	defer func() {
+		if err != nil {
+			if err1 := tx.Rollback(); err1 != nil {
+				log.Error("tx.Rollback() error(%v)", err1)
+			}
+			return
+		}
+		if err = tx.Commit(); err != nil {
+			log.Error("tx.Commit() error(%v)", err)
+		}
+	}()
+	if err = s.dao.BlockAccount(c, ui.Mid, reason); err != nil {
+		log.Error("s.dao.BlockAccount(%d) error(%v)", ui.Mid, err)
+		return
+	}
+	var ueh = &model.UserEventHistory{
+		Mid:        mid,
+		EventID:    conf.Conf.Property.BlockEvent,
+		Score:      ui.Score,
+		BaseScore:  ui.BaseScore,
+		EventScore: ui.EventScore,
+		Remark:     remake,
+		Reason:     "自动封禁",
+	}
+
+	if err = s.dao.TxAddEventHistory(c, tx, ueh); err != nil {
+		log.Error("s.dao.TxAddEventHistory(%+v) error(%v)", ueh, err)
+		return
+	}
+	if err = s.dao.TxAddPunishment(c, tx, ui.Mid, model.PunishmentTypeBlock,
+		reason, s.lastBlockNo(s.c.Property.Block.CycleTimes)); err != nil {
+		log.Error("s.dao.TxAddPunishment(%d,%s) error(%v)", ui.Mid, reason, err)
+		return
+	}
+	// update user state.
+	if err = s.dao.TxUpdateUserState(c, tx, ui); err != nil {
+		log.Error("s.dao.TxUpdateUserState(%v) error(%v)", ui, err)
+		return
+	}
+	s.promBlockInfo.Incr("actual_block_count")
+	return
+}
+
+func (s *Service) canBlock(c context.Context, mid int64) (ui *model.UserInfo, ok bool) {
+	var (
+		err error
+	)
+	ui, err = s.dao.UserInfo(c, mid)
+	if err != nil {
+		log.Error("s.UserInfo(%d) err(%v)", mid, err)
+		return
+	}
+	v, b := s.Config(model.LessBlockScore)
+	if !b {
+		log.Error("scoreLessHandler get config error(%s,%v)", model.LessBlockScore, s.spyConfig)
+		return
+	}
+	// if blocked already , return
+	if ui.State == model.StateBlock || ui.Score > v.(int8) {
+		log.Info("canBlock not block(%v)", ui)
+		return
+	}
+	ok = true
+	return
+}
+
+func (s *Service) blockReason(c context.Context, mid int64) (reason string, remake string) {
+	var (
+		err error
+		hs  []*model.UserEventHistory
+		buf bytes.Buffer
+	)
+	if hs, err = s.dao.HistoryList(c, mid, model.BlockReasonSize); err != nil || len(hs) == 0 {
+		log.Error("s.dao.HistoryList(%d) err(%v)", mid, err)
+		return
+	}
+	m := make(map[string]int)
+	for i, v := range hs {
+		if i == 0 {
+			remake = v.Remark
+		}
+		if m[v.Reason] == 0 {
+			m[v.Reason] = 1
+		} else {
+			m[v.Reason] = m[v.Reason] + 1
+		}
+	}
+	for k, v := range m {
+		buf.WriteString(k)
+		buf.WriteString("x")
+		buf.WriteString(fmt.Sprintf("%d ", v))
+	}
+	reason = buf.String()
+	return
+}

app/job/main/spy/service/block_test.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/spy/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testBlockMid int64 = 4780461
+	testLowScore int8  = 7
+)
+
+func Test_BlockReason(t *testing.T) {
+	Convey("Test_BlockReason get block reason", t, WithService(func(s *Service) {
+		reason, remake := s.blockReason(context.TODO(), testBlockMid)
+		fmt.Println("reason remake", reason, remake)
+		So(reason, ShouldNotBeEmpty)
+		So(remake, ShouldNotBeEmpty)
+	}))
+}
+
+func Test_CanBlock(t *testing.T) {
+	Convey("Test_CanBlock can block ", t, WithService(func(s *Service) {
+		tx, err := s.dao.BeginTran(c)
+		So(err, ShouldBeNil)
+		ui := &model.UserInfo{Mid: testBlockMid, State: model.StateNormal}
+		err = s.dao.TxUpdateUserState(c, tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+
+		ui, ok := s.canBlock(context.TODO(), testBlockMid)
+		fmt.Println("Test_CanBlock ui ", ui, testBlockMid)
+		So(ui, ShouldNotBeNil)
+		So(ok, ShouldBeTrue)
+	}))
+}
+
+func Test_Block(t *testing.T) {
+	Convey("Test_Block block ", t, WithService(func(s *Service) {
+		ui, err := s.dao.UserInfo(context.TODO(), testBlockMid)
+		So(err, ShouldBeNil)
+
+		tx, err := s.dao.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		ui.State = model.StateNormal
+		err = s.dao.TxUpdateUserState(c, tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+
+		ui, err = s.dao.UserInfo(context.TODO(), testBlockMid)
+		So(err, ShouldBeNil)
+		So(ui.State == model.StateNormal, ShouldBeTrue)
+
+		reason, remake := s.blockReason(context.TODO(), testBlockMid)
+		fmt.Println("reason remake", reason, remake)
+		So(reason, ShouldNotBeEmpty)
+		So(remake, ShouldNotBeEmpty)
+
+		Convey("Test_CanBlock do block ", WithService(func(s *Service) {
+			err := s.blockByMid(context.TODO(), testBlockMid)
+			So(err, ShouldBeNil)
+			Convey("Test_CanBlock get block user info ", WithService(func(s *Service) {
+				ui, err := s.dao.UserInfo(context.TODO(), testBlockMid)
+				So(err, ShouldBeNil)
+				So(ui.State == model.StateBlock, ShouldBeTrue)
+			}))
+		}))
+		fmt.Println("Test_Block end ")
+
+	}))
+}

app/job/main/spy/service/rebuild.go

@@ -0,0 +1,58 @@
+/*
+rebuild: user portrait score reset to normal if it's score large than punishment threshold score
+*/
+
+package service
+
+import (
+	"context"
+	"time"
+
+	spy "go-common/app/service/main/spy/model"
+	"go-common/library/log"
+)
+
+const (
+	_normal = 0
+	_ps     = 100
+)
+
+func (s *Service) reBuild() {
+	var (
+		err   error
+		count int64
+	)
+	current := time.Now()
+	before30d, _ := time.ParseDuration("-720h")
+	before31d, _ := time.ParseDuration("-744h")
+	start := current.Add(before31d)
+	end := current.Add(before30d)
+	log.Info("ReBuild task start: start:(%s) end:(%s))", start, end)
+	for t := 0; t < int(s.c.Property.UserInfoShard); t++ {
+		if count, err = s.dao.ReBuildMidCount(context.TODO(), t, _normal, start, end); err != nil {
+			log.Error("s.dao.ReBuildMidCount(%s, %s), err(%v)", start, end, err)
+			continue
+		}
+		log.Info("ReBuild task: index:%d, count:%d)", t, count)
+		if count <= 0 {
+			continue
+		}
+		total := count / _ps
+		log.Info("ReBuild task: shard:%d, count:%d, total:%d)", t, count, total)
+		for i := 0; int64(i) <= total; i++ {
+			midList, err := s.dao.ReBuildMidList(context.TODO(), t, _normal, start, end, _ps)
+			if err != nil {
+				log.Error("s.dao.ReBuildMidList(%s, %s, %d, %d)", start, end, i, _ps)
+				continue
+			}
+			for _, mid := range midList {
+				if err := s.spyRPC.ReBuildPortrait(context.TODO(), &spy.ArgReBuild{Mid: mid, Reason: "自动恢复行为得分"}); err != nil {
+					log.Error("s.spyRPC.ReBuildPortrait(%d), err:%v", mid, err)
+					continue
+				}
+				log.Info("ReBuild task: mid(%d) ReBuild Portrait success)", mid)
+			}
+		}
+	}
+	log.Info("ReBuild task end: start:(%s) end:(%s))", start, end)
+}

app/job/main/spy/service/rebuild_test.go

@@ -0,0 +1,27 @@
+package service
+
+import (
+	"flag"
+	"testing"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/library/log"
+)
+
+func TestServiceReBuild(t *testing.T) {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		t.Errorf("conf.Init() error(%v)", err)
+		t.FailNow()
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	if s == nil {
+		s = New(conf.Conf)
+	}
+	testReBuild(t, s)
+}
+
+func testReBuild(t *testing.T, s *Service) {
+	s.reBuild()
+}

app/job/main/spy/service/report.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/log"
+)
+
+// AddReport add daill report.
+func (s *Service) AddReport(c context.Context) {
+	var (
+		scount      int64
+		pcount      int64
+		dateVersion string
+		err         error
+	)
+	year, month, day := time.Now().Date()
+	stoday := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
+	etoday := time.Date(year, month, day, 23, 59, 59, 999, time.Local)
+	syesday := stoday.AddDate(0, 0, -1)
+	eyesday := etoday.AddDate(0, 0, -1)
+	dateVersion = syesday.Format("20060102")
+	if pcount, err = s.dao.PunishmentCount(c, syesday, eyesday); err != nil {
+		log.Error("s.dao.PunishmentCount(%s, %s), err(%v)", syesday, eyesday, err)
+		return
+	}
+	s.dao.AddReport(c, &model.Report{
+		Name:        model.BlockCount,
+		DateVersion: dateVersion,
+		Val:         pcount,
+		Ctime:       time.Now(),
+	})
+	for i := int64(0); i < conf.Conf.Property.HistoryShard; i++ {
+		var count int64
+		if count, err = s.dao.SecurityLoginCount(c, i, "导入二次验证,恢复行为得分", syesday, eyesday); err != nil {
+			log.Error("s.dao.SecurityLoginCount(%s, %s), err(%v)", syesday, eyesday, err)
+			return
+		}
+		scount = scount + count
+		time.Sleep(s.blockTick)
+	}
+	s.dao.AddReport(c, &model.Report{
+		Name:        model.SecurityLoginCount,
+		DateVersion: dateVersion,
+		Val:         scount,
+		Ctime:       time.Now(),
+	})
+}

app/job/main/spy/service/service.go

@@ -0,0 +1,591 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strconv"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/dao"
+	"go-common/app/job/main/spy/model"
+	cmmdl "go-common/app/service/main/spy/model"
+	spyrpc "go-common/app/service/main/spy/rpc/client"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/stat/prom"
+	xtime "go-common/library/time"
+
+	"github.com/robfig/cron"
+)
+
+const (
+	_bigdataEvent = "bt"
+	_secretEvent  = "st"
+)
+
+// Service service def.
+type Service struct {
+	c               *conf.Config
+	waiter          sync.WaitGroup
+	dao             *dao.Dao
+	eventDatabus    *databus.Databus
+	spystatDatabus  *databus.Databus
+	secLoginDatabus *databus.Databus
+	spyRPC          *spyrpc.Service
+	quit            chan struct{}
+	cachech         chan func()
+	retrych         chan func()
+	spyConfig       map[string]interface{}
+	configLoadTick  time.Duration
+	promBlockInfo   *prom.Prom
+	blockTick       time.Duration
+	blockWaitTick   time.Duration
+	allEventName    map[string]int64
+	loadEventTick   time.Duration
+	// activity events
+	activityEvents map[string]struct{}
+}
+
+// New create a instance of Service and return.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:               c,
+		dao:             dao.New(c),
+		eventDatabus:    databus.New(c.Databus.EventData),
+		spystatDatabus:  databus.New(c.Databus.SpyStatData),
+		secLoginDatabus: databus.New(c.Databus.SecLogin),
+		spyRPC:          spyrpc.New(c.SpyRPC),
+		quit:            make(chan struct{}),
+		cachech:         make(chan func(), 1024),
+		retrych:         make(chan func(), 128),
+		spyConfig:       make(map[string]interface{}),
+		configLoadTick:  time.Duration(c.Property.ConfigLoadTick),
+		promBlockInfo:   prom.New().WithCounter("spy_block_info", []string{"name"}),
+		blockTick:       time.Duration(c.Property.BlockTick),
+		blockWaitTick:   time.Duration(c.Property.BlockWaitTick),
+		allEventName:    make(map[string]int64),
+		loadEventTick:   time.Duration(c.Property.LoadEventTick),
+	}
+
+	if err := s.loadSystemConfig(); err != nil {
+		panic(err)
+	}
+	if err := s.loadeventname(); err != nil {
+		panic(err)
+	}
+	s.initActivityEvents()
+
+	s.waiter.Add(1)
+	go s.consumeproc()
+
+	s.waiter.Add(1)
+	go s.secloginproc()
+
+	s.waiter.Add(1)
+	go s.spystatproc()
+
+	go s.retryproc()
+	go s.cacheproc()
+
+	go s.loadconfig()
+	go s.blockcacheuser()
+
+	go s.loadeventproc()
+
+	t := cron.New()
+	if err := t.AddFunc(s.c.Property.Block.CycleCron, s.cycleblock); err != nil {
+		panic(err)
+	}
+	if err := t.AddFunc(s.c.Property.ReportCron, s.reportjob); err != nil {
+		panic(err)
+	}
+	t.Start()
+
+	return s
+}
+
+func (s *Service) consumeproc() {
+	defer s.waiter.Done()
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("eventproc unknown panic(%v)", x)
+		}
+	}()
+	var (
+		msg       *databus.Message
+		eventMsg  *model.EventMessage
+		ok        bool
+		err       error
+		msgChan   = s.eventDatabus.Messages()
+		c         = context.TODO()
+		preOffset int64
+	)
+	for {
+		select {
+		case msg, ok = <-msgChan:
+			if !ok {
+				log.Info("eventConsumeProc msgChan closed")
+				return
+			}
+		case <-s.quit:
+			log.Info("quit eventConsumeProc")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit error(%v)", err)
+		}
+		if preOffset, err = s.dao.OffsetCache(c, _bigdataEvent, msg.Partition); err != nil {
+			log.Error("s.dao.OffsetCache(%d) error(%v)", msg.Partition, err)
+			preOffset = 0
+		} else {
+			if msg.Offset > preOffset {
+				s.setOffset(_bigdataEvent, msg.Partition, msg.Offset)
+			}
+		}
+		eventMsg = &model.EventMessage{}
+		if err = json.Unmarshal([]byte(msg.Value), eventMsg); err != nil {
+			log.Error("json.Unmarshall(%s) error(%v)", msg.Value, err)
+			s.setOffset(_bigdataEvent, msg.Partition, msg.Offset)
+			continue
+		}
+		if msg.Offset <= preOffset && s.isMsgExpiration(eventMsg.Time) {
+			log.Error("drop expired msg (%+v) (now_offset : %d)", preOffset)
+			continue
+		}
+		if err = s.handleEvent(c, eventMsg); err != nil {
+			log.Error("s.HandleEvent(%v) error(%v)", eventMsg, err)
+			continue
+		}
+		log.Info("s.handleEvent(%v) eventMsg", eventMsg)
+	}
+}
+
+func (s *Service) setOffset(event string, partition int32, offset int64) {
+	s.cachemiss(func() {
+		var err error
+		if err = s.dao.SetOffsetCache(context.TODO(), event, partition, offset); err != nil {
+			log.Error("s.dao.SetOffsetCache(%d,%d) error(%v)", partition, offset, err)
+		}
+	})
+}
+
+func (s *Service) secloginproc() {
+	defer s.waiter.Done()
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("eventproc unknown panic(%v)", x)
+		}
+	}()
+	var (
+		msg       *databus.Message
+		eventMsg  *model.EventMessage
+		ok        bool
+		err       error
+		msgChan   = s.secLoginDatabus.Messages()
+		c         = context.TODO()
+		preOffset int64
+	)
+	for {
+		select {
+		case msg, ok = <-msgChan:
+			if !ok {
+				log.Info("secloginproc msgChan closed")
+				return
+			}
+		case <-s.quit:
+			log.Info("quit secloginproc")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit error(%v)", err)
+		}
+		if preOffset, err = s.dao.OffsetCache(c, _secretEvent, msg.Partition); err != nil {
+			log.Error("s.dao.OffsetCache(%d) error(%v)", msg.Partition, err)
+			preOffset = 0
+		} else {
+			if msg.Offset > preOffset {
+				s.setOffset(_secretEvent, msg.Partition, msg.Offset)
+			}
+		}
+		eventMsg = &model.EventMessage{}
+		if err = json.Unmarshal([]byte(msg.Value), eventMsg); err != nil {
+			log.Error("json.Unmarshall(%s) error(%v)", msg.Value, err)
+			s.setOffset(_secretEvent, msg.Partition, msg.Offset)
+			continue
+		}
+		if msg.Offset <= preOffset && s.isMsgExpiration(eventMsg.Time) {
+			log.Error("drop expired msg (%+v) (now_offset : %d)", preOffset)
+			continue
+		}
+		if err = s.handleEvent(c, eventMsg); err != nil {
+			log.Error("s.HandleEvent(%v) error(%v)", eventMsg, err)
+			continue
+		}
+		log.Info("s.handleEvent(%v) eventMsg", eventMsg)
+	}
+}
+
+func (s *Service) isMsgExpiration(timeStr string) bool {
+	var (
+		eventTime time.Time
+		now       = time.Now()
+		err       error
+	)
+	if eventTime, err = time.Parse("2006-01-02 15:04:05", timeStr); err != nil {
+		return true
+	}
+	return eventTime.AddDate(0, 1, 1).Before(now)
+}
+
+func (s *Service) handleEvent(c context.Context, event *model.EventMessage) (err error) {
+	var (
+		eventTime time.Time
+		argBytes  []byte
+	)
+	if eventTime, err = time.Parse("2006-01-02 15:04:05", event.Time); err != nil {
+		log.Error("time.Parse(%s) errore(%v)", event.Time, err)
+		return
+	}
+	if argBytes, err = json.Marshal(event.Args); err != nil {
+		log.Error("json.Marshal(%v) error(%v), so empty it", event.Args, err)
+		argBytes = []byte("{}")
+	}
+	var argHandleEvent = &cmmdl.ArgHandleEvent{
+		Time:      xtime.Time(eventTime.Unix()),
+		IP:        event.IP,
+		Service:   event.Service,
+		Event:     event.Event,
+		ActiveMid: event.ActiveMid,
+		TargetMid: event.TargetMid,
+		TargetID:  event.TargetID,
+		Args:      string(argBytes),
+		Result:    event.Result,
+		Effect:    event.Effect,
+		RiskLevel: event.RiskLevel,
+	}
+	if err = s.spyRPC.HandleEvent(c, argHandleEvent); err != nil {
+		log.Error("s.spyRPC.HandleEvent(%v) error(%v)", argHandleEvent, err)
+		// s.retrymiss(func() {
+		// 	log.Info("Start retry rpc error(%v)", err)
+		// 	for {
+		// 		if err = s.spyRPC.HandleEvent(context.TODO(), argHandleEvent); err != nil {
+		// 			log.Error("s.spyRPC.HandleEvent(%v) error(%v)", argHandleEvent, err)
+		// 		} else {
+		// 			break
+		// 		}
+		// 	}
+		// 	log.Info("End retry error(%v)", err)
+		// })
+		return
+	}
+	return
+}
+
+func (s *Service) dataproc() {
+	for {
+		select {
+		case <-s.quit:
+			log.Info("quit handleData")
+			return
+		default:
+		}
+		if !s.c.Property.Debug {
+			d := time.Now().AddDate(0, 0, 1)
+			ts := time.Date(d.Year(), d.Month(), d.Day(), 5, 0, 0, 0, time.Local).Sub(time.Now())
+			time.Sleep(ts)
+		} else {
+			time.Sleep(s.c.Property.TaskTimer * time.Second)
+		}
+		s.reBuild()
+	}
+}
+
+// Ping check service health.
+func (s *Service) Ping(c context.Context) (err error) {
+	if err = s.dao.Ping(c); err != nil {
+		log.Error("s.db.Ping() error(%v)", err)
+		return
+	}
+	return
+}
+
+// Close all resource.
+func (s *Service) Close() (err error) {
+	close(s.quit)
+	s.dao.Close()
+	if err = s.eventDatabus.Close(); err != nil {
+		log.Error("s.db.Close() error(%v)", err)
+		return
+	}
+	return
+}
+
+// Wait wait all closed.
+func (s *Service) Wait() {
+	s.waiter.Wait()
+}
+
+func (s *Service) cacheproc() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.cacheproc panic(%v)", x)
+			go s.cacheproc()
+			log.Info("service.cacheproc recover")
+		}
+	}()
+	for {
+		f := <-s.cachech
+		f()
+	}
+}
+
+func (s *Service) retryproc() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.retryproc panic(%v)", x)
+			go s.retryproc()
+			log.Info("service.retryproc recover")
+		}
+	}()
+	for {
+		f := <-s.retrych
+		go f()
+	}
+}
+
+func (s *Service) cachemiss(f func()) {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.cachemiss panic(%v)", x)
+		}
+	}()
+	select {
+	case s.cachech <- f:
+	default:
+		log.Error("service.cachech full")
+	}
+}
+
+func (s *Service) retrymiss(f func()) {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.retrymiss panic(%v)", x)
+		}
+	}()
+	select {
+	case s.retrych <- f:
+	default:
+		log.Error("service.retrych full")
+	}
+}
+
+func (s *Service) loadconfig() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.cycleblock panic(%v)", x)
+		}
+	}()
+	for {
+		time.Sleep(s.configLoadTick)
+		s.loadSystemConfig()
+	}
+}
+
+func (s *Service) loadSystemConfig() (err error) {
+	var (
+		cdb map[string]string
+	)
+	cdb, err = s.dao.Configs(context.TODO())
+	if err != nil {
+		log.Error("sys config db get data err(%v)", err)
+		return
+	}
+	if len(cdb) == 0 {
+		err = errors.New("sys config no data")
+		return
+	}
+	cs := make(map[string]interface{}, len(cdb))
+	for k, v := range cdb {
+		switch k {
+		case model.LimitBlockCount:
+			t, err1 := strconv.ParseInt(v, 10, 64)
+			if err1 != nil {
+				log.Error("sys config err(%s,%v,%v)", model.LimitBlockCount, t, err1)
+				err = err1
+				return
+			}
+			cs[k] = t
+		case model.LessBlockScore:
+			tmp, err1 := strconv.ParseInt(v, 10, 8)
+			if err1 != nil {
+				log.Error("sys config err(%s,%v,%v)", model.LessBlockScore, tmp, err1)
+				err = err1
+				return
+			}
+			cs[k] = int8(tmp)
+		case model.AutoBlock:
+			tmp, err1 := strconv.ParseInt(v, 10, 8)
+			if err1 != nil {
+				log.Error("sys config err(%s,%v,%v)", model.AutoBlock, tmp, err1)
+				err = err1
+				return
+			}
+			cs[k] = int8(tmp)
+		default:
+			cs[k] = v
+		}
+	}
+	s.spyConfig = cs
+	log.Info("loadSystemConfig success(%v)", cs)
+	return
+}
+
+//Config get config.
+func (s *Service) Config(key string) (interface{}, bool) {
+	if s.spyConfig == nil {
+		return nil, false
+	}
+	v, ok := s.spyConfig[key]
+	return v, ok
+}
+
+func (s *Service) cycleblock() {
+	var (
+		c = context.TODO()
+	)
+	log.Info("cycleblock start (%v)", time.Now())
+	if b, err := s.dao.SetNXLockCache(c, model.BlockLockKey, model.DefLockTime); !b || err != nil {
+		log.Error("cycleblock had run (%v,%v)", b, err)
+		return
+	}
+	s.BlockTask(c)
+	s.dao.DelLockCache(c, model.BlockLockKey)
+	log.Info("cycleblock end (%v)", time.Now())
+}
+
+func (s *Service) blockcacheuser() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.blockcacheuser panic(%v)", x)
+			go s.blockcacheuser()
+			log.Info("service.blockcacheuser recover")
+		}
+	}()
+	for {
+		mid, err := s.dao.SPOPBlockCache(context.TODO())
+		if err != nil {
+			log.Error("blockcacheuser err (%v,%v)", mid, err)
+			continue
+		}
+		if mid != 0 {
+			s.blockByMid(context.TODO(), mid)
+			time.Sleep(s.blockTick)
+		} else {
+			// when no user should be block
+			time.Sleep(s.blockWaitTick)
+		}
+	}
+}
+
+func (s *Service) reportjob() {
+	var (
+		c = context.TODO()
+	)
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.reportjob panic(%v)", x)
+			go s.blockcacheuser()
+			log.Info("service.reportjob recover")
+		}
+	}()
+	log.Info("reportjob start (%v)", time.Now())
+	if b, err := s.dao.SetNXLockCache(c, model.ReportJobKey, model.DefLockTime); !b || err != nil {
+		log.Error("reportjob had run (%v,%v)", b, err)
+		return
+	}
+	s.AddReport(c)
+	log.Info("reportjob end (%v)", time.Now())
+}
+
+func (s *Service) spystatproc() {
+	defer s.waiter.Done()
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("sinstatproc unknown panic(%v)", x)
+		}
+	}()
+	var (
+		msg     *databus.Message
+		statMsg *model.SpyStatMessage
+		ok      bool
+		err     error
+		msgChan = s.spystatDatabus.Messages()
+		c       = context.TODO()
+	)
+	for {
+		select {
+		case msg, ok = <-msgChan:
+			if !ok {
+				log.Info("spystatproc msgChan closed")
+				return
+			}
+		case <-s.quit:
+			log.Info("quit spystatproc")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit error(%v)", err)
+		}
+		statMsg = &model.SpyStatMessage{}
+		if err = json.Unmarshal([]byte(msg.Value), statMsg); err != nil {
+			log.Error("json.Unmarshall(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		log.Info(" spystatproc (%v) start", statMsg)
+		// check uuid
+		unique, _ := s.dao.PfaddCache(c, statMsg.UUID)
+		if !unique {
+			log.Error("stat duplicate msg (%s) error", statMsg)
+			continue
+		}
+		s.UpdateStatData(c, statMsg)
+		log.Info(" spystatproc (%v) handle", statMsg)
+	}
+}
+
+func (s *Service) loadeventname() (err error) {
+	var (
+		c  = context.Background()
+		es []*model.Event
+	)
+	es, err = s.dao.AllEvent(c)
+	if err != nil {
+		log.Error("loadeventname allevent error(%v)", err)
+		return
+	}
+	tmp := make(map[string]int64, len(es))
+	for _, e := range es {
+		tmp[e.Name] = e.ID
+	}
+	s.allEventName = tmp
+	log.Info("loadeventname (%v) load success", tmp)
+	return
+}
+
+func (s *Service) loadeventproc() {
+	for {
+		time.Sleep(s.loadEventTick)
+		s.loadeventname()
+	}
+}
+
+func (s *Service) initActivityEvents() {
+	tmp := make(map[string]struct{}, len(s.c.Property.ActivityEvents))
+	for _, name := range s.c.Property.ActivityEvents {
+		tmp[name] = struct{}{}
+	}
+	s.activityEvents = tmp
+}

app/job/main/spy/service/service_test.go

@@ -0,0 +1,132 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/cache/redis"
+
+	"github.com/robfig/cron"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	c          = context.Background()
+	s          *Service
+	testCron   = "*/5 * * * * ?"
+	testCyTime = 5000
+)
+
+func init() {
+	var (
+		err error
+	)
+	dir, _ := filepath.Abs("../cmd/spy-job-dev.toml")
+	flag.Set("conf", dir)
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+	if s == nil {
+		s = New(conf.Conf)
+	}
+	time.Sleep(time.Second)
+}
+
+func CleanCache() {
+	pool := redis.NewPool(conf.Conf.Redis.Config)
+	pool.Get(c).Do("FLUSHDB")
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		Reset(func() { CleanCache() })
+		f(s)
+	}
+}
+
+func Test_LoadSystemConfig(t *testing.T) {
+	Convey("Test_LoadSystemConfig had data", t, WithService(func(s *Service) {
+		fmt.Println(s.spyConfig)
+		So(s.spyConfig, ShouldContainKey, model.LimitBlockCount)
+		So(s.spyConfig, ShouldContainKey, model.LessBlockScore)
+		So(s.spyConfig, ShouldContainKey, model.AutoBlock)
+	}))
+}
+
+func Test_cycleblock(t *testing.T) {
+	Convey("Test_cycleblock cron", t, WithService(func(s *Service) {
+		fmt.Println("Test_cycleblock start ")
+		tx, err := s.dao.BeginTran(c)
+		So(err, ShouldBeNil)
+		ui := &model.UserInfo{Mid: testBlockMid, State: model.StateNormal}
+		err = s.dao.TxUpdateUserState(c, tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+
+		lastBlockNo := s.lastBlockNo(s.c.Property.Block.CycleTimes)
+		t := cron.New()
+		err = t.AddFunc(testCron, s.cycleblock)
+		if err != nil {
+			panic(err)
+		}
+		t.Start()
+
+		Convey("Test_cycleblock user info 1", WithService(func(s *Service) {
+			var ui *model.UserInfo
+			ui, err = s.dao.UserInfo(context.TODO(), testBlockMid)
+			So(err, ShouldBeNil)
+			So(ui.State == model.StateNormal, ShouldBeTrue)
+		}))
+
+		err = s.dao.AddBlockCache(c, testBlockMid, testLowScore, lastBlockNo)
+		So(err, ShouldBeNil)
+		mids, err := s.blockUsers(c, lastBlockNo)
+		So(err, ShouldBeNil)
+		So(mids, ShouldContain, testBlockMid)
+
+		time.Sleep(5000 * time.Millisecond)
+		time.Sleep(time.Duration(s.blockWaitTick))
+
+		Convey("Test_cycleblock user info 2 ", WithService(func(s *Service) {
+			ui, err := s.dao.UserInfo(context.TODO(), testBlockMid)
+			So(err, ShouldBeNil)
+			So(ui.State == model.StateBlock, ShouldBeTrue)
+		}))
+		fmt.Println("Test_cycleblock end ")
+	}))
+}
+
+//  go test  -test.v -test.run TestStat
+func TestStat(t *testing.T) {
+	Convey(" UpdateStatData ", t, WithService(func(s *Service) {
+		err := s.UpdateStatData(c, &model.SpyStatMessage{
+			TargetMid: 1,
+			TargetID:  1,
+			EventName: "init_user_info",
+			Type:      model.IncreaseStat,
+			Quantity:  2,
+			Time:      time.Now().Unix(),
+			UUID:      "123456789qweasdzxcccccccc",
+		})
+		So(err, ShouldBeNil)
+	}))
+	Convey(" UpdateStatData 2", t, WithService(func(s *Service) {
+		err := s.UpdateStatData(c, &model.SpyStatMessage{
+			TargetMid: 1,
+			TargetID:  1,
+			EventName: "auto_block",
+			Type:      model.ResetStat,
+			Quantity:  2,
+			Time:      time.Now().Unix(),
+			UUID:      "123456789qweasdzxcccccccc",
+		})
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/spy/service/stat.go

@@ -0,0 +1,49 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/spy/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// UpdateStatData update spy stat data.
+func (s *Service) UpdateStatData(c context.Context, m *model.SpyStatMessage) (err error) {
+	//TODO check event resaon
+	if s.allEventName[m.EventName] == 0 {
+		log.Error("event name not found %+v", err)
+		err = ecode.SpyEventNotExist
+		return
+	}
+	stat := &model.Statistics{
+		TargetMid: m.TargetMid,
+		TargetID:  m.TargetID,
+		EventID:   s.allEventName[m.EventName],
+		State:     model.WaiteCheck,
+		Quantity:  m.Quantity,
+		Ctime:     time.Now(),
+	}
+	if stat.TargetID != 0 {
+		_, ok := s.activityEvents[m.EventName]
+		if ok {
+			stat.Type = model.ActivityType
+		} else {
+			stat.Type = model.ArchiveType
+		}
+	}
+	// add stat
+	if model.ResetStat == m.Type {
+		if _, err = s.dao.AddStatistics(c, stat); err != nil {
+			log.Error("%+v", err)
+			return
+		}
+	} else {
+		if _, err = s.dao.AddIncrStatistics(c, stat); err != nil {
+			log.Error("%+v", err)
+			return
+		}
+	}
+	return
+}