Create & Init Project...

app/job/main/spy/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/spy/cmd:all-srcs",
+        "//app/job/main/spy/conf:all-srcs",
+        "//app/job/main/spy/dao:all-srcs",
+        "//app/job/main/spy/http:all-srcs",
+        "//app/job/main/spy/model:all-srcs",
+        "//app/job/main/spy/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/CHANGELOG.md

@@ -0,0 +1,59 @@
+v1.3.1
+1. use setnx for uuid
+
+v1.3.0
+1. update hbase client  
+
+v1.2.5
+1. mc ping
+
+v1.2.4
+1. bm http service
+
+v1.2.3
+1. fix WaitGroup
+
+v1.2.2
+1. remove stastd
+
+v1.2.1
+1. topic拆分
+
+v1.2.0
+1. 反作弊案件
+
+v1.1.9
+1. 去掉未使用的identify
+
+v1.1.8
+1. allow handle one month ago event
+
+v1.1.7
+1. add redis,httpclient prom
+
+v1.1.6
+1. 报表读库配置修改
+
+v1.1.5
+1. 报表及监控
+
+v1.1.4
+1. 修改event msg 参数名
+
+v1.1.3
+1. 增加databus消息日志
+
+v1.1.2
+1. 自动封禁系统中，增加过滤条件
+
+v1.1.1
+1. 修复日志
+
+v1.1.0
+1. 反作弊二期，自动封禁相关
+
+v1.0.1
+1. 迁移大仓库
+
+v1.0.0
+1. 初始化项目，更新依赖

app/job/main/spy/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+zhaogangtao
+
+# Author
+zhaogangtao
+yubaihai
+muyang
+
+# Reviewer
+zhaogangtao
+linmiao

app/job/main/spy/OWNERS

@@ -0,0 +1,17 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- muyang
+- yubaihai
+- zhaogangtao
+labels:
+- job
+- job/main/spy
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- linmiao
+- muyang
+- yubaihai
+- zhaogangtao

app/job/main/spy/README.md

@@ -0,0 +1,11 @@
+#### archive-service
+
+##### 项目简介
+> 1.提供反作弊job
+
+##### 编译环境
+> 请只用golang v1.7.x以上版本编译执行。  
+
+##### 依赖包
+> 1.公共包go-common  
+

app/job/main/spy/cmd/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["spy-job-test.toml"],
+    importpath = "go-common/app/job/main/spy/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/http:go_default_library",
+        "//app/job/main/spy/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/os/signal:go_default_library",
+        "//library/syscall:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/cmd/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/http"
+	"go-common/app/job/main/spy/service"
+	"go-common/library/log"
+	"go-common/library/os/signal"
+	"go-common/library/syscall"
+)
+
+var (
+	s *service.Service
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+
+	// service init
+	s = service.New(conf.Conf)
+	http.Init(conf.Conf, s)
+
+	log.Info("spy-job start")
+	signalHandler()
+}
+
+func signalHandler() {
+	var (
+		err error
+		ch  = make(chan os.Signal, 1)
+	)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT, syscall.SIGSTOP)
+	for {
+		si := <-ch
+		switch si {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			log.Info("get a signal %s, stop the spy-job process", si.String())
+			if err = s.Close(); err != nil {
+				log.Error("close spy-job error(%v)", err)
+			}
+			s.Wait()
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/spy/cmd/spy-job-test.toml

@@ -0,0 +1,154 @@
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/spy-job.pid"
+dir = "./"
+perf = "127.0.0.1:6865"
+checkFile = "/data/www/spy-job.html"
+family = "spy-job"
+
+
+[property]
+activityEvents = []
+blockAccountURL = "https://account.bilibili.com/api/member/blockAccount?"
+reportCron = "*/30 * * * * ?"
+taskTimer = 2
+userInfoShard = 100
+historyShard = 100
+configLoadTick = "60s"
+blockEvent = 2
+blockTick = "50ms"
+blockWaitTick = "2s"
+[property.block]
+cycleTimes = 3600
+cycleCron = "*/10 * * * * ?"
+
+[HTTPServer]
+	addr = "0.0.0.0:6866"
+	maxListen = 10
+	timeout = "1s"
+
+[spyRPC]
+    pullInterval = "10s"
+    [spyRPC.client]
+    timeout = "1s"
+    timer = 1000
+    [spyRPC.client.breaker]
+    window ="3s"
+    sleep ="100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+    [[spyRPC.backup]]
+    proto = "tcp"
+    addr = "0.0.0.0:6869"
+    timeout = "1s"
+    timer = 1000
+    [spyRPC.zookeeper]
+    root = "/microservice/spy-service/"
+    addrs = ["172.16.33.54:2181"]
+    timeout = "1s"
+
+[db]
+    addr= "172.16.33.205:3308"
+    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_spy?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "100ms"
+    execTimeout = "100ms"
+    tranTimeout = "200ms"
+    [db.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[databus]
+    [databus.eventData]
+    key = "0QEO9F8JuuIxZzNDvklH"
+    secret= "0QEO9F8JuuIxZzNDvklI"
+    group= "Spy-UGC-S"
+    topic= "Spy-T"
+    action="sub"
+    name = "spy-job/databus"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 2
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+    expire = "1h"
+    [databus.spyStatData]
+    key = "0QEO9F8JuuIxZzNDvklH"
+    secret= "0QEO9F8JuuIxZzNDvklI"
+    group= "Spy-UGC-S"
+    topic= "Spy-T"
+    action="sub"
+    name = "spy-job/databus"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 2
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+    expire = "1h"
+    [databus.secLogin]
+    key = "0QEO9F8JuuIxZzNDvklH"
+    secret= "0QEO9F8JuuIxZzNDvklI"
+    group= "SafeLogin-MainAccountLaw-S"
+    topic= "SafeLogin-T"
+    action="sub"
+    name = "spy-job/databus"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 2
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+    expire = "1h"
+
+[memcache]
+    name = "spy-job/event"
+    proto = "tcp"
+    addr = "172.18.33.60:11234"
+    idle = 5
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[redis]
+name = "spy-service"
+proto = "tcp"
+addr = "172.18.33.61:6879"
+idle = 100
+active = 100
+dialTimeout = "1s"
+readTimeout = "1s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+expire = "24h"
+MsgUUIDExpire = "300s"
+
+[httpClient]
+    dial = "1s"
+    timeout = "3s"
+    keepAlive = "60s"
+    timer = 1000
+    key = "zxnh4k92dwe61t27"
+    secret = "dnu3bwpxyswqwf1ixpsczthury1nqiew"
+    [httpClient.breaker]
+        window  = "3s"
+        sleep   = "100ms"
+        bucket  = 10
+        ratio   = 0.5
+        request = 100
+        switchoff = false

app/job/main/spy/conf/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/spy/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/conf/conf.go

@@ -0,0 +1,140 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+	"time"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"go-common/library/database/hbase.v2"
+
+	"github.com/BurntSushi/toml"
+)
+
+// Conf global config.
+var (
+	confPath string
+	client   *conf.Client
+	// config
+	Conf = &Config{}
+)
+
+// Config info.
+type Config struct {
+	// base
+	// customized property
+	Property *Property
+	// log
+	Xlog *log.Config
+	// db
+	DB *sql.Config
+	// databus
+	Databus *DataSource
+	// rpc to spy-service
+	SpyRPC   *rpc.ClientConfig
+	HBase    *HBaseConfig
+	Memcache *memcache.Config
+	// redis
+	Redis *Redis
+	// http client
+	HTTPClient *bm.ClientConfig
+	// HTTPServer
+	HTTPServer *bm.ServerConfig
+}
+
+// HBaseConfig extra hbase config
+type HBaseConfig struct {
+	*hbase.Config
+	ReadTimeout  time.Duration
+	WriteTimeout time.Duration
+}
+
+// DataSource config all spy job dataSource
+type DataSource struct {
+	EventData   *databus.Config
+	SpyStatData *databus.Config
+	SecLogin    *databus.Config
+}
+
+// Property config for biz logic.
+type Property struct {
+	TaskTimer       time.Duration
+	UserInfoShard   int64
+	Debug           bool
+	ConfigLoadTick  xtime.Duration
+	BlockTick       xtime.Duration
+	BlockWaitTick   xtime.Duration
+	LoadEventTick   xtime.Duration
+	BlockAccountURL string
+	HistoryShard    int64
+	BlockEvent      int64
+	Block           *struct {
+		CycleTimes int64 // unit per seconds
+		CycleCron  string
+	}
+	ReportCron     string
+	ActivityEvents []string
+}
+
+// Redis conf.
+type Redis struct {
+	*redis.Config
+	Expire        xtime.Duration
+	MsgUUIDExpire xtime.Duration
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+}
+
+// Init init conf.
+func Init() (err error) {
+	if confPath == "" {
+		return configCenter()
+	}
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func configCenter() (err error) {
+	if client, err = conf.New(); err != nil {
+		panic(err)
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/spy/dao/BUILD

@@ -0,0 +1,69 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "account_test.go",
+        "dao_test.go",
+        "memcache_test.go",
+        "mysql_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "account.go",
+        "dao.go",
+        "hbase.go",
+        "memcache.go",
+        "mysql.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/spy/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/dao/account.go

@@ -0,0 +1,33 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"go-common/library/log"
+)
+
+// BlockAccount block account
+func (d *Dao) BlockAccount(c context.Context, mid int64, reason string) (err error) {
+	params := url.Values{}
+	params.Set("mid", fmt.Sprintf("%d", mid))
+	params.Set("admin_reason", reason)
+	params.Set("blockType", "6")
+	params.Set("operator", "anticheat")
+	params.Set("type", "json")
+	var resp struct {
+		Code int64 `json:"code"`
+	}
+	// get
+	if err = d.httpClient.Get(c, d.c.Property.BlockAccountURL, "", params, &resp); err != nil {
+		log.Error("httpClient.Do() error(%v)", err)
+		return
+	}
+	if resp.Code != 0 {
+		err = fmt.Errorf("GET block account url resp(%v)", resp)
+		return
+	}
+	log.Info("account user(%d) block suc(%v)", mid, resp)
+	return
+}

app/job/main/spy/dao/account_test.go

@@ -0,0 +1,18 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+	gock "gopkg.in/h2non/gock.v1"
+)
+
+func TestDaoBlockAccount(t *testing.T) {
+	convey.Convey("BlockAccount", t, func() {
+		defer gock.OffAll()
+		httpMock("GET", d.c.Property.BlockAccountURL).Reply(200).JSON(`{"code":0}`)
+		err := d.BlockAccount(context.TODO(), 7593623, "1")
+		convey.So(err, convey.ShouldBeNil)
+	})
+}

app/job/main/spy/dao/dao.go

@@ -0,0 +1,71 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/hbase.v2"
+	"go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao event dao def.
+type Dao struct {
+	c             *conf.Config
+	db            *sql.DB
+	mc            *memcache.Pool
+	hbase         *hbase.Client
+	redis         *redis.Pool
+	httpClient    *bm.Client
+	expire        int
+	msgUUIDExpire int
+}
+
+// New create instance of dao and return.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:             c,
+		db:            sql.NewMySQL(c.DB),
+		mc:            memcache.NewPool(c.Memcache),
+		redis:         redis.NewPool(c.Redis.Config),
+		httpClient:    bm.NewClient(c.HTTPClient),
+		expire:        int(time.Duration(c.Redis.Expire) / time.Second),
+		msgUUIDExpire: int(time.Duration(c.Redis.MsgUUIDExpire) / time.Second),
+	}
+	if c.HBase != nil {
+		d.hbase = hbase.NewClient(c.HBase.Config)
+	}
+	return
+}
+
+// Ping check db health.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.pingMC(c); err != nil {
+		return
+	}
+	if err = d.PingRedis(c); err != nil {
+		return
+	}
+	if d.hbase != nil {
+		if err = d.hbase.Ping(c); err != nil {
+			return
+		}
+	}
+	return d.db.Ping(c)
+}
+
+// Close close all db connections.
+func (d *Dao) Close() {
+	if d.db != nil {
+		d.db.Close()
+	}
+	if d.redis != nil {
+		d.redis.Close()
+	}
+	if d.mc != nil {
+		d.mc.Close()
+	}
+}

app/job/main/spy/dao/dao_test.go

@@ -0,0 +1,58 @@
+package dao
+
+import (
+	"context"
+	"flag"
+	"os"
+	"strings"
+	"testing"
+
+	"go-common/app/job/main/spy/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+	gock "gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d *Dao
+	c = context.Background()
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.account-law.spy-job")
+		flag.Set("conf_appid", "main.account-law.spy-job")
+		flag.Set("conf_token", "1404609bda3db4cd20ad9823bfd20a83")
+		flag.Set("tree_id", "2856")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_env", "10")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/spy-job-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.httpClient.SetTransport(gock.DefaultTransport)
+	os.Exit(m.Run())
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}
+
+func TestPing(t *testing.T) {
+	Convey("TestPing", t, func() {
+		err := d.Ping(c)
+		So(err, ShouldBeNil)
+	})
+
+}

app/job/main/spy/dao/hbase.go

@@ -0,0 +1,55 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/spy/model"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+var (
+	// table
+	_hbaseTableActive = "active_data"
+	// family
+	_familyDays  = "activeDays"
+	_familyDaysB = []byte(_familyDays)
+)
+
+func strRowKey(mid int64) string {
+	return fmt.Sprintf("%d", mid)
+}
+
+// GetActiveData get user active days and watched bangumi video time
+func (dao *Dao) GetActiveData(c context.Context, mid int64) (active *model.Active, err error) {
+	var (
+		result      *hrpc.Result
+		key         = strRowKey(mid)
+		ctx, cancel = context.WithTimeout(c, time.Duration(dao.c.HBase.ReadTimeout))
+	)
+	defer cancel()
+	if result, err = dao.hbase.GetStr(ctx, _hbaseTableActive, key); err != nil {
+		err = errors.Wrapf(err, "hbase.GetStr(%s,%s)", _hbaseTableActive, key)
+		return
+	}
+	active = &model.Active{}
+	for _, c := range result.Cells {
+		h := &model.Active{}
+		if c != nil && bytes.Equal(c.Family, _familyDaysB) {
+			days, err := strconv.ParseInt(string(c.Qualifier), 10, 64)
+			if err != nil {
+				log.Error("strconv.ParseInt err(%v)", err)
+				continue
+			}
+			h.Active = days
+			active = h
+		}
+	}
+	return
+}

app/job/main/spy/dao/memcache.go

@@ -0,0 +1,63 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_partitionKey = "p_%s_%d"
+)
+
+// pingMC ping memcache.
+func (d *Dao) pingMC(ctx context.Context) (err error) {
+	conn := d.mc.Get(ctx)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: "ping", Value: []byte{1}, Expiration: 0}); err != nil {
+		log.Error("conn.Store(set,ping,1) error(%v)", err)
+	}
+	return
+}
+
+func partitionCacheKey(e string, p int32) string {
+	return fmt.Sprintf(_partitionKey, e, p)
+}
+
+// OffsetCache .
+func (d *Dao) OffsetCache(ctx context.Context, event string, p int32) (offset int64, err error) {
+	var (
+		key  = partitionCacheKey(event, p)
+		conn = d.mc.Get(ctx)
+	)
+	defer conn.Close()
+	reply, err := conn.Get(key)
+	if err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+			return
+		}
+		log.Error("conn.Get(get,%d) error(%v)", p, err)
+		return
+	}
+	if err = conn.Scan(reply, &offset); err != nil {
+		log.Error("reply.Scan(%s) error(%v)", string(reply.Value), err)
+	}
+	return
+}
+
+// SetOffsetCache .
+func (d *Dao) SetOffsetCache(ctx context.Context, event string, p int32, offset int64) (err error) {
+	var (
+		key  = partitionCacheKey(event, p)
+		conn = d.mc.Get(ctx)
+	)
+	defer conn.Close()
+
+	if err = conn.Set(&memcache.Item{Key: key, Object: offset, Expiration: 0, Flags: memcache.FlagJSON}); err != nil {
+		log.Error("conn.Set(%s,%d) error(%v)", key, offset, err)
+	}
+	return
+}

app/job/main/spy/dao/memcache_test.go

@@ -0,0 +1,22 @@
+package dao
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestMCInfo(t *testing.T) {
+	Convey("TestMCInfo", t, func() {
+		var (
+			partition int32 = -2233
+			offset1   int64 = 778899
+			offset2   int64
+			err       error
+		)
+		err = d.SetOffsetCache(c, "test", partition, offset1)
+		So(err, ShouldBeNil)
+		offset2, err = d.OffsetCache(c, "test", partition)
+		So(offset1, ShouldEqual, offset2)
+	})
+}

app/job/main/spy/dao/mysql.go

@@ -0,0 +1,295 @@
+package dao
+
+import (
+	"context"
+	xsql "database/sql"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_reBuildMidCountSQL    = "SELECT count(mid) FROM spy_user_info_%02d WHERE state=? AND score>=30 AND mtime BETWEEN ? AND ?;"
+	_reBuildMidSQL         = "SELECT mid FROM spy_user_info_%02d WHERE state=? AND score>=30 AND mtime BETWEEN ? AND ? LIMIT ?;"
+	_getAllConfigSQL       = "SELECT id, property,name,val,ctime FROM spy_system_config;"
+	_addEventHistorySQL    = "INSERT INTO spy_user_event_history_%02d (mid,event_id,score,base_score,event_score,remark,reason,factor_val,ctime) VALUES (?,?,?,?,?,?,?,?,?);"
+	_addPunishmentSQL      = "INSERT INTO spy_punishment (mid,type,reason,batch_no,ctime) VALUES (?,?,?,?,?);"
+	_updateUserStateSQL    = "UPDATE spy_user_info_%02d SET state=? WHERE mid=?"
+	_getLastHistorySQL     = "SELECT id,mid,event_id,score,base_score,event_score,remark,reason,factor_val,ctime FROM spy_user_event_history_%02d WHERE mid=? ORDER BY id DESC LIMIT 1;"
+	_getHistoryListSQL     = "SELECT remark,reason FROM spy_user_event_history_%02d WHERE mid= ? ORDER BY id DESC LIMIT ?;"
+	_updateEventScoreSQL   = "UPDATE spy_user_info_%02d SET event_score=?, score=? WHERE mid=?;"
+	_userInfoSQL           = "SELECT id,mid,score,base_score,event_score,state,ctime,mtime FROM spy_user_info_%02d WHERE mid=? LIMIT 1;"
+	_punishmentCountSQL    = "SELECT COUNT(1) FROM spy_punishment where mtime > ? and mtime < ?;"
+	_securityLoginCountSQL = "SELECT COUNT(1) FROM spy_user_event_history_%02d where reason = ? and mtime > ? and mtime < ?;"
+	_insertReportSQL       = "INSERT INTO `spy_report`(`name`,`date_version`,`val`,`ctime`)VALUES(?,?,?,?);"
+	_insertIncrStatSQL     = "INSERT INTO `spy_statistics`(`target_mid`,`target_id`,`event_id`,`state`,`type`,`quantity`,`ctime`)VALUES(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE quantity=quantity+?;"
+	_insertStatSQL         = "INSERT INTO `spy_statistics`(`target_mid`,`target_id`,`event_id`,`state`,`type`,`quantity`,`ctime`)VALUES(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE quantity=?;"
+	_allEventSQL           = "SELECT id,name,nick_name,service_id,status,ctime,mtime FROM spy_event WHERE status<>0"
+)
+
+func hitHistory(id int64) int64 {
+	return id % conf.Conf.Property.HistoryShard
+}
+
+func hitInfo(id int64) int64 {
+	return id % conf.Conf.Property.UserInfoShard
+}
+
+// BeginTran begin transaction.
+func (d *Dao) BeginTran(c context.Context) (*sql.Tx, error) {
+	return d.db.Begin(c)
+}
+
+// ReBuildMidCount count for need reBuild user.
+func (d *Dao) ReBuildMidCount(c context.Context, i int, state int8, start, end time.Time) (res int64, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_reBuildMidCountSQL, i), state, start, end)
+	if err = row.Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// ReBuildMidList query reBuild user mid list by page.
+func (d *Dao) ReBuildMidList(c context.Context, i int, t int8, start, end time.Time, ps int64) (res []int64, err error) {
+	var rows *sql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_reBuildMidSQL, i), t, start, end, ps); err != nil {
+		log.Error("d.reBuildMidSQL.Query(%d, %s, %s, %d) error(%v)", t, start, end, ps, err)
+		return
+	}
+	defer rows.Close()
+	res = []int64{}
+	for rows.Next() {
+		var r int64
+		if err = rows.Scan(&r); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			res = nil
+			return
+		}
+		res = append(res, r)
+	}
+	err = rows.Err()
+	return
+}
+
+// Configs spy system configs.
+func (d *Dao) Configs(c context.Context) (res map[string]string, err error) {
+	var rows *sql.Rows
+	if rows, err = d.db.Query(c, _getAllConfigSQL); err != nil {
+		log.Error("d.getAllConfigSQL.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	res = map[string]string{}
+	for rows.Next() {
+		var r model.Config
+		if err = rows.Scan(&r.ID, &r.Property, &r.Name, &r.Val, &r.Ctime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			res = nil
+			return
+		}
+		res[r.Property] = r.Val
+	}
+	err = rows.Err()
+	return
+}
+
+// TxAddEventHistory insert user_event_history.
+func (d *Dao) TxAddEventHistory(c context.Context, tx *sql.Tx, ueh *model.UserEventHistory) (err error) {
+	var (
+		now = time.Now()
+	)
+	if _, err = tx.Exec(fmt.Sprintf(_addEventHistorySQL, hitHistory(ueh.Mid)), ueh.Mid, ueh.EventID, ueh.Score, ueh.BaseScore, ueh.EventScore, ueh.Remark, ueh.Reason, ueh.FactorVal, now); err != nil {
+		log.Error("db.Exec(%v) error(%v)", ueh, err)
+		return
+	}
+	return
+}
+
+// TxAddPunishment insert punishment.
+func (d *Dao) TxAddPunishment(c context.Context, tx *sql.Tx, mid int64, t int8, reason string, blockNo int64) (err error) {
+	var (
+		now = time.Now()
+	)
+	if _, err = tx.Exec(_addPunishmentSQL, mid, t, reason, blockNo, now); err != nil {
+		log.Error("db.Exec(%d, %d, %s) error(%v)", mid, t, reason, err)
+		return
+	}
+	return
+}
+
+// History get last one user history.
+func (d *Dao) History(c context.Context, mid int64) (h *model.UserEventHistory, err error) {
+	var (
+		row *sql.Row
+	)
+	h = &model.UserEventHistory{}
+	row = d.db.QueryRow(c, fmt.Sprintf(_getLastHistorySQL, hitHistory(mid)), mid)
+	if err = row.Scan(&h.ID, &h.Mid, &h.EventID, &h.Score, &h.BaseScore, &h.EventScore, &h.Remark, &h.Reason, &h.FactorVal, &h.CTime); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			h = nil
+			return
+		}
+		log.Error("History row.Scan(%d) error(%v)", mid, err)
+	}
+	return
+}
+
+// TxUpdateUserState insert or update  user state by mid.
+func (d *Dao) TxUpdateUserState(c context.Context, tx *sql.Tx, info *model.UserInfo) (err error) {
+	if _, err = d.db.Exec(c, fmt.Sprintf(_updateUserStateSQL, hitInfo(info.Mid)), info.State, info.Mid); err != nil {
+		log.Error("TxUpdateUserState db.Exec(%d, %v) error(%v)", info.Mid, info, err)
+		return
+	}
+	return
+}
+
+// HistoryList query .
+func (d *Dao) HistoryList(c context.Context, mid int64, size int) (res []*model.UserEventHistory, err error) {
+	var rows *sql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_getHistoryListSQL, hitHistory(mid)), mid, size); err != nil {
+		log.Error("d.HistoryList.Query(%d, %d) error(%v)", mid, size, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		r := &model.UserEventHistory{}
+		if err = rows.Scan(&r.Remark, &r.Reason); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			res = nil
+			return
+		}
+		res = append(res, r)
+	}
+	err = rows.Err()
+	return
+}
+
+// UserInfo get info by mid.
+func (d *Dao) UserInfo(c context.Context, mid int64) (res *model.UserInfo, err error) {
+	var (
+		row *sql.Row
+	)
+	res = &model.UserInfo{}
+	row = d.db.QueryRow(c, fmt.Sprintf(_userInfoSQL, hitInfo(mid)), mid)
+	if err = row.Scan(&res.ID, &res.Mid, &res.Score, &res.BaseScore, &res.EventScore, &res.State, &res.CTime, &res.MTime); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			res = nil
+			return
+		}
+		log.Error("row.Scan() error(%v)", err)
+	}
+	return
+}
+
+// TxUpdateEventScore update event score.
+func (d *Dao) TxUpdateEventScore(c context.Context, tx *sql.Tx, mid int64, escore, score int8) (err error) {
+	if _, err = tx.Exec(fmt.Sprintf(_updateEventScoreSQL, hitInfo(mid)), escore, score, mid); err != nil {
+		log.Error("db.TxUpdateEventScore(%s, %d, %d, %d) error(%v)", _updateEventScoreSQL, escore, score, mid, err)
+		return
+	}
+	return
+}
+
+// AddReport add report info.
+func (d *Dao) AddReport(c context.Context, r *model.Report) (affected int64, err error) {
+	var res xsql.Result
+	if res, err = d.db.Exec(c, _insertReportSQL, r.Name, r.DateVersion, r.Val, r.Ctime); err != nil {
+		log.Error("AddReport: db.Exec(%v) error(%v)", r, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// PunishmentCount punishment count.
+func (d *Dao) PunishmentCount(c context.Context, start, end time.Time) (res int64, err error) {
+	row := d.db.QueryRow(c, _punishmentCountSQL, start, end)
+	if err = row.Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// SecurityLoginCount security login count.
+func (d *Dao) SecurityLoginCount(c context.Context, index int64, reason string, stime, etime time.Time) (res int64, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_securityLoginCountSQL, hitHistory(index)), reason, stime, etime)
+	if err = row.Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// AddStatistics add statistics info.
+func (d *Dao) AddStatistics(c context.Context, s *model.Statistics) (id int64, err error) {
+	var res xsql.Result
+	if res, err = d.db.Exec(c, _insertStatSQL, s.TargetMid, s.TargetID, s.EventID, s.State, s.Type, s.Quantity, s.Ctime, s.Quantity); err != nil {
+		return
+	}
+	if id, err = res.LastInsertId(); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}
+
+// AddIncrStatistics add increase statistics info.
+func (d *Dao) AddIncrStatistics(c context.Context, s *model.Statistics) (id int64, err error) {
+	var res xsql.Result
+	if res, err = d.db.Exec(c, _insertIncrStatSQL, s.TargetMid, s.TargetID, s.EventID, s.State, s.Type, s.Quantity, s.Ctime, s.Quantity); err != nil {
+		return
+	}
+	if id, err = res.LastInsertId(); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}
+
+//AllEvent all event.
+func (d *Dao) AllEvent(c context.Context) (list []*model.Event, err error) {
+	var (
+		rows *sql.Rows
+	)
+	list = make([]*model.Event, 0)
+	if rows, err = d.db.Query(c, _allEventSQL); err != nil {
+		log.Error("d.db.Query(%s) error(%v)", _allEventSQL, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var event = &model.Event{}
+		if err = rows.Scan(&event.ID, &event.Name, &event.NickName, &event.ServiceID, &event.Status, &event.Ctime, &event.Mtime); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		list = append(list, &model.Event{
+			ID:        event.ID,
+			Name:      event.Name,
+			NickName:  event.NickName,
+			ServiceID: event.ServiceID,
+			Status:    event.Status,
+			Ctime:     event.Ctime,
+			Mtime:     event.Mtime,
+		})
+	}
+	return
+}

app/job/main/spy/dao/mysql_test.go

@@ -0,0 +1,175 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/spy/model"
+	"go-common/library/database/sql"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testMysqlMid int64 = 15555180
+	size               = 3
+)
+
+func Test_Configs(t *testing.T) {
+	Convey("Test_Configs info data", t, func() {
+		res, err := d.Configs(context.TODO())
+		fmt.Println(res)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+		So(res, ShouldContainKey, model.LimitBlockCount)
+		So(res, ShouldContainKey, model.LessBlockScore)
+		So(res, ShouldContainKey, model.AutoBlock)
+	})
+}
+
+func Test_History(t *testing.T) {
+	Convey("Test_History get history data", t, func() {
+		res, err := d.History(context.TODO(), testMysqlMid)
+		fmt.Println(res)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	})
+}
+
+func Test_HistoryList(t *testing.T) {
+	Convey("Test_HistoryList get history list data", t, func() {
+		res, err := d.HistoryList(context.TODO(), testMysqlMid, size)
+		fmt.Println(res)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	})
+}
+
+func Test_TxUpdateUserState(t *testing.T) {
+	Convey("Test_TxUpdateUserState no err", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+			tx  *sql.Tx
+			ui  = &model.UserInfo{Mid: 15555180}
+		)
+		tx, err = d.db.Begin(c)
+		So(err, ShouldBeNil)
+		ui.State = model.StateNormal
+		err = d.TxUpdateUserState(context.TODO(), tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxAddPunishment(t *testing.T) {
+	Convey("Test_TxAddPunishment add data", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+			tx  *sql.Tx
+		)
+		tx, err = d.db.Begin(c)
+		So(err, ShouldBeNil)
+		err = d.TxAddPunishment(context.TODO(), tx, testMysqlMid, 0, "test封禁", 100000000)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+//  go test  -test.v -test.run Test_Stat
+func Test_Stat(t *testing.T) {
+	Convey("AddIncrStatistics", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+		)
+		stat := &model.Statistics{
+			TargetMid: 1,
+			TargetID:  1,
+			EventID:   1,
+			State:     0,
+			Type:      1,
+			Quantity:  1,
+			Ctime:     time.Now(),
+		}
+		_, err = d.AddIncrStatistics(c, stat)
+		So(err, ShouldBeNil)
+	})
+	Convey("AddStatistics", t, func() {
+		var (
+			c   = context.TODO()
+			err error
+		)
+		stat := &model.Statistics{
+			TargetMid: 1,
+			TargetID:  1,
+			EventID:   2,
+			State:     0,
+			Type:      1,
+			Quantity:  1,
+			Ctime:     time.Now(),
+		}
+		stat.Quantity = 3
+		_, err = d.AddStatistics(c, stat)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxAddEventHistory(t *testing.T) {
+	Convey("test userinfo", t, func() {
+		res, err := d.UserInfo(c, 7593623)
+		So(err, ShouldBeNil)
+		if res == nil {
+			_, err = d.db.Exec(c, "insert into spy_user_info_23 (mid) values (7593623)")
+			So(err, ShouldBeNil)
+		}
+	})
+	var tx *sql.Tx
+	Convey("Test_TxAddEventHistory start", t, func() {
+		var err error
+		tx, err = d.BeginTran(c)
+		So(err, ShouldBeNil)
+	})
+	ueh := &model.UserEventHistory{Mid: 7593634, BaseScore: 100}
+	Convey("Test_TxAddEventHistory", t, func() {
+
+		err := d.TxAddEventHistory(c, tx, ueh)
+		So(err, ShouldBeNil)
+	})
+	Convey("TxUpdateEventScore", t, func() {
+		err := d.TxUpdateEventScore(c, tx, 7593623, 100, 100)
+		So(err, ShouldBeNil)
+	})
+	Convey("Test_TxAddEventHistory commit", t, func() {
+		err := tx.Commit()
+		So(err, ShouldBeNil)
+	})
+	Convey("AllEvent", t, func() {
+		res, err := d.AllEvent(c)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+	})
+	Convey("clean data", t, func() {
+		d.db.Exec(c, "delete from spy_user_event_history_23 where mid = 7593623")
+
+	})
+}
+
+func Test_SecurityLoginCount(t *testing.T) {
+	Convey("Test_SecurityLoginCount", t, func() {
+		_, err := d.SecurityLoginCount(c, 1, "test", time.Now(), time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_ReBuildMidList(t *testing.T) {
+	Convey("Test_ReBuildMidList", t, func() {
+		_, err := d.ReBuildMidList(c, 1, 1, time.Now(), time.Now(), 1)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/spy/dao/redis.go

@@ -0,0 +1,227 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_keyWaitBlock   = "wb_" // b_batch_no wait block
+	_keyBlock       = "bl_" // b_batch_no  block
+	_preLock        = "lk_"
+	_keyUniqueCheck = "uc:"
+	times           = 3
+)
+
+// keyWaitBlock return block cache key.
+func keyWaitBlock(batchNo int64) string {
+	return _keyWaitBlock + fmt.Sprintf("%d", batchNo)
+}
+
+// keyBlock return block cache key.
+func keyBlock() string {
+	return _keyBlock
+}
+
+func lockKey(key string) string {
+	return _preLock + key
+}
+
+func uniqueCheckKey(uuid string) string {
+	return _keyUniqueCheck + uuid
+}
+
+// BlockMidCache get wait block mids.
+func (d *Dao) BlockMidCache(c context.Context, batchNo int64, num int64) (res []int64, err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = keyWaitBlock(batchNo)
+	)
+	defer conn.Close()
+	if res, err = redis.Int64s(conn.Do("ZREVRANGEBYSCORE", key, "+inf", "-inf", "LIMIT", 0, num)); err != nil {
+		log.Error("redis(ZREVRANGEBYSCORE %s,%d) error(%v)", key, num, err)
+		return
+	}
+	return
+}
+
+// DelBlockCache delete the wait block redis.
+func (d *Dao) DelBlockCache(c context.Context, batchNo int64, mid int64) (err error) {
+	var (
+		key  = keyWaitBlock(batchNo)
+		args = []interface{}{key, mid}
+	)
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if err = conn.Send("ZREM", args...); err != nil {
+		log.Error("conn.Send(ZREM %s,%v) error(%v)", key, mid, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	if _, err = conn.Receive(); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+		return
+	}
+	return
+}
+
+//SetNXLockCache redis lock.
+func (d *Dao) SetNXLockCache(c context.Context, k string, times int64) (res bool, err error) {
+	var (
+		key  = lockKey(k)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if res, err = redis.Bool(conn.Do("SETNX", key, "1")); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("conn.Do(SETNX(%d)) error(%v)", key, err)
+			return
+		}
+	}
+	if res {
+		if _, err = redis.Bool(conn.Do("EXPIRE", key, times)); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", key, times, err)
+			return
+		}
+	}
+	return
+}
+
+//DelLockCache del lock cache.
+func (d *Dao) DelLockCache(c context.Context, k string) (err error) {
+	var (
+		key  = lockKey(k)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("DEL", key); err != nil {
+		log.Error("conn.Do(del,%v) err(%v)", key, err)
+	}
+	return
+}
+
+//AddBlockCache add block cache.
+func (d *Dao) AddBlockCache(c context.Context, mid int64, score int8, blockNo int64) (err error) {
+	var (
+		key = keyWaitBlock(blockNo)
+	)
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if err = conn.Send("ZADD", key, score, mid); err != nil {
+		log.Error("conn.Send(ZADD %s,%d,%d) error(%v)", key, score, mid, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.expire); err != nil {
+		log.Error("conn.Send(EXPIRE) error(%v)", err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// SetBlockCache block.
+func (d *Dao) SetBlockCache(c context.Context, mids []int64) (err error) {
+	var (
+		key  = keyBlock()
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, mid := range mids {
+		if err = conn.Send("SADD", key, mid); err != nil {
+			log.Error("SADD conn.Send error(%v)", err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.expire); err != nil {
+		log.Error("EXPIRE conn.Send error(%v)", err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < len(mids); i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("SetBlockCache Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+//SPOPBlockCache pop mid.
+func (d *Dao) SPOPBlockCache(c context.Context) (mid int64, err error) {
+	var (
+		key  = keyBlock()
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if mid, err = redis.Int64(conn.Do("SPOP", key)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("SPOP conn.Do(%s,%v) err(%v)", key, err)
+		}
+	}
+	return
+}
+
+// PingRedis check redis connection
+func (d *Dao) PingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	_, err = conn.Do("SET", "PING", "PONG")
+	conn.Close()
+	return
+}
+
+// PfaddCache SetNX.
+func (d *Dao) PfaddCache(c context.Context, uuid string) (ok bool, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	key := uniqueCheckKey(uuid)
+	if err = conn.Send("SETNX", key, 1); err != nil {
+		log.Error("SETNX conn.Send error(%v)", err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.msgUUIDExpire); err != nil {
+		log.Error("conn.Send(EXPIRE) error(%v)", err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("DelLock conn.Flush() error(%v)", err)
+		return
+	}
+	if ok, err = redis.Bool(conn.Receive()); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+		return
+	}
+	if _, err = conn.Receive(); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+	}
+	return
+}
+
+// TTL get redis cache ttl.
+func (d *Dao) TTL(c context.Context, key string) (ttl int64, err error) {
+	conn := d.redis.Get(c)
+	ttl, err = redis.Int64(conn.Do("TTL", key))
+	conn.Close()
+	return
+}

app/job/main/spy/dao/redis_test.go

@@ -0,0 +1,116 @@
+package dao
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"strings"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testAddMidRedis int64 = 15555180
+	testLowScore    int8  = 7
+	testBlockNo     int64 = 100000
+	testKey               = "test_lock"
+	testMidList           = []int64{15555180}
+)
+
+func Test_PingRedis(t *testing.T) {
+	Convey("ping redis", t, func() {
+		So(d.PingRedis(context.TODO()), ShouldBeNil)
+	})
+}
+
+func Test_AddBlockCache(t *testing.T) {
+	Convey("AddBlockCache redis ", t, func() {
+		c := context.TODO()
+		ret, err := d.BlockMidCache(c, testBlockNo, 10)
+		So(ret, ShouldBeEmpty)
+		So(err, ShouldBeNil)
+
+		err = d.AddBlockCache(c, testAddMidRedis, testLowScore, testBlockNo)
+		So(err, ShouldBeNil)
+
+		ret, err = d.BlockMidCache(c, testBlockNo, 10)
+		So(ret, ShouldContain, testAddMidRedis)
+		So(err, ShouldBeNil)
+
+		err = d.DelBlockCache(c, testBlockNo, testAddMidRedis)
+		So(err, ShouldBeNil)
+
+		ret, err = d.BlockMidCache(c, testBlockNo, 10)
+		So(ret, ShouldNotContain, testAddMidRedis)
+		So(err, ShouldBeNil)
+
+	})
+}
+
+func Test_SetNXLockCache(t *testing.T) {
+	Convey("SetNXLockCache", t, func() {
+		ret, err := d.SetNXLockCache(c, testKey, 2)
+		So(err, ShouldBeNil)
+		So(ret, ShouldBeTrue)
+
+		err = d.DelLockCache(c, testKey)
+		So(err, ShouldBeNil)
+
+		ret, err = d.SetNXLockCache(c, testKey, 2)
+		So(err, ShouldBeNil)
+		So(ret, ShouldBeTrue)
+
+		ret, err = d.SetNXLockCache(c, testKey, 2)
+		So(err, ShouldBeNil)
+		So(ret, ShouldBeFalse)
+	})
+
+}
+
+func Test_SetBlockCache(t *testing.T) {
+	Convey("ping SetBlockCache", t, func() {
+		err := d.SetBlockCache(context.TODO(), testMidList)
+		So(err, ShouldBeNil)
+		Convey("ping SetBlockCache", func() {
+			mid, err := d.SPOPBlockCache(context.TODO())
+			So(err, ShouldBeNil)
+			So(testMidList, ShouldContain, mid)
+			Convey("ping SetBlockCache 2", func() {
+				mid, err := d.SPOPBlockCache(context.TODO())
+				So(err, ShouldBeNil)
+				So(mid == 0, ShouldBeTrue)
+			})
+		})
+	})
+}
+
+// go test  -test.v -test.run TestPfaddCache
+func TestPfaddCache(t *testing.T) {
+	Convey("PfaddCache", t, func() {
+		idx := strings.Replace(randHex(), "-", "", -1)
+		ok, err := d.PfaddCache(context.TODO(), idx)
+		So(err, ShouldBeNil)
+		So(ok, ShouldBeTrue)
+		ok, err = d.PfaddCache(context.TODO(), idx)
+		So(err, ShouldBeNil)
+		So(ok, ShouldBeFalse)
+	})
+}
+
+// go test  -test.v -test.run TestBatchPfaddCache
+func TestBatchPfaddCache(t *testing.T) {
+	Convey("PfaddCache", t, func() {
+		for i := 0; i < 10; i++ {
+			idx := strings.Replace(randHex(), "-", "", -1)
+			_, err := d.PfaddCache(context.TODO(), idx)
+			So(err, ShouldBeNil)
+		}
+	})
+}
+
+func randHex() string {
+	bs := make([]byte, 16)
+	rand.Read(bs)
+	return hex.EncodeToString(bs)
+}

app/job/main/spy/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/spy/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/http/http.go

@@ -0,0 +1,41 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	svr *service.Service
+)
+
+// Init .
+func Init(c *conf.Config, s *service.Service) {
+	svr = s
+	// init inner router
+	engine := bm.DefaultServer(c.HTTPServer)
+	innerRouter(engine)
+	// init local server
+	if err := engine.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	// health check
+	e.Ping(ping)
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	if err := svr.Ping(c); err != nil {
+		log.Error("spy-job service ping error")
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}

app/job/main/spy/model/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "active.go",
+        "config.go",
+        "event.go",
+        "eventmsg.go",
+        "report.go",
+        "stat.go",
+        "user_event_history.go",
+        "user_info.go",
+    ],
+    importpath = "go-common/app/job/main/spy/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/model/active.go

@@ -0,0 +1,7 @@
+package model
+
+type Active struct {
+	Mid     int64
+	Active  int64
+	Watched int64
+}

app/job/main/spy/model/config.go

@@ -0,0 +1,33 @@
+package model
+
+import "time"
+
+// config properties.
+const (
+	LimitBlockCount = "limitBlockCount"
+	LessBlockScore  = "lessBlockScore"
+	AutoBlock       = "autoBlock"
+	AutoBlockOpen   = 1
+)
+
+const (
+	//PunishmentTypeBlock 封禁
+	PunishmentTypeBlock = 1
+	//BlockReasonSize block reason size
+	BlockReasonSize = 3
+	//BlockLockKey cycle block
+	BlockLockKey = "cycleblock"
+	//ReportJobKey report job
+	ReportJobKey = "reportjob"
+	//DefLockTime def.
+	DefLockTime int64 = 60
+)
+
+// Config def.
+type Config struct {
+	ID       int64
+	Property string
+	Name     string
+	Val      string
+	Ctime    time.Time
+}

app/job/main/spy/model/event.go

@@ -0,0 +1,14 @@
+package model
+
+import "go-common/library/time"
+
+// Event def.
+type Event struct {
+	ID        int64
+	Name      string // 事件标示
+	NickName  string // 事件可读昵称
+	ServiceID int64
+	Status    int8 // 0:删除 1:未删除
+	Ctime     time.Time
+	Mtime     time.Time
+}

app/job/main/spy/model/eventmsg.go

@@ -0,0 +1,27 @@
+package model
+
+// EventMessage event from bigdata.
+type EventMessage struct {
+	Time      string      `json:"time"`
+	IP        string      `json:"ip"`
+	Service   string      `json:"service"`
+	Event     string      `json:"event"`
+	ActiveMid int64       `json:"active_mid"`
+	TargetMid int64       `json:"target_mid"`
+	TargetID  int64       `json:"target_id"`
+	Args      interface{} `json:"args"`
+	Result    string      `json:"result"`
+	Effect    string      `json:"effect"`
+	RiskLevel int8        `json:"risk_level"`
+}
+
+// SpyStatMessage sin stat message.
+type SpyStatMessage struct {
+	TargetMid int64  `json:"target_mid"`
+	TargetID  int64  `json:"target_id"`
+	Type      int8   `json:"type"`
+	Quantity  int64  `json:"quantity"`
+	Time      int64  `json:"time"`
+	EventName string `json:"event_name"`
+	UUID      string `json:"uuid"`
+}

app/job/main/spy/model/report.go

@@ -0,0 +1,19 @@
+package model
+
+import "time"
+
+// Report etc.
+type Report struct {
+	ID          int64
+	Name        string
+	DateVersion string
+	Val         int64
+	Ctime       time.Time
+}
+
+const (
+	//BlockCount block count
+	BlockCount = "封禁总数"
+	// SecurityLoginCount security login count
+	SecurityLoginCount = "二次验证总数"
+)

app/job/main/spy/model/stat.go

@@ -0,0 +1,40 @@
+package model
+
+import "time"
+
+const (
+	// IncreaseStat increase spy stat.
+	IncreaseStat int8 = 1
+	// ResetStat reset spy stat.
+	ResetStat int8 = 2
+
+	// WaiteCheck waite for check.
+	WaiteCheck = 0
+)
+
+// type def.
+const (
+	ArchiveType  int8 = 1
+	ActivityType int8 = 2
+)
+
+// Statistics def.
+type Statistics struct {
+	ID        int64     `json:"id"`
+	TargetMid int64     `json:"target_mid"`
+	TargetID  int64     `json:"target_id"`
+	EventID   int64     `json:"event_id"`
+	State     int8      `json:"state"`
+	Type      int8      `json:"type"`
+	Quantity  int64     `json:"quantity"`
+	Ctime     time.Time `json:"ctime"`
+	Mtime     time.Time `json:"mtime"`
+}
+
+// SpyLog def.
+type SpyLog struct {
+	ID        int64     `json:"id"`
+	TargetMid int64     `json:"target_mid"`
+	Ctime     time.Time `json:"ctime"`
+	Mtime     time.Time `json:"mtime"`
+}

app/job/main/spy/model/user_event_history.go

@@ -0,0 +1,19 @@
+package model
+
+import (
+	"go-common/library/time"
+)
+
+// UserEventHistory history of user event def.
+type UserEventHistory struct {
+	ID         int64
+	Mid        int64
+	EventID    int64
+	Score      int8
+	BaseScore  int8
+	EventScore int8
+	Remark     string
+	Reason     string // 事件原因 == eventmsg.Effect
+	FactorVal  float64
+	CTime      time.Time
+}

app/job/main/spy/model/user_info.go

@@ -0,0 +1,24 @@
+package model
+
+import (
+	"go-common/library/time"
+)
+
+const (
+	// StateNormal UserInfo 正常状态
+	StateNormal = 0
+	// StateBlock UserInfo 封禁状态
+	StateBlock = 1
+)
+
+// UserInfo def.
+type UserInfo struct {
+	ID         int64     `json:"id"`
+	Mid        int64     `json:"mid"`
+	Score      int8      `json:"score"`       //真实得分
+	BaseScore  int8      `json:"base_score"`  //基础信息得分
+	EventScore int8      `json:"event_score"` //事件得分
+	State      int8      `json:"state"`       //状态 : 正常/封禁
+	CTime      time.Time `json:"ctime"`
+	MTime      time.Time `json:"mtime"`
+}

app/job/main/spy/service/BUILD

@@ -0,0 +1,69 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "block_test.go",
+        "rebuild_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "block.go",
+        "rebuild.go",
+        "report.go",
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/spy/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/spy/conf:go_default_library",
+        "//app/job/main/spy/dao:go_default_library",
+        "//app/job/main/spy/model:go_default_library",
+        "//app/service/main/spy/model:go_default_library",
+        "//app/service/main/spy/rpc/client:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/spy/service/block.go

@@ -0,0 +1,173 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+func (s *Service) lastBlockNo(seconds int64) int64 {
+	return time.Now().Unix()/seconds - 1 //cron -1 get the last hours block mids
+}
+
+//BlockTask block task.
+func (s *Service) BlockTask(c context.Context) {
+	var (
+		blockNo = s.lastBlockNo(s.c.Property.Block.CycleTimes)
+	)
+	v, ok := s.Config(model.AutoBlock)
+	if !ok {
+		log.Error("Verfiy get config error(%s,%v)", model.AutoBlock, s.spyConfig)
+		return
+	}
+	if v.(int8) != model.AutoBlockOpen {
+		log.Info("autoBlock Close(%s)", blockNo)
+		return
+	}
+	mids, _ := s.blockUsers(c, blockNo)
+	if len(mids) == 0 {
+		log.Info("s.blockUsers len is zero")
+		return
+	}
+	if err := s.dao.SetBlockCache(c, mids); err != nil {
+		log.Error("s.dao.SetBlockCache(%v) error(%v)", mids, err)
+		return
+	}
+}
+
+func (s *Service) blockUsers(c context.Context, blockNo int64) (mids []int64, err error) {
+	v, ok := s.Config(model.LimitBlockCount)
+	if !ok {
+		log.Error("blockUsers get config error(%v)", s.spyConfig)
+		return
+	}
+	if mids, err = s.dao.BlockMidCache(c, blockNo, v.(int64)); err != nil {
+		log.Error("s.dao.BlockMidCache(%s, %d) error(%v)", blockNo, v.(int64), err)
+		return
+	}
+	return
+}
+
+func (s *Service) blockByMid(c context.Context, mid int64) (err error) {
+	ui, ok := s.canBlock(c, mid)
+	if !ok {
+		log.Info("s.canBlock user had block(%d)", mid)
+		return
+	}
+	reason, remake := s.blockReason(c, mid)
+	if err = s.block(c, mid, ui, reason, remake); err != nil {
+		log.Error("s.block(%d,%v,%s) err(%v)", mid, ui, reason, err)
+	}
+	return
+}
+
+func (s *Service) block(c context.Context, mid int64, ui *model.UserInfo, reason string, remake string) (err error) {
+	var (
+		tx *sql.Tx
+	)
+	ui.State = model.StateBlock
+	if tx, err = s.dao.BeginTran(c); err != nil {
+		log.Error("s.dao.BeginTran() err(%v)", err)
+		return
+	}
+	defer func() {
+		if err != nil {
+			if err1 := tx.Rollback(); err1 != nil {
+				log.Error("tx.Rollback() error(%v)", err1)
+			}
+			return
+		}
+		if err = tx.Commit(); err != nil {
+			log.Error("tx.Commit() error(%v)", err)
+		}
+	}()
+	if err = s.dao.BlockAccount(c, ui.Mid, reason); err != nil {
+		log.Error("s.dao.BlockAccount(%d) error(%v)", ui.Mid, err)
+		return
+	}
+	var ueh = &model.UserEventHistory{
+		Mid:        mid,
+		EventID:    conf.Conf.Property.BlockEvent,
+		Score:      ui.Score,
+		BaseScore:  ui.BaseScore,
+		EventScore: ui.EventScore,
+		Remark:     remake,
+		Reason:     "自动封禁",
+	}
+
+	if err = s.dao.TxAddEventHistory(c, tx, ueh); err != nil {
+		log.Error("s.dao.TxAddEventHistory(%+v) error(%v)", ueh, err)
+		return
+	}
+	if err = s.dao.TxAddPunishment(c, tx, ui.Mid, model.PunishmentTypeBlock,
+		reason, s.lastBlockNo(s.c.Property.Block.CycleTimes)); err != nil {
+		log.Error("s.dao.TxAddPunishment(%d,%s) error(%v)", ui.Mid, reason, err)
+		return
+	}
+	// update user state.
+	if err = s.dao.TxUpdateUserState(c, tx, ui); err != nil {
+		log.Error("s.dao.TxUpdateUserState(%v) error(%v)", ui, err)
+		return
+	}
+	s.promBlockInfo.Incr("actual_block_count")
+	return
+}
+
+func (s *Service) canBlock(c context.Context, mid int64) (ui *model.UserInfo, ok bool) {
+	var (
+		err error
+	)
+	ui, err = s.dao.UserInfo(c, mid)
+	if err != nil {
+		log.Error("s.UserInfo(%d) err(%v)", mid, err)
+		return
+	}
+	v, b := s.Config(model.LessBlockScore)
+	if !b {
+		log.Error("scoreLessHandler get config error(%s,%v)", model.LessBlockScore, s.spyConfig)
+		return
+	}
+	// if blocked already , return
+	if ui.State == model.StateBlock || ui.Score > v.(int8) {
+		log.Info("canBlock not block(%v)", ui)
+		return
+	}
+	ok = true
+	return
+}
+
+func (s *Service) blockReason(c context.Context, mid int64) (reason string, remake string) {
+	var (
+		err error
+		hs  []*model.UserEventHistory
+		buf bytes.Buffer
+	)
+	if hs, err = s.dao.HistoryList(c, mid, model.BlockReasonSize); err != nil || len(hs) == 0 {
+		log.Error("s.dao.HistoryList(%d) err(%v)", mid, err)
+		return
+	}
+	m := make(map[string]int)
+	for i, v := range hs {
+		if i == 0 {
+			remake = v.Remark
+		}
+		if m[v.Reason] == 0 {
+			m[v.Reason] = 1
+		} else {
+			m[v.Reason] = m[v.Reason] + 1
+		}
+	}
+	for k, v := range m {
+		buf.WriteString(k)
+		buf.WriteString("x")
+		buf.WriteString(fmt.Sprintf("%d ", v))
+	}
+	reason = buf.String()
+	return
+}

app/job/main/spy/service/block_test.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/spy/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testBlockMid int64 = 4780461
+	testLowScore int8  = 7
+)
+
+func Test_BlockReason(t *testing.T) {
+	Convey("Test_BlockReason get block reason", t, WithService(func(s *Service) {
+		reason, remake := s.blockReason(context.TODO(), testBlockMid)
+		fmt.Println("reason remake", reason, remake)
+		So(reason, ShouldNotBeEmpty)
+		So(remake, ShouldNotBeEmpty)
+	}))
+}
+
+func Test_CanBlock(t *testing.T) {
+	Convey("Test_CanBlock can block ", t, WithService(func(s *Service) {
+		tx, err := s.dao.BeginTran(c)
+		So(err, ShouldBeNil)
+		ui := &model.UserInfo{Mid: testBlockMid, State: model.StateNormal}
+		err = s.dao.TxUpdateUserState(c, tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+
+		ui, ok := s.canBlock(context.TODO(), testBlockMid)
+		fmt.Println("Test_CanBlock ui ", ui, testBlockMid)
+		So(ui, ShouldNotBeNil)
+		So(ok, ShouldBeTrue)
+	}))
+}
+
+func Test_Block(t *testing.T) {
+	Convey("Test_Block block ", t, WithService(func(s *Service) {
+		ui, err := s.dao.UserInfo(context.TODO(), testBlockMid)
+		So(err, ShouldBeNil)
+
+		tx, err := s.dao.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		ui.State = model.StateNormal
+		err = s.dao.TxUpdateUserState(c, tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+
+		ui, err = s.dao.UserInfo(context.TODO(), testBlockMid)
+		So(err, ShouldBeNil)
+		So(ui.State == model.StateNormal, ShouldBeTrue)
+
+		reason, remake := s.blockReason(context.TODO(), testBlockMid)
+		fmt.Println("reason remake", reason, remake)
+		So(reason, ShouldNotBeEmpty)
+		So(remake, ShouldNotBeEmpty)
+
+		Convey("Test_CanBlock do block ", WithService(func(s *Service) {
+			err := s.blockByMid(context.TODO(), testBlockMid)
+			So(err, ShouldBeNil)
+			Convey("Test_CanBlock get block user info ", WithService(func(s *Service) {
+				ui, err := s.dao.UserInfo(context.TODO(), testBlockMid)
+				So(err, ShouldBeNil)
+				So(ui.State == model.StateBlock, ShouldBeTrue)
+			}))
+		}))
+		fmt.Println("Test_Block end ")
+
+	}))
+}

app/job/main/spy/service/rebuild.go

@@ -0,0 +1,58 @@
+/*
+rebuild: user portrait score reset to normal if it's score large than punishment threshold score
+*/
+
+package service
+
+import (
+	"context"
+	"time"
+
+	spy "go-common/app/service/main/spy/model"
+	"go-common/library/log"
+)
+
+const (
+	_normal = 0
+	_ps     = 100
+)
+
+func (s *Service) reBuild() {
+	var (
+		err   error
+		count int64
+	)
+	current := time.Now()
+	before30d, _ := time.ParseDuration("-720h")
+	before31d, _ := time.ParseDuration("-744h")
+	start := current.Add(before31d)
+	end := current.Add(before30d)
+	log.Info("ReBuild task start: start:(%s) end:(%s))", start, end)
+	for t := 0; t < int(s.c.Property.UserInfoShard); t++ {
+		if count, err = s.dao.ReBuildMidCount(context.TODO(), t, _normal, start, end); err != nil {
+			log.Error("s.dao.ReBuildMidCount(%s, %s), err(%v)", start, end, err)
+			continue
+		}
+		log.Info("ReBuild task: index:%d, count:%d)", t, count)
+		if count <= 0 {
+			continue
+		}
+		total := count / _ps
+		log.Info("ReBuild task: shard:%d, count:%d, total:%d)", t, count, total)
+		for i := 0; int64(i) <= total; i++ {
+			midList, err := s.dao.ReBuildMidList(context.TODO(), t, _normal, start, end, _ps)
+			if err != nil {
+				log.Error("s.dao.ReBuildMidList(%s, %s, %d, %d)", start, end, i, _ps)
+				continue
+			}
+			for _, mid := range midList {
+				if err := s.spyRPC.ReBuildPortrait(context.TODO(), &spy.ArgReBuild{Mid: mid, Reason: "自动恢复行为得分"}); err != nil {
+					log.Error("s.spyRPC.ReBuildPortrait(%d), err:%v", mid, err)
+					continue
+				}
+				log.Info("ReBuild task: mid(%d) ReBuild Portrait success)", mid)
+			}
+		}
+	}
+	log.Info("ReBuild task end: start:(%s) end:(%s))", start, end)
+}

app/job/main/spy/service/rebuild_test.go

@@ -0,0 +1,27 @@
+package service
+
+import (
+	"flag"
+	"testing"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/library/log"
+)
+
+func TestServiceReBuild(t *testing.T) {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		t.Errorf("conf.Init() error(%v)", err)
+		t.FailNow()
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	if s == nil {
+		s = New(conf.Conf)
+	}
+	testReBuild(t, s)
+}
+
+func testReBuild(t *testing.T, s *Service) {
+	s.reBuild()
+}

app/job/main/spy/service/report.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/log"
+)
+
+// AddReport add daill report.
+func (s *Service) AddReport(c context.Context) {
+	var (
+		scount      int64
+		pcount      int64
+		dateVersion string
+		err         error
+	)
+	year, month, day := time.Now().Date()
+	stoday := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
+	etoday := time.Date(year, month, day, 23, 59, 59, 999, time.Local)
+	syesday := stoday.AddDate(0, 0, -1)
+	eyesday := etoday.AddDate(0, 0, -1)
+	dateVersion = syesday.Format("20060102")
+	if pcount, err = s.dao.PunishmentCount(c, syesday, eyesday); err != nil {
+		log.Error("s.dao.PunishmentCount(%s, %s), err(%v)", syesday, eyesday, err)
+		return
+	}
+	s.dao.AddReport(c, &model.Report{
+		Name:        model.BlockCount,
+		DateVersion: dateVersion,
+		Val:         pcount,
+		Ctime:       time.Now(),
+	})
+	for i := int64(0); i < conf.Conf.Property.HistoryShard; i++ {
+		var count int64
+		if count, err = s.dao.SecurityLoginCount(c, i, "导入二次验证,恢复行为得分", syesday, eyesday); err != nil {
+			log.Error("s.dao.SecurityLoginCount(%s, %s), err(%v)", syesday, eyesday, err)
+			return
+		}
+		scount = scount + count
+		time.Sleep(s.blockTick)
+	}
+	s.dao.AddReport(c, &model.Report{
+		Name:        model.SecurityLoginCount,
+		DateVersion: dateVersion,
+		Val:         scount,
+		Ctime:       time.Now(),
+	})
+}

app/job/main/spy/service/service.go

@@ -0,0 +1,591 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strconv"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/dao"
+	"go-common/app/job/main/spy/model"
+	cmmdl "go-common/app/service/main/spy/model"
+	spyrpc "go-common/app/service/main/spy/rpc/client"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/stat/prom"
+	xtime "go-common/library/time"
+
+	"github.com/robfig/cron"
+)
+
+const (
+	_bigdataEvent = "bt"
+	_secretEvent  = "st"
+)
+
+// Service service def.
+type Service struct {
+	c               *conf.Config
+	waiter          sync.WaitGroup
+	dao             *dao.Dao
+	eventDatabus    *databus.Databus
+	spystatDatabus  *databus.Databus
+	secLoginDatabus *databus.Databus
+	spyRPC          *spyrpc.Service
+	quit            chan struct{}
+	cachech         chan func()
+	retrych         chan func()
+	spyConfig       map[string]interface{}
+	configLoadTick  time.Duration
+	promBlockInfo   *prom.Prom
+	blockTick       time.Duration
+	blockWaitTick   time.Duration
+	allEventName    map[string]int64
+	loadEventTick   time.Duration
+	// activity events
+	activityEvents map[string]struct{}
+}
+
+// New create a instance of Service and return.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:               c,
+		dao:             dao.New(c),
+		eventDatabus:    databus.New(c.Databus.EventData),
+		spystatDatabus:  databus.New(c.Databus.SpyStatData),
+		secLoginDatabus: databus.New(c.Databus.SecLogin),
+		spyRPC:          spyrpc.New(c.SpyRPC),
+		quit:            make(chan struct{}),
+		cachech:         make(chan func(), 1024),
+		retrych:         make(chan func(), 128),
+		spyConfig:       make(map[string]interface{}),
+		configLoadTick:  time.Duration(c.Property.ConfigLoadTick),
+		promBlockInfo:   prom.New().WithCounter("spy_block_info", []string{"name"}),
+		blockTick:       time.Duration(c.Property.BlockTick),
+		blockWaitTick:   time.Duration(c.Property.BlockWaitTick),
+		allEventName:    make(map[string]int64),
+		loadEventTick:   time.Duration(c.Property.LoadEventTick),
+	}
+
+	if err := s.loadSystemConfig(); err != nil {
+		panic(err)
+	}
+	if err := s.loadeventname(); err != nil {
+		panic(err)
+	}
+	s.initActivityEvents()
+
+	s.waiter.Add(1)
+	go s.consumeproc()
+
+	s.waiter.Add(1)
+	go s.secloginproc()
+
+	s.waiter.Add(1)
+	go s.spystatproc()
+
+	go s.retryproc()
+	go s.cacheproc()
+
+	go s.loadconfig()
+	go s.blockcacheuser()
+
+	go s.loadeventproc()
+
+	t := cron.New()
+	if err := t.AddFunc(s.c.Property.Block.CycleCron, s.cycleblock); err != nil {
+		panic(err)
+	}
+	if err := t.AddFunc(s.c.Property.ReportCron, s.reportjob); err != nil {
+		panic(err)
+	}
+	t.Start()
+
+	return s
+}
+
+func (s *Service) consumeproc() {
+	defer s.waiter.Done()
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("eventproc unknown panic(%v)", x)
+		}
+	}()
+	var (
+		msg       *databus.Message
+		eventMsg  *model.EventMessage
+		ok        bool
+		err       error
+		msgChan   = s.eventDatabus.Messages()
+		c         = context.TODO()
+		preOffset int64
+	)
+	for {
+		select {
+		case msg, ok = <-msgChan:
+			if !ok {
+				log.Info("eventConsumeProc msgChan closed")
+				return
+			}
+		case <-s.quit:
+			log.Info("quit eventConsumeProc")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit error(%v)", err)
+		}
+		if preOffset, err = s.dao.OffsetCache(c, _bigdataEvent, msg.Partition); err != nil {
+			log.Error("s.dao.OffsetCache(%d) error(%v)", msg.Partition, err)
+			preOffset = 0
+		} else {
+			if msg.Offset > preOffset {
+				s.setOffset(_bigdataEvent, msg.Partition, msg.Offset)
+			}
+		}
+		eventMsg = &model.EventMessage{}
+		if err = json.Unmarshal([]byte(msg.Value), eventMsg); err != nil {
+			log.Error("json.Unmarshall(%s) error(%v)", msg.Value, err)
+			s.setOffset(_bigdataEvent, msg.Partition, msg.Offset)
+			continue
+		}
+		if msg.Offset <= preOffset && s.isMsgExpiration(eventMsg.Time) {
+			log.Error("drop expired msg (%+v) (now_offset : %d)", preOffset)
+			continue
+		}
+		if err = s.handleEvent(c, eventMsg); err != nil {
+			log.Error("s.HandleEvent(%v) error(%v)", eventMsg, err)
+			continue
+		}
+		log.Info("s.handleEvent(%v) eventMsg", eventMsg)
+	}
+}
+
+func (s *Service) setOffset(event string, partition int32, offset int64) {
+	s.cachemiss(func() {
+		var err error
+		if err = s.dao.SetOffsetCache(context.TODO(), event, partition, offset); err != nil {
+			log.Error("s.dao.SetOffsetCache(%d,%d) error(%v)", partition, offset, err)
+		}
+	})
+}
+
+func (s *Service) secloginproc() {
+	defer s.waiter.Done()
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("eventproc unknown panic(%v)", x)
+		}
+	}()
+	var (
+		msg       *databus.Message
+		eventMsg  *model.EventMessage
+		ok        bool
+		err       error
+		msgChan   = s.secLoginDatabus.Messages()
+		c         = context.TODO()
+		preOffset int64
+	)
+	for {
+		select {
+		case msg, ok = <-msgChan:
+			if !ok {
+				log.Info("secloginproc msgChan closed")
+				return
+			}
+		case <-s.quit:
+			log.Info("quit secloginproc")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit error(%v)", err)
+		}
+		if preOffset, err = s.dao.OffsetCache(c, _secretEvent, msg.Partition); err != nil {
+			log.Error("s.dao.OffsetCache(%d) error(%v)", msg.Partition, err)
+			preOffset = 0
+		} else {
+			if msg.Offset > preOffset {
+				s.setOffset(_secretEvent, msg.Partition, msg.Offset)
+			}
+		}
+		eventMsg = &model.EventMessage{}
+		if err = json.Unmarshal([]byte(msg.Value), eventMsg); err != nil {
+			log.Error("json.Unmarshall(%s) error(%v)", msg.Value, err)
+			s.setOffset(_secretEvent, msg.Partition, msg.Offset)
+			continue
+		}
+		if msg.Offset <= preOffset && s.isMsgExpiration(eventMsg.Time) {
+			log.Error("drop expired msg (%+v) (now_offset : %d)", preOffset)
+			continue
+		}
+		if err = s.handleEvent(c, eventMsg); err != nil {
+			log.Error("s.HandleEvent(%v) error(%v)", eventMsg, err)
+			continue
+		}
+		log.Info("s.handleEvent(%v) eventMsg", eventMsg)
+	}
+}
+
+func (s *Service) isMsgExpiration(timeStr string) bool {
+	var (
+		eventTime time.Time
+		now       = time.Now()
+		err       error
+	)
+	if eventTime, err = time.Parse("2006-01-02 15:04:05", timeStr); err != nil {
+		return true
+	}
+	return eventTime.AddDate(0, 1, 1).Before(now)
+}
+
+func (s *Service) handleEvent(c context.Context, event *model.EventMessage) (err error) {
+	var (
+		eventTime time.Time
+		argBytes  []byte
+	)
+	if eventTime, err = time.Parse("2006-01-02 15:04:05", event.Time); err != nil {
+		log.Error("time.Parse(%s) errore(%v)", event.Time, err)
+		return
+	}
+	if argBytes, err = json.Marshal(event.Args); err != nil {
+		log.Error("json.Marshal(%v) error(%v), so empty it", event.Args, err)
+		argBytes = []byte("{}")
+	}
+	var argHandleEvent = &cmmdl.ArgHandleEvent{
+		Time:      xtime.Time(eventTime.Unix()),
+		IP:        event.IP,
+		Service:   event.Service,
+		Event:     event.Event,
+		ActiveMid: event.ActiveMid,
+		TargetMid: event.TargetMid,
+		TargetID:  event.TargetID,
+		Args:      string(argBytes),
+		Result:    event.Result,
+		Effect:    event.Effect,
+		RiskLevel: event.RiskLevel,
+	}
+	if err = s.spyRPC.HandleEvent(c, argHandleEvent); err != nil {
+		log.Error("s.spyRPC.HandleEvent(%v) error(%v)", argHandleEvent, err)
+		// s.retrymiss(func() {
+		// 	log.Info("Start retry rpc error(%v)", err)
+		// 	for {
+		// 		if err = s.spyRPC.HandleEvent(context.TODO(), argHandleEvent); err != nil {
+		// 			log.Error("s.spyRPC.HandleEvent(%v) error(%v)", argHandleEvent, err)
+		// 		} else {
+		// 			break
+		// 		}
+		// 	}
+		// 	log.Info("End retry error(%v)", err)
+		// })
+		return
+	}
+	return
+}
+
+func (s *Service) dataproc() {
+	for {
+		select {
+		case <-s.quit:
+			log.Info("quit handleData")
+			return
+		default:
+		}
+		if !s.c.Property.Debug {
+			d := time.Now().AddDate(0, 0, 1)
+			ts := time.Date(d.Year(), d.Month(), d.Day(), 5, 0, 0, 0, time.Local).Sub(time.Now())
+			time.Sleep(ts)
+		} else {
+			time.Sleep(s.c.Property.TaskTimer * time.Second)
+		}
+		s.reBuild()
+	}
+}
+
+// Ping check service health.
+func (s *Service) Ping(c context.Context) (err error) {
+	if err = s.dao.Ping(c); err != nil {
+		log.Error("s.db.Ping() error(%v)", err)
+		return
+	}
+	return
+}
+
+// Close all resource.
+func (s *Service) Close() (err error) {
+	close(s.quit)
+	s.dao.Close()
+	if err = s.eventDatabus.Close(); err != nil {
+		log.Error("s.db.Close() error(%v)", err)
+		return
+	}
+	return
+}
+
+// Wait wait all closed.
+func (s *Service) Wait() {
+	s.waiter.Wait()
+}
+
+func (s *Service) cacheproc() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.cacheproc panic(%v)", x)
+			go s.cacheproc()
+			log.Info("service.cacheproc recover")
+		}
+	}()
+	for {
+		f := <-s.cachech
+		f()
+	}
+}
+
+func (s *Service) retryproc() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.retryproc panic(%v)", x)
+			go s.retryproc()
+			log.Info("service.retryproc recover")
+		}
+	}()
+	for {
+		f := <-s.retrych
+		go f()
+	}
+}
+
+func (s *Service) cachemiss(f func()) {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.cachemiss panic(%v)", x)
+		}
+	}()
+	select {
+	case s.cachech <- f:
+	default:
+		log.Error("service.cachech full")
+	}
+}
+
+func (s *Service) retrymiss(f func()) {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.retrymiss panic(%v)", x)
+		}
+	}()
+	select {
+	case s.retrych <- f:
+	default:
+		log.Error("service.retrych full")
+	}
+}
+
+func (s *Service) loadconfig() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.cycleblock panic(%v)", x)
+		}
+	}()
+	for {
+		time.Sleep(s.configLoadTick)
+		s.loadSystemConfig()
+	}
+}
+
+func (s *Service) loadSystemConfig() (err error) {
+	var (
+		cdb map[string]string
+	)
+	cdb, err = s.dao.Configs(context.TODO())
+	if err != nil {
+		log.Error("sys config db get data err(%v)", err)
+		return
+	}
+	if len(cdb) == 0 {
+		err = errors.New("sys config no data")
+		return
+	}
+	cs := make(map[string]interface{}, len(cdb))
+	for k, v := range cdb {
+		switch k {
+		case model.LimitBlockCount:
+			t, err1 := strconv.ParseInt(v, 10, 64)
+			if err1 != nil {
+				log.Error("sys config err(%s,%v,%v)", model.LimitBlockCount, t, err1)
+				err = err1
+				return
+			}
+			cs[k] = t
+		case model.LessBlockScore:
+			tmp, err1 := strconv.ParseInt(v, 10, 8)
+			if err1 != nil {
+				log.Error("sys config err(%s,%v,%v)", model.LessBlockScore, tmp, err1)
+				err = err1
+				return
+			}
+			cs[k] = int8(tmp)
+		case model.AutoBlock:
+			tmp, err1 := strconv.ParseInt(v, 10, 8)
+			if err1 != nil {
+				log.Error("sys config err(%s,%v,%v)", model.AutoBlock, tmp, err1)
+				err = err1
+				return
+			}
+			cs[k] = int8(tmp)
+		default:
+			cs[k] = v
+		}
+	}
+	s.spyConfig = cs
+	log.Info("loadSystemConfig success(%v)", cs)
+	return
+}
+
+//Config get config.
+func (s *Service) Config(key string) (interface{}, bool) {
+	if s.spyConfig == nil {
+		return nil, false
+	}
+	v, ok := s.spyConfig[key]
+	return v, ok
+}
+
+func (s *Service) cycleblock() {
+	var (
+		c = context.TODO()
+	)
+	log.Info("cycleblock start (%v)", time.Now())
+	if b, err := s.dao.SetNXLockCache(c, model.BlockLockKey, model.DefLockTime); !b || err != nil {
+		log.Error("cycleblock had run (%v,%v)", b, err)
+		return
+	}
+	s.BlockTask(c)
+	s.dao.DelLockCache(c, model.BlockLockKey)
+	log.Info("cycleblock end (%v)", time.Now())
+}
+
+func (s *Service) blockcacheuser() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.blockcacheuser panic(%v)", x)
+			go s.blockcacheuser()
+			log.Info("service.blockcacheuser recover")
+		}
+	}()
+	for {
+		mid, err := s.dao.SPOPBlockCache(context.TODO())
+		if err != nil {
+			log.Error("blockcacheuser err (%v,%v)", mid, err)
+			continue
+		}
+		if mid != 0 {
+			s.blockByMid(context.TODO(), mid)
+			time.Sleep(s.blockTick)
+		} else {
+			// when no user should be block
+			time.Sleep(s.blockWaitTick)
+		}
+	}
+}
+
+func (s *Service) reportjob() {
+	var (
+		c = context.TODO()
+	)
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("service.reportjob panic(%v)", x)
+			go s.blockcacheuser()
+			log.Info("service.reportjob recover")
+		}
+	}()
+	log.Info("reportjob start (%v)", time.Now())
+	if b, err := s.dao.SetNXLockCache(c, model.ReportJobKey, model.DefLockTime); !b || err != nil {
+		log.Error("reportjob had run (%v,%v)", b, err)
+		return
+	}
+	s.AddReport(c)
+	log.Info("reportjob end (%v)", time.Now())
+}
+
+func (s *Service) spystatproc() {
+	defer s.waiter.Done()
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("sinstatproc unknown panic(%v)", x)
+		}
+	}()
+	var (
+		msg     *databus.Message
+		statMsg *model.SpyStatMessage
+		ok      bool
+		err     error
+		msgChan = s.spystatDatabus.Messages()
+		c       = context.TODO()
+	)
+	for {
+		select {
+		case msg, ok = <-msgChan:
+			if !ok {
+				log.Info("spystatproc msgChan closed")
+				return
+			}
+		case <-s.quit:
+			log.Info("quit spystatproc")
+			return
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("msg.Commit error(%v)", err)
+		}
+		statMsg = &model.SpyStatMessage{}
+		if err = json.Unmarshal([]byte(msg.Value), statMsg); err != nil {
+			log.Error("json.Unmarshall(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		log.Info(" spystatproc (%v) start", statMsg)
+		// check uuid
+		unique, _ := s.dao.PfaddCache(c, statMsg.UUID)
+		if !unique {
+			log.Error("stat duplicate msg (%s) error", statMsg)
+			continue
+		}
+		s.UpdateStatData(c, statMsg)
+		log.Info(" spystatproc (%v) handle", statMsg)
+	}
+}
+
+func (s *Service) loadeventname() (err error) {
+	var (
+		c  = context.Background()
+		es []*model.Event
+	)
+	es, err = s.dao.AllEvent(c)
+	if err != nil {
+		log.Error("loadeventname allevent error(%v)", err)
+		return
+	}
+	tmp := make(map[string]int64, len(es))
+	for _, e := range es {
+		tmp[e.Name] = e.ID
+	}
+	s.allEventName = tmp
+	log.Info("loadeventname (%v) load success", tmp)
+	return
+}
+
+func (s *Service) loadeventproc() {
+	for {
+		time.Sleep(s.loadEventTick)
+		s.loadeventname()
+	}
+}
+
+func (s *Service) initActivityEvents() {
+	tmp := make(map[string]struct{}, len(s.c.Property.ActivityEvents))
+	for _, name := range s.c.Property.ActivityEvents {
+		tmp[name] = struct{}{}
+	}
+	s.activityEvents = tmp
+}

app/job/main/spy/service/service_test.go

@@ -0,0 +1,132 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/spy/conf"
+	"go-common/app/job/main/spy/model"
+	"go-common/library/cache/redis"
+
+	"github.com/robfig/cron"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	c          = context.Background()
+	s          *Service
+	testCron   = "*/5 * * * * ?"
+	testCyTime = 5000
+)
+
+func init() {
+	var (
+		err error
+	)
+	dir, _ := filepath.Abs("../cmd/spy-job-dev.toml")
+	flag.Set("conf", dir)
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+	if s == nil {
+		s = New(conf.Conf)
+	}
+	time.Sleep(time.Second)
+}
+
+func CleanCache() {
+	pool := redis.NewPool(conf.Conf.Redis.Config)
+	pool.Get(c).Do("FLUSHDB")
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		Reset(func() { CleanCache() })
+		f(s)
+	}
+}
+
+func Test_LoadSystemConfig(t *testing.T) {
+	Convey("Test_LoadSystemConfig had data", t, WithService(func(s *Service) {
+		fmt.Println(s.spyConfig)
+		So(s.spyConfig, ShouldContainKey, model.LimitBlockCount)
+		So(s.spyConfig, ShouldContainKey, model.LessBlockScore)
+		So(s.spyConfig, ShouldContainKey, model.AutoBlock)
+	}))
+}
+
+func Test_cycleblock(t *testing.T) {
+	Convey("Test_cycleblock cron", t, WithService(func(s *Service) {
+		fmt.Println("Test_cycleblock start ")
+		tx, err := s.dao.BeginTran(c)
+		So(err, ShouldBeNil)
+		ui := &model.UserInfo{Mid: testBlockMid, State: model.StateNormal}
+		err = s.dao.TxUpdateUserState(c, tx, ui)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+
+		lastBlockNo := s.lastBlockNo(s.c.Property.Block.CycleTimes)
+		t := cron.New()
+		err = t.AddFunc(testCron, s.cycleblock)
+		if err != nil {
+			panic(err)
+		}
+		t.Start()
+
+		Convey("Test_cycleblock user info 1", WithService(func(s *Service) {
+			var ui *model.UserInfo
+			ui, err = s.dao.UserInfo(context.TODO(), testBlockMid)
+			So(err, ShouldBeNil)
+			So(ui.State == model.StateNormal, ShouldBeTrue)
+		}))
+
+		err = s.dao.AddBlockCache(c, testBlockMid, testLowScore, lastBlockNo)
+		So(err, ShouldBeNil)
+		mids, err := s.blockUsers(c, lastBlockNo)
+		So(err, ShouldBeNil)
+		So(mids, ShouldContain, testBlockMid)
+
+		time.Sleep(5000 * time.Millisecond)
+		time.Sleep(time.Duration(s.blockWaitTick))
+
+		Convey("Test_cycleblock user info 2 ", WithService(func(s *Service) {
+			ui, err := s.dao.UserInfo(context.TODO(), testBlockMid)
+			So(err, ShouldBeNil)
+			So(ui.State == model.StateBlock, ShouldBeTrue)
+		}))
+		fmt.Println("Test_cycleblock end ")
+	}))
+}
+
+//  go test  -test.v -test.run TestStat
+func TestStat(t *testing.T) {
+	Convey(" UpdateStatData ", t, WithService(func(s *Service) {
+		err := s.UpdateStatData(c, &model.SpyStatMessage{
+			TargetMid: 1,
+			TargetID:  1,
+			EventName: "init_user_info",
+			Type:      model.IncreaseStat,
+			Quantity:  2,
+			Time:      time.Now().Unix(),
+			UUID:      "123456789qweasdzxcccccccc",
+		})
+		So(err, ShouldBeNil)
+	}))
+	Convey(" UpdateStatData 2", t, WithService(func(s *Service) {
+		err := s.UpdateStatData(c, &model.SpyStatMessage{
+			TargetMid: 1,
+			TargetID:  1,
+			EventName: "auto_block",
+			Type:      model.ResetStat,
+			Quantity:  2,
+			Time:      time.Now().Unix(),
+			UUID:      "123456789qweasdzxcccccccc",
+		})
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/spy/service/stat.go

@@ -0,0 +1,49 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/spy/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// UpdateStatData update spy stat data.
+func (s *Service) UpdateStatData(c context.Context, m *model.SpyStatMessage) (err error) {
+	//TODO check event resaon
+	if s.allEventName[m.EventName] == 0 {
+		log.Error("event name not found %+v", err)
+		err = ecode.SpyEventNotExist
+		return
+	}
+	stat := &model.Statistics{
+		TargetMid: m.TargetMid,
+		TargetID:  m.TargetID,
+		EventID:   s.allEventName[m.EventName],
+		State:     model.WaiteCheck,
+		Quantity:  m.Quantity,
+		Ctime:     time.Now(),
+	}
+	if stat.TargetID != 0 {
+		_, ok := s.activityEvents[m.EventName]
+		if ok {
+			stat.Type = model.ActivityType
+		} else {
+			stat.Type = model.ArchiveType
+		}
+	}
+	// add stat
+	if model.ResetStat == m.Type {
+		if _, err = s.dao.AddStatistics(c, stat); err != nil {
+			log.Error("%+v", err)
+			return
+		}
+	} else {
+		if _, err = s.dao.AddIncrStatistics(c, stat); err != nil {
+			log.Error("%+v", err)
+			return
+		}
+	}
+	return
+}