Create & Init Project...

app/job/main/click/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/click/cmd:all-srcs",
+        "//app/job/main/click/conf:all-srcs",
+        "//app/job/main/click/dao:all-srcs",
+        "//app/job/main/click/http:all-srcs",
+        "//app/job/main/click/model:all-srcs",
+        "//app/job/main/click/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/CHANGELOG.md

@@ -0,0 +1,92 @@
+# click-job
+
+### v1.8.5
+> 1. 冷门稿件集中落地的时间由2:00-5:00改为2:00-6:30，保证可以全部处理完
+
+### v1.8.4
+> 1. 修复rtype=2的问题
+
+### v1.8.3
+> 1. 将inline_play_heartbeat改为inline_play_to_view，且暂时兼容老逻辑
+
+### v1.8.2
+> 1. inline播放第一次上报由判断plat为6/7/8/9改为判断UA或者plat（兼容老逻辑），并且上报rtype =2
+> 2. 增加played_time_enough逻辑（inline播放十秒计数），inline播放第二次上报由判断UA中的auto_play改为判断UA中的inline_play_heartbeat/played_time_enough/auto_play（兼容老逻辑）
+
+### v1.8.1
+> 1.增加拜年祭日志
+
+### v1.8.0
+> 1.2019年拜年祭的单品播放数加进主视频的播放数
+
+### v1.7.7
+> 1.lv=-2时，未登录用户也不计算播放数
+> 2.支持黑名单列表
+
+### v1.7.6
+> 1.多线程一起pub
+
+### v1.7.5
+> 1.记录自动播放的buvidToDid的信息，并在之后的上报信息中根据它计算vv
+
+### v1.7.4
+> 1.消费report-click merge后的databus
+
+### v1.7.3
+> 1.自动播放判断
+
+### v1.7.2
+> 1.增加autoplay不计数的逻辑
+
+### v1.7.1
+> 1.迁移BM
+
+### v1.7.0
+> 1. update infoc sdk
+
+### v1.6.1
+> 1.支持plat=5  
+
+### v1.6.0
+> 1.迁移到主站目录下  
+
+### v1.5.0
+> 1.删除消费kafka的所有代码  
+
+### v1.4.1
+> 1.增加trace  
+
+### v1.4.0
+> 1.消费从kafka迁移到databus  
+
+### v1.3.0
+> 1.plat:5增加tv版  
+> 2.forbid增加-2，全面封禁  
+> 3.ugc&pgc计算方式改版 https://www.tapd.cn/20095661/prong/stories/view/1120095661001049183 
+
+### v1.2.0
+> 1.PGC根据epid来计算点击数，提高播放数  
+
+### v1.1.3
+> 1.增加释放内存的时间配置  
+
+### v1.1.2
+> 1.更新计数的时间配置化  
+
+### v1.1.1
+> 1.补充单元测试  
+
+### v1.1.0
+> 1.双写新的databus  
+
+### v1.0.3
+> 1.在凌晨2~4点之间释放内存中的数据  
+
+### v1.0.2
+> 1.处理maxAID逻辑  
+
+### v1.0.1
+> 1.简化程序日志  
+
+### v1.0.0
+> 1.项目初始化  

app/job/main/click/CONTRIBUTORS.md

@@ -0,0 +1,9 @@
+# Owner
+peiyifei
+
+# Author 
+peiyifei
+
+# Reviewer
+peiyifei
+haoguanwei

app/job/main/click/OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- peiyifei
+labels:
+- job
+- job/main/click
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- haoguanwei
+- peiyifei

app/job/main/click/README.md

@@ -0,0 +1,7 @@
+#### click-job
+
+##### 项目简介
+> 1.稿件点击计数
+
+##### 编译环境
+> 请只用golang v1.8.x以上版本编译执行。

app/job/main/click/cmd/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["click-job-test.toml"],
+    importpath = "go-common/app/job/main/click/cmd",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/http:go_default_library",
+        "//app/job/main/click/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/cmd/click-job-test.toml

@@ -0,0 +1,142 @@
+version = "1.0.0"
+user    = "nobody"
+pid     = "/tmp/click-job.pid"
+dir     = "./"
+perf    = "127.0.0.1:6440"
+trace   = false
+debug   = false
+env     = "uat"
+
+chanNum = 50
+hashNum = 500000
+consumeNum = 10
+needInit = false
+lastChangeTime = 75
+releaseTime = 600
+
+[cacheConf]
+ArcUpCacheTime = 600
+PGCReplayTime = 1800
+newAnonymousCacheTime = 60
+NewAnonymousBvCacheTime = 300
+
+[xlog]
+dir = "/data/log/click-job/"
+
+[infoc2]
+taskID = "000025"
+proto = "tcp"
+addr = "172.16.113.149:15140"
+chanSize = 10240
+
+[multiHTTP]
+    [multiHTTP.inner]
+    addrs = ["0.0.0.0:6441"]
+    [multiHTTP.local]
+    addrs = ["0.0.0.0:6442"]
+
+[db]
+addr = "172.22.34.101:3306"
+dsn = "test:test@tcp(172.16.33.205:3308)/archive_click?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8"
+active = 100
+idle = 50
+idleTimeout ="4h"
+queryTimeout = "300ms"
+execTimeout = "300ms"
+tranTimeout = "500ms"
+[db.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+name = "click-job/redis"
+proto = "unix"
+addr = "/tmp/uat-archive-service-redis.sock"
+active = 10
+idle = 1
+dialTimeout = "1s"
+readTimeout = "2s"
+writeTimeout = "2s"
+idleTimeout = "80s"
+
+[statPub]
+key = "0QNB0ZgFozbKUCQhbTq8"
+secret = "0QNB0ZgFozbKUCQhbTq9"
+group = "Stat-UGC-P"
+topic = "Stat-T"
+action = "pub"
+name = "stat-pub/stat-pub"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+active = 5
+idle = 1
+dialTimeout = "1s"
+readTimeout = "1s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[StatViewPub]
+key = "8e27ab7e39270b59"
+secret = "477df6a068d7332a163f95abbad2079c"
+group = "StatView-MainAppSvr-P"
+topic = "StatView-T"
+action = "pub"
+name = "stat-pub/stat-pub"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+active = 5
+idle = 1
+dialTimeout = "1s"
+readTimeout = "1s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[ReportDatabus]
+key = "8e27ab7e39270b59"
+secret = "477df6a068d7332a163f95abbad2079c"
+group = "ArchiveClick-MainArchive-S"
+topic = "ArchiveClick-T"
+action = "sub"
+name = "report-click/sub"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+active = 5
+idle = 1
+dialTimeout = "1s"
+readTimeout = "1s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+
+[HTTPClient]
+key       = "e7482d29be4a95b8"
+secret    = "9e803791cdef756e75faee68e12b7442"
+dial      = "50ms"
+timeout   = "200ms"
+keepAlive = "60s"
+    [HTTPClient.breaker]
+    window  = "1s"
+    sleep   = "100ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[archiveRPC]
+    group = "uat"
+    pullInterval = "10s"
+    [archiveRPC.client]
+    token = "lY0h6KmXH7u9ftRIuhQTkOYqUg10gwHG"
+    timeout = "1s"
+    [archiveRPC.client.breaker]
+    window  ="3s"
+    sleep   ="100ms"
+    bucket  = 10        
+    ratio   = 0.5
+    request = 100
+    [archiveRPC.zookeeper]
+    root = "/microservice/archive-service/"
+    addrs = ["172.18.33.50:2199","172.18.33.51:2199","172.18.33.52:2199"]
+    timeout = "30s"

app/job/main/click/cmd/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/main/click/conf"
+	"go-common/app/job/main/click/http"
+	"go-common/app/job/main/click/service"
+	"go-common/library/log"
+)
+
+var (
+	srv *service.Service
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	log.Init(nil)
+	defer log.Close()
+	log.Info("click-job start")
+	srv = service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	signalHandler()
+}
+
+func signalHandler() {
+	var (
+		err error
+		ch  = make(chan os.Signal, 1)
+	)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		si := <-ch
+		switch si {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("get a signal %s, stop the consume process", si.String())
+			if err = srv.Close(); err != nil {
+				log.Error("srv close consumer error(%v)", err)
+			}
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/click/conf/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/click/conf",
+    tags = ["automanaged"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/conf/conf.go

@@ -0,0 +1,120 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	Conf     = &Config{}
+)
+
+type Config struct {
+	// Env
+	Env            string
+	LastChangeTime int64
+	ReleaseTime    int64
+	BnjMainAid     int64
+	BnjListAids    []int64
+	// interface XLog
+	XLog *log.Config
+	// tracer
+	Tracer *trace.Config
+	// stat databus pub
+	StatPub            *databus.Config
+	StatViewPub        *databus.Config
+	ReportMergeDatabus *databus.Config
+	// click databus pub
+	ClickPub   *databus.Config
+	ArchiveRPC *rpc.ClientConfig
+	// http
+	BM *bm.ServerConfig
+	// redis
+	Redis      *redis.Config
+	HTTPClient *bm.ClientConfig
+	// cache time conf
+	CacheConf struct {
+		PGCReplayTime           int64
+		ArcUpCacheTime          int64
+		NewAnonymousCacheTime   int64
+		NewAnonymousBvCacheTime int64
+	}
+	// db
+	DB *sql.Config
+	// hash number
+	HashNum int64
+	// chan number
+	ChanNum int64
+	// consumer num
+	ConsumeNum int
+	// need Init
+	NeedInit bool
+	// infoc
+	Infoc2 *infoc.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	client.Watch("click-job.toml")
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/click/dao/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "archive_test.go",
+        "bangumi_test.go",
+        "click_test.go",
+        "dao_test.go",
+        "forbid_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "archive.go",
+        "bangumi.go",
+        "click.go",
+        "dao.go",
+        "forbid.go",
+    ],
+    importpath = "go-common/app/job/main/click/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/dao/archive.go

@@ -0,0 +1,30 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/library/log"
+)
+
+const (
+	_maxAIDPath = "http://api.bilibili.co/x/internal/v2/archive/maxAid"
+)
+
+// MaxAID return max aid
+func (d *Dao) MaxAID(c context.Context) (id int64, err error) {
+	var res struct {
+		Code int   `json:"code"`
+		Data int64 `json:"data"`
+	}
+	if err = d.client.Get(c, _maxAIDPath, "", nil, &res); err != nil {
+		log.Error("d.client.MaxAid error(%+v)", err)
+		return
+	}
+	if res.Code != 0 {
+		log.Error("d.client.MaxAid Code(%d)", res.Code)
+		return
+	}
+	log.Info("got MaxAid(%d)", res.Data)
+	id = res.Data
+	return
+}

app/job/main/click/dao/archive_test.go

@@ -0,0 +1,14 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_MaxAID(t *testing.T) {
+	Convey("MaxAID", t, func() {
+		d.MaxAID(context.TODO())
+	})
+}

app/job/main/click/dao/bangumi.go

@@ -0,0 +1,80 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+
+	"go-common/library/log"
+)
+
+const (
+	_aid2epid  = "http://bangumi.bilibili.co/ext/internal/archive/aid2epid"
+	_epidExist = "http://bangumi.bilibili.co/ext/internal/archive/aid/play"
+	_isLegal   = int(1)
+)
+
+var seasonType = []int{1, 2, 3, 4, 5}
+
+// LoadAllBangumi load all bangumi epid -> aid to map
+func (d *Dao) LoadAllBangumi(c context.Context) (etam map[int64]int64, err error) {
+	etam = make(map[int64]int64)
+	for _, t := range seasonType {
+		pageNum := 1
+		for {
+			var resp struct {
+				Code    int             `json:"code"`
+				Message string          `json:"message"`
+				Result  map[int64]int64 `json:"result"`
+			}
+			p := url.Values{}
+			p.Set("build", "0")
+			p.Set("platform", "golang")
+			p.Set("season_type", strconv.Itoa(t))
+			p.Set("page_size", "1000")
+			p.Set("page_no", strconv.Itoa(pageNum))
+			// one time error,all return,wait for next update
+			if err = d.client.Get(c, _aid2epid, "", p, &resp); err != nil {
+				log.Error("d.client.Get(%s) error(%v)", _aid2epid+"?"+p.Encode(), err)
+				return
+			}
+			// record the page number when result is empty
+			if len(resp.Result) == 0 {
+				log.Info("bangumi seasonType(%d) pageNo(%d) is end", t, pageNum)
+				break
+			}
+			for epid, aid := range resp.Result {
+				etam[epid] = aid
+			}
+			pageNum++
+		}
+	}
+	return
+}
+
+// IsLegal check legal by aid epID seasonType
+func (d *Dao) IsLegal(c context.Context, aid, epID int64, seasonType int) (islegal bool, err error) {
+	var resp struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+		Result  struct {
+			Status int `json:"status"`
+		} `json:"result"`
+	}
+	p := url.Values{}
+	p.Set("build", "0")
+	p.Set("platform", "golang")
+	p.Set("season_type", strconv.Itoa(seasonType))
+	p.Set("epid", strconv.FormatInt(epID, 10))
+	p.Set("aid", strconv.FormatInt(aid, 10))
+	if err = d.client.Get(c, _epidExist, "", p, &resp); err != nil {
+		log.Error("d.client.Get(%s) error(%v)", _epidExist+"?"+p.Encode(), err)
+		return
+	}
+	if resp.Result.Status != _isLegal {
+		log.Error("aid(%d) epid(%d) seasonType(%d) is unlegal", aid, epID, seasonType)
+		return
+	}
+	islegal = true
+	return
+}

app/job/main/click/dao/bangumi_test.go

@@ -0,0 +1,27 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_LoadAllBangumi(t *testing.T) {
+	Convey("LoadAllBangumi", t, func() {
+		etam, err := d.LoadAllBangumi(context.TODO())
+		So(err, ShouldBeNil)
+		So(etam, ShouldNotBeNil)
+		for epid, aid := range etam {
+			Printf("epid:%d,aid:%d\n", epid, aid)
+		}
+	})
+}
+
+func Test_IsLegal(t *testing.T) {
+	Convey("IsLegal", t, func() {
+		isLegal, err := d.IsLegal(context.TODO(), 11696747, 157927, 1)
+		So(err, ShouldBeNil)
+		Println(isLegal)
+	})
+}

app/job/main/click/dao/click.go

@@ -0,0 +1,66 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"go-common/app/job/main/click/model"
+	"go-common/library/log"
+)
+
+const (
+	_cliSQL          = "SELECT aid,web,h5,outside,ios,android,android_tv FROM %s WHERE aid=?"
+	_addCliSQL       = "INSERT INTO %s(aid,web,h5,outside,ios,android,android_tv) VALUES(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE web=?,h5=?,outside=?,ios=?,android=?,android_tv=?"
+	_upCliSQL        = "UPDATE %s SET web=web+?,h5=h5+?,outside=outside+?,ios=ios+?,android=android+?,android_tv=android_tv+? WHERE aid=?"
+	_upSpecialCliSQL = "UPDATE %s SET %s=? WHERE aid=?"
+)
+
+func getTable(aid int64) string {
+	return fmt.Sprintf("archive_click_%02d", aid%100)
+}
+
+// Click get click
+func (d *Dao) Click(c context.Context, aid int64) (cli *model.ClickInfo, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_cliSQL, getTable(aid)), aid)
+	cli = &model.ClickInfo{}
+	if err = row.Scan(&cli.Aid, &cli.Web, &cli.H5, &cli.Outer, &cli.Ios, &cli.Android, &cli.AndroidTV); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			cli = nil
+		} else {
+			log.Error("row.Scan error(%v)")
+		}
+	}
+	return
+}
+
+// AddClick add av clicks
+func (d *Dao) AddClick(c context.Context, aid, web, h5, out, ios, android, androidTV int64) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_addCliSQL, getTable(aid)), aid, web, h5, out, ios, android, androidTV, web, h5, out, ios, android, androidTV)
+	if err != nil {
+		log.Error("d.addCliStmt.Exec(%d) error(%v)", aid, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// UpClick update av clicks
+func (d *Dao) UpClick(c context.Context, cli *model.ClickInfo) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_upCliSQL, getTable(cli.Aid)), cli.Web, cli.H5, cli.Outer, cli.Ios, cli.Android, cli.AndroidTV, cli.Aid)
+	if err != nil {
+		log.Error("d.upCliStmt.Exec(+%v) error(%v)", cli, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// UpSpecial update special platform click
+func (d *Dao) UpSpecial(c context.Context, aid int64, tp string, num int64) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_upSpecialCliSQL, getTable(aid), tp), num, aid)
+	if err != nil {
+		log.Error("d.db.Exec error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/click/dao/click_test.go

@@ -0,0 +1,38 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/click/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Click(t *testing.T) {
+	Convey("Click", t, func() {
+		_, err := d.Click(context.TODO(), 1)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_AddClick(t *testing.T) {
+	Convey("AddClick", t, func() {
+		_, err := d.AddClick(context.TODO(), 3, 1, 1, 1, 1, 1, 100)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_UpClick(t *testing.T) {
+	Convey("UpClick", t, func() {
+		rows, err := d.UpClick(context.TODO(), &model.ClickInfo{Aid: 2, AndroidTV: 22222})
+		Println(rows, err)
+	})
+}
+
+func Test_UpSpecial(t *testing.T) {
+	Convey("UpSpecial", t, func() {
+		_, err := d.UpSpecial(context.TODO(), 1, model.TypeForAndroid, 1)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/click/dao/dao.go

@@ -0,0 +1,24 @@
+package dao
+
+import (
+	"go-common/app/job/main/click/conf"
+	"go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao is
+type Dao struct {
+	c      *conf.Config
+	db     *sql.DB
+	client *bm.Client
+}
+
+// New is
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     sql.NewMySQL(c.DB),
+		client: bm.NewClient(c.HTTPClient),
+	}
+	return d
+}

app/job/main/click/dao/dao_test.go

@@ -0,0 +1,19 @@
+package dao
+
+import (
+	"flag"
+	"path/filepath"
+
+	"go-common/app/job/main/click/conf"
+)
+
+var (
+	d *Dao
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/click-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	d = New(conf.Conf)
+}

app/job/main/click/dao/forbid.go

@@ -0,0 +1,76 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/click/model"
+	"go-common/library/log"
+)
+
+const (
+	_forbidSQL        = "SELECT aid,plat,lv,locked FROM archive_click_forbid WHERE locked = ?"
+	_upForbidSQL      = "INSERT INTO archive_click_forbid(aid,plat,lv,locked) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE lv=?,locked=?"
+	_allForbidMidsSQL = "SELECT mid FROM archive_mid_forbid WHERE status=1"
+	_upForbidMidSQL   = "INSERT INTO archive_mid_forbid(mid,status) VALUE(?,?) ON DUPLICATE KEY UPDATE status=?"
+)
+
+// ForbidMids is
+func (d *Dao) ForbidMids(c context.Context) (mids map[int64]struct{}, err error) {
+	rows, err := d.db.Query(c, _allForbidMidsSQL)
+	if err != nil {
+		log.Error("d.db.Query(%s) error(%v)", _allForbidMidsSQL, err)
+		return
+	}
+	defer rows.Close()
+	mids = make(map[int64]struct{})
+	for rows.Next() {
+		var mid int64
+		if err = rows.Scan(&mid); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		mids[mid] = struct{}{}
+	}
+	err = rows.Err()
+	return
+}
+
+// UpMidForbidStatus is
+func (d *Dao) UpMidForbidStatus(c context.Context, mid int64, status int8) (err error) {
+	_, err = d.db.Exec(c, _upForbidMidSQL, mid, status, status)
+	return
+}
+
+// Forbids is
+func (d *Dao) Forbids(c context.Context) (forbids map[int64]map[int8]*model.Forbid, err error) {
+	rows, err := d.db.Query(c, _forbidSQL, model.ValueForLocked)
+	if err != nil {
+		log.Error("d.db.Query(%s) error(%v)", _forbidSQL, model.ValueForLocked, err)
+		return
+	}
+	defer rows.Close()
+	forbids = make(map[int64]map[int8]*model.Forbid)
+	for rows.Next() {
+		var f = &model.Forbid{}
+		if err = rows.Scan(&f.AID, &f.Plat, &f.Lv, &f.Locked); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		if _, ok := forbids[f.AID]; !ok {
+			forbids[f.AID] = make(map[int8]*model.Forbid)
+		}
+		forbids[f.AID][f.Plat] = f
+	}
+	err = rows.Err()
+	return
+}
+
+// UpForbid is
+func (d *Dao) UpForbid(c context.Context, aid int64, plat, lock, lv int8) (rows int64, err error) {
+	res, err := d.db.Exec(c, _upForbidSQL, aid, plat, lv, lock, lv, lock)
+	if err != nil {
+		log.Error("d.db.Exec(%s) error(%v)", _upForbidSQL, err)
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/click/dao/forbid_test.go

@@ -0,0 +1,38 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Forbids(t *testing.T) {
+	Convey("Forbids", t, func() {
+		fs, err := d.Forbids(context.TODO())
+		So(err, ShouldBeNil)
+		Println(fs)
+	})
+}
+
+func Test_UpForbid(t *testing.T) {
+	Convey("UpForbid", t, func() {
+		_, err := d.UpForbid(context.TODO(), 1, 1, 1, 1)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_ForbidMids(t *testing.T) {
+	Convey("ForbidMids", t, func() {
+		mids, err := d.ForbidMids(context.TODO())
+		So(err, ShouldBeNil)
+		Println(mids)
+	})
+}
+
+func Test_UpForbidMid(t *testing.T) {
+	Convey("UpForbidMid", t, func() {
+		err := d.UpMidForbidStatus(context.TODO(), 1684013, 0)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/click/http/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "click.go",
+        "http.go",
+    ],
+    importpath = "go-common/app/job/main/click/http",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/model:go_default_library",
+        "//app/job/main/click/service:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/http/click.go

@@ -0,0 +1,127 @@
+package http
+
+import (
+	"strconv"
+
+	"go-common/app/job/main/click/model"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+)
+
+func click(c *bm.Context) {
+	var (
+		aid, click                    int64
+		aidStr, platformStr, clickStr string
+		err                           error
+	)
+	params := c.Request.Form
+	if clickStr = params.Get("click"); clickStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if click, err = strconv.ParseInt(clickStr, 10, 64); err != nil || click < 0 {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if aidStr = params.Get("aid"); aidStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if aid, err = strconv.ParseInt(aidStr, 10, 64); err != nil || aid <= 0 {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if platformStr = params.Get("platform"); platformStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if platformStr != model.TypeForAndroid &&
+		platformStr != model.TypeForH5 &&
+		platformStr != model.TypeForIOS &&
+		platformStr != model.TypeForOutside &&
+		platformStr != model.TypeForWeb &&
+		platformStr != model.TypeForAndroidTv {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if err = srv.SetSpecial(c, aid, click, platformStr); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	c.JSON(nil, nil)
+}
+
+func lock(c *bm.Context) {
+	var (
+		aid, plat, lv, lock                 int64
+		aidStr, platformStr, lvStr, lockStr string
+		err                                 error
+	)
+	params := c.Request.Form
+	if aidStr = params.Get("aid"); aidStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if platformStr = params.Get("platform"); platformStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if lvStr = params.Get("lv"); lvStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if lockStr = params.Get("lock"); lockStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if aid, err = strconv.ParseInt(aidStr, 10, 64); err != nil || aid <= 0 {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if plat, err = strconv.ParseInt(platformStr, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if lv, err = strconv.ParseInt(lvStr, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if lock, err = strconv.ParseInt(lockStr, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if err = srv.SetLock(c, aid, int8(plat), int8(lock), int8(lv)); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	c.JSON(nil, nil)
+}
+
+func lockMid(c *bm.Context) {
+	var (
+		mid       int64
+		status    int64
+		statusStr string
+		err       error
+	)
+	params := c.Request.Form
+	mid, err = strconv.ParseInt(params.Get("mid"), 10, 64)
+	if mid <= 0 || err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if statusStr = params.Get("status"); statusStr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if status, err = strconv.ParseInt(statusStr, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if status != 0 && status != 1 {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	err = srv.SetMidForbid(c, mid, int8(status))
+	c.JSON(nil, err)
+}

app/job/main/click/http/http.go

@@ -0,0 +1,36 @@
+package http
+
+import (
+	"go-common/app/job/main/click/conf"
+	"go-common/app/job/main/click/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var srv *service.Service
+
+func Init(c *conf.Config, s *service.Service) {
+	srv = s
+	e := bm.DefaultServer(c.BM)
+	innerRouter(e)
+	// init internal server
+	if err := e.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	e.Ping(ping)
+	// path
+	g := e.Group("/x/internal/click")
+	{
+		g.GET("", click)
+		g.GET("/lock", lock)
+		g.GET("/lock/mid", lockMid)
+	}
+}

app/job/main/click/model/BUILD

@@ -0,0 +1,31 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "click.go",
+        "forbid.go",
+        "message.go",
+    ],
+    importpath = "go-common/app/job/main/click/model",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/model/click.go

@@ -0,0 +1,64 @@
+package model
+
+//web+h5+outside+ios+android
+const (
+	TypeForWeb                   = "web"
+	TypeForH5                    = "h5"
+	TypeForOutside               = "outside"
+	TypeForIOS                   = "ios"
+	TypeForAndroid               = "android"
+	TypeForAndroidTv             = "android_tv"
+	PlatForWeb                   = int8(0)
+	PlatForH5                    = int8(1)
+	PlatForOuter                 = int8(2)
+	PlatForIos                   = int8(3)
+	PlatForAndroid               = int8(4)
+	PlatForAndroidTV             = int8(5)
+	PlatForAutoPlayIOS           = int8(6)
+	PlafForAutoPlayInlineIOS     = int8(7)
+	PlatForAutoPlayAndroid       = int8(8)
+	PlatForAutoPlayInlineAndroid = int8(9)
+	_maxDBTimes                  = 6
+)
+
+// ClickInfo is
+type ClickInfo struct {
+	Aid            int64
+	Web            int64
+	H5             int64
+	Outer          int64
+	Ios            int64
+	Android        int64
+	AndroidTV      int64
+	Sum            int64
+	DBTimes        int
+	LastChangeTime int64
+}
+
+// NeedRelease is
+func (c *ClickInfo) NeedRelease() bool {
+	if c.DBTimes > _maxDBTimes {
+		return true
+	}
+	return false
+}
+
+// ArcDuration is
+type ArcDuration struct {
+	Duration int64
+	GotTime  int64
+}
+
+// Ready is
+func (c *ClickInfo) Ready(ts int64) {
+	c.Sum = c.Sum + c.GetSum()
+	c.LastChangeTime = ts
+	c.Web, c.H5, c.Outer, c.Ios, c.Android, c.AndroidTV = 0, 0, 0, 0, 0, 0
+	c.DBTimes++
+}
+
+// GetSum is
+func (c *ClickInfo) GetSum() (sum int64) {
+	sum = c.Web + c.H5 + c.Outer + c.Ios + c.Android + c.AndroidTV
+	return
+}

app/job/main/click/model/forbid.go

@@ -0,0 +1,14 @@
+package model
+
+// is
+const (
+	ValueForLocked = int8(1)
+)
+
+// Forbid is
+type Forbid struct {
+	AID    int64
+	Plat   int8
+	Lv     int8
+	Locked int8
+}

app/job/main/click/model/message.go

@@ -0,0 +1,47 @@
+package model
+
+// plat(web:0,h5:1,outer:2,ios:3,android:4),avid,cid,part,mid,lv,ftime,stime,buvid(device),ip,agent(version)
+
+// is
+const (
+	LogTypeForNotUse      = int8(0)
+	LogTypeForTurly       = int8(1)
+	LogTypeForInlineBegin = int8(2)
+)
+
+// ClickMsg is
+type ClickMsg struct {
+	Plat       int8
+	AID        int64
+	MID        int64
+	Lv         int8
+	Buvid      string
+	Did        string
+	CTime      int64
+	STime      int64
+	IP         string
+	KafkaBs    []byte
+	EpID       int64
+	SeasonType int
+	UserAgent  string
+}
+
+// StatMsg is
+type StatMsg struct {
+	AID   int64 `json:"aid"`
+	Click int   `json:"click"`
+}
+
+// StatViewMsg is
+type StatViewMsg struct {
+	Type  string `json:"type"`
+	ID    int64  `json:"id"`
+	Count int    `json:"count"`
+	Ts    int64  `json:"timestamp"`
+}
+
+// BigDataMsg is
+type BigDataMsg struct {
+	Info string `json:"info"`
+	Tp   int8   `json:"type"`
+}

app/job/main/click/service/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/model:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "click.go",
+        "forbid.go",
+        "redis.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/click/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/dao:go_default_library",
+        "//app/job/main/click/model:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/dgryski/go-farm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/service/click.go

@@ -0,0 +1,201 @@
+package service
+
+import (
+	"context"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/job/main/click/model"
+	"go-common/library/log"
+)
+
+func (s *Service) isAllow(ctx context.Context, c *model.ClickMsg) (rtype int8, err error) {
+	var (
+		f        *model.Forbid
+		ok       bool
+		duration int64
+	)
+	rtype = model.LogTypeForNotUse
+	// 自动播放的恶心逻辑 开始
+	if c.Plat == model.PlatForAutoPlayAndroid || c.Plat == model.PlatForAutoPlayInlineAndroid || c.Plat == model.PlatForAutoPlayIOS || c.Plat == model.PlafForAutoPlayInlineIOS ||
+		strings.Contains(c.UserAgent, "(inline_play_begin)") { // plat的逻辑更换为UA中添加(inline_play_begin)
+		log.Warn("no count! hit autoplay plat(%d) aid(%d)", c.Plat, c.AID)
+		rtype = model.LogTypeForInlineBegin // 2
+		if c.Buvid != "" {
+			if err = s.setRealDid(ctx, c.Buvid, c.AID, c.Did); err != nil {
+				log.Error("s.setRealDid(%s, %s) error(%v)", c.Buvid, c.Did, err)
+				return
+			}
+		}
+		return
+	}
+	// 自动播放的恶心逻辑 结束
+	if c.MID > 0 {
+		if _, ok := s.forbidMids[c.MID]; ok {
+			log.Warn("mid(%d) forbidden", c.MID)
+			return
+		}
+	}
+	if f, ok = s.forbids[c.AID][c.Plat]; ok {
+		if f.Lv == -2 && (strings.HasSuffix(c.UserAgent, "(no_accesskey)") || c.MID == 0) {
+			log.Warn("no count! hit no_accesskey! agent(%s) aid(%d) mid(%d) plat(%d)", c.UserAgent, c.AID, c.MID, c.Plat)
+			return
+		} else if f.Lv == -1 && c.MID == 0 {
+			// 游客不计算点击数
+			log.Warn("no count! hit forbid_lv(%d) mid(%d)", f.Lv, c.MID)
+			return
+		} else if f.Lv >= 0 && (c.Lv <= f.Lv || c.MID == 0) {
+			// 游客和低于锁定等级的不计算点击数
+			return
+		}
+	}
+	if c.EpID > 0 {
+		if err = s.checkEpAvRelation(ctx, c.AID, c.EpID, c.SeasonType); err != nil {
+			log.Error("s.getOid(%d, %d, %d) error(%v)", c.AID, c.EpID, c.SeasonType)
+			return
+		}
+	}
+	if !s.canCount(ctx, c.AID, c.EpID, c.IP, c.STime, c.Did) {
+		log.Warn("same ip(%s) and av(%d) replay", c.IP, c.AID)
+		return
+	}
+	duration = s.ArcDuration(ctx, c.AID)
+	if s.isReplay(ctx, c.MID, c.AID, c.Did, duration) {
+		log.Warn("mid(%d) bvid(%s) aid(%d) epid(%d) isPGC(%v) gapTime(%d) is replay", c.MID, c.Did, c.AID, c.EpID, c.EpID > 0, duration)
+		return
+	}
+	rtype = model.LogTypeForTurly // 1
+	return
+}
+
+func (s *Service) checkEpAvRelation(ctx context.Context, aid, epid int64, seasonType int) (err error) {
+	s.etamMutex.RLock()
+	_, ok := s.eTam[epid]
+	s.etamMutex.RUnlock()
+	if ok {
+		return
+	}
+	var isLegal bool
+	if isLegal, err = s.db.IsLegal(ctx, aid, epid, seasonType); err != nil {
+		// 接口请求失败时按ugc维度走
+		err = nil
+	} else if !isLegal {
+		// epid not exist
+		log.Error("aid(%d) epid(%d) type(%d) not exist", aid, epid, seasonType)
+		return
+	}
+	s.etamMutex.Lock()
+	s.eTam[epid] = aid
+	s.etamMutex.Unlock()
+	return
+}
+
+// 播放计数主方法
+func (s *Service) countClick(ctx context.Context, msg *model.ClickMsg, i int64) (err error) {
+	var (
+		ci  *model.ClickInfo
+		ok  bool
+		now = time.Now().Unix()
+	)
+	if msg == nil {
+		log.Info("svr close s.aidMap length is %d", len(s.aidMap[i]))
+		for _, ci = range s.aidMap[i] {
+			// 反正这些视频点击数很多,不差这些,照顾小透明
+			if ci.GetSum() > 100 {
+				continue
+			}
+			s.upClick(ctx, ci)
+		}
+		return
+	}
+	idx := msg.AID % s.c.ChanNum
+	if atomic.LoadInt64(&s.lockedMap[idx]) == _locked {
+		log.Info("locking aidMap[%d] current length(%d)", idx, len(s.aidMap[idx]))
+		for _, ci := range s.aidMap[idx] {
+			if ci.GetSum() > 100 {
+				continue
+			}
+			s.upClick(ctx, ci)
+			delete(s.aidMap[idx], ci.Aid)
+			time.Sleep(1 * time.Millisecond)
+		}
+		atomic.StoreInt64(&s.lockedMap[idx], _unLock)
+		log.Info("unlocked aidMap[%d] current length(%d)", idx, len(s.aidMap[idx]))
+	}
+	if ci, ok = s.aidMap[idx][msg.AID]; !ok {
+		if ci, err = s.db.Click(ctx, msg.AID); err != nil {
+			log.Error("s.db.Click(%d) error(%v)", msg.AID, err)
+			return
+		}
+		if ci == nil {
+			if _, err = s.db.AddClick(ctx, msg.AID, 0, 0, 0, 0, 0, 0); err != nil {
+				log.Error("s.db.AddClick(%d) error(%v)", msg.AID, err)
+				return
+			}
+			ci = &model.ClickInfo{Aid: msg.AID}
+		}
+		ci.Ready(now)
+		s.aidMap[idx][msg.AID] = ci
+	}
+	switch msg.Plat {
+	case 0:
+		ci.Web++
+	case 1:
+		ci.H5++
+	case 2:
+		ci.Outer++
+	case 3:
+		ci.Ios++
+	case 4:
+		ci.Android++
+	case 5:
+		ci.AndroidTV++
+	}
+	if ci.Sum == 0 || now-ci.LastChangeTime > s.c.LastChangeTime {
+		if err = s.upClick(ctx, ci); err != nil {
+			log.Error("s.upClick(%v) error(%v)", ci, err)
+			return
+		}
+		log.Info("truly add click message(%+v)", msg)
+		if now-ci.LastChangeTime > s.c.ReleaseTime || ci.Aid == s.c.BnjMainAid || ci.NeedRelease() {
+			delete(s.aidMap[idx], msg.AID)
+			return
+		}
+		ci.Ready(now)
+	}
+	return
+}
+
+func (s *Service) upClick(c context.Context, ci *model.ClickInfo) (err error) {
+	if _, err = s.db.UpClick(c, ci); err != nil {
+		log.Error("s.db.UpClick(%+v) error(%v)", ci, err)
+		return
+	}
+	s.busChan <- &model.StatMsg{AID: ci.Aid, Click: int(ci.Sum + ci.GetSum())}
+	// 拜年祭需求 单品视频的播放数算进主视频
+	if _, ok := s.bnjListAidMap[ci.Aid]; ok {
+		bnj := &model.ClickInfo{
+			Aid:       s.c.BnjMainAid,
+			Web:       ci.Web,
+			H5:        ci.H5,
+			Outer:     ci.Outer,
+			Ios:       ci.Ios,
+			Android:   ci.Android,
+			AndroidTV: ci.AndroidTV,
+		}
+		if _, err = s.db.UpClick(c, bnj); err != nil {
+			log.Error("s.db.UpClick(%d) error(%v)", s.c.BnjMainAid, err)
+			return
+		}
+		log.Info("bnjaid(%d) Forced to increase click(%v) by relateAid(%d)", s.c.BnjMainAid, bnj, ci.Aid)
+	}
+	// 拜年祭需求 单品视频的播放数算进主视频
+	return
+}
+
+// SetSpecial http set special aid click
+func (s *Service) SetSpecial(c context.Context, aid, num int64, tp string) (err error) {
+	_, err = s.db.UpSpecial(c, aid, tp, num)
+	return
+}

app/job/main/click/service/forbid.go

@@ -0,0 +1,25 @@
+package service
+
+import (
+	"context"
+
+	"go-common/library/log"
+)
+
+// SetLock set click lock by plat and lv
+func (s *Service) SetLock(c context.Context, aid int64, plat, lock, lv int8) (err error) {
+	if _, err = s.db.UpForbid(c, aid, plat, lock, lv); err != nil {
+		log.Error("s.db.UpForbid(%+v) error(%v)", c, err)
+		return
+	}
+	return
+}
+
+// SetMidForbid is
+func (s *Service) SetMidForbid(c context.Context, mid int64, status int8) (err error) {
+	if err = s.db.UpMidForbidStatus(c, mid, status); err != nil {
+		log.Error("s.db.UpMidForbidStatus(%d, %d) error(%v)", mid, status, err)
+		return
+	}
+	return
+}

app/job/main/click/service/redis.go

@@ -0,0 +1,240 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+
+	farm "github.com/dgryski/go-farm"
+)
+
+const (
+	// aid_ip
+	_anonymousePlayedKey = "nm:%d"
+	// aid bvid
+	_anonymouseBvIDKey = "nb:%d"
+	// bvid's last played aid
+	_bvIDLastPlayedKey = "bv:%d"
+	// Mid last played key
+	_midHashKey    = "mid:%d"
+	_buvidToDidKey = "%d:bdid"
+	// 改短成6分钟
+	_hkeyExpire = 600
+)
+
+func (s *Service) midKey(mid int64) (key string) {
+	key = fmt.Sprintf(_midHashKey, mid%s.c.HashNum)
+	return
+}
+
+func (s *Service) bvKey(bvID string) (key string) {
+	num := int64(farm.Hash32([]byte(bvID)))
+	key = fmt.Sprintf(_bvIDLastPlayedKey, num%s.c.HashNum)
+	return
+}
+
+func (s *Service) buvidToDidKey(buvid string, aid int64) (key string) {
+	num := int64(farm.Hash32([]byte(fmt.Sprintf("%s_%d", buvid, aid))))
+	key = fmt.Sprintf(_buvidToDidKey, num%s.c.HashNum)
+	return
+}
+
+func (s *Service) anonymouseKey(aid, epID int64, ip string) (key string) {
+	var str string
+	if epID == 0 {
+		str = strconv.Itoa(int(aid)) + ip
+	} else {
+		str = strconv.Itoa(int(aid)) + ip + strconv.Itoa(int(epID))
+	}
+	num := int64(farm.Hash32([]byte(str)))
+	key = fmt.Sprintf(_anonymousePlayedKey, num%s.c.HashNum)
+	return
+}
+
+func (s *Service) anonymouseBvIDKey(aid int64, bvid string) (key string) {
+	str := strconv.Itoa(int(aid)) + bvid
+	num := int64(farm.Hash32([]byte(str)))
+	key = fmt.Sprintf(_anonymouseBvIDKey, num%s.c.HashNum)
+	return
+}
+
+// canCount 同一个IP一分钟，同一个bvid五分钟
+func (s *Service) canCount(c context.Context, aid, epID int64, ip string, stime int64, bvid string) (can bool) {
+	var (
+		err            error
+		lastPlayTime   int64
+		bvLastPlayTime int64
+		hKey           = s.anonymouseKey(aid, epID, ip)
+		hbvKey         = s.anonymouseBvIDKey(aid, bvid)
+		conn           = s.redis.Get(c)
+		pTime          = stime + s.c.CacheConf.NewAnonymousCacheTime
+		now            = time.Now().Unix()
+		ipCan          bool
+		bvCan          bool
+	)
+	var field = aid
+	if epID > 0 {
+		field = epID
+	}
+	defer conn.Close()
+	if lastPlayTime, err = redis.Int64(conn.Do("HGET", hKey, field)); err != nil {
+		if err == redis.ErrNil {
+			if _, err = conn.Do("HSET", hKey, field, pTime); err != nil {
+				log.Error("conn.Do(HSET, %s, %d, %d) error(%v)", hKey, field, pTime, err)
+				return
+			}
+			ipCan = true
+		} else {
+			log.Error("conn.Do(HGET, %s, %d) error(%v)", hKey, field, err)
+			return
+		}
+	}
+	if bvLastPlayTime, err = redis.Int64(conn.Do("HGET", hbvKey, field)); err != nil {
+		if err == redis.ErrNil {
+			if _, err = conn.Do("HSET", hbvKey, field, pTime); err != nil {
+				log.Error("conn.Do(HSET, %s, %d, %d)", hbvKey, field, pTime)
+				return
+			}
+			bvCan = true
+		} else {
+			log.Error("conn.Do(HGET, %s, %d) error(%v)", hbvKey, field, err)
+			return
+		}
+	}
+	if ipCan && bvCan {
+		can = true
+		return
+	}
+	if now > lastPlayTime && now > bvLastPlayTime {
+		if err = conn.Send("HSET", hKey, field, now+s.c.CacheConf.NewAnonymousCacheTime); err != nil {
+			log.Error("conn.Send(HSET, %s, %s, %d) error(%v)", hKey, field, now+s.c.CacheConf.NewAnonymousCacheTime, err)
+			return
+		}
+		if err = conn.Send("EXPIRE", hKey, _hkeyExpire); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", hKey, _hkeyExpire, err)
+			return
+		}
+		if err = conn.Send("HSET", hbvKey, field, now+s.c.CacheConf.NewAnonymousBvCacheTime); err != nil {
+			log.Error("conn.Send(HSET, %s, %d, %d) error(%v)", hbvKey, field, now+s.c.CacheConf.NewAnonymousBvCacheTime, err)
+			return
+		}
+		if err = conn.Send("EXPIRE", hbvKey, _hkeyExpire); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", hbvKey, _hkeyExpire, err)
+			return
+		}
+		if err = conn.Flush(); err != nil {
+			log.Error("conn.Flush error(%v)")
+			return
+		}
+		for i := 0; i < 4; i++ {
+			if _, err = conn.Receive(); err != nil {
+				log.Error("conn.Receive error(%v)", err)
+				return
+			}
+		}
+		can = true
+	}
+	return
+}
+
+func (s *Service) isReplay(c context.Context, mid, aid int64, bvID string, gapTime int64) (is bool) {
+	var (
+		hKey  string
+		field string
+		conn  = s.redis.Get(c)
+		value int64
+		err   error
+		now   = time.Now().Unix()
+	)
+	defer conn.Close()
+	if mid > 0 {
+		hKey = s.midKey(mid)
+		field = strconv.Itoa(int(mid))
+	} else {
+		hKey = s.bvKey(bvID)
+		field = bvID
+	}
+	// aid << 32 | ptime
+	if value, err = redis.Int64(conn.Do("HGET", hKey, field)); err != nil {
+		if err != redis.ErrNil {
+			log.Error("conn.Do(HGET, %s, %s) error(%s)", hKey, field, err)
+			return
+		}
+		err = nil
+	}
+	if value != 0 {
+		rOid := value >> 32
+		rNow := value & 0xffffffff
+		if rOid == aid && now-rNow < gapTime {
+			is = true
+			return
+		}
+	}
+	value = aid<<32 | now
+	if err = conn.Send("HSET", hKey, field, value); err != nil {
+		log.Error("conn.Do(HSET, %s, %s, %s) error(%v)", hKey, field, value, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", hKey, _hkeyExpire); err != nil {
+		log.Error("conn.Do(EXPIRE, %s, %d)", hKey, _hkeyExpire)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)")
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+func (s *Service) getRealDid(c context.Context, buvid string, aid int64) (did string, err error) {
+	var (
+		conn = s.redis.Get(c)
+		key  = s.buvidToDidKey(buvid, aid)
+	)
+	defer conn.Close()
+	if did, err = redis.String(conn.Do("HGET", key, buvid)); err != nil {
+		if err != redis.ErrNil {
+			log.Error("redis.String(conn.Do(HGET, %s, %s)) error(%v)", key, buvid, err)
+			return
+		}
+		err = nil
+	}
+	return
+}
+
+func (s *Service) setRealDid(c context.Context, buvid string, aid int64, did string) (err error) {
+	var (
+		conn = s.redis.Get(c)
+		key  = s.buvidToDidKey(buvid, aid)
+	)
+	defer conn.Close()
+	if err = conn.Send("HSET", key, buvid, did); err != nil {
+		log.Error("conn.Do(HSET, %s, %s, %s) error(%v)", key, buvid, did, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, _hkeyExpire); err != nil {
+		log.Error("conn.Send(EXPIRE, %s, %d) error(%v)", key, _hkeyExpire, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}

app/job/main/click/service/service.go

@@ -0,0 +1,439 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/job/main/click/conf"
+	"go-common/app/job/main/click/dao"
+	"go-common/app/job/main/click/model"
+	"go-common/app/service/main/archive/api"
+	arcrpc "go-common/app/service/main/archive/api/gorpc"
+	arcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_unLock = 0
+	_locked = 1
+)
+
+// Service struct
+type Service struct {
+	c  *conf.Config
+	db *dao.Dao
+	// archive
+	reportMergeSub   *databus.Databus
+	statViewPub      *databus.Databus
+	chanWg           sync.WaitGroup
+	redis            *redis.Pool
+	cliChan          []chan *model.ClickMsg
+	closed           bool
+	maxAID           int64
+	gotMaxAIDTime    int64
+	lockedMap        []int64
+	currentLockedIdx int64
+	// aid%50[aid[plat[cnt]]]
+	aidMap []map[int64]*model.ClickInfo
+	// send databus chan
+	busChan     chan *model.StatMsg
+	bigDataChan chan *model.BigDataMsg
+	// forbid cache
+	forbids    map[int64]map[int8]*model.Forbid
+	forbidMids map[int64]struct{}
+	// epid to aid map
+	eTam            map[int64]int64
+	etamMutex       sync.RWMutex
+	infoc2          *infoc.Infoc
+	arcRPC          *arcrpc.Service2
+	arcDurWithMutex struct {
+		Durations map[int64]*model.ArcDuration
+		Mutex     sync.RWMutex
+	}
+	allowPlat     map[int8]struct{}
+	bnjListAidMap map[int64]struct{}
+}
+
+// New is archive service implementation.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:              c,
+		arcRPC:         arcrpc.New2(c.ArchiveRPC),
+		redis:          redis.NewPool(c.Redis),
+		db:             dao.New(c),
+		busChan:        make(chan *model.StatMsg, 10240),
+		bigDataChan:    make(chan *model.BigDataMsg, 10240),
+		reportMergeSub: databus.New(c.ReportMergeDatabus),
+		statViewPub:    databus.New(c.StatViewPub),
+		infoc2:         infoc.New(c.Infoc2),
+		allowPlat:      make(map[int8]struct{}),
+	}
+	s.allowPlat[model.PlatForWeb] = struct{}{}
+	s.allowPlat[model.PlatForH5] = struct{}{}
+	s.allowPlat[model.PlatForOuter] = struct{}{}
+	s.allowPlat[model.PlatForIos] = struct{}{}
+	s.allowPlat[model.PlatForAndroid] = struct{}{}
+	s.allowPlat[model.PlatForAndroidTV] = struct{}{}
+	s.allowPlat[model.PlatForAutoPlayIOS] = struct{}{}
+	s.allowPlat[model.PlafForAutoPlayInlineIOS] = struct{}{}
+	s.allowPlat[model.PlatForAutoPlayAndroid] = struct{}{}
+	s.allowPlat[model.PlatForAutoPlayInlineAndroid] = struct{}{}
+	s.arcDurWithMutex.Durations = make(map[int64]*model.ArcDuration)
+	s.loadConf()
+	go s.confproc()
+	go s.releaseAIDMap()
+	for i := int64(0); i < s.c.ChanNum; i++ {
+		s.aidMap = append(s.aidMap, make(map[int64]*model.ClickInfo, 300000))
+		s.cliChan = append(s.cliChan, make(chan *model.ClickMsg, 256))
+		s.lockedMap = append(s.lockedMap, _unLock)
+	}
+	for i := int64(0); i < s.c.ChanNum; i++ {
+		s.chanWg.Add(1)
+		go s.cliChanProc(i)
+	}
+	for i := 0; i < 10; i++ {
+		s.chanWg.Add(1)
+		go s.sendStat()
+	}
+	s.chanWg.Add(1)
+	go s.sendBigDataMsg()
+	s.chanWg.Add(1)
+	go s.reportMergeSubConsumer()
+	return s
+}
+
+func (s *Service) reportMergeSubConsumer() {
+	defer s.chanWg.Done()
+	msgs := s.reportMergeSub.Messages()
+	for {
+		msg, ok := <-msgs
+		if !ok || s.closed {
+			log.Info("s.reportMergeSub is closed")
+			return
+		}
+		msg.Commit()
+		var (
+			sbs [][]byte
+			err error
+		)
+		if err = json.Unmarshal(msg.Value, &sbs); err != nil {
+			log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
+			continue
+		}
+		for _, bs := range sbs {
+			var (
+				click *model.ClickMsg
+				allow bool
+				now   = time.Now().Unix()
+			)
+			log.Info("split merged message(%s)", strings.Replace(string(bs), "\001", "|", -1))
+			if click, err = s.checkMsgIllegal(bs); err != nil {
+				log.Error("s.checkMsgIllegal(%s) error(%v)", strings.Replace(string(bs), "\001", "|", -1), err)
+				continue
+			}
+			if s.maxAID > 0 && now-s.gotMaxAIDTime < 120 {
+				allow = s.maxAID+300 > click.AID
+			}
+			if !allow {
+				log.Error("maxAid(%d) currentAid(%d) not allow!!!!", s.maxAID, click.AID)
+				continue
+			}
+			log.Info("merge consumer(%d) append to chan", click.AID)
+			s.cliChan[click.AID%s.c.ChanNum] <- click
+		}
+	}
+}
+
+func (s *Service) loadConf() {
+	var (
+		forbids     map[int64]map[int8]*model.Forbid
+		bnjListAids = make(map[int64]struct{})
+		forbidMids  map[int64]struct{}
+		etam        map[int64]int64
+		maxAID      int64
+		err         error
+	)
+	for _, aid := range s.c.BnjListAids {
+		bnjListAids[aid] = struct{}{}
+	}
+	s.bnjListAidMap = bnjListAids
+	if forbidMids, err = s.db.ForbidMids(context.Background()); err == nil {
+		s.forbidMids = forbidMids
+		log.Info("forbid mids(%d)", len(forbidMids))
+	}
+	if forbids, err = s.db.Forbids(context.TODO()); err == nil {
+		s.forbids = forbids
+		log.Info("forbid av(%d)", len(forbids))
+	}
+	if maxAID, err = s.db.MaxAID(context.TODO()); err == nil {
+		s.maxAID = maxAID
+		s.gotMaxAIDTime = time.Now().Unix()
+	}
+	if etam, err = s.db.LoadAllBangumi(context.TODO()); err == nil {
+		s.etamMutex.Lock()
+		s.eTam = etam
+		s.etamMutex.Unlock()
+	}
+}
+
+func (s *Service) releaseAIDMap() {
+	for {
+		time.Sleep(5 * time.Minute)
+		now := time.Now()
+		if (now.Hour() > 1 && now.Hour() < 6) || (now.Hour() == 6 && now.Minute() < 30) { // 2:00 to 6:30
+			if s.currentLockedIdx < int64(len(s.aidMap)) {
+				atomic.StoreInt64(&s.lockedMap[s.currentLockedIdx], _locked)
+			}
+			s.currentLockedIdx++
+			continue
+		}
+		s.currentLockedIdx = 0
+	}
+}
+
+func (s *Service) confproc() {
+	for {
+		time.Sleep(1 * time.Minute)
+		s.loadConf()
+	}
+}
+
+func (s *Service) sendBigDataMsg() {
+	defer s.chanWg.Done()
+	for {
+		var (
+			msg   *model.BigDataMsg
+			msgBs []byte
+			ok    bool
+			err   error
+			infos []interface{}
+		)
+		if msg, ok = <-s.bigDataChan; !ok {
+			break
+		}
+		infos = append(infos, strconv.FormatInt(int64(msg.Tp), 10))
+		for _, v := range strings.Split(msg.Info, "\001") {
+			infos = append(infos, v)
+		}
+		log.Info("truly used %+v", infos)
+		if err = s.infoc2.Info(infos...); err != nil {
+			log.Error("s.infoc2.Info(%s) error(%v)", msgBs, err)
+			continue
+		}
+	}
+}
+
+func (s *Service) sendStat() {
+	defer s.chanWg.Done()
+	for {
+		var (
+			msg *model.StatMsg
+			ok  bool
+			c   = context.TODO()
+			err error
+			key string
+		)
+		if msg, ok = <-s.busChan; !ok {
+			break
+		}
+		key = strconv.FormatInt(msg.AID, 10)
+		vmsg := &model.StatViewMsg{Type: "archive", ID: msg.AID, Count: msg.Click, Ts: time.Now().Unix()}
+		if err = s.statViewPub.Send(c, key, vmsg); err != nil {
+			log.Error("s.statViewPub.Send(%d, %+v) error(%v)", msg.AID, vmsg, err)
+		}
+	}
+}
+
+func (s *Service) cliChanProc(i int64) {
+	defer s.chanWg.Done()
+	var (
+		cli     *model.ClickMsg
+		cliChan = s.cliChan[i]
+		ok      bool
+	)
+	for {
+		if cli, ok = <-cliChan; !ok {
+			s.countClick(context.TODO(), nil, i)
+			return
+		}
+		var (
+			rtype int8
+			err   error
+			c     = context.TODO()
+		)
+		if rtype, err = s.isAllow(c, cli); err != nil {
+			log.Error("cliChanProc Err %v", err)
+		}
+		select {
+		case s.bigDataChan <- &model.BigDataMsg{Info: string(cli.KafkaBs), Tp: rtype}:
+		default:
+			log.Error("s.bigDataChan is full")
+		}
+		if rtype == model.LogTypeForTurly {
+			s.countClick(context.TODO(), cli, i)
+		}
+	}
+}
+
+func (s *Service) checkMsgIllegal(msg []byte) (click *model.ClickMsg, err error) {
+	var (
+		aid        int64
+		clickMsg   []string
+		plat       int64
+		did        string
+		buvid      string
+		mid        int64
+		lv         int64
+		ctime      int64
+		stime      int64
+		epid       int64
+		ip         string
+		seasonType int
+		userAgent  string
+	)
+	clickMsg = strings.Split(string(msg), "\001")
+	if len(clickMsg) < 10 {
+		err = errors.New("click msg error")
+		return
+	}
+	if aid, err = strconv.ParseInt(clickMsg[1], 10, 64); err != nil {
+		err = fmt.Errorf("aid(%s) error", clickMsg[1])
+		return
+	}
+	if aid <= 0 {
+		err = fmt.Errorf("wocao aid(%s) error", clickMsg[1])
+		return
+	}
+	if plat, err = strconv.ParseInt(clickMsg[0], 10, 64); err != nil {
+		err = fmt.Errorf("plat(%s) error", clickMsg[0])
+		return
+	}
+
+	if _, ok := s.allowPlat[int8(plat)]; !ok {
+		err = fmt.Errorf("plat(%d) is illegal", plat)
+		return
+	}
+	userAgent = clickMsg[10]
+	did = clickMsg[8]
+	if did == "" {
+		err = fmt.Errorf("bvID(%s) is illegal", clickMsg[8])
+		return
+	}
+	buvid = clickMsg[11]
+	if clickMsg[4] != "" && clickMsg[4] != "0" {
+		if mid, err = strconv.ParseInt(clickMsg[4], 10, 64); err != nil {
+			err = fmt.Errorf("mid(%s) is illegal", clickMsg[4])
+			return
+		}
+	}
+	if clickMsg[5] != "" {
+		if lv, err = strconv.ParseInt(clickMsg[5], 10, 64); err != nil {
+			err = fmt.Errorf("lv(%s) is illegal", clickMsg[5])
+			return
+		}
+	}
+	if ctime, err = strconv.ParseInt(clickMsg[6], 10, 64); err != nil {
+		err = fmt.Errorf("ctime(%s) is illegal", clickMsg[6])
+		return
+	}
+	if stime, err = strconv.ParseInt(clickMsg[7], 10, 64); err != nil {
+		err = fmt.Errorf("stime(%s) is illegal", clickMsg[7])
+		return
+	}
+	if ip = clickMsg[9]; ip == "" {
+		err = errors.New("ip is illegal")
+		return
+	}
+	if clickMsg[17] != "" {
+		if epid, err = strconv.ParseInt(clickMsg[17], 10, 64); err != nil {
+			err = fmt.Errorf("epid(%s) is illegal", clickMsg[17])
+			return
+		}
+		if clickMsg[15] != "null" {
+			if seasonType, err = strconv.Atoi(clickMsg[15]); err != nil {
+				err = fmt.Errorf("seasonType(%s) is illegal", clickMsg[15])
+				return
+			}
+		}
+	}
+	if strings.Contains(userAgent, "(auto_play)") ||
+		strings.Contains(userAgent, "(inline_play_heartbeat)") ||
+		strings.Contains(userAgent, "(inline_play_to_view)") || // to remove auto_play & inline_play_heartbeat
+		strings.Contains(userAgent, "(played_time_enough)") {
+		if did, err = s.getRealDid(context.TODO(), buvid, aid); err != nil || did == "" {
+			err = fmt.Errorf("bvid(%s) dont have did", buvid)
+			return
+		}
+		did = buvid
+		msg = []byte(strings.Replace(string(msg), buvid, did, 1))
+	}
+	click = &model.ClickMsg{
+		Plat:       int8(plat),
+		AID:        aid,
+		MID:        mid,
+		Lv:         int8(lv),
+		CTime:      ctime,
+		STime:      stime,
+		Did:        did,
+		Buvid:      buvid,
+		IP:         ip,
+		KafkaBs:    msg,
+		EpID:       epid,
+		SeasonType: seasonType,
+		UserAgent:  userAgent,
+	}
+	return
+}
+
+// ArcDuration return archive duration, manager local cache
+func (s *Service) ArcDuration(c context.Context, aid int64) (duration int64) {
+	var (
+		ok     bool
+		arcDur *model.ArcDuration
+		now    = time.Now().Unix()
+		err    error
+	)
+	// duration default
+	duration = s.c.CacheConf.PGCReplayTime
+	s.arcDurWithMutex.Mutex.RLock()
+	arcDur, ok = s.arcDurWithMutex.Durations[aid]
+	s.arcDurWithMutex.Mutex.RUnlock()
+	if ok && now-arcDur.GotTime > s.c.CacheConf.ArcUpCacheTime {
+		duration = arcDur.Duration
+		return
+	}
+	var arc *api.Arc
+	if arc, err = s.arcRPC.Archive3(c, &arcmdl.ArgAid2{Aid: aid}); err != nil {
+		// just log
+		log.Error("s.arcRPC.Archive3(%d) error(%v)", aid, err)
+	} else {
+		duration = arc.Duration
+	}
+	s.arcDurWithMutex.Mutex.Lock()
+	s.arcDurWithMutex.Durations[aid] = &model.ArcDuration{Duration: duration, GotTime: now}
+	s.arcDurWithMutex.Mutex.Unlock()
+	return
+}
+
+// Close kafaka consumer close.
+func (s *Service) Close() (err error) {
+	s.closed = true
+	time.Sleep(time.Second)
+	for i := 0; i < len(s.cliChan); i++ {
+		close(s.cliChan[i])
+	}
+	close(s.bigDataChan)
+	s.chanWg.Wait()
+	s.statViewPub.Close()
+	return
+}

app/job/main/click/service/service_test.go

@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/click/conf"
+	"go-common/app/job/main/click/model"
+	"go-common/library/sync/errgroup"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/click-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+func Test_Archive(t *testing.T) {
+	Convey("isReplay test", t, func() {
+		b := s.isReplay(context.TODO(), 1, 1, "", 1)
+		So(b, ShouldBeFalse)
+	})
+	return
+}
+
+func Test_SetSpecial(t *testing.T) {
+	Convey("SetSpecial", t, func() {
+		s.SetSpecial(context.TODO(), 1, 10, "")
+	})
+}
+
+func Test_ArcDuration(t *testing.T) {
+	Convey("ArcDuration", t, func() {
+		eg, _ := errgroup.WithContext(context.TODO())
+		var aids = []int64{10099744, 10099730, 10099729, 10099728, 10099726, 10099670, 10099669, 10099668, 10099667, 10099660, 10099653, 10099652, 10099651}
+		for _, aid := range aids {
+			id := aid
+			eg.Go(func() (err error) {
+				Println(s.ArcDuration(context.TODO(), id))
+				return
+			})
+		}
+		eg.Wait()
+		Println(s.arcDurWithMutex.Durations)
+		for aid, val := range s.arcDurWithMutex.Durations {
+			Printf("aid(%d) dur(%d) gotTime(%d)\n", aid, val.Duration, val.GotTime)
+		}
+	})
+}
+
+func Test_SendStat(t *testing.T) {
+	Convey("SendStat", t, func() {
+		vmsg := &model.StatViewMsg{Type: "archive", ID: 123456, Count: 123456, Ts: time.Now().Unix()}
+		err := s.statViewPub.Send(context.TODO(), "test", vmsg)
+		So(err, ShouldBeNil)
+	})
+}