Create & Init Project...

app/job/main/click/service/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/model:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "click.go",
+        "forbid.go",
+        "redis.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/click/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/click/conf:go_default_library",
+        "//app/job/main/click/dao:go_default_library",
+        "//app/job/main/click/model:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/dgryski/go-farm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/click/service/click.go

@@ -0,0 +1,201 @@
+package service
+
+import (
+	"context"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/job/main/click/model"
+	"go-common/library/log"
+)
+
+func (s *Service) isAllow(ctx context.Context, c *model.ClickMsg) (rtype int8, err error) {
+	var (
+		f        *model.Forbid
+		ok       bool
+		duration int64
+	)
+	rtype = model.LogTypeForNotUse
+	// 自动播放的恶心逻辑 开始
+	if c.Plat == model.PlatForAutoPlayAndroid || c.Plat == model.PlatForAutoPlayInlineAndroid || c.Plat == model.PlatForAutoPlayIOS || c.Plat == model.PlafForAutoPlayInlineIOS ||
+		strings.Contains(c.UserAgent, "(inline_play_begin)") { // plat的逻辑更换为UA中添加(inline_play_begin)
+		log.Warn("no count! hit autoplay plat(%d) aid(%d)", c.Plat, c.AID)
+		rtype = model.LogTypeForInlineBegin // 2
+		if c.Buvid != "" {
+			if err = s.setRealDid(ctx, c.Buvid, c.AID, c.Did); err != nil {
+				log.Error("s.setRealDid(%s, %s) error(%v)", c.Buvid, c.Did, err)
+				return
+			}
+		}
+		return
+	}
+	// 自动播放的恶心逻辑 结束
+	if c.MID > 0 {
+		if _, ok := s.forbidMids[c.MID]; ok {
+			log.Warn("mid(%d) forbidden", c.MID)
+			return
+		}
+	}
+	if f, ok = s.forbids[c.AID][c.Plat]; ok {
+		if f.Lv == -2 && (strings.HasSuffix(c.UserAgent, "(no_accesskey)") || c.MID == 0) {
+			log.Warn("no count! hit no_accesskey! agent(%s) aid(%d) mid(%d) plat(%d)", c.UserAgent, c.AID, c.MID, c.Plat)
+			return
+		} else if f.Lv == -1 && c.MID == 0 {
+			// 游客不计算点击数
+			log.Warn("no count! hit forbid_lv(%d) mid(%d)", f.Lv, c.MID)
+			return
+		} else if f.Lv >= 0 && (c.Lv <= f.Lv || c.MID == 0) {
+			// 游客和低于锁定等级的不计算点击数
+			return
+		}
+	}
+	if c.EpID > 0 {
+		if err = s.checkEpAvRelation(ctx, c.AID, c.EpID, c.SeasonType); err != nil {
+			log.Error("s.getOid(%d, %d, %d) error(%v)", c.AID, c.EpID, c.SeasonType)
+			return
+		}
+	}
+	if !s.canCount(ctx, c.AID, c.EpID, c.IP, c.STime, c.Did) {
+		log.Warn("same ip(%s) and av(%d) replay", c.IP, c.AID)
+		return
+	}
+	duration = s.ArcDuration(ctx, c.AID)
+	if s.isReplay(ctx, c.MID, c.AID, c.Did, duration) {
+		log.Warn("mid(%d) bvid(%s) aid(%d) epid(%d) isPGC(%v) gapTime(%d) is replay", c.MID, c.Did, c.AID, c.EpID, c.EpID > 0, duration)
+		return
+	}
+	rtype = model.LogTypeForTurly // 1
+	return
+}
+
+func (s *Service) checkEpAvRelation(ctx context.Context, aid, epid int64, seasonType int) (err error) {
+	s.etamMutex.RLock()
+	_, ok := s.eTam[epid]
+	s.etamMutex.RUnlock()
+	if ok {
+		return
+	}
+	var isLegal bool
+	if isLegal, err = s.db.IsLegal(ctx, aid, epid, seasonType); err != nil {
+		// 接口请求失败时按ugc维度走
+		err = nil
+	} else if !isLegal {
+		// epid not exist
+		log.Error("aid(%d) epid(%d) type(%d) not exist", aid, epid, seasonType)
+		return
+	}
+	s.etamMutex.Lock()
+	s.eTam[epid] = aid
+	s.etamMutex.Unlock()
+	return
+}
+
+// 播放计数主方法
+func (s *Service) countClick(ctx context.Context, msg *model.ClickMsg, i int64) (err error) {
+	var (
+		ci  *model.ClickInfo
+		ok  bool
+		now = time.Now().Unix()
+	)
+	if msg == nil {
+		log.Info("svr close s.aidMap length is %d", len(s.aidMap[i]))
+		for _, ci = range s.aidMap[i] {
+			// 反正这些视频点击数很多,不差这些,照顾小透明
+			if ci.GetSum() > 100 {
+				continue
+			}
+			s.upClick(ctx, ci)
+		}
+		return
+	}
+	idx := msg.AID % s.c.ChanNum
+	if atomic.LoadInt64(&s.lockedMap[idx]) == _locked {
+		log.Info("locking aidMap[%d] current length(%d)", idx, len(s.aidMap[idx]))
+		for _, ci := range s.aidMap[idx] {
+			if ci.GetSum() > 100 {
+				continue
+			}
+			s.upClick(ctx, ci)
+			delete(s.aidMap[idx], ci.Aid)
+			time.Sleep(1 * time.Millisecond)
+		}
+		atomic.StoreInt64(&s.lockedMap[idx], _unLock)
+		log.Info("unlocked aidMap[%d] current length(%d)", idx, len(s.aidMap[idx]))
+	}
+	if ci, ok = s.aidMap[idx][msg.AID]; !ok {
+		if ci, err = s.db.Click(ctx, msg.AID); err != nil {
+			log.Error("s.db.Click(%d) error(%v)", msg.AID, err)
+			return
+		}
+		if ci == nil {
+			if _, err = s.db.AddClick(ctx, msg.AID, 0, 0, 0, 0, 0, 0); err != nil {
+				log.Error("s.db.AddClick(%d) error(%v)", msg.AID, err)
+				return
+			}
+			ci = &model.ClickInfo{Aid: msg.AID}
+		}
+		ci.Ready(now)
+		s.aidMap[idx][msg.AID] = ci
+	}
+	switch msg.Plat {
+	case 0:
+		ci.Web++
+	case 1:
+		ci.H5++
+	case 2:
+		ci.Outer++
+	case 3:
+		ci.Ios++
+	case 4:
+		ci.Android++
+	case 5:
+		ci.AndroidTV++
+	}
+	if ci.Sum == 0 || now-ci.LastChangeTime > s.c.LastChangeTime {
+		if err = s.upClick(ctx, ci); err != nil {
+			log.Error("s.upClick(%v) error(%v)", ci, err)
+			return
+		}
+		log.Info("truly add click message(%+v)", msg)
+		if now-ci.LastChangeTime > s.c.ReleaseTime || ci.Aid == s.c.BnjMainAid || ci.NeedRelease() {
+			delete(s.aidMap[idx], msg.AID)
+			return
+		}
+		ci.Ready(now)
+	}
+	return
+}
+
+func (s *Service) upClick(c context.Context, ci *model.ClickInfo) (err error) {
+	if _, err = s.db.UpClick(c, ci); err != nil {
+		log.Error("s.db.UpClick(%+v) error(%v)", ci, err)
+		return
+	}
+	s.busChan <- &model.StatMsg{AID: ci.Aid, Click: int(ci.Sum + ci.GetSum())}
+	// 拜年祭需求 单品视频的播放数算进主视频
+	if _, ok := s.bnjListAidMap[ci.Aid]; ok {
+		bnj := &model.ClickInfo{
+			Aid:       s.c.BnjMainAid,
+			Web:       ci.Web,
+			H5:        ci.H5,
+			Outer:     ci.Outer,
+			Ios:       ci.Ios,
+			Android:   ci.Android,
+			AndroidTV: ci.AndroidTV,
+		}
+		if _, err = s.db.UpClick(c, bnj); err != nil {
+			log.Error("s.db.UpClick(%d) error(%v)", s.c.BnjMainAid, err)
+			return
+		}
+		log.Info("bnjaid(%d) Forced to increase click(%v) by relateAid(%d)", s.c.BnjMainAid, bnj, ci.Aid)
+	}
+	// 拜年祭需求 单品视频的播放数算进主视频
+	return
+}
+
+// SetSpecial http set special aid click
+func (s *Service) SetSpecial(c context.Context, aid, num int64, tp string) (err error) {
+	_, err = s.db.UpSpecial(c, aid, tp, num)
+	return
+}

app/job/main/click/service/forbid.go

@@ -0,0 +1,25 @@
+package service
+
+import (
+	"context"
+
+	"go-common/library/log"
+)
+
+// SetLock set click lock by plat and lv
+func (s *Service) SetLock(c context.Context, aid int64, plat, lock, lv int8) (err error) {
+	if _, err = s.db.UpForbid(c, aid, plat, lock, lv); err != nil {
+		log.Error("s.db.UpForbid(%+v) error(%v)", c, err)
+		return
+	}
+	return
+}
+
+// SetMidForbid is
+func (s *Service) SetMidForbid(c context.Context, mid int64, status int8) (err error) {
+	if err = s.db.UpMidForbidStatus(c, mid, status); err != nil {
+		log.Error("s.db.UpMidForbidStatus(%d, %d) error(%v)", mid, status, err)
+		return
+	}
+	return
+}

app/job/main/click/service/redis.go

@@ -0,0 +1,240 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+
+	farm "github.com/dgryski/go-farm"
+)
+
+const (
+	// aid_ip
+	_anonymousePlayedKey = "nm:%d"
+	// aid bvid
+	_anonymouseBvIDKey = "nb:%d"
+	// bvid's last played aid
+	_bvIDLastPlayedKey = "bv:%d"
+	// Mid last played key
+	_midHashKey    = "mid:%d"
+	_buvidToDidKey = "%d:bdid"
+	// 改短成6分钟
+	_hkeyExpire = 600
+)
+
+func (s *Service) midKey(mid int64) (key string) {
+	key = fmt.Sprintf(_midHashKey, mid%s.c.HashNum)
+	return
+}
+
+func (s *Service) bvKey(bvID string) (key string) {
+	num := int64(farm.Hash32([]byte(bvID)))
+	key = fmt.Sprintf(_bvIDLastPlayedKey, num%s.c.HashNum)
+	return
+}
+
+func (s *Service) buvidToDidKey(buvid string, aid int64) (key string) {
+	num := int64(farm.Hash32([]byte(fmt.Sprintf("%s_%d", buvid, aid))))
+	key = fmt.Sprintf(_buvidToDidKey, num%s.c.HashNum)
+	return
+}
+
+func (s *Service) anonymouseKey(aid, epID int64, ip string) (key string) {
+	var str string
+	if epID == 0 {
+		str = strconv.Itoa(int(aid)) + ip
+	} else {
+		str = strconv.Itoa(int(aid)) + ip + strconv.Itoa(int(epID))
+	}
+	num := int64(farm.Hash32([]byte(str)))
+	key = fmt.Sprintf(_anonymousePlayedKey, num%s.c.HashNum)
+	return
+}
+
+func (s *Service) anonymouseBvIDKey(aid int64, bvid string) (key string) {
+	str := strconv.Itoa(int(aid)) + bvid
+	num := int64(farm.Hash32([]byte(str)))
+	key = fmt.Sprintf(_anonymouseBvIDKey, num%s.c.HashNum)
+	return
+}
+
+// canCount 同一个IP一分钟，同一个bvid五分钟
+func (s *Service) canCount(c context.Context, aid, epID int64, ip string, stime int64, bvid string) (can bool) {
+	var (
+		err            error
+		lastPlayTime   int64
+		bvLastPlayTime int64
+		hKey           = s.anonymouseKey(aid, epID, ip)
+		hbvKey         = s.anonymouseBvIDKey(aid, bvid)
+		conn           = s.redis.Get(c)
+		pTime          = stime + s.c.CacheConf.NewAnonymousCacheTime
+		now            = time.Now().Unix()
+		ipCan          bool
+		bvCan          bool
+	)
+	var field = aid
+	if epID > 0 {
+		field = epID
+	}
+	defer conn.Close()
+	if lastPlayTime, err = redis.Int64(conn.Do("HGET", hKey, field)); err != nil {
+		if err == redis.ErrNil {
+			if _, err = conn.Do("HSET", hKey, field, pTime); err != nil {
+				log.Error("conn.Do(HSET, %s, %d, %d) error(%v)", hKey, field, pTime, err)
+				return
+			}
+			ipCan = true
+		} else {
+			log.Error("conn.Do(HGET, %s, %d) error(%v)", hKey, field, err)
+			return
+		}
+	}
+	if bvLastPlayTime, err = redis.Int64(conn.Do("HGET", hbvKey, field)); err != nil {
+		if err == redis.ErrNil {
+			if _, err = conn.Do("HSET", hbvKey, field, pTime); err != nil {
+				log.Error("conn.Do(HSET, %s, %d, %d)", hbvKey, field, pTime)
+				return
+			}
+			bvCan = true
+		} else {
+			log.Error("conn.Do(HGET, %s, %d) error(%v)", hbvKey, field, err)
+			return
+		}
+	}
+	if ipCan && bvCan {
+		can = true
+		return
+	}
+	if now > lastPlayTime && now > bvLastPlayTime {
+		if err = conn.Send("HSET", hKey, field, now+s.c.CacheConf.NewAnonymousCacheTime); err != nil {
+			log.Error("conn.Send(HSET, %s, %s, %d) error(%v)", hKey, field, now+s.c.CacheConf.NewAnonymousCacheTime, err)
+			return
+		}
+		if err = conn.Send("EXPIRE", hKey, _hkeyExpire); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", hKey, _hkeyExpire, err)
+			return
+		}
+		if err = conn.Send("HSET", hbvKey, field, now+s.c.CacheConf.NewAnonymousBvCacheTime); err != nil {
+			log.Error("conn.Send(HSET, %s, %d, %d) error(%v)", hbvKey, field, now+s.c.CacheConf.NewAnonymousBvCacheTime, err)
+			return
+		}
+		if err = conn.Send("EXPIRE", hbvKey, _hkeyExpire); err != nil {
+			log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", hbvKey, _hkeyExpire, err)
+			return
+		}
+		if err = conn.Flush(); err != nil {
+			log.Error("conn.Flush error(%v)")
+			return
+		}
+		for i := 0; i < 4; i++ {
+			if _, err = conn.Receive(); err != nil {
+				log.Error("conn.Receive error(%v)", err)
+				return
+			}
+		}
+		can = true
+	}
+	return
+}
+
+func (s *Service) isReplay(c context.Context, mid, aid int64, bvID string, gapTime int64) (is bool) {
+	var (
+		hKey  string
+		field string
+		conn  = s.redis.Get(c)
+		value int64
+		err   error
+		now   = time.Now().Unix()
+	)
+	defer conn.Close()
+	if mid > 0 {
+		hKey = s.midKey(mid)
+		field = strconv.Itoa(int(mid))
+	} else {
+		hKey = s.bvKey(bvID)
+		field = bvID
+	}
+	// aid << 32 | ptime
+	if value, err = redis.Int64(conn.Do("HGET", hKey, field)); err != nil {
+		if err != redis.ErrNil {
+			log.Error("conn.Do(HGET, %s, %s) error(%s)", hKey, field, err)
+			return
+		}
+		err = nil
+	}
+	if value != 0 {
+		rOid := value >> 32
+		rNow := value & 0xffffffff
+		if rOid == aid && now-rNow < gapTime {
+			is = true
+			return
+		}
+	}
+	value = aid<<32 | now
+	if err = conn.Send("HSET", hKey, field, value); err != nil {
+		log.Error("conn.Do(HSET, %s, %s, %s) error(%v)", hKey, field, value, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", hKey, _hkeyExpire); err != nil {
+		log.Error("conn.Do(EXPIRE, %s, %d)", hKey, _hkeyExpire)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)")
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+func (s *Service) getRealDid(c context.Context, buvid string, aid int64) (did string, err error) {
+	var (
+		conn = s.redis.Get(c)
+		key  = s.buvidToDidKey(buvid, aid)
+	)
+	defer conn.Close()
+	if did, err = redis.String(conn.Do("HGET", key, buvid)); err != nil {
+		if err != redis.ErrNil {
+			log.Error("redis.String(conn.Do(HGET, %s, %s)) error(%v)", key, buvid, err)
+			return
+		}
+		err = nil
+	}
+	return
+}
+
+func (s *Service) setRealDid(c context.Context, buvid string, aid int64, did string) (err error) {
+	var (
+		conn = s.redis.Get(c)
+		key  = s.buvidToDidKey(buvid, aid)
+	)
+	defer conn.Close()
+	if err = conn.Send("HSET", key, buvid, did); err != nil {
+		log.Error("conn.Do(HSET, %s, %s, %s) error(%v)", key, buvid, did, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, _hkeyExpire); err != nil {
+		log.Error("conn.Send(EXPIRE, %s, %d) error(%v)", key, _hkeyExpire, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}

app/job/main/click/service/service.go

@@ -0,0 +1,439 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/job/main/click/conf"
+	"go-common/app/job/main/click/dao"
+	"go-common/app/job/main/click/model"
+	"go-common/app/service/main/archive/api"
+	arcrpc "go-common/app/service/main/archive/api/gorpc"
+	arcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_unLock = 0
+	_locked = 1
+)
+
+// Service struct
+type Service struct {
+	c  *conf.Config
+	db *dao.Dao
+	// archive
+	reportMergeSub   *databus.Databus
+	statViewPub      *databus.Databus
+	chanWg           sync.WaitGroup
+	redis            *redis.Pool
+	cliChan          []chan *model.ClickMsg
+	closed           bool
+	maxAID           int64
+	gotMaxAIDTime    int64
+	lockedMap        []int64
+	currentLockedIdx int64
+	// aid%50[aid[plat[cnt]]]
+	aidMap []map[int64]*model.ClickInfo
+	// send databus chan
+	busChan     chan *model.StatMsg
+	bigDataChan chan *model.BigDataMsg
+	// forbid cache
+	forbids    map[int64]map[int8]*model.Forbid
+	forbidMids map[int64]struct{}
+	// epid to aid map
+	eTam            map[int64]int64
+	etamMutex       sync.RWMutex
+	infoc2          *infoc.Infoc
+	arcRPC          *arcrpc.Service2
+	arcDurWithMutex struct {
+		Durations map[int64]*model.ArcDuration
+		Mutex     sync.RWMutex
+	}
+	allowPlat     map[int8]struct{}
+	bnjListAidMap map[int64]struct{}
+}
+
+// New is archive service implementation.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:              c,
+		arcRPC:         arcrpc.New2(c.ArchiveRPC),
+		redis:          redis.NewPool(c.Redis),
+		db:             dao.New(c),
+		busChan:        make(chan *model.StatMsg, 10240),
+		bigDataChan:    make(chan *model.BigDataMsg, 10240),
+		reportMergeSub: databus.New(c.ReportMergeDatabus),
+		statViewPub:    databus.New(c.StatViewPub),
+		infoc2:         infoc.New(c.Infoc2),
+		allowPlat:      make(map[int8]struct{}),
+	}
+	s.allowPlat[model.PlatForWeb] = struct{}{}
+	s.allowPlat[model.PlatForH5] = struct{}{}
+	s.allowPlat[model.PlatForOuter] = struct{}{}
+	s.allowPlat[model.PlatForIos] = struct{}{}
+	s.allowPlat[model.PlatForAndroid] = struct{}{}
+	s.allowPlat[model.PlatForAndroidTV] = struct{}{}
+	s.allowPlat[model.PlatForAutoPlayIOS] = struct{}{}
+	s.allowPlat[model.PlafForAutoPlayInlineIOS] = struct{}{}
+	s.allowPlat[model.PlatForAutoPlayAndroid] = struct{}{}
+	s.allowPlat[model.PlatForAutoPlayInlineAndroid] = struct{}{}
+	s.arcDurWithMutex.Durations = make(map[int64]*model.ArcDuration)
+	s.loadConf()
+	go s.confproc()
+	go s.releaseAIDMap()
+	for i := int64(0); i < s.c.ChanNum; i++ {
+		s.aidMap = append(s.aidMap, make(map[int64]*model.ClickInfo, 300000))
+		s.cliChan = append(s.cliChan, make(chan *model.ClickMsg, 256))
+		s.lockedMap = append(s.lockedMap, _unLock)
+	}
+	for i := int64(0); i < s.c.ChanNum; i++ {
+		s.chanWg.Add(1)
+		go s.cliChanProc(i)
+	}
+	for i := 0; i < 10; i++ {
+		s.chanWg.Add(1)
+		go s.sendStat()
+	}
+	s.chanWg.Add(1)
+	go s.sendBigDataMsg()
+	s.chanWg.Add(1)
+	go s.reportMergeSubConsumer()
+	return s
+}
+
+func (s *Service) reportMergeSubConsumer() {
+	defer s.chanWg.Done()
+	msgs := s.reportMergeSub.Messages()
+	for {
+		msg, ok := <-msgs
+		if !ok || s.closed {
+			log.Info("s.reportMergeSub is closed")
+			return
+		}
+		msg.Commit()
+		var (
+			sbs [][]byte
+			err error
+		)
+		if err = json.Unmarshal(msg.Value, &sbs); err != nil {
+			log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
+			continue
+		}
+		for _, bs := range sbs {
+			var (
+				click *model.ClickMsg
+				allow bool
+				now   = time.Now().Unix()
+			)
+			log.Info("split merged message(%s)", strings.Replace(string(bs), "\001", "|", -1))
+			if click, err = s.checkMsgIllegal(bs); err != nil {
+				log.Error("s.checkMsgIllegal(%s) error(%v)", strings.Replace(string(bs), "\001", "|", -1), err)
+				continue
+			}
+			if s.maxAID > 0 && now-s.gotMaxAIDTime < 120 {
+				allow = s.maxAID+300 > click.AID
+			}
+			if !allow {
+				log.Error("maxAid(%d) currentAid(%d) not allow!!!!", s.maxAID, click.AID)
+				continue
+			}
+			log.Info("merge consumer(%d) append to chan", click.AID)
+			s.cliChan[click.AID%s.c.ChanNum] <- click
+		}
+	}
+}
+
+func (s *Service) loadConf() {
+	var (
+		forbids     map[int64]map[int8]*model.Forbid
+		bnjListAids = make(map[int64]struct{})
+		forbidMids  map[int64]struct{}
+		etam        map[int64]int64
+		maxAID      int64
+		err         error
+	)
+	for _, aid := range s.c.BnjListAids {
+		bnjListAids[aid] = struct{}{}
+	}
+	s.bnjListAidMap = bnjListAids
+	if forbidMids, err = s.db.ForbidMids(context.Background()); err == nil {
+		s.forbidMids = forbidMids
+		log.Info("forbid mids(%d)", len(forbidMids))
+	}
+	if forbids, err = s.db.Forbids(context.TODO()); err == nil {
+		s.forbids = forbids
+		log.Info("forbid av(%d)", len(forbids))
+	}
+	if maxAID, err = s.db.MaxAID(context.TODO()); err == nil {
+		s.maxAID = maxAID
+		s.gotMaxAIDTime = time.Now().Unix()
+	}
+	if etam, err = s.db.LoadAllBangumi(context.TODO()); err == nil {
+		s.etamMutex.Lock()
+		s.eTam = etam
+		s.etamMutex.Unlock()
+	}
+}
+
+func (s *Service) releaseAIDMap() {
+	for {
+		time.Sleep(5 * time.Minute)
+		now := time.Now()
+		if (now.Hour() > 1 && now.Hour() < 6) || (now.Hour() == 6 && now.Minute() < 30) { // 2:00 to 6:30
+			if s.currentLockedIdx < int64(len(s.aidMap)) {
+				atomic.StoreInt64(&s.lockedMap[s.currentLockedIdx], _locked)
+			}
+			s.currentLockedIdx++
+			continue
+		}
+		s.currentLockedIdx = 0
+	}
+}
+
+func (s *Service) confproc() {
+	for {
+		time.Sleep(1 * time.Minute)
+		s.loadConf()
+	}
+}
+
+func (s *Service) sendBigDataMsg() {
+	defer s.chanWg.Done()
+	for {
+		var (
+			msg   *model.BigDataMsg
+			msgBs []byte
+			ok    bool
+			err   error
+			infos []interface{}
+		)
+		if msg, ok = <-s.bigDataChan; !ok {
+			break
+		}
+		infos = append(infos, strconv.FormatInt(int64(msg.Tp), 10))
+		for _, v := range strings.Split(msg.Info, "\001") {
+			infos = append(infos, v)
+		}
+		log.Info("truly used %+v", infos)
+		if err = s.infoc2.Info(infos...); err != nil {
+			log.Error("s.infoc2.Info(%s) error(%v)", msgBs, err)
+			continue
+		}
+	}
+}
+
+func (s *Service) sendStat() {
+	defer s.chanWg.Done()
+	for {
+		var (
+			msg *model.StatMsg
+			ok  bool
+			c   = context.TODO()
+			err error
+			key string
+		)
+		if msg, ok = <-s.busChan; !ok {
+			break
+		}
+		key = strconv.FormatInt(msg.AID, 10)
+		vmsg := &model.StatViewMsg{Type: "archive", ID: msg.AID, Count: msg.Click, Ts: time.Now().Unix()}
+		if err = s.statViewPub.Send(c, key, vmsg); err != nil {
+			log.Error("s.statViewPub.Send(%d, %+v) error(%v)", msg.AID, vmsg, err)
+		}
+	}
+}
+
+func (s *Service) cliChanProc(i int64) {
+	defer s.chanWg.Done()
+	var (
+		cli     *model.ClickMsg
+		cliChan = s.cliChan[i]
+		ok      bool
+	)
+	for {
+		if cli, ok = <-cliChan; !ok {
+			s.countClick(context.TODO(), nil, i)
+			return
+		}
+		var (
+			rtype int8
+			err   error
+			c     = context.TODO()
+		)
+		if rtype, err = s.isAllow(c, cli); err != nil {
+			log.Error("cliChanProc Err %v", err)
+		}
+		select {
+		case s.bigDataChan <- &model.BigDataMsg{Info: string(cli.KafkaBs), Tp: rtype}:
+		default:
+			log.Error("s.bigDataChan is full")
+		}
+		if rtype == model.LogTypeForTurly {
+			s.countClick(context.TODO(), cli, i)
+		}
+	}
+}
+
+func (s *Service) checkMsgIllegal(msg []byte) (click *model.ClickMsg, err error) {
+	var (
+		aid        int64
+		clickMsg   []string
+		plat       int64
+		did        string
+		buvid      string
+		mid        int64
+		lv         int64
+		ctime      int64
+		stime      int64
+		epid       int64
+		ip         string
+		seasonType int
+		userAgent  string
+	)
+	clickMsg = strings.Split(string(msg), "\001")
+	if len(clickMsg) < 10 {
+		err = errors.New("click msg error")
+		return
+	}
+	if aid, err = strconv.ParseInt(clickMsg[1], 10, 64); err != nil {
+		err = fmt.Errorf("aid(%s) error", clickMsg[1])
+		return
+	}
+	if aid <= 0 {
+		err = fmt.Errorf("wocao aid(%s) error", clickMsg[1])
+		return
+	}
+	if plat, err = strconv.ParseInt(clickMsg[0], 10, 64); err != nil {
+		err = fmt.Errorf("plat(%s) error", clickMsg[0])
+		return
+	}
+
+	if _, ok := s.allowPlat[int8(plat)]; !ok {
+		err = fmt.Errorf("plat(%d) is illegal", plat)
+		return
+	}
+	userAgent = clickMsg[10]
+	did = clickMsg[8]
+	if did == "" {
+		err = fmt.Errorf("bvID(%s) is illegal", clickMsg[8])
+		return
+	}
+	buvid = clickMsg[11]
+	if clickMsg[4] != "" && clickMsg[4] != "0" {
+		if mid, err = strconv.ParseInt(clickMsg[4], 10, 64); err != nil {
+			err = fmt.Errorf("mid(%s) is illegal", clickMsg[4])
+			return
+		}
+	}
+	if clickMsg[5] != "" {
+		if lv, err = strconv.ParseInt(clickMsg[5], 10, 64); err != nil {
+			err = fmt.Errorf("lv(%s) is illegal", clickMsg[5])
+			return
+		}
+	}
+	if ctime, err = strconv.ParseInt(clickMsg[6], 10, 64); err != nil {
+		err = fmt.Errorf("ctime(%s) is illegal", clickMsg[6])
+		return
+	}
+	if stime, err = strconv.ParseInt(clickMsg[7], 10, 64); err != nil {
+		err = fmt.Errorf("stime(%s) is illegal", clickMsg[7])
+		return
+	}
+	if ip = clickMsg[9]; ip == "" {
+		err = errors.New("ip is illegal")
+		return
+	}
+	if clickMsg[17] != "" {
+		if epid, err = strconv.ParseInt(clickMsg[17], 10, 64); err != nil {
+			err = fmt.Errorf("epid(%s) is illegal", clickMsg[17])
+			return
+		}
+		if clickMsg[15] != "null" {
+			if seasonType, err = strconv.Atoi(clickMsg[15]); err != nil {
+				err = fmt.Errorf("seasonType(%s) is illegal", clickMsg[15])
+				return
+			}
+		}
+	}
+	if strings.Contains(userAgent, "(auto_play)") ||
+		strings.Contains(userAgent, "(inline_play_heartbeat)") ||
+		strings.Contains(userAgent, "(inline_play_to_view)") || // to remove auto_play & inline_play_heartbeat
+		strings.Contains(userAgent, "(played_time_enough)") {
+		if did, err = s.getRealDid(context.TODO(), buvid, aid); err != nil || did == "" {
+			err = fmt.Errorf("bvid(%s) dont have did", buvid)
+			return
+		}
+		did = buvid
+		msg = []byte(strings.Replace(string(msg), buvid, did, 1))
+	}
+	click = &model.ClickMsg{
+		Plat:       int8(plat),
+		AID:        aid,
+		MID:        mid,
+		Lv:         int8(lv),
+		CTime:      ctime,
+		STime:      stime,
+		Did:        did,
+		Buvid:      buvid,
+		IP:         ip,
+		KafkaBs:    msg,
+		EpID:       epid,
+		SeasonType: seasonType,
+		UserAgent:  userAgent,
+	}
+	return
+}
+
+// ArcDuration return archive duration, manager local cache
+func (s *Service) ArcDuration(c context.Context, aid int64) (duration int64) {
+	var (
+		ok     bool
+		arcDur *model.ArcDuration
+		now    = time.Now().Unix()
+		err    error
+	)
+	// duration default
+	duration = s.c.CacheConf.PGCReplayTime
+	s.arcDurWithMutex.Mutex.RLock()
+	arcDur, ok = s.arcDurWithMutex.Durations[aid]
+	s.arcDurWithMutex.Mutex.RUnlock()
+	if ok && now-arcDur.GotTime > s.c.CacheConf.ArcUpCacheTime {
+		duration = arcDur.Duration
+		return
+	}
+	var arc *api.Arc
+	if arc, err = s.arcRPC.Archive3(c, &arcmdl.ArgAid2{Aid: aid}); err != nil {
+		// just log
+		log.Error("s.arcRPC.Archive3(%d) error(%v)", aid, err)
+	} else {
+		duration = arc.Duration
+	}
+	s.arcDurWithMutex.Mutex.Lock()
+	s.arcDurWithMutex.Durations[aid] = &model.ArcDuration{Duration: duration, GotTime: now}
+	s.arcDurWithMutex.Mutex.Unlock()
+	return
+}
+
+// Close kafaka consumer close.
+func (s *Service) Close() (err error) {
+	s.closed = true
+	time.Sleep(time.Second)
+	for i := 0; i < len(s.cliChan); i++ {
+		close(s.cliChan[i])
+	}
+	close(s.bigDataChan)
+	s.chanWg.Wait()
+	s.statViewPub.Close()
+	return
+}

app/job/main/click/service/service_test.go

@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/click/conf"
+	"go-common/app/job/main/click/model"
+	"go-common/library/sync/errgroup"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/click-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+func Test_Archive(t *testing.T) {
+	Convey("isReplay test", t, func() {
+		b := s.isReplay(context.TODO(), 1, 1, "", 1)
+		So(b, ShouldBeFalse)
+	})
+	return
+}
+
+func Test_SetSpecial(t *testing.T) {
+	Convey("SetSpecial", t, func() {
+		s.SetSpecial(context.TODO(), 1, 10, "")
+	})
+}
+
+func Test_ArcDuration(t *testing.T) {
+	Convey("ArcDuration", t, func() {
+		eg, _ := errgroup.WithContext(context.TODO())
+		var aids = []int64{10099744, 10099730, 10099729, 10099728, 10099726, 10099670, 10099669, 10099668, 10099667, 10099660, 10099653, 10099652, 10099651}
+		for _, aid := range aids {
+			id := aid
+			eg.Go(func() (err error) {
+				Println(s.ArcDuration(context.TODO(), id))
+				return
+			})
+		}
+		eg.Wait()
+		Println(s.arcDurWithMutex.Durations)
+		for aid, val := range s.arcDurWithMutex.Durations {
+			Printf("aid(%d) dur(%d) gotTime(%d)\n", aid, val.Duration, val.GotTime)
+		}
+	})
+}
+
+func Test_SendStat(t *testing.T) {
+	Convey("SendStat", t, func() {
+		vmsg := &model.StatViewMsg{Type: "archive", ID: 123456, Count: 123456, Ts: time.Now().Unix()}
+		err := s.statViewPub.Send(context.TODO(), "test", vmsg)
+		So(err, ShouldBeNil)
+	})
+}