342 lines
8.1 KiB
Go
342 lines
8.1 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"sync"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"go-common/app/job/main/videoup-report/conf"
|
||
|
|
arcdao "go-common/app/job/main/videoup-report/dao/archive"
|
||
|
|
"go-common/app/job/main/videoup-report/dao/data"
|
||
|
|
"go-common/app/job/main/videoup-report/dao/email"
|
||
|
|
hbasedao "go-common/app/job/main/videoup-report/dao/hbase"
|
||
|
|
"go-common/app/job/main/videoup-report/dao/manager"
|
||
|
|
"go-common/app/job/main/videoup-report/dao/mission"
|
||
|
|
redisdao "go-common/app/job/main/videoup-report/dao/redis"
|
||
|
|
"go-common/app/job/main/videoup-report/dao/tag"
|
||
|
|
arcmdl "go-common/app/job/main/videoup-report/model/archive"
|
||
|
|
taskmdl "go-common/app/job/main/videoup-report/model/task"
|
||
|
|
account "go-common/app/service/main/account/api"
|
||
|
|
arcrpc "go-common/app/service/main/archive/api/gorpc"
|
||
|
|
upsrpc "go-common/app/service/main/up/api/v1"
|
||
|
|
"go-common/library/log"
|
||
|
|
"go-common/library/queue/databus"
|
||
|
|
"strings"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
_archiveTable = "archive"
|
||
|
|
_videoTable = "archive_video"
|
||
|
|
_upsTable = "ups"
|
||
|
|
_jumpChanSize = int(4000)
|
||
|
|
_logChanSize = int(200000)
|
||
|
|
)
|
||
|
|
|
||
|
|
// Service is service.
|
||
|
|
type Service struct {
|
||
|
|
c *conf.Config
|
||
|
|
arc *arcdao.Dao
|
||
|
|
redis *redisdao.Dao
|
||
|
|
hbase *hbasedao.Dao
|
||
|
|
dataDao *data.Dao
|
||
|
|
email *email.Dao
|
||
|
|
mng *manager.Dao
|
||
|
|
arcUpChs []chan *arcmdl.UpInfo
|
||
|
|
videoUpInfoChs []chan *arcmdl.VideoUpInfo
|
||
|
|
// waiter
|
||
|
|
waiter sync.WaitGroup
|
||
|
|
// databus
|
||
|
|
archiveSub *databus.Databus
|
||
|
|
arcResultSub *databus.Databus
|
||
|
|
videoupSub *databus.Databus
|
||
|
|
ManagerDBSub *databus.Databus
|
||
|
|
// cache
|
||
|
|
sfTpsCache map[int16]*arcmdl.Type
|
||
|
|
adtTpsCache map[int16]struct{}
|
||
|
|
taskCache *arcmdl.TaskCache
|
||
|
|
videoAuditCache *arcmdl.VideoAuditCache
|
||
|
|
arcMoveTypeCache *arcmdl.ArcMoveTypeCache
|
||
|
|
arcRoundFlowCache *arcmdl.ArcRoundFlowCache
|
||
|
|
xcodeTimeCache *arcmdl.XcodeTimeCache
|
||
|
|
assignCache map[int64]*taskmdl.AssignConfig
|
||
|
|
upperCache map[int8]map[int64]struct{}
|
||
|
|
weightCache map[int8]map[int64]*taskmdl.ConfigItem
|
||
|
|
missTagsCache map[string]int
|
||
|
|
|
||
|
|
lastjumpMap map[int64]struct{} //上轮插队的这一轮也更新，否则会出现权重只增不减
|
||
|
|
jumplist *taskmdl.JumpList //插队序列
|
||
|
|
jumpchan chan *taskmdl.WeightLog
|
||
|
|
tasklogchan chan *taskmdl.WeightLog
|
||
|
|
//rpc
|
||
|
|
arcRPCGroup2 *arcrpc.Service2
|
||
|
|
|
||
|
|
//grpc
|
||
|
|
accRPC account.AccountClient
|
||
|
|
upsRPC upsrpc.UpClient
|
||
|
|
|
||
|
|
// closed
|
||
|
|
closed bool
|
||
|
|
tagDao *tag.Dao
|
||
|
|
missionDao *mission.Dao
|
||
|
|
}
|
||
|
|
|
||
|
|
// New is videoup-report-job service implementation.
|
||
|
|
func New(c *conf.Config) (s *Service) {
|
||
|
|
s = &Service{
|
||
|
|
c: c,
|
||
|
|
//dao
|
||
|
|
arc: arcdao.New(c),
|
||
|
|
redis: redisdao.New(c),
|
||
|
|
hbase: hbasedao.New(c),
|
||
|
|
//databus
|
||
|
|
archiveSub: databus.New(c.ArchiveSub),
|
||
|
|
arcResultSub: databus.New(c.ArchiveResultSub),
|
||
|
|
videoupSub: databus.New(c.VideoupSub),
|
||
|
|
ManagerDBSub: databus.New(c.ManagerDBSub),
|
||
|
|
dataDao: data.New(c),
|
||
|
|
email: email.New(c),
|
||
|
|
mng: manager.New(c),
|
||
|
|
// cache
|
||
|
|
taskCache: &arcmdl.TaskCache{
|
||
|
|
Task: make(map[int64]*arcmdl.Task),
|
||
|
|
},
|
||
|
|
videoAuditCache: &arcmdl.VideoAuditCache{
|
||
|
|
Data: make(map[int16]map[string]int),
|
||
|
|
},
|
||
|
|
arcMoveTypeCache: &arcmdl.ArcMoveTypeCache{
|
||
|
|
Data: make(map[int8]map[int16]map[string]int),
|
||
|
|
},
|
||
|
|
arcRoundFlowCache: &arcmdl.ArcRoundFlowCache{
|
||
|
|
Data: make(map[int8]map[int64]map[string]int),
|
||
|
|
},
|
||
|
|
xcodeTimeCache: &arcmdl.XcodeTimeCache{
|
||
|
|
Data: make(map[int8][]int),
|
||
|
|
},
|
||
|
|
arcRPCGroup2: arcrpc.New2(c.ArchiveRPCGroup2),
|
||
|
|
lastjumpMap: make(map[int64]struct{}),
|
||
|
|
jumplist: taskmdl.NewJumpList(),
|
||
|
|
jumpchan: make(chan *taskmdl.WeightLog, _jumpChanSize),
|
||
|
|
tasklogchan: make(chan *taskmdl.WeightLog, _logChanSize),
|
||
|
|
tagDao: tag.New(c),
|
||
|
|
missionDao: mission.New(c),
|
||
|
|
}
|
||
|
|
var err error
|
||
|
|
if s.accRPC, err = account.NewClient(conf.Conf.GRPC.AccRPC); err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
|
||
|
|
if s.upsRPC, err = upsrpc.NewClient(conf.Conf.GRPC.UpsRPC); err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
|
||
|
|
for i := 0; i < s.c.ChanSize; i++ {
|
||
|
|
log.Info("videoup-report-job chanSize starting(%d)", i)
|
||
|
|
s.arcUpChs = append(s.arcUpChs, make(chan *arcmdl.UpInfo, 10240))
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.arcUpdateproc(i)
|
||
|
|
s.videoUpInfoChs = append(s.videoUpInfoChs, make(chan *arcmdl.VideoUpInfo, 10240))
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.upVideoproc(i)
|
||
|
|
}
|
||
|
|
s.loadConf()
|
||
|
|
s.loadType()
|
||
|
|
// load cache.
|
||
|
|
s.loadTask()
|
||
|
|
s.loadTaskTookSort()
|
||
|
|
s.hdlTraffic()
|
||
|
|
s.loadMission()
|
||
|
|
go s.cacheproc()
|
||
|
|
go s.monitorNotifyProc()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.hotarchiveproc()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.arcCanalConsume()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.arcResultConsume()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.taskWeightConsumer()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.taskweightproc()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.movetaskproc()
|
||
|
|
go s.deltaskproc()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.videoupConsumer()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.emailProc()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.emailFastProc()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.retryProc()
|
||
|
|
s.waiter.Add(1)
|
||
|
|
go s.managerDBConsume()
|
||
|
|
|
||
|
|
return s
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) loadType() {
|
||
|
|
tpm, err := s.arc.TypeMapping(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.dede.TypeMapping error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.sfTpsCache = tpm
|
||
|
|
// audit types
|
||
|
|
adt, err := s.arc.AuditTypesConf(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.dede.AuditTypesConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.adtTpsCache = adt
|
||
|
|
|
||
|
|
wvc, err := s.arc.WeightValueConf(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.arc.WeightValueConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
taskmdl.WLVConf = wvc
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) isAuditType(tpID int16) bool {
|
||
|
|
_, isAt := s.adtTpsCache[tpID]
|
||
|
|
return isAt
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) topType(tpID int16) (id int16) {
|
||
|
|
if tp, ok := s.sfTpsCache[tpID]; ok && tp != nil {
|
||
|
|
id = tp.PID
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) typeName(tpID int16) (name string) {
|
||
|
|
if tp, ok := s.sfTpsCache[tpID]; ok && tp != nil {
|
||
|
|
name = tp.Name
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) topTypeName(tpID int16) (name string) {
|
||
|
|
pid := s.topType(tpID)
|
||
|
|
name = s.typeName(pid)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) cacheproc() {
|
||
|
|
for {
|
||
|
|
time.Sleep(1 * time.Minute)
|
||
|
|
// config
|
||
|
|
s.loadConf()
|
||
|
|
// task
|
||
|
|
s.loadTask()
|
||
|
|
s.loadTaskTookSort()
|
||
|
|
// handle task took
|
||
|
|
s.hdlTaskTook()
|
||
|
|
s.hdlTaskTookByHourHalf()
|
||
|
|
// handle video audit
|
||
|
|
s.hdlVideoAuditCount()
|
||
|
|
s.hdlMoveTypeCount()
|
||
|
|
s.hdlRoundFlowCount()
|
||
|
|
//handle calculate video xcode time stats, and save to DB
|
||
|
|
s.hdlXcodeStats()
|
||
|
|
s.hdlTraffic()
|
||
|
|
s.loadMission()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) monitorNotifyProc() {
|
||
|
|
for {
|
||
|
|
s.monitorNotify()
|
||
|
|
time.Sleep(30 * time.Minute)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// s.missTagsCache: missionName or first tag
|
||
|
|
func (s *Service) loadMission() {
|
||
|
|
mm, err := s.missionDao.Missions(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.missionDao.Mission error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.missTagsCache = make(map[string]int)
|
||
|
|
for _, m := range mm {
|
||
|
|
if len(m.Tags) > 0 {
|
||
|
|
splitedTags := strings.Split(m.Tags, ",")
|
||
|
|
s.missTagsCache[splitedTags[0]] = m.ID
|
||
|
|
} else {
|
||
|
|
s.missTagsCache[m.Name] = m.ID
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// hotarchiveproc get hot archive which need to recheck
|
||
|
|
func (s *Service) hotarchiveproc() {
|
||
|
|
defer s.waiter.Done()
|
||
|
|
for {
|
||
|
|
if s.closed {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.addHotRecheck()
|
||
|
|
time.Sleep(10 * time.Minute)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Close consumer close.
|
||
|
|
func (s *Service) Close() {
|
||
|
|
s.closed = true
|
||
|
|
s.archiveSub.Close()
|
||
|
|
s.arcResultSub.Close()
|
||
|
|
s.videoupSub.Close()
|
||
|
|
time.Sleep(2 * time.Second)
|
||
|
|
for i := 0; i < s.c.ChanSize; i++ {
|
||
|
|
log.Info("videoup-report-job chanSize closing(%d)", i)
|
||
|
|
close(s.arcUpChs[i])
|
||
|
|
close(s.videoUpInfoChs[i])
|
||
|
|
}
|
||
|
|
s.arc.Close()
|
||
|
|
s.mng.Close()
|
||
|
|
s.redis.Close()
|
||
|
|
s.email.Close()
|
||
|
|
s.hbase.Close()
|
||
|
|
s.waiter.Wait()
|
||
|
|
}
|
||
|
|
|
||
|
|
// Ping check server ok.
|
||
|
|
func (s *Service) Ping(c context.Context) (err error) {
|
||
|
|
if err = s.arc.Ping(c); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if err = s.mng.Ping(c); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) loadConf() {
|
||
|
|
var (
|
||
|
|
err error
|
||
|
|
assignConf map[int64]*taskmdl.AssignConfig
|
||
|
|
weightConf map[int8]map[int64]*taskmdl.ConfigItem
|
||
|
|
upperCache map[int8]map[int64]struct{}
|
||
|
|
)
|
||
|
|
|
||
|
|
if assignConf, err = s.assignConf(context.TODO()); err != nil {
|
||
|
|
log.Error("s.assignConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.assignCache = assignConf
|
||
|
|
|
||
|
|
upperCache, err = s.upSpecial(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.upSpecial error(%v)", err)
|
||
|
|
} else {
|
||
|
|
s.upperCache = upperCache
|
||
|
|
}
|
||
|
|
|
||
|
|
if weightConf, err = s.weightConf(context.TODO()); err != nil {
|
||
|
|
log.Error(" s.weightConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.weightCache = weightConf
|
||
|
|
}
|