358 lines
8.0 KiB
Go
358 lines
8.0 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"math"
|
||
|
|
"sync"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"go-common/app/job/main/videoup/conf"
|
||
|
|
"go-common/app/job/main/videoup/dao/activity"
|
||
|
|
"go-common/app/job/main/videoup/dao/archive"
|
||
|
|
"go-common/app/job/main/videoup/dao/bvc"
|
||
|
|
"go-common/app/job/main/videoup/dao/manager"
|
||
|
|
"go-common/app/job/main/videoup/dao/message"
|
||
|
|
"go-common/app/job/main/videoup/dao/monitor"
|
||
|
|
"go-common/app/job/main/videoup/dao/redis"
|
||
|
|
mngmdl "go-common/app/job/main/videoup/model/manager"
|
||
|
|
accApi "go-common/app/service/main/account/api"
|
||
|
|
"go-common/library/conf/env"
|
||
|
|
"go-common/library/log"
|
||
|
|
"go-common/library/queue/databus"
|
||
|
|
"go-common/library/stat/prom"
|
||
|
|
|
||
|
|
"github.com/pkg/errors"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Service is service.
|
||
|
|
type Service struct {
|
||
|
|
c *conf.Config
|
||
|
|
// wait group
|
||
|
|
wg sync.WaitGroup
|
||
|
|
// acc rpc
|
||
|
|
accRPC accApi.AccountClient
|
||
|
|
// dao
|
||
|
|
arc *archive.Dao
|
||
|
|
mng *manager.Dao
|
||
|
|
msg *message.Dao
|
||
|
|
redis *redis.Dao
|
||
|
|
monitor *monitor.Dao
|
||
|
|
activity *activity.Dao
|
||
|
|
bvc *bvc.Dao
|
||
|
|
// databus sub
|
||
|
|
bvc2VuSub *databus.Databus
|
||
|
|
videoupSub *databus.Databus
|
||
|
|
arcResultSub *databus.Databus
|
||
|
|
videoshotSub2 *databus.Databus
|
||
|
|
// videoupSub 幂等判断
|
||
|
|
videoupSubIdempotent map[int32]int64
|
||
|
|
statSub *databus.Databus
|
||
|
|
// databus pub
|
||
|
|
videoupPub *databus.Databus
|
||
|
|
blogPub *databus.Databus
|
||
|
|
// cache: type, upper
|
||
|
|
sfTpsCache map[int16]int16
|
||
|
|
TypeMap map[int16]string
|
||
|
|
adtTpsCache map[int16]struct{}
|
||
|
|
thrTpsCache map[int16]int
|
||
|
|
thrMin, thrMax int
|
||
|
|
upperCache map[int8]map[int64]struct{}
|
||
|
|
fansCache int
|
||
|
|
roundTpsCache map[int16]struct{}
|
||
|
|
roundDelayCache int64
|
||
|
|
delayRoundMinTime time.Time
|
||
|
|
specialUp map[int64]struct{}
|
||
|
|
// monitor
|
||
|
|
bvc2VuMo int64
|
||
|
|
bvc2VuDelayMo int64
|
||
|
|
videoupMo int64
|
||
|
|
arcResultMo int64
|
||
|
|
statMo int64
|
||
|
|
|
||
|
|
//prom moni
|
||
|
|
promDatabus *prom.Prom
|
||
|
|
promRetry *prom.Prom
|
||
|
|
//统计差值
|
||
|
|
promVideoS *prom.Prom
|
||
|
|
promVideoE *prom.Prom
|
||
|
|
promPanic *prom.Prom
|
||
|
|
// closed
|
||
|
|
closed bool
|
||
|
|
}
|
||
|
|
|
||
|
|
// New is videoup service implementation.
|
||
|
|
func New(c *conf.Config) (s *Service) {
|
||
|
|
s = &Service{
|
||
|
|
c: c,
|
||
|
|
// dao
|
||
|
|
arc: archive.New(c),
|
||
|
|
mng: manager.New(c),
|
||
|
|
msg: message.New(c),
|
||
|
|
redis: redis.New(c),
|
||
|
|
monitor: monitor.New(c),
|
||
|
|
activity: activity.New(c),
|
||
|
|
bvc: bvc.New(c),
|
||
|
|
|
||
|
|
bvc2VuSub: databus.New(c.Bvc2VuSub),
|
||
|
|
videoupSubIdempotent: make(map[int32]int64),
|
||
|
|
videoupSub: databus.New(c.VideoupSub),
|
||
|
|
arcResultSub: databus.New(c.ArcResultSub),
|
||
|
|
statSub: databus.New(c.StatSub),
|
||
|
|
// databus pub
|
||
|
|
videoupPub: databus.New(c.VideoupPub),
|
||
|
|
blogPub: databus.New(c.BlogPub),
|
||
|
|
|
||
|
|
promDatabus: prom.BusinessInfoCount,
|
||
|
|
promVideoS: prom.CacheHit,
|
||
|
|
promVideoE: prom.CacheMiss,
|
||
|
|
promPanic: prom.CacheMiss,
|
||
|
|
promRetry: prom.BusinessErrCount,
|
||
|
|
}
|
||
|
|
var err error
|
||
|
|
if s.accRPC, err = accApi.NewClient(c.AccRPC); err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
s.specialUp = make(map[int64]struct{}, len(c.SpecialUp))
|
||
|
|
for _, mid := range c.SpecialUp {
|
||
|
|
s.specialUp[mid] = struct{}{}
|
||
|
|
}
|
||
|
|
// load cache
|
||
|
|
s.loadType()
|
||
|
|
s.loadUpper()
|
||
|
|
s.loadConf()
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.bvc2VuConsumer()
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.videoupConsumer()
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.statConsumer()
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.arcResultConsumer()
|
||
|
|
if env.DeployEnv == env.DeployEnvProd {
|
||
|
|
s.videoshotSub2 = databus.New(c.VideoshotSub2)
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.videoshotSHConsumer()
|
||
|
|
}
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.retryproc()
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.QueueProc()
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.delayproc()
|
||
|
|
s.wg.Add(1)
|
||
|
|
go s.roundproc()
|
||
|
|
go s.cacheproc()
|
||
|
|
go s.monitorConsume()
|
||
|
|
go s.edithistoryproc()
|
||
|
|
return s
|
||
|
|
}
|
||
|
|
|
||
|
|
// Ping ping service.
|
||
|
|
func (s *Service) Ping(c context.Context) (err error) {
|
||
|
|
return s.arc.Ping(c)
|
||
|
|
}
|
||
|
|
|
||
|
|
//Rescue runtime panic rescue
|
||
|
|
func (s *Service) Rescue(data interface{}) {
|
||
|
|
r := recover()
|
||
|
|
if r != nil {
|
||
|
|
r = errors.WithStack(r.(error))
|
||
|
|
log.Error("Runtime error caught: %+v and data is %+v", r, data)
|
||
|
|
s.promPanic.Incr("panic")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) edithistoryproc() {
|
||
|
|
for {
|
||
|
|
time.Sleep(nextDay(5))
|
||
|
|
for {
|
||
|
|
rows, _ := s.delArcEditHistory(100)
|
||
|
|
time.Sleep(1 * time.Second)
|
||
|
|
if rows == 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for {
|
||
|
|
rows, _ := s.delArcVideoEditHistory(100)
|
||
|
|
time.Sleep(1 * time.Second)
|
||
|
|
if rows == 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Until next day x hours
|
||
|
|
func nextDay(hour int) time.Duration {
|
||
|
|
n := time.Now().Add(24 * time.Hour)
|
||
|
|
d := time.Date(n.Year(), n.Month(), n.Day(), hour, 0, 0, 0, n.Location())
|
||
|
|
return time.Until(d)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Close consumer close.
|
||
|
|
func (s *Service) Close() {
|
||
|
|
s.bvc2VuSub.Close()
|
||
|
|
s.videoupSub.Close()
|
||
|
|
s.arcResultSub.Close()
|
||
|
|
s.statSub.Close()
|
||
|
|
if env.DeployEnv == env.DeployEnvProd {
|
||
|
|
s.videoshotSub2.Close()
|
||
|
|
}
|
||
|
|
s.closed = true
|
||
|
|
s.wg.Wait()
|
||
|
|
s.redis.Close()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) isMission(c context.Context, aid int64) bool {
|
||
|
|
if addit, _ := s.arc.Addit(c, aid); addit != nil && addit.MissionID > 0 {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) isWhite(mid int64) bool {
|
||
|
|
if ups, ok := s.upperCache[mngmdl.UpperTypeWhite]; ok {
|
||
|
|
_, isWhite := ups[mid]
|
||
|
|
return isWhite
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) isBlack(mid int64) bool {
|
||
|
|
if ups, ok := s.upperCache[mngmdl.UpperTypeBlack]; ok {
|
||
|
|
_, isBlack := ups[mid]
|
||
|
|
return isBlack
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) isAuditType(tpID int16) bool {
|
||
|
|
_, isAt := s.adtTpsCache[tpID]
|
||
|
|
return isAt
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) loadType() {
|
||
|
|
tpMap, err := s.arc.TypeMapping(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.arc.TypeMapping error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.sfTpsCache = tpMap
|
||
|
|
log.Info("s.sfTpsCache Data is (%+v)", s.sfTpsCache)
|
||
|
|
tpNaming, err := s.arc.TypeNaming(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.arc.TypeNaming error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.TypeMap = tpNaming
|
||
|
|
log.Info("s.TypeMap Data is (%+v)", s.TypeMap)
|
||
|
|
// audit types
|
||
|
|
adt, err := s.arc.AuditTypesConf(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.arc.AuditTypesConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.adtTpsCache = adt
|
||
|
|
log.Info("s.adtTpsCache Data is (%+v)", s.adtTpsCache)
|
||
|
|
// threshold
|
||
|
|
thr, err := s.arc.ThresholdConf(context.TODO())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.arc.ThresholdConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.thrTpsCache = thr
|
||
|
|
log.Info("s.thrTpsCache Data is (%+v)", s.thrTpsCache)
|
||
|
|
var min, max = math.MaxInt32, 0
|
||
|
|
for _, t := range thr {
|
||
|
|
if min > t {
|
||
|
|
min = t
|
||
|
|
}
|
||
|
|
if max < t {
|
||
|
|
max = t
|
||
|
|
}
|
||
|
|
}
|
||
|
|
s.thrMin = min
|
||
|
|
s.thrMax = max
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) loadUpper() {
|
||
|
|
var (
|
||
|
|
c = context.TODO()
|
||
|
|
)
|
||
|
|
|
||
|
|
upm, err := s.mng.Uppers(c)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.mng.Uppers error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.upperCache = upm
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) isRoundType(tpID int16) bool {
|
||
|
|
_, in := s.roundTpsCache[tpID]
|
||
|
|
return in
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) loadConf() {
|
||
|
|
var (
|
||
|
|
fans int64
|
||
|
|
days int64
|
||
|
|
err error
|
||
|
|
roundTypes map[int16]struct{}
|
||
|
|
)
|
||
|
|
if fans, err = s.arc.FansConf(context.TODO()); err != nil {
|
||
|
|
log.Error("s.arc.FansConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.fansCache = int(fans)
|
||
|
|
if roundTypes, err = s.arc.RoundTypeConf(context.TODO()); err != nil {
|
||
|
|
log.Error("s.arc.RoundTypeConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.roundTpsCache = roundTypes
|
||
|
|
if days, err = s.arc.RoundEndConf(context.TODO()); err != nil {
|
||
|
|
log.Error("s.arc.RoundEndConf")
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.roundDelayCache = days
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) cacheproc() {
|
||
|
|
for {
|
||
|
|
time.Sleep(1 * time.Minute)
|
||
|
|
s.loadType()
|
||
|
|
s.loadUpper()
|
||
|
|
s.loadConf()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) monitorConsume() {
|
||
|
|
if env.DeployEnv != env.DeployEnvProd {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
var bvc2Vu, videoup, arcResult, stat, bvc2VuDelay int64
|
||
|
|
for {
|
||
|
|
time.Sleep(1 * time.Minute)
|
||
|
|
if s.bvc2VuMo-bvc2Vu == 0 {
|
||
|
|
s.monitor.Send(context.TODO(), "video-job bvc2Video did not consume within a minute")
|
||
|
|
}
|
||
|
|
if s.videoupMo-videoup == 0 {
|
||
|
|
s.monitor.Send(context.TODO(), "video-job videoup did not consume within a minute")
|
||
|
|
}
|
||
|
|
if s.arcResultMo-arcResult == 0 {
|
||
|
|
s.monitor.Send(context.TODO(), "video-job arcResult did not consume within a minute")
|
||
|
|
}
|
||
|
|
if s.statMo-stat == 0 {
|
||
|
|
s.monitor.Send(context.TODO(), "video-job stat did not consume within a minute")
|
||
|
|
}
|
||
|
|
if s.bvc2VuDelayMo-bvc2VuDelay > 0 {
|
||
|
|
s.monitor.Send(context.TODO(), "video-job bvc2videoup consume delayed.")
|
||
|
|
}
|
||
|
|
bvc2Vu = s.bvc2VuMo
|
||
|
|
videoup = s.videoupMo
|
||
|
|
arcResult = s.arcResultMo
|
||
|
|
stat = s.statMo
|
||
|
|
bvc2VuDelay = s.bvc2VuDelayMo
|
||
|
|
}
|
||
|
|
}
|