372 lines
9.3 KiB
Go
372 lines
9.3 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"go-common/app/job/main/videoup-report/model/archive"
|
||
|
|
"go-common/app/job/main/videoup-report/model/task"
|
||
|
|
"go-common/app/job/main/videoup-report/model/utils"
|
||
|
|
"go-common/library/log"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
_pm = int64(3) //period minute
|
||
|
|
//普通任务参数
|
||
|
|
_nc1 = int64(3) // 等待时长9分钟
|
||
|
|
_nc2 = int64(5) // 等待时长15分钟
|
||
|
|
_nc3 = int64(9) // 等待时长27分钟
|
||
|
|
_nc4 = int64(15) // 等待时长45分钟
|
||
|
|
|
||
|
|
//定时任务参数
|
||
|
|
_tc1 = int64(80) // 距离发布4小时
|
||
|
|
_tc2 = int64(40) // 距离发布2小时
|
||
|
|
_tc3 = int64(20) // 距离发布1小时
|
||
|
|
)
|
||
|
|
|
||
|
|
func (s *Service) upTaskWeightCache() (err error) {
|
||
|
|
var (
|
||
|
|
ids []int64
|
||
|
|
firstid int64
|
||
|
|
lastid int64
|
||
|
|
tasksMap map[int64]*task.WeightParams
|
||
|
|
)
|
||
|
|
c := context.TODO()
|
||
|
|
s.jumplist.Reset()
|
||
|
|
|
||
|
|
defer func() {
|
||
|
|
s.lastjumpMap = make(map[int64]struct{})
|
||
|
|
for item := s.jumplist.POP(); item != nil; item = s.jumplist.POP() {
|
||
|
|
select {
|
||
|
|
case s.jumpchan <- item:
|
||
|
|
s.lastjumpMap[item.TaskID] = struct{}{}
|
||
|
|
default:
|
||
|
|
log.Warn("jumpchan full")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}()
|
||
|
|
|
||
|
|
for {
|
||
|
|
cacheMap := make(map[int64]*task.WeightParams)
|
||
|
|
// 先从数据库批量读取出id来
|
||
|
|
if ids, lastid, err = s.arc.TaskIDforWeight(c, firstid); len(ids) == 0 {
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.task.TaskIDforWeight(%d) error(%v)", firstid, err)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
// 先从redis取权重配置,redis挂了从数据库读取权重配置
|
||
|
|
if tasksMap, err = s.redis.GetWeight(c, ids); len(tasksMap) != len(ids) {
|
||
|
|
log.Warn("GetTaskWeight from redis need len(%d) while len(%d)", len(ids), len(tasksMap))
|
||
|
|
if tasksMap, err = s.arc.GetTaskWeight(c, firstid); err != nil {
|
||
|
|
log.Error("s.arc.GetTaskWeight(%d) error(%v)", firstid, err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for id, tp := range tasksMap {
|
||
|
|
if tp.State == 0 {
|
||
|
|
s.setTask(c, id, tp, cacheMap)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(cacheMap) > 0 {
|
||
|
|
s.redis.SetWeight(c, cacheMap)
|
||
|
|
}
|
||
|
|
firstid = lastid
|
||
|
|
time.Sleep(time.Second)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) upTaskWeightDB(c context.Context, item *task.WeightLog) (err error) {
|
||
|
|
// 超时还没入库的直接丢掉，免得阻塞
|
||
|
|
if time.Since(item.Uptime.TimeValue()).Minutes() > 3.0 {
|
||
|
|
log.Warn("task weight item(%v) expired! ", item)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
_, err = s.arc.UpTaskWeight(c, item.TaskID, item.Weight)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("UpTaskWeight(%d,%d) error(%v)", item.Weight, item.TaskID, err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) setTask(c context.Context, taskid int64, tp *task.WeightParams, cm map[int64]*task.WeightParams) {
|
||
|
|
tpc := s.newPriority(c, tp)
|
||
|
|
// 上轮次权重前2000的这轮也必须更新
|
||
|
|
if _, ok := s.lastjumpMap[tpc.TaskID]; ok {
|
||
|
|
select {
|
||
|
|
case s.jumpchan <- tpc:
|
||
|
|
default:
|
||
|
|
log.Warn("jumpchan full")
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
s.jumplist.PUSH(tpc) //插队序列
|
||
|
|
}
|
||
|
|
|
||
|
|
tp.Weight = tpc.Weight
|
||
|
|
if cm != nil {
|
||
|
|
cm[taskid] = tp
|
||
|
|
}
|
||
|
|
select {
|
||
|
|
case s.tasklogchan <- tpc:
|
||
|
|
default:
|
||
|
|
log.Info("s.tasklogchan full(%d)", len(s.tasklogchan))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// 任务权重计算
|
||
|
|
func (s *Service) newPriority(c context.Context, tp *task.WeightParams) (tpc *task.WeightLog) {
|
||
|
|
var round int64
|
||
|
|
var wcf = task.WLVConf
|
||
|
|
tpc = &task.WeightLog{
|
||
|
|
TaskID: tp.TaskID,
|
||
|
|
Mid: tp.Mid,
|
||
|
|
}
|
||
|
|
|
||
|
|
if tp.AccFailed {
|
||
|
|
if tp.Fans, tp.AccFailed = s.getUpperFans(c, tp.Mid); !tp.AccFailed {
|
||
|
|
tp.Special = s.getUpSpecial(tp.Mid, tp.Fans)
|
||
|
|
s.arc.SetUpSpecial(c, tp.TaskID, tp.Special)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
round = int64(time.Since(tp.Ctime.TimeValue()).Minutes()) / _pm
|
||
|
|
//定时任务
|
||
|
|
if !tp.Ptime.TimeValue().IsZero() {
|
||
|
|
pround := int64((time.Until(tp.Ptime.TimeValue()).Minutes())) / 3
|
||
|
|
sumtesmp := wcf.Tlv4 * (int64(tp.Ptime.TimeValue().Sub(tp.Ctime.TimeValue()).Minutes()) - 60*4) / 3
|
||
|
|
switch {
|
||
|
|
case pround >= _tc1:
|
||
|
|
tpc.TWeight = round * wcf.Tlv4
|
||
|
|
case _tc2 <= pround && pround < _tc1:
|
||
|
|
tpc.TWeight = (_tc1-pround)*wcf.Tlv1 + sumtesmp
|
||
|
|
case _tc3 <= pround && pround < _tc2:
|
||
|
|
tpc.TWeight = (_tc2-pround)*wcf.Tlv2 + wcf.Tsum2h + sumtesmp
|
||
|
|
case pround < _tc3:
|
||
|
|
tpc.TWeight = (_tc3-pround)*wcf.Tlv3 + wcf.Tsum1h + sumtesmp
|
||
|
|
}
|
||
|
|
tpc.NWeight = 0
|
||
|
|
} else { // 普通任务加权
|
||
|
|
switch {
|
||
|
|
case round < _nc1:
|
||
|
|
tpc.NWeight = wcf.Nlv5 * round
|
||
|
|
case _nc1 <= round && round < _nc2:
|
||
|
|
tpc.NWeight = wcf.Nlv1*(round-_nc1) + wcf.Nsum9
|
||
|
|
case _nc2 <= round && round < _nc3:
|
||
|
|
tpc.NWeight = wcf.Nlv2*(round-_nc2) + wcf.Nsum15
|
||
|
|
case _nc3 <= round && round < _nc4:
|
||
|
|
tpc.NWeight = wcf.Nlv3*(round-_nc3) + wcf.Nsum27
|
||
|
|
default:
|
||
|
|
tpc.NWeight = wcf.Nlv4*(round-_nc4) + wcf.Nsum45
|
||
|
|
}
|
||
|
|
tpc.TWeight = 0
|
||
|
|
}
|
||
|
|
|
||
|
|
// 特殊任务加权
|
||
|
|
switch tp.Special {
|
||
|
|
case task.UpperBigNormal:
|
||
|
|
tpc.SWeight = wcf.Slv1 * round
|
||
|
|
case task.UpperSuperNormal:
|
||
|
|
tpc.SWeight = wcf.Slv2 * round
|
||
|
|
case task.UpperWhite:
|
||
|
|
tpc.SWeight = wcf.Slv3 * round
|
||
|
|
case task.UpperBigWhite:
|
||
|
|
tpc.SWeight = wcf.Slv4 * round
|
||
|
|
case task.UpperSuperWhite:
|
||
|
|
tpc.SWeight = wcf.Slv5 * round
|
||
|
|
case task.UpperSuperBlack:
|
||
|
|
tpc.SWeight = wcf.Slv6 * round
|
||
|
|
case task.UpperBlack:
|
||
|
|
tpc.SWeight = wcf.Slv7 * round
|
||
|
|
default:
|
||
|
|
tpc.SWeight = 0
|
||
|
|
}
|
||
|
|
|
||
|
|
//配置任务加权
|
||
|
|
tpc.CWeight = 0
|
||
|
|
if len(tp.CfItems) > 0 {
|
||
|
|
tpc.CfItems = tp.CfItems
|
||
|
|
for _, item := range tp.CfItems {
|
||
|
|
if item.Rule == 0 {
|
||
|
|
round2 := int64(time.Since(item.Mtime.TimeValue()).Minutes()) / _pm
|
||
|
|
if round2 < round {
|
||
|
|
round = round2
|
||
|
|
}
|
||
|
|
tpc.CWeight += round * item.Weight
|
||
|
|
} else {
|
||
|
|
tpc.CWeight += item.Weight
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if tpc.CWeight <= wcf.MinWeight {
|
||
|
|
tpc.CWeight = wcf.MinWeight
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
tpc.Weight = tpc.NWeight + tpc.SWeight + tpc.TWeight + tpc.CWeight
|
||
|
|
if tpc.Weight >= wcf.MaxWeight {
|
||
|
|
tpc.Weight = wcf.MaxWeight
|
||
|
|
}
|
||
|
|
tpc.Uptime = utils.NewFormatTime(time.Now())
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// 设置特殊用户的审核任务
|
||
|
|
func (s *Service) setTaskUPSpecial(c context.Context, t *task.Task, mid int64) (fans int64, failed bool) {
|
||
|
|
fans, failed = s.getUpperFans(c, mid)
|
||
|
|
t.UPSpecial = s.getUpSpecial(mid, fans)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) getUpSpecial(mid int64, fans int64) (upspecial int8) {
|
||
|
|
switch {
|
||
|
|
case s.isWhite(mid): //优质
|
||
|
|
if fans >= task.SuperUpperTH {
|
||
|
|
upspecial = task.UpperSuperWhite
|
||
|
|
} else if fans >= task.BigUpperTH {
|
||
|
|
upspecial = task.UpperSuperWhite
|
||
|
|
} else {
|
||
|
|
upspecial = task.UpperWhite
|
||
|
|
}
|
||
|
|
case s.isBlack(mid):
|
||
|
|
if fans >= task.SuperUpperTH {
|
||
|
|
upspecial = task.UpperSuperBlack
|
||
|
|
} else {
|
||
|
|
upspecial = task.UpperBlack
|
||
|
|
}
|
||
|
|
|
||
|
|
default:
|
||
|
|
if fans >= task.SuperUpperTH {
|
||
|
|
upspecial = task.UpperSuperNormal
|
||
|
|
} else if fans >= task.BigUpperTH {
|
||
|
|
upspecial = task.UpperBigNormal
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// 设置定时发布的审核任务
|
||
|
|
func (s *Service) setTaskTimed(c context.Context, t *task.Task) {
|
||
|
|
adelay, err := s.arc.Delay(c, t.Aid)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.arc.Delay(%d) error(%v)", t.Aid, err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if adelay != nil && adelay.State == 0 && !adelay.DTime.IsZero() {
|
||
|
|
t.Ptime = utils.NewFormatTime(adelay.DTime)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) setTaskUpFrom(c context.Context, aid int64, tp *task.WeightParams) {
|
||
|
|
if addit, _ := s.arc.Addit(c, aid); addit != nil {
|
||
|
|
tp.UpFrom = addit.UpFrom
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) setTaskUpGroup(c context.Context, mid int64, tp *task.WeightParams) {
|
||
|
|
ugs := []int8{}
|
||
|
|
for gid, cache := range s.upperCache {
|
||
|
|
if _, ok := cache[mid]; ok {
|
||
|
|
ugs = append(ugs, gid)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
tp.UpGroups = ugs
|
||
|
|
}
|
||
|
|
|
||
|
|
// 设置配置用户权重的审核任务
|
||
|
|
func (s *Service) getConfWeight(c context.Context, t *task.Task, a *archive.Archive) (cfitems []*task.ConfigItem) {
|
||
|
|
// 1. 按照用户
|
||
|
|
if confs, ok := s.weightCache[task.WConfMid]; ok {
|
||
|
|
if conf, ok := confs[a.Mid]; ok {
|
||
|
|
cfitems = append(cfitems, conf)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// 2. 按照分区
|
||
|
|
if confs, ok := s.weightCache[task.WConfType]; ok {
|
||
|
|
if conf, ok := confs[int64(a.TypeID)]; ok {
|
||
|
|
cfitems = append(cfitems, conf)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// 3. 按照投稿来源
|
||
|
|
if confs, ok := s.weightCache[task.WConfUpFrom]; ok {
|
||
|
|
addit, err := s.arc.Addit(c, a.ID)
|
||
|
|
if addit == nil {
|
||
|
|
if err != nil {
|
||
|
|
log.Error(" s.arc.Addit(%d) error(%v)", t.Aid, err)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
if conf, ok := confs[int64(addit.UpFrom)]; ok {
|
||
|
|
cfitems = append(cfitems, conf)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// 权重配置
|
||
|
|
func (s *Service) weightConf(c context.Context) (cfc map[int8]map[int64]*task.ConfigItem, err error) {
|
||
|
|
var (
|
||
|
|
ids []int64
|
||
|
|
arrcfs []*task.ConfigItem
|
||
|
|
)
|
||
|
|
if arrcfs, err = s.arc.WeightConf(context.TODO()); err != nil {
|
||
|
|
log.Error("s.arc.WeightConf error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
cfc = map[int8]map[int64]*task.ConfigItem{}
|
||
|
|
for _, item := range arrcfs {
|
||
|
|
if !item.Et.TimeValue().IsZero() && item.Et.TimeValue().Before(time.Now()) {
|
||
|
|
ids = append(ids, item.ID)
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if _, ok := cfc[item.Radio]; !ok {
|
||
|
|
cfc[item.Radio] = make(map[int64]*task.ConfigItem)
|
||
|
|
}
|
||
|
|
cfc[item.Radio][item.CID] = item
|
||
|
|
}
|
||
|
|
if len(ids) > 0 {
|
||
|
|
log.Info("task config(%v) 权重配置已过期,自动失效", ids)
|
||
|
|
s.arc.DelWeightConfs(c, ids)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) taskWeightConsumer() {
|
||
|
|
var c = context.TODO()
|
||
|
|
defer s.waiter.Done()
|
||
|
|
|
||
|
|
for {
|
||
|
|
if s.closed {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
select {
|
||
|
|
case item, ok := <-s.jumpchan: //插队序列
|
||
|
|
if !ok {
|
||
|
|
log.Error("s.jumpchan closed")
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.upTaskWeightDB(c, item)
|
||
|
|
case item, ok := <-s.tasklogchan:
|
||
|
|
if !ok {
|
||
|
|
log.Error("s.tasklogchan closed")
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.hbase.AddLog(c, item)
|
||
|
|
default:
|
||
|
|
time.Sleep(time.Second)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) taskweightproc() {
|
||
|
|
defer s.waiter.Done()
|
||
|
|
for {
|
||
|
|
if s.closed {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
s.upTaskWeightCache()
|
||
|
|
log.Info("taskweightproc 插队序列(%d) 日志序列(%d)", len(s.jumpchan), len(s.tasklogchan))
|
||
|
|
time.Sleep(3 * time.Minute)
|
||
|
|
}
|
||
|
|
}
|