363 lines
9.2 KiB
Go
363 lines
9.2 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"go-common/app/job/main/aegis/model"
|
||
"go-common/library/log"
|
||
)
|
||
|
||
// WeightManager weight manager
|
||
type WeightManager struct {
|
||
s *Service
|
||
|
||
businessID, flowID int64
|
||
toplen, batchlen int64
|
||
minute int64
|
||
|
||
// cache
|
||
topweightList []*model.WeightItem
|
||
|
||
// channel
|
||
redisWeightList chan *model.WeightItem
|
||
dbWeightList chan *model.WeightItem
|
||
asignList chan *model.Task
|
||
//dbstartSig, dbstopSig chan struct{}
|
||
redisFinish chan struct{}
|
||
|
||
//closeChan chan struct{}
|
||
close bool
|
||
}
|
||
|
||
var _defaultopt = &model.WeightOPT{
|
||
TopListLen: 1000,
|
||
BatchListLen: 1000,
|
||
RedisListLen: 10000,
|
||
DbListLen: 2000,
|
||
AssignLen: 100,
|
||
Minute: 3,
|
||
}
|
||
|
||
// NewWeightManager new
|
||
func NewWeightManager(s *Service, opt *model.WeightOPT, key string) (wm *WeightManager) {
|
||
if opt == nil {
|
||
opt = _defaultopt
|
||
} else {
|
||
if opt.TopListLen <= 0 {
|
||
opt.TopListLen = _defaultopt.TopListLen
|
||
}
|
||
if opt.BatchListLen <= 0 {
|
||
opt.BatchListLen = _defaultopt.BatchListLen
|
||
}
|
||
if opt.RedisListLen <= 0 {
|
||
opt.RedisListLen = _defaultopt.RedisListLen
|
||
}
|
||
if opt.DbListLen <= 0 {
|
||
opt.DbListLen = _defaultopt.DbListLen
|
||
}
|
||
if opt.AssignLen <= 0 {
|
||
opt.AssignLen = _defaultopt.AssignLen
|
||
}
|
||
if opt.Minute <= 0 {
|
||
opt.Minute = _defaultopt.Minute
|
||
}
|
||
}
|
||
|
||
if len(key) > 0 {
|
||
bizid, flowid := parseKey(key)
|
||
opt.BusinessID = int64(bizid)
|
||
opt.FlowID = int64(flowid)
|
||
}
|
||
|
||
wm = &WeightManager{
|
||
s: s,
|
||
businessID: opt.BusinessID,
|
||
flowID: opt.FlowID,
|
||
toplen: opt.TopListLen,
|
||
batchlen: opt.BatchListLen,
|
||
minute: opt.Minute,
|
||
redisWeightList: make(chan *model.WeightItem, opt.RedisListLen),
|
||
dbWeightList: make(chan *model.WeightItem, opt.DbListLen),
|
||
asignList: make(chan *model.Task, opt.AssignLen),
|
||
redisFinish: make(chan struct{}),
|
||
}
|
||
|
||
go wm.weightProc()
|
||
go wm.weightWatcher()
|
||
log.Info("启动权重计算器 bizid(%d) flowid(%d) opt(%+v)", wm.businessID, wm.flowID, opt)
|
||
return
|
||
}
|
||
|
||
func parseKey(key string) (bizid, flowid int) {
|
||
pos := strings.Index(key, "-")
|
||
bizids := key[:pos]
|
||
flowids := key[pos+1:]
|
||
bizid, _ = strconv.Atoi(bizids)
|
||
flowid, _ = strconv.Atoi(flowids)
|
||
return
|
||
}
|
||
|
||
func (s *Service) startWeightManager() {
|
||
// 1.当前的所有业务线，需要计算权重的先枚举出来
|
||
s.wmHash = make(map[string]*WeightManager)
|
||
for key := range s.newactiveBizFlow {
|
||
bizid, _ := parseKey(key)
|
||
s.wmHash[key] = NewWeightManager(s, s.getWeightOpt(bizid), key)
|
||
}
|
||
}
|
||
|
||
func (w *WeightManager) weightProc() {
|
||
for !w.close {
|
||
if err := w.weightRedisProcess(); err != nil {
|
||
w.weightDBProcess()
|
||
}
|
||
time.Sleep(time.Duration(w.minute) * time.Minute)
|
||
}
|
||
}
|
||
|
||
func (w *WeightManager) weightWatcher() {
|
||
for !w.close {
|
||
select {
|
||
case <-w.redisFinish: //取出权重最大的一批，更新到数据库
|
||
log.Info("redisFinish(%d-%d:%d)", w.businessID, w.flowID, w.toplen)
|
||
w.handleRedisFinish(context.Background())
|
||
case wi := <-w.redisWeightList:
|
||
w.handleRedisWeightList(context.Background(), wi)
|
||
case wi := <-w.dbWeightList:
|
||
w.handleDBWeightList(context.Background(), wi)
|
||
case task := <-w.asignList:
|
||
w.handleAssign(context.Background(), task)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (w *WeightManager) weightRedisProcess() (err error) {
|
||
var c = context.Background()
|
||
if err = w.s.dao.CreateUnionSet(c, w.businessID, w.flowID); err != nil {
|
||
return
|
||
}
|
||
|
||
var (
|
||
start = int64(0)
|
||
stop = w.batchlen
|
||
)
|
||
for {
|
||
wis, err := w.s.dao.RangeUinonSet(c, w.businessID, w.flowID, start, stop)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
log.Info("weightRedisProcess length(%d) start(%d) stop(%d)", len(wis), start, stop)
|
||
start += w.batchlen
|
||
stop += w.batchlen
|
||
if len(wis) == 0 {
|
||
break
|
||
}
|
||
for _, wi := range wis {
|
||
if w.caculateWeight(c, wi) {
|
||
log.Warn("weightRedisProcess 任务未找到 wi(%+v)", wi)
|
||
continue
|
||
}
|
||
w.s.dao.SetWeight(c, w.businessID, w.flowID, wi.ID, wi.Weight)
|
||
}
|
||
time.Sleep(time.Second)
|
||
}
|
||
w.redisFinish <- struct{}{}
|
||
w.s.dao.DeleteUinonSet(c, w.businessID, w.flowID)
|
||
return nil
|
||
}
|
||
|
||
func (w *WeightManager) caculateWeight(c context.Context, wi *model.WeightItem) (skip bool) {
|
||
task, err := w.s.dao.GetTask(c, wi.ID)
|
||
if err != nil {
|
||
return true
|
||
}
|
||
w.reAssign(c, task)
|
||
|
||
wm := int64(time.Since(task.Ctime.Time()).Minutes())
|
||
wl := &model.WeightLog{
|
||
UPtime: time.Now().Format("2006-01-02 15:04:05"),
|
||
Mid: task.MID,
|
||
Fans: task.Fans,
|
||
Group: task.Group,
|
||
WaitTime: model.WaitTime(task.Ctime.Time()),
|
||
}
|
||
|
||
var wtRange, wtEqual int64
|
||
|
||
wci, ewc := w.s.getWeightCache(c, task.BusinessID, task.FlowID)
|
||
if wci != nil {
|
||
wtRange = w.rangeCaculate(c, wci, task, wm, wl)
|
||
}
|
||
if ewc != nil {
|
||
wtEqual = w.equalCaculate(c, ewc, task, wm, wl)
|
||
}
|
||
wi.Weight = wtRange + wtEqual
|
||
wl.Weight = wi.Weight
|
||
|
||
w.s.sendWeightLog(c, task, wl)
|
||
return
|
||
}
|
||
|
||
func (w *WeightManager) rangeCaculate(c context.Context, wci map[string]*model.RangeWeightConfig, task *model.Task, wt int64, wl *model.WeightLog) (weight int64) {
|
||
var wtWeight, fanWeight, groupWeight int64
|
||
|
||
if cfg, ok := wci["waittime"]; ok {
|
||
if wtlen := len(cfg.Range); wtlen > 0 { // 等待时长，要把之前等级的权重加上去
|
||
for i := wtlen - 1; i >= 0; i-- {
|
||
if wt >= cfg.Range[i].Threshold { // 命中配置
|
||
wtWeight += cfg.Range[i].Weight * ((wt - cfg.Range[i].Threshold) / w.minute)
|
||
|
||
// 计算0 到 (i-1) 累计权重
|
||
for j := 0; j <= i-1; j++ {
|
||
wtWeight += cfg.Range[j].Weight * ((cfg.Range[j+1].Threshold - cfg.Range[j].Threshold) / w.minute)
|
||
}
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if cfg, ok := wci["fans"]; ok {
|
||
if fanLen := len(cfg.Range); fanLen > 0 {
|
||
for i := fanLen - 1; i >= 0; i-- {
|
||
if task.Fans >= cfg.Range[i].Threshold {
|
||
fanWeight = cfg.Range[i].Weight * (wt / w.minute)
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if cfg, ok := wci["group"]; ok {
|
||
if len(cfg.Range) > 0 {
|
||
for _, item := range cfg.Range {
|
||
if strings.Contains(","+task.Group+",", fmt.Sprintf(",%d,", item.Threshold)) {
|
||
groupWeight = item.Weight * (wt / w.minute)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
weight = wtWeight + fanWeight + groupWeight
|
||
wl.WaitWeight = wtWeight
|
||
wl.FansWeight = fanWeight
|
||
wl.GroupWeight = groupWeight
|
||
return
|
||
}
|
||
|
||
func (w *WeightManager) equalCaculate(c context.Context, ewc []*model.EqualWeightConfig, task *model.Task, wt int64, wl *model.WeightLog) (weight int64) {
|
||
var midweight, taskweight int64
|
||
for _, item := range ewc {
|
||
if item.Name == "mid" {
|
||
if strings.Contains(","+item.IDs+",", fmt.Sprintf(",%d,", task.MID)) {
|
||
if item.Type == model.WeightTypeCycle {
|
||
midweight += item.Weight * (wt / w.minute)
|
||
} else {
|
||
midweight += item.Weight
|
||
}
|
||
log.Info("equalCaculate task(%+v) hit (%+v)", task, item)
|
||
wl.ConfigItems = append(wl.ConfigItems, &model.ConfigItem{
|
||
Name: item.Name,
|
||
Desc: item.Description,
|
||
Uname: item.Uname,
|
||
})
|
||
}
|
||
}
|
||
if item.Name == "taskid" || item.Name == "task_id" {
|
||
if strings.Contains(","+item.IDs+",", fmt.Sprintf(",%d,", task.ID)) {
|
||
if item.Type == model.WeightTypeCycle {
|
||
taskweight += item.Weight * (wt / w.minute)
|
||
} else {
|
||
taskweight += item.Weight
|
||
}
|
||
log.Info("equalCaculate task(%+v) hit (%+v)", task, item)
|
||
wl.ConfigItems = append(wl.ConfigItems, &model.ConfigItem{
|
||
Name: item.Name,
|
||
Desc: item.Description,
|
||
Uname: item.Uname,
|
||
})
|
||
}
|
||
}
|
||
}
|
||
weight = midweight + taskweight
|
||
|
||
wl.EqualWeight = weight
|
||
return
|
||
}
|
||
|
||
func (w *WeightManager) reAssign(c context.Context, task *model.Task) {
|
||
if task.UID == 0 {
|
||
select {
|
||
case w.asignList <- task:
|
||
log.Info("指派判断 reAssign（%+v）", task)
|
||
case <-time.NewTimer(10 * time.Millisecond).C:
|
||
log.Warn("chan asignList full,len:%d", len(w.dbWeightList))
|
||
}
|
||
}
|
||
}
|
||
|
||
func (w *WeightManager) weightDBProcess() (err error) {
|
||
// TODO 只用db更新权重的策略
|
||
return nil
|
||
}
|
||
|
||
func (w *WeightManager) handleAssign(c context.Context, task *model.Task) (err error) {
|
||
if w.s.setAssign(c, task) {
|
||
if rows, err := w.s.dao.AssignTask(c, task); err == nil && rows == 1 {
|
||
w.s.dao.SetTask(c, task)
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func (w *WeightManager) handleRedisWeightList(c context.Context, wi *model.WeightItem) (err error) {
|
||
return w.s.dao.SetWeight(c, w.businessID, w.flowID, wi.ID, wi.Weight)
|
||
}
|
||
|
||
func (w *WeightManager) handleDBWeightList(c context.Context, wi *model.WeightItem) (rows int64, err error) {
|
||
return w.s.dao.SetWeightDB(c, wi.ID, wi.Weight)
|
||
}
|
||
|
||
func (w *WeightManager) handleRedisFinish(c context.Context) (err error) {
|
||
log.Info("handleRedisFinish")
|
||
wis, err := w.s.dao.TopWeights(c, w.businessID, w.flowID, w.toplen)
|
||
if err != nil {
|
||
return
|
||
}
|
||
|
||
tempMap := make(map[int64]struct{})
|
||
for _, wi := range wis {
|
||
log.Info("handleRedisFinish:(%+v)", wi)
|
||
w.addToDBList(wi)
|
||
tempMap[wi.ID] = struct{}{}
|
||
}
|
||
|
||
for _, wi := range w.topweightList {
|
||
if _, ok := tempMap[wi.ID]; !ok {
|
||
weight, err := w.s.dao.GetWeight(c, w.businessID, w.flowID, wi.ID)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
wi.Weight = weight
|
||
w.addToDBList(wi)
|
||
}
|
||
}
|
||
w.topweightList = wis
|
||
log.Info("handleRedisFinish:topweightList(%d)", len(wis))
|
||
|
||
return
|
||
}
|
||
|
||
func (w *WeightManager) addToDBList(wi *model.WeightItem) {
|
||
select {
|
||
case w.dbWeightList <- wi:
|
||
log.Info("addToDBList (%+v)", wi)
|
||
case <-time.NewTimer(10 * time.Millisecond).C:
|
||
log.Warn("chan dbWeightList full,len:%d", len(w.dbWeightList))
|
||
}
|
||
}
|