Create & Init Project...

app/job/main/videoup-report/service/BUILD

@@ -0,0 +1,101 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "archive_test.go",
+        "excitation_test.go",
+        "monitor_test.go",
+        "reply_test.go",
+        "retry_test.go",
+        "service_test.go",
+        "tag_test.go",
+        "ups_test.go",
+        "video_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/videoup-report/conf:go_default_library",
+        "//app/job/main/videoup-report/model/archive:go_default_library",
+        "//app/job/main/videoup-report/model/email:go_default_library",
+        "//app/job/main/videoup-report/model/manager:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "archive.go",
+        "archive_result.go",
+        "databus.go",
+        "email.go",
+        "excitation.go",
+        "extra_func.go",
+        "flow.go",
+        "monitor.go",
+        "reply.go",
+        "retry.go",
+        "service.go",
+        "tag.go",
+        "task.go",
+        "task_assign.go",
+        "task_dispatch.go",
+        "task_weight.go",
+        "track.go",
+        "ups.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/main/videoup-report/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/videoup-report/conf:go_default_library",
+        "//app/job/main/videoup-report/dao/archive:go_default_library",
+        "//app/job/main/videoup-report/dao/data:go_default_library",
+        "//app/job/main/videoup-report/dao/email:go_default_library",
+        "//app/job/main/videoup-report/dao/hbase:go_default_library",
+        "//app/job/main/videoup-report/dao/manager:go_default_library",
+        "//app/job/main/videoup-report/dao/mission:go_default_library",
+        "//app/job/main/videoup-report/dao/redis:go_default_library",
+        "//app/job/main/videoup-report/dao/tag:go_default_library",
+        "//app/job/main/videoup-report/model/archive:go_default_library",
+        "//app/job/main/videoup-report/model/email:go_default_library",
+        "//app/job/main/videoup-report/model/manager:go_default_library",
+        "//app/job/main/videoup-report/model/monitor:go_default_library",
+        "//app/job/main/videoup-report/model/task:go_default_library",
+        "//app/job/main/videoup-report/model/utils:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/up/api/v1:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/report:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/videoup-report/service/archive.go

@@ -0,0 +1,545 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"fmt"
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+// hdlArchiveMessage deal with archive action
+func (s *Service) hdlArchiveMessage(action string, nwMsg []byte, oldMsg []byte) {
+	var (
+		err    error
+		arc    = &archive.Archive{}
+		oldArc = &archive.Archive{}
+	)
+	if action != _updateAct {
+		return
+	}
+	if err = json.Unmarshal(nwMsg, arc); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", nwMsg, err)
+		return
+	}
+	if err = json.Unmarshal(oldMsg, oldArc); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", oldMsg, err)
+		return
+	}
+	if arc.TypeID != oldArc.TypeID {
+		s.hdlMoveType(arc, oldArc)
+	}
+	if arc.Round != oldArc.Round {
+		s.hdlRoundFlow(arc, oldArc)
+	}
+	if arc.State != oldArc.State && arc.State == archive.StateForbidUpDelete {
+		s.arc.DelDispatchByAid(context.TODO(), arc.ID)
+	}
+}
+
+// hdlMoveType deal with archive move typeid
+func (s *Service) hdlMoveType(arc *archive.Archive, oldArc *archive.Archive) {
+	if _, ok := archive.ReportArchiveRound[arc.Round]; !ok {
+		return
+	}
+	s.arcMoveTypeCache.Lock()
+	defer s.arcMoveTypeCache.Unlock()
+	if _, ok := s.arcMoveTypeCache.Data[arc.Round]; !ok {
+		s.arcMoveTypeCache.Data[arc.Round] = make(map[int16]map[string]int)
+	}
+	if _, ok := s.arcMoveTypeCache.Data[arc.Round][oldArc.TypeID]; !ok {
+		s.arcMoveTypeCache.Data[arc.Round][oldArc.TypeID] = make(map[string]int)
+	}
+	if _, ok := s.arcMoveTypeCache.Data[arc.Round][arc.TypeID]; !ok {
+		s.arcMoveTypeCache.Data[arc.Round][arc.TypeID] = make(map[string]int)
+	}
+	s.arcMoveTypeCache.Data[arc.Round][oldArc.TypeID]["out"]++
+	s.arcMoveTypeCache.Data[arc.Round][arc.TypeID]["in"]++
+}
+
+// hdlRoundFlow deal with archive round flow
+func (s *Service) hdlRoundFlow(arc *archive.Archive, oldArc *archive.Archive) {
+	var (
+		oper    *archive.Oper
+		newOper *archive.Oper
+		err     error
+	)
+	if _, ok := archive.ReportArchiveRound[oldArc.Round]; !ok {
+		return
+	}
+	if oper, err = s.arc.LastRoundOper(context.TODO(), oldArc.ID, oldArc.Round); err != nil {
+		log.Error("s.LastRoundOper(%d,%d) 获取archive_oper记录失败 error(%v)", oldArc.ID, oldArc.Round, err)
+		return
+	}
+	if newOper, err = s.arc.NextRoundOper(context.TODO(), oper.ID, oldArc.ID, oldArc.Round); err != nil {
+		log.Error("s.NextRoundOper(%d,%d,%d) 获取archive_oper记录失败 error(%v)", oper.ID, oldArc.ID, oldArc.Round, err)
+		return
+	}
+	s.arcRoundFlowCache.Lock()
+	defer s.arcRoundFlowCache.Unlock()
+	if _, ok := s.arcRoundFlowCache.Data[oldArc.Round]; !ok {
+		s.arcRoundFlowCache.Data[oldArc.Round] = make(map[int64]map[string]int)
+	}
+	if _, ok := s.arcRoundFlowCache.Data[oldArc.Round][oldArc.ID]; !ok {
+		s.arcRoundFlowCache.Data[oldArc.Round][oldArc.ID] = make(map[string]int)
+	}
+	s.arcRoundFlowCache.Data[oldArc.Round][oldArc.ID]["take_time"] = int(newOper.MTime.Unix() - oper.MTime.Unix())
+	s.arcRoundFlowCache.Data[oldArc.Round][oldArc.ID]["uid"] = int(newOper.UID)
+	log.Info("s.arcRoundFlowCache.Data: %v", s.arcRoundFlowCache.Data)
+}
+
+// hdlMoveTypeCount check and write archive move type stats to db
+func (s *Service) hdlMoveTypeCount() {
+	var (
+		report *archive.Report
+		err    error
+		bs     []byte
+		ctime  = time.Now()
+		mtime  = ctime
+	)
+	s.arcMoveTypeCache.Lock()
+	defer s.arcMoveTypeCache.Unlock()
+	if len(s.arcMoveTypeCache.Data) < 1 {
+		log.Info("s.hdlMoveTypeCount() 统计内容为空，忽略：%v", s.arcMoveTypeCache.Data)
+		return
+	}
+	if report, err = s.arc.ReportLast(context.TODO(), archive.ReportTypeArcMoveType); err != nil {
+		log.Error("s.arc.ReportLast(%d) error(%v)", archive.ReportTypeArcMoveType, err)
+		return
+	}
+	if report != nil && time.Now().Unix()-report.CTime.Unix() < 5*60 {
+		log.Info("s.arc.ReportLast(%d) 距离上一次写入还没过5分钟!", archive.ReportTypeArcMoveType)
+		return
+	}
+	if bs, err = json.Marshal(s.arcMoveTypeCache.Data); err != nil {
+		log.Error("json.Marshal(%v) error(%v)", s.arcMoveTypeCache.Data, err)
+		return
+	}
+	if _, err = s.arc.ReportAdd(context.TODO(), archive.ReportTypeArcMoveType, string(bs), ctime, mtime); err != nil {
+		log.Error("s.arc.ReportAdd(%d,%s,%v,%v) error(%v)", archive.ReportTypeArcMoveType, string(bs), ctime, mtime, err)
+		return
+	}
+	s.arcMoveTypeCache.Data = make(map[int8]map[int16]map[string]int)
+}
+
+// hdlRoundFlowCount check and write archive round flow stats to db
+func (s *Service) hdlRoundFlowCount() {
+	var (
+		report *archive.Report
+		err    error
+		bs     []byte
+		ctime  = time.Now()
+		mtime  = ctime
+	)
+	s.arcRoundFlowCache.Lock()
+	defer s.arcRoundFlowCache.Unlock()
+	if len(s.arcRoundFlowCache.Data) < 1 {
+		log.Info("s.hdlRoundFlowCount() 统计内容为空，忽略：%v", s.arcRoundFlowCache.Data)
+		return
+	}
+	if report, err = s.arc.ReportLast(context.TODO(), archive.ReportTypeArcRoundFlow); err != nil {
+		log.Error("s.arc.ReportLast(%d) error(%v)", archive.ReportTypeArcRoundFlow, err)
+		return
+	}
+	if report != nil && time.Now().Unix()-report.CTime.Unix() < 5*60 {
+		log.Info("s.arc.ReportLast(%d) 距离上一次写入还没过5分钟!", archive.ReportTypeArcRoundFlow)
+		return
+	}
+	if bs, err = json.Marshal(s.arcRoundFlowCache.Data); err != nil {
+		log.Error("json.Marshal(%v) error(%v)", s.arcRoundFlowCache.Data, err)
+		return
+	}
+	if _, err = s.arc.ReportAdd(context.TODO(), archive.ReportTypeArcRoundFlow, string(bs), ctime, mtime); err != nil {
+		log.Error("s.arc.ReportAdd(%d,%s,%v,%v) error(%v)", archive.ReportTypeArcRoundFlow, string(bs), ctime, mtime, err)
+		return
+	}
+	s.arcRoundFlowCache.Data = make(map[int8]map[int64]map[string]int)
+}
+
+// MoveType get archive move type stats by typeid
+func (s *Service) MoveType(c context.Context, stime, etime time.Time) (reports []*archive.Report, err error) {
+	if reports, err = s.arc.Reports(c, archive.ReportTypeArcMoveType, stime, etime); err != nil {
+		log.Error("s.arc.Reports(%d) err(%v)", archive.ReportTypeArcMoveType, err)
+		return
+	}
+	return
+}
+
+// RoundFlow get archive round flow take time records
+func (s *Service) RoundFlow(c context.Context, stime, etime time.Time) (reports []*archive.Report, err error) {
+	if reports, err = s.arc.Reports(c, archive.ReportTypeArcRoundFlow, stime, etime); err != nil {
+		log.Error("s.arc.Reports(%d) err(%v)", archive.ReportTypeArcRoundFlow, err)
+		return
+	}
+	return
+}
+
+func (s *Service) arcUpdateproc(k int) {
+	defer s.waiter.Done()
+	for {
+		var (
+			upInfo *archive.UpInfo
+			ok     bool
+		)
+		if upInfo, ok = <-s.arcUpChs[k]; !ok {
+			log.Info("s.arcUpChs[k] closed", k)
+			return
+		}
+		go s.hdlExcitation(upInfo.Nw, upInfo.Old)
+		go s.hdlMonitorArc(upInfo.Nw, upInfo.Old)
+		s.trackArchive(upInfo.Nw, upInfo.Old)
+		go s.arcStateChange(upInfo.Nw, upInfo.Old, true)
+	}
+}
+
+func (s *Service) putArcChan(action string, nwMsg []byte, oldMsg []byte) {
+	var (
+		err      error
+		chanSize = int64(s.c.ChanSize)
+	)
+	nw := &archive.Archive{}
+	if err = json.Unmarshal(nwMsg, nw); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", nwMsg, err)
+		return
+	}
+	switch action {
+	case _insertAct:
+		s.arcUpChs[nw.ID%chanSize] <- &archive.UpInfo{Nw: nw, Old: nil}
+	case _updateAct:
+		old := &archive.Archive{}
+		if err = json.Unmarshal(oldMsg, old); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", oldMsg, err)
+			return
+		}
+		s.arcUpChs[nw.ID%chanSize] <- &archive.UpInfo{Nw: nw, Old: old}
+	}
+}
+
+// secondRound 接收到databus的second_round消息。
+func (s *Service) secondRound(c context.Context, m *archive.VideoupMsg) (err error) {
+	var (
+		a *archive.Archive
+	)
+	if a, err = s.arc.ArchiveByAid(c, m.Aid); err != nil || a.ID <= 0 {
+		log.Error("secondRound s.arc.ArchiveByAid error(%v)/not found aid(%d)", err, m.Aid)
+		return
+	}
+	s.dealFromList(c, m)
+	s.dealMissionTag(c, m, a)
+	//开评论逻辑判断
+	s.arcStateChange(a, nil, true)
+	if archive.NormalState(a.State) {
+		s.adminBindTag(c, a.Mid, a.ID, a.Tag, a.TypeID)
+	}
+	//邮件发送开关
+	if m.SendEmail {
+		if m.AdminChange && !s.isPGC(a.ID) {
+			s.sendMail(c, a, nil)
+		}
+		s.sendArchivePrivateEmail(c, a)
+	}
+
+	return
+}
+
+// dealFromList 处理from list流程
+func (s *Service) dealFromList(c context.Context, m *archive.VideoupMsg) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.dealFromList panic(%v)", pErr)
+		}
+	}()
+	switch m.FromList {
+	case archive.FromListHotReview: //热门回查
+		var (
+			has   = false
+			state = archive.RecheckStateWait
+		)
+		//查询flow_design中是否存在禁止
+		if has, err = s.arc.HasFlowGroup(c, archive.FlowPoolRecheck, archive.FlowGroupIDHot, m.Aid); err != nil {
+			log.Error("s.updateRecheckState(%d,%d,%d,%d) error(%v)", archive.TypeHotRecheck, archive.FlowPoolRecheck, archive.FlowGroupIDHot, m.Aid, err)
+			return
+		}
+		if has {
+			state = archive.RecheckStateForbid
+		} else {
+			state = archive.RecheckStateNoForbid
+		}
+		s.updateRecheckState(c, archive.TypeHotRecheck, m.Aid, state)
+	case archive.FromListExcitation: //激励回查
+		s.updateRecheckState(c, archive.TypeExcitationRecheck, m.Aid, archive.RecheckStateNoForbid)
+	default:
+		log.Warn("Unknown message from_list (%s)", m.FromList)
+	}
+	return
+}
+
+// dealMissionTag 处理活动tag
+func (s *Service) dealMissionTag(c context.Context, m *archive.VideoupMsg, a *archive.Archive) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.dealMissionTag panic(%v)", pErr)
+		}
+	}()
+	if m.MissionID != 0 { //消息里的mission_id是修改之前的
+		addit, err := s.arc.Addit(c, m.Aid)
+		if err != nil {
+			log.Error("s.arc.Addit(%d) error(%v)", m.Aid, err)
+		} else if addit == nil {
+			log.Warn("s.arc.Addit(%d) warn(addit is nil)", m.Aid)
+		} else if addit.MissionID == 0 {
+			//取消活动资格，去掉活动tag
+			tags, err := s.removeMissionTags(c, a)
+			if err == nil {
+				a.Tag = strings.Join(tags, ",")
+			}
+		}
+	}
+	return
+}
+
+// updateRecheckState 回查提交时的事件
+func (s *Service) updateRecheckState(c context.Context, tp int, aid int64, state int8) (err error) {
+
+	//修改archive_recheck的state
+	if err = s.arc.UpdateRecheckState(c, tp, aid, state); err != nil {
+		return
+	}
+	a, err := s.arc.ArchiveByAid(c, aid)
+	if err != nil {
+		log.Error("s.arc.ArchiveByAid error(%v)", err)
+		err = nil
+		return
+	}
+	tpStr := archive.RecheckType(tp)
+	if tpStr != "" {
+		s.arc.AddArchiveOper(c, aid, a.Attribute, a.TypeID, a.State, a.Round, 0, "", "已"+tpStr)
+	}
+	return
+}
+
+// addHotRecheck get hot archive from data api, and insert to archive_recheck table
+func (s *Service) addHotRecheck() (err error) {
+	var (
+		c    = context.TODO()
+		aids []int64
+	)
+	if aids, err = s.dataDao.HotArchive(c); err != nil {
+		log.Error("s.addHotRecheck() s.dataDao.HotArchive() error(%v)", err)
+		return
+	}
+	if err = s.arc.AddRecheckAids(c, archive.TypeHotRecheck, aids, true); err != nil {
+		log.Error("s.addHotRecheck() s.arc.AddRecheckAids error(%v)", err)
+		return
+	}
+	return
+}
+
+func (s *Service) addArchive(c context.Context, m *archive.VideoupMsg) (err error) {
+	var (
+		a                       *archive.Archive
+		addit                   *archive.Addit
+		tx                      *sql.Tx
+		channelDiff, operRemark string
+		operConts               []string
+	)
+
+	if a, err = s.arc.ArchiveByAid(c, m.Aid); err != nil || a.ID <= 0 {
+		log.Error("addArchive s.arc.ArchiveByAid error(%v)/not found aid(%d)", err, m.Aid)
+		return
+	}
+	//同步到tag服务方，以便在前台显示
+	if err = s.upBindTag(c, a.Mid, m.Aid, a.Tag, a.TypeID); err != nil {
+		return
+	}
+	if addit, err = s.arc.Addit(c, m.Aid); err != nil {
+		log.Error("modifyArchive s.arc.Addit error(%v) aid(%d)", err, m.Aid)
+		return
+	}
+	if tx, err = s.arc.BeginTran(c); err != nil {
+		log.Error("addArchive s.arc.BeginTran error(%v)", err)
+		return
+	}
+	//非活动的ugc稿件
+	if addit == nil || (!addit.IsPGC() && addit.MissionID <= 0) {
+		if channelDiff, operRemark, err = s.txAddChannelReview(c, tx, m.Aid); err != nil {
+			log.Error("addArchive s.txAddChannelReview(%d) error(%v)", m.Aid, err)
+			tx.Rollback()
+			return
+		}
+		if channelDiff != "" {
+			operConts = append(operConts, channelDiff)
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("addArchive tx.Commit() error(%v) aid(%d)", err, m.Aid)
+		return
+	}
+	if len(operConts) > 0 && operRemark != "" {
+		s.arc.AddArchiveOper(c, m.Aid, a.Attribute, a.TypeID, a.State, a.Round, 0, strings.Join(operConts, ","), operRemark)
+	}
+	return
+}
+
+func (s *Service) modifyArchive(c context.Context, m *archive.VideoupMsg) (err error) {
+	var (
+		a                       *archive.Archive
+		addit                   *archive.Addit
+		tx                      *sql.Tx
+		channelDiff, operRemark string
+		operConts               []string
+	)
+
+	if a, err = s.arc.ArchiveByAid(c, m.Aid); err != nil || a.ID <= 0 {
+		log.Error("modifyArchive s.arc.ArchiveByAid error(%v)/not found aid(%d)", err, m.Aid)
+		return
+	}
+	//tag修改或分区修改时，同步到tag服务方，以便在前台显示,即使失败也不影响后续
+	if m.TagChange || m.ChangeTypeID {
+		s.upBindTag(c, a.Mid, m.Aid, a.Tag, a.TypeID)
+	}
+
+	if addit, err = s.arc.Addit(c, m.Aid); err != nil {
+		log.Error("modifyArchive s.arc.Addit error(%v) aid(%d)", err, m.Aid)
+		return
+	}
+	if tx, err = s.arc.BeginTran(c); err != nil {
+		log.Error("modifyArchive s.arc.BeginTran error(%v)", err)
+		return
+	}
+	//新增视频 且 非活动的ugc稿件
+	if m.AddVideos && (addit == nil || (!addit.IsPGC() && addit.MissionID <= 0)) {
+		log.Info("begin to check channel review aid(%d)", m.Aid)
+		if channelDiff, operRemark, err = s.txAddChannelReview(c, tx, m.Aid); err != nil {
+			log.Error("modifyArchive s.txAddChannelReview(%d) error(%v)", m.Aid, err)
+			tx.Rollback()
+			return
+		}
+		if channelDiff != "" {
+			operConts = append(operConts, channelDiff)
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("modifyArchive tx.Commit() error(%v) aid(%d)", err, m.Aid)
+		return
+	}
+	if len(operConts) > 0 && operRemark != "" {
+		s.arc.AddArchiveOper(c, m.Aid, a.Attribute, a.TypeID, a.State, a.Round, 0, strings.Join(operConts, ","), operRemark)
+	}
+	return
+}
+
+func (s *Service) autoOpen(c context.Context, m *archive.VideoupMsg) (err error) {
+	var (
+		a *archive.Archive
+	)
+	if a, err = s.arc.ArchiveByAid(c, m.Aid); err != nil || a.ID <= 0 {
+		log.Error("autoOpen s.arc.ArchiveByAid error(%v)/not found aid(%d)", err, m.Aid)
+		return
+	}
+	s.adminBindTag(c, a.Mid, a.ID, a.Tag, a.TypeID)
+	return
+}
+
+func (s *Service) delayOpen(c context.Context, m *archive.VideoupMsg) (err error) {
+	var (
+		a *archive.Archive
+	)
+	if a, err = s.arc.ArchiveByAid(c, m.Aid); err != nil || a.ID <= 0 {
+		log.Error("autoOpen s.arc.ArchiveByAid error(%v)/not found aid(%d)", err, m.Aid)
+		return
+	}
+	s.adminBindTag(c, a.Mid, a.ID, a.Tag, a.TypeID)
+	return
+}
+
+func (s *Service) postFirstRound(c context.Context, m *archive.VideoupMsg) (err error) {
+	var (
+		v *archive.Video
+		a *archive.Archive
+	)
+	if v, err = s.arc.NewVideo(c, m.Filename); err != nil || v == nil {
+		log.Error("postFirstRound s.arc.NewVideo error(%v)/not found filename(%d)", err, m.Filename)
+		return
+	}
+	if a, err = s.arc.ArchiveByAid(c, m.Aid); err != nil || a.ID <= 0 {
+		log.Error("postFirstRound s.arc.ArchiveByAid error(%v)/not found aid(%d)", err, m.Aid)
+		return
+	}
+	if a.State == archive.StateForbidUpDelete {
+		log.Warn("postFirstRound archive(%d) filename(%s) state(%d) is deleted", a.ID, v.Filename, a.State)
+		return
+	}
+
+	if m.AdminChange && !s.isPGC(a.ID) {
+		s.sendMail(c, a, v)
+	}
+	s.sendVideoPrivateEmail(c, a, v)
+	return
+}
+
+func (s *Service) isPGC(aid int64) (is bool) {
+	is = false
+	if ad, _ := s.arc.Addit(context.TODO(), aid); ad != nil && ad.IsPGC() {
+		is = true
+	}
+	return
+}
+
+func (s *Service) arcStateChange(nw *archive.Archive, old *archive.Archive, canOpen bool) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.arcStateChange panic(%v)", pErr)
+		}
+	}()
+	if nw == nil {
+		return
+	}
+
+	oldValue := 0
+	if old != nil {
+		oldValue = isOpenReplyState(old.State)
+	}
+	switchVal := isOpenReplyState(nw.State) - oldValue
+	//关评论
+	if switchVal < 0 && !canOpen {
+		s.arcReply(context.TODO(), nw, archive.ReplyOff)
+	}
+	//开评论
+	if switchVal > 0 && canOpen {
+		s.arcReply(context.TODO(), nw, archive.ReplyOn)
+	}
+
+	return
+}
+
+//removeMissionTags 删除活动tag
+func (s *Service) removeMissionTags(c context.Context, a *archive.Archive) (tags []string, err error) {
+	tags = strings.Split(a.Tag, ",")
+
+	for i := 0; i < len(tags); i++ {
+		if _, ok := s.missTagsCache[tags[i]]; ok {
+			tags = append(tags[:i], tags[i+1:]...)
+			i--
+			continue
+		}
+	}
+	tagStr := strings.Join(tags, ",")
+	if err = s.adminBindTag(c, a.Mid, a.ID, tagStr, a.TypeID); err != nil {
+		log.Error("removeMissionTags(%v) s.adminBindTag() error(%v)", a, err)
+		return
+	}
+	if _, err = s.arc.UpTag(c, a.ID, tagStr); err != nil {
+		log.Error("s.arc.UpTag(%d,%s) error(%v)", a.ID, tagStr, err)
+		err = nil
+	}
+	if _, err = s.arc.AddArchiveOper(c, a.ID, a.Attribute, a.TypeID, a.State, a.Round, 0, fmt.Sprintf("[Tag]从[%s]设为[%s]", a.Tag, tagStr), "因被取消活动资格"); err != nil {
+		log.Error("s.arc.AddArchiveOper() archive(%v) error(%v)", a, err)
+		err = nil
+	}
+	return
+}

app/job/main/videoup-report/service/archive_result.go

@@ -0,0 +1,58 @@
+package service
+
+import (
+	"encoding/json"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/library/log"
+)
+
+func (s *Service) arcResultConsume() {
+	defer s.waiter.Done()
+	var (
+		err  error
+		msgs = s.arcResultSub.Messages()
+	)
+	for {
+		msg, open := <-msgs
+		if !open {
+			log.Info("arcResultConsume s.arcResultSub.Messages is closed")
+			return
+		}
+		msg.Commit()
+		if msg == nil {
+			continue
+		}
+		log.Info("arcResultConsume consume key(%s) offset(%d) value(%s)", msg.Key, msg.Offset, string(msg.Value))
+
+		m := &archive.Message{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("arcResultConsume json.Unmarshal error(%v)", err)
+			continue
+		}
+		if m.Table != _archive {
+			continue
+		}
+
+		nw := &archive.Archive{}
+		if err = json.Unmarshal(m.New, nw); err != nil {
+			log.Error("arcResultConsume json.Unmarshal error(%v) msg.new(%s)", err, string(m.New))
+			continue
+		}
+		nw.ID = nw.AID
+
+		var old *archive.Archive
+		if m.Action == _insertAct {
+			old = nil
+		} else if m.Action == _updateAct {
+			old = &archive.Archive{}
+			if err = json.Unmarshal(m.Old, old); err != nil {
+				log.Error("arcResultConsume json.Unmarshal error(%v) msg.old(%s)", err, string(m.Old))
+				continue
+			}
+			old.ID = old.AID
+		}
+
+		go s.arcStateChange(nw, old, false)
+	}
+}

app/job/main/videoup-report/service/archive_test.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+)
+
+func TestService_Addarchive(t *testing.T) {
+	Convey("addarchive", t, func() {
+		err := s.addArchive(context.Background(), &archive.VideoupMsg{
+			Aid: 10,
+		})
+		t.Logf("err(%+v)", err)
+	})
+}
+
+func TestService_Modifyarchive(t *testing.T) {
+	Convey("Modifyarchive", t, func() {
+		err := s.modifyArchive(context.Background(), &archive.VideoupMsg{
+			Aid:       11,
+			TagChange: true,
+			AddVideos: true,
+		})
+		t.Logf("err(%+v)", err)
+	})
+}
+
+func TestService_arcStateChange(t *testing.T) {
+	var (
+		err   error
+		state int64
+		aid   = int64(12)
+	)
+	Convey("arcStateChange", t, func() {
+		a, _ := s.arc.ArchiveByAid(context.TODO(), aid)
+		nw := &archive.Archive{
+			ID:    a.ID,
+			State: archive.StateOpen,
+		}
+		old := &archive.Archive{
+			ID:    a.ID,
+			State: archive.StateForbidRecicle,
+		}
+		err = s.dataDao.CloseReply(context.TODO(), a.ID, a.Mid)
+		So(err, ShouldBeNil)
+
+		//只允许关的情况下，状态联动从关-》开不起作用
+		err = s.arcStateChange(nw, old, false)
+		So(err, ShouldBeNil)
+		state, err = s.dataDao.CheckReply(context.TODO(), nw.ID)
+		So(err, ShouldBeNil)
+		So(state, ShouldEqual, archive.ReplyOff)
+
+		//只允许开的情况下，状态联动从关->开起作用
+		err = s.arcStateChange(nw, old, true)
+		So(err, ShouldBeNil)
+		state, err = s.dataDao.CheckReply(context.TODO(), nw.ID)
+		So(err, ShouldBeNil)
+		So(state, ShouldEqual, archive.ReplyOn)
+
+		//只允许开的情况下，状态联动从开->关不起作用
+		err = s.arcStateChange(old, nw, true)
+		So(err, ShouldBeNil)
+		state, err = s.dataDao.CheckReply(context.TODO(), nw.ID)
+		So(err, ShouldBeNil)
+		So(state, ShouldEqual, archive.ReplyOn)
+
+		//只允许关的情况下，状态联动从开->关起作用
+		err = s.arcStateChange(old, nw, false)
+		So(err, ShouldBeNil)
+		state, err = s.dataDao.CheckReply(context.TODO(), nw.ID)
+		So(err, ShouldBeNil)
+		So(state, ShouldEqual, archive.ReplyOff)
+	})
+}

app/job/main/videoup-report/service/databus.go

@@ -0,0 +1,135 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/manager"
+	"go-common/library/log"
+)
+
+var (
+	_archive   = "archive"
+	_video     = "archive_video"
+	_insertAct = "insert"
+	_updateAct = "update"
+	//_delete    = "delete"
+)
+
+// consumer binlog
+func (s *Service) arcCanalConsume() {
+	defer s.waiter.Done()
+	var (
+		msgs = s.archiveSub.Messages()
+		err  error
+	)
+	for {
+		msg, ok := <-msgs
+		if !ok {
+			log.Error("s.archiveSub.Message closed", err)
+			return
+		}
+		msg.Commit()
+		m := &archive.Message{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		log.Info("arcCanalConsume msg(%s)", msg.Value)
+		log.Info("arcCanalConsume topic(%s) partition(%d) offset(%d)  commit start", msg.Topic, msg.Partition, msg.Offset)
+		if msg.Offset >= s.c.BeginOffset {
+			log.Info("arcCanalConsume offset(%d) is hit BeginOffset(%d) and start track data", msg.Offset, s.c.BeginOffset)
+			if m.Table == _archiveTable {
+				s.putArcChan(m.Action, m.New, m.Old)
+			}
+			if m.Table == _videoTable {
+				s.putVideoChan(m.Action, m.New, m.Old)
+			}
+		} else {
+			log.Info("arcCanalConsume offset(%d) not hit BeginOffset(%d) and pass", msg.Offset, s.c.BeginOffset)
+		}
+		//todo 异步消费
+		if m.Table == _video && m.Action == _updateAct {
+			s.hdlVideoUpdateBinLog(m.New, m.Old)
+		}
+		if m.Table == _archive {
+			s.hdlArchiveMessage(m.Action, m.New, m.Old)
+		}
+	}
+}
+
+func (s *Service) videoupConsumer() {
+	defer s.waiter.Done()
+	var (
+		msgs = s.videoupSub.Messages()
+		err  error
+		c    = context.TODO()
+	)
+	for {
+		msg, ok := <-msgs
+		if !ok {
+			log.Error("s.videoupSub.Messages closed")
+			return
+		}
+		msg.Commit()
+		m := &archive.VideoupMsg{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
+			return
+		}
+		log.Info("videoupMessage key(%s) value(%s) partition(%d) offset(%d) route(%s) commit start", msg.Key, msg.Value, msg.Partition, msg.Offset, m.Route)
+		switch m.Route {
+		case archive.RoutePostFirstRound:
+			err = s.postFirstRound(c, m)
+		case archive.RouteSecondRound:
+			err = s.secondRound(c, m)
+		case archive.RouteAddArchive:
+			err = s.addArchive(c, m)
+		case archive.RouteModifyArchive:
+			err = s.modifyArchive(c, m)
+		case archive.RouteAutoOpen:
+			err = s.autoOpen(c, m)
+		case archive.RouteDelayOpen:
+			err = s.delayOpen(c, m)
+		default:
+			log.Warn("videoupConsumer unknown message route(%s)", m.Route)
+		}
+		if err == nil {
+			log.Info("videoupMessage key(%s) value(%s) partition(%d) offset(%d) end", msg.Key, msg.Value, msg.Partition, msg.Offset)
+		} else {
+			log.Error("videoupMessage key(%s) value(%s) partition(%d) offset(%d) error(%v)", msg.Key, msg.Value, msg.Partition, msg.Offset, err)
+		}
+	}
+}
+
+// managerDBConsume 消费manager binlog
+func (s *Service) managerDBConsume() {
+	defer s.waiter.Done()
+	var (
+		err  error
+		msgs = s.ManagerDBSub.Messages()
+	)
+	for {
+		msg, open := <-msgs
+		if !open {
+			log.Info("managerDBConsume s.arcResultSub.Messages is closed")
+			return
+		}
+		if msg == nil {
+			continue
+		}
+		msg.Commit()
+		log.Info("managerDBConsume consume key(%s) offset(%d) value(%s)", msg.Key, msg.Offset, string(msg.Value))
+
+		m := &manager.BinMsg{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("managerDBConsume json.Unmarshal error(%v)", err)
+			continue
+		}
+		switch m.Table {
+		case _upsTable:
+			s.hdlManagerUpsBinlog(m)
+		}
+	}
+}

app/job/main/videoup-report/service/email.go

@@ -0,0 +1,322 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	emailmdl "go-common/app/job/main/videoup-report/model/email"
+	"go-common/app/job/main/videoup-report/model/manager"
+	account "go-common/app/service/main/account/api"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+//一审备注中有"私单报备"， 则发送邮件
+func (s *Service) sendVideoPrivateEmail(c context.Context, a *archive.Archive, v *archive.Video) (err error) {
+	if a == nil || v == nil {
+		return
+	}
+	note, err := s.arc.VideoAuditNote(c, v.ID)
+	if err != nil {
+		log.Error("sendVideoPrivateEmail s.arc.VideoAuditNote error(%v)", err)
+		return
+	}
+	if !s.needPrivateEmail(a.TypeID, note) {
+		log.Info("sendVideoPrivateEmail no need to send email: vid(%d), aid(%d), note(%s), typeId(%d)", v.ID, v.Aid, note, a.TypeID)
+		return
+	}
+
+	log.Info("start to sendVideoPrivateEmail: note(%s), typeId(%d), aid(%d), video(%v), archive(%v)", note, a.TypeID, a.ID, v, a)
+	//up主信息
+	pfl, err := s.profile(c, a.Mid)
+	if err != nil || pfl == nil {
+		log.Error("sendVideoPrivateEmail s.profile error(%v) or nil, mid(%d), vid(%d), aid(%d)", err, a.Mid, v.ID, a.ID)
+		return
+	}
+
+	//审核人员信息
+	mngUID, err := s.arc.LastVideoOperUID(c, v.ID)
+	if err != nil || mngUID <= 0 {
+		log.Error("sendVideoPrivateEmail s.arc.LastVideoOperUID error(%v) or zero(%d) vid(%d) aid(%d)", err, mngUID, v.ID, v.Aid)
+		return
+	}
+	mngUser, err := s.mng.User(c, mngUID)
+	if err != nil || mngUser == nil || mngUser.ID != mngUID {
+		log.Error("sendVideoPrivateEmail s.mng.User(%d) error(%v) or not found(%+v) uid(%d)", mngUID, err, mngUser)
+		return
+	}
+
+	//禁止项状态
+	noRankAttr, noDynamicAttr, noRecommend := "关", "关", "关"
+	if v.AttrVal(archive.AttrBitNoRank) == archive.AttrYes {
+		noRankAttr = "开"
+	}
+	if v.AttrVal(archive.AttrBitNoDynamic) == archive.AttrYes {
+		noDynamicAttr = "开"
+	}
+	if v.AttrVal(archive.AttrBitNoRecommend) == archive.AttrYes {
+		noRecommend = "开"
+	}
+
+	//组合邮件参数
+	params := map[string]string{
+		"upName":          pfl.Profile.Name,
+		"aid":             strconv.FormatInt(v.Aid, 10),
+		"arcTitle":        a.Title,
+		"arcState":        archive.StateMean[a.State],
+		"noRankAttr":      noRankAttr,
+		"noDynamicAttr":   noDynamicAttr,
+		"noRecommendAttr": noRecommend,
+		"upFans":          strconv.FormatInt(pfl.Follower, 10),
+		"uid":             strconv.FormatInt(mngUID, 10),
+		"mngName":         mngUser.Username,
+		"mngDepartment":   mngUser.Department,
+		"note":            note,
+		"typeId":          strconv.Itoa(int(s.topType(a.TypeID))),
+		"emailType":       emailmdl.EmailPrivateVideo,
+	}
+	tpl := s.email.PrivateEmailTemplate(params)
+	s.email.PushToRedis(c, tpl)
+	return
+}
+
+//稿件备注中有"私单报备"， 则发送邮件
+func (s *Service) sendArchivePrivateEmail(c context.Context, a *archive.Archive) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.sendArchivePrivateEmail panic(%v)", pErr)
+		}
+	}()
+	if a == nil {
+		return
+	}
+	note, err := s.arc.ArchiveNote(c, a.ID)
+	if err != nil {
+		log.Error("sendArchivePrivateEmail s.arc.ArchiveNote error(%v) aid(%d)", err, a.ID)
+		return
+	}
+	if !s.needPrivateEmail(a.TypeID, note) {
+		log.Info("sendArchivePrivateEmail: no need to send email: aid(%d), note(%s), typeId(%d)", a.ID, note, a.TypeID)
+		return
+	}
+
+	log.Info("start to sendArchivePrivateEmail: note(%s), typeId(%d), aid(%d), archive(%v)", note, a.TypeID, a.ID, a)
+	//up主信息
+	pfl, err := s.profile(c, a.Mid)
+	if err != nil || pfl == nil {
+		log.Error("sendArchivePrivateEmail s.profile error(%v) or nil, mid(%d), aid(%d)", err, a.Mid, a.ID)
+		return
+	}
+
+	//审核人员信息
+	arcOper, err := s.arc.LastArcOper(c, a.ID)
+	if err != nil || arcOper == nil || arcOper.AID <= 0 {
+		log.Error("sendArchivePrivateEmail s.arc.LastArcOper(%d), error(%v) or not found(%+v)", a.ID, err, arcOper)
+		return
+	}
+	mngUser, err := s.mng.User(c, arcOper.UID)
+	if err != nil || mngUser == nil || mngUser.ID != arcOper.UID {
+		log.Error("sendArchivePrivateEmail s.mng.User error(%v) or not found(%+v), aid(%d), uid(%d)", err, mngUser, a.ID, arcOper.UID)
+		return
+	}
+
+	//禁止项状态
+	noRankAttr, noDynamicAttr, noRecommend := "关", "关", "关"
+	if a.AttrVal(archive.AttrBitNoRank) == archive.AttrYes {
+		noRankAttr = "开"
+	}
+	if a.AttrVal(archive.AttrBitNoDynamic) == archive.AttrYes {
+		noDynamicAttr = "开"
+	}
+	if a.AttrVal(archive.AttrBitNoRecommend) == archive.AttrYes {
+		noRecommend = "开"
+	}
+
+	//组合邮件参数
+	params := map[string]string{
+		"upName":          pfl.Profile.Name,
+		"aid":             strconv.FormatInt(a.ID, 10),
+		"arcTitle":        a.Title,
+		"arcState":        archive.StateMean[a.State],
+		"noRankAttr":      noRankAttr,
+		"noDynamicAttr":   noDynamicAttr,
+		"noRecommendAttr": noRecommend,
+		"upFans":          strconv.FormatInt(pfl.Follower, 10),
+		"uid":             strconv.FormatInt(mngUser.ID, 10),
+		"mngName":         mngUser.Username,
+		"mngDepartment":   mngUser.Department,
+		"note":            note,
+		"typeId":          strconv.Itoa(int(s.topType(a.TypeID))),
+		"emailType":       emailmdl.EmailPrivateArchive,
+	}
+	tpl := s.email.PrivateEmailTemplate(params)
+	s.email.PushToRedis(c, tpl)
+	return
+}
+
+//触发私单报备邮件的条件
+func (s *Service) needPrivateEmail(typeID int16, note string) (matched bool) {
+	typeI := int(s.topType(typeID))
+	_, exist := s.email.PrivateAddr[strconv.Itoa(typeI)]
+	matched = exist && strings.Contains(note, "私单报备")
+	return
+}
+
+func (s *Service) sendMail(c context.Context, a *archive.Archive, v *archive.Video) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.sendMail panic(%v)", pErr)
+		}
+	}()
+	var (
+		condition, additionCondition, content []string
+		mngUser                               *manager.User
+		pfl                                   *account.ProfileStatReply
+		remark, additionType                  string
+		mngUID                                int64
+		videoOper                             *archive.VideoOper
+		arcOper                               *archive.Oper
+	)
+	//触发条件
+	if s.isSigned(a.Mid) {
+		additionCondition = append(additionCondition, "签约UP主")
+		additionType = "signed"
+	}
+	if s.isWhite(a.Mid) {
+		condition = append(condition, "优质UP主")
+	}
+	if s.isPolitices(a.Mid) {
+		condition = append(condition, "时政UP主")
+	}
+	if s.isEnterprise(a.Mid) {
+		condition = append(condition, "企业机构实名认证UP主")
+	}
+	if pfl, _ = s.profile(c, a.Mid); pfl != nil && pfl.Follower > 100000 {
+		condition = append(condition, fmt.Sprintf("10万以上粉丝（%d个粉丝）", pfl.Follower))
+	}
+	if len(condition)+len(additionCondition) <= 0 {
+		log.Info("sendMail aid(%d) mid(%d) is not signed uper & whith uper && shizheng uper && qiye uper && funs < 10W", a.ID, a.Mid)
+		return
+	}
+
+	content = append(content, fmt.Sprintf("[审核状态]: %s", archive.StateMean[a.State]))
+	fromVideo := v != nil
+	if fromVideo {
+		//视频审核操作
+		videoOper, err = s.arc.LastVideoOper(c, v.ID)
+		if err != nil || videoOper == nil || videoOper.VID <= 0 {
+			log.Error("sendMail s.arc.LastVideoOper(vid(%d)) error(%v) or not found", v.ID, err)
+			return
+		}
+
+		mngUID = videoOper.UID
+		if len(videoOper.Content) != 0 {
+			content = append(content, videoOper.Content)
+		}
+		remark = strings.TrimSpace(videoOper.Remark)
+		if len(remark) != 0 {
+			content = append(content, fmt.Sprintf("备注：%s", strings.Replace(remark, "\n", ",", -1)))
+		}
+	} else {
+		//稿件审核操作
+		if arcOper, err = s.arc.LastArcOper(c, a.ID); err != nil || arcOper == nil || arcOper.AID <= 0 {
+			log.Error("sendMail s.arc.LastArcOper(aid(%d)) error(%v) or not found", a.ID, err)
+			return
+		}
+		mngUID = arcOper.UID
+		if len(arcOper.Content) != 0 {
+			content = append(content, arcOper.Content)
+		}
+		remark = strings.TrimSpace(arcOper.Remark)
+		if len(remark) != 0 {
+			content = append(content, fmt.Sprintf("备注: %s", strings.Replace(remark, "\n", ",", -1)))
+		}
+	}
+
+	//审核人员信息
+	if mngUser, err = s.mng.User(c, mngUID); err != nil || mngUser == nil || mngUser.ID != mngUID {
+		log.Error("s.mng.User(%d) error(%v) or not found(%d)", arcOper.UID, err, mngUser)
+		return
+	}
+
+	if len(condition) > 0 {
+		params := map[string]string{
+			"aid":        strconv.FormatInt(a.ID, 10),
+			"title":      a.Title,
+			"upName":     pfl.Profile.Name,
+			"condition":  strings.Join(condition, "/"),
+			"change":     strings.Join(content, " ,"),
+			"uid":        strconv.FormatInt(mngUser.ID, 10),
+			"username":   mngUser.Username,
+			"department": mngUser.Department,
+			"typeId":     strconv.Itoa(int(s.topType(a.TypeID))),
+			"fromVideo":  strconv.FormatBool(fromVideo),
+		}
+		tpl := s.email.NotifyEmailTemplate(params)
+		s.email.PushToRedis(c, tpl)
+	}
+	if len(additionCondition) > 0 {
+		params := map[string]string{
+			"aid":        strconv.FormatInt(a.ID, 10),
+			"title":      a.Title,
+			"upName":     pfl.Profile.Name,
+			"condition":  strings.Join(additionCondition, "/"),
+			"change":     strings.Join(content, " ,"),
+			"uid":        strconv.FormatInt(mngUser.ID, 10),
+			"username":   mngUser.Username,
+			"department": mngUser.Department,
+			"typeId":     additionType,
+			"fromVideo":  strconv.FormatBool(fromVideo),
+		}
+		tpl := s.email.NotifyEmailTemplate(params)
+		s.email.PushToRedis(c, tpl)
+	}
+	return
+}
+
+func (s *Service) emailProc() {
+	defer s.waiter.Done()
+
+	for {
+		if s.closed {
+			return
+		}
+
+		s.email.Start(emailmdl.MailKey)
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+func (s *Service) emailFastProc() {
+	defer s.waiter.Done()
+	var (
+		err error
+	)
+
+	for {
+		if s.closed {
+			return
+		}
+
+		<-s.email.FastChan()
+		log.Info("emailFastProc start to handle")
+
+		for {
+			if s.closed {
+				return
+			}
+
+			err = s.email.Start(emailmdl.MailFastKey)
+			if err == redis.ErrNil {
+				log.Info("emailFastProc start to rest")
+				break
+			}
+			time.Sleep(200 * time.Millisecond)
+		}
+
+	}
+}

app/job/main/videoup-report/service/excitation.go

@@ -0,0 +1,132 @@
+package service
+
+import (
+	"context"
+	"go-common/app/job/main/videoup-report/dao/data"
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/manager"
+	"go-common/library/log"
+	"time"
+)
+
+// hdlExcitation 激励回查逻辑
+func (s *Service) hdlExcitation(nw, old *archive.Archive) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.hdlExcitation(%v,%v) panic(%v)", nw, old, pErr)
+		}
+	}()
+	var (
+		c        = context.TODO()
+		state    int8
+		archives []*archive.Archive
+		addits   map[int64]*archive.Addit
+		now      = time.Now()
+		aids     []int64
+		isIgn    bool
+		aCount   = 20 //自制投稿阀值
+	)
+	log.Info("hdlExcitation() begin new archive(%v) old archive(%v)", nw, old)
+	if old != nil { //只有新投稿才需要继续
+		return
+	}
+	//检查UP主是否在白名单中
+	if isIgn, err = s.isIgnMidExcitation(c, nw.Mid); err != nil {
+		log.Error("s.hdlExcitation() new(%v) old(%v) s.isIgnMidExcitation() error(%v)", nw, old, err)
+		go s.hdlExcitationRetry(nw, old)
+		return
+	} else if isIgn {
+		log.Error("s.hdlExcitation() new(%v) old(%v) ignore white mid(%d)", nw, old, nw.Mid)
+		return
+	}
+	if state, err = s.dataDao.UpProfitState(c, nw.Mid); err != nil {
+		log.Error("hdlExcitation() s.dataDao.UpProfitState(%d) error(%v)", nw.Mid, err)
+		go s.hdlExcitationRetry(nw, old)
+		return
+	}
+	if state != data.UpProfitStateSigned {
+		log.Info("hdlExcitation() ignore 非签约UP主(%d) state(%d)", nw.Mid, state)
+		return
+	}
+	//查询7天内自制、过审、定时过审稿件
+	st := now.Add(-168 * time.Hour)
+	if archives, err = s.arc.ExcitationArchivesByTime(c, nw.Mid, st, now); err != nil {
+		log.Error("hdlExcitation() s.arc.ExcitationAidsByTime(%d,%v,%v) error(%v)", nw.Mid, st, now, err)
+		go s.hdlExcitationRetry(nw, old)
+		return
+	}
+	if len(archives) == 0 {
+		log.Info("hdlExcitation() ignore UP主(%d) 在 %v 到 %v 时间内没有自制、非特殊分区、非私单、非商单稿件。", nw.Mid, st, now)
+		return
+	}
+	for _, a := range archives {
+		if s.isAuditType(a.TypeID) || a.AttrVal(archive.AttrBitIsPOrder) == archive.AttrYes {
+			log.Info("hdlExcitation() 忽略特殊分区、私单 archive(%v)", a)
+			continue
+		}
+		aids = append(aids, a.ID)
+	}
+	if addits, err = s.arc.Addits(c, aids); err != nil {
+		log.Error("hdlExcitation() aid(%d) s.arc.Addits(%v) error(%v)", nw.ID, aids, err)
+		go s.hdlExcitationRetry(nw, old)
+		return
+	}
+
+	for i := 0; i < len(aids); i++ {
+		if _, ok := addits[aids[i]]; ok && addits[aids[i]].OrderID > 0 {
+			log.Info("hdlExcitation() 忽略商单 addit(%v)", addits[aids[i]])
+			aids = append(aids[:i], aids[i+1:]...)
+			i--
+			continue
+		}
+	}
+
+	if len(aids) < aCount {
+		log.Info("hdlExcitation() ignore UP主(%d) 在 %v 到 %v 时间内只有 %d 个自制、非特殊分区、非私单、非商单稿件没有达到 %d 个。aids:%v", nw.Mid, st, now, len(aids), aCount, aids)
+		return
+	}
+	log.Info("hdlExcitation() 符合激励回查的UP主(%d) 正在插入激励回查aids(%v)", nw.Mid, aids)
+	if err = s.arc.AddRecheckAids(c, archive.TypeExcitationRecheck, aids, true); err != nil {
+		log.Error("s.hdlExcitation() s.arc.AddRecheckAids error(%v)", err)
+		go s.hdlExcitationRetry(nw, old)
+		return
+	}
+	return
+}
+
+// isIgnMidExcitation 检查UP主是否在白名单中
+func (s *Service) isIgnMidExcitation(c context.Context, mid int64) (is bool, err error) {
+	groups, err := s.dataDao.MidGroups(c, mid)
+	if err != nil {
+		log.Error("s.isIgnMidExcitation(%d) error(%v)", mid, err)
+		return
+	}
+	if _, ok := groups[manager.UpTypeExcitationWhite]; ok {
+		is = true
+		return
+	}
+	return
+}
+
+// ignoreUpsExcitation 将UP主的未回查的稿件从激励回查去除
+func (s *Service) ignoreUpsExcitation(c context.Context, mid int64) (err error) {
+	log.Info("s.ignoreUpsExcitation(%d)", mid)
+	if err = s.arc.UpdateMidRecheckState(c, archive.TypeExcitationRecheck, mid, archive.RecheckStateIgnore); err != nil {
+		log.Error("s.ignoreUpsExcitation(%d) error(%v)", mid, err)
+		return
+	}
+	return
+}
+
+// hdlExcitationRetry 激励回查重试
+func (s *Service) hdlExcitationRetry(nw, old *archive.Archive) (err error) {
+	mTime, _ := time.ParseInLocation("2006-01-02 15:04:05", nw.MTime, time.Local)
+	if time.Now().Unix()-mTime.Unix() > 3600 {
+		log.Error("s.hdlExcitationRetry() new(%v) old(%v) the retry handler finish. (over 60 mins)", nw, old)
+		return
+	}
+	log.Error("s.hdlExcitationRetry() new(%v) old(%v) the handler will retry in 10 seconds", nw, old)
+	time.Sleep(10 * time.Second)
+	err = s.hdlExcitation(nw, old)
+	return
+}

app/job/main/videoup-report/service/excitation_test.go

@@ -0,0 +1,18 @@
+package service
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/job/main/videoup-report/model/archive"
+	"testing"
+)
+
+func TestService_HdlExcitation(t *testing.T) {
+	Convey("HdlExcitation", t, func() {
+		n := &archive.Archive{
+			ID:  17191032,
+			Mid: 27515256,
+		}
+		err := s.hdlExcitation(n, nil)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/videoup-report/service/extra_func.go

@@ -0,0 +1,186 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	tmod "go-common/app/job/main/videoup-report/model/task"
+	account "go-common/app/service/main/account/api"
+	upsrpc "go-common/app/service/main/up/api/v1"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+
+	"fmt"
+	"math"
+)
+
+//ERROR
+var (
+	ErrRPCEmpty = errors.New("rpc reply empty")
+)
+
+func (s *Service) upGroupMids(c context.Context, gid int64) (mids []int64, err error) {
+	var (
+		total int
+		maxps = 10000
+		req   = &upsrpc.UpGroupMidsReq{
+			Pn:      1,
+			GroupID: gid,
+			Ps:      maxps,
+		}
+		reply *upsrpc.UpGroupMidsReply
+	)
+
+	for {
+		reply, err = s.upsRPC.UpGroupMids(c, req)
+		if err == nil && (reply == nil || reply.Mids == nil) {
+			err = ErrRPCEmpty
+		}
+		if err != nil {
+			log.Error("UpGroupMids req(%+v) error(%v)", req, err)
+			return
+		}
+		total = reply.Total
+		mids = append(mids, reply.Mids...)
+		if reply.Size() != maxps {
+			break
+		}
+		req.Pn++
+	}
+	log.Info("upGroupMids(%d) reply total(%d) len(%d)", gid, total, len(mids))
+	return
+}
+
+func (s *Service) upSpecial(c context.Context) (ups map[int8]map[int64]struct{}, err error) {
+	var (
+		g                                                                  errgroup.Group
+		whitegroup, blackgroup, policesgroup, enterprisegroup, signedgroup map[int64]struct{}
+	)
+	ups = make(map[int8]map[int64]struct{})
+
+	f := func(gid int8) (map[int64]struct{}, error) {
+		group := make(map[int64]struct{})
+		mids, e := s.upGroupMids(c, int64(gid))
+		if e != nil {
+			return group, e
+		}
+		for _, mid := range mids {
+			group[mid] = struct{}{}
+		}
+		return group, nil
+	}
+	g.Go(func() error {
+		whitegroup, err = f(tmod.UpperTypeWhite)
+		return err
+	})
+	g.Go(func() error {
+		blackgroup, err = f(tmod.UpperTypeBlack)
+		return err
+	})
+	g.Go(func() error {
+		policesgroup, err = f(tmod.UpperTypePolitices)
+		return err
+	})
+	g.Go(func() error {
+		enterprisegroup, err = f(tmod.UpperTypeEnterprise)
+		return err
+	})
+	g.Go(func() error {
+		signedgroup, err = f(tmod.UpperTypeSigned)
+		return err
+	})
+	if err = g.Wait(); err != nil {
+		return
+	}
+
+	ups[tmod.UpperTypeWhite] = whitegroup
+	ups[tmod.UpperTypeBlack] = blackgroup
+	ups[tmod.UpperTypePolitices] = policesgroup
+	ups[tmod.UpperTypeEnterprise] = enterprisegroup
+	ups[tmod.UpperTypeSigned] = signedgroup
+	return
+
+}
+
+//
+func (s *Service) profile(c context.Context, mid int64) (p *account.ProfileStatReply, err error) {
+	if p, err = s.accRPC.ProfileWithStat3(c, &account.MidReq{Mid: mid}); err != nil {
+		p = nil
+		log.Error("s.accRPC.ProfileWithStat3(%d) error(%v)", mid, err)
+	}
+	return
+}
+
+func (s *Service) getUpperFans(c context.Context, mid int64) (fans int64, failed bool) {
+	card, err := s.profile(c, mid)
+	if err != nil {
+		failed = true
+		log.Error("s.profile(mid=%d) error(%v)", mid, err)
+		return
+	}
+
+	fans = card.Follower
+	log.Info("s.profile(mid=%d) fans(%d)", mid, fans)
+	return
+}
+
+func (s *Service) isWhite(mid int64) bool {
+	if ups, ok := s.upperCache[tmod.UpperTypeWhite]; ok {
+		_, isWhite := ups[mid]
+		return isWhite
+	}
+	return false
+}
+
+func (s *Service) isBlack(mid int64) bool {
+	if ups, ok := s.upperCache[tmod.UpperTypeBlack]; ok {
+		_, isBlack := ups[mid]
+		return isBlack
+	}
+	return false
+}
+
+func (s *Service) isPolitices(mid int64) bool {
+	if ups, ok := s.upperCache[tmod.UpperTypePolitices]; ok {
+		_, isShiZheng := ups[mid]
+		return isShiZheng
+	}
+	return false
+}
+
+func (s *Service) isEnterprise(mid int64) bool {
+	if ups, ok := s.upperCache[tmod.UpperTypeEnterprise]; ok {
+		_, isQiYe := ups[mid]
+		return isQiYe
+	}
+	return false
+}
+
+func (s *Service) isSigned(mid int64) bool {
+	if ups, ok := s.upperCache[tmod.UpperTypeSigned]; ok {
+		_, signed := ups[mid]
+		return signed
+	}
+	return false
+}
+
+// Until next day x hours
+func nextDay(hour int) time.Duration {
+	n := time.Now().Add(24 * time.Hour)
+	d := time.Date(n.Year(), n.Month(), n.Day(), hour, 0, 0, 0, n.Location())
+	return time.Until(d)
+}
+
+func secondsFormat(sec int) (str string) {
+	if sec < 0 {
+		return "--:--:--"
+	}
+	if sec == 0 {
+		return "00:00:00"
+	}
+	h := math.Floor(float64(sec) / 3600)
+	m := math.Floor((float64(sec) - 3600*h) / 60)
+	se := sec % 60
+	return fmt.Sprintf("%02d:%02d:%02d", int64(h), int64(m), se)
+}

app/job/main/videoup-report/service/flow.go

@@ -0,0 +1,109 @@
+package service
+
+import (
+	"context"
+	"strings"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+/**
+ * txAddFlow 新增流量套餐的记录
+ * return int64, error/nil
+ */
+func (s *Service) txAddFlow(tx *sql.Tx, pool int8, oid, groupID, uid int64, remark string) (id int64, err error) {
+	if id, err = s.arc.TxAddFlow(tx, pool, oid, uid, groupID, remark); err != nil {
+		log.Error("txAddFlow s.arc.TxAddFlow(%d,%d,%d,%d,%s) error(%v)", pool, oid, uid, groupID, remark, err)
+		return
+	}
+	if id <= 0 {
+		return
+	}
+	if _, err = s.arc.TxAddFlowLog(tx, pool, archive.FlowLogAdd, oid, uid, groupID, remark); err != nil {
+		log.Error("txAddFlow s.arc.TxAddFlowLog(%d,%d,%d,%d,%s) error(%v)", pool, oid, uid, groupID, remark, err)
+		return
+	}
+	return
+}
+
+/**
+ * txUpFlowState 更新流量套餐的状态
+ * return error/nil
+ */
+func (s *Service) txUpFlowState(tx *sql.Tx, state int8, uid int64, f *archive.FlowData) (err error) {
+	if f == nil {
+		return
+	}
+	var rows int64
+	if rows, err = s.arc.TxUpFlowState(tx, f.ID, state); err != nil {
+		log.Error("updateFlowState s.arc.TxUpFlowState error(%v) id(%d) state(%d)", err, f.ID, state)
+		return
+	}
+	if rows <= 0 {
+		return
+	}
+
+	action := archive.FlowLogUpdate
+	if state == archive.FlowDelete {
+		action = archive.FlowLogDel
+	}
+	if _, err = s.arc.TxAddFlowLog(tx, f.Pool, action, f.OID, uid, f.GroupID, "审核后台修改状态"); err != nil {
+		log.Error("updateFlowState s.arc.TxAddFlowLog error(%v) pool(%d) oid(%d) uid(%d) state(%d)", err, f.Pool, f.OID, uid, state)
+		return
+	}
+	return
+}
+
+/**
+ * txAddOrUpdateFlowState 新增或更新流量套餐的状态
+ * return *archive.FlowData/nil, bool, error/nil
+ */
+func (s *Service) txAddOrUpdateFlowState(c context.Context, tx *sql.Tx, oid, groupID, uid int64, pool, state int8, remark string) (flow *archive.FlowData, diff string, err error) {
+	var (
+		old, nw int8
+	)
+
+	defer func() {
+		if err == nil && old != nw {
+			tagID := archive.FlowOperType[groupID]
+			if tagID > 0 {
+				stateMap := map[int8]string{archive.FlowOpen: "是", archive.FlowDelete: "否"}
+				diff = strings.TrimSpace(archive.Operformat(tagID, stateMap[old], stateMap[nw], archive.OperStyleOne))
+			}
+		}
+	}()
+
+	if flow, err = s.arc.FlowUnique(c, oid, groupID, pool); err != nil {
+		log.Error("txAddOrUpdateFlowState s.arc.FlowUnique(%d,%d,%d) error(%v) state(%d)", oid, groupID, pool, err, state)
+		return
+	}
+	//无数据前提下，新状态=state就没必要添加数据啦
+	if flow == nil && state == archive.FlowDelete {
+		return
+	}
+	if flow == nil {
+		flow = &archive.FlowData{Pool: pool, OID: oid, GroupID: groupID, State: archive.FlowOpen}
+		if flow.ID, err = s.txAddFlow(tx, flow.Pool, flow.OID, flow.GroupID, uid, remark); err != nil {
+			log.Error("txAddOrUpdateFlowState s.txAddFlow error(%v) flow(%+v) state(%d)", err, flow, state)
+			return
+		}
+		old = archive.FlowDelete
+		nw = archive.FlowOpen
+	} else {
+		old = flow.State
+		nw = state
+	}
+	if flow.State == state {
+		return
+	}
+
+	if err = s.txUpFlowState(tx, state, uid, flow); err != nil {
+		log.Error("txAddOrUpdateFlowState s.txUpdateFlowState error(%v) flow(%+v) state(%d) ", err, flow, state)
+		return
+	}
+	flow.State = state
+	nw = state
+	return
+}

app/job/main/videoup-report/service/monitor.go

@@ -0,0 +1,140 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/monitor"
+	"go-common/library/log"
+	"time"
+)
+
+// hdlMonitorArc deal with archive stay stats
+func (s *Service) hdlMonitorArc(nw, old *archive.Archive) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.hdlMonitorArc panic(%v)", pErr)
+		}
+	}()
+	var (
+		oKey, nKey string
+		kFormat    = monitor.RedisPrefix + monitor.SuffixArc
+		addit      *archive.Addit
+	)
+	log.Info("hdlMonitorArc (%v,%v)", nw, old)
+	if addit, err = s.arc.Addit(context.TODO(), nw.ID); err != nil {
+		log.Error("s.hdlMonitorArc() s.arc.Addit(%d) error(%v)", nw.ID, err)
+		return
+	}
+	//去掉PGC稿件
+	if addit != nil && (addit.UpFrom == archive.UpFromPGC || addit.UpFrom == archive.UpFromPGCSecret || addit.UpFrom == archive.UpFromCoopera) {
+		return
+	}
+	if old != nil {
+		if nw.Round == old.Round && nw.State == old.State {
+			return
+		}
+		oKey = fmt.Sprintf(kFormat, monitor.BusArc, old.Round, old.State)
+	}
+	//忽略Round 99 state不为-6,-1,0,1的数据
+	if nw.Round != archive.RoundEnd && (nw.State == archive.StateForbidFixed || nw.State == archive.StateForbidWait || nw.State == archive.StateOpen || nw.State == archive.StateOrange) {
+		nKey = fmt.Sprintf(kFormat, monitor.BusArc, nw.Round, nw.State)
+	}
+	//回查忽略活动稿件
+	if addit != nil && addit.MissionID > 0 && (nw.Round == archive.RoundReviewFirst || nw.Round == archive.RoundReviewFirstWaitTrigger || nw.Round == archive.RoundReviewSecond || nw.Round == archive.RoundTriggerClick) {
+		nKey = ""
+	}
+	log.Info("hdlMonitorArc () s.monitorSave(%s,%s,%d)", oKey, nKey, nw.ID)
+	err = s.monitorSave(oKey, nKey, nw.ID)
+	return
+}
+
+// hdlMonitorVideo 视频审核监控
+func (s *Service) hdlMonitorVideo(nv, ov *archive.Video) (err error) {
+	var (
+		oKey, nKey string
+		kFormat    = monitor.RedisPrefix + monitor.SuffixVideo
+	)
+	log.Info("hdlMonitorVideo (%v,%v)", nv, ov)
+	if ov != nil {
+		if nv.Status == ov.Status {
+			return
+		}
+		oKey = fmt.Sprintf(kFormat, monitor.BusVideo, ov.Status)
+	}
+	if nv.Status == archive.VideoStatusSubmit || nv.Status == archive.VideoStatusWait {
+		nKey = fmt.Sprintf(kFormat, monitor.BusVideo, nv.Status)
+	}
+	log.Info("hdlMonitorVideo () s.monitorSave(%s,%s,%d) filename(%s)", oKey, nKey, nv.ID, nv.Filename)
+	err = s.monitorSave(oKey, nKey, nv.ID)
+	return
+}
+
+func (s *Service) monitorSave(oKey, nKey string, oid int64) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.monitorSave panic(%v)", pErr)
+		}
+	}()
+	var c = context.TODO()
+	if oKey != "" {
+		if err = s.redis.RemMonitorStats(c, oKey, oid); err != nil {
+			log.Error("s.monitorSave() RemMonitorStats(%s,%d) error(%v)", oKey, oid, err)
+		}
+		s.redis.ClearMonitorStats(c, oKey)
+	}
+	if nKey != "" {
+		if err = s.redis.AddMonitorStats(c, nKey, oid); err != nil {
+			log.Error("s.monitorSave() AddMonitorStats(%s,%d) error(%v)", nKey, oid, err)
+			return
+		}
+		s.redis.ClearMonitorStats(c, nKey)
+	}
+	if err != nil {
+		log.Error("s.monitorSave(%s,%s,%d) error(%v)", oKey, nKey, oid, err)
+	}
+	return
+}
+
+//monitorNotifyEmail 发送监控通知
+func (s *Service) monitorNotify() {
+	var (
+		c    = context.TODO()
+		data []*monitor.RuleResultData
+		err  error
+	)
+	defer func() {
+		if err := recover(); err != nil {
+			log.Error("monitorNotifyEmail() panic(%v)", err)
+		}
+	}()
+	// 从admin获取报警数据
+	if data, err = s.dataDao.MonitorNotify(c); err != nil {
+		log.Error("s.dataDao.MonitorNotify() error(%v)", err)
+		return
+	}
+	for _, v := range data {
+		if v.Rule.State != monitor.RuleStateOK {
+			log.Error("monitorNotify() ignore rule(%d) state(%d)", v.Rule.ID, v.Rule.State)
+			continue
+		}
+		subject := fmt.Sprintf("%s监控", v.Rule.Name)
+		body := fmt.Sprintf("当前滞留时间为%s超过阀值，滞留量为%d，整体量为%d \n%s", secondsFormat(v.Stats.MaxTime), v.Stats.MoniCount, v.Stats.TotalCount, time.Now().Format("2006-01-02 15:04:05"))
+		url := ""
+		switch v.Rule.Business {
+		case monitor.BusVideo:
+			url = fmt.Sprintf("http://manager.bilibili.co/#!/video/list?monitor_list=%d_%d_%d", v.Rule.Type, v.Rule.Business, v.Rule.ID)
+		case monitor.BusArc:
+			url = fmt.Sprintf("http://manager.bilibili.co/#!/archive_utils/all?monitor_list=%d_%d_%d", v.Rule.Type, v.Rule.Business, v.Rule.ID)
+		}
+		body += fmt.Sprintf("\n跳转链接：%s", url)
+		if v.Rule.RuleConf.Notify.Way == monitor.NotifyTypeEmail {
+			tpl := s.email.MonitorNotifyTemplate(subject, body, v.Rule.RuleConf.Notify.Member)
+			log.Info("monitorNotify() email template(%v)", *tpl)
+			s.email.PushToRedis(c, tpl)
+		} else {
+			log.Error("monitorNotify() unknown notify rule(%d) type(%s)", v.Rule.ID, v.Rule.RuleConf.Notify.Way)
+			continue
+		}
+	}
+}

app/job/main/videoup-report/service/monitor_test.go

@@ -0,0 +1,32 @@
+package service
+
+import (
+	"context"
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/job/main/videoup-report/model/archive"
+	"testing"
+)
+
+func Test_ArcMonitorStats(t *testing.T) {
+	Convey("ArcMonitorStats", t, func() {
+		//var o *archive.Archive
+		o := &archive.Video{
+			ID:    10018978,
+			State: -30,
+		}
+		n := &archive.Video{
+			ID:    10018978,
+			State: -1,
+		}
+		err := s.hdlMonitorVideo(n, o)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestMonitorNotify(t *testing.T) {
+	Convey("MonitorNotify", t, func() {
+		tpl := s.email.MonitorNotifyTemplate("", "http://uat-manager.bilibili.co/#!/video/list?monitor_list=1_1_5", []string{"liusiming@bilibili.com"})
+		s.email.PushToRedis(context.TODO(), tpl)
+		s.email.Start("f_mail_list_fast")
+	})
+}

app/job/main/videoup-report/service/reply.go

@@ -0,0 +1,92 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/email"
+	"go-common/library/log"
+	"go-common/library/queue/databus/report"
+)
+
+func (s *Service) arcReply(c context.Context, a *archive.Archive, replySwitch int64) (err error) {
+	if replySwitch != archive.ReplyOn && replySwitch != archive.ReplyOff {
+		log.Error("arcReply aid(%d) archive replySwitch(%d) 's state is unknow!", a.ID, replySwitch)
+		return
+	}
+
+	replyState, _ := s.dataDao.CheckReply(c, a.ID)
+
+	//删除之前的重试机会，避免延迟重试覆盖最新结果
+	//s.removeRetry(c, a.ID, email.RetryActionReply)
+	if replySwitch == archive.ReplyOn {
+		err = s.openReply(c, a, replyState)
+	} else {
+		err = s.closeReply(c, a, replyState)
+	}
+	return
+}
+
+func (s *Service) openReply(c context.Context, a *archive.Archive, oldState int64) (err error) {
+	if a == nil {
+		return
+	}
+	if err = s.dataDao.OpenReply(c, a.ID, a.Mid); err != nil {
+		log.Error("openReply s.dataDao.OpenReply(%d,%d) error(%v)", a.ID, a.Mid, err)
+		s.addRetry(c, a.ID, email.RetryActionReply, archive.ReplyOn, oldState)
+		return
+	}
+	if oldState == archive.ReplyOn {
+		return
+	}
+	if oldState != archive.ReplyOn && oldState != archive.ReplyOff {
+		oldState = archive.ReplyDefault
+	}
+
+	s.logJob(a, archive.ReplyDesc[archive.ReplyOn], archive.ReplyDesc[oldState])
+	return
+}
+
+func (s *Service) closeReply(c context.Context, a *archive.Archive, oldState int64) (err error) {
+	if a == nil {
+		return
+	}
+	if err = s.dataDao.CloseReply(c, a.ID, a.Mid); err != nil {
+		log.Error("closeReply s.dataDao.CloseReply(%d,%d) error(%v)", a.ID, a.Mid, err)
+		s.addRetry(c, a.ID, email.RetryActionReply, archive.ReplyOff, oldState)
+		return
+	}
+	if oldState == archive.ReplyOff {
+		return
+	}
+	if oldState != archive.ReplyOn && oldState != archive.ReplyOff {
+		oldState = archive.ReplyDefault
+	}
+
+	s.logJob(a, archive.ReplyDesc[archive.ReplyOff], archive.ReplyDesc[oldState])
+	return
+}
+
+func (s *Service) logJob(a *archive.Archive, action string, oldAction string) {
+	info := &report.ManagerInfo{
+		Uname:    "videoup-job",
+		UID:      399,
+		Business: archive.LogBusJob,
+		Type:     archive.LogTypeReply,
+		Oid:      a.ID,
+		Action:   action,
+		Ctime:    time.Now(),
+		Index:    []interface{}{a.State},
+		Content:  map[string]interface{}{"old": oldAction},
+	}
+	log.Info("logJob (%+v)", info)
+	report.Manager(info)
+}
+
+func isOpenReplyState(state int) (val int) {
+	if archive.NormalState(state) || state == archive.StateForbidFixed {
+		val = 1
+	}
+	return
+}

app/job/main/videoup-report/service/reply_test.go

@@ -0,0 +1,55 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+	"go-common/app/job/main/videoup-report/model/archive"
+)
+
+var a = &archive.Archive{
+	ID:        10098208,
+	Mid:       10920044,
+	Attribute: 1097728,
+	State:     0,
+	Round:     99,
+	TypeID:    21,
+}
+
+func TestServicearcReply(t *testing.T) {
+	var (
+		c           = context.Background()
+		replySwitch = int64(1)
+	)
+	convey.Convey("arcReply", t, func(ctx convey.C) {
+		err := s.arcReply(c, a, replySwitch)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestServiceopenReply(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("openReply", t, func(ctx convey.C) {
+		err := s.openReply(c, a, archive.ReplyOff)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestServicecloseReply(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("closeReply", t, func(ctx convey.C) {
+		err := s.closeReply(c, a, archive.ReplyOn)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/job/main/videoup-report/service/retry.go

@@ -0,0 +1,116 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/email"
+	"go-common/library/log"
+)
+
+func (s *Service) addRetry(c context.Context, aid int64, action string, flag int64, flagA int64) (err error) {
+	member := email.Retry{
+		AID:        aid,
+		Action:     action,
+		Flag:       flag,
+		FlagA:      flagA,
+		CreateTime: time.Now().Unix(),
+	}
+
+	err = s.email.PushRedis(c, member, email.RetryListKey)
+	return
+}
+
+//popRetry spop a retry element from redis set
+func (s *Service) popRetry(c context.Context, key string) (member email.Retry, err error) {
+	var bs []byte
+	if bs, err = s.email.PopRedis(c, key); err != nil {
+		return
+	}
+
+	if err = json.Unmarshal(bs, &member); err != nil {
+		log.Error("PopRetry(%s) json.Unmarshal(%s) error(%v)", key, string(bs), err)
+	}
+	return
+}
+
+func (s *Service) removeRetry(c context.Context, aid int64, action string) (err error) {
+	var (
+		bs    []byte
+		list  []interface{}
+		reply int
+	)
+
+	for _, snew := range archive.ReplyState {
+		for _, sold := range archive.ReplyState {
+			member := email.Retry{
+				AID:    aid,
+				Action: action,
+				Flag:   snew,
+				FlagA:  sold,
+			}
+			if bs, err = json.Marshal(member); err != nil {
+				log.Error("removeRetry json.Marshal error(%v) member(%+v)", err, member)
+				err = nil
+				continue
+			}
+			list = append(list, string(bs))
+		}
+	}
+
+	if reply, err = s.email.RemoveRedis(c, email.RetryListKey, list...); err != nil {
+		log.Error("removeRetry s.email.RemoveRedis error(%v) aid(%d) action(%s)", err, aid, action)
+	} else {
+		log.Info("removeRetry s.email.RemoveRedis success reply(%d) aid(%d) action(%s)", reply, aid, action)
+	}
+	return
+}
+
+func (s *Service) retryProc() {
+	defer s.waiter.Done()
+	for {
+		if s.closed {
+			return
+		}
+
+		c := context.TODO()
+		member, err := s.popRetry(c, email.RetryListKey)
+		if err != nil {
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		log.Info("retry member(%+v)", member)
+
+		if member.CreateTime > 0 && (time.Now().Unix()-member.CreateTime >= 180) {
+			log.Error("retry list too long for 3min")
+			time.Sleep(5 * time.Millisecond)
+			continue
+		}
+
+		switch member.Action {
+		case email.RetryActionReply:
+			a, err := s.arc.ArchiveByAid(c, member.AID)
+			if err != nil {
+				log.Error("retryProc s.arc.ArchiveByAid(%d) error(%v) member(%+v)", member.AID, err, member)
+				s.addRetry(c, member.AID, member.Action, member.Flag, member.FlagA)
+				continue
+			}
+			isOpen := isOpenReplyState(a.State) > 0
+			if member.Flag == archive.ReplyOn && isOpen {
+				err = s.openReply(c, a, member.FlagA)
+			}
+			if member.Flag == archive.ReplyOff && !isOpen {
+				err = s.closeReply(c, a, member.FlagA)
+			}
+			//if err != nil {
+			//s.addRetry(c, member.AID, member.Action, member.Flag, member.FlagA)
+			//}
+		default:
+			log.Warn("retryProc unknown action(%s) member(%+v)", member.Action, member)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+}

app/job/main/videoup-report/service/retry_test.go

@@ -0,0 +1,55 @@
+package service
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/email"
+)
+
+var (
+	aid    = a.ID
+	action = email.RetryActionReply
+	flags  = archive.ReplyOn
+	flagA  = archive.ReplyOff
+)
+
+func TestServiceaddRetry(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("addRetry", t, func(ctx convey.C) {
+		err := s.addRetry(c, aid, action, flags, flagA)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestServiceremoveRetry(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	TestServiceaddRetry(t)
+	convey.Convey("removeRetry", t, func(ctx convey.C) {
+		err := s.removeRetry(c, aid, action)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestServiceretryProc(t *testing.T) {
+	convey.Convey("retryProc", t, func(ctx convey.C) {
+		go func() {
+			time.Sleep(1 * time.Second)
+			s.closed = true
+		}()
+		s.retryProc()
+		ctx.Convey("No return values", func(ctx convey.C) {
+		})
+	})
+}

app/job/main/videoup-report/service/service.go

@@ -0,0 +1,341 @@
+package service
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/videoup-report/conf"
+	arcdao "go-common/app/job/main/videoup-report/dao/archive"
+	"go-common/app/job/main/videoup-report/dao/data"
+	"go-common/app/job/main/videoup-report/dao/email"
+	hbasedao "go-common/app/job/main/videoup-report/dao/hbase"
+	"go-common/app/job/main/videoup-report/dao/manager"
+	"go-common/app/job/main/videoup-report/dao/mission"
+	redisdao "go-common/app/job/main/videoup-report/dao/redis"
+	"go-common/app/job/main/videoup-report/dao/tag"
+	arcmdl "go-common/app/job/main/videoup-report/model/archive"
+	taskmdl "go-common/app/job/main/videoup-report/model/task"
+	account "go-common/app/service/main/account/api"
+	arcrpc "go-common/app/service/main/archive/api/gorpc"
+	upsrpc "go-common/app/service/main/up/api/v1"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"strings"
+)
+
+const (
+	_archiveTable = "archive"
+	_videoTable   = "archive_video"
+	_upsTable     = "ups"
+	_jumpChanSize = int(4000)
+	_logChanSize  = int(200000)
+)
+
+// Service is service.
+type Service struct {
+	c              *conf.Config
+	arc            *arcdao.Dao
+	redis          *redisdao.Dao
+	hbase          *hbasedao.Dao
+	dataDao        *data.Dao
+	email          *email.Dao
+	mng            *manager.Dao
+	arcUpChs       []chan *arcmdl.UpInfo
+	videoUpInfoChs []chan *arcmdl.VideoUpInfo
+	// waiter
+	waiter sync.WaitGroup
+	// databus
+	archiveSub   *databus.Databus
+	arcResultSub *databus.Databus
+	videoupSub   *databus.Databus
+	ManagerDBSub *databus.Databus
+	// cache
+	sfTpsCache        map[int16]*arcmdl.Type
+	adtTpsCache       map[int16]struct{}
+	taskCache         *arcmdl.TaskCache
+	videoAuditCache   *arcmdl.VideoAuditCache
+	arcMoveTypeCache  *arcmdl.ArcMoveTypeCache
+	arcRoundFlowCache *arcmdl.ArcRoundFlowCache
+	xcodeTimeCache    *arcmdl.XcodeTimeCache
+	assignCache       map[int64]*taskmdl.AssignConfig
+	upperCache        map[int8]map[int64]struct{}
+	weightCache       map[int8]map[int64]*taskmdl.ConfigItem
+	missTagsCache     map[string]int
+
+	lastjumpMap map[int64]struct{} //上轮插队的这一轮也更新，否则会出现权重只增不减
+	jumplist    *taskmdl.JumpList  //插队序列
+	jumpchan    chan *taskmdl.WeightLog
+	tasklogchan chan *taskmdl.WeightLog
+	//rpc
+	arcRPCGroup2 *arcrpc.Service2
+
+	//grpc
+	accRPC account.AccountClient
+	upsRPC upsrpc.UpClient
+
+	// closed
+	closed     bool
+	tagDao     *tag.Dao
+	missionDao *mission.Dao
+}
+
+// New is videoup-report-job service implementation.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c: c,
+		//dao
+		arc:   arcdao.New(c),
+		redis: redisdao.New(c),
+		hbase: hbasedao.New(c),
+		//databus
+		archiveSub:   databus.New(c.ArchiveSub),
+		arcResultSub: databus.New(c.ArchiveResultSub),
+		videoupSub:   databus.New(c.VideoupSub),
+		ManagerDBSub: databus.New(c.ManagerDBSub),
+		dataDao:      data.New(c),
+		email:        email.New(c),
+		mng:          manager.New(c),
+		// cache
+		taskCache: &arcmdl.TaskCache{
+			Task: make(map[int64]*arcmdl.Task),
+		},
+		videoAuditCache: &arcmdl.VideoAuditCache{
+			Data: make(map[int16]map[string]int),
+		},
+		arcMoveTypeCache: &arcmdl.ArcMoveTypeCache{
+			Data: make(map[int8]map[int16]map[string]int),
+		},
+		arcRoundFlowCache: &arcmdl.ArcRoundFlowCache{
+			Data: make(map[int8]map[int64]map[string]int),
+		},
+		xcodeTimeCache: &arcmdl.XcodeTimeCache{
+			Data: make(map[int8][]int),
+		},
+		arcRPCGroup2: arcrpc.New2(c.ArchiveRPCGroup2),
+		lastjumpMap:  make(map[int64]struct{}),
+		jumplist:     taskmdl.NewJumpList(),
+		jumpchan:     make(chan *taskmdl.WeightLog, _jumpChanSize),
+		tasklogchan:  make(chan *taskmdl.WeightLog, _logChanSize),
+		tagDao:       tag.New(c),
+		missionDao:   mission.New(c),
+	}
+	var err error
+	if s.accRPC, err = account.NewClient(conf.Conf.GRPC.AccRPC); err != nil {
+		panic(err)
+	}
+
+	if s.upsRPC, err = upsrpc.NewClient(conf.Conf.GRPC.UpsRPC); err != nil {
+		panic(err)
+	}
+
+	for i := 0; i < s.c.ChanSize; i++ {
+		log.Info("videoup-report-job chanSize starting(%d)", i)
+		s.arcUpChs = append(s.arcUpChs, make(chan *arcmdl.UpInfo, 10240))
+		s.waiter.Add(1)
+		go s.arcUpdateproc(i)
+		s.videoUpInfoChs = append(s.videoUpInfoChs, make(chan *arcmdl.VideoUpInfo, 10240))
+		s.waiter.Add(1)
+		go s.upVideoproc(i)
+	}
+	s.loadConf()
+	s.loadType()
+	// load cache.
+	s.loadTask()
+	s.loadTaskTookSort()
+	s.hdlTraffic()
+	s.loadMission()
+	go s.cacheproc()
+	go s.monitorNotifyProc()
+	s.waiter.Add(1)
+	go s.hotarchiveproc()
+	s.waiter.Add(1)
+	go s.arcCanalConsume()
+	s.waiter.Add(1)
+	go s.arcResultConsume()
+	s.waiter.Add(1)
+	go s.taskWeightConsumer()
+	s.waiter.Add(1)
+	go s.taskweightproc()
+	s.waiter.Add(1)
+	go s.movetaskproc()
+	go s.deltaskproc()
+	s.waiter.Add(1)
+	go s.videoupConsumer()
+	s.waiter.Add(1)
+	go s.emailProc()
+	s.waiter.Add(1)
+	go s.emailFastProc()
+	s.waiter.Add(1)
+	go s.retryProc()
+	s.waiter.Add(1)
+	go s.managerDBConsume()
+
+	return s
+}
+
+func (s *Service) loadType() {
+	tpm, err := s.arc.TypeMapping(context.TODO())
+	if err != nil {
+		log.Error("s.dede.TypeMapping error(%v)", err)
+		return
+	}
+	s.sfTpsCache = tpm
+	// audit types
+	adt, err := s.arc.AuditTypesConf(context.TODO())
+	if err != nil {
+		log.Error("s.dede.AuditTypesConf error(%v)", err)
+		return
+	}
+	s.adtTpsCache = adt
+
+	wvc, err := s.arc.WeightValueConf(context.TODO())
+	if err != nil {
+		log.Error("s.arc.WeightValueConf error(%v)", err)
+		return
+	}
+	taskmdl.WLVConf = wvc
+}
+
+func (s *Service) isAuditType(tpID int16) bool {
+	_, isAt := s.adtTpsCache[tpID]
+	return isAt
+}
+
+func (s *Service) topType(tpID int16) (id int16) {
+	if tp, ok := s.sfTpsCache[tpID]; ok && tp != nil {
+		id = tp.PID
+	}
+	return
+}
+
+func (s *Service) typeName(tpID int16) (name string) {
+	if tp, ok := s.sfTpsCache[tpID]; ok && tp != nil {
+		name = tp.Name
+	}
+	return
+}
+
+func (s *Service) topTypeName(tpID int16) (name string) {
+	pid := s.topType(tpID)
+	name = s.typeName(pid)
+	return
+}
+
+func (s *Service) cacheproc() {
+	for {
+		time.Sleep(1 * time.Minute)
+		// config
+		s.loadConf()
+		// task
+		s.loadTask()
+		s.loadTaskTookSort()
+		// handle task took
+		s.hdlTaskTook()
+		s.hdlTaskTookByHourHalf()
+		// handle video audit
+		s.hdlVideoAuditCount()
+		s.hdlMoveTypeCount()
+		s.hdlRoundFlowCount()
+		//handle calculate video xcode time stats, and save to DB
+		s.hdlXcodeStats()
+		s.hdlTraffic()
+		s.loadMission()
+	}
+}
+
+func (s *Service) monitorNotifyProc() {
+	for {
+		s.monitorNotify()
+		time.Sleep(30 * time.Minute)
+	}
+}
+
+// s.missTagsCache: missionName or first tag
+func (s *Service) loadMission() {
+	mm, err := s.missionDao.Missions(context.TODO())
+	if err != nil {
+		log.Error("s.missionDao.Mission error(%v)", err)
+		return
+	}
+	s.missTagsCache = make(map[string]int)
+	for _, m := range mm {
+		if len(m.Tags) > 0 {
+			splitedTags := strings.Split(m.Tags, ",")
+			s.missTagsCache[splitedTags[0]] = m.ID
+		} else {
+			s.missTagsCache[m.Name] = m.ID
+		}
+	}
+}
+
+// hotarchiveproc get hot archive which need to recheck
+func (s *Service) hotarchiveproc() {
+	defer s.waiter.Done()
+	for {
+		if s.closed {
+			return
+		}
+		s.addHotRecheck()
+		time.Sleep(10 * time.Minute)
+	}
+}
+
+// Close  consumer close.
+func (s *Service) Close() {
+	s.closed = true
+	s.archiveSub.Close()
+	s.arcResultSub.Close()
+	s.videoupSub.Close()
+	time.Sleep(2 * time.Second)
+	for i := 0; i < s.c.ChanSize; i++ {
+		log.Info("videoup-report-job chanSize closing(%d)", i)
+		close(s.arcUpChs[i])
+		close(s.videoUpInfoChs[i])
+	}
+	s.arc.Close()
+	s.mng.Close()
+	s.redis.Close()
+	s.email.Close()
+	s.hbase.Close()
+	s.waiter.Wait()
+}
+
+// Ping check server ok.
+func (s *Service) Ping(c context.Context) (err error) {
+	if err = s.arc.Ping(c); err != nil {
+		return
+	}
+	if err = s.mng.Ping(c); err != nil {
+		return
+	}
+	return
+}
+
+func (s *Service) loadConf() {
+	var (
+		err        error
+		assignConf map[int64]*taskmdl.AssignConfig
+		weightConf map[int8]map[int64]*taskmdl.ConfigItem
+		upperCache map[int8]map[int64]struct{}
+	)
+
+	if assignConf, err = s.assignConf(context.TODO()); err != nil {
+		log.Error("s.assignConf error(%v)", err)
+		return
+	}
+	s.assignCache = assignConf
+
+	upperCache, err = s.upSpecial(context.TODO())
+	if err != nil {
+		log.Error("s.upSpecial error(%v)", err)
+	} else {
+		s.upperCache = upperCache
+	}
+
+	if weightConf, err = s.weightConf(context.TODO()); err != nil {
+		log.Error(" s.weightConf error(%v)", err)
+		return
+	}
+	s.weightCache = weightConf
+}

app/job/main/videoup-report/service/service_test.go

@@ -0,0 +1,91 @@
+package service
+
+import (
+	"context"
+	"flag"
+	. "github.com/smartystreets/goconvey/convey"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/videoup-report/conf"
+	"go-common/app/job/main/videoup-report/model/archive"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/videoup-report-job.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+func Test_loadType(t *testing.T) {
+	Convey("loadType", t, func() {
+		s.loadType()
+	})
+}
+
+func Test_VideoAudit(t *testing.T) {
+	Convey("VideoAudit", t, func() {
+		_, err := s.VideoAudit(context.Background(), time.Now(), time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_Ping(t *testing.T) {
+	Convey("Ping", t, func() {
+		err := s.Ping(context.Background())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TaskTooksByHalfHour(t *testing.T) {
+	Convey("TaskTooksByHalfHour", t, func() {
+		_, err := s.TaskTooksByHalfHour(context.Background(), time.Now(), time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_AddArchiveHotRecheck(t *testing.T) {
+	Convey("AddArchiveHotRecheck", t, func() {
+		time.Sleep(time.Second)
+		err := s.addHotRecheck()
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_SecondRound(t *testing.T) {
+	Convey("SecondRound", t, func() {
+		m := &archive.VideoupMsg{
+			Route:    "second_round",
+			Aid:      24320325,
+			FromList: "hot_review",
+		}
+		err := s.secondRound(context.Background(), m)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_SecondRoundCancelMission(t *testing.T) {
+	Convey("SecondRound", t, func() {
+		m := &archive.VideoupMsg{
+			Route:     "second_round",
+			Aid:       17191032,
+			MissionID: 1,
+		}
+		err := s.secondRound(context.Background(), m)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_SecondFormat(t *testing.T) {
+	Convey("SecondFormat", t, func() {
+		m := 5556
+		format := secondsFormat(m)
+		So(format, ShouldNotBeNil)
+	})
+}

app/job/main/videoup-report/service/tag.go

@@ -0,0 +1,116 @@
+package service
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+func (s *Service) upBindTag(c context.Context, mid, aid int64, tags string, typeID int16) (err error) {
+	typeName := s.typeName(typeID)
+	topTypeName := s.topTypeName(typeID)
+	if topTypeName != "" {
+		typeName = fmt.Sprintf("%s,%s", typeName, topTypeName)
+	}
+
+	for i := 0; i < 3; i++ {
+		if err = s.tagDao.UpBind(c, mid, aid, tags, typeName, ""); err == nil {
+			return
+		}
+	}
+	if err != nil {
+		log.Error("upBindTag s.tagDao.UpBind(%d,%d,%s,%s) typeid(%d) error(%v)", mid, aid, tags, typeName, typeID, err)
+	}
+	return
+}
+
+func (s *Service) adminBindTag(c context.Context, mid, aid int64, tags string, typeID int16) (err error) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.adminBindTag panic(%v)", pErr)
+		}
+	}()
+	typeName := s.typeName(typeID)
+	topTypeName := s.topTypeName(typeID)
+	if topTypeName != "" {
+		typeName = fmt.Sprintf("%s,%s", typeName, topTypeName)
+	}
+
+	for i := 0; i < 3; i++ {
+		if err = s.tagDao.AdminBind(c, mid, aid, tags, typeName, ""); err == nil {
+			return
+		}
+	}
+	if err != nil {
+		log.Error("adminBindTag s.tagDao.AdminBind(%d,%d,%s,%s) typeid(%d) error(%v)", mid, aid, tags, typeName, typeID, err)
+	}
+	return
+}
+
+func (s *Service) checkChannelReview(c context.Context, aid int64) (channelReview bool, channelIDs string, err error) {
+	var (
+		tmp bool
+	)
+	if tmp, channelIDs, err = s.tagDao.CheckChannelReview(c, aid); err != nil {
+		log.Error("checkChannelReview s.tagDao.CheckChannelReview error(%v) aid(%d)", err, aid)
+		return
+	}
+	if !tmp {
+		log.Info("checkChannelReview s.tagDao.CheckChannelReview aid(%d) not in channelreview", aid)
+		return
+	}
+
+	channelReview = true
+	return
+}
+
+func (s *Service) txAddOrUpRecheckState(c context.Context, tx *sql.Tx, tp int, aid int64, state int8) (recheck *archive.Recheck, err error) {
+	if recheck, err = s.arc.RecheckByAid(c, tp, aid); err != nil {
+		log.Error("txAddOrUpRecheckState s.arc.RecheckByAid(%d,%d) error(%v)", tp, aid, err)
+		return
+	}
+	if recheck != nil && recheck.State == state {
+		return
+	}
+
+	if recheck == nil {
+		recheck = &archive.Recheck{Type: tp, Aid: aid, State: archive.RecheckStateWait}
+		if recheck.ID, err = s.arc.TxAddRecheckAID(tx, tp, aid); err != nil {
+			log.Error("txAddOrUpRecheckState s.arc.TxAddRecheckAID(%d,%d) error(%v)", tp, aid, err)
+			return
+		}
+	}
+	if recheck.State != state {
+		if _, err = s.arc.TxUpRecheckState(tx, tp, aid, state); err != nil {
+			log.Error("txAddOrUpRecheckState s.arc.TxUpRecheckState(%d,%d,%d) error(%v)", tp, aid, state, err)
+			return
+		}
+		recheck.State = state
+	}
+	return
+}
+
+func (s *Service) txAddChannelReview(c context.Context, tx *sql.Tx, aid int64) (operCont string, operRemark string, err error) {
+	var (
+		need       bool
+		channelIDs string
+	)
+	if need, channelIDs, err = s.checkChannelReview(c, aid); err != nil || !need {
+		return
+	}
+	log.Info("start to add channel review aid(%d)", aid)
+	if _, err = s.txAddOrUpRecheckState(c, tx, archive.TypeChannelRecheck, aid, archive.RecheckStateWait); err != nil {
+		log.Error("txAddChannelReview s.txAddOrUpRecheckState(%d) error(%v)", aid, err)
+		return
+	}
+
+	if _, operCont, err = s.txAddOrUpdateFlowState(c, tx, aid, archive.FLowGroupIDChannel, 399, archive.PoolArcForbid, archive.FlowOpen, "投稿开启频道禁止"); err != nil {
+		log.Error("txAddChannelReview s.txAddOrUpdateFlowState(%d) error(%v)", aid, err)
+		return
+	}
+	operRemark = fmt.Sprintf("待频道回查,频道ID:%s", channelIDs)
+	return
+}

app/job/main/videoup-report/service/tag_test.go

@@ -0,0 +1,15 @@
+package service
+
+import (
+	"context"
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func TestService_upbind(t *testing.T) {
+	Convey("upbind", t, func() {
+		a, _ := s.arc.ArchiveByAid(context.Background(), 10110255)
+		err := s.upBindTag(context.Background(), a.Mid, a.ID, a.Tag, a.TypeID)
+		t.Logf("err(%+v)", err)
+	})
+}

app/job/main/videoup-report/service/task.go

@@ -0,0 +1,185 @@
+package service
+
+import (
+	"context"
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/library/log"
+	"sort"
+	"time"
+)
+
+func (s *Service) loadTask() {
+	var (
+		err   error
+		took  *archive.TaskTook
+		tooks []*archive.TaskTook
+		tasks []*archive.Task
+	)
+	s.taskCache.Lock()
+	defer s.taskCache.Unlock()
+	if len(s.taskCache.Took) == 0 && len(s.taskCache.Task) == 0 {
+		if took, err = s.arc.TaskTookByHalfHour(context.TODO()); err != nil {
+			log.Error("s.arc.TaskTookByHalfHour error(%v)", err)
+			return
+		}
+		if took != nil {
+			if tooks, err = s.arc.TaskTooks(context.TODO(), took.Ctime); err != nil {
+				log.Error("s.arc.TaskTooks(%v) error(%v)", took.Ctime, err)
+				return
+			}
+			s.taskCache.Took = tooks
+		}
+		if tasks, err = s.arc.TaskByUntreated(context.TODO()); err != nil {
+			log.Error("s.arc.TaskByUntreated() error(%v)", err)
+			return
+		}
+	} else {
+		var tasksOrig, tasksDone []*archive.Task
+		if tasksOrig, err = s.arc.TaskByMtime(context.TODO(), s.taskCache.Mtime.Add(-time.Minute*1)); err != nil {
+			log.Error("s.arc.TaskByMtime(%v) error(%v)", s.taskCache.Mtime, err)
+			return
+		}
+		if tasksDone, err = s.arc.TaskDoneByMtime(context.TODO(), s.taskCache.Mtime.Add(-time.Minute*1)); err != nil {
+			log.Error("s.arc.TaskDoneByMtime(%v) error(%v)", s.taskCache.Mtime, err)
+			return
+		}
+		tasks = make([]*archive.Task, len(tasksOrig)+len(tasksDone))
+		copy(tasks, tasksOrig)
+		copy(tasks[len(tasksOrig):], tasksDone)
+	}
+	for _, task := range tasks {
+		_, ok := s.taskCache.Task[task.ID]
+		if ok && (task.State != archive.TaskStateUnclaimed && task.State != archive.TaskStateUntreated) {
+			delete(s.taskCache.Task, task.ID)
+		} else if task.State == archive.TaskStateUnclaimed || task.State == archive.TaskStateUntreated {
+			s.taskCache.Task[task.ID] = task
+		}
+	}
+}
+
+func (s *Service) loadTaskTookSort() {
+	var (
+		took         int
+		tooks        []int
+		taskMinCtime *archive.Task
+	)
+	s.taskCache.Lock()
+	defer s.taskCache.Unlock()
+	for _, task := range s.taskCache.Task {
+		if (s.taskCache.Mtime == time.Time{} || s.taskCache.Mtime.Unix() < task.Mtime.Unix()) {
+			s.taskCache.Mtime = task.Mtime
+		}
+		if taskMinCtime == nil || taskMinCtime.Ctime.Unix() > task.Ctime.Unix() {
+			taskMinCtime = task
+		}
+		took = int(time.Now().Unix() - task.Ctime.Unix())
+		tooks = append(tooks, took)
+	}
+	if len(tooks) == 0 {
+		return
+	}
+	sort.Ints(tooks)
+	s.taskCache.Sort = tooks
+	log.Info("s.loadTaskTookSort() 本轮统计: 耗时最久id(%d) ctime(%v)", taskMinCtime.ID, taskMinCtime.Ctime)
+}
+
+func (s *Service) hdlTaskTook() (lastID int64, err error) {
+	s.taskCache.Lock()
+	defer s.taskCache.Unlock()
+	var (
+		spacing         float32
+		m50             float32
+		m50Index        float32
+		m50IndexPoint   float32
+		m50Value        int
+		m60             float32
+		m60Index        float32
+		m60IndexPoint   float32
+		m60Value        int
+		m80             float32
+		m80Index        float32
+		m80IndexPoint   float32
+		m80Value        int
+		m90             float32
+		m90Index        float32
+		m90IndexPoint   float32
+		m90Value        int
+		took            *archive.TaskTook
+		taskTookSortLen = len(s.taskCache.Sort)
+	)
+	if taskTookSortLen > 1 {
+		spacing = float32(taskTookSortLen-1) / 10
+		m50Index = 1 + spacing*5
+		m50IndexPoint = m50Index - float32(int(m50Index))
+		m50Value = s.taskCache.Sort[int(m50Index)-1]
+		m50 = float32(s.taskCache.Sort[int(m50Index)]-m50Value)*m50IndexPoint + float32(m50Value)
+		m60Index = 1 + spacing*6
+		m60IndexPoint = m60Index - float32(int(m60Index))
+		m60Value = s.taskCache.Sort[int(m60Index)-1]
+		m60 = float32(s.taskCache.Sort[int(m60Index)]-m60Value)*m60IndexPoint + float32(m60Value)
+		m80Index = 1 + spacing*8
+		m80IndexPoint = m80Index - float32(int(m80Index))
+		m80Value = s.taskCache.Sort[int(m80Index)-1]
+		m80 = float32(s.taskCache.Sort[int(m80Index)]-m80Value)*m80IndexPoint + float32(m80Value)
+		m90Index = 1 + spacing*9
+		m90IndexPoint = m90Index - float32(int(m90Index))
+		m90Value = s.taskCache.Sort[int(m90Index)-1]
+		m90 = float32(s.taskCache.Sort[int(m90Index)]-m90Value)*m90IndexPoint + float32(m90Value)
+		took = &archive.TaskTook{}
+		took.M50 = int(m50 + 0.5)
+		took.M60 = int(m60 + 0.5)
+		took.M80 = int(m80 + 0.5)
+		took.M90 = int(m90 + 0.5)
+		took.TypeID = archive.TookTypeMinute
+		took.Ctime = time.Now()
+		took.Mtime = took.Ctime
+		s.taskCache.Took = append(s.taskCache.Took, took)
+		lastID, err = s.arc.AddTaskTook(context.TODO(), took)
+	}
+	return
+}
+
+func (s *Service) hdlTaskTookByHourHalf() (lastID int64, err error) {
+	s.taskCache.Lock()
+	defer s.taskCache.Unlock()
+	var (
+		m50     int
+		m60     int
+		m80     int
+		m90     int
+		took    *archive.TaskTook
+		tookLen = len(s.taskCache.Took)
+	)
+	for _, v := range s.taskCache.Took {
+		m50 += v.M50
+		m60 += v.M60
+		m80 += v.M80
+		m90 += v.M90
+	}
+	if tookLen >= 30 {
+		took = &archive.TaskTook{}
+		m50 /= int(float32(tookLen) + 0.5)
+		m60 /= int(float32(tookLen) + 0.5)
+		m80 /= int(float32(tookLen) + 0.5)
+		m90 /= int(float32(tookLen) + 0.5)
+		took.M50 = m50
+		took.M60 = m60
+		took.M80 = m80
+		took.M90 = m90
+		took.Ctime = time.Now()
+		took.Mtime = took.Ctime
+		took.TypeID = archive.TookTypeHalfHour
+		lastID, err = s.arc.AddTaskTook(context.TODO(), took)
+		s.taskCache.Took = nil
+	}
+	return
+}
+
+// TaskTooksByHalfHour get task books by ctime
+func (s *Service) TaskTooksByHalfHour(c context.Context, stime, etime time.Time) (tooks []*archive.TaskTook, err error) {
+	if tooks, err = s.arc.TaskTooksByHalfHour(c, stime, etime); err != nil {
+		log.Error("s.arc.TaskTooksByHalfHour(%v,%v)", stime, etime)
+		return
+	}
+	return
+}

app/job/main/videoup-report/service/task_assign.go

@@ -0,0 +1,105 @@
+package service
+
+import (
+	"context"
+	"math/rand"
+	"time"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/task"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+// 设置指派任务
+func (s *Service) setTaskAssign(c context.Context, a *archive.Archive, v *archive.Video) (t *task.Task) {
+	var (
+		now    = time.Now()
+		arruid = []int64{}
+		mapUID = make(map[int64]int64)
+	)
+
+	t = &task.Task{
+		Pool:    task.PoolForFirst,
+		Aid:     a.ID,
+		Cid:     v.Cid,
+		Subject: task.SubjectForNormal,
+		AdminID: int64(0),
+		UID:     int64(0),
+		State:   task.StateForTaskDefault,
+	}
+
+	for _, tc := range s.assignCache {
+		log.Info("task doing(%v) aid(%d) type(%d) filename(%s) tc(%d)", t, a.ID, a.TypeID, v.Filename, tc.ID)
+		if tc.STime.After(now) || tc.ETime.Before(now) {
+			log.Error("task time is error stime(%v) etime(%v)", tc.STime, tc.ETime)
+			continue
+		}
+		var midOk, tidOk, durationOk = true, true, true
+		if len(tc.MIDs) > 0 {
+			if _, midOk = tc.MIDs[a.Mid]; !midOk {
+				log.Info("task mid(%d) wrong", a.Mid)
+			}
+		}
+		if len(tc.TIDs) > 0 {
+			if _, tidOk = tc.TIDs[a.TypeID]; !tidOk {
+				log.Info("task type(%d) wrong", a.TypeID)
+			}
+		}
+		if tc.MinDuration != tc.MaxDuration && (v.Duration < tc.MinDuration || v.Duration > tc.MaxDuration) {
+			log.Error("task minDur(%d) maxDur(%d) wrong", tc.MinDuration, tc.MaxDuration)
+			durationOk = false
+		}
+		if midOk && tidOk && durationOk {
+			for _, uid := range tc.UIDs {
+				if _, ok := mapUID[uid]; !ok {
+					mapUID[uid] = tc.AdminID
+					arruid = append(arruid, uid)
+				}
+			}
+		}
+	}
+	if len(arruid) > 0 {
+		uids, err := s.arc.ConsumerOnline(c, xstr.JoinInts(arruid))
+		if err != nil || len(uids) == 0 {
+			log.Warn("task s.arc.ConsumerOnline(%v) (%v) err(%v)", arruid, uids, err)
+			return
+		}
+		if len(uids) == 1 {
+			t.UID = uids[0]
+		} else {
+			inx := rand.Intn(len(uids) - 1)
+			log.Info("task uids(%v) rand inx(%d)", uids, inx)
+			t.UID = uids[inx]
+		}
+	}
+
+	if t.UID != 0 {
+		t.Subject = task.SubjectForTask
+		t.AdminID = mapUID[t.UID] // 命中多个指派者配置，选择其中一个就行
+		t.State = task.StateForTaskDefault
+	}
+
+	return
+}
+
+// 指派配置
+func (s *Service) assignConf(c context.Context) (tcs map[int64]*task.AssignConfig, err error) {
+	var ids []int64
+	if tcs, err = s.arc.AssignConfigs(context.TODO()); err != nil {
+		log.Error("s.arc.AssignConfigs(%v) error(%v)", err)
+		return
+	}
+
+	for k, v := range tcs {
+		if !v.ETime.IsZero() && v.ETime.Before(time.Now()) {
+			delete(tcs, k)
+			ids = append(ids, k)
+		}
+	}
+	if len(ids) > 0 {
+		log.Info("task config(%v) 指派配置已过期,自动失效", ids)
+		s.arc.DelAssignConfs(c, ids)
+	}
+	return
+}

app/job/main/videoup-report/service/task_dispatch.go

@@ -0,0 +1,301 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	tmod "go-common/app/job/main/videoup-report/model/task"
+	"go-common/app/job/main/videoup-report/model/utils"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+func (s *Service) hdlVideoTask(c context.Context, fn string) (err error) {
+	var (
+		v     *archive.Video
+		a     *archive.Archive
+		state int8
+		dID   int64
+	)
+	if v, a, err = s.archiveVideo(c, fn); err != nil {
+		log.Error("s.archiveVideo(%s) error(%v)", fn, err)
+		return
+	}
+	if a.State == archive.StateForbidUpDelete {
+		log.Info("task archive(%d) deleted", a.ID)
+		return
+	}
+
+	if v.Status != archive.VideoStatusWait {
+		log.Info("task archive(%d) filename(%s) already status(%d)", a.ID, v.Filename, v.Status)
+		return
+	}
+
+	if dID, state, err = s.arc.DispatchState(c, v.Aid, v.Cid); err != nil {
+		log.Error("task s.arc.DispatchState(%d,%d) error(%v)", v.Aid, v.Cid, err)
+		return
+	}
+	if dID != 0 && state <= tmod.StateForTaskWork {
+		log.Info("task aid(%d) cid(%d) filename(%s) already in dispatch state(%d)", v.Aid, v.Cid, v.Filename, state)
+		return
+	}
+
+	log.Info("archive(%d) filename(%s) video(%d) tranVideoTask begin", a.ID, v.Filename, v.Cid)
+	if err = s.addVideoTask(c, a, v); err != nil {
+		log.Error("task s.addVideoTask error(%v)", err)
+		return
+	}
+	return
+}
+
+func (s *Service) archiveVideo(c context.Context, filename string) (v *archive.Video, a *archive.Archive, err error) {
+	if v, err = s.arc.NewVideo(c, filename); err != nil {
+		log.Error("s.arc.NewVideo(%s) error(%v)", filename, err)
+		return
+	}
+	if v == nil {
+		log.Error("s.arc.NewVideo(%s) video is nil", filename)
+		err = fmt.Errorf("video(%s) is not exists", filename)
+		return
+	}
+	if a, err = s.arc.ArchiveByAid(c, v.Aid); err != nil {
+		log.Error("s.arc.ArchiveByAid(%d) filename(%s) error(%v)", v.Aid, filename, err)
+		return
+	}
+	return
+}
+
+func (s *Service) addVideoTask(c context.Context, a *archive.Archive, v *archive.Video) (err error) {
+	var (
+		task         *tmod.Task
+		lastID, fans int64
+		cfitems      []*tmod.ConfigItem
+		descb        []byte
+		accfailed    bool
+	)
+
+	task = s.setTaskAssign(c, a, v)
+	fans, accfailed = s.setTaskUPSpecial(c, task, a.Mid)
+	s.setTaskTimed(c, task)
+	cfitems = s.getConfWeight(c, task, a)
+
+	if lastID, err = s.arc.AddDispatch(c, task); err != nil {
+		log.Error("s.arc.AddDispatch error(%v)", err)
+		return
+	}
+	// 允许日志记录错误
+	if _, err = s.arc.AddTaskHis(c, tmod.PoolForFirst, 6, lastID, v.Cid, task.UID, v.Status, "videoup-job"); err != nil {
+		log.Error("s.arc.AddTaskHis error(%v)", err)
+	}
+
+	log.Info("archive(%d) filename(%s) taskUid(%d)", a.ID, v.Filename, task.UID)
+
+	// 保存权重配置信息,错误不影响正常流程
+	tp := &tmod.WeightParams{
+		TaskID:    lastID,
+		Mid:       a.Mid,
+		Special:   task.UPSpecial,
+		Ctime:     utils.NewFormatTime(time.Now()),
+		Ptime:     task.Ptime,
+		CfItems:   cfitems,
+		Fans:      fans,
+		AccFailed: accfailed,
+
+		TypeID: a.TypeID,
+	}
+
+	s.setTaskUpFrom(c, a.ID, tp)
+	s.setTaskUpGroup(c, a.Mid, tp)
+
+	s.redis.SetWeight(c, map[int64]*tmod.WeightParams{lastID: tp})
+	if len(cfitems) > 0 {
+		if descb, err = json.Marshal(cfitems); err != nil {
+			log.Error("json.Marshal error(%v)", err)
+		} else {
+			if _, err = s.arc.InDispatchExtend(c, lastID, string(descb)); err != nil {
+				log.Error("s.task.InDispatchExtend(%d) error(%v)", lastID, err)
+			}
+		}
+		err = nil
+	}
+	return
+}
+
+func (s *Service) moveDispatch() (err error) {
+	var (
+		tx               *sql.Tx
+		c                = context.TODO()
+		mtime            = time.Now().Add(-24 * time.Hour)
+		startTime        = time.Date(mtime.Year(), mtime.Month(), mtime.Day(), mtime.Hour(), 0, 0, 0, mtime.Location())
+		endTime          = time.Date(mtime.Year(), mtime.Month(), mtime.Day(), mtime.Hour(), 59, 59, 0, mtime.Location())
+		dispatchRows     int64
+		dispatchDoneRows int64
+	)
+	if tx, err = s.arc.BeginTran(c); err != nil {
+		log.Error("s.arc.BeginTran error(%v)")
+		return
+	}
+	if dispatchRows, err = s.arc.TxAddDispatchDone(c, tx, startTime, endTime); err != nil {
+		tx.Rollback()
+		log.Error("s.arc.TxAddDispatchDone(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	if dispatchDoneRows, err = s.arc.TxDelDispatchByTime(c, tx, startTime, endTime); err != nil {
+		tx.Rollback()
+		log.Error("s.arc.TxDelDispatchByTime(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	if dispatchRows != dispatchDoneRows {
+		// no way here !
+		tx.Rollback()
+		log.Error("moveDispatch error dispatchRows(%d) dispatchDoneRows(%d)", dispatchRows, dispatchDoneRows)
+		return
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("tx.Commit error(%v)")
+		return
+	}
+	log.Info("moveDispatch mtime(%s) to mtime(%s) rows(%d)", startTime.Format("2006-01-02 15:04:05"), endTime.Format("2006-01-02 15:04:05"), dispatchDoneRows)
+	return
+}
+
+func (s *Service) moveTaskOperHis(limit int64) (moved int64, err error) {
+	var (
+		tx        *sql.Tx
+		c         = context.TODO()
+		mtime     = time.Now().Add(-2 * 30 * 24 * time.Hour)
+		before    = time.Date(mtime.Year(), mtime.Month(), mtime.Day(), mtime.Hour(), 0, 0, 0, mtime.Location())
+		movedRows int64
+		delRows   int64
+	)
+	if tx, err = s.arc.BeginTran(c); err != nil {
+		log.Error("s.arc.BeginTran error(%v)")
+		return
+	}
+	if movedRows, err = s.arc.TxMoveTaskOperDone(tx, before, limit); err != nil {
+		tx.Rollback()
+		log.Error("s.arc.TxMoveTaskOperDone(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	if delRows, err = s.arc.TxDelTaskOper(tx, before, limit); err != nil {
+		tx.Rollback()
+		log.Error("s.arc.TxDelTaskOper(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	if movedRows != delRows {
+		tx.Rollback()
+		log.Error("moveOperHistory error mvRows(%d) delRows(%d)", movedRows, delRows)
+		return
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("tx.Commit error(%v)")
+		return
+	}
+	log.Info("moveTaskOperHistory before mtime(%s) rows(%d)", before.Format("2006-01-02 15:04:05"), movedRows)
+	return movedRows, nil
+}
+
+func (s *Service) delTaskDispatchDone(limit int64) (delRows int64, err error) {
+	var (
+		c      = context.TODO()
+		mtime  = time.Now().Add(-30 * 24 * time.Hour)
+		before = time.Date(mtime.Year(), mtime.Month(), mtime.Day(), mtime.Hour(), 0, 0, 0, mtime.Location())
+	)
+	if delRows, err = s.arc.DelTaskDoneBefore(c, before, limit); err != nil {
+		log.Error("s.arc.DelTaskDoneBefore(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	if delRows > 0 {
+		log.Info("delTaskDispatchDone before mtime(%s) rows(%d)", before.Format("2006-01-02 15:04:05"), delRows)
+	}
+
+	if delRows, err = s.arc.DelTaskBefore(c, before, limit); err != nil {
+		log.Error("s.arc.DelTaskBefore(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	if delRows > 0 {
+		log.Info("DelTaskBefore before mtime(%s) rows(%d)", before.Format("2006-01-02 15:04:05"), delRows)
+	}
+	return
+}
+
+func (s *Service) delTaskHistoryDone(limit int64) (delRows int64, err error) {
+	var (
+		c      = context.TODO()
+		mtime  = time.Now().Add(-3 * 30 * 24 * time.Hour)
+		before = time.Date(mtime.Year(), mtime.Month(), mtime.Day(), mtime.Hour(), 0, 0, 0, mtime.Location())
+	)
+	if delRows, err = s.arc.DelTaskHistoryDone(c, before, limit); err != nil {
+		log.Error("s.arc.DelTaskHistoryDone(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	log.Info("delTaskHistoryDone before mtime(%s) rows(%d)", before.Format("2006-01-02 15:04:05"), delRows)
+	return
+}
+
+func (s *Service) delTaskExtend(limit int64) (delRows int64, err error) {
+	var (
+		c      = context.TODO()
+		mtime  = time.Now().Add(-20 * 24 * time.Hour)
+		before = time.Date(mtime.Year(), mtime.Month(), mtime.Day(), mtime.Hour(), 0, 0, 0, mtime.Location())
+	)
+	if delRows, err = s.arc.DelTaskExtend(c, before, limit); err != nil {
+		log.Error("s.arc.DelTaskExtend(%s) error(%v)", mtime.Format("2006-01-02 15:04:05"), err)
+		return
+	}
+	log.Info("delTaskExtend before mtime(%s) rows(%d)", before.Format("2006-01-02 15:04:05"), delRows)
+	return
+}
+
+/*
+	1.移动task_dispatch到task_dispatch_done
+	2.移动task_oper_history到task_oper_history_done
+	3.删除过于久远的task_dispatch_done,task_oper_history_done,task_dispatch_extend
+*/
+func (s *Service) movetaskproc() {
+	defer s.waiter.Done()
+	for {
+		if s.closed {
+			return
+		}
+		s.moveDispatch()
+		time.Sleep(1 * time.Hour)
+	}
+}
+
+func (s *Service) deltaskproc() {
+	for {
+		for {
+			rows, _ := s.moveTaskOperHis(100)
+			time.Sleep(1 * time.Second)
+			if rows == 0 {
+				break
+			}
+		}
+		for {
+			rows, _ := s.delTaskDispatchDone(100)
+			time.Sleep(1 * time.Second)
+			if rows == 0 {
+				break
+			}
+		}
+		for {
+			rows, _ := s.delTaskHistoryDone(100)
+			time.Sleep(1 * time.Second)
+			if rows == 0 {
+				break
+			}
+		}
+		for {
+			rows, _ := s.delTaskExtend(100)
+			time.Sleep(1 * time.Second)
+			if rows == 0 {
+				break
+			}
+		}
+		time.Sleep(nextDay(10))
+	}
+}

app/job/main/videoup-report/service/task_weight.go

@@ -0,0 +1,371 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/app/job/main/videoup-report/model/task"
+	"go-common/app/job/main/videoup-report/model/utils"
+	"go-common/library/log"
+)
+
+const (
+	_pm = int64(3) //period minute
+	//普通任务参数
+	_nc1 = int64(3)  // 等待时长9分钟
+	_nc2 = int64(5)  // 等待时长15分钟
+	_nc3 = int64(9)  // 等待时长27分钟
+	_nc4 = int64(15) // 等待时长45分钟
+
+	//定时任务参数
+	_tc1 = int64(80) // 距离发布4小时
+	_tc2 = int64(40) // 距离发布2小时
+	_tc3 = int64(20) // 距离发布1小时
+)
+
+func (s *Service) upTaskWeightCache() (err error) {
+	var (
+		ids      []int64
+		firstid  int64
+		lastid   int64
+		tasksMap map[int64]*task.WeightParams
+	)
+	c := context.TODO()
+	s.jumplist.Reset()
+
+	defer func() {
+		s.lastjumpMap = make(map[int64]struct{})
+		for item := s.jumplist.POP(); item != nil; item = s.jumplist.POP() {
+			select {
+			case s.jumpchan <- item:
+				s.lastjumpMap[item.TaskID] = struct{}{}
+			default:
+				log.Warn("jumpchan full")
+			}
+		}
+	}()
+
+	for {
+		cacheMap := make(map[int64]*task.WeightParams)
+		// 先从数据库批量读取出id来
+		if ids, lastid, err = s.arc.TaskIDforWeight(c, firstid); len(ids) == 0 {
+			if err != nil {
+				log.Error("s.task.TaskIDforWeight(%d) error(%v)", firstid, err)
+			}
+			return
+		}
+		// 先从redis取权重配置,redis挂了从数据库读取权重配置
+		if tasksMap, err = s.redis.GetWeight(c, ids); len(tasksMap) != len(ids) {
+			log.Warn("GetTaskWeight from redis need len(%d) while len(%d)", len(ids), len(tasksMap))
+			if tasksMap, err = s.arc.GetTaskWeight(c, firstid); err != nil {
+				log.Error("s.arc.GetTaskWeight(%d) error(%v)", firstid, err)
+				return
+			}
+		}
+		for id, tp := range tasksMap {
+			if tp.State == 0 {
+				s.setTask(c, id, tp, cacheMap)
+			}
+		}
+		if len(cacheMap) > 0 {
+			s.redis.SetWeight(c, cacheMap)
+		}
+		firstid = lastid
+		time.Sleep(time.Second)
+	}
+}
+
+func (s *Service) upTaskWeightDB(c context.Context, item *task.WeightLog) (err error) {
+	// 超时还没入库的直接丢掉，免得阻塞
+	if time.Since(item.Uptime.TimeValue()).Minutes() > 3.0 {
+		log.Warn("task weight item(%v) expired! ", item)
+		return
+	}
+	_, err = s.arc.UpTaskWeight(c, item.TaskID, item.Weight)
+	if err != nil {
+		log.Error("UpTaskWeight(%d,%d) error(%v)", item.Weight, item.TaskID, err)
+		return
+	}
+	return
+}
+
+func (s *Service) setTask(c context.Context, taskid int64, tp *task.WeightParams, cm map[int64]*task.WeightParams) {
+	tpc := s.newPriority(c, tp)
+	// 上轮次权重前2000的这轮也必须更新
+	if _, ok := s.lastjumpMap[tpc.TaskID]; ok {
+		select {
+		case s.jumpchan <- tpc:
+		default:
+			log.Warn("jumpchan full")
+		}
+	} else {
+		s.jumplist.PUSH(tpc) //插队序列
+	}
+
+	tp.Weight = tpc.Weight
+	if cm != nil {
+		cm[taskid] = tp
+	}
+	select {
+	case s.tasklogchan <- tpc:
+	default:
+		log.Info("s.tasklogchan full(%d)", len(s.tasklogchan))
+	}
+}
+
+// 任务权重计算
+func (s *Service) newPriority(c context.Context, tp *task.WeightParams) (tpc *task.WeightLog) {
+	var round int64
+	var wcf = task.WLVConf
+	tpc = &task.WeightLog{
+		TaskID: tp.TaskID,
+		Mid:    tp.Mid,
+	}
+
+	if tp.AccFailed {
+		if tp.Fans, tp.AccFailed = s.getUpperFans(c, tp.Mid); !tp.AccFailed {
+			tp.Special = s.getUpSpecial(tp.Mid, tp.Fans)
+			s.arc.SetUpSpecial(c, tp.TaskID, tp.Special)
+		}
+	}
+
+	round = int64(time.Since(tp.Ctime.TimeValue()).Minutes()) / _pm
+	//定时任务
+	if !tp.Ptime.TimeValue().IsZero() {
+		pround := int64((time.Until(tp.Ptime.TimeValue()).Minutes())) / 3
+		sumtesmp := wcf.Tlv4 * (int64(tp.Ptime.TimeValue().Sub(tp.Ctime.TimeValue()).Minutes()) - 60*4) / 3
+		switch {
+		case pround >= _tc1:
+			tpc.TWeight = round * wcf.Tlv4
+		case _tc2 <= pround && pround < _tc1:
+			tpc.TWeight = (_tc1-pround)*wcf.Tlv1 + sumtesmp
+		case _tc3 <= pround && pround < _tc2:
+			tpc.TWeight = (_tc2-pround)*wcf.Tlv2 + wcf.Tsum2h + sumtesmp
+		case pround < _tc3:
+			tpc.TWeight = (_tc3-pround)*wcf.Tlv3 + wcf.Tsum1h + sumtesmp
+		}
+		tpc.NWeight = 0
+	} else { // 普通任务加权
+		switch {
+		case round < _nc1:
+			tpc.NWeight = wcf.Nlv5 * round
+		case _nc1 <= round && round < _nc2:
+			tpc.NWeight = wcf.Nlv1*(round-_nc1) + wcf.Nsum9
+		case _nc2 <= round && round < _nc3:
+			tpc.NWeight = wcf.Nlv2*(round-_nc2) + wcf.Nsum15
+		case _nc3 <= round && round < _nc4:
+			tpc.NWeight = wcf.Nlv3*(round-_nc3) + wcf.Nsum27
+		default:
+			tpc.NWeight = wcf.Nlv4*(round-_nc4) + wcf.Nsum45
+		}
+		tpc.TWeight = 0
+	}
+
+	// 特殊任务加权
+	switch tp.Special {
+	case task.UpperBigNormal:
+		tpc.SWeight = wcf.Slv1 * round
+	case task.UpperSuperNormal:
+		tpc.SWeight = wcf.Slv2 * round
+	case task.UpperWhite:
+		tpc.SWeight = wcf.Slv3 * round
+	case task.UpperBigWhite:
+		tpc.SWeight = wcf.Slv4 * round
+	case task.UpperSuperWhite:
+		tpc.SWeight = wcf.Slv5 * round
+	case task.UpperSuperBlack:
+		tpc.SWeight = wcf.Slv6 * round
+	case task.UpperBlack:
+		tpc.SWeight = wcf.Slv7 * round
+	default:
+		tpc.SWeight = 0
+	}
+
+	//配置任务加权
+	tpc.CWeight = 0
+	if len(tp.CfItems) > 0 {
+		tpc.CfItems = tp.CfItems
+		for _, item := range tp.CfItems {
+			if item.Rule == 0 {
+				round2 := int64(time.Since(item.Mtime.TimeValue()).Minutes()) / _pm
+				if round2 < round {
+					round = round2
+				}
+				tpc.CWeight += round * item.Weight
+			} else {
+				tpc.CWeight += item.Weight
+			}
+		}
+		if tpc.CWeight <= wcf.MinWeight {
+			tpc.CWeight = wcf.MinWeight
+		}
+	}
+
+	tpc.Weight = tpc.NWeight + tpc.SWeight + tpc.TWeight + tpc.CWeight
+	if tpc.Weight >= wcf.MaxWeight {
+		tpc.Weight = wcf.MaxWeight
+	}
+	tpc.Uptime = utils.NewFormatTime(time.Now())
+	return
+}
+
+// 设置特殊用户的审核任务
+func (s *Service) setTaskUPSpecial(c context.Context, t *task.Task, mid int64) (fans int64, failed bool) {
+	fans, failed = s.getUpperFans(c, mid)
+	t.UPSpecial = s.getUpSpecial(mid, fans)
+	return
+}
+
+func (s *Service) getUpSpecial(mid int64, fans int64) (upspecial int8) {
+	switch {
+	case s.isWhite(mid): //优质
+		if fans >= task.SuperUpperTH {
+			upspecial = task.UpperSuperWhite
+		} else if fans >= task.BigUpperTH {
+			upspecial = task.UpperSuperWhite
+		} else {
+			upspecial = task.UpperWhite
+		}
+	case s.isBlack(mid):
+		if fans >= task.SuperUpperTH {
+			upspecial = task.UpperSuperBlack
+		} else {
+			upspecial = task.UpperBlack
+		}
+
+	default:
+		if fans >= task.SuperUpperTH {
+			upspecial = task.UpperSuperNormal
+		} else if fans >= task.BigUpperTH {
+			upspecial = task.UpperBigNormal
+		}
+	}
+	return
+}
+
+// 设置定时发布的审核任务
+func (s *Service) setTaskTimed(c context.Context, t *task.Task) {
+	adelay, err := s.arc.Delay(c, t.Aid)
+	if err != nil {
+		log.Error("s.arc.Delay(%d) error(%v)", t.Aid, err)
+		return
+	}
+	if adelay != nil && adelay.State == 0 && !adelay.DTime.IsZero() {
+		t.Ptime = utils.NewFormatTime(adelay.DTime)
+	}
+}
+
+func (s *Service) setTaskUpFrom(c context.Context, aid int64, tp *task.WeightParams) {
+	if addit, _ := s.arc.Addit(c, aid); addit != nil {
+		tp.UpFrom = addit.UpFrom
+	}
+}
+
+func (s *Service) setTaskUpGroup(c context.Context, mid int64, tp *task.WeightParams) {
+	ugs := []int8{}
+	for gid, cache := range s.upperCache {
+		if _, ok := cache[mid]; ok {
+			ugs = append(ugs, gid)
+		}
+	}
+	tp.UpGroups = ugs
+}
+
+// 设置配置用户权重的审核任务
+func (s *Service) getConfWeight(c context.Context, t *task.Task, a *archive.Archive) (cfitems []*task.ConfigItem) {
+	// 1. 按照用户
+	if confs, ok := s.weightCache[task.WConfMid]; ok {
+		if conf, ok := confs[a.Mid]; ok {
+			cfitems = append(cfitems, conf)
+		}
+	}
+	// 2. 按照分区
+	if confs, ok := s.weightCache[task.WConfType]; ok {
+		if conf, ok := confs[int64(a.TypeID)]; ok {
+			cfitems = append(cfitems, conf)
+		}
+	}
+	// 3. 按照投稿来源
+	if confs, ok := s.weightCache[task.WConfUpFrom]; ok {
+		addit, err := s.arc.Addit(c, a.ID)
+		if addit == nil {
+			if err != nil {
+				log.Error(" s.arc.Addit(%d) error(%v)", t.Aid, err)
+			}
+			return
+		}
+
+		if conf, ok := confs[int64(addit.UpFrom)]; ok {
+			cfitems = append(cfitems, conf)
+		}
+	}
+	return
+}
+
+// 权重配置
+func (s *Service) weightConf(c context.Context) (cfc map[int8]map[int64]*task.ConfigItem, err error) {
+	var (
+		ids    []int64
+		arrcfs []*task.ConfigItem
+	)
+	if arrcfs, err = s.arc.WeightConf(context.TODO()); err != nil {
+		log.Error("s.arc.WeightConf error(%v)", err)
+		return
+	}
+	cfc = map[int8]map[int64]*task.ConfigItem{}
+	for _, item := range arrcfs {
+		if !item.Et.TimeValue().IsZero() && item.Et.TimeValue().Before(time.Now()) {
+			ids = append(ids, item.ID)
+			continue
+		}
+		if _, ok := cfc[item.Radio]; !ok {
+			cfc[item.Radio] = make(map[int64]*task.ConfigItem)
+		}
+		cfc[item.Radio][item.CID] = item
+	}
+	if len(ids) > 0 {
+		log.Info("task config(%v) 权重配置已过期,自动失效", ids)
+		s.arc.DelWeightConfs(c, ids)
+	}
+	return
+}
+
+func (s *Service) taskWeightConsumer() {
+	var c = context.TODO()
+	defer s.waiter.Done()
+
+	for {
+		if s.closed {
+			return
+		}
+		select {
+		case item, ok := <-s.jumpchan: //插队序列
+			if !ok {
+				log.Error("s.jumpchan closed")
+				return
+			}
+			s.upTaskWeightDB(c, item)
+		case item, ok := <-s.tasklogchan:
+			if !ok {
+				log.Error("s.tasklogchan closed")
+				return
+			}
+			s.hbase.AddLog(c, item)
+		default:
+			time.Sleep(time.Second)
+		}
+	}
+}
+
+func (s *Service) taskweightproc() {
+	defer s.waiter.Done()
+	for {
+		if s.closed {
+			return
+		}
+		s.upTaskWeightCache()
+		log.Info("taskweightproc 插队序列(%d) 日志序列(%d)", len(s.jumpchan), len(s.tasklogchan))
+		time.Sleep(3 * time.Minute)
+	}
+}

app/job/main/videoup-report/service/track.go

@@ -0,0 +1,91 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+)
+
+func (s *Service) trackArchive(nw *archive.Archive, old *archive.Archive) (err error) {
+	var (
+		bs      []byte
+		remarks = make(map[string]string)
+	)
+	if addit, _ := s.arc.Addit(context.TODO(), nw.ID); addit != nil {
+		remarks["dynamic"] = addit.Dynamic
+		if addit.MissionID > 0 {
+			remarks["mission_id"] = strconv.FormatInt(addit.MissionID, 10)
+		}
+	}
+	if old == nil {
+		remarks["cover"] = nw.Cover
+		remarks["desc"] = nw.Content
+		remarks["title"] = nw.Title
+		remarks["typeid"] = strconv.Itoa(int(nw.TypeID))
+		remarks["copyright"] = strconv.Itoa(int(nw.Copyright))
+		bs, _ = json.Marshal(remarks)
+	} else if nw.State != old.State || nw.Access != old.Access || nw.Round != old.Round || nw.Content != old.Content ||
+		nw.Cover != old.Cover || nw.Title != old.Title || nw.TypeID != old.TypeID || nw.Copyright != old.Copyright || nw.Attribute != old.Attribute {
+		if nw.Cover != old.Cover {
+			remarks["cover"] = nw.Cover
+		}
+		if nw.Content != old.Content {
+			remarks["desc"] = nw.Content
+		}
+		if nw.Title != old.Title {
+			remarks["title"] = nw.Title
+		}
+		if nw.TypeID != old.TypeID {
+			remarks["typeid"] = strconv.Itoa(int(nw.TypeID))
+		}
+		if nw.Copyright != old.Copyright {
+			remarks["copyright"] = strconv.Itoa(int(nw.Copyright))
+		}
+		if len(remarks) != 0 {
+			bs, _ = json.Marshal(remarks)
+		}
+		if nw.State >= int(archive.StateOpen) && nw.Access == int(archive.AccessMember) {
+			nw.State = int(archive.AccessMember)
+		}
+	} else {
+		// NOTE: nothing modify
+		return
+	}
+	s.arc.AddTrack(context.TODO(), nw.ID, nw.State, nw.Round, nw.Attribute, string(bs), nw.MTime, nw.MTime)
+	return
+}
+
+func (s *Service) trackVideo(nw *archive.Video, old *archive.Video) (err error) {
+	var (
+		remarks = make(map[string]interface{})
+		bs      []byte
+	)
+	if old == nil {
+		if nw.Title != "" {
+			remarks["title"] = nw.Title
+		}
+		if nw.Desc != "" {
+			remarks["desc"] = nw.Desc
+		}
+	} else if nw.XcodeState != old.XcodeState || nw.Status != old.Status || nw.Title != old.Title || nw.Desc != old.Desc || nw.Attribute != old.Attribute {
+		if nw.FailCode != archive.XcodeFailZero {
+			remarks["xcode_fail"] = nw.FailCode
+		}
+		if nw.Title != old.Title && nw.Title != "" {
+			remarks["title"] = nw.Title
+		}
+		if nw.Desc != old.Desc && nw.Desc != "" {
+			remarks["desc"] = nw.Desc
+		}
+	} else {
+		// no change
+		return
+	}
+	if len(remarks) != 0 {
+		bs, err = json.Marshal(remarks)
+	}
+	s.arc.AddVideoTrack(context.TODO(), nw.Aid, nw.Filename, nw.Status, nw.XcodeState, string(bs), nw.MTime, nw.MTime)
+	return
+}

app/job/main/videoup-report/service/ups.go

@@ -0,0 +1,37 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"go-common/app/job/main/videoup-report/model/manager"
+	"go-common/library/log"
+)
+
+// hdlManagerUpsBinlog UP主分组表变更事件
+func (s *Service) hdlManagerUpsBinlog(msg *manager.BinMsg) {
+	defer func() {
+		if pErr := recover(); pErr != nil {
+			log.Error("s.hdlManagerUpsBinlog() panic(%v)", pErr)
+		}
+	}()
+	var (
+		oldUps, newUps *manager.Ups
+		err            error
+		c              = context.TODO()
+	)
+	if msg.New != nil {
+		if err = json.Unmarshal(msg.New, &newUps); err != nil {
+			log.Error("s.hdlManagerUpsBinlog() json.Unmarshal error(%v) msg.new(%s)", err, string(msg.New))
+			return
+		}
+	}
+	if msg.Old != nil {
+		if err = json.Unmarshal(msg.Old, &oldUps); err != nil {
+			log.Error("s.hdlManagerUpsBinlog() json.Unmarshal error(%v) msg.old(%s)", err, string(msg.Old))
+			return
+		}
+	}
+	if newUps != nil && newUps.Type == manager.UpTypeExcitationWhite { //新增UP主到激励回查白名单，需要将该UP主的未回查的稿件从激励回查去除
+		go s.ignoreUpsExcitation(c, newUps.MID)
+	}
+}

app/job/main/videoup-report/service/ups_test.go

@@ -0,0 +1,20 @@
+package service
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/job/main/videoup-report/model/manager"
+	"testing"
+)
+
+func TestHdlManagerUpsBinlog(t *testing.T) {
+	Convey("hdlManagerUpsBinlog", t, func() {
+		str := "{\"ctime\":\"2018-10-31 15:29:18\",\"id\":502,\"mid\":27515256,\"mtime\":\"2018-10-31 15:29:18\",\"note\":\"1111\",\"type\":18,\"uid\":277}"
+		bs := []byte(str)
+		m := &manager.BinMsg{
+			Action: "insert",
+			Table:  "ups",
+			New:    bs,
+		}
+		s.hdlManagerUpsBinlog(m)
+	})
+}

app/job/main/videoup-report/service/video.go

@@ -0,0 +1,364 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"sort"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/videoup-report/model/archive"
+	"go-common/library/log"
+)
+
+// VideoReports get video report record from DB
+func (s *Service) VideoReports(c context.Context, t int8, stime, etime time.Time) (reports []*archive.Report, err error) {
+	if reports, err = s.arc.Reports(c, t, stime, etime); err != nil {
+		log.Error("s.arc.Reports(%d) err(%v)", t, err)
+		return
+	}
+	return
+}
+
+// hdlVideoUpdateBinLog handle bilibili_archive's video table update bin log
+func (s *Service) hdlVideoUpdateBinLog(nMsg, oMsg []byte) {
+	var (
+		nv  = &archive.Video{}
+		ov  = &archive.Video{}
+		err error
+	)
+	if err = json.Unmarshal(nMsg, nv); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", nMsg, err)
+		return
+	}
+	if err = json.Unmarshal(oMsg, ov); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", oMsg, err)
+		return
+	}
+	if nv.Status != ov.Status {
+		s.hdlVideoAudit(*nv, *ov)
+	}
+	if ov.XcodeState != nv.XcodeState {
+		s.hdlXcodeTime(*nv, *ov)
+	}
+	// 视频状态变为待审核(视频信息改动或者一转完成)
+	if nv.Status != ov.Status {
+		if nv.Status == archive.VideoStatusWait { //待审核
+			s.hdlVideoTask(context.TODO(), nv.Filename)
+		}
+		if nv.Status == archive.VideoStatusDelete { //视频删除
+			s.arc.DelDispatch(context.TODO(), nv.Aid, nv.Cid)
+		}
+	}
+}
+
+// hdlVideoAudit handle video audit stats
+func (s *Service) hdlVideoAudit(video, oldVideo archive.Video) {
+	var (
+		err error
+		arc = &archive.Archive{}
+	)
+	if arc, err = s.arc.ArchiveByAid(context.TODO(), video.Aid); err != nil {
+		log.Error("s.arc.ArchiveByAid(%d) error(%v)", video.Aid, err)
+		return
+	}
+	s.videoAuditCache.Lock()
+	defer s.videoAuditCache.Unlock()
+	if _, ok := s.videoAuditCache.Data[arc.TypeID]; !ok {
+		s.videoAuditCache.Data[arc.TypeID] = make(map[string]int)
+	}
+	switch video.Status {
+	case archive.VideoStatusWait:
+		s.videoAuditCache.Data[arc.TypeID]["auditing"]++
+	case archive.VideoStatusOpen:
+		s.videoAuditCache.Data[arc.TypeID]["audited"]++
+	}
+}
+
+// hdlVideoAuditCount handle audit stats count
+func (s *Service) hdlVideoAuditCount() {
+	var (
+		err    error
+		report *archive.Report
+		ctime  = time.Now()
+		mtime  = ctime
+		bs     []byte
+	)
+	if report, err = s.arc.ReportLast(context.TODO(), archive.ReportTypeVideoAudit); err != nil {
+		log.Error("s.arc.ReportLast(%d) error(%v)", archive.ReportTypeVideoAudit, err)
+		return
+	}
+	if report != nil && time.Now().Unix()-report.CTime.Unix() < 60*5 {
+		log.Info("s.arc.ReportLast(%d) 距离上一次写入还没过5分钟!", archive.ReportTypeVideoAudit)
+		return
+	}
+	s.videoAuditCache.Lock()
+	defer s.videoAuditCache.Unlock()
+	if bs, err = json.Marshal(s.videoAuditCache.Data); err != nil {
+		log.Error("json.Marshal(%v) error(%v)", s.videoAuditCache.Data, err)
+		return
+	}
+	if _, err = s.arc.ReportAdd(context.TODO(), archive.ReportTypeVideoAudit, string(bs), ctime, mtime); err != nil {
+		log.Error("s.arc.ReportAdd(%d,%s,%v,%v) error(%v)", archive.ReportTypeVideoAudit, string(bs), ctime, mtime, err)
+		return
+	}
+	s.videoAuditCache.Data = make(map[int16]map[string]int)
+}
+
+// VideoAudit get video audit by typeid
+func (s *Service) VideoAudit(c context.Context, stime, etime time.Time) (reports []*archive.Report, err error) {
+	if reports, err = s.arc.Reports(c, archive.ReportTypeVideoAudit, stime, etime); err != nil {
+		log.Error("s.arc.Reports(%d) err(%v)", archive.ReportTypeVideoAudit, err)
+		return
+	}
+	return
+}
+
+// hdlXcodeTime Stats video xcode spend time.
+func (s *Service) hdlXcodeTime(nv, ov archive.Video) {
+	if nv.XcodeState != archive.VideoXcodeSDFinish && nv.XcodeState != archive.VideoXcodeHDFinish && nv.XcodeState != archive.VideoDispatchFinish {
+		return
+	}
+	var (
+		nMt time.Time
+		oMt time.Time
+		err error
+	)
+	s.xcodeTimeCache.Lock()
+	defer s.xcodeTimeCache.Unlock()
+	if nMt, err = time.ParseInLocation("2006-01-02 15:04:05", nv.MTime, time.Local); err != nil {
+		log.Error("time.ParseInLocation(%s) err(%v)", nv.MTime, err)
+		return
+	}
+	if oMt, err = time.ParseInLocation("2006-01-02 15:04:05", ov.MTime, time.Local); err != nil {
+		log.Error("time.ParseInLocation(%s) err(%v)", ov.MTime, err)
+		return
+	}
+	t := int(nMt.Unix() - oMt.Unix())
+	if t <= 0 {
+		log.Info("warning: xcode spend time: %d", t)
+		return
+	}
+	s.xcodeTimeCache.Data[nv.XcodeState] = append(s.xcodeTimeCache.Data[nv.XcodeState], t)
+}
+
+// hdlXcodeStats handle calculate and save hdlXcodeTime() stats result
+func (s *Service) hdlXcodeStats() {
+	var (
+		c          = context.TODO()
+		states     = []int8{archive.VideoXcodeSDFinish, archive.VideoXcodeHDFinish, archive.VideoDispatchFinish} //xcode states need stats
+		levels     = []int8{50, 60, 80, 90}
+		xcodeStats = make(map[int8]map[string]int)
+		bs         []byte
+		err        error
+		ctime      = time.Now()
+		mtime      = ctime
+	)
+
+	for _, st := range states {
+		if _, ok := s.xcodeTimeCache.Data[st]; !ok {
+			continue
+		}
+		sort.Ints(s.xcodeTimeCache.Data[st])
+		seconds := s.xcodeTimeCache.Data[st]
+		if len(seconds) < 1 {
+			continue
+		}
+		for _, l := range levels {
+			m := "m" + strconv.Itoa(int(l))
+			o := int(math.Floor(float64(len(seconds))*(float64(l)/100)+0.5)) - 1 //seconds offset
+			if o < 0 {
+				continue
+			}
+			if o < 0 || o >= len(seconds) {
+				log.Error("s.hdlVideoXcodeStats() index out of range. seconds(%d)", o)
+				continue
+			}
+			if _, ok := xcodeStats[st]; !ok {
+				xcodeStats[st] = make(map[string]int)
+			}
+			xcodeStats[st][m] = seconds[o]
+		}
+	}
+	if bs, err = json.Marshal(xcodeStats); err != nil {
+		log.Error("s.hdlVideoXcodeStats() json.Marshal error(%v)", err)
+		return
+	}
+	log.Info("s.hdlVideoXcodeStats() end xcode stats xcodeStats:%s", bs)
+	if len(xcodeStats) < 1 {
+		log.Info("s.hdlVideoXcodeStats() end xcode stats ignore empty data")
+		return
+	}
+	if _, err = s.arc.ReportAdd(c, archive.ReportTypeXcode, string(bs), ctime, mtime); err != nil {
+		log.Error("s.hdlVideoXcodeStats() s.arc.ReportAdd error(%v)", err)
+		return
+	}
+	s.xcodeTimeCache.Lock()
+	defer s.xcodeTimeCache.Unlock()
+	s.xcodeTimeCache.Data = make(map[int8][]int)
+}
+
+// hdlTraffic Calculate how long it took to check video flow in ten minutes.
+// Stats result include sd_xcode,video check,hd_xcode,dispatch time.
+func (s *Service) hdlTraffic() {
+	var (
+		err        error
+		ctx        = context.TODO()
+		report     *archive.Report                                                                                                        //Single report type
+		reports    []*archive.Report                                                                                                      //Report type slice
+		tooks      []*archive.TaskTook                                                                                                    //Task took time stats
+		statsCache = make(map[int8]map[string][]int)                                                                                      //Event took time list
+		traffic    = make(map[int8]map[string]int)                                                                                        //Event took time stats result
+		bs         []byte                                                                                                                 //Json byte
+		ctime      = time.Now()                                                                                                           //Stats create time
+		mtime      = ctime                                                                                                                //Stats modify time
+		states     = []int8{archive.VideoUploadInfo, archive.VideoXcodeSDFinish, archive.VideoXcodeHDFinish, archive.VideoDispatchFinish} //xcode states need stats
+	)
+
+	//0.Get the last report write time. If less than 10 minutes, then return.
+	if report, err = s.arc.ReportLast(ctx, archive.ReportTypeTraffic); err != nil {
+		log.Error("s.arc.ReportLast(%d) error(%v)", archive.ReportTypeTraffic, err)
+		return
+	}
+	if report != nil && time.Now().Unix()-report.CTime.Unix() < 60*6 {
+		log.Info("s.arc.ReportLast(%d) 距离上一次写入还没过6分钟!", archive.ReportTypeTraffic)
+		return
+	}
+	now := time.Now()
+	stime := now.Add(-10 * time.Minute)
+
+	//1.Get video task time stats.
+	if tooks, err = s.arc.TaskTooks(ctx, stime); err != nil {
+		log.Error("s.arc.TaskTooks(%v) error(%v)", stime, err)
+		return
+	}
+	statsCache[archive.VideoUploadInfo] = make(map[string][]int)
+	for _, v := range tooks {
+		statsCache[archive.VideoUploadInfo]["m50"] = append(statsCache[archive.VideoUploadInfo]["m50"], v.M50)
+		statsCache[archive.VideoUploadInfo]["m60"] = append(statsCache[archive.VideoUploadInfo]["m60"], v.M60)
+		statsCache[archive.VideoUploadInfo]["m80"] = append(statsCache[archive.VideoUploadInfo]["m80"], v.M80)
+		statsCache[archive.VideoUploadInfo]["m90"] = append(statsCache[archive.VideoUploadInfo]["m90"], v.M90)
+	}
+
+	//2.Get sd_xcode,hd_xcode,dispatch time stats.
+	if reports, err = s.arc.Reports(ctx, archive.ReportTypeXcode, stime, now); err != nil {
+		log.Error("s.arc.Reports(%d) err(%v)", archive.ReportTypeXcode, err)
+		return
+	}
+	xcodeStats := make(map[int8]map[string]int)
+	for _, v := range reports {
+		err = json.Unmarshal([]byte(v.Content), &xcodeStats)
+		if err != nil {
+			log.Error("json.Unmarshal(%s) err(%v)", v.Content, err)
+			continue
+		}
+		for state, stats := range xcodeStats {
+			if _, ok := statsCache[state]; !ok {
+				statsCache[state] = make(map[string][]int)
+			}
+			totalTime := 0
+			for level, val := range stats {
+				totalTime += val
+				statsCache[state][level] = append(statsCache[state][level], val)
+			}
+		}
+	}
+
+	//3.Calculate total time stats.
+	for state, stats := range statsCache {
+		for level, vals := range stats {
+			total := 0
+			for _, v := range vals {
+				total += v
+			}
+			if _, ok := traffic[state]; !ok {
+				traffic[state] = make(map[string]int)
+			}
+			traffic[state][level] = total / len(vals)
+		}
+	}
+
+	//4.Save stats result
+	if len(traffic) < 1 {
+		log.Info("s.hdlTraffic() end traffic stats ignore empty data")
+		return
+	}
+	if bs, err = json.Marshal(traffic); err != nil {
+		log.Error("s.hdlTraffic() json.Marshal error(%v)", err)
+		return
+	}
+	log.Info("s.hdlTraffic() end traffic stats traffic:%s", bs)
+	if _, err = s.arc.ReportAdd(ctx, archive.ReportTypeTraffic, string(bs), ctime, mtime); err != nil {
+		log.Error("s.hdlVideoXcodeStats() s.arc.ReportAdd error(%v)", err)
+		return
+	}
+
+	//5.Update video traffic jam time
+	jamTime := 0
+	stateOk := true
+	for _, s := range states {
+		if _, ok := traffic[s]; !ok {
+			stateOk = false
+			break
+		}
+		if _, ok := traffic[s]["m60"]; !ok {
+			stateOk = false
+			break
+		}
+		if _, ok := traffic[s]["m80"]; !ok {
+			stateOk = false
+			break
+		}
+		jamTime += traffic[s]["m60"]
+		jamTime += traffic[s]["m80"]
+	}
+	if !stateOk {
+		log.Error("s.hdlTraffic() 一审耗时计算失败！traffic：%v", traffic)
+	} else {
+		err = s.redis.SetVideoJam(ctx, jamTime)
+		log.Info("s.hdlTraffic() s.redis.SetVideoJam(%d)", jamTime)
+		if err != nil {
+			log.Error("s.hdlTraffic() 更新Redis失败！error(%v)", err)
+		}
+	}
+}
+
+func (s *Service) putVideoChan(action string, nwMsg []byte, oldMsg []byte) {
+	var (
+		err      error
+		chanSize = int64(s.c.ChanSize)
+	)
+	nw := &archive.Video{}
+	if err = json.Unmarshal(nwMsg, nw); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", nwMsg, err)
+		return
+	}
+	switch action {
+	case _insertAct:
+		s.videoUpInfoChs[nw.Aid%chanSize] <- &archive.VideoUpInfo{Nw: nw, Old: nil}
+	case _updateAct:
+		old := &archive.Video{}
+		if err = json.Unmarshal(oldMsg, old); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", oldMsg, err)
+			return
+		}
+		s.videoUpInfoChs[nw.Aid%chanSize] <- &archive.VideoUpInfo{Nw: nw, Old: old}
+	}
+}
+
+func (s *Service) upVideoproc(k int) {
+	defer s.waiter.Done()
+	for {
+		var (
+			ok     bool
+			upInfo *archive.VideoUpInfo
+		)
+		if upInfo, ok = <-s.videoUpInfoChs[k]; !ok {
+			log.Info("s.videoUpInfoCh[%d] closed", k)
+			return
+		}
+		s.trackVideo(upInfo.Nw, upInfo.Old)
+		go s.hdlMonitorVideo(upInfo.Nw, upInfo.Old)
+	}
+}

app/job/main/videoup-report/service/video_test.go

@@ -0,0 +1,16 @@
+package service
+
+import (
+	"go-common/app/job/main/videoup-report/conf"
+	"testing"
+)
+
+// TestHdlTraffic Test calculate video check spend time function.
+func TestHdlTraffic(t *testing.T) {
+	err := conf.Init()
+	if err != nil {
+		return
+	}
+	s := New(conf.Conf)
+	s.hdlTraffic()
+}