package service import ( "context" "encoding/json" "strconv" "strings" "time" "go-common/app/job/main/creative/model" "go-common/library/log" ) const ( _archive = "archive" _insert = "insert" _update = "update" fromArchiveUp = 0 fromArchiveNewUp = 1 _relationMidTable = "user_relation_mid_" _relationStatTable = "user_relation_stat_" ) //pub up auth msg. func (s *Service) pub(mid int64, from, isAuthor int) (err error) { c := context.TODO() msg := &model.Msg{ MID: mid, From: from, IsAuthor: isAuthor, TimeStamp: time.Now().Unix(), } if err = s.upPub.Send(c, strconv.FormatInt(mid, 10), msg); err != nil { log.Error("pub mid(%d) error(%v)", mid, err) } return } func (s *Service) arcNotifyCanalConsume() { var err error for msg := range s.arcNotifySub.Messages() { msg.Commit() s.arcNotifyMo++ m := &model.CanalMsg{} if err = json.Unmarshal(msg.Value, m); err != nil { log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if m.Table == _archive { s.arcNotifyMsg(m) log.Info("arcNotifyCanalConsume key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) } } s.wg.Done() } func (s *Service) arcNotifyMsg(m *model.CanalMsg) { var ( err error newArc = &model.Archive{} oldArc = &model.Archive{} ) if err = json.Unmarshal(m.New, newArc); err != nil { log.Error("arcNotifyMsg newMsg json.Unmarshal(%s) error(%v)", string(m.New), err) return } mid := newArc.MID if mid <= 0 { log.Error("arcNotifyMsg mid (%d) error", mid) return } if m.Action == _insert && newArc.State >= 0 { //0->1 s.pub(mid, fromArchiveUp, 1) } else if m.Action == _update { if err = json.Unmarshal(m.Old, oldArc); err != nil { log.Error("arcNotifyMsg oldMsg json.Unmarshal(%s) error(%v)", string(m.Old), err) return } if oldArc.State < 0 && (newArc.State >= 0 || newArc.State == -6) { //0->1 s.pub(mid, fromArchiveUp, 1) } if (oldArc.State >= 0 || oldArc.State == -6) && newArc.State < 0 { //1->0 cnt, err := s.arc.UpCount(context.Background(), mid) if err != nil { log.Error("arcNotifyMsg s.arc.UpCount(%d) error(%v)", mid, err) return } if cnt <= 0 { s.pub(mid, fromArchiveUp, 0) } } } } func (s *Service) arcCanalConsume() { var err error for msg := range s.arcSub.Messages() { msg.Commit() s.arcMo++ m := &model.CanalMsg{} if err = json.Unmarshal(msg.Value, m); err != nil { log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if m.Table == _archive && m.Action == _insert { arc := &model.Archive{} if err = json.Unmarshal(m.New, arc); err != nil { log.Error("creative-job binglog newMsg json.Unmarshal(%s) error(%v)", m.New, err) continue } if arc.MID > 0 { s.pub(arc.MID, fromArchiveNewUp, 1) } log.Info("arcCanalConsume key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) } } s.wg.Done() } func (s *Service) task() { for msg := range s.taskSub.Messages() { msg.Commit() log.Info("databus task key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.taskSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- msg } for _, c := range s.taskSubQueue { close(c) } s.wg.Done() } func (s *Service) share() { for msg := range s.shareSub.Messages() { msg.Commit() m := &model.ShareMsg{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus share json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if m.TP != 3 { continue } mid, authorMID := m.MID, s.getMIDByAID(m.OID) if mid != authorMID { //不是该用户分享的稿件则不做任何处理 log.Warn("s.arc.Archive mid(%d)|author mid(%d)", mid, authorMID) continue } log.Info("databus share key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.shareSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m } for _, c := range s.shareSubQueue { close(c) } s.wg.Done() } func (s *Service) relation() { var err error newF, advancedF := s.c.Task.NewFollower, s.c.Task.AdvancedFollower for msg := range s.relationSub.Messages() { msg.Commit() rl := &model.RelaMessage{} if err = json.Unmarshal(msg.Value, rl); err != nil { log.Error("databus relation json.Unmarshal (%v) error(%v)", msg.Value, err) continue } if !strings.HasPrefix(rl.Table, _relationStatTable) && !strings.HasPrefix(rl.Table, _relationMidTable) { continue } if strings.HasPrefix(rl.Table, _relationStatTable) { ost := &model.Stat{} st := &model.Stat{} if rl.Action == "update" { if err = json.Unmarshal(rl.Old, ost); err != nil { log.Error("relation old msg json.Unmarshal(%s) error(%v)", string(rl.Old), err) continue } } if err = json.Unmarshal(rl.New, st); err != nil { log.Error("relation new msg json.Unmarshal(%s) error(%v)", string(rl.New), err) continue } isFollower := false if ost.Follower < newF && st.Follower >= newF { //新手任务粉丝数限制 isFollower = true } if ost.Follower < advancedF && st.Follower >= advancedF { //进阶任务粉丝数限制 isFollower = true } if isFollower { log.Info("databus relation key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.followerQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- st } } if strings.HasPrefix(rl.Table, _relationMidTable) { fl := &model.Relation{} if err = json.Unmarshal(rl.New, fl); err != nil { log.Error("json.Unmarshal(%s) error(%v)", string(rl.New), err) continue } if fl.FID != s.c.Task.BiliMID { //过滤关注哔哩哔哩创作中心 continue } log.Info("databus relation key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.relationQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- fl } } for _, c := range s.relationQueue { close(c) } for _, c := range s.followerQueue { close(c) } s.wg.Done() } func (s *Service) statView() { statView, statViewUp := s.c.Task.StatView, s.c.Task.StatViewUp for msg := range s.statViewSub.Messages() { msg.Commit() m := &model.StatView{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus statView json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if !strings.EqualFold(m.Type, "archive") { continue } if m.Count >= statView && m.Count <= statViewUp { s.statViewSubQueue[s.shardingQueueIndex(msg.Key, s.statViewQueueLen)] <- m } } for _, c := range s.statViewSubQueue { close(c) } s.wg.Done() } func (s *Service) statLike() { statLike, statLikeUp := s.c.Task.StatLike, s.c.Task.StatLikeUp for msg := range s.statLikeSub.Messages() { msg.Commit() m := &model.StatLike{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus statLike json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if !strings.EqualFold(m.Type, "archive") { continue } if m.Count >= statLike && m.Count <= statLikeUp { s.statLikeSubQueue[s.shardingQueueIndex(msg.Key, s.statLikeQueueLen)] <- m } } for _, c := range s.statLikeSubQueue { close(c) } s.wg.Done() } func (s *Service) statShare() { statShare, statShareUp := s.c.Task.StatShare, s.c.Task.StatShareUp for msg := range s.statShareSub.Messages() { msg.Commit() m := &model.StatShare{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus statShare json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if !strings.EqualFold(m.Type, "archive") { continue } if m.Count >= statShare && m.Count <= statShareUp { log.Info("databus statShare key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.statShareSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m } } for _, c := range s.statShareSubQueue { close(c) } s.wg.Done() } func (s *Service) statCoin() { statCoin, statCoinUp := s.c.Task.StatCoin, s.c.Task.StatCoinUp for msg := range s.statCoinSub.Messages() { msg.Commit() m := &model.StatCoin{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus statCoin json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if !strings.EqualFold(m.Type, "archive") { continue } if m.Count >= statCoin && m.Count <= statCoinUp { log.Info("databus statCoin key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.statCoinSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m } } for _, c := range s.statCoinSubQueue { close(c) } s.wg.Done() } func (s *Service) statFav() { statFav, statFavUp := s.c.Task.StatFav, s.c.Task.StatFavUp for msg := range s.statFavSub.Messages() { msg.Commit() m := &model.StatFav{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus statFav json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if !strings.EqualFold(m.Type, "archive") { continue } if m.Count >= statFav && m.Count <= statFavUp { log.Info("databus statFav key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.statFavSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m } } for _, c := range s.statFavSubQueue { close(c) } s.wg.Done() } func (s *Service) statReply() { statReply, statReplyUp := s.c.Task.StatReply, s.c.Task.StatReplyUp for msg := range s.statReplySub.Messages() { msg.Commit() m := &model.StatReply{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus statReply json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if !strings.EqualFold(m.Type, "archive") || m.Count < s.c.Task.StatReply { continue } if m.Count >= statReply && m.Count <= statReplyUp { log.Info("databus statReply key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.statReplySubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m } } for _, c := range s.statReplySubQueue { close(c) } s.wg.Done() } func (s *Service) statDM() { statDM, statDMUp := s.c.Task.StatDM, s.c.Task.StatDMUp for msg := range s.statDMSub.Messages() { msg.Commit() m := &model.StatDM{} if err := json.Unmarshal(msg.Value, m); err != nil { log.Error("databus statDM json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if !strings.EqualFold(m.Type, "archive") || m.Count < s.c.Task.StatDM { continue } if m.Count >= statDM && m.Count <= statDMUp { log.Info("databus statDM key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset) s.statDMSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m } } for _, c := range s.statDMSubQueue { close(c) } s.wg.Done() } func (s *Service) newUp() { var err error for msg := range s.newUpSub.Messages() { msg.Commit() m := &model.CanalMsg{} if err = json.Unmarshal(msg.Value, m); err != nil { log.Error("databus newUp json.Unmarshal(%v) error(%v)", string(msg.Value), err) continue } if m.Table != _archive { continue } newArc := &model.Archive{} if err = json.Unmarshal(m.New, newArc); err != nil { log.Error("databus newUp newMsg json.Unmarshal(%s) error(%v)", string(m.New), err) continue } mid, aid := newArc.MID, newArc.AID if mid == 0 || aid == 0 { log.Error("databus newUp mid(%d) | aid(%d) error", mid, aid) continue } isUp := false if m.Action == "insert" && newArc.State >= 0 { isUp = true } else if m.Action == "update" { oldArc := &model.Archive{} if err = json.Unmarshal(m.Old, oldArc); err != nil { log.Error("newUp oldMsg json.Unmarshal(%s) error(%v)", string(m.Old), err) continue } if oldArc.State < 0 && (newArc.State >= 0 || newArc.State == -6) { //0->1 isUp = true } } if isUp { av, err := s.arc.View(context.Background(), mid, aid) //获取投稿来源 if err != nil { log.Error("newUp s.arc.View mid(%d) mid(%d) av(%+v) error(%v)", mid, aid, av, err) } if av != nil && av.Archive != nil && (av.Archive.UpFrom == 3 || av.Archive.UpFrom == 8 || av.Archive.UpFrom == 9) { // 3-App , 8-Android , 9-IOS log.Info("databus mobile mid(%d) aid(%d) av(%+v)", mid, aid, av) s.mobileUpQueue[s.shardingQueueIndex(strconv.FormatInt(mid, 10), s.databusQueueLen)] <- &model.Up{AID: aid, MID: mid} } cnt, err := s.arc.UpCount(context.Background(), mid) if err != nil { log.Error("newUp s.arc.UpCount(%d) error(%v)", mid, err) continue } if cnt == 1 { //新手投下自己的第一个稿件 log.Info("databus newUp mid(%d) aid(%d) count(%d)", mid, aid, cnt) s.newUpQueue[s.shardingQueueIndex(strconv.FormatInt(mid, 10), s.databusQueueLen)] <- &model.Up{AID: aid, MID: mid} } if cnt >= 5 { //进阶任务视频投稿超过5个 log.Info("databus oldUp mid(%d) aid(%d) count(%d)", mid, aid, cnt) s.oldUpQueue[s.shardingQueueIndex(strconv.FormatInt(mid, 10), s.databusQueueLen)] <- &model.Up{AID: aid, MID: mid} } } } for _, c := range s.newUpQueue { close(c) } for _, c := range s.oldUpQueue { close(c) } for _, c := range s.mobileUpQueue { close(c) } s.wg.Done() }