go-common/app/job/main/creative/service/databus.go

480 lines
13 KiB
Go
Raw Normal View History

2019-04-22 10:49:16 +00:00
package service
import (
"context"
"encoding/json"
"strconv"
"strings"
"time"
"go-common/app/job/main/creative/model"
"go-common/library/log"
)
const (
_archive = "archive"
_insert = "insert"
_update = "update"
fromArchiveUp = 0
fromArchiveNewUp = 1
_relationMidTable = "user_relation_mid_"
_relationStatTable = "user_relation_stat_"
)
//pub up auth msg.
func (s *Service) pub(mid int64, from, isAuthor int) (err error) {
c := context.TODO()
msg := &model.Msg{
MID: mid,
From: from,
IsAuthor: isAuthor,
TimeStamp: time.Now().Unix(),
}
if err = s.upPub.Send(c, strconv.FormatInt(mid, 10), msg); err != nil {
log.Error("pub mid(%d) error(%v)", mid, err)
}
return
}
func (s *Service) arcNotifyCanalConsume() {
var err error
for msg := range s.arcNotifySub.Messages() {
msg.Commit()
s.arcNotifyMo++
m := &model.CanalMsg{}
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if m.Table == _archive {
s.arcNotifyMsg(m)
log.Info("arcNotifyCanalConsume key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
}
}
s.wg.Done()
}
func (s *Service) arcNotifyMsg(m *model.CanalMsg) {
var (
err error
newArc = &model.Archive{}
oldArc = &model.Archive{}
)
if err = json.Unmarshal(m.New, newArc); err != nil {
log.Error("arcNotifyMsg newMsg json.Unmarshal(%s) error(%v)", string(m.New), err)
return
}
mid := newArc.MID
if mid <= 0 {
log.Error("arcNotifyMsg mid (%d) error", mid)
return
}
if m.Action == _insert && newArc.State >= 0 { //0->1
s.pub(mid, fromArchiveUp, 1)
} else if m.Action == _update {
if err = json.Unmarshal(m.Old, oldArc); err != nil {
log.Error("arcNotifyMsg oldMsg json.Unmarshal(%s) error(%v)", string(m.Old), err)
return
}
if oldArc.State < 0 && (newArc.State >= 0 || newArc.State == -6) { //0->1
s.pub(mid, fromArchiveUp, 1)
}
if (oldArc.State >= 0 || oldArc.State == -6) && newArc.State < 0 { //1->0
cnt, err := s.arc.UpCount(context.Background(), mid)
if err != nil {
log.Error("arcNotifyMsg s.arc.UpCount(%d) error(%v)", mid, err)
return
}
if cnt <= 0 {
s.pub(mid, fromArchiveUp, 0)
}
}
}
}
func (s *Service) arcCanalConsume() {
var err error
for msg := range s.arcSub.Messages() {
msg.Commit()
s.arcMo++
m := &model.CanalMsg{}
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if m.Table == _archive && m.Action == _insert {
arc := &model.Archive{}
if err = json.Unmarshal(m.New, arc); err != nil {
log.Error("creative-job binglog newMsg json.Unmarshal(%s) error(%v)", m.New, err)
continue
}
if arc.MID > 0 {
s.pub(arc.MID, fromArchiveNewUp, 1)
}
log.Info("arcCanalConsume key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
}
}
s.wg.Done()
}
func (s *Service) task() {
for msg := range s.taskSub.Messages() {
msg.Commit()
log.Info("databus task key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.taskSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- msg
}
for _, c := range s.taskSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) share() {
for msg := range s.shareSub.Messages() {
msg.Commit()
m := &model.ShareMsg{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus share json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if m.TP != 3 {
continue
}
mid, authorMID := m.MID, s.getMIDByAID(m.OID)
if mid != authorMID { //不是该用户分享的稿件则不做任何处理
log.Warn("s.arc.Archive mid(%d)|author mid(%d)", mid, authorMID)
continue
}
log.Info("databus share key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.shareSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m
}
for _, c := range s.shareSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) relation() {
var err error
newF, advancedF := s.c.Task.NewFollower, s.c.Task.AdvancedFollower
for msg := range s.relationSub.Messages() {
msg.Commit()
rl := &model.RelaMessage{}
if err = json.Unmarshal(msg.Value, rl); err != nil {
log.Error("databus relation json.Unmarshal (%v) error(%v)", msg.Value, err)
continue
}
if !strings.HasPrefix(rl.Table, _relationStatTable) && !strings.HasPrefix(rl.Table, _relationMidTable) {
continue
}
if strings.HasPrefix(rl.Table, _relationStatTable) {
ost := &model.Stat{}
st := &model.Stat{}
if rl.Action == "update" {
if err = json.Unmarshal(rl.Old, ost); err != nil {
log.Error("relation old msg json.Unmarshal(%s) error(%v)", string(rl.Old), err)
continue
}
}
if err = json.Unmarshal(rl.New, st); err != nil {
log.Error("relation new msg json.Unmarshal(%s) error(%v)", string(rl.New), err)
continue
}
isFollower := false
if ost.Follower < newF && st.Follower >= newF { //新手任务粉丝数限制
isFollower = true
}
if ost.Follower < advancedF && st.Follower >= advancedF { //进阶任务粉丝数限制
isFollower = true
}
if isFollower {
log.Info("databus relation key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.followerQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- st
}
}
if strings.HasPrefix(rl.Table, _relationMidTable) {
fl := &model.Relation{}
if err = json.Unmarshal(rl.New, fl); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", string(rl.New), err)
continue
}
if fl.FID != s.c.Task.BiliMID { //过滤关注哔哩哔哩创作中心
continue
}
log.Info("databus relation key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.relationQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- fl
}
}
for _, c := range s.relationQueue {
close(c)
}
for _, c := range s.followerQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) statView() {
statView, statViewUp := s.c.Task.StatView, s.c.Task.StatViewUp
for msg := range s.statViewSub.Messages() {
msg.Commit()
m := &model.StatView{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus statView json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if !strings.EqualFold(m.Type, "archive") {
continue
}
if m.Count >= statView && m.Count <= statViewUp {
s.statViewSubQueue[s.shardingQueueIndex(msg.Key, s.statViewQueueLen)] <- m
}
}
for _, c := range s.statViewSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) statLike() {
statLike, statLikeUp := s.c.Task.StatLike, s.c.Task.StatLikeUp
for msg := range s.statLikeSub.Messages() {
msg.Commit()
m := &model.StatLike{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus statLike json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if !strings.EqualFold(m.Type, "archive") {
continue
}
if m.Count >= statLike && m.Count <= statLikeUp {
s.statLikeSubQueue[s.shardingQueueIndex(msg.Key, s.statLikeQueueLen)] <- m
}
}
for _, c := range s.statLikeSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) statShare() {
statShare, statShareUp := s.c.Task.StatShare, s.c.Task.StatShareUp
for msg := range s.statShareSub.Messages() {
msg.Commit()
m := &model.StatShare{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus statShare json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if !strings.EqualFold(m.Type, "archive") {
continue
}
if m.Count >= statShare && m.Count <= statShareUp {
log.Info("databus statShare key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.statShareSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m
}
}
for _, c := range s.statShareSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) statCoin() {
statCoin, statCoinUp := s.c.Task.StatCoin, s.c.Task.StatCoinUp
for msg := range s.statCoinSub.Messages() {
msg.Commit()
m := &model.StatCoin{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus statCoin json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if !strings.EqualFold(m.Type, "archive") {
continue
}
if m.Count >= statCoin && m.Count <= statCoinUp {
log.Info("databus statCoin key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.statCoinSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m
}
}
for _, c := range s.statCoinSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) statFav() {
statFav, statFavUp := s.c.Task.StatFav, s.c.Task.StatFavUp
for msg := range s.statFavSub.Messages() {
msg.Commit()
m := &model.StatFav{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus statFav json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if !strings.EqualFold(m.Type, "archive") {
continue
}
if m.Count >= statFav && m.Count <= statFavUp {
log.Info("databus statFav key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.statFavSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m
}
}
for _, c := range s.statFavSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) statReply() {
statReply, statReplyUp := s.c.Task.StatReply, s.c.Task.StatReplyUp
for msg := range s.statReplySub.Messages() {
msg.Commit()
m := &model.StatReply{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus statReply json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if !strings.EqualFold(m.Type, "archive") || m.Count < s.c.Task.StatReply {
continue
}
if m.Count >= statReply && m.Count <= statReplyUp {
log.Info("databus statReply key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.statReplySubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m
}
}
for _, c := range s.statReplySubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) statDM() {
statDM, statDMUp := s.c.Task.StatDM, s.c.Task.StatDMUp
for msg := range s.statDMSub.Messages() {
msg.Commit()
m := &model.StatDM{}
if err := json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus statDM json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if !strings.EqualFold(m.Type, "archive") || m.Count < s.c.Task.StatDM {
continue
}
if m.Count >= statDM && m.Count <= statDMUp {
log.Info("databus statDM key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, msg.Value, msg.Partition, msg.Offset)
s.statDMSubQueue[s.shardingQueueIndex(msg.Key, s.databusQueueLen)] <- m
}
}
for _, c := range s.statDMSubQueue {
close(c)
}
s.wg.Done()
}
func (s *Service) newUp() {
var err error
for msg := range s.newUpSub.Messages() {
msg.Commit()
m := &model.CanalMsg{}
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("databus newUp json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
if m.Table != _archive {
continue
}
newArc := &model.Archive{}
if err = json.Unmarshal(m.New, newArc); err != nil {
log.Error("databus newUp newMsg json.Unmarshal(%s) error(%v)", string(m.New), err)
continue
}
mid, aid := newArc.MID, newArc.AID
if mid == 0 || aid == 0 {
log.Error("databus newUp mid(%d) | aid(%d) error", mid, aid)
continue
}
isUp := false
if m.Action == "insert" && newArc.State >= 0 {
isUp = true
} else if m.Action == "update" {
oldArc := &model.Archive{}
if err = json.Unmarshal(m.Old, oldArc); err != nil {
log.Error("newUp oldMsg json.Unmarshal(%s) error(%v)", string(m.Old), err)
continue
}
if oldArc.State < 0 && (newArc.State >= 0 || newArc.State == -6) { //0->1
isUp = true
}
}
if isUp {
av, err := s.arc.View(context.Background(), mid, aid) //获取投稿来源
if err != nil {
log.Error("newUp s.arc.View mid(%d) mid(%d) av(%+v) error(%v)", mid, aid, av, err)
}
if av != nil && av.Archive != nil && (av.Archive.UpFrom == 3 || av.Archive.UpFrom == 8 || av.Archive.UpFrom == 9) { // 3-App , 8-Android , 9-IOS
log.Info("databus mobile mid(%d) aid(%d) av(%+v)", mid, aid, av)
s.mobileUpQueue[s.shardingQueueIndex(strconv.FormatInt(mid, 10), s.databusQueueLen)] <- &model.Up{AID: aid, MID: mid}
}
cnt, err := s.arc.UpCount(context.Background(), mid)
if err != nil {
log.Error("newUp s.arc.UpCount(%d) error(%v)", mid, err)
continue
}
if cnt == 1 { //新手投下自己的第一个稿件
log.Info("databus newUp mid(%d) aid(%d) count(%d)", mid, aid, cnt)
s.newUpQueue[s.shardingQueueIndex(strconv.FormatInt(mid, 10), s.databusQueueLen)] <- &model.Up{AID: aid, MID: mid}
}
if cnt >= 5 { //进阶任务视频投稿超过5个
log.Info("databus oldUp mid(%d) aid(%d) count(%d)", mid, aid, cnt)
s.oldUpQueue[s.shardingQueueIndex(strconv.FormatInt(mid, 10), s.databusQueueLen)] <- &model.Up{AID: aid, MID: mid}
}
}
}
for _, c := range s.newUpQueue {
close(c)
}
for _, c := range s.oldUpQueue {
close(c)
}
for _, c := range s.mobileUpQueue {
close(c)
}
s.wg.Done()
}