334 lines
10 KiB
Go
334 lines
10 KiB
Go
|
package service
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"encoding/json"
|
|||
|
|
"fmt"
|
|||
|
|
"strings"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"go-common/app/interface/main/push-archive/dao"
|
|||
|
|
"go-common/app/interface/main/push-archive/model"
|
|||
|
|
accmdl "go-common/app/service/main/account/model"
|
|||
|
|
"go-common/library/log"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
_insertAct = "insert"
|
|||
|
|
_updateAct = "update"
|
|||
|
|
_deleteAct = "delete"
|
|||
|
|
|
|||
|
|
_archiveTable = "archive"
|
|||
|
|
|
|||
|
|
_pushRetryTimes = 3
|
|||
|
|
_pushPartSize = 500000 // 每次请求最多推50w mid
|
|||
|
|
|
|||
|
|
_hbaseBatch = 100
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
func (s *Service) consumeArchiveproc() {
|
|||
|
|
defer s.wg.Done()
|
|||
|
|
var (
|
|||
|
|
err error
|
|||
|
|
msgs = s.archiveSub.Messages()
|
|||
|
|
)
|
|||
|
|
for {
|
|||
|
|
msg, ok := <-msgs
|
|||
|
|
if !ok {
|
|||
|
|
log.Warn("s.ArchiveSub has been closed.")
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
msg.Commit()
|
|||
|
|
s.arcMo++
|
|||
|
|
log.Info("consume archive key(%s) offset(%d) message(%s)", msg.Key, msg.Offset, msg.Value)
|
|||
|
|
m := new(model.Message)
|
|||
|
|
if err = json.Unmarshal(msg.Value, &m); err != nil {
|
|||
|
|
log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if m.Table != _archiveTable {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
n := new(model.Archive)
|
|||
|
|
if err = json.Unmarshal(m.New, n); err != nil {
|
|||
|
|
log.Error("json.Unmarshal(%s) error(%v)", m.New, err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// 稿件过审
|
|||
|
|
switch m.Action {
|
|||
|
|
case _insertAct:
|
|||
|
|
if !n.IsNormal() {
|
|||
|
|
log.Info("archive (%d) upper(%d) is not normal when insert, no need to push", n.ID, n.Mid)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
case _updateAct:
|
|||
|
|
o := new(model.Archive)
|
|||
|
|
if err = json.Unmarshal(m.Old, o); err != nil {
|
|||
|
|
log.Error("json.Unmarshal(%s) error(%v)", m.Old, err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if !n.IsNormal() || o.State == n.State || o.PubTime == n.PubTime {
|
|||
|
|
log.Info("archive (%d) upper(%d) is not normal when update, no need to push", n.ID, n.Mid)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
default:
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
dao.PromInfo("archive_push")
|
|||
|
|
// 在指定周期内，up主只推送一次
|
|||
|
|
if s.limit(n.Mid) {
|
|||
|
|
log.Info("upper's push limit upper(%d) aid(%d)", n.Mid, n.ID)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
log.Info("noticeFans start, mid(%d) aid(%d)", n.Mid, n.ID)
|
|||
|
|
err = s.noticeFans(n) // notice fans
|
|||
|
|
log.Info("noticeFans end, mid(%d) aid(%d) error(%v)", n.Mid, n.ID, err)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// isForbidTime 当前时间是否被禁止推送消息
|
|||
|
|
func (s *Service) isForbidTime() (forbid bool, err error) {
|
|||
|
|
var forbidStart, forbidEnd time.Time
|
|||
|
|
now := time.Now()
|
|||
|
|
for _, f := range s.ForbidTimes {
|
|||
|
|
forbidStart, err = s.getTodayTime(f.PushForbidStartTime)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("isForbidTime getTodayTime(%s) error(%v)", f.PushForbidStartTime, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
forbidEnd, err = s.getTodayTime(f.PushForbidEndTime)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("isForbidTime getTodayTime(%s) error(%v)", f.PushForbidEndTime, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if now.After(forbidStart) && now.Before(forbidEnd) {
|
|||
|
|
forbid = true
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) noticeFans(arc *model.Archive) (err error) {
|
|||
|
|
isPGC := s.isPGC(arc)
|
|||
|
|
mids, counter, err := s.fans(arc.Mid, arc.ID, isPGC)
|
|||
|
|
if err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
log.Info("noticeFans get fans mid(%d) aid(%d) fans number(%d)", arc.Mid, arc.ID, counter)
|
|||
|
|
arc.Author = s.name(arc.Mid)
|
|||
|
|
for gKey, list := range mids {
|
|||
|
|
g := s.dao.FanGroups[gKey]
|
|||
|
|
params := model.NewBatchParam(map[string]interface{}{
|
|||
|
|
"relationType": g.RelationType,
|
|||
|
|
"archive": arc,
|
|||
|
|
"group": s.getGroupParam(g),
|
|||
|
|
"msgTemplate": g.MsgTemplate,
|
|||
|
|
}, model.PushParamHandler)
|
|||
|
|
dao.Batch(&list, _pushPartSize, _pushRetryTimes, params, s.dao.NoticeFans)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) fans(upper int64, aid int64, isPGC bool) (mids map[string][]int64, counter int, err error) {
|
|||
|
|
mids = make(map[string][]int64)
|
|||
|
|
// 从 HBase 获取upper粉丝数据
|
|||
|
|
fans, err := s.dao.Fans(context.TODO(), upper, isPGC)
|
|||
|
|
ln := len(fans)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("fans from hbase upper(%d) error(%v)", upper, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
log.Info("filter fans -- get all fans of upper(%d), fans num(%d)", upper, ln)
|
|||
|
|
if ln == 0 {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
dao.PromInfoAdd("archive_fans", int64(ln))
|
|||
|
|
// 筛选粉丝，分配到对应的实验组
|
|||
|
|
hitFans := s.group(upper, fans)
|
|||
|
|
hour := time.Now().Hour()
|
|||
|
|
for gKey, hits := range hitFans {
|
|||
|
|
log.Info("before filter by active time, upper(%d) group(%s) fans(%d)", upper, gKey, len(hits))
|
|||
|
|
if len(hits) <= 0 {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
g := s.dao.FanGroups[gKey]
|
|||
|
|
var activeFans, excluded []int64
|
|||
|
|
if g.RelationType == model.RelationAttention {
|
|||
|
|
activeFans, excluded = s.dao.FansByActiveTime(hour, &hits)
|
|||
|
|
noLimitFans := s.noPushLimitFans(upper, gKey, &activeFans)
|
|||
|
|
for _, mid := range activeFans {
|
|||
|
|
if !s.filterUserSetting(mid, g.RelationType) {
|
|||
|
|
excluded = append(excluded, mid)
|
|||
|
|
log.Info("filter fans: excluded by usersettings, fan(%d), relationtype(%d), upper(%d) table(%s)", mid, g.RelationType, upper, g.HBaseTable)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if !s.pushLimit(mid, upper, g, &noLimitFans) {
|
|||
|
|
excluded = append(excluded, mid)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
mids[gKey] = append(mids[gKey], mid)
|
|||
|
|
dao.PromInfo(fmt.Sprintf("%s__send", g.Name))
|
|||
|
|
log.Info("filter fans: included by pushlimit: upper(%d)'s fan(%d) group.name(%s)", upper, mid, g.Name)
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
mids[gKey] = hits
|
|||
|
|
}
|
|||
|
|
counter += len(mids[gKey])
|
|||
|
|
log.Info("upper(%d) aid(%d) group(%s) fans(%d)", upper, aid, gKey, len(mids[gKey]))
|
|||
|
|
// 统计数据落表
|
|||
|
|
s.formStatisticsProc(aid, g.Name, mids[gKey], &excluded)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// group 粉丝分组: 分组优先级(优先级 & 分组的可用性)-->分组规则(默认default,hbase过滤走hbase)
|
|||
|
|
func (s *Service) group(upper int64, fans map[int64]int) (list map[string][]int64) {
|
|||
|
|
list = make(map[string][]int64)
|
|||
|
|
// 将粉丝分为 特殊关注 和 普通关注 两类
|
|||
|
|
attentions, specials := s.dao.FansByProportion(upper, fans)
|
|||
|
|
log.Info("group count upper(%d) attentions(%d) specials(%d)", upper, len(attentions), len(specials))
|
|||
|
|
// 禁止推送时间判断, 免打扰用户(针对普通关注)
|
|||
|
|
isForbid, err := s.isForbidTime()
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("isForbidTime upper(%d) error(%v)", upper, err)
|
|||
|
|
}
|
|||
|
|
// 按顺序筛选出每个分组的数据
|
|||
|
|
for _, gkey := range s.dao.GroupOrder {
|
|||
|
|
g := s.dao.FanGroups[gkey]
|
|||
|
|
if g.RelationType == model.RelationAttention && isForbid {
|
|||
|
|
log.Info("forbid by time upper(%d) group(%+v)", upper, g)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// 优先处理abtest的流量
|
|||
|
|
if g.Hitby == model.GroupDataTypeAbtest {
|
|||
|
|
switch g.RelationType {
|
|||
|
|
case model.RelationAttention:
|
|||
|
|
var pool, exists, ex []int64
|
|||
|
|
pool, attentions = s.fansByAbtest(g, attentions)
|
|||
|
|
for _, gk := range s.dao.GroupOrder {
|
|||
|
|
gg := s.dao.FanGroups[gk]
|
|||
|
|
if gg.RelationType != model.RelationAttention ||
|
|||
|
|
gg.Hitby != model.GroupDataTypeHBase ||
|
|||
|
|
gg.Name == "ai:pushlist_follow_recent" {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
ex, pool = s.dao.FansByHBase(upper, gk, &pool)
|
|||
|
|
exists = append(exists, ex...)
|
|||
|
|
}
|
|||
|
|
list[gkey] = s.filterByBlacklist(upper, exists)
|
|||
|
|
case model.RelationSpecial:
|
|||
|
|
list[gkey], specials = s.fansByAbtest(g, specials)
|
|||
|
|
}
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if g.Hitby == model.GroupDataTypeAbComparison {
|
|||
|
|
// 对照组保持原来逻辑
|
|||
|
|
switch g.RelationType {
|
|||
|
|
case model.RelationAttention:
|
|||
|
|
var pool, exists []int64
|
|||
|
|
pool, attentions = s.fansByAbtest(g, attentions)
|
|||
|
|
for _, gk := range s.dao.GroupOrder {
|
|||
|
|
gg := s.dao.FanGroups[gk]
|
|||
|
|
if gg.RelationType != model.RelationAttention || gg.Hitby != model.GroupDataTypeHBase {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
exists, pool = s.dao.FansByHBase(upper, gk, &pool)
|
|||
|
|
list[gkey] = append(list[gkey], exists...)
|
|||
|
|
}
|
|||
|
|
case model.RelationSpecial:
|
|||
|
|
list[gkey], specials = s.fansByAbtest(g, specials)
|
|||
|
|
}
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
switch {
|
|||
|
|
case g.RelationType == model.RelationAttention && g.Hitby == model.GroupDataTypeDefault:
|
|||
|
|
list[gkey] = attentions
|
|||
|
|
attentions = []int64{}
|
|||
|
|
case g.RelationType == model.RelationSpecial && g.Hitby == model.GroupDataTypeDefault:
|
|||
|
|
list[gkey] = specials
|
|||
|
|
specials = []int64{}
|
|||
|
|
case g.RelationType == model.RelationAttention && g.Hitby == model.GroupDataTypeHBase:
|
|||
|
|
list[gkey], attentions = s.dao.FansByHBase(upper, gkey, &attentions)
|
|||
|
|
case g.RelationType == model.RelationSpecial && g.Hitby == model.GroupDataTypeHBase:
|
|||
|
|
list[gkey], specials = s.dao.FansByHBase(upper, gkey, &specials)
|
|||
|
|
default:
|
|||
|
|
log.Error("group failed for grouporder(%s) & group.relationtype(%d) & hitby(%s)", gkey, g.RelationType, g.Hitby)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) filterByBlacklist(upper int64, fans []int64) (result []int64) {
|
|||
|
|
for {
|
|||
|
|
var mids []int64
|
|||
|
|
l := len(fans)
|
|||
|
|
if l == 0 {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if l <= _hbaseBatch {
|
|||
|
|
mids = fans[:]
|
|||
|
|
fans = []int64{}
|
|||
|
|
} else {
|
|||
|
|
mids = fans[:_hbaseBatch]
|
|||
|
|
fans = fans[_hbaseBatch:]
|
|||
|
|
}
|
|||
|
|
exists, notExists := s.dao.ExistsInBlacklist(context.Background(), upper, mids)
|
|||
|
|
result = append(result, notExists...) // 不在黑名单中的用户直接可推
|
|||
|
|
exists, notExists = s.dao.ExistsInWhitelist(context.Background(), upper, exists)
|
|||
|
|
result = append(result, exists...) // 存在黑名单中但是后来被恢复到白名单中的用户可推
|
|||
|
|
for _, mid := range notExists {
|
|||
|
|
// 只出现在黑名单中的用户不推
|
|||
|
|
log.Warn("filter by blacklist, mid(%d)", mid)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// getGroupParam g.name被_分隔后的第1个/2个值（比如：ai:pushlist_offline_up的是offline, special的就是special）
|
|||
|
|
func (s *Service) getGroupParam(g *dao.FanGroup) string {
|
|||
|
|
p := strings.Split(g.Name, "_")
|
|||
|
|
if len(p) == 1 {
|
|||
|
|
return p[0]
|
|||
|
|
}
|
|||
|
|
if g.RelationType == model.RelationSpecial {
|
|||
|
|
return "s_" + p[1]
|
|||
|
|
}
|
|||
|
|
return p[1]
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// filterUserSetting 普通关注up主的粉丝中，推送开启配置为全部推送的人很少，因此，只要其开启了特殊关注/全部关注／没有设置的就发送；特殊关注等同
|
|||
|
|
func (s *Service) filterUserSetting(mid int64, relationType int) bool {
|
|||
|
|
if relationType == model.RelationSpecial && s.userSettings[mid] != nil && s.userSettings[mid].Type == model.PushTypeForbid {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if relationType == model.RelationAttention && s.userSettings[mid] != nil && s.userSettings[mid].Type == model.PushTypeForbid {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) name(mid int64) (name string) {
|
|||
|
|
arg := &accmdl.ArgMid{Mid: mid}
|
|||
|
|
info, err := s.accRPC.Info3(context.TODO(), arg)
|
|||
|
|
if err != nil {
|
|||
|
|
dao.PromError("archive:获取作者信息")
|
|||
|
|
log.Error("s.accRPC.Info3(%+v) error(%v)", arg, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
name = info.Name
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) isPGC(arc *model.Archive) bool {
|
|||
|
|
return (arc.Attribute >> model.AttrBitIsPGC & 1) == 1
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Test for test
|
|||
|
|
// TODO delete, for test
|
|||
|
|
func (s *Service) Test(arc *model.Archive) {
|
|||
|
|
if arc.ID == 0 || arc.Mid == 0 {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
go s.noticeFans(arc)
|
|||
|
|
} // TODO delete, for test
|