458 lines
12 KiB
Go
458 lines
12 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"regexp"
|
||
"time"
|
||
|
||
"go-common/app/job/main/dm2/conf"
|
||
"go-common/app/job/main/dm2/dao"
|
||
"go-common/app/job/main/dm2/model"
|
||
"go-common/app/job/main/dm2/model/oplog"
|
||
arcCli "go-common/app/service/main/archive/api/gorpc"
|
||
filterCli "go-common/app/service/main/filter/api/grpc/v1"
|
||
seqMdl "go-common/app/service/main/seq-server/model"
|
||
seqCli "go-common/app/service/main/seq-server/rpc/client"
|
||
"go-common/library/log"
|
||
"go-common/library/log/infoc"
|
||
"go-common/library/queue/databus"
|
||
"go-common/library/sync/pipeline/fanout"
|
||
"go-common/library/xstr"
|
||
)
|
||
|
||
const (
|
||
_routineSizeDefault = 10
|
||
_chanSize = 10240
|
||
_batchSize = 1000
|
||
_maxUpRecent = 1000
|
||
)
|
||
|
||
// Service service struct
|
||
type Service struct {
|
||
conf *conf.Config
|
||
dao *dao.Dao
|
||
cache *fanout.Fanout
|
||
arcRPC *arcCli.Service2
|
||
// seq serer
|
||
seqArg *seqMdl.ArgBusiness
|
||
seqRPC *seqCli.Service2
|
||
indexCsmr *databus.Databus
|
||
subjectCsmr *databus.Databus
|
||
actionCsmr *databus.Databus
|
||
reportCsmr *databus.Databus
|
||
videoupCsmr *databus.Databus
|
||
subtitleAuditCsmr *databus.Databus
|
||
flushMergeChan []chan *model.Flush
|
||
flushSegChan []chan *model.FlushDMSeg
|
||
dmRecentChan []chan *model.DM
|
||
routineSize int
|
||
realname map[int64]int64 // key：分区id，value:cid，即该分区中大于cid的视频开启实名制
|
||
// filter service
|
||
filterRPC filterCli.FilterClient
|
||
maskMid []int64
|
||
dmOperationLogSvc *infoc.Infoc
|
||
opsLogCh chan *oplog.Infoc
|
||
// bnj
|
||
bnjAid int64
|
||
bnjSubAids map[int64]struct{}
|
||
bnjCsmr *databus.Databus
|
||
bnjliveRoomID int64
|
||
bnjStart time.Time
|
||
bnjIgnoreBeginTime time.Duration
|
||
bnjIgnoreEndTime time.Duration
|
||
bnjArcVideos []*model.Video
|
||
bnjIgnoreRate int64
|
||
bnjUserLevel int32
|
||
}
|
||
|
||
// New new service.
|
||
func New(c *conf.Config) (s *Service) {
|
||
s = &Service{
|
||
conf: c,
|
||
dao: dao.New(c),
|
||
cache: fanout.New("cache", fanout.Worker(1), fanout.Buffer(1024)),
|
||
arcRPC: arcCli.New2(c.ArchiveRPC),
|
||
seqArg: &seqMdl.ArgBusiness{BusinessID: c.Seq.BusinessID, Token: c.Seq.Token},
|
||
seqRPC: seqCli.New2(c.SeqRPC),
|
||
subjectCsmr: databus.New(c.Databus.SubjectCsmr),
|
||
indexCsmr: databus.New(c.Databus.IndexCsmr),
|
||
actionCsmr: databus.New(c.Databus.ActionCsmr),
|
||
reportCsmr: databus.New(c.Databus.ReportCsmr),
|
||
videoupCsmr: databus.New(c.Databus.VideoupCsmr),
|
||
subtitleAuditCsmr: databus.New(c.Databus.SubtitleAuditCsmr),
|
||
routineSize: c.RoutineSize,
|
||
realname: make(map[int64]int64),
|
||
dmOperationLogSvc: infoc.New(c.Infoc2),
|
||
opsLogCh: make(chan *oplog.Infoc, 1024),
|
||
}
|
||
if c.RoutineSize <= 0 {
|
||
s.routineSize = _routineSizeDefault
|
||
}
|
||
s.flushMergeChan = make([]chan *model.Flush, s.routineSize)
|
||
s.flushSegChan = make([]chan *model.FlushDMSeg, s.routineSize)
|
||
s.dmRecentChan = make([]chan *model.DM, s.routineSize)
|
||
filterRPC, err := filterCli.NewClient(c.FliterRPC)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
s.filterRPC = filterRPC
|
||
for idStr, cid := range conf.Conf.Realname {
|
||
ids, err := xstr.SplitInts(idStr)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
for _, id := range ids {
|
||
if _, ok := s.realname[id]; !ok {
|
||
s.realname[id] = cid
|
||
}
|
||
}
|
||
}
|
||
// laji bnj
|
||
if s.conf.BNJ != nil {
|
||
//bnj count
|
||
s.initBnj()
|
||
}
|
||
//消费DMReport-T消息
|
||
go s.reportCsmproc()
|
||
// 消费DMAction-T消息
|
||
go s.actionCsmproc()
|
||
// 消费DMSubject-T消息
|
||
go s.subjectCsmproc()
|
||
// 消费DMMeta-T消息
|
||
go s.indexCsmproc()
|
||
// 消费 Videoup2Bvc消息
|
||
go s.videoupCsmrproc()
|
||
// 消费 字幕 提交 消息
|
||
go s.subtitleAuditProc()
|
||
// 刷新全段弹幕
|
||
for i := 0; i < s.routineSize; i++ {
|
||
flushChan := make(chan *model.Flush, _chanSize)
|
||
s.flushMergeChan[i] = flushChan
|
||
go s.flushmergeproc(flushChan)
|
||
}
|
||
// 刷新分段弹幕
|
||
for i := 0; i < s.routineSize; i++ {
|
||
flushSegChan := make(chan *model.FlushDMSeg, _chanSize)
|
||
s.flushSegChan[i] = flushSegChan
|
||
go s.flushSegproc(flushSegChan)
|
||
}
|
||
// 异步处理创作中心最新弹幕列表缓存
|
||
for i := 0; i < s.routineSize; i++ {
|
||
recentChan := make(chan *model.DM, _chanSize)
|
||
s.dmRecentChan[i] = recentChan
|
||
go s.dmRecentproc(recentChan)
|
||
}
|
||
go s.transferProc()
|
||
// 处理热门二级分类视频的弹幕蒙版
|
||
go s.maskProc()
|
||
// 刷新开启蒙版mid
|
||
s.maskMid, _ = s.dao.MaskMids(context.TODO())
|
||
log.Info("update mask mid(%v)", s.maskMid)
|
||
go s.maskMidProc()
|
||
// dm task
|
||
go s.taskResProc()
|
||
go s.taskDelProc()
|
||
// oplog
|
||
go s.oplogproc()
|
||
return
|
||
}
|
||
|
||
// Ping check thrid resource.
|
||
func (s *Service) Ping(c context.Context) (err error) {
|
||
return s.dao.Ping(c)
|
||
}
|
||
|
||
func (s *Service) subjectCsmproc() {
|
||
var (
|
||
err error
|
||
c = context.TODO()
|
||
regexSubject = regexp.MustCompile("dm_subject_[0-9]+")
|
||
)
|
||
for {
|
||
msg, ok := <-s.subjectCsmr.Messages()
|
||
if !ok {
|
||
log.Error("subject binlog consumer exit")
|
||
return
|
||
}
|
||
m := &model.BinlogMsg{}
|
||
if err = json.Unmarshal(msg.Value, &m); err != nil {
|
||
log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
|
||
continue
|
||
}
|
||
log.Info("%s", m)
|
||
if regexSubject.MatchString(m.Table) {
|
||
if err = s.trackSubject(c, m); err != nil {
|
||
log.Error("s.trackSubject(%s) error(%v)", m, err)
|
||
continue
|
||
}
|
||
}
|
||
if err = msg.Commit(); err != nil {
|
||
log.Error("commit offset(%v) error(%v)", msg, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) indexCsmproc() {
|
||
var (
|
||
err error
|
||
c = context.TODO()
|
||
regexIndex = regexp.MustCompile("dm_index_[0-9]+")
|
||
)
|
||
for {
|
||
msg, ok := <-s.indexCsmr.Messages()
|
||
if !ok {
|
||
log.Error("index binlog consumer exit")
|
||
return
|
||
}
|
||
m := &model.BinlogMsg{}
|
||
if err = json.Unmarshal(msg.Value, &m); err != nil {
|
||
log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
|
||
continue
|
||
}
|
||
log.Info("%s", m)
|
||
if regexIndex.MatchString(m.Table) {
|
||
if err = s.trackIndex(c, m); err != nil {
|
||
log.Error("s.traceIndex(%s) error(%v)", m, err)
|
||
continue
|
||
}
|
||
}
|
||
if err = msg.Commit(); err != nil {
|
||
log.Error("commit offset(%v) error(%v)", msg, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) flushmergeproc(flushChan chan *model.Flush) {
|
||
var (
|
||
flushs = make(map[int64]*model.Flush)
|
||
ticker = time.NewTicker(60 * time.Second)
|
||
err error
|
||
)
|
||
for {
|
||
select {
|
||
case flush, ok := <-flushChan:
|
||
if !ok {
|
||
log.Error("action channel closed")
|
||
return
|
||
}
|
||
if _, ok := flushs[flush.Oid]; !ok || flush.Force { // key不存在或者需要强制刷新的
|
||
flushs[flush.Oid] = flush
|
||
}
|
||
if len(flushs) < _batchSize {
|
||
continue
|
||
}
|
||
case <-ticker.C:
|
||
}
|
||
if len(flushs) > 0 {
|
||
for _, flush := range flushs {
|
||
if err = s.flushDmCache(context.TODO(), flush); err != nil {
|
||
log.Error("action:flushmergeproc,flush:%+v,error(%v)", flush, err)
|
||
}
|
||
}
|
||
flushs = make(map[int64]*model.Flush)
|
||
}
|
||
}
|
||
}
|
||
|
||
func keySegFlush(tp int32, oid, total, num int64) string {
|
||
return fmt.Sprintf("f_%d_%d_%d_%d", tp, oid, total, num)
|
||
}
|
||
|
||
func (s *Service) flushSegproc(ch chan *model.FlushDMSeg) {
|
||
var (
|
||
key string
|
||
merge = make(map[string]*model.FlushDMSeg)
|
||
ticker = time.NewTicker(60 * time.Second)
|
||
err error
|
||
)
|
||
for {
|
||
select {
|
||
case msg, ok := <-ch:
|
||
if !ok {
|
||
log.Error("action channel closed")
|
||
return
|
||
}
|
||
key = keySegFlush(msg.Type, msg.Oid, msg.Page.Total, msg.Page.Num)
|
||
if _, ok := merge[key]; !ok || msg.Force { // key不存在或者需要强制刷新的
|
||
merge[key] = msg
|
||
}
|
||
if len(merge) < _batchSize {
|
||
continue
|
||
}
|
||
case <-ticker.C:
|
||
}
|
||
if len(merge) > 0 {
|
||
for _, v := range merge {
|
||
if err = s.flushDmSegCache(context.TODO(), v); err != nil {
|
||
log.Error("action:flushSegproc,data:%+v,error(%v)", v, err)
|
||
continue
|
||
}
|
||
}
|
||
merge = make(map[string]*model.FlushDMSeg)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) actionCsmproc() {
|
||
for {
|
||
msg, ok := <-s.actionCsmr.Messages()
|
||
if !ok {
|
||
log.Error("action consumer exit")
|
||
return
|
||
}
|
||
act := &model.Action{}
|
||
err := json.Unmarshal(msg.Value, &act)
|
||
if err != nil {
|
||
log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
|
||
continue
|
||
}
|
||
start := time.Now()
|
||
if err = s.actionAct(context.TODO(), act); err != nil {
|
||
log.Error("action:%s,data:%s,error(%v)", act.Action, act.Data, err)
|
||
continue
|
||
}
|
||
log.Info("partition:%d,offset:%d,key:%s,value:%s costing:%+v", msg.Partition, msg.Offset, msg.Key, msg.Value, time.Since(start))
|
||
if err = msg.Commit(); err != nil {
|
||
log.Error("commit offset(%v) error(%v)", msg, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// add recent dm
|
||
func (s *Service) dmRecentproc(dmChan chan *model.DM) {
|
||
var (
|
||
count int64
|
||
c = context.TODO()
|
||
)
|
||
for {
|
||
dm, ok := <-dmChan
|
||
if !ok {
|
||
log.Error("recent dm channel is closed")
|
||
return
|
||
}
|
||
sub, err := s.subject(c, dm.Type, dm.Oid)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if dm.State != model.StateNormal && dm.State != model.StateHide && dm.State != model.StateMonitorAfter {
|
||
if err = s.dao.ZRemRecentDM(c, sub.Mid, dm.ID); err != nil {
|
||
continue
|
||
}
|
||
} else {
|
||
if dm.Content == nil {
|
||
if dm.Content, err = s.dao.Content(c, dm.Oid, dm.ID); err != nil {
|
||
continue
|
||
}
|
||
}
|
||
if dm.Pool == model.PoolSpecial && dm.ContentSpe == nil {
|
||
if dm.ContentSpe, err = s.dao.ContentSpecial(c, dm.ID); err != nil {
|
||
continue
|
||
}
|
||
}
|
||
if count, err = s.dao.AddRecentDM(c, sub.Mid, dm); err != nil {
|
||
continue
|
||
}
|
||
if trimCnt := count - _maxUpRecent; trimCnt > 0 {
|
||
if err = s.dao.TrimRecentDM(c, sub.Mid, trimCnt); err != nil {
|
||
continue
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) asyncAddRecent(c context.Context, dm *model.DM) {
|
||
select {
|
||
case s.dmRecentChan[dm.Oid%int64(s.routineSize)] <- dm:
|
||
default:
|
||
log.Warn("dm recent channel is full,dm(%+v)", dm)
|
||
}
|
||
}
|
||
|
||
func (s *Service) reportCsmproc() {
|
||
for {
|
||
msg, ok := <-s.reportCsmr.Messages()
|
||
if !ok {
|
||
log.Error("report consumer exit")
|
||
return
|
||
}
|
||
log.Info("partition:%d,offset:%d,value:%s", msg.Partition, msg.Offset, msg.Value)
|
||
act := &model.ReportAction{}
|
||
err := json.Unmarshal(msg.Value, &act)
|
||
if err != nil {
|
||
log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
|
||
continue
|
||
}
|
||
if diff := act.HideTime - time.Now().Unix(); diff > 0 {
|
||
log.Info("action:%+v will be processed after %d seconds", act, diff)
|
||
time.Sleep(time.Duration(diff) * time.Second)
|
||
}
|
||
if _, err = s.dao.DelDMHideState(context.TODO(), 1, act.Cid, act.Did); err != nil {
|
||
log.Error("DelDMHideState(%+v) error(%v)", act, err)
|
||
} else {
|
||
log.Info("DelDMHideState(%+v) success ", act)
|
||
}
|
||
if err = msg.Commit(); err != nil {
|
||
log.Error("commit offset(%v) error(%v)", msg, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) videoupCsmrproc() {
|
||
var (
|
||
err error
|
||
c = context.TODO()
|
||
)
|
||
for {
|
||
msg, ok := <-s.videoupCsmr.Messages()
|
||
if !ok {
|
||
log.Error("videoup consumer exit")
|
||
return
|
||
}
|
||
log.Info("partition:%d,offset:%d,key:%s,value:%s", msg.Partition, msg.Offset, msg.Key, msg.Value)
|
||
m := &model.VideoupMsg{}
|
||
if err = json.Unmarshal(msg.Value, m); err != nil {
|
||
log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
|
||
continue
|
||
}
|
||
if m.Route == model.RouteSecondRound || m.Route == model.RouteAutoOpen ||
|
||
m.Route == model.RouteForceSync || m.Route == model.RouteDelayOpen {
|
||
if err = s.trackVideoup(c, m.Aid); err != nil {
|
||
continue
|
||
}
|
||
}
|
||
if err = msg.Commit(); err != nil {
|
||
log.Error("commit offset(%v) error(%v)", msg, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) subtitleAuditProc() {
|
||
var (
|
||
err error
|
||
c = context.Background()
|
||
)
|
||
for {
|
||
msg, ok := <-s.subtitleAuditCsmr.Messages()
|
||
if !ok {
|
||
log.Error("subtitle_audit consumer exit")
|
||
return
|
||
}
|
||
log.Info("partition:%d,offset:%d,key:%s,value:%s", msg.Partition, msg.Offset, msg.Key, msg.Value)
|
||
m := &model.SubtitleAuditMsg{}
|
||
if err = json.Unmarshal(msg.Value, m); err != nil {
|
||
log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
|
||
continue
|
||
}
|
||
if err = s.SubtitleFilter(c, m.Oid, m.SubtitleID); err != nil {
|
||
log.Error("SubtitleFilter(oid:%v,subtitleID:%v),error(%v)", m.Oid, m.SubtitleID, err)
|
||
continue
|
||
}
|
||
if err = msg.Commit(); err != nil {
|
||
log.Error("commit offset(%v) error(%v)", msg, err)
|
||
}
|
||
}
|
||
}
|