216 lines
6.2 KiB
Go
216 lines
6.2 KiB
Go
package ugc
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"go-common/app/job/main/tv/dao/app"
|
|
"go-common/app/job/main/tv/model/ugc"
|
|
"go-common/library/log"
|
|
)
|
|
|
|
const (
|
|
_errSleep = 500 * time.Millisecond
|
|
_succSleep = 10 * time.Millisecond
|
|
)
|
|
|
|
func errMid(funcName string, mid int64, err error) {
|
|
log.Error("Func:[%s], Step:[%s], Mid:[%d], Err:[%v]", "fullRefresh", funcName, mid, err)
|
|
}
|
|
|
|
func errArcPce(funcName string, mid int64, numPce int, err error) {
|
|
log.Error("Func:[%s], Step:[%s], Mid:[%d], NumPce:[%d], Err:[%v]", "fullRefresh-ArcPce", funcName, mid, numPce, err)
|
|
}
|
|
|
|
func infoArc(funcName string, aid int64, msg string) {
|
|
log.Info("Func:[%s], Step:[%s], Aid:[%d], Msg:[%s]", "fullRefresh-ArcPce-Arc", funcName, aid, msg)
|
|
time.Sleep(_errSleep)
|
|
}
|
|
|
|
func errArc(funcName string, aid int64, err error) {
|
|
log.Error("Func:[%s], Step:[%s], Aid:[%d], Err:[%v]", "fullRefresh-ArcPce-Arc", funcName, aid, err)
|
|
time.Sleep(_errSleep)
|
|
}
|
|
|
|
func errArcVideos(funcName string, aid int64, cids []int64, err error) {
|
|
log.Error("Func:[%s], Step:[%s], Aid:[%d], Cids: [%v], Err:[%v]", "fullRefresh-ArcPce-Arc-Videos", funcName, aid, cids, err)
|
|
time.Sleep(_errSleep)
|
|
}
|
|
|
|
func infoArcVideos(funcName string, aid int64, cids []int64, msg string) {
|
|
log.Info("Func:[%s], Step:[%s], Aid:[%d], Cids: [%v], Msg:[%s]", "fullRefresh-ArcPce-Arc-Videos", funcName, aid, cids, msg)
|
|
time.Sleep(_succSleep)
|
|
}
|
|
|
|
func (s *Service) fullRefreshproc() {
|
|
for {
|
|
s.fullRefresh()
|
|
time.Sleep(time.Duration(s.c.UgcSync.Frequency.FullRefreshFre))
|
|
}
|
|
}
|
|
|
|
func (s *Service) fullRefresh() {
|
|
var (
|
|
fullName = "fullRefresh"
|
|
pagesize = s.c.UgcSync.Batch.ProducerPS
|
|
begin = time.Now()
|
|
totalArcs = 0
|
|
treatedUp = 0
|
|
totalUp = len(s.activeUps)
|
|
)
|
|
if totalUp == 0 {
|
|
log.Error("[%s] ActiveUps Empty", fullName)
|
|
return
|
|
}
|
|
log.Info("fullRefresh Total Uppers Len %d", totalUp)
|
|
for mid := range s.activeUps {
|
|
var (
|
|
upArcCnt int
|
|
err error
|
|
)
|
|
if upArcCnt, err = s.dao.UpArcsCnt(ctx, int64(mid)); err != nil {
|
|
errMid("CountUpArcs", mid, err)
|
|
continue
|
|
}
|
|
if upArcCnt == 0 {
|
|
errMid("CountUpArcs", mid, fmt.Errorf("Empty Arcs"))
|
|
continue
|
|
}
|
|
for arcPce := 0; arcPce < app.NumPce(int(upArcCnt), pagesize); arcPce++ { // travel the upper's archive by piece
|
|
var upArcs []*ugc.ArcFull
|
|
if upArcs, err = s.dao.PickUpArcs(ctx, int(mid), arcPce, pagesize); err != nil {
|
|
errArcPce("PickUpArcs", mid, arcPce, err)
|
|
continue
|
|
}
|
|
if len(upArcs) == 0 {
|
|
errArcPce("PickUpArcs", mid, arcPce, fmt.Errorf("Empty Arcs, Stop Picking"))
|
|
break
|
|
}
|
|
if err = s.fullArcs(ctx, upArcs); err != nil {
|
|
errArcPce("FullArcs", mid, arcPce, err)
|
|
}
|
|
time.Sleep(time.Duration(s.c.UgcSync.Frequency.FullRefArcFre)) // pause between each archives pce treatment
|
|
}
|
|
treatedUp = treatedUp + 1
|
|
totalArcs = totalArcs + upArcCnt
|
|
log.Info("fullRefresh Total Up %d, Treated Up %d, Treated Arcs %d, Time Used %v", totalUp, treatedUp, totalArcs, time.Since(begin))
|
|
}
|
|
log.Info("fullRefresh Ends! Len Uppers %d, Time Used %v", len(s.activeUps), time.Since(begin))
|
|
}
|
|
|
|
func (s *Service) fullArcs(ctx context.Context, arcs []*ugc.ArcFull) (err error) {
|
|
for _, arc := range arcs {
|
|
var (
|
|
arcOk, actVideos, shouldAudit bool
|
|
aid = arc.AID
|
|
transCids []int64
|
|
arcAllow = &ugc.ArcAllow{}
|
|
)
|
|
if err = s.dao.SetArcCMS(ctx, &arc.ArcCMS); err != nil { // set cache
|
|
errArc("SetArcCMS", aid, err) // cache error, ignore
|
|
}
|
|
if arc.Deleted == 1 {
|
|
if actVideos, err = s.dao.ActVideos(ctx, aid); err != nil {
|
|
errArc("actVideos", aid, err) // db error
|
|
continue
|
|
}
|
|
if !actVideos {
|
|
infoArc("actVideos", aid, "Arc Deleted && No Active Videos, Jump to the next")
|
|
continue
|
|
} else {
|
|
if err = s.dao.DelVideos(ctx, aid); err != nil { // delete also the videos
|
|
errArc("actVideos", aid, err)
|
|
continue
|
|
}
|
|
infoArc("actVideos", aid, "Arc Deleted, So we delete the rest videos")
|
|
}
|
|
}
|
|
arcAllow.FromArcFull(arc)
|
|
if arcOk = s.arcAllowImport(arcAllow); !arcOk {
|
|
log.Warn("[fullRefresh-ArcPce-Arc]")
|
|
continue
|
|
}
|
|
if arcOk, transCids, err = s.transFailTreat(ctx, aid); err != nil {
|
|
errArcVideos("TransFailVideos-DelVideos", aid, transCids, err) // db error
|
|
continue
|
|
}
|
|
if !arcOk {
|
|
continue
|
|
}
|
|
if shouldAudit, err = s.dao.ShouldAudit(ctx, aid); err != nil {
|
|
errArc("ShouldAudit", aid, err)
|
|
continue
|
|
}
|
|
if shouldAudit {
|
|
log.Info("fullRefresh addAudCid cAid %d", aid)
|
|
s.audAidCh <- []int64{aid} // add aid into channel to treat
|
|
}
|
|
if err = s.refArcVideo(ctx, aid); err != nil {
|
|
errArc("refArcVideo", aid, err)
|
|
continue
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
return
|
|
}
|
|
|
|
func (s *Service) transFailTreat(ctx context.Context, aid int64) (arcOk bool, failCids []int64, err error) {
|
|
arcOk = true
|
|
if failCids, err = s.dao.TransFailVideos(ctx, aid); err != nil { // delete transcoding failed cids
|
|
errArc("TransFailVideos", aid, err) // db error, stop this archive here
|
|
return
|
|
}
|
|
if len(failCids) == 0 {
|
|
// infoArcVideos("TransFailVideos", aid, failCids, "No Fail Cids")
|
|
return
|
|
}
|
|
if arcOk, err = s.dao.DelVideoArc(ctx, &ugc.DelVideos{
|
|
AID: aid,
|
|
CIDs: failCids,
|
|
}); err != nil {
|
|
return
|
|
}
|
|
if !arcOk {
|
|
infoArcVideos("TransFailVideos", aid, failCids, " Delete Videos & Arc succ")
|
|
return
|
|
}
|
|
infoArcVideos("TransFailVideos", aid, failCids, " Delete Videos succ")
|
|
return
|
|
}
|
|
|
|
func (s *Service) refArcVideo(ctx context.Context, cAid int64) (err error) {
|
|
var (
|
|
proName = "videoProducer-video"
|
|
pagesize = s.c.UgcSync.Batch.ProducerPS
|
|
videoCnt int
|
|
maxID = 0
|
|
)
|
|
if videoCnt, err = s.dao.ArcVideoCnt(ctx, cAid); err != nil {
|
|
log.Error("[%s] CountArcs Aid %d, error [%v]", proName, cAid, err)
|
|
return
|
|
}
|
|
if videoCnt == 0 {
|
|
return
|
|
}
|
|
nbPiece := app.NumPce(videoCnt, pagesize)
|
|
log.Info("[%s] NumPiece %d, Pagesize %d", proName, nbPiece, pagesize)
|
|
for i := 0; i < nbPiece; i++ {
|
|
videos, newMaxID, errR := s.dao.PickArcVideo(ctx, cAid, maxID, pagesize)
|
|
if errR != nil {
|
|
log.Error("[%s] Pick Piece %d Error, Ignore it", proName, i)
|
|
continue
|
|
}
|
|
if newMaxID <= maxID {
|
|
log.Error("[%s] MaxID is not increasing! [%d,%d]", proName, newMaxID, maxID)
|
|
return
|
|
}
|
|
maxID = newMaxID
|
|
for _, v := range videos {
|
|
s.dao.SetVideoCMS(ctx, v)
|
|
}
|
|
time.Sleep(500 * time.Millisecond)
|
|
}
|
|
return
|
|
}
|