412 lines
12 KiB
Go
412 lines
12 KiB
Go
package service
|
||
|
||
import (
|
||
"bufio"
|
||
"context"
|
||
"fmt"
|
||
"go-common/app/job/bbq/video/dao"
|
||
searchv1 "go-common/app/service/bbq/search/api/grpc/v1"
|
||
"go-common/library/log"
|
||
"io"
|
||
"net/url"
|
||
"os"
|
||
|
||
"strconv"
|
||
"time"
|
||
)
|
||
|
||
const (
|
||
_retryTimes = 3
|
||
|
||
_selection = 5 //运营精选状态
|
||
)
|
||
|
||
//taskCheckVideoDBVSES 视频全量脚本
|
||
func (s *Service) taskSyncVideo2ES() {
|
||
var step time.Duration
|
||
var id int64
|
||
for {
|
||
ids, videos, err := s.dao.VideoList(context.Background(), id)
|
||
if err != nil {
|
||
log.Error("sync video err(%v)", err)
|
||
return
|
||
}
|
||
if videos == nil {
|
||
return
|
||
}
|
||
videoStatisticsHive, _ := s.dao.VideoStatisticsHiveList(context.Background(), ids)
|
||
videoStatistics, _ := s.dao.VideoStatisticsList(context.Background(), ids)
|
||
videoTags, _ := s.dao.VideoTagsList(context.Background(), ids)
|
||
req := new(searchv1.SaveVideoRequest)
|
||
for _, v := range videos {
|
||
fmt.Println(v.SVID)
|
||
id = v.SVID
|
||
tmp := &searchv1.VideoESInfo{
|
||
SVID: v.SVID,
|
||
Title: v.Title,
|
||
Content: v.Content,
|
||
MID: v.MID,
|
||
CID: v.CID,
|
||
Pubtime: int64(v.Pubtime),
|
||
Ctime: int64(v.Ctime),
|
||
Mtime: int64(v.Mtime),
|
||
Duration: v.Duration,
|
||
Original: v.Original,
|
||
State: v.State,
|
||
VerID: v.VerID,
|
||
Ver: v.Ver,
|
||
From: v.From,
|
||
AVID: v.AVID,
|
||
Tid: v.Tid,
|
||
SubTid: v.SubTid,
|
||
ISFullScreen: v.ISFullScreen,
|
||
Score: v.Score,
|
||
}
|
||
if videoStatisticsHive[id] != nil {
|
||
tmp.PlayHive = videoStatisticsHive[id].PlayHive
|
||
tmp.FavHive = videoStatisticsHive[id].FavHive
|
||
tmp.CoinHive = videoStatisticsHive[id].CoinHive
|
||
tmp.SubtitlesHive = videoStatisticsHive[id].SubtitlesHive
|
||
tmp.LikesHive = videoStatisticsHive[id].LikesHive
|
||
tmp.ShareHive = videoStatisticsHive[id].ShareHive
|
||
tmp.ReportHive = videoStatisticsHive[id].ReportHive
|
||
tmp.DurationDailyHive = videoStatisticsHive[id].DurationDailyHive
|
||
tmp.DurationAllHive = videoStatisticsHive[id].DurationAllHive
|
||
tmp.ReplyHive = videoStatisticsHive[id].ReplyHive
|
||
tmp.ShareDailyHive = videoStatisticsHive[id].ShareDailyHive
|
||
tmp.PlayDailyHive = videoStatisticsHive[id].PlayDailyHive
|
||
tmp.SubtitlesDailyHive = videoStatisticsHive[id].SubtitlesDailyHive
|
||
tmp.LikesDailyHive = videoStatisticsHive[id].LikesDailyHive
|
||
tmp.FavDailyHive = videoStatisticsHive[id].FavDailyHive
|
||
tmp.ReplyDailyHive = videoStatisticsHive[id].ReplyDailyHive
|
||
tmp.AccessHive = videoStatisticsHive[id].AccessHive
|
||
}
|
||
if videoStatistics[id] != nil {
|
||
tmp.Play = videoStatistics[id].Play
|
||
tmp.Subtitles = videoStatistics[id].Subtitles
|
||
tmp.Like = videoStatistics[id].Like
|
||
tmp.Share = videoStatistics[id].Share
|
||
tmp.Report = videoStatistics[id].Report
|
||
}
|
||
if videoTags[id] != nil {
|
||
tmp.Tags = videoTags[id]
|
||
}
|
||
req.List = append(req.List, tmp)
|
||
}
|
||
step = 1
|
||
for {
|
||
if _, err := s.dao.SearchClient.SaveVideo(context.Background(), req); err != nil {
|
||
time.Sleep(step * time.Second)
|
||
step++
|
||
continue
|
||
}
|
||
break
|
||
}
|
||
}
|
||
|
||
}
|
||
|
||
//SaveVideo2ES 保存视频到es
|
||
func (s *Service) SaveVideo2ES(ids string) (res bool) {
|
||
res = true
|
||
if len(ids) == 0 {
|
||
return
|
||
}
|
||
videos, err := s.dao.VideoListByIDs(context.Background(), ids)
|
||
if err != nil || videos == nil {
|
||
res = false
|
||
return
|
||
}
|
||
videoStatisticsHive, _ := s.dao.VideoStatisticsHiveList(context.Background(), ids)
|
||
videoStatistics, _ := s.dao.VideoStatisticsList(context.Background(), ids)
|
||
// videoTags, _ := s.dao.VideoTagsList(context.Background(), ids)
|
||
var step time.Duration
|
||
var id int64
|
||
req := new(searchv1.SaveVideoRequest)
|
||
for _, v := range videos {
|
||
id = v.SVID
|
||
fmt.Println(id)
|
||
tmp := &searchv1.VideoESInfo{
|
||
SVID: v.SVID,
|
||
Title: v.Title,
|
||
Content: v.Content,
|
||
MID: v.MID,
|
||
CID: v.CID,
|
||
Pubtime: int64(v.Pubtime),
|
||
Ctime: int64(v.Ctime),
|
||
Mtime: int64(v.Mtime),
|
||
Duration: v.Duration,
|
||
Original: v.Original,
|
||
State: v.State,
|
||
VerID: v.VerID,
|
||
Ver: v.Ver,
|
||
From: v.From,
|
||
AVID: v.AVID,
|
||
Tid: v.Tid,
|
||
SubTid: v.SubTid,
|
||
ISFullScreen: v.ISFullScreen,
|
||
Score: v.Score,
|
||
}
|
||
if videoStatisticsHive[id] != nil {
|
||
tmp.PlayHive = videoStatisticsHive[id].PlayHive
|
||
tmp.FavHive = videoStatisticsHive[id].FavHive
|
||
tmp.CoinHive = videoStatisticsHive[id].CoinHive
|
||
tmp.SubtitlesHive = videoStatisticsHive[id].SubtitlesHive
|
||
tmp.LikesHive = videoStatisticsHive[id].LikesHive
|
||
tmp.ShareHive = videoStatisticsHive[id].ShareHive
|
||
tmp.ReportHive = videoStatisticsHive[id].ReportHive
|
||
tmp.DurationDailyHive = videoStatisticsHive[id].DurationDailyHive
|
||
tmp.DurationAllHive = videoStatisticsHive[id].DurationAllHive
|
||
tmp.ReplyHive = videoStatisticsHive[id].ReplyHive
|
||
tmp.ShareDailyHive = videoStatisticsHive[id].ShareDailyHive
|
||
tmp.PlayDailyHive = videoStatisticsHive[id].PlayDailyHive
|
||
tmp.SubtitlesDailyHive = videoStatisticsHive[id].SubtitlesDailyHive
|
||
tmp.LikesDailyHive = videoStatisticsHive[id].LikesDailyHive
|
||
tmp.FavDailyHive = videoStatisticsHive[id].FavDailyHive
|
||
tmp.ReplyDailyHive = videoStatisticsHive[id].ReplyDailyHive
|
||
tmp.AccessHive = videoStatisticsHive[id].AccessHive
|
||
}
|
||
if videoStatistics[id] != nil {
|
||
tmp.Play = videoStatistics[id].Play
|
||
tmp.Subtitles = videoStatistics[id].Subtitles
|
||
tmp.Like = videoStatistics[id].Like
|
||
tmp.Share = videoStatistics[id].Share
|
||
tmp.Report = videoStatistics[id].Report
|
||
}
|
||
// if videoTags[id] != nil {
|
||
// tmp.Tags = videoTags[id]
|
||
// }
|
||
req.List = append(req.List, tmp)
|
||
}
|
||
step = 1
|
||
for {
|
||
if _, err := s.dao.SearchClient.SaveVideo(context.Background(), req); err != nil {
|
||
if step == 11 {
|
||
log.Error("save es err(%v) ids(%s)", err, ids)
|
||
res = false
|
||
break
|
||
}
|
||
time.Sleep(step * time.Second)
|
||
step++
|
||
continue
|
||
}
|
||
break
|
||
}
|
||
return
|
||
}
|
||
|
||
func formArrayString(arr []int64) string {
|
||
var res string
|
||
for i, v := range arr {
|
||
if i != 0 {
|
||
res += ","
|
||
}
|
||
res += strconv.FormatInt(v, 10)
|
||
}
|
||
return res
|
||
}
|
||
|
||
//deltaSync2ES 为不同表进行增量同步的脚本，baseTableQuery指明不同表的查询语句
|
||
func (s *Service) deltaSync2ES(taskName string, baseTableQuery string) {
|
||
task, err := s.dao.RawCheckTask(context.Background(), taskName)
|
||
if err != nil {
|
||
log.Error("get last_chek_time fail: task=%s", taskName)
|
||
return
|
||
}
|
||
log.Info("get last_chek_time succ: task=%s, last_check_time=%d", taskName, task.LastCheck)
|
||
|
||
// 获得所有变更的svid
|
||
ids, mtime, err := s.dao.RawGetIDByMtime(baseTableQuery, task.LastCheck)
|
||
if err != nil {
|
||
log.Error("get raw id by mtime fail: task=%s, last_mtime=%d, base_table_query=%s",
|
||
taskName, mtime, baseTableQuery)
|
||
return
|
||
}
|
||
idsNum := len(ids)
|
||
log.Info("get changed svids: task=%s, id_num=%d", taskName, idsNum)
|
||
if idsNum == 0 {
|
||
return
|
||
}
|
||
task.LastCheck = mtime
|
||
|
||
// 对所有变更的svid分批次进行同步到es
|
||
for i := 0; i < idsNum; i += dao.MaxSyncESNum {
|
||
last := i + dao.MaxSyncESNum
|
||
if last > idsNum {
|
||
last = idsNum
|
||
}
|
||
selectedIDs := ids[i:last]
|
||
idsStr := formArrayString(selectedIDs)
|
||
if res := s.SaveVideo2ES(idsStr); !res {
|
||
log.Error("sync video 2 es fail: task=%s, offset=%d, id_num=%d, base_table_query=%s",
|
||
taskName, i, idsNum, baseTableQuery)
|
||
return
|
||
}
|
||
log.Info("one sync video 2 es: task=%s, offset=%d, id_num=%d", taskName, i, idsNum)
|
||
}
|
||
|
||
// 更新task最近check的时间点
|
||
if _, err := s.dao.UpdateTaskLastCheck(context.Background(), taskName, task.LastCheck); err != nil {
|
||
log.Error("update task last check time fail: task=%s, last_mtime=%d, base_table_query=%s",
|
||
taskName, task.LastCheck, baseTableQuery)
|
||
return
|
||
}
|
||
log.Info("sync video 2 es: task=%s, id_num=%d, last_mtime=%d", taskName, idsNum, task.LastCheck)
|
||
}
|
||
|
||
//taskCheckVideo video表增量脚本
|
||
func (s *Service) taskCheckVideo() {
|
||
taskName := "checkVideo"
|
||
s.deltaSync2ES(taskName, dao.QueryVideoByMtime)
|
||
}
|
||
|
||
//taskCheckVideoStatistics video_statistics表增量脚本
|
||
func (s *Service) taskCheckVideoStatistics() {
|
||
taskName := "checkVideoSt"
|
||
s.deltaSync2ES(taskName, dao.QueryVideoStatisticsByMtime)
|
||
}
|
||
|
||
//taskCheckVideoStatisticsHive video_statistics_hive表增量脚本
|
||
func (s *Service) taskCheckVideoStatisticsHive() {
|
||
taskName := "checkVideoStHv"
|
||
s.deltaSync2ES(taskName, dao.QueryVideoStatisticsHiveByMtime)
|
||
}
|
||
|
||
//taskCheckVideoTag video_tag表增量脚本
|
||
func (s *Service) taskCheckVideoTag() {
|
||
taskName := "checkVideoTag"
|
||
s.deltaSync2ES(taskName, dao.QueryVideoTagByMtime)
|
||
}
|
||
|
||
//taskCheckTag tag表增量脚本
|
||
func (s *Service) taskCheckTag() {
|
||
taskName := "checkTag"
|
||
task, err := s.dao.RawCheckTask(context.Background(), taskName)
|
||
if err != nil {
|
||
log.Error("get last_chek_time fail: task=%s", taskName)
|
||
return
|
||
}
|
||
log.Info("get last_chek_time succ: task=%s, last_check_time=%d", taskName, task.LastCheck)
|
||
for {
|
||
ids, mtime, err := s.dao.RawTagByMtime(context.Background(), task.LastCheck)
|
||
if err != nil || len(ids) == 0 {
|
||
return
|
||
}
|
||
id := int64(0)
|
||
for {
|
||
svids, temp, err := s.dao.RawVideoTagByIDs(context.Background(), ids, id)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if len(svids) == 0 {
|
||
break
|
||
}
|
||
if flag := s.SaveVideo2ES(svids); !flag {
|
||
return
|
||
}
|
||
id = temp
|
||
}
|
||
if num, err := s.dao.UpdateTaskLastCheck(context.Background(), taskName, mtime); err != nil || num == 0 {
|
||
return
|
||
}
|
||
task.LastCheck = mtime
|
||
}
|
||
}
|
||
|
||
// taskRmInvalidES 删除es中多余的视频
|
||
func (s *Service) taskRmInvalidES() {
|
||
fmt.Println("aaa")
|
||
esReq := new(searchv1.ESVideoDataRequest)
|
||
delReq := new(searchv1.DelVideoBySVIDRequest)
|
||
svid := int64(0)
|
||
query := `{"query":{"range":{"svid":{"gt":%d}}},"sort":[{"svid":"asc"}],"from":0,"size":10}`
|
||
for {
|
||
esReq.Query = fmt.Sprintf(query, svid)
|
||
res, err := s.dao.SearchClient.ESVideoData(context.Background(), esReq)
|
||
if err != nil {
|
||
return
|
||
}
|
||
svids := make([]string, 0)
|
||
for _, v := range res.List {
|
||
svids = append(svids, strconv.Itoa(int(v.SVID)))
|
||
svid = v.SVID
|
||
}
|
||
vs, err := s.dao.RawVideoBySVIDS(context.Background(), svids)
|
||
if err != nil {
|
||
return
|
||
}
|
||
notList := make([]int64, 0)
|
||
for _, v := range res.List {
|
||
if _, ok := vs[v.SVID]; !ok {
|
||
fmt.Println(v.SVID)
|
||
notList = append(notList, v.SVID)
|
||
}
|
||
}
|
||
if len(notList) != 0 {
|
||
delReq.SVIDs = notList
|
||
s.dao.SearchClient.DelVideoBySVID(context.Background(), delReq)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (s *Service) commitCID() {
|
||
ctx := context.Background()
|
||
path := s.c.URLs["bvc_push"]
|
||
if path == "" {
|
||
return
|
||
}
|
||
srcPath := s.c.Path["cids"]
|
||
if srcPath == "" {
|
||
return
|
||
}
|
||
if srcPath == "" {
|
||
log.Error("sugsrc path is empty")
|
||
return
|
||
}
|
||
src, err := os.Open(srcPath)
|
||
if err != nil {
|
||
log.Error("writeSug os.Open source sug error(%v)", err)
|
||
return
|
||
}
|
||
defer src.Close()
|
||
br := bufio.NewReader(src)
|
||
i := 1
|
||
for {
|
||
a, _, c := br.ReadLine()
|
||
if c == io.EOF {
|
||
break
|
||
}
|
||
cid, err := strconv.ParseInt(string(a), 10, 64)
|
||
if err != nil {
|
||
log.Error("parse err [%v]", err)
|
||
continue
|
||
}
|
||
svid, err := s.dao.GetSvidByCid(ctx, cid)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
params := url.Values{}
|
||
params.Set("svid", strconv.FormatInt(svid, 10))
|
||
params.Set("cid", string(a))
|
||
req, err := s.dao.HTTPClient.NewRequest("GET", path, "", params)
|
||
if err != nil {
|
||
log.Error("error(%v)", err)
|
||
continue
|
||
}
|
||
var res struct {
|
||
Code int `json:"code"`
|
||
Msg string `json:"message"`
|
||
}
|
||
if err = s.dao.HTTPClient.Do(ctx, req, &res); err != nil {
|
||
log.Errorv(ctx, log.KV("log", fmt.Sprintf("err[%v]", err)))
|
||
continue
|
||
}
|
||
if res.Code != 0 {
|
||
log.Errorv(ctx, log.KV("log", fmt.Sprintf("error(%v)", err)))
|
||
} else {
|
||
log.Info("commit svid:%d cid:%d success No.%d", svid, cid, i)
|
||
}
|
||
i++
|
||
}
|
||
}
|