Create & Init Project...

app/job/bbq/video/service/BUILD

@@ -0,0 +1,57 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "binlogv.go",
+        "binlogvr.go",
+        "bvcsub.go",
+        "cmsvideo.go",
+        "comment.go",
+        "email.go",
+        "ftp.go",
+        "notice.go",
+        "service.go",
+        "user.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/video/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/dao:go_default_library",
+        "//app/job/bbq/video/model:go_default_library",
+        "//app/service/bbq/common:go_default_library",
+        "//app/service/bbq/notice-service/api/v1:go_default_library",
+        "//app/service/bbq/search/api/grpc/v1:go_default_library",
+        "//app/service/bbq/topic/api:go_default_library",
+        "//app/service/bbq/video/api/grpc/v1:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/ftp-master:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/service/binlogv.go

@@ -0,0 +1,197 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/model"
+	"go-common/app/service/bbq/common"
+	topic "go-common/app/service/bbq/topic/api"
+	"go-common/library/log"
+	"strconv"
+)
+
+// videoConsumeproc 视频表消费
+func (s *Service) videoBinlogSub() {
+	var msgs = s.videoSub.Messages()
+	for {
+		var err error
+		msg, ok := <-msgs
+		if !ok {
+			log.Info("userCanal databus Consumer exit")
+			return
+		}
+		res := &model.DatabusRes{}
+		log.Infov(context.Background(), log.KV("log", fmt.Sprintf("canal message %s", string(msg.Value))))
+		if err = json.Unmarshal(msg.Value, &res); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			msg.Commit()
+			continue
+		}
+
+		if res.Table != "video" || (res.Action != "update" && res.Action != "insert") {
+			msg.Commit()
+			continue
+		}
+
+		var vNew, vOld *model.VideoRaw
+		if res.Action == "insert" || res.Action == "update" {
+			if err = json.Unmarshal(res.New, &vNew); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.New))
+				continue
+			}
+		}
+		if res.Action == "update" {
+			if err = json.Unmarshal(res.Old, &vOld); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.Old))
+				continue
+			}
+		}
+
+		//idempotent consume
+		for i := 0; i < _retryTimes; i++ {
+			//fetured video state subscription
+			if err = s.VideoStateSub(vNew, vOld); err == nil {
+				break
+			}
+		}
+		//s.UpdateCms(context.Background(), vNew)
+		//register comment
+		for i := 0; i < _retryTimes; i++ {
+			//merge related information subscription
+			if err = s.CommentReg(context.Background(), vNew.SVID, model.StateActive); err == nil {
+				break
+			}
+		}
+
+		if res.Action == "insert" {
+			for i := 0; i < _retryTimes; i++ {
+				log.V(1).Infow(context.Background(), "log", "merge up info", "retry_time", i, "mid", vNew.MID, "svid", vNew.SVID)
+				//merge related information subscription
+				if err = s.MergeUpInfoSub(vNew); err == nil {
+					break
+				}
+			}
+		}
+
+		//unidempotent consume
+		if res.Action == "update" {
+			s.UpdateStaInfoSub(vNew, vOld)
+		} else if res.Action == "insert" {
+			s.AddSVTotal(vNew)
+		}
+		msg.Commit()
+	}
+}
+
+//UpdateCms ..
+func (s *Service) UpdateCms(c context.Context, vNew *model.VideoRaw) (err error) {
+	if err = s.dao.UpdateCms(c, vNew); err != nil {
+		log.Warnw(c, "event", fmt.Sprintf("updateCms err:%v,param:%v", err, vNew))
+	}
+	return
+}
+
+// VideoStateSub 视频状态变更消费
+func (s *Service) VideoStateSub(vNew *model.VideoRaw, vOld *model.VideoRaw) (err error) {
+	log.Infow(context.Background(), "log", "one video state sub", "svid", vNew.SVID)
+
+	s.SaveVideo2ES(strconv.Itoa(int(vNew.SVID)))
+
+	if vOld == nil || vNew.State != vOld.State {
+		var ids []int64
+		ids, err = s.dao.GetRecallOpVideo(context.Background())
+		if err != nil {
+			log.Warnw(context.Background(), "log", "get recall op video fail")
+			return
+		}
+		needSetRecallOpVideo := true
+		if vNew.State == _selection {
+			for _, id := range ids {
+				if id == vNew.SVID {
+					needSetRecallOpVideo = false
+					break
+				}
+			}
+			ids = append(ids, vNew.SVID)
+		} else if vNew.State != _selection {
+			index := -1
+			for i, id := range ids {
+				if id == vNew.SVID {
+					index = i
+					break
+				}
+			}
+			if index != -1 {
+				ids = append(ids[:index], ids[index+1:]...)
+			} else {
+				needSetRecallOpVideo = false
+			}
+		}
+		if needSetRecallOpVideo {
+			if err = s.dao.SetRecallOpVideo(context.Background(), ids); err != nil {
+				log.Warnw(context.Background(), "log", "get recall op video fail")
+				return
+			}
+		}
+	}
+
+	// 话题状态变更
+	needUpdateTopicVideoState := false
+	topicState := topic.TopicVideoStateUnAvailable
+	if vOld == nil {
+		needUpdateTopicVideoState = true
+		// update topic video
+		if common.IsTopicSvStateAvailable(int64(vNew.State)) {
+			topicState = topic.TopicVideoStateAvailable
+		}
+	} else {
+		if common.IsTopicSvStateAvailable(int64(vNew.State)) != common.IsTopicSvStateAvailable(int64(vOld.State)) {
+			needUpdateTopicVideoState = true
+			if common.IsTopicSvStateAvailable(int64(vNew.State)) {
+				topicState = topic.TopicVideoStateAvailable
+			}
+		}
+	}
+	if needUpdateTopicVideoState {
+		_, err = s.topicClient.UpdateVideoState(context.Background(), &topic.UpdateVideoStateReq{Svid: vNew.SVID, State: int32(topicState)})
+		log.Infow(context.Background(), "log", "update topic video state", "svid", vNew.SVID, "new_state", vNew.State)
+		if err != nil {
+			log.Warnw(context.Background(), "log", "update topic video state", "svid", vNew.SVID, "new_state", vNew.State)
+			return
+		}
+	}
+	return
+}
+
+//MergeUpInfoSub ..
+func (s *Service) MergeUpInfoSub(vNew *model.VideoRaw) (err error) {
+	mid := vNew.MID
+	if err = s.dao.MergeUpInfo(mid); err != nil {
+		log.Error("MergeUpInfo failed,err:%v,mid:%d", err, mid)
+	}
+	return
+}
+
+//UpdateStaInfoSub ...
+func (s *Service) UpdateStaInfoSub(vNew *model.VideoRaw, vOld *model.VideoRaw) {
+	if vOld == nil || vNew.State == vOld.State {
+		return
+	}
+	if vNew.State == model.VideoStInactive {
+		s.dao.UpdateUVSt(vNew.MID, "unshelf_av_total")
+	} else if vNew.State == model.VideoStDeleted {
+		s.dao.UpdateUVStDel(vNew.MID, "av_total")
+	}
+
+	if vOld.State == model.VideoStInactive {
+		s.dao.UpdateUVStDel(vNew.MID, "unshelf_av_total")
+	} else if vNew.State == model.VideoStDeleted {
+		s.dao.UpdateUVSt(vOld.MID, "av_total")
+	}
+}
+
+//AddSVTotal ...
+func (s *Service) AddSVTotal(vNew *model.VideoRaw) {
+	s.dao.AddSVTotal(vNew.MID)
+}

app/job/bbq/video/service/binlogvr.go

@@ -0,0 +1,103 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/model"
+	videov1 "go-common/app/service/bbq/video/api/grpc/v1"
+	"go-common/library/log"
+)
+
+//videoRepositorySub video_repository subscription .
+func (s *Service) videoRepositoryBinlogSub() {
+	msgs := s.videoRep.Messages()
+	for {
+		var err error
+		msg, ok := <-msgs
+		if !ok {
+			log.Info("video_repository databus Consumer exit")
+			return
+		}
+		res := &model.DatabusRes{}
+		log.Infov(context.Background(), log.KV("log", fmt.Sprintf("canal message %s", string(msg.Value))))
+		if err = json.Unmarshal(msg.Value, &res); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			msg.Commit()
+			continue
+		}
+
+		if res.Table != "video_repository" || (res.Action != "update" && res.Action != "insert") {
+			msg.Commit()
+			continue
+		}
+
+		//unserialize databus struct
+		var vNew, vOld *model.VideoRepRaw
+		if res.Action == "insert" || res.Action == "update" {
+			if err = json.Unmarshal(res.New, &vNew); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.New))
+				msg.Commit()
+				continue
+			}
+		}
+		if res.Action == "update" {
+			if err = json.Unmarshal(res.Old, &vOld); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.Old))
+				msg.Commit()
+				continue
+			}
+		}
+
+		if res.Action == "insert" {
+			for i := 0; i < _retryTimes; i++ {
+				if err = s.PepareResource(vNew); err == nil {
+					break
+				}
+			}
+		}
+		msg.Commit()
+	}
+}
+
+//PepareResource ...
+func (s *Service) PepareResource(vNew *model.VideoRepRaw) (err error) {
+	var (
+		ctx  = context.Background()
+		SVID int64
+		row  *model.VideoRepRaw
+	)
+	//bbq/cms video not trans to bvc
+	if vNew.From == model.VideoFromBBQ || vNew.From == model.VideoFromCMS {
+		return
+	}
+	if row, err = s.dao.RawVideoByID(ctx, vNew.ID); err != nil {
+		return
+	}
+	if row.SVID > 0 {
+		SVID = row.SVID
+	} else {
+		req := &videov1.CreateIDRequest{
+			Mid: vNew.MID,
+		}
+		var rep *videov1.CreateIDResponse
+		if rep, err = s.dao.VideoClient.CreateID(ctx, req); err != nil {
+			log.Error("Numbering device return err:%v", err)
+			return
+		}
+		if err = s.dao.UpdateSvid(context.Background(), vNew.ID, rep.NewId); err != nil {
+			return
+		}
+		SVID = rep.NewId
+	}
+	reqBvc := &videov1.BVideoTransRequset{
+		SVID: SVID,
+		CID:  vNew.CID,
+	}
+	log.Info("bvc trans commit req:%v", reqBvc)
+	if _, err = s.dao.VideoClient.BVCTransCommit(ctx, reqBvc); err != nil {
+		log.Error("BVCTransCommit err :%v,req:%v", err, reqBvc)
+	}
+	s.dao.UpdateSyncStatus(ctx, SVID, model.SourceRequest)
+	return
+}

app/job/bbq/video/service/bvcsub.go

@@ -0,0 +1,120 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/model"
+	videov1 "go-common/app/service/bbq/video/api/grpc/v1"
+	"go-common/library/log"
+	"strings"
+)
+
+//BvcTransSub ...
+func (s *Service) BvcTransSub() {
+	msgs := s.bvcSub.Messages()
+	for {
+		var (
+			err error
+			vr  *model.VideoRepRaw
+		)
+		c := context.Background()
+		msg, ok := <-msgs
+
+		//release subscription
+		if s.c.SubBvcControl.Control == 2 {
+			msg.Commit()
+			continue
+		}
+		if !ok {
+			log.Info("BvcTransSub databus Consumer exit")
+			return
+		}
+		res := &model.DatabusBVCTransSub{}
+		log.Infov(context.Background(), log.KV("log", fmt.Sprintf("databus message %s", string(msg.Value))))
+		if err = json.Unmarshal(msg.Value, &res); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			msg.Commit()
+			continue
+		}
+		if vr, err = s.dao.RawVideo(c, res.SVID); err != nil {
+			msg.Commit()
+			continue
+		}
+		//resource check
+		if vr.SyncStatus&model.SourceXcodeCover > 0 {
+			if err = s.importVideo(c, vr); err != nil {
+				log.Errorw(c, "errmsg", "importVideo err", "req", vr, "err", err)
+				msg.Commit()
+				continue
+			}
+			s.syncTag(c, vr.Tag)
+
+			s.dao.UpdateSyncStatus(c, vr.SVID, model.SourceOnshelf)
+		}
+
+		msg.Commit()
+	}
+}
+
+//importVideo put video on shelf
+func (s *Service) importVideo(c context.Context, vr *model.VideoRepRaw) (err error) {
+	// var (
+	// 	st int64
+	// )
+	// if vr.From == model.VideoFromBILI {
+	// 	st = model.VideoStPassReview
+	// } else {
+	// 	st = model.VideoStPendingPassReview
+	// }
+	req := &videov1.ImportVideoInfo{
+		AVID:        vr.AVID,
+		Svid:        vr.SVID,
+		MID:         vr.MID,
+		CID:         vr.CID,
+		SubTID:      vr.SubTID,
+		TID:         vr.TID,
+		Title:       vr.Title,
+		Pubtime:     vr.Pubtime,
+		From:        int64(vr.From),
+		CoverUrl:    vr.CoverURL,
+		CoverHeight: vr.CoverHeight,
+		CoverWidth:  vr.CoverWidth,
+		//State:         st,
+		HomeImgHeight: vr.HomeImgHeight,
+		HomeImgUrl:    vr.HomeImgURL,
+		HomeImgWidth:  vr.HomeImgWidth,
+	}
+	for i := 0; i < _retryTimes; i++ {
+		if _, err = s.dao.VideoClient.ImportVideo(c, req); err == nil {
+			break
+		}
+	}
+	return
+}
+
+//syncTag sync video from bilibili common tag
+func (s *Service) syncTag(c context.Context, t string) (err error) {
+	if t == "" {
+		return
+	}
+	var (
+		arrTag []string
+		tag    []*videov1.TagInfo
+	)
+	arrTag = strings.Split(t, ",")
+	for _, v := range arrTag {
+		tmp := &videov1.TagInfo{
+			TagName: v,
+			TagType: 3,
+		}
+		tag = append(tag, tmp)
+	}
+	reqTag := &videov1.SyncVideoTagRequest{
+		TagInfos: tag,
+	}
+	if _, err = s.dao.VideoClient.SyncTag(c, reqTag); err != nil {
+		log.Error("sync tag err :%v,tag:%v", err, tag)
+	}
+	return
+}

app/job/bbq/video/service/cmsvideo.go

@@ -0,0 +1,17 @@
+package service
+
+import (
+	"go-common/library/log"
+)
+
+//TransToCheckBack ..
+func (s *Service) TransToCheckBack() {
+	log.Info("deliveryNewVdieoToCms begin")
+	s.dao.TransToCheckBack()
+}
+
+//TransToReview ...
+func (s *Service) TransToReview() {
+	log.Info("TransToReview begin")
+	s.dao.TransToReview()
+}

app/job/bbq/video/service/comment.go

@@ -0,0 +1,76 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"go-common/library/log"
+	"sync"
+	"sync/atomic"
+)
+
+const (
+	_stateActive = 0
+	_defaultType = 23
+)
+
+//CommentReg 评论开关
+func (s *Service) CommentReg(c context.Context, oid int64, state int16) (err error) {
+	req := map[string]interface{}{
+		"oid":   oid,
+		"type":  _defaultType,
+		"adid":  0,
+		"mid":   0,
+		"state": state,
+	}
+	err = s.dao.ReplyReg(c, req)
+	return
+}
+
+// AutoRegAll 批量注册评论
+func (s *Service) AutoRegAll(c context.Context) {
+	var (
+		IDs     []int64
+		e       int
+		err     error
+		lastID  int64
+		num     int64
+		failNum int64
+	)
+	for {
+		IDs, lastID, _ = s.dao.GetVideoByLastID(c, lastID)
+		if len(IDs) == 0 {
+			break
+		}
+		sum := len(IDs)
+		partNum := 200
+		rnum := int(sum / partNum)
+		log.Info("rountine start num[%d]", rnum)
+		var wg sync.WaitGroup
+		for i := 0; i < rnum+1; i++ {
+			wg.Add(1)
+			if e = (i + 1) * partNum; e > sum {
+				e = sum
+			}
+			r := IDs[i*partNum : e]
+			go func(ids []int64, k int) {
+				defer wg.Done()
+				for _, svid := range ids {
+					err = s.CommentReg(c, svid, _stateActive)
+					if err != nil {
+						atomic.AddInt64(&failNum, 1)
+						fmt.Printf("Comment active fail [%v]\n", err)
+						log.Errorv(c, log.KV("log", fmt.Sprintf("Comment active fail [%v]", err)))
+					} else {
+						atomic.AddInt64(&num, 1)
+						fmt.Printf("Comment active success oid:[%d]\n", svid)
+						log.Infov(c, log.KV("log", fmt.Sprintf("Comment active success oid:[%d]", svid)))
+					}
+				}
+			}(r, i)
+		}
+		wg.Wait()
+	}
+	log.Info("comment reg complete succ[%d] fail[%d]", num, failNum)
+	fmt.Printf("comment reg complete succ[%d] fail[%d]", num, failNum)
+
+}

app/job/bbq/video/service/email.go

@@ -0,0 +1,34 @@
+package service
+
+import (
+	"go-common/app/job/bbq/video/model"
+
+	gomail "gopkg.in/gomail.v2"
+)
+
+//SendMail ...
+func (s *Service) SendMail(mailType int) (err error) {
+	var (
+		m       = gomail.NewMessage()
+		message = map[string][]string{}
+		cType   string
+		cBody   string
+	)
+
+	switch mailType {
+	//运营i后台脚本导入完成推送邮件
+	case model.JobFinishNotice:
+		message["To"] = s.c.Mail.To
+		message["Subject"] = []string{"同步运营筛选视频任务已完成"}
+		cBody = "运营筛选视频已经导入完成！"
+		cType = "text/plain"
+	case 2:
+		message["To"] = []string{"write your address"}
+	}
+
+	m.SetHeaders(message)
+	m.SetBody(cType, cBody)
+
+	err = s.dao.SendMail(m)
+	return
+}

app/job/bbq/video/service/ftp.go

@@ -0,0 +1,357 @@
+package service
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"go-common/app/job/bbq/video/model"
+	"go-common/library/log"
+
+	ftp "github.com/ftp-master"
+)
+
+const (
+	filePathKey      = "search"
+	videoFileName    = "bbqvideo"
+	videoFileMD5Name = "bbqvideo.md5"
+	userFileName     = "bbquser"
+	userFileMD5Name  = "bbquser.md5"
+	sugFileName      = "bbqsug"
+	sugFileMD5Name   = "bbqsug.md5"
+	sugSrcIDIdx      = 0
+	sugSrcTermIdx    = 1
+	sugSrcTypeIdx    = 2
+	sugSrcScoreIdx   = 3
+	defaultSugPath   = "/src/go-common/app/job/bbq/video/cmd/sug"
+)
+
+// writeSug .
+func (s *Service) writeSug(path string, filename string, md5name string) (err error) {
+	var num int64
+	filePath := path + filename
+	file, err := os.OpenFile(filePath, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		log.Error("writeSug os.OpenFile(%s) error(%v)", filename, err)
+		return
+	}
+	defer file.Close()
+	srcPath := s.c.FTP.LocalPath["sugsrc"]
+	if srcPath == "" {
+		srcPath = os.Getenv("GOPATH") + defaultSugPath
+	}
+	if srcPath == "" {
+		log.Error("sugsrc path is empty")
+		return
+	}
+	src, err := os.Open(srcPath)
+	if err != nil {
+		log.Error("writeSug os.Open source sug error(%v)", err)
+		return
+	}
+	defer src.Close()
+	br := bufio.NewReader(src)
+	for {
+		a, _, c := br.ReadLine()
+		if c == io.EOF {
+			break
+		}
+		strArr := strings.Split(string(a), ",")
+		var bs []byte
+		var b []byte
+		id, _ := strconv.ParseInt(strArr[sugSrcIDIdx], 10, 64)
+		score, _ := strconv.ParseInt(strArr[sugSrcScoreIdx], 10, 64)
+		sd := &model.Sug{
+			ID:    id,
+			Term:  strArr[sugSrcTermIdx],
+			Type:  strArr[sugSrcTypeIdx],
+			Score: score,
+		}
+		if b, err = json.Marshal(sd); err != nil {
+			log.Warn("json.Marshal(%v) error(%v)", sd, err)
+			continue
+		}
+		bs = append(bs, b...)
+		bs = append(bs, '\n')
+		if _, err = file.Write(bs); err != nil {
+			log.Error("writeSug file.Write error(%v)", err)
+			return
+		}
+		num++
+		// log.Info("write sug term:[%s]", strArr[sugSrcTermIdx])
+	}
+	log.Info("writeSug success num (%d)", num)
+	if err = s.writeMD5(path, filename, md5name); err != nil {
+		return
+	}
+	s.uploadFile(path, filename, md5name)
+	return
+}
+
+// writeUserInfo .
+func (s *Service) writeUserInfo(path string, filename string, md5name string) (err error) {
+	var lastID int64
+	var num int64
+	filePath := path + filename
+	file, err := os.OpenFile(filePath, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		log.Error("writeUserInfo os.OpenFile(%s) error(%v)", filename, err)
+		return
+	}
+	defer file.Close()
+	for {
+		var (
+			dataList []*model.UserBaseDB
+			bs       []byte
+		)
+		if dataList, err = s.dao.UsersByLast(context.Background(), lastID); err != nil {
+			return
+		}
+		if len(dataList) == 0 {
+			break
+		}
+		for _, data := range dataList {
+			var b []byte
+			sd := &model.UserSearch{
+				ID:    data.MID,
+				Uname: data.Uname,
+			}
+			if b, err = json.Marshal(sd); err != nil {
+				log.Warn("json.Marshal(%v) error(%v)", sd, err)
+				continue
+			}
+			bs = append(bs, b...)
+			bs = append(bs, '\n')
+			lastID = data.ID
+			num++
+			// log.Info("append user mid:[%d]", data.MID)
+		}
+		if _, err = file.Write(bs); err != nil {
+			log.Error("writeUserInfo file.Write error(%v)", err)
+			return
+		}
+	}
+	log.Info("writeUserInfo success num (%d)", num)
+	if err = s.writeMD5(path, filename, md5name); err != nil {
+		return
+	}
+	s.uploadFile(path, filename, md5name)
+	return
+}
+
+// writeVideoInfo .
+func (s *Service) writeVideoInfo(path string) (err error) {
+	var lastID int64
+	var num int64
+	var RelateID int64
+	filePath := path + videoFileName
+	file, err := os.OpenFile(filePath, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		log.Error("writeVideoInfo os.OpenFile(%s) error(%v)", videoFileName, err)
+		return
+	}
+	defer file.Close()
+	for {
+		var (
+			dataList []*model.VideoDB
+			bs       []byte
+		)
+		if dataList, err = s.dao.VideosByLast(context.Background(), lastID); err != nil {
+			return
+		}
+		if len(dataList) == 0 {
+			break
+		}
+		for _, data := range dataList {
+			var (
+				b []byte
+			)
+			RelateID, err = strconv.ParseInt(fmt.Sprintf("%d00", data.AutoID), 10, 32)
+			if err != nil {
+				log.Error("relate id parse err [%v]", err)
+				continue
+			}
+			sd := &model.VideoSearch{
+				ID:      int32(RelateID),
+				Title:   data.Title,
+				PubTime: int32(data.Pubtime),
+			}
+			if b, err = json.Marshal(sd); err != nil {
+				log.Warn("json.Marshal(%v) error(%v)", sd, err)
+				continue
+			}
+			bs = append(bs, b...)
+			bs = append(bs, '\n')
+			lastID = data.AutoID
+			num++
+			// log.Info("append video svid:[%d]", data.ID)
+		}
+		if _, err = file.Write(bs); err != nil {
+			log.Error("writeVideoInfo file.Write error(%v)", err)
+			return
+		}
+	}
+	log.Info("writeVideoInfo success num (%d)", num)
+	if err = s.writeMD5(path, videoFileName, videoFileMD5Name); err != nil {
+		return
+	}
+	s.uploadFile(path, videoFileName, videoFileMD5Name)
+	return
+}
+
+// writeMD5 .
+func (s *Service) writeMD5(path string, fname string, md5name string) (err error) {
+	var out bytes.Buffer
+	f, err := exec.LookPath("md5sum")
+	if err != nil {
+		log.Error("writeMD5 exec.LookPath(md5sum) error(%v)", err)
+		f = "md5sum"
+	}
+	cmd := exec.Command(f, path+fname)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		log.Error("writeMD5 cmd.Run() error(%v)", err)
+		return
+	}
+	outStrs := strings.Split(out.String(), " ")
+	if len(outStrs) == 0 {
+		log.Error("writeMD5 len(outStrs) == 0")
+		return errors.New("NONE MD5")
+	}
+	md5FileName := path + md5name
+	md5File, err := os.OpenFile(md5FileName, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY, 0600)
+	if err != nil {
+		log.Error("writeMD5 os.OpenFile(md5 file) error(%v)", err)
+		return
+	}
+	defer md5File.Close()
+	_, err = md5File.WriteString(outStrs[0])
+	return
+}
+
+// SyncVideo2Search 同步视频
+func (s *Service) SyncVideo2Search() {
+	err := s.writeVideoInfo(s.c.FTP.LocalPath[filePathKey])
+	if err != nil {
+		log.Error("SyncVideo err[%v]", err)
+	} else {
+		log.Info("SyncVideo complete")
+	}
+}
+
+// SyncUser2Search 同步用户
+func (s *Service) SyncUser2Search() {
+	err := s.writeUserInfo(s.c.FTP.LocalPath[filePathKey], userFileName, userFileMD5Name)
+	if err != nil {
+		log.Error("syncUser err[%v]", err)
+	} else {
+		log.Info("syncUser complete")
+	}
+}
+
+// SyncSug2Search 同步sug
+func (s *Service) SyncSug2Search() {
+	err := s.writeSug(s.c.FTP.LocalPath[filePathKey], sugFileName, sugFileMD5Name)
+	if err != nil {
+		log.Error("syncSug err[%v]", err)
+	} else {
+		log.Info("syncSug complete")
+	}
+}
+
+// SyncSearch 全量同步搜索各文件
+func (s *Service) SyncSearch() {
+	for {
+		s.searchChan <- "SyncSearch Complete"
+		var wg sync.WaitGroup
+		fmt.Println("SyncSearch Start")
+		stratAt := time.Now()
+		wg.Add(3)
+		go func() {
+			defer wg.Done()
+			s.SyncVideo2Search()
+		}()
+		go func() {
+			defer wg.Done()
+			s.SyncUser2Search()
+		}()
+		go func() {
+			defer wg.Done()
+			s.SyncSug2Search()
+		}()
+		wg.Wait()
+		//耗时统计
+		elapsed := time.Since(stratAt)
+		log.Info("SyncSearch elapsed: %s", elapsed)
+		fmt.Println(<-s.searchChan)
+		//间隔1min
+		time.Sleep(time.Minute * 1)
+	}
+}
+
+// uploadFile .
+func (s *Service) uploadFile(path string, fname string, md5name string) (err error) {
+	ftp, err := ftp.Connect(s.c.FTP.Addr)
+	if err != nil {
+		log.Error("connect to ftp(%s) error(%v)", s.c.FTP.Addr, err)
+		return
+	}
+	defer ftp.Quit()
+	err = ftp.Login(s.c.FTP.User, s.c.FTP.Password)
+	if err != nil {
+		log.Error("ftp login(user:%s) error(%v)", s.c.FTP.User, err)
+		return
+	}
+	defer ftp.Logout()
+	ftpDir := fmt.Sprintf(s.c.FTP.RemotePath[filePathKey], fname)
+	err = ftp.ChangeDir(ftpDir)
+	if err != nil {
+		log.Error("enter ftp path [%s] err [%v]", ftpDir, err)
+		return
+	}
+	// delete source file, and upload file to ftp.
+	if err = ftp.Delete(fname); err != nil {
+		log.Error("ftp.Delete(%s) error(%v)", fname, err)
+	}
+	filePath := path + fname
+	file, err := os.Open(filePath)
+	if err != nil {
+		log.Error("os.Open(%s) error(%v)", filePath, err)
+		return
+	}
+	defer file.Close()
+	if err = ftp.Stor(fname, file); err != nil {
+		log.Error("ftp.Stor(%s) error(%v)", fname, err)
+		return
+	}
+	log.Info("ftp upload success(%s)", fname)
+	// delete source.md5 file, and upload file to ftp.
+	if err = ftp.Delete(md5name); err != nil {
+		log.Error("ftp.Delete(%s) error(%v)", md5name, err)
+	}
+	md5FilePath := path + md5name
+	md5File, err := os.Open(md5FilePath)
+	if err != nil {
+		log.Error("os.Open(%s) error(%v)", md5name, err)
+		return
+	}
+	defer md5File.Close()
+	if err = ftp.Stor(md5name, md5File); err != nil {
+		log.Error("ftp.Stor(%s) error(%v)", md5name, err)
+		return
+	}
+	log.Info("ftp upload success(%s)", md5name)
+	return
+}

app/job/bbq/video/service/notice.go

@@ -0,0 +1,112 @@
+package service
+
+import (
+	"context"
+	"go-common/app/job/bbq/video/dao"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"sync/atomic"
+	"time"
+
+	notice "go-common/app/service/bbq/notice-service/api/v1"
+)
+
+// 信号量，为了避免task执行超过周期，连续执行两个任务会出现问题
+var i32 int32
+
+// SysMsgTask 推送消息
+func (s *Service) SysMsgTask() {
+	ctx := context.Background()
+	newValue := atomic.AddInt32(&i32, 1)
+	defer atomic.AddInt32(&i32, -1)
+	if newValue > 1 {
+		log.Errorv(ctx, log.KV("log", "sysMsgTask pending"))
+		return
+	}
+
+	res, err := s.dao.RawCheckTask(ctx, "checkSysMsg")
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "get last sysMsgTask id fail"))
+		return
+	}
+
+	lastSysMsgID := res.LastCheck
+	curSysMsgID := res.LastCheck
+	list, err := s.dao.GetNewSysMsg(ctx, curSysMsgID)
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "get new sysMsg fail"))
+		return
+	}
+	if len(list) == 0 {
+		log.Infov(ctx, log.KV("log", "no new sys msg to be sync to notice"))
+		return
+	}
+
+	var mids []int64
+	for _, item := range list {
+		curSysMsgID = item.Id
+
+		notice := notice.NoticeBase{
+			Mid:        item.Receiver,
+			ActionMid:  item.Sender,
+			SvId:       0,
+			NoticeType: 4,
+			Text:       item.Text,
+			JumpUrl:    item.JumpUrl,
+			BizType:    dao.NoticeBizTypeSysMsg,
+			BizId:      item.Id,
+		}
+		// 全量系统消息
+		if item.Receiver == 0 {
+			lastUserID := int64(0)
+			if len(mids) == 0 {
+				for {
+					if userBases, err := s.dao.UsersByLast(ctx, lastUserID); err != nil {
+						log.Errorv(ctx, log.KV("log", "sys msg task: get user base fail"))
+						break
+					} else {
+						for _, userBase := range userBases {
+							mids = append(mids, userBase.MID)
+						}
+						if len(userBases) > 0 {
+							lastUserID = userBases[len(userBases)-1].ID
+						} else {
+							break
+						}
+					}
+
+				}
+			}
+			midChan := make(chan int64, 20)
+			go func() {
+				for _, mid := range mids {
+					midChan <- mid
+				}
+				close(midChan)
+			}()
+			startTime := time.Now()
+			g := errgroup.Group{}
+			for i := 0; i < 10; i++ {
+				g.Go(func() error {
+					subNotice := notice
+					for mid := range midChan {
+						subNotice.Mid = mid
+						s.dao.CreateNotice(ctx, &subNotice)
+					}
+					return nil
+				})
+			}
+			g.Wait()
+			log.Info("total sys msg notice push: cost_time=%f, mid_len=%d", time.Since(startTime).Seconds(), len(mids))
+		} else {
+			s.dao.CreateNotice(ctx, &notice)
+		}
+	}
+
+	if _, err := s.dao.UpdateTaskLastCheck(ctx, "checkSysMsg", curSysMsgID); err != nil {
+		log.Errorv(ctx, log.KV("log", "update check_task mysql fail"))
+		return
+	}
+
+	log.Infov(ctx, log.KV("log", "no new sys msg to be sync to notice"), log.KV("last_sys_id", lastSysMsgID), log.KV("cur_sys_id", curSysMsgID))
+}

app/job/bbq/video/service/service.go

@@ -0,0 +1,180 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"go-common/library/conf/env"
+	"go-common/library/queue/databus"
+	"reflect"
+	"time"
+
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/dao"
+	"go-common/library/log"
+
+	topic "go-common/app/service/bbq/topic/api"
+
+	"github.com/robfig/cron"
+)
+
+var (
+	srvName string
+)
+
+// Service struct
+type Service struct {
+	c           *conf.Config
+	dao         *dao.Dao
+	searchChan  chan string
+	videoSub    *databus.Databus
+	videoRep    *databus.Databus
+	bvcSub      *databus.Databus
+	scheFunc    map[string]func()
+	topicClient topic.TopicClient
+}
+
+func init() {
+	flag.StringVar(&srvName, "srv", "", "service name")
+}
+
+func newTopicClient() topic.TopicClient {
+	topicClient, err := topic.NewClient(nil)
+	if err != nil {
+		log.Errorw(context.Background(), "log", "get topic client fail")
+		panic(err)
+	}
+	return topicClient
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:           c,
+		dao:         dao.New(c),
+		searchChan:  make(chan string, 1),
+		topicClient: newTopicClient(),
+	}
+	s.scheFunc = s.initScheduleFunc()
+	if srvName != "" {
+		switch srvName {
+		case "test":
+			s.Test()
+		case "syncsv2es":
+			s.taskSyncVideo2ES()
+		case "syncuserdmg":
+			s.taskSyncUserDmg()
+		case "rminvalides":
+			s.taskRmInvalidES()
+		case "regcmtall":
+			s.AutoRegAll(context.Background())
+		case "syncuserbase":
+			s.taskSyncUsrBaseFromVideo(context.Background())
+		case "syncuserbasic":
+			s.taskSyncPegasusUserBasic()
+		case "syncsearchvideo":
+			s.SyncVideo2Search()
+		case "syncsearchuser":
+			s.SyncUser2Search()
+		case "syncsearchsug":
+			s.SyncSug2Search()
+		case "upubface":
+			s.UpdateUsrBaseFace()
+		case "SysMsgTask":
+			s.SysMsgTask()
+		case "UserProfile":
+			s.UserProfileUpdate()
+		case "pushbvc":
+			s.commitCID()
+		case "cmscheckback":
+			s.TransToCheckBack()
+		}
+		return s
+	}
+	//初始化databus
+	s.initDatabus()
+	//启动相关rountine
+	s.launchCor()
+	//定时任务启动
+	if env.DeployEnv == env.DeployEnvProd {
+		s.runScheduler(c.Scheduler)
+	}
+	return s
+}
+
+//runScheduler .1
+func (s *Service) runScheduler(c *conf.Scheduler) {
+	sche := cron.New()
+
+	t := reflect.TypeOf(*c)
+	v := reflect.ValueOf(*c)
+	for i := 0; i < v.NumField(); i++ {
+		//排除配置为空的任务
+		if job := v.Field(i).String(); job != "" {
+			fn := t.Field(i).Name
+			//从映射集中取出对应函数
+			if f, ok := s.scheFunc[fn]; !ok {
+				fmt.Printf("skip[%s]\n", fn)
+				continue
+			} else {
+				fmt.Printf("run[%s]\n", fn)
+				if err := sche.AddFunc(job, f); err != nil {
+					panic(err)
+				}
+			}
+		}
+	}
+	sche.Start()
+}
+
+//Test 测试
+func (s *Service) Test() {
+	log.Info("HeartBeat:%s", time.Now())
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}
+
+// initScheduleFunc 任务结构与函数映射关系
+// map {key: conf.Scheduler结构体中字段名 value: 对应执行的函数}
+func (s *Service) initScheduleFunc() map[string]func() {
+	return map[string]func(){
+		"Test":             s.Test,
+		"CheckVideo2ES":    s.taskSyncVideo2ES,
+		"SyncUserDmg":      s.taskSyncUserDmg,
+		"SyncUpUserDmg":    s.taskSyncUpUserDmg,
+		"CheckVideo":       s.taskCheckVideo,
+		"CheckVideoSt":     s.taskCheckVideoStatistics,
+		"CheckVideoStHv":   s.taskCheckVideoStatisticsHive,
+		"CheckVideoTag":    s.taskCheckVideoTag,
+		"CheckTag":         s.taskCheckTag,
+		"SyncUsrSta":       s.taskSyncUsrStaFromHive,
+		"SysMsgTask":       s.SysMsgTask,
+		"UserProfileBbq":   s.UserProfileUpdate,
+		"TransToReview":    s.TransToReview,
+		"TransToCheckBack": s.TransToCheckBack,
+	}
+}
+
+func (s *Service) launchCor() {
+	time.Sleep(time.Second * 3)
+	if env.DeployEnv == env.DeployEnvProd {
+		go s.SyncSearch()
+	}
+	go s.videoBinlogSub()
+	go s.videoRepositoryBinlogSub()
+	go s.BvcTransSub()
+}
+
+func (s *Service) initDatabus() {
+	s.videoSub = databus.New(conf.Conf.Databus["videosub"])
+	s.videoRep = databus.New(conf.Conf.Databus["videorep"])
+	s.bvcSub = databus.New(conf.Conf.Databus["bvcsub"])
+}

app/job/bbq/video/service/user.go

@@ -0,0 +1,237 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	video "go-common/app/service/bbq/video/api/grpc/v1"
+	"go-common/library/log"
+	"os"
+	"time"
+)
+
+//taskSyncUserDmg 同步用户画像
+func (s *Service) taskSyncUserDmg() {
+	jobURL, err := s.dao.QueryUserDmg(context.Background())
+	if err != nil {
+		log.Error("get user dmg err(%v)", err)
+		return
+	}
+	urls, err := s.dao.QueryJobStatus(context.Background(), jobURL)
+	if err != nil {
+		log.Error("get user dmg job result err(%v)", err)
+		return
+	}
+	for _, url := range urls {
+		go func(url string) {
+			fpath, err := s.dao.Download(url, "")
+			if err != nil {
+				return
+			}
+			s.dao.ReadLine(fpath, s.dao.HandlerUserDmg)
+			os.RemoveAll(fpath)
+		}(url)
+	}
+}
+
+//UserProfileUpdate bbq_user_profile
+func (s *Service) UserProfileUpdate() {
+	_, err := s.dao.UserProfileGet(context.Background())
+	if err != nil {
+		log.Error("get user dmg err(%v)", err)
+	}
+}
+
+//taskSyncUserDmg 同步天马推荐用户画像
+func (s *Service) taskSyncPegasusUserBasic() {
+	_, err := s.dao.QueryUserBasic(context.Background())
+	if err != nil {
+		log.Error("get user dmg err(%v)", err)
+	}
+}
+
+//taskSyncUpUserDmg 同步up主画像
+func (s *Service) taskSyncUpUserDmg() {
+	log.Infov(context.Background(), log.KV("event", "sync_up_user_dmg"))
+	var mid = int64(0)
+	for {
+		log.Infov(context.Background(), log.KV("event", "one_sync_up_user_dmg"), log.KV("mid", mid))
+		upUserDmgs, err := s.dao.QueryUpUserDmg(context.Background(), mid)
+		if err != nil {
+			log.Error("get up user dmg err(%v)", err)
+			return
+		}
+
+		if len(upUserDmgs) == 0 {
+			break
+		}
+
+		for _, upUserDmg := range upUserDmgs {
+			mid = upUserDmg.MID
+			fmt.Println(upUserDmg.MID)
+			if err = s.dao.InsertOnDup(context.Background(), upUserDmg); err != nil {
+				log.Error("user dmg insert on dup failed,mid :%v, err:%v", upUserDmg.MID, err)
+			}
+		}
+	}
+	// TODO: 考虑个好方法
+	// 万一对方接口有问题，那就都完了
+	//s.dao.DelUpUserDmg(context.Background())
+}
+
+// 同步up主画像 从hive更新到user_statistics_hive
+func (s *Service) taskSyncUsrStaFromHive() {
+	log.Infov(context.Background(), log.KV("event", "taskSyncUsrStaFromHive"))
+	var (
+		err    error
+		jobURL string
+		urls   []string
+		url    string
+		fpath  string
+		try    int
+		date   = time.Now().AddDate(0, 0, -1).Format("20060102")
+	)
+	for try = 1; try <= 3; try++ {
+		//发起hive查询,拿到url
+		if jobURL, err = s.dao.QueryUpMid(context.Background(), date); err != nil {
+			log.Warn("taskSyncUsrStaFromHive try and init query hive failed, err(%v)", err)
+			continue
+		}
+		log.Info("taskSyncUsrStaFromHive init query hive success")
+		//查询job状态
+		if urls, err = s.dao.QueryJobStatus(context.Background(), jobURL); err != nil {
+			log.Warn("taskSyncUsrStaFromHive try and get hive query status failed, err(%v)", err)
+			continue
+		}
+		break
+	}
+	if err != nil {
+		log.Error("taskSyncUsrStaFromHive init and get hive query status failed, err(%v)", err)
+		return
+	}
+	for _, url = range urls {
+		for try = 0; try <= 3; try++ {
+			if fpath, err = s.dao.Download(url, ""); err != nil {
+				log.Warn("taskSyncUsrInfoFromHive try and download file (%v) failed, err(%v)", url, err)
+				time.Sleep(time.Duration(try*10) * time.Second)
+				continue
+			}
+			s.dao.ReadLines(fpath, s.dao.HandlerMids)
+			os.RemoveAll(fpath)
+			return
+		}
+		if err != nil {
+			log.Error("taskSyncUsrInfoFromHive download file (%v) failed, err(%v)", url, err)
+		}
+	}
+}
+
+//从video表同步up画像
+//func (s *Service) taskSyncUsrBaseFromVideo(c context.Context) {
+//	log.Infov(context.Background(), log.KV("event", "taskSyncUsrInfoFromVideo"))
+//	//get mids
+//	var (
+//		mids *[]int64
+//		err  error
+//		i    int8
+//		mid  int64
+//		req  *video.SyncUserBaseResponse
+//	)
+//	for i := 0; i <= 3; i++ {
+//		mids, err = s.dao.SelMidFromVideo()
+//		if err != nil {
+//			log.Info("taskSyncUsrInfoFromVideo try and get up mid failed , err(%v)", err)
+//		} else {
+//			break
+//		}
+//	}
+//	if err != nil {
+//		log.Error("taskSyncUsrInfoFromVideo get up mid failed, err(%v)", err)
+//	}
+//	//get userinfo and update
+//	for _, mid = range *mids {
+//		//重试
+//		for i = 0; i <= 3; i++ {
+//			if req, err = s.dao.VideoClient.SyncUserBase(c, &video.SyncMidRequset{MID: mid}); err != nil || req.Affc == -1 {
+//				log.Info("taskSyncUsrInfoFromVideo try and failed, mid(%v), err(%v)", mid, err)
+//			} else {
+//				break
+//			}
+//		}
+//		if err != nil {
+//			log.Error("taskSyncUsrInfoFromVideo failed, mid(%v), err(%v)", mid, err)
+//		}
+//	}
+//}
+
+//从video表同步up画像
+func (s *Service) taskSyncUsrBaseFromVideo(c context.Context) {
+	fmt.Println("taskSyncUsrBaseFromVideo start")
+	log.Infov(context.Background(), log.KV("event", "taskSyncUsrInfoFromVideo"))
+	//get mids
+	var (
+		mids []int64
+		err  error
+		req  *video.SyncUserBaseResponse
+	)
+	for try := 0; try <= 3; try++ {
+		mids, err = s.dao.SelMidFromVideo()
+		if err != nil {
+			log.Info("taskSyncUsrInfoFromVideo try and get up mid failed , err(%v)", err)
+		} else {
+			break
+		}
+	}
+	if err != nil {
+		log.Error("taskSyncUsrInfoFromVideo get up mid failed, err(%v)", err)
+	}
+	i := len(mids) / 50
+	//get userinfo and update
+	for j := 1; j <= i; j++ {
+		for try := 0; try <= 3; try++ {
+			if req, err = s.dao.VideoClient.SyncUserBases(c, &video.SyncMidsRequset{MIDS: mids[(j-1)*50 : j*50]}); err != nil {
+				log.Info("taskSyncUsrInfoFromVideo try and failed, err(%v)", err)
+			} else {
+				break
+			}
+		}
+		if err != nil {
+			log.Error("taskSyncUsrInfoFromVideo failed, err(%v)", err)
+		} else {
+			log.Info("taskSyncUsrInfoFromVideo success ,affected %v rows", req.Affc)
+		}
+	}
+	if i*50 < len(mids) {
+		for try := 0; try <= 3; try++ {
+			if req, err = s.dao.VideoClient.SyncUserBases(c, &video.SyncMidsRequset{MIDS: mids[i*50:]}); err != nil {
+				log.Info("taskSyncUsrInfoFromVideo try and failed, err(%v)", err)
+			} else {
+				break
+			}
+		}
+		if err != nil {
+			log.Error("taskSyncUsrInfoFromVideo failed, err(%v)", err)
+		} else {
+			log.Info("taskSyncUsrInfoFromVideo success ,affected %v rows", req.Affc)
+		}
+	}
+}
+
+// UpdateUsrBaseFace 更新user_base里的face字段
+func (s *Service) UpdateUsrBaseFace() (err error) {
+	var (
+		mids []int64
+	)
+	log.Infov(context.Background(), log.KV("event", "UpdateUsrBaseFace"))
+	for i := 0; ; i++ {
+		mids, err = s.dao.SelMidFromUserBase(i * 1000)
+		if err != nil {
+			log.Error("UpdateUsrBaseFace select mid failed")
+			return
+		}
+		if len(mids) == 0 {
+			break
+		}
+		s.dao.UpUserBases(context.Background(), mids)
+	}
+	return
+}

app/job/bbq/video/service/video.go

@@ -0,0 +1,411 @@
+package service
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"go-common/app/job/bbq/video/dao"
+	searchv1 "go-common/app/service/bbq/search/api/grpc/v1"
+	"go-common/library/log"
+	"io"
+	"net/url"
+	"os"
+
+	"strconv"
+	"time"
+)
+
+const (
+	_retryTimes = 3
+
+	_selection = 5 //运营精选状态
+)
+
+//taskCheckVideoDBVSES 视频全量脚本
+func (s *Service) taskSyncVideo2ES() {
+	var step time.Duration
+	var id int64
+	for {
+		ids, videos, err := s.dao.VideoList(context.Background(), id)
+		if err != nil {
+			log.Error("sync video err(%v)", err)
+			return
+		}
+		if videos == nil {
+			return
+		}
+		videoStatisticsHive, _ := s.dao.VideoStatisticsHiveList(context.Background(), ids)
+		videoStatistics, _ := s.dao.VideoStatisticsList(context.Background(), ids)
+		videoTags, _ := s.dao.VideoTagsList(context.Background(), ids)
+		req := new(searchv1.SaveVideoRequest)
+		for _, v := range videos {
+			fmt.Println(v.SVID)
+			id = v.SVID
+			tmp := &searchv1.VideoESInfo{
+				SVID:         v.SVID,
+				Title:        v.Title,
+				Content:      v.Content,
+				MID:          v.MID,
+				CID:          v.CID,
+				Pubtime:      int64(v.Pubtime),
+				Ctime:        int64(v.Ctime),
+				Mtime:        int64(v.Mtime),
+				Duration:     v.Duration,
+				Original:     v.Original,
+				State:        v.State,
+				VerID:        v.VerID,
+				Ver:          v.Ver,
+				From:         v.From,
+				AVID:         v.AVID,
+				Tid:          v.Tid,
+				SubTid:       v.SubTid,
+				ISFullScreen: v.ISFullScreen,
+				Score:        v.Score,
+			}
+			if videoStatisticsHive[id] != nil {
+				tmp.PlayHive = videoStatisticsHive[id].PlayHive
+				tmp.FavHive = videoStatisticsHive[id].FavHive
+				tmp.CoinHive = videoStatisticsHive[id].CoinHive
+				tmp.SubtitlesHive = videoStatisticsHive[id].SubtitlesHive
+				tmp.LikesHive = videoStatisticsHive[id].LikesHive
+				tmp.ShareHive = videoStatisticsHive[id].ShareHive
+				tmp.ReportHive = videoStatisticsHive[id].ReportHive
+				tmp.DurationDailyHive = videoStatisticsHive[id].DurationDailyHive
+				tmp.DurationAllHive = videoStatisticsHive[id].DurationAllHive
+				tmp.ReplyHive = videoStatisticsHive[id].ReplyHive
+				tmp.ShareDailyHive = videoStatisticsHive[id].ShareDailyHive
+				tmp.PlayDailyHive = videoStatisticsHive[id].PlayDailyHive
+				tmp.SubtitlesDailyHive = videoStatisticsHive[id].SubtitlesDailyHive
+				tmp.LikesDailyHive = videoStatisticsHive[id].LikesDailyHive
+				tmp.FavDailyHive = videoStatisticsHive[id].FavDailyHive
+				tmp.ReplyDailyHive = videoStatisticsHive[id].ReplyDailyHive
+				tmp.AccessHive = videoStatisticsHive[id].AccessHive
+			}
+			if videoStatistics[id] != nil {
+				tmp.Play = videoStatistics[id].Play
+				tmp.Subtitles = videoStatistics[id].Subtitles
+				tmp.Like = videoStatistics[id].Like
+				tmp.Share = videoStatistics[id].Share
+				tmp.Report = videoStatistics[id].Report
+			}
+			if videoTags[id] != nil {
+				tmp.Tags = videoTags[id]
+			}
+			req.List = append(req.List, tmp)
+		}
+		step = 1
+		for {
+			if _, err := s.dao.SearchClient.SaveVideo(context.Background(), req); err != nil {
+				time.Sleep(step * time.Second)
+				step++
+				continue
+			}
+			break
+		}
+	}
+
+}
+
+//SaveVideo2ES 保存视频到es
+func (s *Service) SaveVideo2ES(ids string) (res bool) {
+	res = true
+	if len(ids) == 0 {
+		return
+	}
+	videos, err := s.dao.VideoListByIDs(context.Background(), ids)
+	if err != nil || videos == nil {
+		res = false
+		return
+	}
+	videoStatisticsHive, _ := s.dao.VideoStatisticsHiveList(context.Background(), ids)
+	videoStatistics, _ := s.dao.VideoStatisticsList(context.Background(), ids)
+	// videoTags, _ := s.dao.VideoTagsList(context.Background(), ids)
+	var step time.Duration
+	var id int64
+	req := new(searchv1.SaveVideoRequest)
+	for _, v := range videos {
+		id = v.SVID
+		fmt.Println(id)
+		tmp := &searchv1.VideoESInfo{
+			SVID:         v.SVID,
+			Title:        v.Title,
+			Content:      v.Content,
+			MID:          v.MID,
+			CID:          v.CID,
+			Pubtime:      int64(v.Pubtime),
+			Ctime:        int64(v.Ctime),
+			Mtime:        int64(v.Mtime),
+			Duration:     v.Duration,
+			Original:     v.Original,
+			State:        v.State,
+			VerID:        v.VerID,
+			Ver:          v.Ver,
+			From:         v.From,
+			AVID:         v.AVID,
+			Tid:          v.Tid,
+			SubTid:       v.SubTid,
+			ISFullScreen: v.ISFullScreen,
+			Score:        v.Score,
+		}
+		if videoStatisticsHive[id] != nil {
+			tmp.PlayHive = videoStatisticsHive[id].PlayHive
+			tmp.FavHive = videoStatisticsHive[id].FavHive
+			tmp.CoinHive = videoStatisticsHive[id].CoinHive
+			tmp.SubtitlesHive = videoStatisticsHive[id].SubtitlesHive
+			tmp.LikesHive = videoStatisticsHive[id].LikesHive
+			tmp.ShareHive = videoStatisticsHive[id].ShareHive
+			tmp.ReportHive = videoStatisticsHive[id].ReportHive
+			tmp.DurationDailyHive = videoStatisticsHive[id].DurationDailyHive
+			tmp.DurationAllHive = videoStatisticsHive[id].DurationAllHive
+			tmp.ReplyHive = videoStatisticsHive[id].ReplyHive
+			tmp.ShareDailyHive = videoStatisticsHive[id].ShareDailyHive
+			tmp.PlayDailyHive = videoStatisticsHive[id].PlayDailyHive
+			tmp.SubtitlesDailyHive = videoStatisticsHive[id].SubtitlesDailyHive
+			tmp.LikesDailyHive = videoStatisticsHive[id].LikesDailyHive
+			tmp.FavDailyHive = videoStatisticsHive[id].FavDailyHive
+			tmp.ReplyDailyHive = videoStatisticsHive[id].ReplyDailyHive
+			tmp.AccessHive = videoStatisticsHive[id].AccessHive
+		}
+		if videoStatistics[id] != nil {
+			tmp.Play = videoStatistics[id].Play
+			tmp.Subtitles = videoStatistics[id].Subtitles
+			tmp.Like = videoStatistics[id].Like
+			tmp.Share = videoStatistics[id].Share
+			tmp.Report = videoStatistics[id].Report
+		}
+		// if videoTags[id] != nil {
+		// 	tmp.Tags = videoTags[id]
+		// }
+		req.List = append(req.List, tmp)
+	}
+	step = 1
+	for {
+		if _, err := s.dao.SearchClient.SaveVideo(context.Background(), req); err != nil {
+			if step == 11 {
+				log.Error("save es err(%v) ids(%s)", err, ids)
+				res = false
+				break
+			}
+			time.Sleep(step * time.Second)
+			step++
+			continue
+		}
+		break
+	}
+	return
+}
+
+func formArrayString(arr []int64) string {
+	var res string
+	for i, v := range arr {
+		if i != 0 {
+			res += ","
+		}
+		res += strconv.FormatInt(v, 10)
+	}
+	return res
+}
+
+//deltaSync2ES 为不同表进行增量同步的脚本，baseTableQuery指明不同表的查询语句
+func (s *Service) deltaSync2ES(taskName string, baseTableQuery string) {
+	task, err := s.dao.RawCheckTask(context.Background(), taskName)
+	if err != nil {
+		log.Error("get last_chek_time fail: task=%s", taskName)
+		return
+	}
+	log.Info("get last_chek_time succ: task=%s, last_check_time=%d", taskName, task.LastCheck)
+
+	// 获得所有变更的svid
+	ids, mtime, err := s.dao.RawGetIDByMtime(baseTableQuery, task.LastCheck)
+	if err != nil {
+		log.Error("get raw id by mtime fail: task=%s, last_mtime=%d, base_table_query=%s",
+			taskName, mtime, baseTableQuery)
+		return
+	}
+	idsNum := len(ids)
+	log.Info("get changed svids: task=%s, id_num=%d", taskName, idsNum)
+	if idsNum == 0 {
+		return
+	}
+	task.LastCheck = mtime
+
+	// 对所有变更的svid分批次进行同步到es
+	for i := 0; i < idsNum; i += dao.MaxSyncESNum {
+		last := i + dao.MaxSyncESNum
+		if last > idsNum {
+			last = idsNum
+		}
+		selectedIDs := ids[i:last]
+		idsStr := formArrayString(selectedIDs)
+		if res := s.SaveVideo2ES(idsStr); !res {
+			log.Error("sync video 2 es fail: task=%s, offset=%d, id_num=%d, base_table_query=%s",
+				taskName, i, idsNum, baseTableQuery)
+			return
+		}
+		log.Info("one sync video 2 es: task=%s, offset=%d, id_num=%d", taskName, i, idsNum)
+	}
+
+	// 更新task最近check的时间点
+	if _, err := s.dao.UpdateTaskLastCheck(context.Background(), taskName, task.LastCheck); err != nil {
+		log.Error("update task last check time fail: task=%s, last_mtime=%d, base_table_query=%s",
+			taskName, task.LastCheck, baseTableQuery)
+		return
+	}
+	log.Info("sync video 2 es: task=%s, id_num=%d, last_mtime=%d", taskName, idsNum, task.LastCheck)
+}
+
+//taskCheckVideo video表增量脚本
+func (s *Service) taskCheckVideo() {
+	taskName := "checkVideo"
+	s.deltaSync2ES(taskName, dao.QueryVideoByMtime)
+}
+
+//taskCheckVideoStatistics video_statistics表增量脚本
+func (s *Service) taskCheckVideoStatistics() {
+	taskName := "checkVideoSt"
+	s.deltaSync2ES(taskName, dao.QueryVideoStatisticsByMtime)
+}
+
+//taskCheckVideoStatisticsHive video_statistics_hive表增量脚本
+func (s *Service) taskCheckVideoStatisticsHive() {
+	taskName := "checkVideoStHv"
+	s.deltaSync2ES(taskName, dao.QueryVideoStatisticsHiveByMtime)
+}
+
+//taskCheckVideoTag video_tag表增量脚本
+func (s *Service) taskCheckVideoTag() {
+	taskName := "checkVideoTag"
+	s.deltaSync2ES(taskName, dao.QueryVideoTagByMtime)
+}
+
+//taskCheckTag tag表增量脚本
+func (s *Service) taskCheckTag() {
+	taskName := "checkTag"
+	task, err := s.dao.RawCheckTask(context.Background(), taskName)
+	if err != nil {
+		log.Error("get last_chek_time fail: task=%s", taskName)
+		return
+	}
+	log.Info("get last_chek_time succ: task=%s, last_check_time=%d", taskName, task.LastCheck)
+	for {
+		ids, mtime, err := s.dao.RawTagByMtime(context.Background(), task.LastCheck)
+		if err != nil || len(ids) == 0 {
+			return
+		}
+		id := int64(0)
+		for {
+			svids, temp, err := s.dao.RawVideoTagByIDs(context.Background(), ids, id)
+			if err != nil {
+				return
+			}
+			if len(svids) == 0 {
+				break
+			}
+			if flag := s.SaveVideo2ES(svids); !flag {
+				return
+			}
+			id = temp
+		}
+		if num, err := s.dao.UpdateTaskLastCheck(context.Background(), taskName, mtime); err != nil || num == 0 {
+			return
+		}
+		task.LastCheck = mtime
+	}
+}
+
+// taskRmInvalidES 删除es中多余的视频
+func (s *Service) taskRmInvalidES() {
+	fmt.Println("aaa")
+	esReq := new(searchv1.ESVideoDataRequest)
+	delReq := new(searchv1.DelVideoBySVIDRequest)
+	svid := int64(0)
+	query := `{"query":{"range":{"svid":{"gt":%d}}},"sort":[{"svid":"asc"}],"from":0,"size":10}`
+	for {
+		esReq.Query = fmt.Sprintf(query, svid)
+		res, err := s.dao.SearchClient.ESVideoData(context.Background(), esReq)
+		if err != nil {
+			return
+		}
+		svids := make([]string, 0)
+		for _, v := range res.List {
+			svids = append(svids, strconv.Itoa(int(v.SVID)))
+			svid = v.SVID
+		}
+		vs, err := s.dao.RawVideoBySVIDS(context.Background(), svids)
+		if err != nil {
+			return
+		}
+		notList := make([]int64, 0)
+		for _, v := range res.List {
+			if _, ok := vs[v.SVID]; !ok {
+				fmt.Println(v.SVID)
+				notList = append(notList, v.SVID)
+			}
+		}
+		if len(notList) != 0 {
+			delReq.SVIDs = notList
+			s.dao.SearchClient.DelVideoBySVID(context.Background(), delReq)
+		}
+	}
+}
+
+func (s *Service) commitCID() {
+	ctx := context.Background()
+	path := s.c.URLs["bvc_push"]
+	if path == "" {
+		return
+	}
+	srcPath := s.c.Path["cids"]
+	if srcPath == "" {
+		return
+	}
+	if srcPath == "" {
+		log.Error("sugsrc path is empty")
+		return
+	}
+	src, err := os.Open(srcPath)
+	if err != nil {
+		log.Error("writeSug os.Open source sug error(%v)", err)
+		return
+	}
+	defer src.Close()
+	br := bufio.NewReader(src)
+	i := 1
+	for {
+		a, _, c := br.ReadLine()
+		if c == io.EOF {
+			break
+		}
+		cid, err := strconv.ParseInt(string(a), 10, 64)
+		if err != nil {
+			log.Error("parse err [%v]", err)
+			continue
+		}
+		svid, err := s.dao.GetSvidByCid(ctx, cid)
+		if err != nil {
+			continue
+		}
+		params := url.Values{}
+		params.Set("svid", strconv.FormatInt(svid, 10))
+		params.Set("cid", string(a))
+		req, err := s.dao.HTTPClient.NewRequest("GET", path, "", params)
+		if err != nil {
+			log.Error("error(%v)", err)
+			continue
+		}
+		var res struct {
+			Code int    `json:"code"`
+			Msg  string `json:"message"`
+		}
+		if err = s.dao.HTTPClient.Do(ctx, req, &res); err != nil {
+			log.Errorv(ctx, log.KV("log", fmt.Sprintf("err[%v]", err)))
+			continue
+		}
+		if res.Code != 0 {
+			log.Errorv(ctx, log.KV("log", fmt.Sprintf("error(%v)", err)))
+		} else {
+			log.Info("commit svid:%d cid:%d success No.%d", svid, cid, i)
+		}
+		i++
+	}
+}