Create & Init Project...

app/job/bbq/video/dao/BUILD

@@ -0,0 +1,58 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "cmsvideo.go",
+        "comment.go",
+        "dao.go",
+        "email.go",
+        "notice.go",
+        "user.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/video/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/model:go_default_library",
+        "//app/service/bbq/notice-service/api/v1:go_default_library",
+        "//app/service/bbq/search/api/grpc/v1:go_default_library",
+        "//app/service/bbq/sys-msg/api/v1:go_default_library",
+        "//app/service/bbq/video/api/grpc/v1:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/dao/berserker.go

@@ -0,0 +1,454 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/model"
+	"go-common/library/conf/env"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	xhttp "net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"io/ioutil"
+
+	pkgerr "github.com/pkg/errors"
+)
+
+const (
+	_jobStatusSuccess = 1
+	_jobStatusFailed  = 2
+	_jobStatusDoing   = 3
+	_jobStatusWaiting = 4
+	//_httpHeaderUser     = "x1-bilispy-user"
+	//_httpHeaderColor    = "x1-bilispy-color"
+	//_httpHeaderTimeout  = "x1-bilispy-timeout"
+	_httpHeaderRemoteIP = "x-backend-bili-real-ip"
+	_userAgent          = "User-Agent"
+	_noKickUserAgent    = "yangyucheng@bilibili.com"
+	_queryJSON          = `{"select":[],"where":{"log_date":{"in":["%s"]}},"page":{"limit":1000},"sort":{"play":-1}}`
+	_queryJSONOper      = `{"select":[],"where":{"log_date":{"in":["%s"]},"cid":{"gt":%d}},"page":{"limit":5000},"sort":{"cid":1}}`
+	_hscUserAgent       = "huangshancheng@bilibili.com"
+	_lzqUserAgent       = "liuzhiquan@bilibili.com"
+	_chmUserAgent       = "caiheming@bilibili.com"
+	_ljUserAgent        = "liujin@bilibili.com"
+	//_userDmgQueryJSON   = `{"select":[],"where":{"log_date":{"in":["%s"]},"mid":{"gt":"%s"}},"sort":{"mid":1},"page":{"limit":200}}`
+	_upUserDmgQueryJSON = `{"select":[],"where":{"mid":{"gt":%d}},"sort":{"mid":1},"page":{"limit":200}}`
+	_userDmgQueryHive   = `select mid, gender, age, geo, content_tag, viewed_video, content_zone, content_count, follow_ups from sycpb.hbase_dmp_tag where last_active_date >= %s and length(viewed_video) > 0`
+	_upMidQueryHive     = `select mid from ods.ods_member_relation_stat where log_date = %s  and follower>= 10000 limit 100`
+	//_upMidQueryHive = `{"select":["name":"mid"],"where":{"log_date":{"in":["%s"]},"follower":{"gte":10000}, "pages":{"limit":10}}`
+	_basePathUserProfile      = "/tmp/"
+	_basePathUserProfileBuvid = "/data/"
+)
+
+var (
+	signParams = []string{"appKey", "timestamp", "version"}
+)
+
+// QueryPlayDaily get video play rank list from berserker
+func (d *Dao) QueryPlayDaily(c context.Context, date string) (vlist []*model.VideoHiveInfo, err error) {
+	v := make(url.Values, 8)
+	query := fmt.Sprintf(_queryJSON, date)
+	v.Set("query", query)
+	var res struct {
+		Code   int                   `json:"code"`
+		Result []model.VideoHiveInfo `json:"result"`
+	}
+
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Rankdaily, "", v, d.c.Berserker.Key.YYC, _noKickUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+	if res.Code != 200 || len(res.Result) == 0 {
+		err = ecode.NothingFound
+		log.Warn("Berserker return err, url:%s;res:%d", d.c.Berserker.API.Rankdaily+"?"+v.Encode(), res.Code)
+		return
+	}
+	for _, info := range res.Result {
+		i := info
+		vlist = append(vlist, &i)
+	}
+	return
+}
+
+//QueryOperaVideo query operation video once
+func (d *Dao) QueryOperaVideo(c context.Context, date string, ch chan<- *model.VideoHiveInfo) (err error) {
+	i := int64(0)
+	var mid int64
+	for {
+		v := make(url.Values, 8)
+		var res struct {
+			Code   int                   `json:"code"`
+			Result []model.VideoHiveInfo `json:"result"`
+		}
+		query := fmt.Sprintf(_queryJSONOper, date, i)
+		v.Set("query", query)
+		if err = d.doHTTPGet(c, d.c.Berserker.API.Operaonce, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res); err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+			return
+		}
+		if res.Code == 200 && len(res.Result) == 0 {
+			return
+		}
+		if res.Code != 200 {
+			err = ecode.NothingFound
+			log.Warn("Berserker return err, url:%s;res:%d", d.c.Berserker.API.Operaonce+"?"+v.Encode(), res.Code)
+			return
+		}
+
+		for _, info := range res.Result {
+			ch <- &info
+			mid = info.CID
+		}
+		i = mid
+	}
+}
+
+//QueryUserBasic ...
+func (d *Dao) QueryUserBasic(c context.Context) (jobURL string, err error) {
+
+	v := make(url.Values, 8)
+	var res struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+	query := "{}"
+	v.Set("query", query)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Userbasic, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+	for i, file := range res.Result {
+		query = fmt.Sprintf("{\"fileSuffix\": \"%s\"}", file)
+		v.Set("query", query)
+		bs, err := d.doHTTPGetRaw(c, d.c.Berserker.API.Userbasic, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res)
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf("/data/basic_profile/part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+		}
+
+	}
+
+	return
+}
+
+//UserProfileGet ...
+func (d *Dao) UserProfileGet(c context.Context) (jobURL []string, err error) {
+	//
+	v := make(url.Values, 8)
+	var res struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+	query := "{}"
+	v.Set("query", query)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.UserProfile, "", v, d.c.Berserker.Key.HM, _chmUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+
+	for i, file := range res.Result {
+		query = fmt.Sprintf("{\"fileSuffix\": \"/%s\"}", file)
+		//fmt.Printf("query: %v\n", query)
+		v.Set("query", query)
+
+		time.Sleep(3 * time.Second)
+		var bs []byte
+		bs, err = d.doHTTPGetRaw(c, d.c.Berserker.API.UserProfile, "", v, d.c.Berserker.Key.HM, _chmUserAgent, &res)
+
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf(_basePathUserProfile+"part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+			d.ReadLine(fmt.Sprintf(_basePathUserProfile+"part_%d", i), d.HandlerUserBbqDmg)
+			os.RemoveAll(fmt.Sprintf(_basePathUserProfile+"part_%d", i))
+		}
+
+	}
+
+	time.Sleep(3 * time.Second)
+	v2 := make(url.Values, 8)
+	var res2 struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+
+	query2 := "{}"
+	v2.Set("query2", query2)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.UserProfileBuvid, "", v2, d.c.Berserker.Key.HM, _chmUserAgent, &res2); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+
+	for i, file := range res2.Result {
+		query2 = fmt.Sprintf("{\"fileSuffix\": \"/%s\"}", file)
+		//fmt.Printf("query: %v\n", query)
+		v2.Set("query", query2)
+
+		time.Sleep(3 * time.Second)
+		bs, err := d.doHTTPGetRaw(c, d.c.Berserker.API.UserProfileBuvid, "", v2, d.c.Berserker.Key.HM, _chmUserAgent, &res2)
+
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+			d.ReadLine(fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i), d.HandlerUserBbqDmgBuvid)
+			os.RemoveAll(fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i))
+		}
+
+	}
+	return
+}
+
+// doHttpRequest make a http request for data platform api
+func (d *Dao) doHTTPGet(c context.Context, uri, realIP string, params url.Values, key *conf.BerSerkerKey, userAgent string, res interface{}) (err error) {
+
+	enc, err := d.berserkeSign(params, key)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+
+	fmt.Printf("Req: %s ", req.URL)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+
+	req.Header.Set(_userAgent, userAgent+" "+env.AppID)
+	if err != nil {
+		return
+	}
+
+	if realIP != "" {
+		req.Header.Set(_httpHeaderRemoteIP, realIP)
+	}
+
+	return d.HTTPClient.Do(c, req, res)
+}
+
+// doHTTPGetRaw make a http request for data platform api
+func (d *Dao) doHTTPGetRaw(c context.Context, uri, realIP string, params url.Values, key *conf.BerSerkerKey, userAgent string, res interface{}) (bs []byte, err error) {
+	enc, err := d.berserkeSign(params, key)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+
+	req.Header.Set(_userAgent, userAgent+" "+env.AppID)
+	if err != nil {
+		return
+	}
+
+	if realIP != "" {
+		req.Header.Set(_httpHeaderRemoteIP, realIP)
+	}
+
+	return d.HTTPClient.Raw(c, req)
+}
+
+// Sign calc appkey and appsecret sign.
+func (d *Dao) berserkeSign(params url.Values, key *conf.BerSerkerKey) (query string, err error) {
+	params.Set("appKey", key.Appkey)
+	params.Set("signMethod", "md5")
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	tmp := params.Encode()
+	signTmp := d.encode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(key.Secret)
+	b.WriteString(signTmp)
+	b.WriteString(key.Secret)
+	mh := md5.Sum(b.Bytes())
+	// query
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+// Encode encodes the values into ``URL encoded'' form
+// ("bar=baz&foo=quux") sorted by key.
+func (d *Dao) encode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range signParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}
+
+// QueryUserDmg .
+func (d *Dao) QueryUserDmg(c context.Context) (jobURL string, err error) {
+	logDay := time.Now().AddDate(0, 0, -1).Format("20060102")
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_userDmgQueryHive, logDay))
+	var res struct {
+		Code         int    `json:"code"`
+		Msg          string `json:"msg"`
+		JobStatusURL string `json:"jobStatusUrl"`
+	}
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Userdmg, "", params, d.c.Berserker.Key.HSC, _hscUserAgent, &res); err != nil {
+		return
+	}
+	if res.Code != 200 {
+		log.Error("Berserker user_dmg err(%v)", err)
+		return
+	}
+	jobURL = res.JobStatusURL
+	return
+}
+
+// QueryJobStatus 查询hive脚本执行结果
+func (d *Dao) QueryJobStatus(c context.Context, jobURL string) (urls []string, err error) {
+	var res struct {
+		Code      int      `json:"code"`
+		Msg       string   `json:"msg"`
+		StatusID  int      `json:"statusId"`
+		StatusMsg string   `json:"statusMsg"`
+		HdfsPath  []string `json:"hdfsPath"`
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, jobURL, nil)
+	if err != nil {
+		log.Error("QueryJobStatus NewRequest, err(%v)", err)
+		return
+	}
+	for {
+		if err = d.HTTPClient.Do(c, req, &res); err != nil {
+			log.Error("QueryJobStatus do get failed, joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.Code != 200 {
+			log.Error("QueryJobStatus http code error, joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.StatusID == _jobStatusDoing || res.StatusID == _jobStatusWaiting {
+			//等待1min
+			log.Info("QueryJobStatus got job status %v, joburl(%v)", res.StatusID, jobURL)
+			time.Sleep(60 * time.Second)
+			continue
+		}
+		if res.StatusID == _jobStatusFailed {
+			log.Error("QueryJobStatus got job status failed joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.StatusID == _jobStatusSuccess {
+			log.Info("QueryJobStatus got job status success joburl(%v), err(%v)", jobURL, err)
+			urls = res.HdfsPath
+			return
+		}
+		if res.StatusID != _jobStatusSuccess && res.StatusID != _jobStatusFailed && res.StatusID != _jobStatusDoing && res.StatusID != _jobStatusWaiting {
+			log.Error("QueryJobStatus got wrong job status status(%v), joburl(%v)", res.StatusID, jobURL)
+			return
+		}
+	}
+}
+
+//QueryUpUserDmg .
+func (d *Dao) QueryUpUserDmg(c context.Context, mid int64) (upUserDmg []*model.UpUserDmg, err error) {
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_upUserDmgQueryJSON, mid))
+	var res struct {
+		Code   int                `json:"code"`
+		Result []*model.UpUserDmg `json:"result"`
+	}
+
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Upuserdmg, "", params, d.c.Berserker.Key.HSC, _hscUserAgent, &res); err != nil {
+		return
+	}
+	if res.Code != 200 {
+		log.Error("Berserker up_user_dmg err(%v)", err)
+		return
+	}
+	upUserDmg = res.Result
+	return
+}
+
+//QueryUpMid .发起hive查询，取粉丝数大于1万的up mid
+func (d *Dao) QueryUpMid(c context.Context, date string) (jobURL string, err error) {
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_upMidQueryHive, date))
+	var res struct {
+		Code         int    `json:"code"`
+		Msg          string `json:"msg"`
+		JobStatusURL string `json:"jobStatusUrl"`
+	}
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Upmid, "", params, d.c.Berserker.Key.LJ, _ljUserAgent, &res); err != nil {
+		log.Error("hive QueryUpMid failed, err(%v)", err)
+		return
+	}
+	if res.Code != 200 {
+		fmt.Println(res.Code)
+		log.Error("hive QueryUpMid failed, err(%v), httpcode(%v)", err, res.Code)
+		return
+	}
+	jobURL = res.JobStatusURL
+	fmt.Println(jobURL)
+	return
+}

app/job/bbq/video/dao/cmsvideo.go

@@ -0,0 +1,175 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/job/bbq/video/model"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_updatevideostatus    = "update video set state = ? where svid = ?"
+	_updateCmsvideoStatus = "insert into cms_video (svid,cms_status,sv_status,`from`,title,pubtime,mid) values (?,?,?,?,?,?,?) on duplicate key update cms_status = values(cms_status),sv_status=values(sv_status),cms_uname = '',`from` = values(`from`),title=values(title),pubtime=values(pubtime),mid = values(mid)"
+	_selectVideoRows      = "select id,svid,title,mid,`from`,pubtime from video where state = ? and id > ? limit 1000"
+	_updateVR             = "update video_repository set state = ? where svid=?"
+)
+
+//UpdateCms ..
+func (d *Dao) UpdateCms(c context.Context, v *model.VideoRaw) (err error) {
+	if _, err = d.dbCms.Exec(c,
+		_updateCmsvideoStatus,
+		v.SVID,
+		v.State,
+		v.State,
+		v.From,
+		v.Title,
+		v.Pubtime,
+		v.MID,
+	); err != nil {
+		log.Error("DeliveryNewVdieoToCms insert cms_video err,svid : %v,err :%v", v.SVID, err)
+		return
+	}
+	return
+}
+
+//TransToCheckBack ..
+func (d *Dao) TransToCheckBack() (err error) {
+	var (
+		rows  *xsql.Rows
+		count int64
+		id    int64
+		c     = context.Background()
+	)
+	for {
+		if rows, err = d.db.Query(c, _selectVideoRows, model.VideoStPassReview, count); err != nil {
+			log.Error("DeliveryNewVdieoToCms select video failed ,err:%v", err)
+			return
+		}
+		flag := false
+		for rows.Next() {
+			videoinfo := model.VideoInfo{}
+			if err = rows.Scan(
+				&id,
+				&videoinfo.SVID,
+				&videoinfo.Title,
+				&videoinfo.MID,
+				&videoinfo.From,
+				&videoinfo.Pubtime,
+			); err != nil {
+				if err == xsql.ErrNoRows {
+					return
+				}
+				continue
+			}
+			count = id
+			//满足运营导入规则
+			if d.CmsRule(videoinfo.SVID) {
+				if _, err = d.dbCms.Exec(c,
+					_updateCmsvideoStatus,
+					videoinfo.SVID,
+					model.VideoStCheckBack,
+					model.VideoStCheckBack,
+					videoinfo.From,
+					videoinfo.Title,
+					videoinfo.Pubtime,
+					videoinfo.MID,
+				); err != nil {
+					log.Error("DeliveryNewVdieoToCms insert cms_video err,svid : %v,err :%v", videoinfo.SVID, err)
+					continue
+				}
+				if _, err = d.db.Exec(c,
+					_updatevideostatus,
+					model.VideoStCheckBack,
+					videoinfo.SVID,
+				); err != nil {
+					log.Error("DeliveryNewVdieoToCms update video status err : %v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+				if _, err = d.dbCms.Exec(c, _updateVR, model.VideoStCheckBack, videoinfo.SVID); err != nil {
+					log.Error("DeliveryNewVdieoToCms update vr err :%v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+			}
+			flag = true
+		}
+		rows.Close()
+		if !flag {
+			return
+		}
+	}
+}
+
+// CmsRule ...
+func (d *Dao) CmsRule(svid int64) (flag bool) {
+	return true
+}
+
+//TransToReview ...
+func (d *Dao) TransToReview() (err error) {
+	var (
+		rows  *xsql.Rows
+		count int64
+		id    int64
+		c     = context.Background()
+	)
+	for {
+		if rows, err = d.db.Query(c, _selectVideoRows, model.VideoStPendingPassReview, count); err != nil {
+			log.Error("TransToReview select video failed ,err:%v", err)
+			continue
+		}
+		flag := false
+		for rows.Next() {
+			videoinfo := model.VideoInfo{}
+			if err = rows.Scan(
+				&id,
+				&videoinfo.SVID,
+				&videoinfo.Title,
+				&videoinfo.MID,
+				&videoinfo.From,
+				&videoinfo.Pubtime,
+			); err != nil {
+				if err == xsql.ErrNoRows {
+					return
+				}
+				continue
+			}
+			count = id
+			//满足运营导入规则
+			if d.CmsRule(videoinfo.SVID) {
+				var st int
+				if videoinfo.From == model.VideoFromBILI || videoinfo.From == model.VideoFromCMS {
+					st = model.VideoStPassReview
+				} else {
+					st = model.VideoStPassReviewReject
+				}
+
+				if _, err = d.dbCms.Exec(c,
+					_updateCmsvideoStatus,
+					videoinfo.SVID,
+					st,
+					st,
+					videoinfo.From,
+					videoinfo.Title,
+					videoinfo.Pubtime,
+					videoinfo.MID,
+				); err != nil {
+					log.Error("TransToReview insert cms_video err,svid : %v,err :%v", videoinfo.SVID, err)
+					continue
+				}
+				if _, err = d.db.Exec(c,
+					_updatevideostatus,
+					st,
+					videoinfo.SVID,
+				); err != nil {
+					log.Error("TransToReview update video status err : %v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+			}
+			flag = true
+		}
+		rows.Close()
+		if !flag {
+			return
+		}
+	}
+}

app/job/bbq/video/dao/comment.go

@@ -0,0 +1,13 @@
+package dao
+
+import (
+	"context"
+	"go-common/library/net/metadata"
+)
+
+//ReplyReg 评论注册/冻结
+func (d *Dao) ReplyReg(c context.Context, req map[string]interface{}) (err error) {
+	ip := metadata.String(c, metadata.RemoteIP)
+	_, err = replyHTTPCommon(c, d.HTTPClient, d.c.URLs["reply_reg"], "POST", req, ip)
+	return
+}

app/job/bbq/video/dao/dao.go

@@ -0,0 +1,157 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	notice "go-common/app/service/bbq/notice-service/api/v1"
+	searchv1 "go-common/app/service/bbq/search/api/grpc/v1"
+	videov1 "go-common/app/service/bbq/video/api/grpc/v1"
+	account "go-common/app/service/main/account/api"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc/warden"
+	"net/url"
+	"reflect"
+	"strconv"
+
+	jsoniter "github.com/json-iterator/go"
+	gomail "gopkg.in/gomail.v2"
+)
+
+// Dao dao
+type Dao struct {
+	c             *conf.Config
+	redis         *redis.Pool
+	bfredis       *redis.Pool
+	db            *xsql.DB
+	dbCms         *xsql.DB
+	HTTPClient    *bm.Client
+	SearchClient  searchv1.SearchClient
+	VideoClient   videov1.VideoClient
+	AccountClient account.AccountClient
+	email         *gomail.Dialer
+	noticeClient  notice.NoticeClient
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:             c,
+		redis:         redis.NewPool(c.Redis),
+		bfredis:       redis.NewPool(c.BfRedis),
+		db:            xsql.NewMySQL(c.MySQL),
+		dbCms:         xsql.NewMySQL(c.MySQLCms),
+		HTTPClient:    bm.NewClient(c.BM.Client),
+		SearchClient:  newSearchClient(c.GRPCClient["search"]),
+		VideoClient:   newVideoClient(c.GRPCClient["video"]),
+		AccountClient: newAccountClient(c.GRPCClient["account"]),
+		email:         gomail.NewDialer(c.Mail.Host, c.Mail.Port, c.Mail.From, c.Mail.Password),
+		noticeClient:  newNoticeClient(c.GRPCClient["notice"]),
+	}
+	return
+}
+
+// newNoticeClient .
+func newNoticeClient(cfg *conf.GRPCConf) notice.NoticeClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return notice.NewNoticeClient(cc)
+}
+
+//newSearchClient .
+func newSearchClient(cfg *conf.GRPCConf) searchv1.SearchClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return searchv1.NewSearchClient(cc)
+}
+
+//newAccountClient .
+func newAccountClient(cfg *conf.GRPCConf) account.AccountClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return account.NewAccountClient(cc)
+}
+
+//newVideoClient
+func newVideoClient(cfg *conf.GRPCConf) videov1.VideoClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return videov1.NewVideoClient(cc)
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return d.db.Ping(c)
+}
+
+// BeginTran begin mysql transaction
+func (d *Dao) BeginTran(c context.Context) (*xsql.Tx, error) {
+	return d.db.Begin(c)
+}
+
+// ReplyHTTPCommon 评论公用请求
+func replyHTTPCommon(c context.Context, httpClient *bm.Client, path string, method string, data map[string]interface{}, ip string) (r []byte, err error) {
+
+	params := url.Values{}
+	t := reflect.TypeOf(data).Kind()
+	if t == reflect.Map {
+		for k, v := range data {
+			// params.Set(k, v.(string))
+			switch reflect.TypeOf(v).Kind() {
+			case reflect.Int64:
+				params.Set(k, strconv.FormatInt(v.(int64), 10))
+			case reflect.Int16:
+				params.Set(k, strconv.FormatInt(int64(v.(int16)), 10))
+			case reflect.String:
+				params.Set(k, v.(string))
+			case reflect.Int:
+				params.Set(k, strconv.FormatInt(int64(v.(int)), 10))
+			}
+		}
+	}
+	log.V(5).Infov(c, log.KV("log", fmt.Sprintf("reply req url(%s)", path+"?"+params.Encode())))
+	req, err := httpClient.NewRequest(method, path, ip, params)
+	if err != nil {
+		log.Errorv(c, log.KV("log", fmt.Sprintf("reply url(%s) error(%v)", path+"?"+params.Encode(), err)))
+		return
+	}
+	var res struct {
+		Code int             `json:"code"`
+		Msg  string          `json:"message"`
+		Data json.RawMessage `json:"data"`
+	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = httpClient.Do(c, req, &res); err != nil {
+		str, _ := json.Marshal(res)
+		log.Errorv(c, log.KV("log", fmt.Sprintf("reply ret data(%s) err[%v]", str, err)))
+		return
+	}
+	str, _ := json.Marshal(res)
+	log.V(5).Infov(c, log.KV("log", fmt.Sprintf("reply ret data(%s)", str)))
+	if res.Code != 0 {
+		err = ecode.Int(res.Code)
+		log.Warnv(c, log.KV("log", fmt.Sprintf("reply url(%s) error(%v)", path+"?"+params.Encode(), err)))
+	}
+	r = res.Data
+	return
+}

app/job/bbq/video/dao/email.go

@@ -0,0 +1,12 @@
+package dao
+
+import (
+	gomail "gopkg.in/gomail.v2"
+)
+
+// SendMail asynchronous send mail.
+func (d *Dao) SendMail(message *gomail.Message) (err error) {
+	message.SetAddressHeader("From", d.c.Mail.From, "bbq")
+	err = d.email.DialAndSend(message)
+	return
+}

app/job/bbq/video/dao/notice.go

@@ -0,0 +1,61 @@
+package dao
+
+import (
+	"context"
+	notice "go-common/app/service/bbq/notice-service/api/v1"
+	msg "go-common/app/service/bbq/sys-msg/api/v1"
+	"go-common/library/log"
+)
+
+// 通知的业务类型
+const (
+	NoticeBizTypeSv      = 1
+	NoticeBizTypeComment = 2
+	NoticeBizTypeUser    = 3
+	NoticeBizTypeSysMsg  = 4
+)
+
+// 通知类型
+const (
+	NoticeTypeLike    = 1
+	NoticeTypeComment = 2
+	NoticeTypeFan     = 3
+	NoticeTypeSysMsg  = 4
+)
+
+const (
+	_selectSQL = "select id, type, sender, receiver, jump_url, text, ctime, state from sys_msg where id > ? order by id asc"
+)
+
+// CreateNotice 创建通知
+func (d *Dao) CreateNotice(ctx context.Context, notice *notice.NoticeBase) (err error) {
+	_, err = d.noticeClient.CreateNotice(ctx, notice)
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "create notice fail: notice="+notice.String()))
+		return
+	}
+
+	log.V(10).Infov(ctx, log.KV("log", "create notice: notice="+notice.String()))
+	return
+}
+
+// GetNewSysMsg 获取未被推送的系统消息
+func (d *Dao) GetNewSysMsg(ctx context.Context, id int64) (list []*msg.SysMsg, err error) {
+
+	rows, err := d.db.Query(ctx, _selectSQL, id)
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "query mysql sys msg fail"))
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var msg msg.SysMsg
+		if err = rows.Scan(&msg.Id, &msg.Type, &msg.Sender, &msg.Receiver, &msg.JumpUrl, &msg.Text, &msg.Ctime, &msg.State); err != nil {
+			log.Errorv(ctx, log.KV("log", "scan mysql sys msg fail"))
+			return
+		}
+		list = append(list, &msg)
+	}
+	return
+}

app/job/bbq/video/dao/user.go

@@ -0,0 +1,543 @@
+package dao
+
+import (
+	"bufio"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/model"
+	video "go-common/app/service/bbq/video/api/grpc/v1"
+	acc "go-common/app/service/main/account/api"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	_userDmgCacheKey      = "bbq:user:profile:%s"
+	_userDmgCacheKeyBuvid = "bbq:device:profile:{buvid}:%s"
+
+	//_userDmgCacheKeyBbq="bbq:user:profile"
+	_userDmgCacheTimeout = 86400
+	_incrUpUserDmgSQL    = "insert into user_statistics_hive (`mid`,`uname`,`play_total`,`fan_total`,`av_total`,`like_total`) value (?,?,?,?,?,?)"
+	_updateUpUserDmgSQL  = "update user_statistics_hive set `uname` = ? , `play_total` = ? , `fan_total` = ? , `av_total` = ? , `like_total` = ? , `mtime` = ? where `mid` = ?"
+	_delUpUserDmgSQL     = "delete from user_statistics_hive where mtime < ?"
+	_insertOnDupUpDmg    = "insert into user_statistics_hive (`mid`,`uname`,`play_total`,`fan_total`,`av_total`,`like_total`) value (?,?,?,?,?,?) on duplicate key update `uname`=?,`play_total`=?,`fan_total`=?,`av_total`=?,`like_total`=?"
+	_selMidFromVideo     = "select distinct mid from video"
+	_queryUsersByLast    = "select id,mid,uname from user_base where id > ? order by id ASC limit ?"
+	_selMidFromUserBase  = "SELECT DISTINCT mid fROM user_base limit ?, 1000"
+	_upUserBase          = "UPDATE user_base SET face = ? where mid = ?"
+)
+
+//getUserDmgKey .
+func getUserDmgKey(mid string) (key string) {
+	return fmt.Sprintf(_userDmgCacheKey, mid)
+}
+
+//getUserBuvidDmgKey .
+func getUserBuvidDmgKey(buvid string) (key string) {
+	return fmt.Sprintf(_userDmgCacheKeyBuvid, buvid)
+}
+
+// InsertOnDup ...
+func (d *Dao) InsertOnDup(c context.Context, upUserDmg *model.UpUserDmg) (err error) {
+	_, err = d.db.Exec(c, _insertOnDupUpDmg, upUserDmg.MID, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes)
+	return
+}
+
+//CacheUserDmg ...
+func (d *Dao) CacheUserDmg(c context.Context, userDmg *model.UserDmg) (mid string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	var b []byte
+	if b, err = json.Marshal(userDmg); err != nil {
+		log.Error("cache user dmg marshal err(%v)", err)
+		return
+	}
+	cacheKey := getUserDmgKey(userDmg.MID)
+	fmt.Println(cacheKey)
+	if _, err = conn.Do("SET", cacheKey, b, "EX", _userDmgCacheTimeout); err != nil {
+		log.Error("cache user dmg redis set err(%v)", err)
+		return
+	}
+	return
+}
+
+//CacheUserBbqDmg ...
+func (d *Dao) CacheUserBbqDmg(c context.Context, userBbqDmg *model.UserBbqDmg) (mid string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	tag2 := strings.Join(userBbqDmg.Tag2, ",")
+	tag3 := strings.Join(userBbqDmg.Tag3, ",")
+	up := strings.Join(userBbqDmg.Up, ",")
+	cacheKey := getUserDmgKey(userBbqDmg.MID)
+
+	if err = conn.Send("HSET", cacheKey, "zone", tag2); err != nil {
+		log.Error("cache user bbq dmg redis set tag2 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "tag", tag3); err != nil {
+		log.Error("cache user bbq dmg redis set tag3 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "up", up); err != nil {
+		log.Error("cache user bbq dmg redis set up err(%v)", err)
+		return
+	}
+
+	return
+}
+
+//CacheUserBbqDmgBuvid ...
+func (d *Dao) CacheUserBbqDmgBuvid(c context.Context, userBbqDmgBuvid *model.UserBbqBuvidDmg) (Buvid string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	tag2 := strings.Join(userBbqDmgBuvid.Tag2, ",")
+	tag3 := strings.Join(userBbqDmgBuvid.Tag3, ",")
+	up := strings.Join(userBbqDmgBuvid.Up, ",")
+	cacheKey := getUserBuvidDmgKey(userBbqDmgBuvid.Buvid)
+
+	if err = conn.Send("HSET", cacheKey, "zone", tag2); err != nil {
+		log.Error("cache user bbq buvid dmg redis set tag2 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "tag", tag3); err != nil {
+		log.Error("cache user bbq buvid dmg redis set tag3 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "up", up); err != nil {
+		log.Error("cache user bbq buvid dmg redis set up err(%v)", err)
+		return
+	}
+
+	return
+}
+
+// AddUpUserDmg .
+func (d *Dao) AddUpUserDmg(c context.Context, upUserDmg *model.UpUserDmg) (num int64, err error) {
+	var res sql.Result
+	if res, err = d.db.Exec(c, _incrUpUserDmgSQL, upUserDmg.MID, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes); err != nil {
+		return 0, err
+	}
+	return res.LastInsertId()
+}
+
+// UpdateUpUserDmg .
+func (d *Dao) UpdateUpUserDmg(c context.Context, upUserDmg *model.UpUserDmg) (num int64, err error) {
+	t := time.Now().AddDate(0, 0, 0).Format("2006-01-02 15:04:05")
+	var res sql.Result
+	if res, err = d.db.Exec(c, _updateUpUserDmgSQL, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes, t, upUserDmg.MID); err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
+// DelUpUserDmg .
+func (d *Dao) DelUpUserDmg(c context.Context) (num int64, err error) {
+	t := time.Unix(time.Now().Unix(), -int64(36*time.Hour)).Format("2006-01-02 15:04:05")
+	var res sql.Result
+	if res, err = d.db.Exec(c, _delUpUserDmgSQL, t); err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
+//Download 下载文件
+func (d *Dao) Download(url string, name string) (fpath string, err error) {
+	if name == "" {
+		u := strings.Split(url, "/")
+		l := len(u)
+		name = u[l-1]
+	}
+	t := time.Now().AddDate(0, 0, 0).Format("20060102")
+	path := conf.Conf.Download.File + t
+	err = d.CreateDir(path)
+	if err != nil {
+		log.Error("create dir(%s) err(%v)", path, err)
+		return
+	}
+	fpath = path + "/" + name
+	newFile, err := os.Create(fpath)
+
+	if err != nil {
+		log.Error("create path(%s) err(%v)", fpath, err)
+		return
+	}
+	defer newFile.Close()
+
+	client := http.Client{}
+	resp, err := client.Get(url)
+	if err != nil {
+		log.Error("download url(%s) err(%v)", url, err)
+		return
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(newFile, resp.Body)
+	if err != nil {
+		log.Error("copy err(%v)", err)
+		return
+	}
+	return
+}
+
+//CreateDir 创建文件夹
+func (d *Dao) CreateDir(path string) (err error) {
+	_, err = os.Stat(path)
+	defer func() {
+		if os.IsExist(err) {
+			err = nil
+		}
+	}()
+	if os.IsNotExist(err) {
+		err = os.Mkdir(path, os.ModePerm)
+	}
+	return
+}
+
+// ReadLine 按行读取文件，hander回调
+func (d *Dao) ReadLine(path string, handler func(string)) (err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Error("open path(%s) err(%v)", path, err)
+		return
+	}
+	defer f.Close()
+	buf := bufio.NewReader(f)
+	for {
+
+		line, err := buf.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			log.Error("read path(%s) err(%v)", path, err)
+			return nil
+		}
+		line = strings.TrimSpace(line)
+		handler(line)
+		time.Sleep(time.Duration(1) * time.Second)
+	}
+}
+
+// ReadLines 50条发起一次grpc请求
+func (d *Dao) ReadLines(path string, handler func([]int64)) (err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Error("ReadLine open path(%s) err(%v)", path, err)
+		return
+	}
+	defer f.Close()
+	buf := bufio.NewReader(f)
+	mids := make([]int64, 0, 50)
+	i := 0
+	for {
+		line, err := buf.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			}
+			log.Error("read path(%s) err(%v)", path, err)
+			break
+		}
+		mid, _ := strconv.ParseInt(strings.TrimSpace(line), 10, 64)
+		mids = append(mids, mid)
+		i++
+		if i == 50 {
+			handler(mids)
+			mids = make([]int64, 0, 50)
+			i = 0
+			time.Sleep(time.Duration(1) * time.Second)
+		}
+	}
+	if len(mids) != 0 {
+		handler(mids)
+	}
+	return
+}
+
+//HandlerUserDmg mid, gender, age, geo, content_tag, viewed_video, content_zone, content_count, follow_ups
+func (d *Dao) HandlerUserDmg(user string) {
+	u := strings.Split(user, "\u0001")
+	userDmg := &model.UserDmg{
+		MID:          u[0],
+		Gender:       u[1],
+		Age:          u[2],
+		Geo:          u[3],
+		ContentTag:   u[4],
+		ViewedVideo:  d.HandlerViewedVideo(u[5]),
+		ContentZone:  u[6],
+		ContentCount: u[7],
+		FollowUps:    u[8],
+	}
+	d.CacheUserDmg(context.Background(), userDmg)
+}
+
+//HandlerUserBbqDmg ..
+func (d *Dao) HandlerUserBbqDmg(user string) {
+	u := strings.Split(user, ",")
+	userBbqDmg := &model.UserBbqDmg{
+		MID:  u[0],
+		Tag2: strings.Split(u[1], "\u0002"),
+		Tag3: strings.Split(u[2], "\u0002"),
+		Up:   strings.Split(u[3], "\u0002"),
+	}
+	d.CacheUserBbqDmg(context.Background(), userBbqDmg)
+}
+
+//HandlerUserBbqDmgBuvid ..
+func (d *Dao) HandlerUserBbqDmgBuvid(user string) {
+	u := strings.Split(user, ",")
+	UserBbqBuvidDmg := &model.UserBbqBuvidDmg{
+		Buvid: u[0],
+		Tag2:  strings.Split(u[1], "\u0002"),
+		Tag3:  strings.Split(u[2], "\u0002"),
+		Up:    strings.Split(u[3], "\u0002"),
+	}
+	d.CacheUserBbqDmgBuvid(context.Background(), UserBbqBuvidDmg)
+}
+
+// HandlerMids update userbase by mids
+func (d *Dao) HandlerMids(mids []int64) {
+	res, err := d.VideoClient.SyncUserStas(context.Background(), &video.SyncMidsRequset{MIDS: mids})
+	if err != nil {
+		log.Error("userbases update failes, mids(%v), err(%v)", mids, err)
+		return
+	}
+	log.Info("userbases update success, affected %v rows", res.Affc)
+}
+
+// HandlerMid update userbase by mid
+func (d *Dao) HandlerMid(s string) {
+	mid, _ := strconv.ParseInt(s, 10, 64)
+	res, err := d.VideoClient.SyncUserSta(context.Background(), &video.SyncMidRequset{MID: mid})
+	if err != nil {
+		log.Error("userbase update failes, mid(%v), err(%v)", mid, err)
+		return
+	}
+	if res.Affc == 1 {
+		log.Info("userbase insert success ,mid(%v)", mid)
+	} else if res.Affc == 2 {
+		log.Info("userbase update success , mid(%v)", mid)
+	}
+}
+
+//HandlerViewedVideo 处理看过的视频，保存最近看过的100个
+func (d *Dao) HandlerViewedVideo(v string) (res map[int64]string) {
+	res = make(map[int64]string)
+	var vv [][]interface{}
+	var dd string
+
+	err := json.Unmarshal([]byte(v), &vv)
+	if err != nil {
+		return
+	}
+	l := len(vv)
+	n := 1
+	for i := l - 1; i >= 0; i-- {
+		for _, a := range vv[i] {
+			switch b := a.(type) {
+			case string:
+				dd = b
+			case []interface{}:
+				ll := len(b)
+				for j := ll - 1; j >= 0; j-- {
+					switch c := b[j].(type) {
+					case float64:
+						k := int64(c)
+						if _, ok := res[k]; !ok {
+							res[k] = dd
+							n++
+						}
+					}
+					if n > 100 {
+						return
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+// SelMidFromVideo get distinct mid list from table video
+func (d *Dao) SelMidFromVideo() (mids []int64, err error) {
+	rows, err := d.db.Query(context.Background(), _selMidFromVideo)
+	if err != nil {
+		log.Error("SelMidFromVideo failed, err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var s string
+		if err = rows.Scan(&s); err != nil {
+			panic(err.Error())
+		}
+		var mid int64
+		if mid, err = strconv.ParseInt(s, 10, 64); err != nil {
+			log.Error("strconv.ParseInt(%s) error(%v)", s, err)
+			return
+		}
+		mids = append(mids, mid)
+	}
+	return
+}
+
+//MergeUpInfo merge up info
+func (d *Dao) MergeUpInfo(mid int64) (err error) {
+	var (
+		ctx    = context.Background()
+		params = url.Values{}
+		req    = &http.Request{}
+		id     int64
+		res    struct {
+			Code int
+			Data model.UpUserInfoRes
+		}
+	)
+	err = d.db.QueryRow(ctx, "select mid from user_base where mid = ?", mid).Scan(&id)
+	if err == nil {
+		log.Infow(ctx, "log", "already has mid in user_base", "mid", mid)
+		return
+	}
+	if err == sql.ErrNoRows {
+		params.Set("mid", strconv.FormatInt(mid, 10))
+		req, err = d.HTTPClient.NewRequest("GET", d.c.URLs["account"], "", params)
+		if err != nil {
+			log.Error("MergeUpInfo error(%v)", err)
+			return
+		}
+		if err = d.HTTPClient.Do(ctx, req, &res); err != nil {
+			log.Error("MergeUpInfo http req failed ,err:%v", err)
+			return
+		}
+		res := res.Data
+		var sex int
+		switch res.Sex {
+		case "男":
+			sex = 1
+		case "女":
+			sex = 2
+		default:
+			sex = 3
+		}
+		_, err = d.db.Exec(ctx,
+			"insert into user_base (mid,uname,face,sex,user_type,complete_degree)values(?,?,?,?,?,?)",
+			res.MID,
+			res.Name,
+			res.Face,
+			sex,
+			model.UserTypeUp,
+			0)
+		if err != nil {
+			log.Error("MergeUpInfo insert upinfo failed,err:%v", err)
+			return
+		}
+	} else {
+		log.Error("MergeUpInfo query sql failed,err:%v", err)
+	}
+	if err = d.db.QueryRow(ctx, "select id from user_statistics where mid = ?", mid).Scan(&id); err != nil {
+		if err == sql.ErrNoRows {
+			if _, err = d.db.Exec(ctx, "insert into user_statistics (mid) values (?)", mid); err != nil {
+				log.Error("init insert user_statistics failed,err:%v", err)
+			}
+		} else {
+			log.Error("init query user_statistics failed,err:%v", err)
+		}
+	}
+	return
+}
+
+//UsersByLast 使用lastid批量获取用户
+func (d *Dao) UsersByLast(c context.Context, lastid int64) (r []*model.UserBaseDB, err error) {
+	var rows *xsql.Rows
+	rows, err = d.db.Query(c, _queryUsersByLast, lastid, _limitSize)
+	if err != nil {
+		log.Error("db _queryVideos err(%v)", err)
+		return
+	}
+	for rows.Next() {
+		u := new(model.UserBaseDB)
+		if err = rows.Scan(&u.ID, &u.MID, &u.Uname); err != nil {
+			log.Error("scan err(%v)", err)
+			continue
+		}
+		r = append(r, u)
+	}
+	return
+}
+
+// SelMidFromUserBase get distinct mid list from table user_base
+func (d *Dao) SelMidFromUserBase(start int) (mids []int64, err error) {
+	var mid int64
+	rows, err := d.db.Query(context.Background(), _selMidFromUserBase, start)
+	if err != nil {
+		log.Error("SelMidFromUserBase failed, err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var s string
+		if err = rows.Scan(&s); err != nil {
+			panic(err.Error())
+		}
+		if mid, err = strconv.ParseInt(s, 10, 64); err != nil {
+			log.Error("strconv.ParseInt(%s) error(%v)", s, err)
+			return
+		}
+		mids = append(mids, mid)
+	}
+	return
+}
+
+// UpUserBases 根据mids更新用户基本信息
+func (d *Dao) UpUserBases(c context.Context, mids []int64) (err error) {
+	var (
+		tx *xsql.Tx
+	)
+	midsReq := &acc.MidsReq{
+		Mids:   mids,
+		RealIp: metadata.String(c, metadata.RemoteIP)}
+	infosReply, err := d.AccountClient.Infos3(c, midsReq)
+	if infosReply == nil {
+		log.Error("查询infos3失败,err（%v）", err)
+		fmt.Printf("查询infos3失败,err（%v）", err)
+		return
+	}
+	if tx, err = d.BeginTran(c); err != nil {
+		log.Error("begin transaction error(%v)", err)
+		return
+	}
+	for _, info := range infosReply.Infos {
+		if info.Mid != 0 {
+			if len(info.Face) > 255 {
+				info.Face = "http://i0.hdslb.com/bfs/bbq/video-image/userface/1558868601542006937.png"
+			}
+			for try := 0; try < 3; try++ {
+				if _, err = tx.Exec(_upUserBase, info.Face, info.Mid); err == nil {
+					break
+				}
+			}
+			if err != nil {
+				log.Error("mid(%v) update failed", info.Mid)
+			}
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("UpUserBases commit failed err(%v)", err)
+	}
+	return
+}

app/job/bbq/video/dao/video.go

@@ -0,0 +1,524 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strconv"
+	"strings"
+	xtime "time"
+
+	"go-common/app/job/bbq/video/model"
+	"go-common/app/service/bbq/search/api/grpc/v1"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/time"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+const (
+	_limitSize = 2000
+	// MaxSyncESNum 限制每次更新到es的数量
+	MaxSyncESNum = 100
+	// QueryVideoByMtime 根据mtime获取视频基础信息
+	QueryVideoByMtime = "select `svid`,`mtime` from video where mtime >= ? order by mtime asc"
+	// QueryVideoStatisticsByMtime 根据mtime获取视频相关播放信息
+	QueryVideoStatisticsByMtime = "select `svid`,`mtime` from video_statistics where mtime >= ? order by mtime asc"
+	// QueryVideoStatisticsHiveByMtime 根据mtime获取视频主站信息
+	QueryVideoStatisticsHiveByMtime = "select `svid`,`mtime` from video_statistics_hive where mtime >= ? order by mtime asc"
+	// QueryVideoTagByMtime 根据mtime获取视频tag信息
+	QueryVideoTagByMtime           = "select `svid`,`mtime` from video_tag where mtime >= ? order by mtime asc"
+	_recRecallOpVideoKey           = "job:bbq:rec:op"
+	_syncOperVideoTagKey           = "job:bbq:video:syncvideotagkey"
+	_syncOperVideoTimeKey          = "job:bbq:video:syncvideotimekey"
+	_selectVideoInfo               = "select `svid`,`title`,`content`,`mid`,`cid`,`pubtime`,`ctime`,`mtime`,`duration`,`original`,`state`,`is_full_screen`,`ver_id`,`ver`,`from`,`avid`,`tid`,`sub_tid`,`score` from video where id > ? order by id asc limit 100"
+	_selectVideoInfoByIDs          = "select `svid`,`title`,`content`,`mid`,`cid`,`pubtime`,`ctime`,`mtime`,`duration`,`original`,`state`,`is_full_screen`,`ver_id`,`ver`,`from`,`avid`,`tid`,`sub_tid`,`score` from video where svid in (%s)"
+	_selectVideoStatisticsHiveInfo = "select `svid`,`play`,`fav`,`coin`,`subtitles`,`likes`,`share`,`report`,`duration_daily`,`duration_all`,`reply`,`share_daily`,`play_daily`,`subtitles_daily`,`likes_daily`,`fav_daily`,`access`,`reply_daily` from video_statistics_hive where svid in (%s)"
+	_selectVideoStatisticsInfo     = "select `svid`,`play`,`subtitles`,`like`,`share`,`report` from video_statistics where svid in (%s)"
+	_selectVideoTagsInfo           = "select v.svid,t.id,t.name,t.type from video_tag v inner join tag t on v.tag_id = t.id where v.svid in (%s)"
+	_queryCheckTask                = "select `task_id`,`task_name`,`last_check` from check_task where `task_name` = ?"
+	_updateTaskLastCheck           = "update check_task set last_check = ? where `task_name` = ?"
+	_queryTagByMtime               = "select `id`,`mtime` from tag where mtime > ? order by mtime asc limit 10"
+	_queryVideoTagByTagID          = "select `id`,`svid` from video_tag where tag_id in (%s) and id > ? order by id asc limit 100"
+	_queryVideoBySVIDs             = "select `svid`,`title` from video where svid in (%s)"
+	_queryIDs                      = "select `id`,`svid` from video where id > %d order by id asc limit %d"
+	_queryOutPutVideos             = "select id,svid,title,pubtime from video where id > ? and state in (%s) order by id ASC limit ?"
+	_updateUVSt                    = "update user_statistics set %s = %s+1 where mid = %d"
+	_updateUVStDel                 = "update user_statistics set %s = %s-1 where mid = %d and %s >0"
+	_updateSVTotal                 = "update user_statistics set av_total = av_total + 1 where mid = ?"
+	_getSvidByCid                  = "select svid from video where cid = ?"
+	_updateSVID                    = "update video_repository set svid = ? where id = ?"
+	_updateSyncStatus              = "update video_repository set sync_status = sync_status|? where svid = ?"
+	_queryCMSOne                   = "select `home_img_url`,`home_img_width`,`home_img_height`,`from`,`sync_status`, `tag`, `avid`, `cid`, `svid`, `title`, `mid`, `content`, `pubtime`,`duration`,`original`,`is_full_screen`,`tid`,`sub_tid`,`cover_url`,`cover_width`,`cover_height` from video_repository where svid = ?"
+	_queryCMSOneByID               = "select `tag`, `avid`, `cid`, `svid`, `title`, `mid`, `content`, `pubtime`,`duration`,`original`,`is_full_screen`,`tid`,`sub_tid`,`cover_url`,`cover_width`,`cover_height` from video_repository where id = ?"
+	_queryBbqVideo                 = "select title,mid,cid,state,tid,sub_tid,svid from video where svid in (%s)"
+)
+
+// RawVideo 从数据库获取视频信息
+func (d *Dao) RawVideo(ctx context.Context, SVID int64) (res *model.VideoRepRaw, err error) {
+	res = new(model.VideoRepRaw)
+	err = d.dbCms.QueryRow(ctx, _queryCMSOne, SVID).Scan(&res.HomeImgURL, &res.CoverWidth, &res.HomeImgHeight, &res.From, &res.SyncStatus, &res.Tag, &res.AVID, &res.CID, &res.SVID, &res.Title, &res.MID, &res.Content, &res.Pubtime, &res.Duration, &res.Original, &res.IsFull, &res.TID, &res.SubTID, &res.CoverURL, &res.CoverWidth, &res.CoverHeight)
+	if err != nil {
+		log.Error("RawVideo error(%v),svid:%d", err, SVID)
+	}
+	return
+}
+
+//RawBbqVideo ..
+func (d *Dao) RawBbqVideo(ctx context.Context, SVID []int64) (res *model.VideoRaw, err error) {
+	res = new(model.VideoRaw)
+
+	svids := strings.Trim(strings.Join(strings.Split(fmt.Sprint(SVID), " "), ","), "[]")
+	err = d.db.QueryRow(ctx, fmt.Sprintf(_queryBbqVideo, svids)).Scan(&res.Title, &res.MID, &res.CID, &res.State, &res.TID, &res.SubTID, &res.SVID)
+	if err != nil {
+		log.Errorw(ctx, "event", "RawBbqVideo queryrow scan err:[%v],SVID :[%v]", err, SVID)
+	}
+	return
+
+}
+
+//RawVideoByID ...get video info by id
+func (d *Dao) RawVideoByID(ctx context.Context, ID int64) (res *model.VideoRepRaw, err error) {
+	res = new(model.VideoRepRaw)
+	err = d.dbCms.QueryRow(ctx, _queryCMSOneByID, ID).Scan(&res.Tag, &res.AVID, &res.CID, &res.SVID, &res.Title, &res.MID, &res.Content, &res.Pubtime, &res.Duration, &res.Original, &res.IsFull, &res.TID, &res.SubTID, &res.CoverURL, &res.CoverWidth, &res.CoverHeight)
+	if err != nil {
+		log.Error("RawVideoByID error(%v),id:%d", err, ID)
+	}
+	return
+}
+
+//UpdateSyncStatus update video_repository sync_status
+func (d *Dao) UpdateSyncStatus(ctx context.Context, SVID int64, st int64) (err error) {
+	if _, err = d.dbCms.Exec(ctx, _updateSyncStatus, st, SVID); err != nil {
+		log.Error("UpdateSyncStatus err :%v,svid :", err, SVID)
+	}
+	return
+}
+
+//UpdateVideoUploadProcessStatus ...
+func (d *Dao) UpdateVideoUploadProcessStatus(ctx context.Context, SVID int64, st int64) (err error) {
+	if _, err = d.db.Exec(ctx, "update video_upload_process set status = ? where svid = ?", st, SVID); err != nil {
+		log.Errorw(ctx, "errmsg", "UpdateVideoUploadProcessStatus update failed", "err", err)
+	}
+	return
+}
+
+//UpdateSvid ...
+func (d *Dao) UpdateSvid(c context.Context, id int64, svid int64) (err error) {
+	if _, err = d.dbCms.Exec(c, _updateSVID, svid, id); err != nil {
+		log.Error("distribution svid err:%v,svid:%d", err, svid)
+	}
+	return
+}
+
+//AddSVTotal ...
+func (d *Dao) AddSVTotal(mid int64) (err error) {
+	_, err = d.db.Exec(context.Background(), _updateSVTotal, mid)
+	if err != nil {
+		log.Error("AddSVTotal ,mid:%s,err:%v", mid, err)
+	}
+	return
+}
+
+//UpdateUVSt 更新用户视频统计信息
+func (d *Dao) UpdateUVSt(mid int64, field string) (err error) {
+	_, err = d.db.Exec(context.Background(), fmt.Sprintf(_updateUVSt, field, field, mid))
+	if err != nil {
+		log.Error("UpdateUVSt ,mid:%s,field:%s,op:%d,err:%v", mid, field, err)
+	}
+	return
+}
+
+//UpdateUVStDel ...
+func (d *Dao) UpdateUVStDel(mid int64, field string) (err error) {
+	_, err = d.db.Exec(context.Background(), fmt.Sprintf(_updateUVStDel, field, field, mid, field))
+	if err != nil {
+		log.Error("UpdateUVStDel ,mid:%s,field:%s,op:%d,err:%v", mid, field, err)
+	}
+	return
+}
+
+//VideoList 获取视频基础信息
+func (d *Dao) VideoList(c context.Context, id int64) (ids string, res []*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _selectVideoInfo, id); err != nil {
+		log.Error("select videos err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		pubtime  time.Time
+		ctime    time.Time
+		mtime    time.Time
+		idstring []string
+	)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.Title, &tmp.Content, &tmp.MID, &tmp.CID, &pubtime, &ctime, &mtime, &tmp.Duration, &tmp.Original, &tmp.State, &tmp.ISFullScreen, &tmp.VerID, &tmp.Ver, &tmp.From, &tmp.AVID, &tmp.Tid, &tmp.SubTid, &tmp.Score); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			return
+		}
+		tmp.Pubtime = int64(pubtime)
+		tmp.Ctime = int64(ctime)
+		tmp.Mtime = int64(mtime)
+		idstring = append(idstring, strconv.FormatInt(tmp.SVID, 10))
+		res = append(res, tmp)
+	}
+	ids = strings.Join(idstring, ",")
+	return
+}
+
+//VideoListByIDs 根据视频id获取视频基础信息
+func (d *Dao) VideoListByIDs(c context.Context, ids string) (res []*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoInfoByIDs, ids)); err != nil {
+		log.Error("select videos by ids err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		pubtime time.Time
+		ctime   time.Time
+		mtime   time.Time
+	)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.Title, &tmp.Content, &tmp.MID, &tmp.CID, &pubtime, &ctime, &mtime, &tmp.Duration, &tmp.Original, &tmp.State, &tmp.ISFullScreen, &tmp.VerID, &tmp.Ver, &tmp.From, &tmp.AVID, &tmp.Tid, &tmp.SubTid, &tmp.Score); err != nil {
+			log.Error("select videos scan err(%v)", err)
+			return
+		}
+		tmp.Pubtime = int64(pubtime)
+		tmp.Ctime = int64(ctime)
+		tmp.Mtime = int64(mtime)
+		res = append(res, tmp)
+	}
+	return
+}
+
+//VideoStatisticsHiveList 获取视频互动信息，hive表
+func (d *Dao) VideoStatisticsHiveList(c context.Context, ids string) (res map[int64]*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoStatisticsHiveInfo, ids)); err != nil {
+		log.Error("select video statistics hive err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[int64]*v1.VideoESInfo)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.PlayHive, &tmp.FavHive, &tmp.CoinHive, &tmp.SubtitlesHive, &tmp.LikesHive, &tmp.ShareHive, &tmp.ReportHive, &tmp.DurationDailyHive, &tmp.DurationAllHive, &tmp.ReplyHive, &tmp.ShareDailyHive, &tmp.PlayDailyHive, &tmp.SubtitlesDailyHive, &tmp.LikesDailyHive, &tmp.FavDailyHive, &tmp.AccessHive, &tmp.ReplyDailyHive); err != nil {
+			log.Error("select video statistics hive scan err(%v)", err)
+			return
+		}
+		res[tmp.SVID] = tmp
+	}
+	return
+}
+
+//VideoStatisticsList 获取视频互动信息
+func (d *Dao) VideoStatisticsList(c context.Context, ids string) (res map[int64]*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoStatisticsInfo, ids)); err != nil {
+		log.Error("select video statistics err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[int64]*v1.VideoESInfo)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.Play, &tmp.SubtitlesHive, &tmp.Like, &tmp.Share, &tmp.Report); err != nil {
+			log.Error("select video statistics scan err(%v)", err)
+			return
+		}
+		res[tmp.SVID] = tmp
+	}
+	return
+}
+
+//VideoTagsList 获取视频tags
+func (d *Dao) VideoTagsList(c context.Context, ids string) (res map[int64][]*v1.VideoESTags, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoTagsInfo, ids)); err != nil {
+		fmt.Println(rows)
+		log.Error("select video statistics err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var svid int64
+	res = make(map[int64][]*v1.VideoESTags)
+	for rows.Next() {
+		tmp := new(v1.VideoESTags)
+		if err = rows.Scan(&svid, &tmp.ID, &tmp.Name, &tmp.Type); err != nil {
+			log.Error("select video statistics scan err(%v)", err)
+			return
+		}
+		res[svid] = append(res[svid], tmp)
+	}
+	return
+}
+
+//RawCheckTask 查询脚本任务
+func (d *Dao) RawCheckTask(c context.Context, taskName string) (res *model.CheckTask, err error) {
+	res = new(model.CheckTask)
+	raw := d.db.QueryRow(c, _queryCheckTask, taskName)
+	if err = raw.Scan(&res.TaskID, &res.TaskName, &res.LastCheck); err != nil {
+		log.Error("query check task name(%s) err(%v)", taskName, err)
+	}
+	return
+}
+
+//UpdateTaskLastCheck 更新上次执行时间
+func (d *Dao) UpdateTaskLastCheck(c context.Context, taskName string, lastCheck int64) (num int64, err error) {
+	res, err := d.db.Exec(c, _updateTaskLastCheck, lastCheck, taskName)
+	if err != nil {
+		log.Error("update task last check name(%s)(%d) err(%v)", taskName, lastCheck, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// RawGetIDByMtime 获取最近更新的那些svid，该函数可以用于多个表的查询，只需传入不同表的查询语句即可
+func (d *Dao) RawGetIDByMtime(baseTableQuery string, mtime int64) (ids []int64, lastMtime int64, err error) {
+	mtimeStr := xtime.Unix(mtime, 0).Format("2006-01-02 15:04:05")
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(context.Background(), baseTableQuery, mtimeStr); err != nil {
+		log.Error("select ids fail: err=%v, mtime=%s, sql=%s", err, mtimeStr, baseTableQuery)
+		return
+	}
+	defer rows.Close()
+	var (
+		temp time.Time
+		svid int64
+	)
+	for rows.Next() {
+		if err = rows.Scan(&svid, &temp); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			log.Error("select videos by mtime scan fail: err=%v, mtime=%s, sql=%s", err, mtimeStr, baseTableQuery)
+			return
+		}
+		lastMtime = int64(temp)
+		ids = append(ids, svid)
+	}
+	return
+}
+
+//RawTagByMtime 根据mtime获取tag信息
+func (d *Dao) RawTagByMtime(c context.Context, mtime int64) (ids string, res int64, err error) {
+	str := xtime.Unix(mtime, 0).Format("2006-01-02 15:04:05")
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _queryTagByMtime, str); err != nil {
+		log.Error("select tag err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		temp     time.Time
+		id       int64
+		idstring []string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&id, &temp); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			log.Error("select tag by mtime scan err(%v)", err)
+			return
+		}
+		res = int64(temp)
+		idstring = append(idstring, strconv.FormatInt(id, 10))
+	}
+	ids = strings.Join(idstring, ",")
+	return
+}
+
+//RawVideoTagByIDs .
+func (d *Dao) RawVideoTagByIDs(c context.Context, ids string, id int64) (svids string, res int64, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_queryVideoTagByTagID, ids), id); err != nil {
+		log.Error("select video tag err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		temp     int64
+		svid     int64
+		idstring []string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&temp, &svid); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			} else {
+				log.Error("select video tag by id scan err(%v)", err)
+			}
+			return
+		}
+		res = int64(temp)
+		idstring = append(idstring, strconv.FormatInt(svid, 10))
+	}
+	svids = strings.Join(idstring, ",")
+	return
+}
+
+//GetSyncOperVideoFlag 获取同步信号灯
+func (d *Dao) GetSyncOperVideoFlag(c context.Context) (tag int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	val, err := conn.Do("get", _syncOperVideoTagKey)
+	if err != nil {
+		log.Error("cache sync oper video tag get err(%v)", err)
+		return
+	}
+	if val == nil {
+		if err = d.SetSyncOperVideoFlag(c, model.DenySyncOperVideoTag); err != nil {
+			log.Error("set sync oper video flag faild")
+			return
+		}
+		tag = model.DenySyncOperVideoTag
+	} else {
+		tag, err = redis.Int64(val, err)
+	}
+	return
+}
+
+//SetSyncOperVideoFlag 设置同步信号灯
+func (d *Dao) SetSyncOperVideoFlag(c context.Context, v int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("set", _syncOperVideoTagKey, v); err != nil {
+		log.Error("cache sync oper video tag set err(%v)", err)
+		return
+	}
+	return
+}
+
+//GetSyncOperVideoExportTime ...
+func (d *Dao) GetSyncOperVideoExportTime(c context.Context) (t string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	val, err := conn.Do("get", _syncOperVideoTimeKey)
+	if err == nil {
+		if val == nil {
+			conn.Do("set", _syncOperVideoTimeKey, "")
+		}
+		t, err = redis.String(val, err)
+	} else {
+		log.Error("get sync oper video export time err,errinfo:%v", err)
+	}
+	return
+}
+
+//RawVideoBySVIDS 根据svids获取视频
+func (d *Dao) RawVideoBySVIDS(c context.Context, svids []string) (res map[int64]string, err error) {
+	res = make(map[int64]string)
+	str := strings.Join(svids, ",")
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_queryVideoBySVIDs, str)); err != nil {
+		log.Error("select videos by svids err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		svid  int64
+		title string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&svid, &title); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			log.Error("select videos by svids scan err(%v)", err)
+			return
+		}
+		res[svid] = title
+	}
+	return
+}
+
+//GetVideoByLastID 获取所有SVID
+func (d *Dao) GetVideoByLastID(c context.Context, last int64) (IDs []int64, lastRet int64, err error) {
+	length := 1000 //分批大小
+	var rows *xsql.Rows
+	rows, err = d.db.Query(c, fmt.Sprintf(_queryIDs, last, length))
+	if err != nil {
+		log.Error("db _queryIDs err(%v)", err)
+		return
+	}
+	for rows.Next() {
+		var svid int64
+		if err = rows.Scan(&lastRet, &svid); err != nil {
+			log.Error("scan err(%v)", err)
+			continue
+		}
+		IDs = append(IDs, svid)
+	}
+	return
+}
+
+//GetRecallOpVideo 获取精选视频
+func (d *Dao) GetRecallOpVideo(c context.Context) (ids []int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	val, err := redis.Bytes(conn.Do("GET", _recRecallOpVideoKey))
+	if err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("cache rec recall op video get redis err (%v)", err)
+		}
+		return ids, err
+	}
+	if err = jsoniter.Unmarshal(val, &ids); err != nil {
+		log.Error("rec recall op video unmarshal err (%v)", err)
+	}
+	return
+}
+
+//SetRecallOpVideo 写入精选视频
+func (d *Dao) SetRecallOpVideo(c context.Context, ids []int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	bytes, _ := jsoniter.Marshal(ids)
+	_, err = conn.Do("SET", _recRecallOpVideoKey, bytes)
+	if err != nil {
+		log.Error("rec recall op video set redis error(%v) ", err)
+	}
+	return
+}
+
+//VideosByLast 使用lastid批量获取视屏
+func (d *Dao) VideosByLast(c context.Context, lastid int64) (svinfo []*model.VideoDB, err error) {
+	var rows *xsql.Rows
+	query := fmt.Sprintf(_queryOutPutVideos, model.VideoStateOutPut)
+	rows, err = d.db.Query(c, query, lastid, _limitSize)
+	if err != nil {
+		log.Error("db _queryVideos err(%v)", err)
+		return
+	}
+	for rows.Next() {
+		video := new(model.VideoDB)
+		if err = rows.Scan(&video.AutoID, &video.ID, &video.Title, &video.Pubtime); err != nil {
+			log.Error("scan err(%v)", err)
+			continue
+		}
+		svinfo = append(svinfo, video)
+	}
+	return
+}
+
+// GetSvidByCid 根据cid获取svid
+func (d *Dao) GetSvidByCid(c context.Context, cid int64) (svid int64, err error) {
+	err = d.db.QueryRow(c, _getSvidByCid, cid).Scan(&svid)
+	if err != nil {
+		log.Warn("db _getSvidByCid err(%v)", err)
+		return
+	}
+	return
+}