Create & Init Project...

app/job/bbq/video/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq/video/cmd:all-srcs",
+        "//app/job/bbq/video/conf:all-srcs",
+        "//app/job/bbq/video/dao:all-srcs",
+        "//app/job/bbq/video/model:all-srcs",
+        "//app/job/bbq/video/server/http:all-srcs",
+        "//app/job/bbq/video/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/CHANGELOG.md

@@ -0,0 +1,39 @@
+# v1.0.10
+修复user_type的设置
+
+# v1.0.9
+1. fix dead  loop
+2. pr
+# v1.0.8
+修复video消费
+
+# v1.0.7
+1. 增加video表的state消费，同步修改到topic
+
+# v1.0.6
+1. 删除无效方法
+
+# v1.0.5
+1. 增加观看历史bloomfilter小时级重建
+2. 导入视频删除同步b站同步信息
+
+# v1.0.4
+1. 增加binlog补全用户信息
+2. merge master
+
+# v1.0.3
+1. 添加25w临时导入脚本
+2. merge master
+
+# v1.0.2
+1. 修改视频导入逻辑
+2. merge master
+3. 添加 std log
+
+# v1.0.1
+1. 添加批量注册评论服务
+2. 上线功能xxx
+
+# v1.0.6
+1. 删除无效方法
+2. 修复

app/job/bbq/video/CONTRIBUTORS.md

@@ -0,0 +1,9 @@
+# Owner
+yangyucheng
+luxiaowei
+daiwei
+jiangdongqi
+
+# Author
+
+# Reviewer

app/job/bbq/video/OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- daiwei
+- jiangdongqi
+- luxiaowei
+- yangyucheng
+labels:
+- bbq
+- job
+- job/bbq/video
+options:
+  no_parent_owners: true

app/job/bbq/video/README.md

@@ -0,0 +1,15 @@
+# video-job
+
+# 项目简介
+1.BBQ项目脚本
+
+# 编译环境
+
+
+# 依赖包
+
+
+# 编译执行
+```bash
+# go run main.go -conf test.toml -deploy.env uat
+```

app/job/bbq/video/api/http/api.md

@@ -0,0 +1 @@
+# HTTP API文档

app/job/bbq/video/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["test.toml"],
+    importpath = "go-common/app/job/bbq/video/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/server/http:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/cmd/cids

@@ -0,0 +1 @@
+10150892

app/job/bbq/video/cmd/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/server/http"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("start")
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	http.Init(conf.Conf)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("exit")
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/bbq/video/cmd/sug

@@ -0,0 +1,2 @@
+111001917,抹茶,bbq_user,0
+4,舞蹈,bbq_video,0

app/job/bbq/video/cmd/test.toml

@@ -0,0 +1,366 @@
+[log]
+    # dir = "/tmp/log/bbq"
+	stdout = true
+
+[download]
+	file = "/data/"
+
+[bm]
+    [bm.server]
+        addr = "0.0.0.0:9999"
+        timeout = "1s"
+    [bm.client]
+        key = "c8c48e784e05acfb"
+        secret = "aa63ee0a10afa358d02a07e7abcec546"
+        dial = "1000ms"
+        timeout = "30s"
+        keepAlive = "60s"
+        timer = 10
+    [bm.client.breaker]
+        window  = "3s"
+        sleep   = "1000ms"
+        bucket  = 10
+        ratio   = 0.5
+        request = 100
+
+[mysql]
+	addr = "172.16.38.91:3306"
+	dsn = "root:123456@tcp(172.16.38.91:3306)/bbq?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["root:123456@tcp(172.16.38.91:3306)/bbq?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 10
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "1000ms"
+	execTimeout = "2000ms"
+	tranTimeout = "2000ms"
+[mysqlcms]
+	addr = "172.16.38.91:3306"
+	dsn = "root:123456@tcp(172.16.38.91:3306)/bbq_cms?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["root:123456@tcp(172.16.38.91:3306)/bbq_cms?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 10
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "1000ms"
+	execTimeout = "2000ms"
+	tranTimeout = "2000ms"
+
+[redis]
+	name = "video-job"
+	proto = "tcp"
+	addr = "172.16.38.91:6379"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[bfredis]
+	name = "video-job-bloomfilter"
+	proto = "tcp"
+	addr = "172.16.38.91:6379"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[memcache]
+	name = "video-job"
+	proto = "tcp"
+	addr = ""
+	active = 50
+	idle = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "24h"
+
+[grpcClient]
+	[grpcClient.video]
+        addr = "discovery://default/bbq.service.video"
+        [grpcClient.video.wardenconf]
+            dial = "300ms"
+            timeout = "2s"
+    [grpcClient.search]
+		addr = "discovery://default/bbq.service.search"
+		[grpcClient.search.wardenconf]
+        	dial = "100ms"
+        	timeout = "500ms"
+	[grpcClient.account]
+		addr = "discovery://default/account.service"
+        [grpcClient.account.wardenconf]
+            dial = "1s"
+            timeout = "2s"
+    [grpcClient.notice]
+        addr = "discovery://default/bbq.service.notice"
+        [grpcClient.notice.wardenconf]
+            dial = "3000ms"
+            timeout = "5000ms"
+
+[scheduler]
+    # #每天4点，同步全量视频数据到es
+    # checkVideo2ES = "0 0 4 * * *"
+    # #每天0点10分，同步用户画像
+    # syncUserDmg = "0 30 0 * * *"
+    # #每天3点00分，同步主站视频库
+    # SyncVideo = "0 0 3 * * *"
+    # #每天1点00分，同步主站up主信息
+    # SyncUpUserDmg = "0 0 1 * * *"
+    # #心跳测试
+    # Test = "*/1 * * * * *"
+    #每1分，视频增量脚本
+    checkVideo = "0 */1 * * * *"
+    checkVideoSt = "0 */1 * * * *"
+    checkVideoStHv = "0 */1 * * * *"
+    checkVideoTag = "0 */1 * * * *"
+    checkTag = "0 */1 * * * *"
+    #运营筛选视频导入，1分钟check一次信号
+    SyncVideoOper = "0 0 2 * * *"
+    #同步cms任务
+    TransToCheckBack = "1 * * * * *"
+	#每天晚上9点更新userbase
+	SyncUsrSta = "0 30 21 * * *"
+    # checkVideo = "0 */1 * * * *"
+    # checkVideoSt = "0 */1 * * * *"
+    # checkVideoStHv = "0 */1 * * * *"
+    # checkVideoTag = "0 */1 * * * *"
+    # checkTag = "0 */1 * * * *"
+    # #运营筛选视频导入，1分钟check一次信号
+    # SyncVideoOper = "0 0 2 * * *"
+    # #同步cms任务
+    # DeliveryNewVideoToCms = "0 0 3 * * *"
+	## 去重BloomFilter重建
+	videoViewHistory = "0 */1 * * * *"
+	# 系统消息任务
+	SysMsgTask = "0 */1 * * * *"
+	## UsrProfileBbq
+	UserProfileBbq = "0 0 9 * * *"
+
+[berserker]
+	[berserker.key]
+		[berserker.key.yyc]
+		    appkey = "dcf555f4f314ccdfad7cb8a3288f1643"
+		    secret = "4efd3443561e66832b3ba9128a1ad720"
+		[berserker.key.hsc]
+		    appkey = "0bcf67ecf921576350a4f916c1edecbf"
+		    secret = "1003e5a9babe66ef6834fe839ce8223f"
+		[berserker.key.lzq]
+			appkey = "e51d04f43e9c9bca8678dcb4586c76db"
+			secret = "7c53bf6d17c2d51253b927bc057abeae"
+		[berserker.key.lj]
+			appkey = "4ddb82b57e863ebd888713061a08ba10"
+			secret = "49c4e3eb4fd8aff9d3862447113e032f"
+		[berserker.key.dw]
+			appkey = "66ed588a742c72408bc2d876afa8f7ca"
+			secret = "7d7a63e37fe6af52a58fb4755b62360e"
+		[berserker.key.hm]
+			appkey = "0b685519ff66471faf3f000cc67661eb"
+			secret = "e345a3281f0009b51016fab9a9076877"
+	[berserker.api]
+        rankmonthly = "http://berserker.bilibili.co/avenger/api/92/query"
+		rankdaily = "http://berserker.bilibili.co/avenger/api/102/query"
+		userdmg = "http://berserker.bilibili.co/avenger/api/137/query"
+		upuserdmg = "http://berserker.bilibili.co/avenger/api/125/query"
+		operaonce = "http://berserker.bilibili.co/avenger/api/121/query"
+		userbasic = "http://berserker.bilibili.co/avenger/api/147/query"
+		upmid = "http://berserker.bilibili.co/avenger/api/149/query"
+		videoView = "http://berserker.bilibili.co/avenger/api/154/query"
+		UserProfile = "http://berserker.bilibili.co/avenger/api/178/query"
+		UserProfileBuvid = "http://berserker.bilibili.co/avenger/api/186/query"
+
+[tagmap]
+	[tagmap.tagtidmap]
+		1   = "动画"
+		3   = "音乐"
+		4   = "游戏"
+		5   = "娱乐"
+		11  = "电视剧"
+		13  = "番剧"
+		23  = "电影"
+		36  = "科技"
+		119 = "鬼畜"
+		129 = "舞蹈"
+		155 = "时尚"
+		160 = "生活"
+		165 = "广告"
+		167 = "国创"
+		177 = "纪录片"
+		181 = "影视"
+	[tagmap.tagsubtidmap]
+		24=  "MAD·AMV"
+		25=  "MMD·3D"
+		27=  "综合"
+		47=  "短片·手书·配音"
+		28=  "原创音乐"
+		29=  "三次元音乐"
+		30=  "VOCALOID·UTAU"
+		31=  "翻唱"
+		54=  "OP/ED/OST"
+		59=  "演奏"
+		130= "音乐选集"
+		17=  "单机联机"
+		19=  "Mugen"
+		65=  "网络游戏"
+		121= "GMV"
+		136= "音游"
+		171= "电子竞技"
+		172= "手机游戏"
+		173= "桌游棋牌"
+		71=  "综艺"
+		131= "Korea相关"
+		137= "明星"
+		185= "国产剧"
+		187= "海外剧"
+		32=  "完结动画"
+		33=  "连载动画"
+		51=  "资讯"
+		152= "官方延伸"
+		83=  "其他国家"
+		145= "欧美电影"
+		146= "日本电影"
+		147= "国产电影"
+		39=  "演讲• 公开课"
+		95=  "数码"
+		96=  "星海"
+		98=  "机械"
+		122= "野生技术协会"
+		124= "趣味科普人文"
+		176= "汽车"
+		22=  "鬼畜调教"
+		26=  "音MAD"
+		126= "人力VOCALOID"
+		127= "教程演示"
+		20=  "宅舞"
+		154= "三次元舞蹈"
+		156= "舞蹈教程"
+		157= "美妆"
+		158= "服饰"
+		159= "资讯"
+		164= "健身"
+		21=  "日常"
+		75=  "动物圈"
+		76=  "美食圈"
+		138= "搞笑"
+		161= "手工"
+		162= "绘画"
+		163= "运动"
+		174= "其他"
+		175= "ASMR"
+		166= "广告"
+		153= "国产动画"
+		168= "国产原创相关"
+		169= "布袋戏"
+		170= "资讯"
+		37=  "人文历史"
+		178= "科学探索"
+		179= "热血军事"
+		180= "舌尖上的旅行"
+		85=  "短片"
+		86=  "特摄"
+		182= "影视杂谈"
+		183= "影视剪辑"
+		184= "预告·资讯"
+
+[databus]
+    [databus.videosub]
+        key = "36ff3e402f7c310a"
+        secret = "dbd11b140486dc0bc263cf7ec540186c"
+        group = "BBQVideo-BbqBbq-S"
+        topic = "BBQVideo-T"
+        action = "sub"
+        buffer = 1024
+        name = "app-job/video"
+        proto = "tcp"
+        addr = "172.22.33.174:6205 "
+        idle = 1
+        active = 1
+        dialTimeout = "1s"
+        readTimeout = "60s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+	[databus.videosub.discovery]
+			domain = "uat-api.bilibili.co"
+			key = "c8c48e784e05acfb"
+			secret = "aa63ee0a10afa358d02a07e7abcec546"
+			region = "sh"
+			zone = "sh001"
+			env = "uat"
+    [databus.videorep]
+        key = "36ff3e402f7c310a"
+        secret = "dbd11b140486dc0bc263cf7ec540186c"
+        group = "BBQVideoRep-BbqBbq-S"
+        topic = "BBQVideoRep-T"
+        action = "sub"
+        buffer = 1024
+        name = "history"
+        proto = "tcp"
+        addr = "172.22.33.174:6205 "
+        idle = 1
+        active = 1
+        dialTimeout = "1s"
+        readTimeout = "60s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+		[databus.videorep.discovery]
+			domain = "uat-api.bilibili.co"
+			key = "c8c48e784e05acfb"
+			secret = "aa63ee0a10afa358d02a07e7abcec546"
+			region = "sh"
+			zone = "sh001"
+			env = "uat"
+	[databus.bvcsub]
+        key = "36ff3e402f7c310a"
+        secret = "dbd11b140486dc0bc263cf7ec540186c"
+        group = "BBQCms-BbqBbq-S"
+        topic = "BBQCms-T"
+        action = "sub"
+        buffer = 1024
+        name = "history"
+        proto = "tcp"
+        addr = "172.22.33.174:6205"
+        idle = 1
+        active = 1
+        dialTimeout = "1s"
+        readTimeout = "60s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+		[databus.bvcsub.discovery]
+			domain = "uat-api.bilibili.co"
+			key = "c8c48e784e05acfb"
+			secret = "aa63ee0a10afa358d02a07e7abcec546"
+			region = "sh"
+			zone = "sh001"
+			env = "uat"
+		
+[mail]
+host = "smtp.exmail.qq.com"
+port = 465
+from = "jiangdongqi@bilibili.com"
+password = "l"
+to = ["jiangdongqi@bilibili.com"]
+
+[urls]
+	reply_reg = "http://uat-api.bilibili.co/x/internal/v2/reply/subject/regist"
+	account = "http://uat-api.bilibili.co/x/internal/v3/account/info"
+	bvc_push = "http://127.0.0.1:8802/bbq/internal/sv/trans/commit"
+
+[ftp]
+ addr = "172.16.33.203:21"
+ user = "work"
+ password = "MhxzKhl"
+ 	[ftp.remotepath]
+		search = "/home/work/open/%s"
+ timeout = "1000ms"
+	[ftp.localpath]
+		search = "/tmp/"
+[vst]
+	tmpstatus = -3
+[path]
+	cids="/Users/Cheney/go/src/go-common/app/job/bbq/video/cmd/cids"
+[subbvccontrol]
+	control = 1

app/job/bbq/video/conf/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/bbq/video/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/conf/conf.go

@@ -0,0 +1,223 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"go-common/library/net/rpc/warden"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config .
+type Config struct {
+	Log          *log.Config
+	BM           *HTTPGeneral
+	Verify       *verify.Config
+	Tracer       *trace.Config
+	Redis        *redis.Config
+	BfRedis      *redis.Config
+	Memcache     *memcache.Config
+	MySQL        *sql.Config
+	MySQLCms     *sql.Config
+	MySQLOffline *sql.Config
+	Ecode        *ecode.Config
+	Scheduler    *Scheduler
+	GRPCClient   map[string]*GRPCConf
+	//berserker
+	Berserker     *Berserker
+	TagMap        *TagMap
+	Databus       map[string]*databus.Config
+	Mail          *Mail
+	URLs          map[string]string
+	Download      *Download
+	FTP           *FTP
+	VST           *Vst
+	Path          map[string]string
+	SubBvcControl *Sub
+}
+
+//Sub ...
+type Sub struct {
+	Control int8
+}
+
+//Vst ...
+type Vst struct {
+	TmpStatus int64
+}
+
+// FTP FTP.
+type FTP struct {
+	Addr       string
+	User       string
+	Password   string
+	RemotePath map[string]string
+	Timeout    xtime.Duration
+	LocalPath  map[string]string
+}
+
+//GRPCConf .
+type GRPCConf struct {
+	WardenConf *warden.ClientConfig
+	Addr       string
+}
+
+// Download .
+type Download struct {
+	File string
+}
+
+//Mail ...
+type Mail struct {
+	Host     string
+	Port     int
+	From     string
+	Password string
+	To       []string
+}
+
+//TagMap ...
+type TagMap struct {
+	TagTidMap    map[string]string
+	TagSubTidMap map[string]string
+}
+
+// HTTPGeneral conf
+type HTTPGeneral struct {
+	Server *bm.ServerConfig
+	Client *bm.ClientConfig
+}
+
+// Berserker conf
+type Berserker struct {
+	Key *BerSerkerKeyList
+	API *BerserkerAPI
+}
+
+// BerserkerAPI conf
+type BerserkerAPI struct {
+	Rankdaily        string
+	Userdmg          string
+	Upuserdmg        string
+	Operaonce        string
+	Userbasic        string
+	Upmid            string
+	VideoView        string
+	UserProfile      string
+	UserProfileBuvid string
+}
+
+// BerSerkerKeyList conf
+type BerSerkerKeyList struct {
+	YYC *BerSerkerKey
+	HSC *BerSerkerKey
+	LZQ *BerSerkerKey
+	LJ  *BerSerkerKey
+	DW  *BerSerkerKey
+	HM  *BerSerkerKey
+}
+
+// BerSerkerKey conf
+type BerSerkerKey struct {
+	Appkey string
+	Secret string
+}
+
+//Scheduler .
+type Scheduler struct {
+	CheckVideo2ES         string
+	SyncUserDmg           string
+	Test                  string
+	SyncUpUserDmg         string
+	CheckVideo            string
+	CheckVideoSt          string
+	CheckVideoStHv        string
+	CheckVideoTag         string
+	CheckTag              string
+	SyncVideoOper         string
+	DeliveryNewVideoToCms string
+	SyncUsrSta            string
+	SyncSearch            string
+	VideoViewHistory      string
+	SysMsgTask            string
+	UserProfileBbq        string
+	TransToReview         string
+	TransToCheckBack      string
+}
+
+// Databus .
+type Databus struct {
+	Video    *databus.Config
+	VideoRep *databus.Config
+	BvcSub   *databus.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/bbq/video/dao/BUILD

@@ -0,0 +1,58 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "cmsvideo.go",
+        "comment.go",
+        "dao.go",
+        "email.go",
+        "notice.go",
+        "user.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/video/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/model:go_default_library",
+        "//app/service/bbq/notice-service/api/v1:go_default_library",
+        "//app/service/bbq/search/api/grpc/v1:go_default_library",
+        "//app/service/bbq/sys-msg/api/v1:go_default_library",
+        "//app/service/bbq/video/api/grpc/v1:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/dao/berserker.go

@@ -0,0 +1,454 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/model"
+	"go-common/library/conf/env"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	xhttp "net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"io/ioutil"
+
+	pkgerr "github.com/pkg/errors"
+)
+
+const (
+	_jobStatusSuccess = 1
+	_jobStatusFailed  = 2
+	_jobStatusDoing   = 3
+	_jobStatusWaiting = 4
+	//_httpHeaderUser     = "x1-bilispy-user"
+	//_httpHeaderColor    = "x1-bilispy-color"
+	//_httpHeaderTimeout  = "x1-bilispy-timeout"
+	_httpHeaderRemoteIP = "x-backend-bili-real-ip"
+	_userAgent          = "User-Agent"
+	_noKickUserAgent    = "yangyucheng@bilibili.com"
+	_queryJSON          = `{"select":[],"where":{"log_date":{"in":["%s"]}},"page":{"limit":1000},"sort":{"play":-1}}`
+	_queryJSONOper      = `{"select":[],"where":{"log_date":{"in":["%s"]},"cid":{"gt":%d}},"page":{"limit":5000},"sort":{"cid":1}}`
+	_hscUserAgent       = "huangshancheng@bilibili.com"
+	_lzqUserAgent       = "liuzhiquan@bilibili.com"
+	_chmUserAgent       = "caiheming@bilibili.com"
+	_ljUserAgent        = "liujin@bilibili.com"
+	//_userDmgQueryJSON   = `{"select":[],"where":{"log_date":{"in":["%s"]},"mid":{"gt":"%s"}},"sort":{"mid":1},"page":{"limit":200}}`
+	_upUserDmgQueryJSON = `{"select":[],"where":{"mid":{"gt":%d}},"sort":{"mid":1},"page":{"limit":200}}`
+	_userDmgQueryHive   = `select mid, gender, age, geo, content_tag, viewed_video, content_zone, content_count, follow_ups from sycpb.hbase_dmp_tag where last_active_date >= %s and length(viewed_video) > 0`
+	_upMidQueryHive     = `select mid from ods.ods_member_relation_stat where log_date = %s  and follower>= 10000 limit 100`
+	//_upMidQueryHive = `{"select":["name":"mid"],"where":{"log_date":{"in":["%s"]},"follower":{"gte":10000}, "pages":{"limit":10}}`
+	_basePathUserProfile      = "/tmp/"
+	_basePathUserProfileBuvid = "/data/"
+)
+
+var (
+	signParams = []string{"appKey", "timestamp", "version"}
+)
+
+// QueryPlayDaily get video play rank list from berserker
+func (d *Dao) QueryPlayDaily(c context.Context, date string) (vlist []*model.VideoHiveInfo, err error) {
+	v := make(url.Values, 8)
+	query := fmt.Sprintf(_queryJSON, date)
+	v.Set("query", query)
+	var res struct {
+		Code   int                   `json:"code"`
+		Result []model.VideoHiveInfo `json:"result"`
+	}
+
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Rankdaily, "", v, d.c.Berserker.Key.YYC, _noKickUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+	if res.Code != 200 || len(res.Result) == 0 {
+		err = ecode.NothingFound
+		log.Warn("Berserker return err, url:%s;res:%d", d.c.Berserker.API.Rankdaily+"?"+v.Encode(), res.Code)
+		return
+	}
+	for _, info := range res.Result {
+		i := info
+		vlist = append(vlist, &i)
+	}
+	return
+}
+
+//QueryOperaVideo query operation video once
+func (d *Dao) QueryOperaVideo(c context.Context, date string, ch chan<- *model.VideoHiveInfo) (err error) {
+	i := int64(0)
+	var mid int64
+	for {
+		v := make(url.Values, 8)
+		var res struct {
+			Code   int                   `json:"code"`
+			Result []model.VideoHiveInfo `json:"result"`
+		}
+		query := fmt.Sprintf(_queryJSONOper, date, i)
+		v.Set("query", query)
+		if err = d.doHTTPGet(c, d.c.Berserker.API.Operaonce, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res); err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+			return
+		}
+		if res.Code == 200 && len(res.Result) == 0 {
+			return
+		}
+		if res.Code != 200 {
+			err = ecode.NothingFound
+			log.Warn("Berserker return err, url:%s;res:%d", d.c.Berserker.API.Operaonce+"?"+v.Encode(), res.Code)
+			return
+		}
+
+		for _, info := range res.Result {
+			ch <- &info
+			mid = info.CID
+		}
+		i = mid
+	}
+}
+
+//QueryUserBasic ...
+func (d *Dao) QueryUserBasic(c context.Context) (jobURL string, err error) {
+
+	v := make(url.Values, 8)
+	var res struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+	query := "{}"
+	v.Set("query", query)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Userbasic, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+	for i, file := range res.Result {
+		query = fmt.Sprintf("{\"fileSuffix\": \"%s\"}", file)
+		v.Set("query", query)
+		bs, err := d.doHTTPGetRaw(c, d.c.Berserker.API.Userbasic, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res)
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf("/data/basic_profile/part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+		}
+
+	}
+
+	return
+}
+
+//UserProfileGet ...
+func (d *Dao) UserProfileGet(c context.Context) (jobURL []string, err error) {
+	//
+	v := make(url.Values, 8)
+	var res struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+	query := "{}"
+	v.Set("query", query)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.UserProfile, "", v, d.c.Berserker.Key.HM, _chmUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+
+	for i, file := range res.Result {
+		query = fmt.Sprintf("{\"fileSuffix\": \"/%s\"}", file)
+		//fmt.Printf("query: %v\n", query)
+		v.Set("query", query)
+
+		time.Sleep(3 * time.Second)
+		var bs []byte
+		bs, err = d.doHTTPGetRaw(c, d.c.Berserker.API.UserProfile, "", v, d.c.Berserker.Key.HM, _chmUserAgent, &res)
+
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf(_basePathUserProfile+"part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+			d.ReadLine(fmt.Sprintf(_basePathUserProfile+"part_%d", i), d.HandlerUserBbqDmg)
+			os.RemoveAll(fmt.Sprintf(_basePathUserProfile+"part_%d", i))
+		}
+
+	}
+
+	time.Sleep(3 * time.Second)
+	v2 := make(url.Values, 8)
+	var res2 struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+
+	query2 := "{}"
+	v2.Set("query2", query2)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.UserProfileBuvid, "", v2, d.c.Berserker.Key.HM, _chmUserAgent, &res2); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+
+	for i, file := range res2.Result {
+		query2 = fmt.Sprintf("{\"fileSuffix\": \"/%s\"}", file)
+		//fmt.Printf("query: %v\n", query)
+		v2.Set("query", query2)
+
+		time.Sleep(3 * time.Second)
+		bs, err := d.doHTTPGetRaw(c, d.c.Berserker.API.UserProfileBuvid, "", v2, d.c.Berserker.Key.HM, _chmUserAgent, &res2)
+
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+			d.ReadLine(fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i), d.HandlerUserBbqDmgBuvid)
+			os.RemoveAll(fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i))
+		}
+
+	}
+	return
+}
+
+// doHttpRequest make a http request for data platform api
+func (d *Dao) doHTTPGet(c context.Context, uri, realIP string, params url.Values, key *conf.BerSerkerKey, userAgent string, res interface{}) (err error) {
+
+	enc, err := d.berserkeSign(params, key)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+
+	fmt.Printf("Req: %s ", req.URL)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+
+	req.Header.Set(_userAgent, userAgent+" "+env.AppID)
+	if err != nil {
+		return
+	}
+
+	if realIP != "" {
+		req.Header.Set(_httpHeaderRemoteIP, realIP)
+	}
+
+	return d.HTTPClient.Do(c, req, res)
+}
+
+// doHTTPGetRaw make a http request for data platform api
+func (d *Dao) doHTTPGetRaw(c context.Context, uri, realIP string, params url.Values, key *conf.BerSerkerKey, userAgent string, res interface{}) (bs []byte, err error) {
+	enc, err := d.berserkeSign(params, key)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+
+	req.Header.Set(_userAgent, userAgent+" "+env.AppID)
+	if err != nil {
+		return
+	}
+
+	if realIP != "" {
+		req.Header.Set(_httpHeaderRemoteIP, realIP)
+	}
+
+	return d.HTTPClient.Raw(c, req)
+}
+
+// Sign calc appkey and appsecret sign.
+func (d *Dao) berserkeSign(params url.Values, key *conf.BerSerkerKey) (query string, err error) {
+	params.Set("appKey", key.Appkey)
+	params.Set("signMethod", "md5")
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	tmp := params.Encode()
+	signTmp := d.encode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(key.Secret)
+	b.WriteString(signTmp)
+	b.WriteString(key.Secret)
+	mh := md5.Sum(b.Bytes())
+	// query
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+// Encode encodes the values into ``URL encoded'' form
+// ("bar=baz&foo=quux") sorted by key.
+func (d *Dao) encode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range signParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}
+
+// QueryUserDmg .
+func (d *Dao) QueryUserDmg(c context.Context) (jobURL string, err error) {
+	logDay := time.Now().AddDate(0, 0, -1).Format("20060102")
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_userDmgQueryHive, logDay))
+	var res struct {
+		Code         int    `json:"code"`
+		Msg          string `json:"msg"`
+		JobStatusURL string `json:"jobStatusUrl"`
+	}
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Userdmg, "", params, d.c.Berserker.Key.HSC, _hscUserAgent, &res); err != nil {
+		return
+	}
+	if res.Code != 200 {
+		log.Error("Berserker user_dmg err(%v)", err)
+		return
+	}
+	jobURL = res.JobStatusURL
+	return
+}
+
+// QueryJobStatus 查询hive脚本执行结果
+func (d *Dao) QueryJobStatus(c context.Context, jobURL string) (urls []string, err error) {
+	var res struct {
+		Code      int      `json:"code"`
+		Msg       string   `json:"msg"`
+		StatusID  int      `json:"statusId"`
+		StatusMsg string   `json:"statusMsg"`
+		HdfsPath  []string `json:"hdfsPath"`
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, jobURL, nil)
+	if err != nil {
+		log.Error("QueryJobStatus NewRequest, err(%v)", err)
+		return
+	}
+	for {
+		if err = d.HTTPClient.Do(c, req, &res); err != nil {
+			log.Error("QueryJobStatus do get failed, joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.Code != 200 {
+			log.Error("QueryJobStatus http code error, joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.StatusID == _jobStatusDoing || res.StatusID == _jobStatusWaiting {
+			//等待1min
+			log.Info("QueryJobStatus got job status %v, joburl(%v)", res.StatusID, jobURL)
+			time.Sleep(60 * time.Second)
+			continue
+		}
+		if res.StatusID == _jobStatusFailed {
+			log.Error("QueryJobStatus got job status failed joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.StatusID == _jobStatusSuccess {
+			log.Info("QueryJobStatus got job status success joburl(%v), err(%v)", jobURL, err)
+			urls = res.HdfsPath
+			return
+		}
+		if res.StatusID != _jobStatusSuccess && res.StatusID != _jobStatusFailed && res.StatusID != _jobStatusDoing && res.StatusID != _jobStatusWaiting {
+			log.Error("QueryJobStatus got wrong job status status(%v), joburl(%v)", res.StatusID, jobURL)
+			return
+		}
+	}
+}
+
+//QueryUpUserDmg .
+func (d *Dao) QueryUpUserDmg(c context.Context, mid int64) (upUserDmg []*model.UpUserDmg, err error) {
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_upUserDmgQueryJSON, mid))
+	var res struct {
+		Code   int                `json:"code"`
+		Result []*model.UpUserDmg `json:"result"`
+	}
+
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Upuserdmg, "", params, d.c.Berserker.Key.HSC, _hscUserAgent, &res); err != nil {
+		return
+	}
+	if res.Code != 200 {
+		log.Error("Berserker up_user_dmg err(%v)", err)
+		return
+	}
+	upUserDmg = res.Result
+	return
+}
+
+//QueryUpMid .发起hive查询，取粉丝数大于1万的up mid
+func (d *Dao) QueryUpMid(c context.Context, date string) (jobURL string, err error) {
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_upMidQueryHive, date))
+	var res struct {
+		Code         int    `json:"code"`
+		Msg          string `json:"msg"`
+		JobStatusURL string `json:"jobStatusUrl"`
+	}
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Upmid, "", params, d.c.Berserker.Key.LJ, _ljUserAgent, &res); err != nil {
+		log.Error("hive QueryUpMid failed, err(%v)", err)
+		return
+	}
+	if res.Code != 200 {
+		fmt.Println(res.Code)
+		log.Error("hive QueryUpMid failed, err(%v), httpcode(%v)", err, res.Code)
+		return
+	}
+	jobURL = res.JobStatusURL
+	fmt.Println(jobURL)
+	return
+}

app/job/bbq/video/dao/cmsvideo.go

@@ -0,0 +1,175 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/job/bbq/video/model"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_updatevideostatus    = "update video set state = ? where svid = ?"
+	_updateCmsvideoStatus = "insert into cms_video (svid,cms_status,sv_status,`from`,title,pubtime,mid) values (?,?,?,?,?,?,?) on duplicate key update cms_status = values(cms_status),sv_status=values(sv_status),cms_uname = '',`from` = values(`from`),title=values(title),pubtime=values(pubtime),mid = values(mid)"
+	_selectVideoRows      = "select id,svid,title,mid,`from`,pubtime from video where state = ? and id > ? limit 1000"
+	_updateVR             = "update video_repository set state = ? where svid=?"
+)
+
+//UpdateCms ..
+func (d *Dao) UpdateCms(c context.Context, v *model.VideoRaw) (err error) {
+	if _, err = d.dbCms.Exec(c,
+		_updateCmsvideoStatus,
+		v.SVID,
+		v.State,
+		v.State,
+		v.From,
+		v.Title,
+		v.Pubtime,
+		v.MID,
+	); err != nil {
+		log.Error("DeliveryNewVdieoToCms insert cms_video err,svid : %v,err :%v", v.SVID, err)
+		return
+	}
+	return
+}
+
+//TransToCheckBack ..
+func (d *Dao) TransToCheckBack() (err error) {
+	var (
+		rows  *xsql.Rows
+		count int64
+		id    int64
+		c     = context.Background()
+	)
+	for {
+		if rows, err = d.db.Query(c, _selectVideoRows, model.VideoStPassReview, count); err != nil {
+			log.Error("DeliveryNewVdieoToCms select video failed ,err:%v", err)
+			return
+		}
+		flag := false
+		for rows.Next() {
+			videoinfo := model.VideoInfo{}
+			if err = rows.Scan(
+				&id,
+				&videoinfo.SVID,
+				&videoinfo.Title,
+				&videoinfo.MID,
+				&videoinfo.From,
+				&videoinfo.Pubtime,
+			); err != nil {
+				if err == xsql.ErrNoRows {
+					return
+				}
+				continue
+			}
+			count = id
+			//满足运营导入规则
+			if d.CmsRule(videoinfo.SVID) {
+				if _, err = d.dbCms.Exec(c,
+					_updateCmsvideoStatus,
+					videoinfo.SVID,
+					model.VideoStCheckBack,
+					model.VideoStCheckBack,
+					videoinfo.From,
+					videoinfo.Title,
+					videoinfo.Pubtime,
+					videoinfo.MID,
+				); err != nil {
+					log.Error("DeliveryNewVdieoToCms insert cms_video err,svid : %v,err :%v", videoinfo.SVID, err)
+					continue
+				}
+				if _, err = d.db.Exec(c,
+					_updatevideostatus,
+					model.VideoStCheckBack,
+					videoinfo.SVID,
+				); err != nil {
+					log.Error("DeliveryNewVdieoToCms update video status err : %v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+				if _, err = d.dbCms.Exec(c, _updateVR, model.VideoStCheckBack, videoinfo.SVID); err != nil {
+					log.Error("DeliveryNewVdieoToCms update vr err :%v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+			}
+			flag = true
+		}
+		rows.Close()
+		if !flag {
+			return
+		}
+	}
+}
+
+// CmsRule ...
+func (d *Dao) CmsRule(svid int64) (flag bool) {
+	return true
+}
+
+//TransToReview ...
+func (d *Dao) TransToReview() (err error) {
+	var (
+		rows  *xsql.Rows
+		count int64
+		id    int64
+		c     = context.Background()
+	)
+	for {
+		if rows, err = d.db.Query(c, _selectVideoRows, model.VideoStPendingPassReview, count); err != nil {
+			log.Error("TransToReview select video failed ,err:%v", err)
+			continue
+		}
+		flag := false
+		for rows.Next() {
+			videoinfo := model.VideoInfo{}
+			if err = rows.Scan(
+				&id,
+				&videoinfo.SVID,
+				&videoinfo.Title,
+				&videoinfo.MID,
+				&videoinfo.From,
+				&videoinfo.Pubtime,
+			); err != nil {
+				if err == xsql.ErrNoRows {
+					return
+				}
+				continue
+			}
+			count = id
+			//满足运营导入规则
+			if d.CmsRule(videoinfo.SVID) {
+				var st int
+				if videoinfo.From == model.VideoFromBILI || videoinfo.From == model.VideoFromCMS {
+					st = model.VideoStPassReview
+				} else {
+					st = model.VideoStPassReviewReject
+				}
+
+				if _, err = d.dbCms.Exec(c,
+					_updateCmsvideoStatus,
+					videoinfo.SVID,
+					st,
+					st,
+					videoinfo.From,
+					videoinfo.Title,
+					videoinfo.Pubtime,
+					videoinfo.MID,
+				); err != nil {
+					log.Error("TransToReview insert cms_video err,svid : %v,err :%v", videoinfo.SVID, err)
+					continue
+				}
+				if _, err = d.db.Exec(c,
+					_updatevideostatus,
+					st,
+					videoinfo.SVID,
+				); err != nil {
+					log.Error("TransToReview update video status err : %v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+			}
+			flag = true
+		}
+		rows.Close()
+		if !flag {
+			return
+		}
+	}
+}

app/job/bbq/video/dao/comment.go

@@ -0,0 +1,13 @@
+package dao
+
+import (
+	"context"
+	"go-common/library/net/metadata"
+)
+
+//ReplyReg 评论注册/冻结
+func (d *Dao) ReplyReg(c context.Context, req map[string]interface{}) (err error) {
+	ip := metadata.String(c, metadata.RemoteIP)
+	_, err = replyHTTPCommon(c, d.HTTPClient, d.c.URLs["reply_reg"], "POST", req, ip)
+	return
+}

app/job/bbq/video/dao/dao.go

@@ -0,0 +1,157 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	notice "go-common/app/service/bbq/notice-service/api/v1"
+	searchv1 "go-common/app/service/bbq/search/api/grpc/v1"
+	videov1 "go-common/app/service/bbq/video/api/grpc/v1"
+	account "go-common/app/service/main/account/api"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc/warden"
+	"net/url"
+	"reflect"
+	"strconv"
+
+	jsoniter "github.com/json-iterator/go"
+	gomail "gopkg.in/gomail.v2"
+)
+
+// Dao dao
+type Dao struct {
+	c             *conf.Config
+	redis         *redis.Pool
+	bfredis       *redis.Pool
+	db            *xsql.DB
+	dbCms         *xsql.DB
+	HTTPClient    *bm.Client
+	SearchClient  searchv1.SearchClient
+	VideoClient   videov1.VideoClient
+	AccountClient account.AccountClient
+	email         *gomail.Dialer
+	noticeClient  notice.NoticeClient
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:             c,
+		redis:         redis.NewPool(c.Redis),
+		bfredis:       redis.NewPool(c.BfRedis),
+		db:            xsql.NewMySQL(c.MySQL),
+		dbCms:         xsql.NewMySQL(c.MySQLCms),
+		HTTPClient:    bm.NewClient(c.BM.Client),
+		SearchClient:  newSearchClient(c.GRPCClient["search"]),
+		VideoClient:   newVideoClient(c.GRPCClient["video"]),
+		AccountClient: newAccountClient(c.GRPCClient["account"]),
+		email:         gomail.NewDialer(c.Mail.Host, c.Mail.Port, c.Mail.From, c.Mail.Password),
+		noticeClient:  newNoticeClient(c.GRPCClient["notice"]),
+	}
+	return
+}
+
+// newNoticeClient .
+func newNoticeClient(cfg *conf.GRPCConf) notice.NoticeClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return notice.NewNoticeClient(cc)
+}
+
+//newSearchClient .
+func newSearchClient(cfg *conf.GRPCConf) searchv1.SearchClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return searchv1.NewSearchClient(cc)
+}
+
+//newAccountClient .
+func newAccountClient(cfg *conf.GRPCConf) account.AccountClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return account.NewAccountClient(cc)
+}
+
+//newVideoClient
+func newVideoClient(cfg *conf.GRPCConf) videov1.VideoClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return videov1.NewVideoClient(cc)
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return d.db.Ping(c)
+}
+
+// BeginTran begin mysql transaction
+func (d *Dao) BeginTran(c context.Context) (*xsql.Tx, error) {
+	return d.db.Begin(c)
+}
+
+// ReplyHTTPCommon 评论公用请求
+func replyHTTPCommon(c context.Context, httpClient *bm.Client, path string, method string, data map[string]interface{}, ip string) (r []byte, err error) {
+
+	params := url.Values{}
+	t := reflect.TypeOf(data).Kind()
+	if t == reflect.Map {
+		for k, v := range data {
+			// params.Set(k, v.(string))
+			switch reflect.TypeOf(v).Kind() {
+			case reflect.Int64:
+				params.Set(k, strconv.FormatInt(v.(int64), 10))
+			case reflect.Int16:
+				params.Set(k, strconv.FormatInt(int64(v.(int16)), 10))
+			case reflect.String:
+				params.Set(k, v.(string))
+			case reflect.Int:
+				params.Set(k, strconv.FormatInt(int64(v.(int)), 10))
+			}
+		}
+	}
+	log.V(5).Infov(c, log.KV("log", fmt.Sprintf("reply req url(%s)", path+"?"+params.Encode())))
+	req, err := httpClient.NewRequest(method, path, ip, params)
+	if err != nil {
+		log.Errorv(c, log.KV("log", fmt.Sprintf("reply url(%s) error(%v)", path+"?"+params.Encode(), err)))
+		return
+	}
+	var res struct {
+		Code int             `json:"code"`
+		Msg  string          `json:"message"`
+		Data json.RawMessage `json:"data"`
+	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = httpClient.Do(c, req, &res); err != nil {
+		str, _ := json.Marshal(res)
+		log.Errorv(c, log.KV("log", fmt.Sprintf("reply ret data(%s) err[%v]", str, err)))
+		return
+	}
+	str, _ := json.Marshal(res)
+	log.V(5).Infov(c, log.KV("log", fmt.Sprintf("reply ret data(%s)", str)))
+	if res.Code != 0 {
+		err = ecode.Int(res.Code)
+		log.Warnv(c, log.KV("log", fmt.Sprintf("reply url(%s) error(%v)", path+"?"+params.Encode(), err)))
+	}
+	r = res.Data
+	return
+}

app/job/bbq/video/dao/email.go

@@ -0,0 +1,12 @@
+package dao
+
+import (
+	gomail "gopkg.in/gomail.v2"
+)
+
+// SendMail asynchronous send mail.
+func (d *Dao) SendMail(message *gomail.Message) (err error) {
+	message.SetAddressHeader("From", d.c.Mail.From, "bbq")
+	err = d.email.DialAndSend(message)
+	return
+}

app/job/bbq/video/dao/notice.go

@@ -0,0 +1,61 @@
+package dao
+
+import (
+	"context"
+	notice "go-common/app/service/bbq/notice-service/api/v1"
+	msg "go-common/app/service/bbq/sys-msg/api/v1"
+	"go-common/library/log"
+)
+
+// 通知的业务类型
+const (
+	NoticeBizTypeSv      = 1
+	NoticeBizTypeComment = 2
+	NoticeBizTypeUser    = 3
+	NoticeBizTypeSysMsg  = 4
+)
+
+// 通知类型
+const (
+	NoticeTypeLike    = 1
+	NoticeTypeComment = 2
+	NoticeTypeFan     = 3
+	NoticeTypeSysMsg  = 4
+)
+
+const (
+	_selectSQL = "select id, type, sender, receiver, jump_url, text, ctime, state from sys_msg where id > ? order by id asc"
+)
+
+// CreateNotice 创建通知
+func (d *Dao) CreateNotice(ctx context.Context, notice *notice.NoticeBase) (err error) {
+	_, err = d.noticeClient.CreateNotice(ctx, notice)
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "create notice fail: notice="+notice.String()))
+		return
+	}
+
+	log.V(10).Infov(ctx, log.KV("log", "create notice: notice="+notice.String()))
+	return
+}
+
+// GetNewSysMsg 获取未被推送的系统消息
+func (d *Dao) GetNewSysMsg(ctx context.Context, id int64) (list []*msg.SysMsg, err error) {
+
+	rows, err := d.db.Query(ctx, _selectSQL, id)
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "query mysql sys msg fail"))
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var msg msg.SysMsg
+		if err = rows.Scan(&msg.Id, &msg.Type, &msg.Sender, &msg.Receiver, &msg.JumpUrl, &msg.Text, &msg.Ctime, &msg.State); err != nil {
+			log.Errorv(ctx, log.KV("log", "scan mysql sys msg fail"))
+			return
+		}
+		list = append(list, &msg)
+	}
+	return
+}

app/job/bbq/video/dao/user.go

@@ -0,0 +1,543 @@
+package dao
+
+import (
+	"bufio"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/model"
+	video "go-common/app/service/bbq/video/api/grpc/v1"
+	acc "go-common/app/service/main/account/api"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	_userDmgCacheKey      = "bbq:user:profile:%s"
+	_userDmgCacheKeyBuvid = "bbq:device:profile:{buvid}:%s"
+
+	//_userDmgCacheKeyBbq="bbq:user:profile"
+	_userDmgCacheTimeout = 86400
+	_incrUpUserDmgSQL    = "insert into user_statistics_hive (`mid`,`uname`,`play_total`,`fan_total`,`av_total`,`like_total`) value (?,?,?,?,?,?)"
+	_updateUpUserDmgSQL  = "update user_statistics_hive set `uname` = ? , `play_total` = ? , `fan_total` = ? , `av_total` = ? , `like_total` = ? , `mtime` = ? where `mid` = ?"
+	_delUpUserDmgSQL     = "delete from user_statistics_hive where mtime < ?"
+	_insertOnDupUpDmg    = "insert into user_statistics_hive (`mid`,`uname`,`play_total`,`fan_total`,`av_total`,`like_total`) value (?,?,?,?,?,?) on duplicate key update `uname`=?,`play_total`=?,`fan_total`=?,`av_total`=?,`like_total`=?"
+	_selMidFromVideo     = "select distinct mid from video"
+	_queryUsersByLast    = "select id,mid,uname from user_base where id > ? order by id ASC limit ?"
+	_selMidFromUserBase  = "SELECT DISTINCT mid fROM user_base limit ?, 1000"
+	_upUserBase          = "UPDATE user_base SET face = ? where mid = ?"
+)
+
+//getUserDmgKey .
+func getUserDmgKey(mid string) (key string) {
+	return fmt.Sprintf(_userDmgCacheKey, mid)
+}
+
+//getUserBuvidDmgKey .
+func getUserBuvidDmgKey(buvid string) (key string) {
+	return fmt.Sprintf(_userDmgCacheKeyBuvid, buvid)
+}
+
+// InsertOnDup ...
+func (d *Dao) InsertOnDup(c context.Context, upUserDmg *model.UpUserDmg) (err error) {
+	_, err = d.db.Exec(c, _insertOnDupUpDmg, upUserDmg.MID, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes)
+	return
+}
+
+//CacheUserDmg ...
+func (d *Dao) CacheUserDmg(c context.Context, userDmg *model.UserDmg) (mid string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	var b []byte
+	if b, err = json.Marshal(userDmg); err != nil {
+		log.Error("cache user dmg marshal err(%v)", err)
+		return
+	}
+	cacheKey := getUserDmgKey(userDmg.MID)
+	fmt.Println(cacheKey)
+	if _, err = conn.Do("SET", cacheKey, b, "EX", _userDmgCacheTimeout); err != nil {
+		log.Error("cache user dmg redis set err(%v)", err)
+		return
+	}
+	return
+}
+
+//CacheUserBbqDmg ...
+func (d *Dao) CacheUserBbqDmg(c context.Context, userBbqDmg *model.UserBbqDmg) (mid string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	tag2 := strings.Join(userBbqDmg.Tag2, ",")
+	tag3 := strings.Join(userBbqDmg.Tag3, ",")
+	up := strings.Join(userBbqDmg.Up, ",")
+	cacheKey := getUserDmgKey(userBbqDmg.MID)
+
+	if err = conn.Send("HSET", cacheKey, "zone", tag2); err != nil {
+		log.Error("cache user bbq dmg redis set tag2 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "tag", tag3); err != nil {
+		log.Error("cache user bbq dmg redis set tag3 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "up", up); err != nil {
+		log.Error("cache user bbq dmg redis set up err(%v)", err)
+		return
+	}
+
+	return
+}
+
+//CacheUserBbqDmgBuvid ...
+func (d *Dao) CacheUserBbqDmgBuvid(c context.Context, userBbqDmgBuvid *model.UserBbqBuvidDmg) (Buvid string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	tag2 := strings.Join(userBbqDmgBuvid.Tag2, ",")
+	tag3 := strings.Join(userBbqDmgBuvid.Tag3, ",")
+	up := strings.Join(userBbqDmgBuvid.Up, ",")
+	cacheKey := getUserBuvidDmgKey(userBbqDmgBuvid.Buvid)
+
+	if err = conn.Send("HSET", cacheKey, "zone", tag2); err != nil {
+		log.Error("cache user bbq buvid dmg redis set tag2 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "tag", tag3); err != nil {
+		log.Error("cache user bbq buvid dmg redis set tag3 err(%v)", err)
+		return
+	}
+
+	if err = conn.Send("HSET", cacheKey, "up", up); err != nil {
+		log.Error("cache user bbq buvid dmg redis set up err(%v)", err)
+		return
+	}
+
+	return
+}
+
+// AddUpUserDmg .
+func (d *Dao) AddUpUserDmg(c context.Context, upUserDmg *model.UpUserDmg) (num int64, err error) {
+	var res sql.Result
+	if res, err = d.db.Exec(c, _incrUpUserDmgSQL, upUserDmg.MID, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes); err != nil {
+		return 0, err
+	}
+	return res.LastInsertId()
+}
+
+// UpdateUpUserDmg .
+func (d *Dao) UpdateUpUserDmg(c context.Context, upUserDmg *model.UpUserDmg) (num int64, err error) {
+	t := time.Now().AddDate(0, 0, 0).Format("2006-01-02 15:04:05")
+	var res sql.Result
+	if res, err = d.db.Exec(c, _updateUpUserDmgSQL, upUserDmg.Uname, upUserDmg.Play, upUserDmg.Fans, upUserDmg.AVs, upUserDmg.Likes, t, upUserDmg.MID); err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
+// DelUpUserDmg .
+func (d *Dao) DelUpUserDmg(c context.Context) (num int64, err error) {
+	t := time.Unix(time.Now().Unix(), -int64(36*time.Hour)).Format("2006-01-02 15:04:05")
+	var res sql.Result
+	if res, err = d.db.Exec(c, _delUpUserDmgSQL, t); err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
+//Download 下载文件
+func (d *Dao) Download(url string, name string) (fpath string, err error) {
+	if name == "" {
+		u := strings.Split(url, "/")
+		l := len(u)
+		name = u[l-1]
+	}
+	t := time.Now().AddDate(0, 0, 0).Format("20060102")
+	path := conf.Conf.Download.File + t
+	err = d.CreateDir(path)
+	if err != nil {
+		log.Error("create dir(%s) err(%v)", path, err)
+		return
+	}
+	fpath = path + "/" + name
+	newFile, err := os.Create(fpath)
+
+	if err != nil {
+		log.Error("create path(%s) err(%v)", fpath, err)
+		return
+	}
+	defer newFile.Close()
+
+	client := http.Client{}
+	resp, err := client.Get(url)
+	if err != nil {
+		log.Error("download url(%s) err(%v)", url, err)
+		return
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(newFile, resp.Body)
+	if err != nil {
+		log.Error("copy err(%v)", err)
+		return
+	}
+	return
+}
+
+//CreateDir 创建文件夹
+func (d *Dao) CreateDir(path string) (err error) {
+	_, err = os.Stat(path)
+	defer func() {
+		if os.IsExist(err) {
+			err = nil
+		}
+	}()
+	if os.IsNotExist(err) {
+		err = os.Mkdir(path, os.ModePerm)
+	}
+	return
+}
+
+// ReadLine 按行读取文件，hander回调
+func (d *Dao) ReadLine(path string, handler func(string)) (err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Error("open path(%s) err(%v)", path, err)
+		return
+	}
+	defer f.Close()
+	buf := bufio.NewReader(f)
+	for {
+
+		line, err := buf.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			log.Error("read path(%s) err(%v)", path, err)
+			return nil
+		}
+		line = strings.TrimSpace(line)
+		handler(line)
+		time.Sleep(time.Duration(1) * time.Second)
+	}
+}
+
+// ReadLines 50条发起一次grpc请求
+func (d *Dao) ReadLines(path string, handler func([]int64)) (err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Error("ReadLine open path(%s) err(%v)", path, err)
+		return
+	}
+	defer f.Close()
+	buf := bufio.NewReader(f)
+	mids := make([]int64, 0, 50)
+	i := 0
+	for {
+		line, err := buf.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			}
+			log.Error("read path(%s) err(%v)", path, err)
+			break
+		}
+		mid, _ := strconv.ParseInt(strings.TrimSpace(line), 10, 64)
+		mids = append(mids, mid)
+		i++
+		if i == 50 {
+			handler(mids)
+			mids = make([]int64, 0, 50)
+			i = 0
+			time.Sleep(time.Duration(1) * time.Second)
+		}
+	}
+	if len(mids) != 0 {
+		handler(mids)
+	}
+	return
+}
+
+//HandlerUserDmg mid, gender, age, geo, content_tag, viewed_video, content_zone, content_count, follow_ups
+func (d *Dao) HandlerUserDmg(user string) {
+	u := strings.Split(user, "\u0001")
+	userDmg := &model.UserDmg{
+		MID:          u[0],
+		Gender:       u[1],
+		Age:          u[2],
+		Geo:          u[3],
+		ContentTag:   u[4],
+		ViewedVideo:  d.HandlerViewedVideo(u[5]),
+		ContentZone:  u[6],
+		ContentCount: u[7],
+		FollowUps:    u[8],
+	}
+	d.CacheUserDmg(context.Background(), userDmg)
+}
+
+//HandlerUserBbqDmg ..
+func (d *Dao) HandlerUserBbqDmg(user string) {
+	u := strings.Split(user, ",")
+	userBbqDmg := &model.UserBbqDmg{
+		MID:  u[0],
+		Tag2: strings.Split(u[1], "\u0002"),
+		Tag3: strings.Split(u[2], "\u0002"),
+		Up:   strings.Split(u[3], "\u0002"),
+	}
+	d.CacheUserBbqDmg(context.Background(), userBbqDmg)
+}
+
+//HandlerUserBbqDmgBuvid ..
+func (d *Dao) HandlerUserBbqDmgBuvid(user string) {
+	u := strings.Split(user, ",")
+	UserBbqBuvidDmg := &model.UserBbqBuvidDmg{
+		Buvid: u[0],
+		Tag2:  strings.Split(u[1], "\u0002"),
+		Tag3:  strings.Split(u[2], "\u0002"),
+		Up:    strings.Split(u[3], "\u0002"),
+	}
+	d.CacheUserBbqDmgBuvid(context.Background(), UserBbqBuvidDmg)
+}
+
+// HandlerMids update userbase by mids
+func (d *Dao) HandlerMids(mids []int64) {
+	res, err := d.VideoClient.SyncUserStas(context.Background(), &video.SyncMidsRequset{MIDS: mids})
+	if err != nil {
+		log.Error("userbases update failes, mids(%v), err(%v)", mids, err)
+		return
+	}
+	log.Info("userbases update success, affected %v rows", res.Affc)
+}
+
+// HandlerMid update userbase by mid
+func (d *Dao) HandlerMid(s string) {
+	mid, _ := strconv.ParseInt(s, 10, 64)
+	res, err := d.VideoClient.SyncUserSta(context.Background(), &video.SyncMidRequset{MID: mid})
+	if err != nil {
+		log.Error("userbase update failes, mid(%v), err(%v)", mid, err)
+		return
+	}
+	if res.Affc == 1 {
+		log.Info("userbase insert success ,mid(%v)", mid)
+	} else if res.Affc == 2 {
+		log.Info("userbase update success , mid(%v)", mid)
+	}
+}
+
+//HandlerViewedVideo 处理看过的视频，保存最近看过的100个
+func (d *Dao) HandlerViewedVideo(v string) (res map[int64]string) {
+	res = make(map[int64]string)
+	var vv [][]interface{}
+	var dd string
+
+	err := json.Unmarshal([]byte(v), &vv)
+	if err != nil {
+		return
+	}
+	l := len(vv)
+	n := 1
+	for i := l - 1; i >= 0; i-- {
+		for _, a := range vv[i] {
+			switch b := a.(type) {
+			case string:
+				dd = b
+			case []interface{}:
+				ll := len(b)
+				for j := ll - 1; j >= 0; j-- {
+					switch c := b[j].(type) {
+					case float64:
+						k := int64(c)
+						if _, ok := res[k]; !ok {
+							res[k] = dd
+							n++
+						}
+					}
+					if n > 100 {
+						return
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+// SelMidFromVideo get distinct mid list from table video
+func (d *Dao) SelMidFromVideo() (mids []int64, err error) {
+	rows, err := d.db.Query(context.Background(), _selMidFromVideo)
+	if err != nil {
+		log.Error("SelMidFromVideo failed, err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var s string
+		if err = rows.Scan(&s); err != nil {
+			panic(err.Error())
+		}
+		var mid int64
+		if mid, err = strconv.ParseInt(s, 10, 64); err != nil {
+			log.Error("strconv.ParseInt(%s) error(%v)", s, err)
+			return
+		}
+		mids = append(mids, mid)
+	}
+	return
+}
+
+//MergeUpInfo merge up info
+func (d *Dao) MergeUpInfo(mid int64) (err error) {
+	var (
+		ctx    = context.Background()
+		params = url.Values{}
+		req    = &http.Request{}
+		id     int64
+		res    struct {
+			Code int
+			Data model.UpUserInfoRes
+		}
+	)
+	err = d.db.QueryRow(ctx, "select mid from user_base where mid = ?", mid).Scan(&id)
+	if err == nil {
+		log.Infow(ctx, "log", "already has mid in user_base", "mid", mid)
+		return
+	}
+	if err == sql.ErrNoRows {
+		params.Set("mid", strconv.FormatInt(mid, 10))
+		req, err = d.HTTPClient.NewRequest("GET", d.c.URLs["account"], "", params)
+		if err != nil {
+			log.Error("MergeUpInfo error(%v)", err)
+			return
+		}
+		if err = d.HTTPClient.Do(ctx, req, &res); err != nil {
+			log.Error("MergeUpInfo http req failed ,err:%v", err)
+			return
+		}
+		res := res.Data
+		var sex int
+		switch res.Sex {
+		case "男":
+			sex = 1
+		case "女":
+			sex = 2
+		default:
+			sex = 3
+		}
+		_, err = d.db.Exec(ctx,
+			"insert into user_base (mid,uname,face,sex,user_type,complete_degree)values(?,?,?,?,?,?)",
+			res.MID,
+			res.Name,
+			res.Face,
+			sex,
+			model.UserTypeUp,
+			0)
+		if err != nil {
+			log.Error("MergeUpInfo insert upinfo failed,err:%v", err)
+			return
+		}
+	} else {
+		log.Error("MergeUpInfo query sql failed,err:%v", err)
+	}
+	if err = d.db.QueryRow(ctx, "select id from user_statistics where mid = ?", mid).Scan(&id); err != nil {
+		if err == sql.ErrNoRows {
+			if _, err = d.db.Exec(ctx, "insert into user_statistics (mid) values (?)", mid); err != nil {
+				log.Error("init insert user_statistics failed,err:%v", err)
+			}
+		} else {
+			log.Error("init query user_statistics failed,err:%v", err)
+		}
+	}
+	return
+}
+
+//UsersByLast 使用lastid批量获取用户
+func (d *Dao) UsersByLast(c context.Context, lastid int64) (r []*model.UserBaseDB, err error) {
+	var rows *xsql.Rows
+	rows, err = d.db.Query(c, _queryUsersByLast, lastid, _limitSize)
+	if err != nil {
+		log.Error("db _queryVideos err(%v)", err)
+		return
+	}
+	for rows.Next() {
+		u := new(model.UserBaseDB)
+		if err = rows.Scan(&u.ID, &u.MID, &u.Uname); err != nil {
+			log.Error("scan err(%v)", err)
+			continue
+		}
+		r = append(r, u)
+	}
+	return
+}
+
+// SelMidFromUserBase get distinct mid list from table user_base
+func (d *Dao) SelMidFromUserBase(start int) (mids []int64, err error) {
+	var mid int64
+	rows, err := d.db.Query(context.Background(), _selMidFromUserBase, start)
+	if err != nil {
+		log.Error("SelMidFromUserBase failed, err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var s string
+		if err = rows.Scan(&s); err != nil {
+			panic(err.Error())
+		}
+		if mid, err = strconv.ParseInt(s, 10, 64); err != nil {
+			log.Error("strconv.ParseInt(%s) error(%v)", s, err)
+			return
+		}
+		mids = append(mids, mid)
+	}
+	return
+}
+
+// UpUserBases 根据mids更新用户基本信息
+func (d *Dao) UpUserBases(c context.Context, mids []int64) (err error) {
+	var (
+		tx *xsql.Tx
+	)
+	midsReq := &acc.MidsReq{
+		Mids:   mids,
+		RealIp: metadata.String(c, metadata.RemoteIP)}
+	infosReply, err := d.AccountClient.Infos3(c, midsReq)
+	if infosReply == nil {
+		log.Error("查询infos3失败,err（%v）", err)
+		fmt.Printf("查询infos3失败,err（%v）", err)
+		return
+	}
+	if tx, err = d.BeginTran(c); err != nil {
+		log.Error("begin transaction error(%v)", err)
+		return
+	}
+	for _, info := range infosReply.Infos {
+		if info.Mid != 0 {
+			if len(info.Face) > 255 {
+				info.Face = "http://i0.hdslb.com/bfs/bbq/video-image/userface/1558868601542006937.png"
+			}
+			for try := 0; try < 3; try++ {
+				if _, err = tx.Exec(_upUserBase, info.Face, info.Mid); err == nil {
+					break
+				}
+			}
+			if err != nil {
+				log.Error("mid(%v) update failed", info.Mid)
+			}
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("UpUserBases commit failed err(%v)", err)
+	}
+	return
+}

app/job/bbq/video/dao/video.go

@@ -0,0 +1,524 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strconv"
+	"strings"
+	xtime "time"
+
+	"go-common/app/job/bbq/video/model"
+	"go-common/app/service/bbq/search/api/grpc/v1"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/time"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+const (
+	_limitSize = 2000
+	// MaxSyncESNum 限制每次更新到es的数量
+	MaxSyncESNum = 100
+	// QueryVideoByMtime 根据mtime获取视频基础信息
+	QueryVideoByMtime = "select `svid`,`mtime` from video where mtime >= ? order by mtime asc"
+	// QueryVideoStatisticsByMtime 根据mtime获取视频相关播放信息
+	QueryVideoStatisticsByMtime = "select `svid`,`mtime` from video_statistics where mtime >= ? order by mtime asc"
+	// QueryVideoStatisticsHiveByMtime 根据mtime获取视频主站信息
+	QueryVideoStatisticsHiveByMtime = "select `svid`,`mtime` from video_statistics_hive where mtime >= ? order by mtime asc"
+	// QueryVideoTagByMtime 根据mtime获取视频tag信息
+	QueryVideoTagByMtime           = "select `svid`,`mtime` from video_tag where mtime >= ? order by mtime asc"
+	_recRecallOpVideoKey           = "job:bbq:rec:op"
+	_syncOperVideoTagKey           = "job:bbq:video:syncvideotagkey"
+	_syncOperVideoTimeKey          = "job:bbq:video:syncvideotimekey"
+	_selectVideoInfo               = "select `svid`,`title`,`content`,`mid`,`cid`,`pubtime`,`ctime`,`mtime`,`duration`,`original`,`state`,`is_full_screen`,`ver_id`,`ver`,`from`,`avid`,`tid`,`sub_tid`,`score` from video where id > ? order by id asc limit 100"
+	_selectVideoInfoByIDs          = "select `svid`,`title`,`content`,`mid`,`cid`,`pubtime`,`ctime`,`mtime`,`duration`,`original`,`state`,`is_full_screen`,`ver_id`,`ver`,`from`,`avid`,`tid`,`sub_tid`,`score` from video where svid in (%s)"
+	_selectVideoStatisticsHiveInfo = "select `svid`,`play`,`fav`,`coin`,`subtitles`,`likes`,`share`,`report`,`duration_daily`,`duration_all`,`reply`,`share_daily`,`play_daily`,`subtitles_daily`,`likes_daily`,`fav_daily`,`access`,`reply_daily` from video_statistics_hive where svid in (%s)"
+	_selectVideoStatisticsInfo     = "select `svid`,`play`,`subtitles`,`like`,`share`,`report` from video_statistics where svid in (%s)"
+	_selectVideoTagsInfo           = "select v.svid,t.id,t.name,t.type from video_tag v inner join tag t on v.tag_id = t.id where v.svid in (%s)"
+	_queryCheckTask                = "select `task_id`,`task_name`,`last_check` from check_task where `task_name` = ?"
+	_updateTaskLastCheck           = "update check_task set last_check = ? where `task_name` = ?"
+	_queryTagByMtime               = "select `id`,`mtime` from tag where mtime > ? order by mtime asc limit 10"
+	_queryVideoTagByTagID          = "select `id`,`svid` from video_tag where tag_id in (%s) and id > ? order by id asc limit 100"
+	_queryVideoBySVIDs             = "select `svid`,`title` from video where svid in (%s)"
+	_queryIDs                      = "select `id`,`svid` from video where id > %d order by id asc limit %d"
+	_queryOutPutVideos             = "select id,svid,title,pubtime from video where id > ? and state in (%s) order by id ASC limit ?"
+	_updateUVSt                    = "update user_statistics set %s = %s+1 where mid = %d"
+	_updateUVStDel                 = "update user_statistics set %s = %s-1 where mid = %d and %s >0"
+	_updateSVTotal                 = "update user_statistics set av_total = av_total + 1 where mid = ?"
+	_getSvidByCid                  = "select svid from video where cid = ?"
+	_updateSVID                    = "update video_repository set svid = ? where id = ?"
+	_updateSyncStatus              = "update video_repository set sync_status = sync_status|? where svid = ?"
+	_queryCMSOne                   = "select `home_img_url`,`home_img_width`,`home_img_height`,`from`,`sync_status`, `tag`, `avid`, `cid`, `svid`, `title`, `mid`, `content`, `pubtime`,`duration`,`original`,`is_full_screen`,`tid`,`sub_tid`,`cover_url`,`cover_width`,`cover_height` from video_repository where svid = ?"
+	_queryCMSOneByID               = "select `tag`, `avid`, `cid`, `svid`, `title`, `mid`, `content`, `pubtime`,`duration`,`original`,`is_full_screen`,`tid`,`sub_tid`,`cover_url`,`cover_width`,`cover_height` from video_repository where id = ?"
+	_queryBbqVideo                 = "select title,mid,cid,state,tid,sub_tid,svid from video where svid in (%s)"
+)
+
+// RawVideo 从数据库获取视频信息
+func (d *Dao) RawVideo(ctx context.Context, SVID int64) (res *model.VideoRepRaw, err error) {
+	res = new(model.VideoRepRaw)
+	err = d.dbCms.QueryRow(ctx, _queryCMSOne, SVID).Scan(&res.HomeImgURL, &res.CoverWidth, &res.HomeImgHeight, &res.From, &res.SyncStatus, &res.Tag, &res.AVID, &res.CID, &res.SVID, &res.Title, &res.MID, &res.Content, &res.Pubtime, &res.Duration, &res.Original, &res.IsFull, &res.TID, &res.SubTID, &res.CoverURL, &res.CoverWidth, &res.CoverHeight)
+	if err != nil {
+		log.Error("RawVideo error(%v),svid:%d", err, SVID)
+	}
+	return
+}
+
+//RawBbqVideo ..
+func (d *Dao) RawBbqVideo(ctx context.Context, SVID []int64) (res *model.VideoRaw, err error) {
+	res = new(model.VideoRaw)
+
+	svids := strings.Trim(strings.Join(strings.Split(fmt.Sprint(SVID), " "), ","), "[]")
+	err = d.db.QueryRow(ctx, fmt.Sprintf(_queryBbqVideo, svids)).Scan(&res.Title, &res.MID, &res.CID, &res.State, &res.TID, &res.SubTID, &res.SVID)
+	if err != nil {
+		log.Errorw(ctx, "event", "RawBbqVideo queryrow scan err:[%v],SVID :[%v]", err, SVID)
+	}
+	return
+
+}
+
+//RawVideoByID ...get video info by id
+func (d *Dao) RawVideoByID(ctx context.Context, ID int64) (res *model.VideoRepRaw, err error) {
+	res = new(model.VideoRepRaw)
+	err = d.dbCms.QueryRow(ctx, _queryCMSOneByID, ID).Scan(&res.Tag, &res.AVID, &res.CID, &res.SVID, &res.Title, &res.MID, &res.Content, &res.Pubtime, &res.Duration, &res.Original, &res.IsFull, &res.TID, &res.SubTID, &res.CoverURL, &res.CoverWidth, &res.CoverHeight)
+	if err != nil {
+		log.Error("RawVideoByID error(%v),id:%d", err, ID)
+	}
+	return
+}
+
+//UpdateSyncStatus update video_repository sync_status
+func (d *Dao) UpdateSyncStatus(ctx context.Context, SVID int64, st int64) (err error) {
+	if _, err = d.dbCms.Exec(ctx, _updateSyncStatus, st, SVID); err != nil {
+		log.Error("UpdateSyncStatus err :%v,svid :", err, SVID)
+	}
+	return
+}
+
+//UpdateVideoUploadProcessStatus ...
+func (d *Dao) UpdateVideoUploadProcessStatus(ctx context.Context, SVID int64, st int64) (err error) {
+	if _, err = d.db.Exec(ctx, "update video_upload_process set status = ? where svid = ?", st, SVID); err != nil {
+		log.Errorw(ctx, "errmsg", "UpdateVideoUploadProcessStatus update failed", "err", err)
+	}
+	return
+}
+
+//UpdateSvid ...
+func (d *Dao) UpdateSvid(c context.Context, id int64, svid int64) (err error) {
+	if _, err = d.dbCms.Exec(c, _updateSVID, svid, id); err != nil {
+		log.Error("distribution svid err:%v,svid:%d", err, svid)
+	}
+	return
+}
+
+//AddSVTotal ...
+func (d *Dao) AddSVTotal(mid int64) (err error) {
+	_, err = d.db.Exec(context.Background(), _updateSVTotal, mid)
+	if err != nil {
+		log.Error("AddSVTotal ,mid:%s,err:%v", mid, err)
+	}
+	return
+}
+
+//UpdateUVSt 更新用户视频统计信息
+func (d *Dao) UpdateUVSt(mid int64, field string) (err error) {
+	_, err = d.db.Exec(context.Background(), fmt.Sprintf(_updateUVSt, field, field, mid))
+	if err != nil {
+		log.Error("UpdateUVSt ,mid:%s,field:%s,op:%d,err:%v", mid, field, err)
+	}
+	return
+}
+
+//UpdateUVStDel ...
+func (d *Dao) UpdateUVStDel(mid int64, field string) (err error) {
+	_, err = d.db.Exec(context.Background(), fmt.Sprintf(_updateUVStDel, field, field, mid, field))
+	if err != nil {
+		log.Error("UpdateUVStDel ,mid:%s,field:%s,op:%d,err:%v", mid, field, err)
+	}
+	return
+}
+
+//VideoList 获取视频基础信息
+func (d *Dao) VideoList(c context.Context, id int64) (ids string, res []*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _selectVideoInfo, id); err != nil {
+		log.Error("select videos err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		pubtime  time.Time
+		ctime    time.Time
+		mtime    time.Time
+		idstring []string
+	)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.Title, &tmp.Content, &tmp.MID, &tmp.CID, &pubtime, &ctime, &mtime, &tmp.Duration, &tmp.Original, &tmp.State, &tmp.ISFullScreen, &tmp.VerID, &tmp.Ver, &tmp.From, &tmp.AVID, &tmp.Tid, &tmp.SubTid, &tmp.Score); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			return
+		}
+		tmp.Pubtime = int64(pubtime)
+		tmp.Ctime = int64(ctime)
+		tmp.Mtime = int64(mtime)
+		idstring = append(idstring, strconv.FormatInt(tmp.SVID, 10))
+		res = append(res, tmp)
+	}
+	ids = strings.Join(idstring, ",")
+	return
+}
+
+//VideoListByIDs 根据视频id获取视频基础信息
+func (d *Dao) VideoListByIDs(c context.Context, ids string) (res []*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoInfoByIDs, ids)); err != nil {
+		log.Error("select videos by ids err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		pubtime time.Time
+		ctime   time.Time
+		mtime   time.Time
+	)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.Title, &tmp.Content, &tmp.MID, &tmp.CID, &pubtime, &ctime, &mtime, &tmp.Duration, &tmp.Original, &tmp.State, &tmp.ISFullScreen, &tmp.VerID, &tmp.Ver, &tmp.From, &tmp.AVID, &tmp.Tid, &tmp.SubTid, &tmp.Score); err != nil {
+			log.Error("select videos scan err(%v)", err)
+			return
+		}
+		tmp.Pubtime = int64(pubtime)
+		tmp.Ctime = int64(ctime)
+		tmp.Mtime = int64(mtime)
+		res = append(res, tmp)
+	}
+	return
+}
+
+//VideoStatisticsHiveList 获取视频互动信息，hive表
+func (d *Dao) VideoStatisticsHiveList(c context.Context, ids string) (res map[int64]*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoStatisticsHiveInfo, ids)); err != nil {
+		log.Error("select video statistics hive err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[int64]*v1.VideoESInfo)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.PlayHive, &tmp.FavHive, &tmp.CoinHive, &tmp.SubtitlesHive, &tmp.LikesHive, &tmp.ShareHive, &tmp.ReportHive, &tmp.DurationDailyHive, &tmp.DurationAllHive, &tmp.ReplyHive, &tmp.ShareDailyHive, &tmp.PlayDailyHive, &tmp.SubtitlesDailyHive, &tmp.LikesDailyHive, &tmp.FavDailyHive, &tmp.AccessHive, &tmp.ReplyDailyHive); err != nil {
+			log.Error("select video statistics hive scan err(%v)", err)
+			return
+		}
+		res[tmp.SVID] = tmp
+	}
+	return
+}
+
+//VideoStatisticsList 获取视频互动信息
+func (d *Dao) VideoStatisticsList(c context.Context, ids string) (res map[int64]*v1.VideoESInfo, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoStatisticsInfo, ids)); err != nil {
+		log.Error("select video statistics err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[int64]*v1.VideoESInfo)
+	for rows.Next() {
+		tmp := new(v1.VideoESInfo)
+		if err = rows.Scan(&tmp.SVID, &tmp.Play, &tmp.SubtitlesHive, &tmp.Like, &tmp.Share, &tmp.Report); err != nil {
+			log.Error("select video statistics scan err(%v)", err)
+			return
+		}
+		res[tmp.SVID] = tmp
+	}
+	return
+}
+
+//VideoTagsList 获取视频tags
+func (d *Dao) VideoTagsList(c context.Context, ids string) (res map[int64][]*v1.VideoESTags, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_selectVideoTagsInfo, ids)); err != nil {
+		fmt.Println(rows)
+		log.Error("select video statistics err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var svid int64
+	res = make(map[int64][]*v1.VideoESTags)
+	for rows.Next() {
+		tmp := new(v1.VideoESTags)
+		if err = rows.Scan(&svid, &tmp.ID, &tmp.Name, &tmp.Type); err != nil {
+			log.Error("select video statistics scan err(%v)", err)
+			return
+		}
+		res[svid] = append(res[svid], tmp)
+	}
+	return
+}
+
+//RawCheckTask 查询脚本任务
+func (d *Dao) RawCheckTask(c context.Context, taskName string) (res *model.CheckTask, err error) {
+	res = new(model.CheckTask)
+	raw := d.db.QueryRow(c, _queryCheckTask, taskName)
+	if err = raw.Scan(&res.TaskID, &res.TaskName, &res.LastCheck); err != nil {
+		log.Error("query check task name(%s) err(%v)", taskName, err)
+	}
+	return
+}
+
+//UpdateTaskLastCheck 更新上次执行时间
+func (d *Dao) UpdateTaskLastCheck(c context.Context, taskName string, lastCheck int64) (num int64, err error) {
+	res, err := d.db.Exec(c, _updateTaskLastCheck, lastCheck, taskName)
+	if err != nil {
+		log.Error("update task last check name(%s)(%d) err(%v)", taskName, lastCheck, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// RawGetIDByMtime 获取最近更新的那些svid，该函数可以用于多个表的查询，只需传入不同表的查询语句即可
+func (d *Dao) RawGetIDByMtime(baseTableQuery string, mtime int64) (ids []int64, lastMtime int64, err error) {
+	mtimeStr := xtime.Unix(mtime, 0).Format("2006-01-02 15:04:05")
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(context.Background(), baseTableQuery, mtimeStr); err != nil {
+		log.Error("select ids fail: err=%v, mtime=%s, sql=%s", err, mtimeStr, baseTableQuery)
+		return
+	}
+	defer rows.Close()
+	var (
+		temp time.Time
+		svid int64
+	)
+	for rows.Next() {
+		if err = rows.Scan(&svid, &temp); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			log.Error("select videos by mtime scan fail: err=%v, mtime=%s, sql=%s", err, mtimeStr, baseTableQuery)
+			return
+		}
+		lastMtime = int64(temp)
+		ids = append(ids, svid)
+	}
+	return
+}
+
+//RawTagByMtime 根据mtime获取tag信息
+func (d *Dao) RawTagByMtime(c context.Context, mtime int64) (ids string, res int64, err error) {
+	str := xtime.Unix(mtime, 0).Format("2006-01-02 15:04:05")
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _queryTagByMtime, str); err != nil {
+		log.Error("select tag err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		temp     time.Time
+		id       int64
+		idstring []string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&id, &temp); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			log.Error("select tag by mtime scan err(%v)", err)
+			return
+		}
+		res = int64(temp)
+		idstring = append(idstring, strconv.FormatInt(id, 10))
+	}
+	ids = strings.Join(idstring, ",")
+	return
+}
+
+//RawVideoTagByIDs .
+func (d *Dao) RawVideoTagByIDs(c context.Context, ids string, id int64) (svids string, res int64, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_queryVideoTagByTagID, ids), id); err != nil {
+		log.Error("select video tag err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		temp     int64
+		svid     int64
+		idstring []string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&temp, &svid); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			} else {
+				log.Error("select video tag by id scan err(%v)", err)
+			}
+			return
+		}
+		res = int64(temp)
+		idstring = append(idstring, strconv.FormatInt(svid, 10))
+	}
+	svids = strings.Join(idstring, ",")
+	return
+}
+
+//GetSyncOperVideoFlag 获取同步信号灯
+func (d *Dao) GetSyncOperVideoFlag(c context.Context) (tag int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	val, err := conn.Do("get", _syncOperVideoTagKey)
+	if err != nil {
+		log.Error("cache sync oper video tag get err(%v)", err)
+		return
+	}
+	if val == nil {
+		if err = d.SetSyncOperVideoFlag(c, model.DenySyncOperVideoTag); err != nil {
+			log.Error("set sync oper video flag faild")
+			return
+		}
+		tag = model.DenySyncOperVideoTag
+	} else {
+		tag, err = redis.Int64(val, err)
+	}
+	return
+}
+
+//SetSyncOperVideoFlag 设置同步信号灯
+func (d *Dao) SetSyncOperVideoFlag(c context.Context, v int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("set", _syncOperVideoTagKey, v); err != nil {
+		log.Error("cache sync oper video tag set err(%v)", err)
+		return
+	}
+	return
+}
+
+//GetSyncOperVideoExportTime ...
+func (d *Dao) GetSyncOperVideoExportTime(c context.Context) (t string, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	val, err := conn.Do("get", _syncOperVideoTimeKey)
+	if err == nil {
+		if val == nil {
+			conn.Do("set", _syncOperVideoTimeKey, "")
+		}
+		t, err = redis.String(val, err)
+	} else {
+		log.Error("get sync oper video export time err,errinfo:%v", err)
+	}
+	return
+}
+
+//RawVideoBySVIDS 根据svids获取视频
+func (d *Dao) RawVideoBySVIDS(c context.Context, svids []string) (res map[int64]string, err error) {
+	res = make(map[int64]string)
+	str := strings.Join(svids, ",")
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, fmt.Sprintf(_queryVideoBySVIDs, str)); err != nil {
+		log.Error("select videos by svids err(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var (
+		svid  int64
+		title string
+	)
+	for rows.Next() {
+		if err = rows.Scan(&svid, &title); err != nil {
+			if err == sql.ErrNoRows {
+				err = nil
+			}
+			log.Error("select videos by svids scan err(%v)", err)
+			return
+		}
+		res[svid] = title
+	}
+	return
+}
+
+//GetVideoByLastID 获取所有SVID
+func (d *Dao) GetVideoByLastID(c context.Context, last int64) (IDs []int64, lastRet int64, err error) {
+	length := 1000 //分批大小
+	var rows *xsql.Rows
+	rows, err = d.db.Query(c, fmt.Sprintf(_queryIDs, last, length))
+	if err != nil {
+		log.Error("db _queryIDs err(%v)", err)
+		return
+	}
+	for rows.Next() {
+		var svid int64
+		if err = rows.Scan(&lastRet, &svid); err != nil {
+			log.Error("scan err(%v)", err)
+			continue
+		}
+		IDs = append(IDs, svid)
+	}
+	return
+}
+
+//GetRecallOpVideo 获取精选视频
+func (d *Dao) GetRecallOpVideo(c context.Context) (ids []int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	val, err := redis.Bytes(conn.Do("GET", _recRecallOpVideoKey))
+	if err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("cache rec recall op video get redis err (%v)", err)
+		}
+		return ids, err
+	}
+	if err = jsoniter.Unmarshal(val, &ids); err != nil {
+		log.Error("rec recall op video unmarshal err (%v)", err)
+	}
+	return
+}
+
+//SetRecallOpVideo 写入精选视频
+func (d *Dao) SetRecallOpVideo(c context.Context, ids []int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	bytes, _ := jsoniter.Marshal(ids)
+	_, err = conn.Do("SET", _recRecallOpVideoKey, bytes)
+	if err != nil {
+		log.Error("rec recall op video set redis error(%v) ", err)
+	}
+	return
+}
+
+//VideosByLast 使用lastid批量获取视屏
+func (d *Dao) VideosByLast(c context.Context, lastid int64) (svinfo []*model.VideoDB, err error) {
+	var rows *xsql.Rows
+	query := fmt.Sprintf(_queryOutPutVideos, model.VideoStateOutPut)
+	rows, err = d.db.Query(c, query, lastid, _limitSize)
+	if err != nil {
+		log.Error("db _queryVideos err(%v)", err)
+		return
+	}
+	for rows.Next() {
+		video := new(model.VideoDB)
+		if err = rows.Scan(&video.AutoID, &video.ID, &video.Title, &video.Pubtime); err != nil {
+			log.Error("scan err(%v)", err)
+			continue
+		}
+		svinfo = append(svinfo, video)
+	}
+	return
+}
+
+// GetSvidByCid 根据cid获取svid
+func (d *Dao) GetSvidByCid(c context.Context, cid int64) (svid int64, err error) {
+	err = d.db.QueryRow(c, _getSvidByCid, cid).Scan(&svid)
+	if err != nil {
+		log.Warn("db _getSvidByCid err(%v)", err)
+		return
+	}
+	return
+}

app/job/bbq/video/model/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "model.go",
+        "search.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/video/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/model/model.go

@@ -0,0 +1,347 @@
+package model
+
+import (
+	"encoding/json"
+	"go-common/library/time"
+)
+
+//静态变量
+const (
+	IsDeletedFalse = 0     //未删除标识
+	DefaultVer     = "1.0" // DefaultVer 默认初始化版本
+	FromMain       = 1     // FromMain 渠道来自主站
+
+	//版本状态（1-草稿 2-待审核 3-待上架 4-已上架 -1-已下架 -2-强制下架）
+	OnShelf = 4
+
+	//Tag类型
+	OPTag                 = 0 //运营标签
+	TIDTag                = 1 //一级分区标签
+	SubTIDTag             = 2 //二级分区标签
+	NormalTag             = 3 //普通标签
+	AllowSyncOperVideoTag = int64(1)
+	DenySyncOperVideoTag  = int64(2)
+	JobFinishNotice       = 1 //运营导入脚本完成邮件推送类型
+
+	VideoStCheckBack         = 2  //视频状态回查
+	VideoStPassReview        = 1  //审核通过
+	VideoStPendingPassReview = 0  //原始稿件状态，等待安全审核
+	VideoStPassReviewReject  = -1 //回查不通过
+	VideoStCanPlay           = 3  //可放出
+	VideoStHighGrade         = 4  //优质
+	VideoStRecommend         = 5  //推荐
+	VideoStInactive          = -3 //视频下架
+	VideoStDeleted           = -4 //视频硬删除
+
+	//origin sync st abandon
+	VideoRepSyncStOrigin = 0
+	//sub bvc commit
+	VideoRepSyncStBvcCommit = 10
+	//receive bvc resource
+	VideoRepSyncStInsertBvcInfo = 20
+	//video onshelf
+	VideoRepSyncStOnshelf = 30
+
+	UVStOpAdd = 1  //add
+	UVStOpDel = -1 //delete
+	//StateActive 评论状态
+	StateActive = int16(0)
+	//DefaultType ..
+	DefaultType = int16(23)
+	UserTypeUp  = 1
+	//VideoFromBILI ..
+	VideoFromBILI = 0
+	//VideoFromBBQ ..
+	VideoFromBBQ = 1
+	//VideoFromCMS ..
+	VideoFromCMS = 2
+
+	//SourceRequest video_repository.sync_status source request
+	SourceRequest = 1
+	//SourceXcodeCover video_repository.sync_status xcode/cover
+	SourceXcodeCover = 2
+	//SourceAI video_repository.sync_status ai source
+	SourceAI = 4
+	//SourceOnshelf video_repository.sync_status video on shelf
+	SourceOnshelf = 8
+
+	VideoUploadProcessStatusFailed    = -1
+	VideoUploadProcessStatusPending   = 0
+	VideoUploadProcessStatusSuccessed = 1
+)
+
+//Tag .
+type Tag struct {
+	ID   int64  `json:"id"`
+	Name string `json:"name"`
+	Type int64  `json:"type"`
+}
+
+//VideoHiveInfo struct
+type VideoHiveInfo struct {
+	AVID           int64  `json:"avid"`
+	CID            int64  `json:"cid"`
+	MID            int64  `json:"mid"`
+	Title          string `json:"title"`
+	Content        string `json:"content"`
+	Original       int16  `json:"original"`
+	Report         int64  `json:"report"`
+	DurationAll    int64  `json:"duration_all"`
+	Play           int64  `json:"play"`
+	PlayGuest      int64  `json:"play_guest"`
+	PlayFans       int64  `json:"play_fans"`
+	Access         int64  `json:"access"`
+	Reply          int64  `json:"reply"`
+	Fav            int64  `json:"fav"`
+	Likes          int64  `json:"likes"`
+	Coin           int64  `json:"coin"`
+	Share          int64  `json:"share"`
+	Danmu          int64  `json:"danmu"`
+	ElecPay        int64  `json:"elec_pay"`
+	ElecNum        int64  `json:"elec_num"`
+	ElecUser       int64  `json:"elec_user"`
+	Duration       int64  `json:"duration"`
+	State          int64  `json:"state"`
+	Tag            string `json:"tag"`
+	ShareDaily     int64  `json:"share_daily"`
+	PlayDaily      int64  `json:"play_daily"`
+	FavDaily       int64  `json:"fav_daily"`
+	ReplyDaily     int64  `json:"reply_daily"`
+	DanmuDaily     int64  `json:"danmu_daily"`
+	LikesDaily     int64  `json:"likes_daily"`
+	DurationDaily  int64  `json:"duration_daily"`
+	Pubtime        string `json:"pubtime"`
+	LogDate        string `json:"log_date"`
+	TID            int64  `json:"tid"`
+	SubTID         int64  `json:"sub_tid"`
+	Ctime          string `json:"ctime"`
+	DispatchStatus int64  `json:"dispatch_status"`
+	IsFullScreen   int16  `json:"is_full_screen"`
+}
+
+// VideoInfo 一般视频信息
+type VideoInfo struct {
+	SVID     int64     `json:"svid"`
+	TID      int64     `json:"tid"`
+	SubTID   int64     `json:"sub_tid"`
+	Title    string    `json:"title"`
+	Content  string    `json:"content"`
+	MID      int64     `json:"mid"`
+	Report   int64     `json:"report"`
+	Duration int64     `json:"duration"`
+	Pubtime  string    `json:"pubtime"`
+	Ctime    time.Time `json:"ctime"`
+	AVID     int64     `json:"avid"`
+	CID      int64     `json:"cid"`
+	State    int16     `json:"state"`
+	Original int16     `json:"original"`
+	From     int16     `json:"from"`
+	VerID    int64     `json:"ver_id"`
+	Ver      int64     `json:"ver"`
+}
+
+// VideoStHive 视频hive统计数据
+type VideoStHive struct {
+	SVID           int64 `json:"svid"`
+	Play           int64 `json:"play"`
+	Report         int64 `json:"report"`
+	DurationAll    int64 `json:"duration_all"`
+	Access         int64 `json:"access"`
+	Reply          int64 `json:"reply"`
+	Fav            int64 `json:"fav"`
+	Likes          int64 `json:"likes"`
+	Coin           int64 `json:"coin"`
+	Share          int64 `json:"share"`
+	Subtitles      int64 `json:"subtitles"`
+	ElecPay        int64 `json:"elec_pay"`
+	ElecNum        int64 `json:"elec_num"`
+	ElecUser       int64 `json:"elec_user"`
+	DurationDaily  int64 `json:"duration_daily"`
+	ShareDaily     int64 `json:"share_daily"`
+	PlayDaily      int64 `json:"play_daily"`
+	FavDaily       int64 `json:"fav_daily"`
+	ReplyDaily     int64 `json:"reply_daily"`
+	SubtitlesDaily int64 `json:"subtitles_daily"`
+	LikesDaily     int64 `json:"likes_daily"`
+}
+
+// UserBaseDB 用户基础表字段
+type UserBaseDB struct {
+	ID        int64     `json:"id"`
+	MID       int64     `json:"mid"`
+	Uname     string    `json:"uname"`
+	Face      string    `json:"face"`
+	Birthday  string    `json:"birthday"`
+	Exp       int64     `json:"exp"`
+	Level     int64     `json:"level"`
+	Ctime     time.Time `json:"ctime"`
+	Mtime     time.Time `json:"mtime"`
+	Signature string    `json:"signature"`
+	Region    int64     `json:"region"`
+	Sex       int16     `json:"sex"`
+}
+
+//UserDmg 用户画像
+type UserDmg struct {
+	MID          string           `json:"mid"`
+	Gender       string           `json:"gender"`
+	Age          string           `json:"age"`
+	Geo          string           `json:"geo"`
+	ContentTag   string           `json:"content_tag"`
+	ViewedVideo  map[int64]string `json:"viewed_video"`
+	ContentZone  string           `json:"content_zone"`
+	ContentCount string           `json:"content_count"`
+	FollowUps    string           `json:"follow_ups"`
+}
+
+//UserBbqDmg 用户画像
+type UserBbqDmg struct {
+	MID  string   `json:"mid"`
+	Tag2 []string `json:"tag2"`
+	Tag3 []string `json:"tag3"`
+	Up   []string `json:"up"`
+}
+
+//UserBbqBuvidDmg 用户画像buvid
+type UserBbqBuvidDmg struct {
+	Buvid string   `json:"mid"`
+	Tag2  []string `json:"tag2"`
+	Tag3  []string `json:"tag3"`
+	Up    []string `json:"up"`
+}
+
+//UpUserDmg 主站up主用户画像
+type UpUserDmg struct {
+	MID   int64  `json:"mid"`
+	Uname string `json:"uname"`
+	Play  int64  `json:"play"`
+	Fans  int64  `json:"fans"`
+	AVs   int64  `json:"avs"`
+	Likes int64  `json:"likes"`
+}
+
+// CheckTask .
+type CheckTask struct {
+	TaskID    int64  `json:"task_id"`
+	TaskName  string `json:"task_name"`
+	LastCheck int64  `json:"last_check"`
+}
+
+// DatabusRes canal standary message
+type DatabusRes struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+//DatabusBVCTransSub ...
+type DatabusBVCTransSub struct {
+	SVID int64 `json:"svid"`
+}
+
+// VideoDB 视频表数据库字段
+type VideoDB struct {
+	AutoID      int64     `json:"auto_id"`
+	ID          int64     `json:"id"`
+	Title       string    `json:"title"`
+	Content     string    `json:"content"`
+	MID         int64     `json:"mid"`
+	CID         int64     `json:"cid"`
+	Pubtime     time.Time `json:"pubtime"`
+	Ctime       string    `json:"ctime"`
+	Duration    int64     `json:"duration"`
+	Original    int16     `json:"original"`
+	State       int16     `json:"state"`
+	IsFull      int16     `json:"is_full_screen"`
+	VerID       int64     `json:"ver_id"`
+	Ver         string    `json:"ver"`
+	From        int16     `json:"from"`
+	AVID        int64     `json:"avid"`
+	TID         int64     `json:"tid"`
+	SubTID      int64     `json:"sub_tid"`
+	Score       int64     `json:"score"`
+	CoverURL    string    `json:"cover_url"`
+	CoverWidth  int64     `json:"cover_width"`
+	CoverHeight int64     `json:"cover_height"`
+}
+
+// VideoRaw 视频原生表数据库字段
+type VideoRaw struct {
+	ID          int64  `json:"id"`
+	Title       string `json:"title"`
+	Content     string `json:"content"`
+	MID         int64  `json:"mid"`
+	CID         int64  `json:"cid"`
+	Pubtime     string `json:"pubtime"`
+	Ctime       string `json:"ctime"`
+	Duration    int64  `json:"duration"`
+	Original    int16  `json:"original"`
+	State       int16  `json:"state"`
+	IsFull      int16  `json:"is_full_screen"`
+	VerID       int64  `json:"ver_id"`
+	Ver         string `json:"ver"`
+	From        int16  `json:"from"`
+	AVID        int64  `json:"avid"`
+	TID         int64  `json:"tid"`
+	SubTID      int64  `json:"sub_tid"`
+	Score       int64  `json:"score"`
+	CoverURL    string `json:"cover_url"`
+	CoverWidth  int64  `json:"cover_width"`
+	CoverHeight int64  `json:"cover_height"`
+	SVID        int64  `json:"svid"`
+}
+
+// VideoRepRaw 视频原生表数据库字段
+type VideoRepRaw struct {
+	ID            int64  `json:"id"`
+	SVID          int64  `json:"svid"`
+	Title         string `json:"title"`
+	Content       string `json:"content"`
+	MID           int64  `json:"mid"`
+	CID           int64  `json:"cid"`
+	Pubtime       string `json:"pubtime"`
+	Duration      int64  `json:"duration"`
+	Original      int16  `json:"original"`
+	IsFull        int16  `json:"is_full_screen"`
+	From          int16  `json:"from"`
+	AVID          int64  `json:"avid"`
+	TID           int64  `json:"tid"`
+	SubTID        int64  `json:"sub_tid"`
+	Score         int64  `json:"score"`
+	CoverURL      string `json:"cover_url"`
+	CoverWidth    int64  `json:"cover_width"`
+	CoverHeight   int64  `json:"cover_height"`
+	Tag           string `json:"tag"`
+	SyncStatus    int64  `json:"sync_status"`
+	HomeImgURL    string `json:"home_img_url" form:"home_img_url"`
+	HomeImgWidth  int64  `json:"home_img_width" form:"home_img_width"`
+	HomeImgHeight int64  `json:"home_img_height" form:"home_img_height"`
+}
+
+//UpUserInfoRes account服务返回信息
+type UpUserInfoRes struct {
+	MID  int64  `json:"mid"`
+	Name string `json:"name"`
+	Sex  string `json:"sex"`
+	Face string `json:"face"`
+	Sign string `json:"sign"`
+	Rank int64  `json:"rank"`
+}
+
+// UserBase .
+type UserBase struct {
+	Mid   int64  `json:"mid"`
+	Uname string `json:"uname"`
+	Face  string `json:"face"`
+}
+
+//CmsVideo ..
+type CmsVideo struct {
+	ID       int64  `json:"id"`
+	SVStatus int64  `json:"sv_status"`
+	Pubtime  string `json:"pubtime"`
+	Mid      int64  `json:"mid"`
+	Title    string `json:"title"`
+	From     int64  `json:"from"`
+}

app/job/bbq/video/model/search.go

@@ -0,0 +1,26 @@
+package model
+
+// VideoSearch 视频搜索json
+type VideoSearch struct {
+	ID      int32  `json:"id"`
+	Title   string `json:"title"`
+	Play    int32  `json:"play"`
+	Review  int32  `json:"review"`
+	PubTime int32  `json:"pubtime"`
+}
+
+// UserSearch 用户搜索json
+type UserSearch struct {
+	ID      int64  `json:"id"`
+	Uname   string `json:"uname"`
+	Fans    int64  `json:"fans"`
+	Article int64  `json:"article"`
+}
+
+// Sug Sug文本结构
+type Sug struct {
+	ID    int64  `json:"id"`
+	Term  string `json:"term"`
+	Type  string `json:"type"`
+	Score int64  `json:"score"`
+}

app/job/bbq/video/model/video.go

@@ -0,0 +1,6 @@
+package model
+
+const (
+	//VideoStateOutPut 视频可放出状态
+	VideoStateOutPut = "0,1,3,4,5"
+)

app/job/bbq/video/server/http/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/bbq/video/server/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/server/http/http.go

@@ -0,0 +1,53 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	srv *service.Service
+	vfy *verify.Verify
+)
+
+// Init init
+func Init(c *conf.Config) {
+	srv = service.New(c)
+	vfy = verify.New(c.Verify)
+	engine := bm.DefaultServer(c.BM.Server)
+	route(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("bm Start error(%v)", err)
+		panic(err)
+	}
+}
+
+func route(e *bm.Engine) {
+	e.Ping(ping)
+	e.Register(register)
+	g := e.Group("/x/video")
+	{
+		g.GET("/start", vfy.Verify, howToStart)
+	}
+}
+
+func ping(c *bm.Context) {
+	if err := srv.Ping(c); err != nil {
+		log.Error("ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}
+
+// example for http request handler
+func howToStart(c *bm.Context) {
+	c.String(0, "Golang 大法好 !!!")
+}

app/job/bbq/video/service/BUILD

@@ -0,0 +1,57 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "binlogv.go",
+        "binlogvr.go",
+        "bvcsub.go",
+        "cmsvideo.go",
+        "comment.go",
+        "email.go",
+        "ftp.go",
+        "notice.go",
+        "service.go",
+        "user.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/video/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/dao:go_default_library",
+        "//app/job/bbq/video/model:go_default_library",
+        "//app/service/bbq/common:go_default_library",
+        "//app/service/bbq/notice-service/api/v1:go_default_library",
+        "//app/service/bbq/search/api/grpc/v1:go_default_library",
+        "//app/service/bbq/topic/api:go_default_library",
+        "//app/service/bbq/video/api/grpc/v1:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/ftp-master:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/service/binlogv.go

@@ -0,0 +1,197 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/model"
+	"go-common/app/service/bbq/common"
+	topic "go-common/app/service/bbq/topic/api"
+	"go-common/library/log"
+	"strconv"
+)
+
+// videoConsumeproc 视频表消费
+func (s *Service) videoBinlogSub() {
+	var msgs = s.videoSub.Messages()
+	for {
+		var err error
+		msg, ok := <-msgs
+		if !ok {
+			log.Info("userCanal databus Consumer exit")
+			return
+		}
+		res := &model.DatabusRes{}
+		log.Infov(context.Background(), log.KV("log", fmt.Sprintf("canal message %s", string(msg.Value))))
+		if err = json.Unmarshal(msg.Value, &res); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			msg.Commit()
+			continue
+		}
+
+		if res.Table != "video" || (res.Action != "update" && res.Action != "insert") {
+			msg.Commit()
+			continue
+		}
+
+		var vNew, vOld *model.VideoRaw
+		if res.Action == "insert" || res.Action == "update" {
+			if err = json.Unmarshal(res.New, &vNew); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.New))
+				continue
+			}
+		}
+		if res.Action == "update" {
+			if err = json.Unmarshal(res.Old, &vOld); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.Old))
+				continue
+			}
+		}
+
+		//idempotent consume
+		for i := 0; i < _retryTimes; i++ {
+			//fetured video state subscription
+			if err = s.VideoStateSub(vNew, vOld); err == nil {
+				break
+			}
+		}
+		//s.UpdateCms(context.Background(), vNew)
+		//register comment
+		for i := 0; i < _retryTimes; i++ {
+			//merge related information subscription
+			if err = s.CommentReg(context.Background(), vNew.SVID, model.StateActive); err == nil {
+				break
+			}
+		}
+
+		if res.Action == "insert" {
+			for i := 0; i < _retryTimes; i++ {
+				log.V(1).Infow(context.Background(), "log", "merge up info", "retry_time", i, "mid", vNew.MID, "svid", vNew.SVID)
+				//merge related information subscription
+				if err = s.MergeUpInfoSub(vNew); err == nil {
+					break
+				}
+			}
+		}
+
+		//unidempotent consume
+		if res.Action == "update" {
+			s.UpdateStaInfoSub(vNew, vOld)
+		} else if res.Action == "insert" {
+			s.AddSVTotal(vNew)
+		}
+		msg.Commit()
+	}
+}
+
+//UpdateCms ..
+func (s *Service) UpdateCms(c context.Context, vNew *model.VideoRaw) (err error) {
+	if err = s.dao.UpdateCms(c, vNew); err != nil {
+		log.Warnw(c, "event", fmt.Sprintf("updateCms err:%v,param:%v", err, vNew))
+	}
+	return
+}
+
+// VideoStateSub 视频状态变更消费
+func (s *Service) VideoStateSub(vNew *model.VideoRaw, vOld *model.VideoRaw) (err error) {
+	log.Infow(context.Background(), "log", "one video state sub", "svid", vNew.SVID)
+
+	s.SaveVideo2ES(strconv.Itoa(int(vNew.SVID)))
+
+	if vOld == nil || vNew.State != vOld.State {
+		var ids []int64
+		ids, err = s.dao.GetRecallOpVideo(context.Background())
+		if err != nil {
+			log.Warnw(context.Background(), "log", "get recall op video fail")
+			return
+		}
+		needSetRecallOpVideo := true
+		if vNew.State == _selection {
+			for _, id := range ids {
+				if id == vNew.SVID {
+					needSetRecallOpVideo = false
+					break
+				}
+			}
+			ids = append(ids, vNew.SVID)
+		} else if vNew.State != _selection {
+			index := -1
+			for i, id := range ids {
+				if id == vNew.SVID {
+					index = i
+					break
+				}
+			}
+			if index != -1 {
+				ids = append(ids[:index], ids[index+1:]...)
+			} else {
+				needSetRecallOpVideo = false
+			}
+		}
+		if needSetRecallOpVideo {
+			if err = s.dao.SetRecallOpVideo(context.Background(), ids); err != nil {
+				log.Warnw(context.Background(), "log", "get recall op video fail")
+				return
+			}
+		}
+	}
+
+	// 话题状态变更
+	needUpdateTopicVideoState := false
+	topicState := topic.TopicVideoStateUnAvailable
+	if vOld == nil {
+		needUpdateTopicVideoState = true
+		// update topic video
+		if common.IsTopicSvStateAvailable(int64(vNew.State)) {
+			topicState = topic.TopicVideoStateAvailable
+		}
+	} else {
+		if common.IsTopicSvStateAvailable(int64(vNew.State)) != common.IsTopicSvStateAvailable(int64(vOld.State)) {
+			needUpdateTopicVideoState = true
+			if common.IsTopicSvStateAvailable(int64(vNew.State)) {
+				topicState = topic.TopicVideoStateAvailable
+			}
+		}
+	}
+	if needUpdateTopicVideoState {
+		_, err = s.topicClient.UpdateVideoState(context.Background(), &topic.UpdateVideoStateReq{Svid: vNew.SVID, State: int32(topicState)})
+		log.Infow(context.Background(), "log", "update topic video state", "svid", vNew.SVID, "new_state", vNew.State)
+		if err != nil {
+			log.Warnw(context.Background(), "log", "update topic video state", "svid", vNew.SVID, "new_state", vNew.State)
+			return
+		}
+	}
+	return
+}
+
+//MergeUpInfoSub ..
+func (s *Service) MergeUpInfoSub(vNew *model.VideoRaw) (err error) {
+	mid := vNew.MID
+	if err = s.dao.MergeUpInfo(mid); err != nil {
+		log.Error("MergeUpInfo failed,err:%v,mid:%d", err, mid)
+	}
+	return
+}
+
+//UpdateStaInfoSub ...
+func (s *Service) UpdateStaInfoSub(vNew *model.VideoRaw, vOld *model.VideoRaw) {
+	if vOld == nil || vNew.State == vOld.State {
+		return
+	}
+	if vNew.State == model.VideoStInactive {
+		s.dao.UpdateUVSt(vNew.MID, "unshelf_av_total")
+	} else if vNew.State == model.VideoStDeleted {
+		s.dao.UpdateUVStDel(vNew.MID, "av_total")
+	}
+
+	if vOld.State == model.VideoStInactive {
+		s.dao.UpdateUVStDel(vNew.MID, "unshelf_av_total")
+	} else if vNew.State == model.VideoStDeleted {
+		s.dao.UpdateUVSt(vOld.MID, "av_total")
+	}
+}
+
+//AddSVTotal ...
+func (s *Service) AddSVTotal(vNew *model.VideoRaw) {
+	s.dao.AddSVTotal(vNew.MID)
+}

app/job/bbq/video/service/binlogvr.go

@@ -0,0 +1,103 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/model"
+	videov1 "go-common/app/service/bbq/video/api/grpc/v1"
+	"go-common/library/log"
+)
+
+//videoRepositorySub video_repository subscription .
+func (s *Service) videoRepositoryBinlogSub() {
+	msgs := s.videoRep.Messages()
+	for {
+		var err error
+		msg, ok := <-msgs
+		if !ok {
+			log.Info("video_repository databus Consumer exit")
+			return
+		}
+		res := &model.DatabusRes{}
+		log.Infov(context.Background(), log.KV("log", fmt.Sprintf("canal message %s", string(msg.Value))))
+		if err = json.Unmarshal(msg.Value, &res); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			msg.Commit()
+			continue
+		}
+
+		if res.Table != "video_repository" || (res.Action != "update" && res.Action != "insert") {
+			msg.Commit()
+			continue
+		}
+
+		//unserialize databus struct
+		var vNew, vOld *model.VideoRepRaw
+		if res.Action == "insert" || res.Action == "update" {
+			if err = json.Unmarshal(res.New, &vNew); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.New))
+				msg.Commit()
+				continue
+			}
+		}
+		if res.Action == "update" {
+			if err = json.Unmarshal(res.Old, &vOld); err != nil {
+				log.Error("video unmarshal err(%v) data[%s]", err, string(res.Old))
+				msg.Commit()
+				continue
+			}
+		}
+
+		if res.Action == "insert" {
+			for i := 0; i < _retryTimes; i++ {
+				if err = s.PepareResource(vNew); err == nil {
+					break
+				}
+			}
+		}
+		msg.Commit()
+	}
+}
+
+//PepareResource ...
+func (s *Service) PepareResource(vNew *model.VideoRepRaw) (err error) {
+	var (
+		ctx  = context.Background()
+		SVID int64
+		row  *model.VideoRepRaw
+	)
+	//bbq/cms video not trans to bvc
+	if vNew.From == model.VideoFromBBQ || vNew.From == model.VideoFromCMS {
+		return
+	}
+	if row, err = s.dao.RawVideoByID(ctx, vNew.ID); err != nil {
+		return
+	}
+	if row.SVID > 0 {
+		SVID = row.SVID
+	} else {
+		req := &videov1.CreateIDRequest{
+			Mid: vNew.MID,
+		}
+		var rep *videov1.CreateIDResponse
+		if rep, err = s.dao.VideoClient.CreateID(ctx, req); err != nil {
+			log.Error("Numbering device return err:%v", err)
+			return
+		}
+		if err = s.dao.UpdateSvid(context.Background(), vNew.ID, rep.NewId); err != nil {
+			return
+		}
+		SVID = rep.NewId
+	}
+	reqBvc := &videov1.BVideoTransRequset{
+		SVID: SVID,
+		CID:  vNew.CID,
+	}
+	log.Info("bvc trans commit req:%v", reqBvc)
+	if _, err = s.dao.VideoClient.BVCTransCommit(ctx, reqBvc); err != nil {
+		log.Error("BVCTransCommit err :%v,req:%v", err, reqBvc)
+	}
+	s.dao.UpdateSyncStatus(ctx, SVID, model.SourceRequest)
+	return
+}

app/job/bbq/video/service/bvcsub.go

@@ -0,0 +1,120 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/model"
+	videov1 "go-common/app/service/bbq/video/api/grpc/v1"
+	"go-common/library/log"
+	"strings"
+)
+
+//BvcTransSub ...
+func (s *Service) BvcTransSub() {
+	msgs := s.bvcSub.Messages()
+	for {
+		var (
+			err error
+			vr  *model.VideoRepRaw
+		)
+		c := context.Background()
+		msg, ok := <-msgs
+
+		//release subscription
+		if s.c.SubBvcControl.Control == 2 {
+			msg.Commit()
+			continue
+		}
+		if !ok {
+			log.Info("BvcTransSub databus Consumer exit")
+			return
+		}
+		res := &model.DatabusBVCTransSub{}
+		log.Infov(context.Background(), log.KV("log", fmt.Sprintf("databus message %s", string(msg.Value))))
+		if err = json.Unmarshal(msg.Value, &res); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			msg.Commit()
+			continue
+		}
+		if vr, err = s.dao.RawVideo(c, res.SVID); err != nil {
+			msg.Commit()
+			continue
+		}
+		//resource check
+		if vr.SyncStatus&model.SourceXcodeCover > 0 {
+			if err = s.importVideo(c, vr); err != nil {
+				log.Errorw(c, "errmsg", "importVideo err", "req", vr, "err", err)
+				msg.Commit()
+				continue
+			}
+			s.syncTag(c, vr.Tag)
+
+			s.dao.UpdateSyncStatus(c, vr.SVID, model.SourceOnshelf)
+		}
+
+		msg.Commit()
+	}
+}
+
+//importVideo put video on shelf
+func (s *Service) importVideo(c context.Context, vr *model.VideoRepRaw) (err error) {
+	// var (
+	// 	st int64
+	// )
+	// if vr.From == model.VideoFromBILI {
+	// 	st = model.VideoStPassReview
+	// } else {
+	// 	st = model.VideoStPendingPassReview
+	// }
+	req := &videov1.ImportVideoInfo{
+		AVID:        vr.AVID,
+		Svid:        vr.SVID,
+		MID:         vr.MID,
+		CID:         vr.CID,
+		SubTID:      vr.SubTID,
+		TID:         vr.TID,
+		Title:       vr.Title,
+		Pubtime:     vr.Pubtime,
+		From:        int64(vr.From),
+		CoverUrl:    vr.CoverURL,
+		CoverHeight: vr.CoverHeight,
+		CoverWidth:  vr.CoverWidth,
+		//State:         st,
+		HomeImgHeight: vr.HomeImgHeight,
+		HomeImgUrl:    vr.HomeImgURL,
+		HomeImgWidth:  vr.HomeImgWidth,
+	}
+	for i := 0; i < _retryTimes; i++ {
+		if _, err = s.dao.VideoClient.ImportVideo(c, req); err == nil {
+			break
+		}
+	}
+	return
+}
+
+//syncTag sync video from bilibili common tag
+func (s *Service) syncTag(c context.Context, t string) (err error) {
+	if t == "" {
+		return
+	}
+	var (
+		arrTag []string
+		tag    []*videov1.TagInfo
+	)
+	arrTag = strings.Split(t, ",")
+	for _, v := range arrTag {
+		tmp := &videov1.TagInfo{
+			TagName: v,
+			TagType: 3,
+		}
+		tag = append(tag, tmp)
+	}
+	reqTag := &videov1.SyncVideoTagRequest{
+		TagInfos: tag,
+	}
+	if _, err = s.dao.VideoClient.SyncTag(c, reqTag); err != nil {
+		log.Error("sync tag err :%v,tag:%v", err, tag)
+	}
+	return
+}

app/job/bbq/video/service/cmsvideo.go

@@ -0,0 +1,17 @@
+package service
+
+import (
+	"go-common/library/log"
+)
+
+//TransToCheckBack ..
+func (s *Service) TransToCheckBack() {
+	log.Info("deliveryNewVdieoToCms begin")
+	s.dao.TransToCheckBack()
+}
+
+//TransToReview ...
+func (s *Service) TransToReview() {
+	log.Info("TransToReview begin")
+	s.dao.TransToReview()
+}

app/job/bbq/video/service/comment.go

@@ -0,0 +1,76 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"go-common/library/log"
+	"sync"
+	"sync/atomic"
+)
+
+const (
+	_stateActive = 0
+	_defaultType = 23
+)
+
+//CommentReg 评论开关
+func (s *Service) CommentReg(c context.Context, oid int64, state int16) (err error) {
+	req := map[string]interface{}{
+		"oid":   oid,
+		"type":  _defaultType,
+		"adid":  0,
+		"mid":   0,
+		"state": state,
+	}
+	err = s.dao.ReplyReg(c, req)
+	return
+}
+
+// AutoRegAll 批量注册评论
+func (s *Service) AutoRegAll(c context.Context) {
+	var (
+		IDs     []int64
+		e       int
+		err     error
+		lastID  int64
+		num     int64
+		failNum int64
+	)
+	for {
+		IDs, lastID, _ = s.dao.GetVideoByLastID(c, lastID)
+		if len(IDs) == 0 {
+			break
+		}
+		sum := len(IDs)
+		partNum := 200
+		rnum := int(sum / partNum)
+		log.Info("rountine start num[%d]", rnum)
+		var wg sync.WaitGroup
+		for i := 0; i < rnum+1; i++ {
+			wg.Add(1)
+			if e = (i + 1) * partNum; e > sum {
+				e = sum
+			}
+			r := IDs[i*partNum : e]
+			go func(ids []int64, k int) {
+				defer wg.Done()
+				for _, svid := range ids {
+					err = s.CommentReg(c, svid, _stateActive)
+					if err != nil {
+						atomic.AddInt64(&failNum, 1)
+						fmt.Printf("Comment active fail [%v]\n", err)
+						log.Errorv(c, log.KV("log", fmt.Sprintf("Comment active fail [%v]", err)))
+					} else {
+						atomic.AddInt64(&num, 1)
+						fmt.Printf("Comment active success oid:[%d]\n", svid)
+						log.Infov(c, log.KV("log", fmt.Sprintf("Comment active success oid:[%d]", svid)))
+					}
+				}
+			}(r, i)
+		}
+		wg.Wait()
+	}
+	log.Info("comment reg complete succ[%d] fail[%d]", num, failNum)
+	fmt.Printf("comment reg complete succ[%d] fail[%d]", num, failNum)
+
+}

app/job/bbq/video/service/email.go

@@ -0,0 +1,34 @@
+package service
+
+import (
+	"go-common/app/job/bbq/video/model"
+
+	gomail "gopkg.in/gomail.v2"
+)
+
+//SendMail ...
+func (s *Service) SendMail(mailType int) (err error) {
+	var (
+		m       = gomail.NewMessage()
+		message = map[string][]string{}
+		cType   string
+		cBody   string
+	)
+
+	switch mailType {
+	//运营i后台脚本导入完成推送邮件
+	case model.JobFinishNotice:
+		message["To"] = s.c.Mail.To
+		message["Subject"] = []string{"同步运营筛选视频任务已完成"}
+		cBody = "运营筛选视频已经导入完成！"
+		cType = "text/plain"
+	case 2:
+		message["To"] = []string{"write your address"}
+	}
+
+	m.SetHeaders(message)
+	m.SetBody(cType, cBody)
+
+	err = s.dao.SendMail(m)
+	return
+}

app/job/bbq/video/service/ftp.go

@@ -0,0 +1,357 @@
+package service
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"go-common/app/job/bbq/video/model"
+	"go-common/library/log"
+
+	ftp "github.com/ftp-master"
+)
+
+const (
+	filePathKey      = "search"
+	videoFileName    = "bbqvideo"
+	videoFileMD5Name = "bbqvideo.md5"
+	userFileName     = "bbquser"
+	userFileMD5Name  = "bbquser.md5"
+	sugFileName      = "bbqsug"
+	sugFileMD5Name   = "bbqsug.md5"
+	sugSrcIDIdx      = 0
+	sugSrcTermIdx    = 1
+	sugSrcTypeIdx    = 2
+	sugSrcScoreIdx   = 3
+	defaultSugPath   = "/src/go-common/app/job/bbq/video/cmd/sug"
+)
+
+// writeSug .
+func (s *Service) writeSug(path string, filename string, md5name string) (err error) {
+	var num int64
+	filePath := path + filename
+	file, err := os.OpenFile(filePath, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		log.Error("writeSug os.OpenFile(%s) error(%v)", filename, err)
+		return
+	}
+	defer file.Close()
+	srcPath := s.c.FTP.LocalPath["sugsrc"]
+	if srcPath == "" {
+		srcPath = os.Getenv("GOPATH") + defaultSugPath
+	}
+	if srcPath == "" {
+		log.Error("sugsrc path is empty")
+		return
+	}
+	src, err := os.Open(srcPath)
+	if err != nil {
+		log.Error("writeSug os.Open source sug error(%v)", err)
+		return
+	}
+	defer src.Close()
+	br := bufio.NewReader(src)
+	for {
+		a, _, c := br.ReadLine()
+		if c == io.EOF {
+			break
+		}
+		strArr := strings.Split(string(a), ",")
+		var bs []byte
+		var b []byte
+		id, _ := strconv.ParseInt(strArr[sugSrcIDIdx], 10, 64)
+		score, _ := strconv.ParseInt(strArr[sugSrcScoreIdx], 10, 64)
+		sd := &model.Sug{
+			ID:    id,
+			Term:  strArr[sugSrcTermIdx],
+			Type:  strArr[sugSrcTypeIdx],
+			Score: score,
+		}
+		if b, err = json.Marshal(sd); err != nil {
+			log.Warn("json.Marshal(%v) error(%v)", sd, err)
+			continue
+		}
+		bs = append(bs, b...)
+		bs = append(bs, '\n')
+		if _, err = file.Write(bs); err != nil {
+			log.Error("writeSug file.Write error(%v)", err)
+			return
+		}
+		num++
+		// log.Info("write sug term:[%s]", strArr[sugSrcTermIdx])
+	}
+	log.Info("writeSug success num (%d)", num)
+	if err = s.writeMD5(path, filename, md5name); err != nil {
+		return
+	}
+	s.uploadFile(path, filename, md5name)
+	return
+}
+
+// writeUserInfo .
+func (s *Service) writeUserInfo(path string, filename string, md5name string) (err error) {
+	var lastID int64
+	var num int64
+	filePath := path + filename
+	file, err := os.OpenFile(filePath, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		log.Error("writeUserInfo os.OpenFile(%s) error(%v)", filename, err)
+		return
+	}
+	defer file.Close()
+	for {
+		var (
+			dataList []*model.UserBaseDB
+			bs       []byte
+		)
+		if dataList, err = s.dao.UsersByLast(context.Background(), lastID); err != nil {
+			return
+		}
+		if len(dataList) == 0 {
+			break
+		}
+		for _, data := range dataList {
+			var b []byte
+			sd := &model.UserSearch{
+				ID:    data.MID,
+				Uname: data.Uname,
+			}
+			if b, err = json.Marshal(sd); err != nil {
+				log.Warn("json.Marshal(%v) error(%v)", sd, err)
+				continue
+			}
+			bs = append(bs, b...)
+			bs = append(bs, '\n')
+			lastID = data.ID
+			num++
+			// log.Info("append user mid:[%d]", data.MID)
+		}
+		if _, err = file.Write(bs); err != nil {
+			log.Error("writeUserInfo file.Write error(%v)", err)
+			return
+		}
+	}
+	log.Info("writeUserInfo success num (%d)", num)
+	if err = s.writeMD5(path, filename, md5name); err != nil {
+		return
+	}
+	s.uploadFile(path, filename, md5name)
+	return
+}
+
+// writeVideoInfo .
+func (s *Service) writeVideoInfo(path string) (err error) {
+	var lastID int64
+	var num int64
+	var RelateID int64
+	filePath := path + videoFileName
+	file, err := os.OpenFile(filePath, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		log.Error("writeVideoInfo os.OpenFile(%s) error(%v)", videoFileName, err)
+		return
+	}
+	defer file.Close()
+	for {
+		var (
+			dataList []*model.VideoDB
+			bs       []byte
+		)
+		if dataList, err = s.dao.VideosByLast(context.Background(), lastID); err != nil {
+			return
+		}
+		if len(dataList) == 0 {
+			break
+		}
+		for _, data := range dataList {
+			var (
+				b []byte
+			)
+			RelateID, err = strconv.ParseInt(fmt.Sprintf("%d00", data.AutoID), 10, 32)
+			if err != nil {
+				log.Error("relate id parse err [%v]", err)
+				continue
+			}
+			sd := &model.VideoSearch{
+				ID:      int32(RelateID),
+				Title:   data.Title,
+				PubTime: int32(data.Pubtime),
+			}
+			if b, err = json.Marshal(sd); err != nil {
+				log.Warn("json.Marshal(%v) error(%v)", sd, err)
+				continue
+			}
+			bs = append(bs, b...)
+			bs = append(bs, '\n')
+			lastID = data.AutoID
+			num++
+			// log.Info("append video svid:[%d]", data.ID)
+		}
+		if _, err = file.Write(bs); err != nil {
+			log.Error("writeVideoInfo file.Write error(%v)", err)
+			return
+		}
+	}
+	log.Info("writeVideoInfo success num (%d)", num)
+	if err = s.writeMD5(path, videoFileName, videoFileMD5Name); err != nil {
+		return
+	}
+	s.uploadFile(path, videoFileName, videoFileMD5Name)
+	return
+}
+
+// writeMD5 .
+func (s *Service) writeMD5(path string, fname string, md5name string) (err error) {
+	var out bytes.Buffer
+	f, err := exec.LookPath("md5sum")
+	if err != nil {
+		log.Error("writeMD5 exec.LookPath(md5sum) error(%v)", err)
+		f = "md5sum"
+	}
+	cmd := exec.Command(f, path+fname)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		log.Error("writeMD5 cmd.Run() error(%v)", err)
+		return
+	}
+	outStrs := strings.Split(out.String(), " ")
+	if len(outStrs) == 0 {
+		log.Error("writeMD5 len(outStrs) == 0")
+		return errors.New("NONE MD5")
+	}
+	md5FileName := path + md5name
+	md5File, err := os.OpenFile(md5FileName, os.O_CREATE|syscall.O_TRUNC|os.O_WRONLY, 0600)
+	if err != nil {
+		log.Error("writeMD5 os.OpenFile(md5 file) error(%v)", err)
+		return
+	}
+	defer md5File.Close()
+	_, err = md5File.WriteString(outStrs[0])
+	return
+}
+
+// SyncVideo2Search 同步视频
+func (s *Service) SyncVideo2Search() {
+	err := s.writeVideoInfo(s.c.FTP.LocalPath[filePathKey])
+	if err != nil {
+		log.Error("SyncVideo err[%v]", err)
+	} else {
+		log.Info("SyncVideo complete")
+	}
+}
+
+// SyncUser2Search 同步用户
+func (s *Service) SyncUser2Search() {
+	err := s.writeUserInfo(s.c.FTP.LocalPath[filePathKey], userFileName, userFileMD5Name)
+	if err != nil {
+		log.Error("syncUser err[%v]", err)
+	} else {
+		log.Info("syncUser complete")
+	}
+}
+
+// SyncSug2Search 同步sug
+func (s *Service) SyncSug2Search() {
+	err := s.writeSug(s.c.FTP.LocalPath[filePathKey], sugFileName, sugFileMD5Name)
+	if err != nil {
+		log.Error("syncSug err[%v]", err)
+	} else {
+		log.Info("syncSug complete")
+	}
+}
+
+// SyncSearch 全量同步搜索各文件
+func (s *Service) SyncSearch() {
+	for {
+		s.searchChan <- "SyncSearch Complete"
+		var wg sync.WaitGroup
+		fmt.Println("SyncSearch Start")
+		stratAt := time.Now()
+		wg.Add(3)
+		go func() {
+			defer wg.Done()
+			s.SyncVideo2Search()
+		}()
+		go func() {
+			defer wg.Done()
+			s.SyncUser2Search()
+		}()
+		go func() {
+			defer wg.Done()
+			s.SyncSug2Search()
+		}()
+		wg.Wait()
+		//耗时统计
+		elapsed := time.Since(stratAt)
+		log.Info("SyncSearch elapsed: %s", elapsed)
+		fmt.Println(<-s.searchChan)
+		//间隔1min
+		time.Sleep(time.Minute * 1)
+	}
+}
+
+// uploadFile .
+func (s *Service) uploadFile(path string, fname string, md5name string) (err error) {
+	ftp, err := ftp.Connect(s.c.FTP.Addr)
+	if err != nil {
+		log.Error("connect to ftp(%s) error(%v)", s.c.FTP.Addr, err)
+		return
+	}
+	defer ftp.Quit()
+	err = ftp.Login(s.c.FTP.User, s.c.FTP.Password)
+	if err != nil {
+		log.Error("ftp login(user:%s) error(%v)", s.c.FTP.User, err)
+		return
+	}
+	defer ftp.Logout()
+	ftpDir := fmt.Sprintf(s.c.FTP.RemotePath[filePathKey], fname)
+	err = ftp.ChangeDir(ftpDir)
+	if err != nil {
+		log.Error("enter ftp path [%s] err [%v]", ftpDir, err)
+		return
+	}
+	// delete source file, and upload file to ftp.
+	if err = ftp.Delete(fname); err != nil {
+		log.Error("ftp.Delete(%s) error(%v)", fname, err)
+	}
+	filePath := path + fname
+	file, err := os.Open(filePath)
+	if err != nil {
+		log.Error("os.Open(%s) error(%v)", filePath, err)
+		return
+	}
+	defer file.Close()
+	if err = ftp.Stor(fname, file); err != nil {
+		log.Error("ftp.Stor(%s) error(%v)", fname, err)
+		return
+	}
+	log.Info("ftp upload success(%s)", fname)
+	// delete source.md5 file, and upload file to ftp.
+	if err = ftp.Delete(md5name); err != nil {
+		log.Error("ftp.Delete(%s) error(%v)", md5name, err)
+	}
+	md5FilePath := path + md5name
+	md5File, err := os.Open(md5FilePath)
+	if err != nil {
+		log.Error("os.Open(%s) error(%v)", md5name, err)
+		return
+	}
+	defer md5File.Close()
+	if err = ftp.Stor(md5name, md5File); err != nil {
+		log.Error("ftp.Stor(%s) error(%v)", md5name, err)
+		return
+	}
+	log.Info("ftp upload success(%s)", md5name)
+	return
+}

app/job/bbq/video/service/notice.go

@@ -0,0 +1,112 @@
+package service
+
+import (
+	"context"
+	"go-common/app/job/bbq/video/dao"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"sync/atomic"
+	"time"
+
+	notice "go-common/app/service/bbq/notice-service/api/v1"
+)
+
+// 信号量，为了避免task执行超过周期，连续执行两个任务会出现问题
+var i32 int32
+
+// SysMsgTask 推送消息
+func (s *Service) SysMsgTask() {
+	ctx := context.Background()
+	newValue := atomic.AddInt32(&i32, 1)
+	defer atomic.AddInt32(&i32, -1)
+	if newValue > 1 {
+		log.Errorv(ctx, log.KV("log", "sysMsgTask pending"))
+		return
+	}
+
+	res, err := s.dao.RawCheckTask(ctx, "checkSysMsg")
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "get last sysMsgTask id fail"))
+		return
+	}
+
+	lastSysMsgID := res.LastCheck
+	curSysMsgID := res.LastCheck
+	list, err := s.dao.GetNewSysMsg(ctx, curSysMsgID)
+	if err != nil {
+		log.Errorv(ctx, log.KV("log", "get new sysMsg fail"))
+		return
+	}
+	if len(list) == 0 {
+		log.Infov(ctx, log.KV("log", "no new sys msg to be sync to notice"))
+		return
+	}
+
+	var mids []int64
+	for _, item := range list {
+		curSysMsgID = item.Id
+
+		notice := notice.NoticeBase{
+			Mid:        item.Receiver,
+			ActionMid:  item.Sender,
+			SvId:       0,
+			NoticeType: 4,
+			Text:       item.Text,
+			JumpUrl:    item.JumpUrl,
+			BizType:    dao.NoticeBizTypeSysMsg,
+			BizId:      item.Id,
+		}
+		// 全量系统消息
+		if item.Receiver == 0 {
+			lastUserID := int64(0)
+			if len(mids) == 0 {
+				for {
+					if userBases, err := s.dao.UsersByLast(ctx, lastUserID); err != nil {
+						log.Errorv(ctx, log.KV("log", "sys msg task: get user base fail"))
+						break
+					} else {
+						for _, userBase := range userBases {
+							mids = append(mids, userBase.MID)
+						}
+						if len(userBases) > 0 {
+							lastUserID = userBases[len(userBases)-1].ID
+						} else {
+							break
+						}
+					}
+
+				}
+			}
+			midChan := make(chan int64, 20)
+			go func() {
+				for _, mid := range mids {
+					midChan <- mid
+				}
+				close(midChan)
+			}()
+			startTime := time.Now()
+			g := errgroup.Group{}
+			for i := 0; i < 10; i++ {
+				g.Go(func() error {
+					subNotice := notice
+					for mid := range midChan {
+						subNotice.Mid = mid
+						s.dao.CreateNotice(ctx, &subNotice)
+					}
+					return nil
+				})
+			}
+			g.Wait()
+			log.Info("total sys msg notice push: cost_time=%f, mid_len=%d", time.Since(startTime).Seconds(), len(mids))
+		} else {
+			s.dao.CreateNotice(ctx, &notice)
+		}
+	}
+
+	if _, err := s.dao.UpdateTaskLastCheck(ctx, "checkSysMsg", curSysMsgID); err != nil {
+		log.Errorv(ctx, log.KV("log", "update check_task mysql fail"))
+		return
+	}
+
+	log.Infov(ctx, log.KV("log", "no new sys msg to be sync to notice"), log.KV("last_sys_id", lastSysMsgID), log.KV("cur_sys_id", curSysMsgID))
+}

app/job/bbq/video/service/service.go

@@ -0,0 +1,180 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"go-common/library/conf/env"
+	"go-common/library/queue/databus"
+	"reflect"
+	"time"
+
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/dao"
+	"go-common/library/log"
+
+	topic "go-common/app/service/bbq/topic/api"
+
+	"github.com/robfig/cron"
+)
+
+var (
+	srvName string
+)
+
+// Service struct
+type Service struct {
+	c           *conf.Config
+	dao         *dao.Dao
+	searchChan  chan string
+	videoSub    *databus.Databus
+	videoRep    *databus.Databus
+	bvcSub      *databus.Databus
+	scheFunc    map[string]func()
+	topicClient topic.TopicClient
+}
+
+func init() {
+	flag.StringVar(&srvName, "srv", "", "service name")
+}
+
+func newTopicClient() topic.TopicClient {
+	topicClient, err := topic.NewClient(nil)
+	if err != nil {
+		log.Errorw(context.Background(), "log", "get topic client fail")
+		panic(err)
+	}
+	return topicClient
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:           c,
+		dao:         dao.New(c),
+		searchChan:  make(chan string, 1),
+		topicClient: newTopicClient(),
+	}
+	s.scheFunc = s.initScheduleFunc()
+	if srvName != "" {
+		switch srvName {
+		case "test":
+			s.Test()
+		case "syncsv2es":
+			s.taskSyncVideo2ES()
+		case "syncuserdmg":
+			s.taskSyncUserDmg()
+		case "rminvalides":
+			s.taskRmInvalidES()
+		case "regcmtall":
+			s.AutoRegAll(context.Background())
+		case "syncuserbase":
+			s.taskSyncUsrBaseFromVideo(context.Background())
+		case "syncuserbasic":
+			s.taskSyncPegasusUserBasic()
+		case "syncsearchvideo":
+			s.SyncVideo2Search()
+		case "syncsearchuser":
+			s.SyncUser2Search()
+		case "syncsearchsug":
+			s.SyncSug2Search()
+		case "upubface":
+			s.UpdateUsrBaseFace()
+		case "SysMsgTask":
+			s.SysMsgTask()
+		case "UserProfile":
+			s.UserProfileUpdate()
+		case "pushbvc":
+			s.commitCID()
+		case "cmscheckback":
+			s.TransToCheckBack()
+		}
+		return s
+	}
+	//初始化databus
+	s.initDatabus()
+	//启动相关rountine
+	s.launchCor()
+	//定时任务启动
+	if env.DeployEnv == env.DeployEnvProd {
+		s.runScheduler(c.Scheduler)
+	}
+	return s
+}
+
+//runScheduler .1
+func (s *Service) runScheduler(c *conf.Scheduler) {
+	sche := cron.New()
+
+	t := reflect.TypeOf(*c)
+	v := reflect.ValueOf(*c)
+	for i := 0; i < v.NumField(); i++ {
+		//排除配置为空的任务
+		if job := v.Field(i).String(); job != "" {
+			fn := t.Field(i).Name
+			//从映射集中取出对应函数
+			if f, ok := s.scheFunc[fn]; !ok {
+				fmt.Printf("skip[%s]\n", fn)
+				continue
+			} else {
+				fmt.Printf("run[%s]\n", fn)
+				if err := sche.AddFunc(job, f); err != nil {
+					panic(err)
+				}
+			}
+		}
+	}
+	sche.Start()
+}
+
+//Test 测试
+func (s *Service) Test() {
+	log.Info("HeartBeat:%s", time.Now())
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}
+
+// initScheduleFunc 任务结构与函数映射关系
+// map {key: conf.Scheduler结构体中字段名 value: 对应执行的函数}
+func (s *Service) initScheduleFunc() map[string]func() {
+	return map[string]func(){
+		"Test":             s.Test,
+		"CheckVideo2ES":    s.taskSyncVideo2ES,
+		"SyncUserDmg":      s.taskSyncUserDmg,
+		"SyncUpUserDmg":    s.taskSyncUpUserDmg,
+		"CheckVideo":       s.taskCheckVideo,
+		"CheckVideoSt":     s.taskCheckVideoStatistics,
+		"CheckVideoStHv":   s.taskCheckVideoStatisticsHive,
+		"CheckVideoTag":    s.taskCheckVideoTag,
+		"CheckTag":         s.taskCheckTag,
+		"SyncUsrSta":       s.taskSyncUsrStaFromHive,
+		"SysMsgTask":       s.SysMsgTask,
+		"UserProfileBbq":   s.UserProfileUpdate,
+		"TransToReview":    s.TransToReview,
+		"TransToCheckBack": s.TransToCheckBack,
+	}
+}
+
+func (s *Service) launchCor() {
+	time.Sleep(time.Second * 3)
+	if env.DeployEnv == env.DeployEnvProd {
+		go s.SyncSearch()
+	}
+	go s.videoBinlogSub()
+	go s.videoRepositoryBinlogSub()
+	go s.BvcTransSub()
+}
+
+func (s *Service) initDatabus() {
+	s.videoSub = databus.New(conf.Conf.Databus["videosub"])
+	s.videoRep = databus.New(conf.Conf.Databus["videorep"])
+	s.bvcSub = databus.New(conf.Conf.Databus["bvcsub"])
+}

app/job/bbq/video/service/user.go

@@ -0,0 +1,237 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	video "go-common/app/service/bbq/video/api/grpc/v1"
+	"go-common/library/log"
+	"os"
+	"time"
+)
+
+//taskSyncUserDmg 同步用户画像
+func (s *Service) taskSyncUserDmg() {
+	jobURL, err := s.dao.QueryUserDmg(context.Background())
+	if err != nil {
+		log.Error("get user dmg err(%v)", err)
+		return
+	}
+	urls, err := s.dao.QueryJobStatus(context.Background(), jobURL)
+	if err != nil {
+		log.Error("get user dmg job result err(%v)", err)
+		return
+	}
+	for _, url := range urls {
+		go func(url string) {
+			fpath, err := s.dao.Download(url, "")
+			if err != nil {
+				return
+			}
+			s.dao.ReadLine(fpath, s.dao.HandlerUserDmg)
+			os.RemoveAll(fpath)
+		}(url)
+	}
+}
+
+//UserProfileUpdate bbq_user_profile
+func (s *Service) UserProfileUpdate() {
+	_, err := s.dao.UserProfileGet(context.Background())
+	if err != nil {
+		log.Error("get user dmg err(%v)", err)
+	}
+}
+
+//taskSyncUserDmg 同步天马推荐用户画像
+func (s *Service) taskSyncPegasusUserBasic() {
+	_, err := s.dao.QueryUserBasic(context.Background())
+	if err != nil {
+		log.Error("get user dmg err(%v)", err)
+	}
+}
+
+//taskSyncUpUserDmg 同步up主画像
+func (s *Service) taskSyncUpUserDmg() {
+	log.Infov(context.Background(), log.KV("event", "sync_up_user_dmg"))
+	var mid = int64(0)
+	for {
+		log.Infov(context.Background(), log.KV("event", "one_sync_up_user_dmg"), log.KV("mid", mid))
+		upUserDmgs, err := s.dao.QueryUpUserDmg(context.Background(), mid)
+		if err != nil {
+			log.Error("get up user dmg err(%v)", err)
+			return
+		}
+
+		if len(upUserDmgs) == 0 {
+			break
+		}
+
+		for _, upUserDmg := range upUserDmgs {
+			mid = upUserDmg.MID
+			fmt.Println(upUserDmg.MID)
+			if err = s.dao.InsertOnDup(context.Background(), upUserDmg); err != nil {
+				log.Error("user dmg insert on dup failed,mid :%v, err:%v", upUserDmg.MID, err)
+			}
+		}
+	}
+	// TODO: 考虑个好方法
+	// 万一对方接口有问题，那就都完了
+	//s.dao.DelUpUserDmg(context.Background())
+}
+
+// 同步up主画像 从hive更新到user_statistics_hive
+func (s *Service) taskSyncUsrStaFromHive() {
+	log.Infov(context.Background(), log.KV("event", "taskSyncUsrStaFromHive"))
+	var (
+		err    error
+		jobURL string
+		urls   []string
+		url    string
+		fpath  string
+		try    int
+		date   = time.Now().AddDate(0, 0, -1).Format("20060102")
+	)
+	for try = 1; try <= 3; try++ {
+		//发起hive查询,拿到url
+		if jobURL, err = s.dao.QueryUpMid(context.Background(), date); err != nil {
+			log.Warn("taskSyncUsrStaFromHive try and init query hive failed, err(%v)", err)
+			continue
+		}
+		log.Info("taskSyncUsrStaFromHive init query hive success")
+		//查询job状态
+		if urls, err = s.dao.QueryJobStatus(context.Background(), jobURL); err != nil {
+			log.Warn("taskSyncUsrStaFromHive try and get hive query status failed, err(%v)", err)
+			continue
+		}
+		break
+	}
+	if err != nil {
+		log.Error("taskSyncUsrStaFromHive init and get hive query status failed, err(%v)", err)
+		return
+	}
+	for _, url = range urls {
+		for try = 0; try <= 3; try++ {
+			if fpath, err = s.dao.Download(url, ""); err != nil {
+				log.Warn("taskSyncUsrInfoFromHive try and download file (%v) failed, err(%v)", url, err)
+				time.Sleep(time.Duration(try*10) * time.Second)
+				continue
+			}
+			s.dao.ReadLines(fpath, s.dao.HandlerMids)
+			os.RemoveAll(fpath)
+			return
+		}
+		if err != nil {
+			log.Error("taskSyncUsrInfoFromHive download file (%v) failed, err(%v)", url, err)
+		}
+	}
+}
+
+//从video表同步up画像
+//func (s *Service) taskSyncUsrBaseFromVideo(c context.Context) {
+//	log.Infov(context.Background(), log.KV("event", "taskSyncUsrInfoFromVideo"))
+//	//get mids
+//	var (
+//		mids *[]int64
+//		err  error
+//		i    int8
+//		mid  int64
+//		req  *video.SyncUserBaseResponse
+//	)
+//	for i := 0; i <= 3; i++ {
+//		mids, err = s.dao.SelMidFromVideo()
+//		if err != nil {
+//			log.Info("taskSyncUsrInfoFromVideo try and get up mid failed , err(%v)", err)
+//		} else {
+//			break
+//		}
+//	}
+//	if err != nil {
+//		log.Error("taskSyncUsrInfoFromVideo get up mid failed, err(%v)", err)
+//	}
+//	//get userinfo and update
+//	for _, mid = range *mids {
+//		//重试
+//		for i = 0; i <= 3; i++ {
+//			if req, err = s.dao.VideoClient.SyncUserBase(c, &video.SyncMidRequset{MID: mid}); err != nil || req.Affc == -1 {
+//				log.Info("taskSyncUsrInfoFromVideo try and failed, mid(%v), err(%v)", mid, err)
+//			} else {
+//				break
+//			}
+//		}
+//		if err != nil {
+//			log.Error("taskSyncUsrInfoFromVideo failed, mid(%v), err(%v)", mid, err)
+//		}
+//	}
+//}
+
+//从video表同步up画像
+func (s *Service) taskSyncUsrBaseFromVideo(c context.Context) {
+	fmt.Println("taskSyncUsrBaseFromVideo start")
+	log.Infov(context.Background(), log.KV("event", "taskSyncUsrInfoFromVideo"))
+	//get mids
+	var (
+		mids []int64
+		err  error
+		req  *video.SyncUserBaseResponse
+	)
+	for try := 0; try <= 3; try++ {
+		mids, err = s.dao.SelMidFromVideo()
+		if err != nil {
+			log.Info("taskSyncUsrInfoFromVideo try and get up mid failed , err(%v)", err)
+		} else {
+			break
+		}
+	}
+	if err != nil {
+		log.Error("taskSyncUsrInfoFromVideo get up mid failed, err(%v)", err)
+	}
+	i := len(mids) / 50
+	//get userinfo and update
+	for j := 1; j <= i; j++ {
+		for try := 0; try <= 3; try++ {
+			if req, err = s.dao.VideoClient.SyncUserBases(c, &video.SyncMidsRequset{MIDS: mids[(j-1)*50 : j*50]}); err != nil {
+				log.Info("taskSyncUsrInfoFromVideo try and failed, err(%v)", err)
+			} else {
+				break
+			}
+		}
+		if err != nil {
+			log.Error("taskSyncUsrInfoFromVideo failed, err(%v)", err)
+		} else {
+			log.Info("taskSyncUsrInfoFromVideo success ,affected %v rows", req.Affc)
+		}
+	}
+	if i*50 < len(mids) {
+		for try := 0; try <= 3; try++ {
+			if req, err = s.dao.VideoClient.SyncUserBases(c, &video.SyncMidsRequset{MIDS: mids[i*50:]}); err != nil {
+				log.Info("taskSyncUsrInfoFromVideo try and failed, err(%v)", err)
+			} else {
+				break
+			}
+		}
+		if err != nil {
+			log.Error("taskSyncUsrInfoFromVideo failed, err(%v)", err)
+		} else {
+			log.Info("taskSyncUsrInfoFromVideo success ,affected %v rows", req.Affc)
+		}
+	}
+}
+
+// UpdateUsrBaseFace 更新user_base里的face字段
+func (s *Service) UpdateUsrBaseFace() (err error) {
+	var (
+		mids []int64
+	)
+	log.Infov(context.Background(), log.KV("event", "UpdateUsrBaseFace"))
+	for i := 0; ; i++ {
+		mids, err = s.dao.SelMidFromUserBase(i * 1000)
+		if err != nil {
+			log.Error("UpdateUsrBaseFace select mid failed")
+			return
+		}
+		if len(mids) == 0 {
+			break
+		}
+		s.dao.UpUserBases(context.Background(), mids)
+	}
+	return
+}

app/job/bbq/video/service/video.go

@@ -0,0 +1,411 @@
+package service
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"go-common/app/job/bbq/video/dao"
+	searchv1 "go-common/app/service/bbq/search/api/grpc/v1"
+	"go-common/library/log"
+	"io"
+	"net/url"
+	"os"
+
+	"strconv"
+	"time"
+)
+
+const (
+	_retryTimes = 3
+
+	_selection = 5 //运营精选状态
+)
+
+//taskCheckVideoDBVSES 视频全量脚本
+func (s *Service) taskSyncVideo2ES() {
+	var step time.Duration
+	var id int64
+	for {
+		ids, videos, err := s.dao.VideoList(context.Background(), id)
+		if err != nil {
+			log.Error("sync video err(%v)", err)
+			return
+		}
+		if videos == nil {
+			return
+		}
+		videoStatisticsHive, _ := s.dao.VideoStatisticsHiveList(context.Background(), ids)
+		videoStatistics, _ := s.dao.VideoStatisticsList(context.Background(), ids)
+		videoTags, _ := s.dao.VideoTagsList(context.Background(), ids)
+		req := new(searchv1.SaveVideoRequest)
+		for _, v := range videos {
+			fmt.Println(v.SVID)
+			id = v.SVID
+			tmp := &searchv1.VideoESInfo{
+				SVID:         v.SVID,
+				Title:        v.Title,
+				Content:      v.Content,
+				MID:          v.MID,
+				CID:          v.CID,
+				Pubtime:      int64(v.Pubtime),
+				Ctime:        int64(v.Ctime),
+				Mtime:        int64(v.Mtime),
+				Duration:     v.Duration,
+				Original:     v.Original,
+				State:        v.State,
+				VerID:        v.VerID,
+				Ver:          v.Ver,
+				From:         v.From,
+				AVID:         v.AVID,
+				Tid:          v.Tid,
+				SubTid:       v.SubTid,
+				ISFullScreen: v.ISFullScreen,
+				Score:        v.Score,
+			}
+			if videoStatisticsHive[id] != nil {
+				tmp.PlayHive = videoStatisticsHive[id].PlayHive
+				tmp.FavHive = videoStatisticsHive[id].FavHive
+				tmp.CoinHive = videoStatisticsHive[id].CoinHive
+				tmp.SubtitlesHive = videoStatisticsHive[id].SubtitlesHive
+				tmp.LikesHive = videoStatisticsHive[id].LikesHive
+				tmp.ShareHive = videoStatisticsHive[id].ShareHive
+				tmp.ReportHive = videoStatisticsHive[id].ReportHive
+				tmp.DurationDailyHive = videoStatisticsHive[id].DurationDailyHive
+				tmp.DurationAllHive = videoStatisticsHive[id].DurationAllHive
+				tmp.ReplyHive = videoStatisticsHive[id].ReplyHive
+				tmp.ShareDailyHive = videoStatisticsHive[id].ShareDailyHive
+				tmp.PlayDailyHive = videoStatisticsHive[id].PlayDailyHive
+				tmp.SubtitlesDailyHive = videoStatisticsHive[id].SubtitlesDailyHive
+				tmp.LikesDailyHive = videoStatisticsHive[id].LikesDailyHive
+				tmp.FavDailyHive = videoStatisticsHive[id].FavDailyHive
+				tmp.ReplyDailyHive = videoStatisticsHive[id].ReplyDailyHive
+				tmp.AccessHive = videoStatisticsHive[id].AccessHive
+			}
+			if videoStatistics[id] != nil {
+				tmp.Play = videoStatistics[id].Play
+				tmp.Subtitles = videoStatistics[id].Subtitles
+				tmp.Like = videoStatistics[id].Like
+				tmp.Share = videoStatistics[id].Share
+				tmp.Report = videoStatistics[id].Report
+			}
+			if videoTags[id] != nil {
+				tmp.Tags = videoTags[id]
+			}
+			req.List = append(req.List, tmp)
+		}
+		step = 1
+		for {
+			if _, err := s.dao.SearchClient.SaveVideo(context.Background(), req); err != nil {
+				time.Sleep(step * time.Second)
+				step++
+				continue
+			}
+			break
+		}
+	}
+
+}
+
+//SaveVideo2ES 保存视频到es
+func (s *Service) SaveVideo2ES(ids string) (res bool) {
+	res = true
+	if len(ids) == 0 {
+		return
+	}
+	videos, err := s.dao.VideoListByIDs(context.Background(), ids)
+	if err != nil || videos == nil {
+		res = false
+		return
+	}
+	videoStatisticsHive, _ := s.dao.VideoStatisticsHiveList(context.Background(), ids)
+	videoStatistics, _ := s.dao.VideoStatisticsList(context.Background(), ids)
+	// videoTags, _ := s.dao.VideoTagsList(context.Background(), ids)
+	var step time.Duration
+	var id int64
+	req := new(searchv1.SaveVideoRequest)
+	for _, v := range videos {
+		id = v.SVID
+		fmt.Println(id)
+		tmp := &searchv1.VideoESInfo{
+			SVID:         v.SVID,
+			Title:        v.Title,
+			Content:      v.Content,
+			MID:          v.MID,
+			CID:          v.CID,
+			Pubtime:      int64(v.Pubtime),
+			Ctime:        int64(v.Ctime),
+			Mtime:        int64(v.Mtime),
+			Duration:     v.Duration,
+			Original:     v.Original,
+			State:        v.State,
+			VerID:        v.VerID,
+			Ver:          v.Ver,
+			From:         v.From,
+			AVID:         v.AVID,
+			Tid:          v.Tid,
+			SubTid:       v.SubTid,
+			ISFullScreen: v.ISFullScreen,
+			Score:        v.Score,
+		}
+		if videoStatisticsHive[id] != nil {
+			tmp.PlayHive = videoStatisticsHive[id].PlayHive
+			tmp.FavHive = videoStatisticsHive[id].FavHive
+			tmp.CoinHive = videoStatisticsHive[id].CoinHive
+			tmp.SubtitlesHive = videoStatisticsHive[id].SubtitlesHive
+			tmp.LikesHive = videoStatisticsHive[id].LikesHive
+			tmp.ShareHive = videoStatisticsHive[id].ShareHive
+			tmp.ReportHive = videoStatisticsHive[id].ReportHive
+			tmp.DurationDailyHive = videoStatisticsHive[id].DurationDailyHive
+			tmp.DurationAllHive = videoStatisticsHive[id].DurationAllHive
+			tmp.ReplyHive = videoStatisticsHive[id].ReplyHive
+			tmp.ShareDailyHive = videoStatisticsHive[id].ShareDailyHive
+			tmp.PlayDailyHive = videoStatisticsHive[id].PlayDailyHive
+			tmp.SubtitlesDailyHive = videoStatisticsHive[id].SubtitlesDailyHive
+			tmp.LikesDailyHive = videoStatisticsHive[id].LikesDailyHive
+			tmp.FavDailyHive = videoStatisticsHive[id].FavDailyHive
+			tmp.ReplyDailyHive = videoStatisticsHive[id].ReplyDailyHive
+			tmp.AccessHive = videoStatisticsHive[id].AccessHive
+		}
+		if videoStatistics[id] != nil {
+			tmp.Play = videoStatistics[id].Play
+			tmp.Subtitles = videoStatistics[id].Subtitles
+			tmp.Like = videoStatistics[id].Like
+			tmp.Share = videoStatistics[id].Share
+			tmp.Report = videoStatistics[id].Report
+		}
+		// if videoTags[id] != nil {
+		// 	tmp.Tags = videoTags[id]
+		// }
+		req.List = append(req.List, tmp)
+	}
+	step = 1
+	for {
+		if _, err := s.dao.SearchClient.SaveVideo(context.Background(), req); err != nil {
+			if step == 11 {
+				log.Error("save es err(%v) ids(%s)", err, ids)
+				res = false
+				break
+			}
+			time.Sleep(step * time.Second)
+			step++
+			continue
+		}
+		break
+	}
+	return
+}
+
+func formArrayString(arr []int64) string {
+	var res string
+	for i, v := range arr {
+		if i != 0 {
+			res += ","
+		}
+		res += strconv.FormatInt(v, 10)
+	}
+	return res
+}
+
+//deltaSync2ES 为不同表进行增量同步的脚本，baseTableQuery指明不同表的查询语句
+func (s *Service) deltaSync2ES(taskName string, baseTableQuery string) {
+	task, err := s.dao.RawCheckTask(context.Background(), taskName)
+	if err != nil {
+		log.Error("get last_chek_time fail: task=%s", taskName)
+		return
+	}
+	log.Info("get last_chek_time succ: task=%s, last_check_time=%d", taskName, task.LastCheck)
+
+	// 获得所有变更的svid
+	ids, mtime, err := s.dao.RawGetIDByMtime(baseTableQuery, task.LastCheck)
+	if err != nil {
+		log.Error("get raw id by mtime fail: task=%s, last_mtime=%d, base_table_query=%s",
+			taskName, mtime, baseTableQuery)
+		return
+	}
+	idsNum := len(ids)
+	log.Info("get changed svids: task=%s, id_num=%d", taskName, idsNum)
+	if idsNum == 0 {
+		return
+	}
+	task.LastCheck = mtime
+
+	// 对所有变更的svid分批次进行同步到es
+	for i := 0; i < idsNum; i += dao.MaxSyncESNum {
+		last := i + dao.MaxSyncESNum
+		if last > idsNum {
+			last = idsNum
+		}
+		selectedIDs := ids[i:last]
+		idsStr := formArrayString(selectedIDs)
+		if res := s.SaveVideo2ES(idsStr); !res {
+			log.Error("sync video 2 es fail: task=%s, offset=%d, id_num=%d, base_table_query=%s",
+				taskName, i, idsNum, baseTableQuery)
+			return
+		}
+		log.Info("one sync video 2 es: task=%s, offset=%d, id_num=%d", taskName, i, idsNum)
+	}
+
+	// 更新task最近check的时间点
+	if _, err := s.dao.UpdateTaskLastCheck(context.Background(), taskName, task.LastCheck); err != nil {
+		log.Error("update task last check time fail: task=%s, last_mtime=%d, base_table_query=%s",
+			taskName, task.LastCheck, baseTableQuery)
+		return
+	}
+	log.Info("sync video 2 es: task=%s, id_num=%d, last_mtime=%d", taskName, idsNum, task.LastCheck)
+}
+
+//taskCheckVideo video表增量脚本
+func (s *Service) taskCheckVideo() {
+	taskName := "checkVideo"
+	s.deltaSync2ES(taskName, dao.QueryVideoByMtime)
+}
+
+//taskCheckVideoStatistics video_statistics表增量脚本
+func (s *Service) taskCheckVideoStatistics() {
+	taskName := "checkVideoSt"
+	s.deltaSync2ES(taskName, dao.QueryVideoStatisticsByMtime)
+}
+
+//taskCheckVideoStatisticsHive video_statistics_hive表增量脚本
+func (s *Service) taskCheckVideoStatisticsHive() {
+	taskName := "checkVideoStHv"
+	s.deltaSync2ES(taskName, dao.QueryVideoStatisticsHiveByMtime)
+}
+
+//taskCheckVideoTag video_tag表增量脚本
+func (s *Service) taskCheckVideoTag() {
+	taskName := "checkVideoTag"
+	s.deltaSync2ES(taskName, dao.QueryVideoTagByMtime)
+}
+
+//taskCheckTag tag表增量脚本
+func (s *Service) taskCheckTag() {
+	taskName := "checkTag"
+	task, err := s.dao.RawCheckTask(context.Background(), taskName)
+	if err != nil {
+		log.Error("get last_chek_time fail: task=%s", taskName)
+		return
+	}
+	log.Info("get last_chek_time succ: task=%s, last_check_time=%d", taskName, task.LastCheck)
+	for {
+		ids, mtime, err := s.dao.RawTagByMtime(context.Background(), task.LastCheck)
+		if err != nil || len(ids) == 0 {
+			return
+		}
+		id := int64(0)
+		for {
+			svids, temp, err := s.dao.RawVideoTagByIDs(context.Background(), ids, id)
+			if err != nil {
+				return
+			}
+			if len(svids) == 0 {
+				break
+			}
+			if flag := s.SaveVideo2ES(svids); !flag {
+				return
+			}
+			id = temp
+		}
+		if num, err := s.dao.UpdateTaskLastCheck(context.Background(), taskName, mtime); err != nil || num == 0 {
+			return
+		}
+		task.LastCheck = mtime
+	}
+}
+
+// taskRmInvalidES 删除es中多余的视频
+func (s *Service) taskRmInvalidES() {
+	fmt.Println("aaa")
+	esReq := new(searchv1.ESVideoDataRequest)
+	delReq := new(searchv1.DelVideoBySVIDRequest)
+	svid := int64(0)
+	query := `{"query":{"range":{"svid":{"gt":%d}}},"sort":[{"svid":"asc"}],"from":0,"size":10}`
+	for {
+		esReq.Query = fmt.Sprintf(query, svid)
+		res, err := s.dao.SearchClient.ESVideoData(context.Background(), esReq)
+		if err != nil {
+			return
+		}
+		svids := make([]string, 0)
+		for _, v := range res.List {
+			svids = append(svids, strconv.Itoa(int(v.SVID)))
+			svid = v.SVID
+		}
+		vs, err := s.dao.RawVideoBySVIDS(context.Background(), svids)
+		if err != nil {
+			return
+		}
+		notList := make([]int64, 0)
+		for _, v := range res.List {
+			if _, ok := vs[v.SVID]; !ok {
+				fmt.Println(v.SVID)
+				notList = append(notList, v.SVID)
+			}
+		}
+		if len(notList) != 0 {
+			delReq.SVIDs = notList
+			s.dao.SearchClient.DelVideoBySVID(context.Background(), delReq)
+		}
+	}
+}
+
+func (s *Service) commitCID() {
+	ctx := context.Background()
+	path := s.c.URLs["bvc_push"]
+	if path == "" {
+		return
+	}
+	srcPath := s.c.Path["cids"]
+	if srcPath == "" {
+		return
+	}
+	if srcPath == "" {
+		log.Error("sugsrc path is empty")
+		return
+	}
+	src, err := os.Open(srcPath)
+	if err != nil {
+		log.Error("writeSug os.Open source sug error(%v)", err)
+		return
+	}
+	defer src.Close()
+	br := bufio.NewReader(src)
+	i := 1
+	for {
+		a, _, c := br.ReadLine()
+		if c == io.EOF {
+			break
+		}
+		cid, err := strconv.ParseInt(string(a), 10, 64)
+		if err != nil {
+			log.Error("parse err [%v]", err)
+			continue
+		}
+		svid, err := s.dao.GetSvidByCid(ctx, cid)
+		if err != nil {
+			continue
+		}
+		params := url.Values{}
+		params.Set("svid", strconv.FormatInt(svid, 10))
+		params.Set("cid", string(a))
+		req, err := s.dao.HTTPClient.NewRequest("GET", path, "", params)
+		if err != nil {
+			log.Error("error(%v)", err)
+			continue
+		}
+		var res struct {
+			Code int    `json:"code"`
+			Msg  string `json:"message"`
+		}
+		if err = s.dao.HTTPClient.Do(ctx, req, &res); err != nil {
+			log.Errorv(ctx, log.KV("log", fmt.Sprintf("err[%v]", err)))
+			continue
+		}
+		if res.Code != 0 {
+			log.Errorv(ctx, log.KV("log", fmt.Sprintf("error(%v)", err)))
+		} else {
+			log.Info("commit svid:%d cid:%d success No.%d", svid, cid, i)
+		}
+		i++
+	}
+}