Create & Init Project...

app/job/main/archive/BUILD

@@ -0,0 +1,30 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/archive/cmd:all-srcs",
+        "//app/job/main/archive/conf:all-srcs",
+        "//app/job/main/archive/dao/archive:all-srcs",
+        "//app/job/main/archive/dao/email:all-srcs",
+        "//app/job/main/archive/dao/monitor:all-srcs",
+        "//app/job/main/archive/dao/reply:all-srcs",
+        "//app/job/main/archive/dao/result:all-srcs",
+        "//app/job/main/archive/http:all-srcs",
+        "//app/job/main/archive/model/archive:all-srcs",
+        "//app/job/main/archive/model/databus:all-srcs",
+        "//app/job/main/archive/model/dm:all-srcs",
+        "//app/job/main/archive/model/pgc:all-srcs",
+        "//app/job/main/archive/model/result:all-srcs",
+        "//app/job/main/archive/model/retry:all-srcs",
+        "//app/job/main/archive/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/CHANGELOG.md

@@ -0,0 +1,431 @@
+#### 稿件字段同步
+
+##### Version 2.13.7
+> 1.去除track表迁移first_pass表
+
+##### Version 2.13.6
+> 1.同步联合创作人archive_staff
+
+##### Version 2.13.5
+> 1.修复err后重复retry tranResult & -->chan
+
+##### Version 2.13.4
+> 1.走account的grpc
+
+##### Version 2.13.3
+> 1.扩展archive-service的配置
+
+##### Version 2.13.2
+> 1.账号信息空判断
+
+##### Version 2.13.1
+> 1.账号databus有重复消息
+
+##### Version 2.13.0
+> 1.账号notify走databus，记录日志
+
+##### Version 2.12.13
+> 1.账号notify走databus
+
+##### Version 2.12.12
+> 1.增加databus消费，重新生成用户昵称和头像缓存
+
+##### Version 2.12.11
+> 1.修改第一P的判断逻辑
+
+##### Version 2.12.10
+> 1.同步稿件时判断是否为vupload
+
+##### Version 2.12.9
+> 1.更新archive表的视频id，分辨率字段
+
+##### Version 2.12.8
+> 1.新表字段修改
+
+##### Version 2.12.7
+> 1.增加分辨率字段
+
+##### Version 2.12.6
+> 1.全量新表
+
+##### Version 2.12.5
+> 1.灰度新表
+
+##### Version 2.12.4
+> 1.切换BM
+
+##### Version 2.12.3
+> 1.计算稿件总时长
+
+##### Version 2.12.2
+> 1.聚合所有topic消费  
+
+##### Version 2.12.1
+> 1.发送statDm-T消息  
+
+##### Version 2.12.0
+> 1.迁移到主站目录下  
+
+##### Version 2.11.1
+> 1.弹幕计数做聚合  
+
+##### Version 2.11.0
+> 1.弹幕计数  
+
+##### Version 2.10.4
+> 1.删除所有插入track的代码  
+
+##### Version 2.10.3
+> 1.配置offset，停止track记录  
+
+##### Version 2.10.2
+> 1.补充单元测试  
+
+##### Version 2.10.2
+> 1.改进报警文案  
+
+##### Version 2.10.1
+> 1.pgc异步  
+
+##### Version 2.10.0
+> 1.增加监控  
+> 2.pgc与ugc分开  
+
+##### Version 2.9.4
+> 1.增加无限塞回重试逻辑  
+
+##### Version 2.9.3
+> 1.增加force_sync消息  
+
+##### Version 2.9.2
+> 1.处理first_round消息  
+
+##### Version 2.9.1
+> 1.全量通过databus更新稿件缓存  
+
+##### Version 2.9.0
+> 1.通过databus消息更新稿件缓存  
+> 2.灰度30%  
+
+##### Version 2.8.1
+> 1.调整 passed 然后又改了回来 = =
+
+##### Version 2.8.0
+> 1.archive_track 迁移至 hbase  
+> 2.调整 passed 逻辑改查 archive_oper 表  
+
+##### Version 2.7.1
+> 1.databus通知吐出dynamic字段  
+
+##### Version 2.7.0
+> 1.删除hzxs缓存清楚  
+
+##### Version 2.6.1
+> 1.archive增加dynamic字段  
+
+##### Version 2.6.0
+> 1.去除pgc的databus订阅  
+
+##### Version 2.5.2
+> 1.回滚，video relation表数据有误  
+
+##### Version 2.5.1
+> 1.result库支持video分表逻辑(灰度10%的稿件)   
+
+##### Version 2.5.0
+> 1.删除cdn代码  
+> 2.完善更新db的日志  
+
+##### Version 2.4.1
+> 1.切新的httpclient  
+
+##### Version 2.4.0
+> 1.同步result库判断微调  
+
+##### Version 2.3.0
+> 1.删除purge cdn逻辑  
+
+##### Version 2.2.7
+> 1.routine数量配置化&优化报警逻辑  
+
+##### Version 2.2.6
+> 1.多routine更新数据库&缓存&每分钟报警  
+
+##### Version 2.2.5
+> 1.cid为0不同步result  
+
+##### Version 2.2.4
+> 1.add error log  
+
+##### Version 2.2.3
+> 1.fix cids 0  
+
+##### Version 2.2.1
+> 1.修复pgc同步result逻辑  
+
+##### Version 2.2.0
+> 1.video缓存更新逻辑优化  
+
+##### Version 2.1.2
+> 1.pgc第二次过审不同步result库  
+
+##### Version 2.1.1
+> 1.修复videos插入失败的bug   
+
+##### Version 2.1.0
+> 1.模块分层  
+
+##### Version 2.0.0
+> 1.大仓库版本，依赖新的go-common  
+> 2.archive_result库的archive_video表有数据更新会调archive-service接口更新分P详情信息  
+
+##### Version 1.14.0
+> 1.databus send error 时无限重试  
+
+##### Version 1.13.0
+> 1.暂时去掉HK节点的通知  
+> 2.增加notify的databus  
+
+##### Version 1.12.0
+> 1.增加https页面的cdn purge  
+
+##### Version 1.11.0
+> 1.增加result库的partition消费监控  
+
+##### Version 1.10.0
+> 1.update field增加group2  
+
+##### Version 1.9.1
+> 1.archive的channal根据aid取余，每个channal只有一个goroutine消费，避免消费乱序  
+
+##### Version 1.9.0
+> 1.升级go-common&go-business  
+> 2.兼容manager后台修改稿件归属的mid缓存  
+
+##### Version 1.8.0
+> 1.所有缓存清理走archive_result库  
+
+##### Version 1.7.15
+> 1.迁移archive-service的group2缓存到result库  
+
+##### Version 1.7.14
+> 1.增加monitor/ping   
+
+##### Version 1.7.13
+> 1.group1使用archive_result库更新缓存  
+
+##### Version 1.7.12
+> 1.bugfix,任务分发:稿件修改分区，写redis逻辑错误    
+
+##### Version 1.7.11
+> 1.移除所有group1的调用  
+
+##### Version 1.7.10
+> 1.bugfix,修复track的redis  
+
+##### Version 1.7.9
+> 1.bugfix,修复track的redis  
+
+##### Version 1.7.8
+> 1.bugfix,修复sql语句  
+
+##### Version 1.7.7
+> 1.增加香港节点的配置  
+
+##### Version 1.7.6
+> 1.已删除的稿件不记录track  
+
+##### Version 1.7.5
+> 1.dede_arctype替换为archive_type并修改相关逻辑  
+> 2.添加杭州group  
+
+##### Version 1.7.5
+> 1.视频track变更增加标题简介  
+
+##### Version 1.7.4
+> 1.去掉feed push  
+
+##### Version 1.7.3
+> 1.升级go-common和go-business  
+> 2.接入新版配置中心  
+
+##### Version 1.7.2
+> 1.增加proc处理  
+> 2.修改track video待审重复计数bug  
+
+##### Version 1.7.1
+> 1.增加track video  
+
+##### Version 1.7.0
+> 1.增加track信息记录  
+> 2.计数更新硬币数  
+
+##### Version 1.6.24
+> 1.增加后台报表统计  
+> 2.track表备注增加活动id  
+> 3.增加upcache错误重试  
+
+##### Version 1.6.23
+> 1.增加接口失败重试  
+
+##### Version 1.6.22
+> 1.稿件变更无脑add/del  
+> 2.基于无脑变更，去掉发布时间和属性变化通知  
+
+##### Version 1.6.21
+> 1.升级vendor  
+
+##### Version 1.6.20
+> 1.增加archive-service group2 缓存增量更新  
+
+##### Version 1.6.19
+> 1.修复日志错误  
+
+##### Version 1.6.18
+> 1.移除stat的databus消费
+
+##### Version 1.6.17
+> 1.移除评论注册  
+
+##### Version 1.6.16
+> 1.archive缓存走RPC  
+
+##### Version 1.6.15
+> 1.根据state状态记录access变更  
+
+##### Version 1.6.14
+> 1.track去除没必要的记录  
+
+##### Version 1.6.13
+> 1.内部接口地址规范化  
+
+##### Version 1.6.12
+> 1.track兼容round改动2  
+
+##### Version 1.6.11
+> 1.track兼容round的改动  
+
+##### Version 1.6.10
+> 1.track增加attr和access  
+
+##### Version 1.6.9
+> 1.增加attr不在列表输出的判断  
+
+##### Version 1.6.8
+> 1.增加纪录片的表同步  
+
+##### Version 1.6.7
+> 1.up更改后通知评论替换subject的mid  
+
+##### Version 1.6.6
+> 1.修复track时间错误  
+
+##### Version 1.6.5
+> 1.修复xcode判断错误  
+
+##### Version 1.6.4
+> 1.增加PGC表同步逻辑  
+
+##### Version 1.6.3
+> 1.临时去除attr发邮件判断  
+
+##### Version 1.6.2
+> 1.增加track追踪  
+
+##### Version 1.6.1
+> 1.修复insert时的json解析错误  
+
+##### Version 1.6.0
+
+> 1.稿件依赖databus  
+
+##### Version 1.5.7
+
+> 1.kafka to databus
+
+##### Version 1.5.6
+
+> 1.增加insert事件 评论subject注册  
+
+##### Version 1.5.5
+
+> 1.增加monitor监控  
+
+##### Version 1.5.4
+
+> 1.fix sub databus bug  
+
+##### Version 1.5.3
+
+> 1.刷新cdn加条件  
+
+##### Version 1.5.2
+
+> 1.稿件计数消费databus  
+
+##### Version 1.5.1
+
+> 1.更新stat cache的Rpc为2  
+
+##### Version 1.5.0
+
+> 1.番剧和电影状态变更需要发送邮件  
+
+##### Version 1.4.4
+
+> 1.[consumer]去除archive binlog调用tag change接口
+
+##### Version 1.4.3
+
+> 1.[consumer]稿件推送动态改为过审就推  
+
+##### Version 1.4.2
+
+> 1.[consumer]修复tag过审同步问题  
+
+##### Version 1.4.1
+
+> 1.[consumer]增加sleep控制消费速度  
+
+##### Version 1.4.0
+
+> 1.[consumer]增加稿件变动purge  
+
+##### Version 1.3.0
+
+> 1.[consumer]增加archive的事件字段变更，新老字段  
+
+##### Version 1.2.0
+
+> 1.[consumer]修改archive的事件注册，分通过、不通过、无差别删除cache  
+
+##### Version 1.1.5
+
+> 1.[consumer]修改archive的cache更新接口  
+
+##### Version 1.1.4
+
+> 1.[consumer]修复没过审视频被推tag动态问题  
+
+##### Version 1.1.3
+
+> 1.[consumer]修正通过tag服务批量获取tag信息签名错误  
+> 2.[consumer]修改调用tag change 接口mid错误  
+
+##### Version 1.1.2
+
+> 1.[consumer]feed调用历史数量判断修改，防止feed不成功  
+
+##### Version 1.1.1
+
+> 1.[consumer]feed调用加稿件审核历史判断  
+
+##### Version 1.1.0
+
+> 1.[consumer]增加分区视频列表cache更新和删除  
+> 2.[consumer]修复改tag的更新  
+
+##### Version 1.0.0
+
+> 1.[producer]binlog同步  
+> 2.[consumer]信息消费更新缓存等

app/job/main/archive/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+peiyifei
+
+# Author 
+peiyifei
+haoguanwei
+yujia
+
+# Reviewer
+peiyifei
+haoguanwei

app/job/main/archive/OWNERS

@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- haoguanwei
+- peiyifei
+- yujia
+labels:
+- job
+- job/main/archive
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- haoguanwei
+- peiyifei
+- yujia

app/job/main/archive/README.md

@@ -0,0 +1,13 @@
+#### archive-job
+
+##### 项目简介
+> 1.提供稿件相关的自动化逻辑  
+
+##### 编译环境
+> 请只用golang v1.8.x以上版本编译执行。  
+
+##### 依赖包
+> 1.公共包go-common  
+
+##### 特别说明
+> 1.model目录可能会被其他项目引用，请谨慎请改并通知各方。  

app/job/main/archive/cmd/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = [
+        "archive-job-example.toml",
+        "archive-job-test.toml",
+    ],
+    importpath = "go-common/app/job/main/archive/cmd",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/http:go_default_library",
+        "//app/job/main/archive/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/cmd/archive-job-example.toml

@@ -0,0 +1,77 @@
+version = "1.0.0"
+user    = "nobody"
+pid     = "/tmp/archive-job.pid"
+dir     = "./"
+perf    = "127.0.0.1:6161"
+trace   = false
+debug   = false
+env     = "test"
+
+tick = "50ms"
+
+[xlog]
+dir = "/data/log/archive-job/"
+
+[app]
+key    = "e7482d29be4a95b8"
+secret = "9e803791cdef756e75faee68e12b7442"
+
+[host]
+apiCo = "http://api.bilibili.co"
+
+[httpClient]
+dial      = "100ms"
+timeout   = "1s"
+keepAlive = "60s"
+timer     = 128
+
+[databus]
+key = "0Pub71WwEMKXu63qtztu"
+secret = "0Pub71WwEMKXu63qtztv"
+group = "Stat-UGC-S"
+topic = "Stat-T"
+action = "sub"
+    [databus.redis]
+    name = "archive-job/databus"
+    proto = "tcp"
+    addr = "172.16.33.56:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[kafka]
+group = "archive-job-consumer"
+topics = ["archive-push", "stat"]
+offset = false
+    [kafka.zookeeper]
+    root = "/kafka"
+    addrs = ["172.16.0.148:2181"]
+    timeout = "30s"
+
+[archiveRPC]
+    pullInterval = "10s"
+    [archiveRPC.client]
+    proto = "tcp"
+    addr = "172.16.0.26:6089"
+    timeout = "1s"
+    timer = 1000
+    [[archiveRPC.backup]]
+    proto = "tcp"
+    addr = "172.16.0.26:6089"
+    timeout = "1s"
+    timer = 1000
+    [archiveRPC.zookeeper]
+    root = "/microservice/archive-service/"
+    addrs = ["172.16.0.148:2181"]
+    timeout = "30s"
+
+[mail]
+host = "smtp.exmail.qq.com"
+port = 465
+username = "2333@bilibili.com"
+password = "2333"
+bangumi = ["233@bilibili.com", "2333@bilibili.com"]
+movie = ["233@bilibili.com", "2333@bilibili.com"]

app/job/main/archive/cmd/archive-job-test.toml

@@ -0,0 +1,223 @@
+env = "uat"
+chanSize = 50
+PGCAsync = 0
+UGCAsync = 0
+MonitorSize = 30
+WeChatToken = "uYsRqXPjnbyXDCba"
+WeChatSecret = "tUILeGIXJHFSUdwfLjBHDZJXmLgYpgEy"
+WeChantUsers = "peiyifei,haoguanwei"
+tick = "50ms"
+
+[bm]
+addr = "0.0.0.0:6161"
+timeout = "1s"
+[app]
+key    = "53e2fa226f5ad348"
+secret = "3cf6bd1b0ff671021da5f424fea4b04a"
+
+[host]
+apiCo = "http://api.bilibili.co"
+
+[httpClient]
+key    = "53e2fa226f5ad348"
+secret = "3cf6bd1b0ff671021da5f424fea4b04a"
+dial      = "100ms"
+timeout   = "1s"
+keepAlive = "60s"
+[httpClient.breaker]
+    window  = "10s"
+    sleep   = "100ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[db]
+    [db.archive]
+    name = "172.18.33.49:3306"
+    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_archive?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 20
+    idle = 10
+    idleTimeout = "4h"
+    queryTimeout = "5s"
+    execTimeout = "5s"
+    tranTimeout = "5s"
+    [db.archive.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+    [db.result]
+    name = "172.18.33.49:3306"
+    dsn = "test:test@tcp(172.16.33.205:3308)/archive_result?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8"
+    active = 20
+    idle = 10
+    idleTimeout = "4h"
+    queryTimeout = "5s"
+    execTimeout = "5s"
+    tranTimeout = "5s"
+    [db.result.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+    [db.dede]
+    name = "172.18.33.49:3307"
+    dsn = "dede:jh3GWwFgzsLHvkWsDPiSxy5Ge2vNt4Ys@tcp(172.18.33.49:3307)/bilibili?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8"
+    active = 20
+    idle = 10
+    idleTimeout ="4h"
+    queryTimeout = "5s"
+    execTimeout = "5s"
+    tranTimeout = "5s"
+    [db.dede.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+name = "archive-job/trackreport"
+proto = "tcp"
+addr = "172.18.33.61:6807"
+active = 10
+idle = 1
+dialTimeout = "1s"
+readTimeout = "1s"
+writeTimeout = "1s"
+idleTimeout = "80s"
+
+[archiveResultPub]
+key = "0QNB0ZgFozbKUCQhbTq8"
+secret = "0QNB0ZgFozbKUCQhbTq9"
+group = "ArchiveNotify-ArchiveJob-P"
+topic = "ArchiveNotify-T"
+action = "pub"
+name = "archive-job/archivepub"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 10
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[statSub]
+key = "0Q2UxI6hM1i6k6LRzDmx"
+secret= "0Q2UxI6hM1i6k6LRzDmy"
+group= "Stat-UGC-S"
+topic= "Stat-T"
+action="sub"
+name = "archive-job/stat"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+active = 1
+idle = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+expire = "1h"
+
+[DmSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "DMSubject-MainAppSvr-S"
+topic = "DMSubject-T"
+action = "sub"
+name = "archive-job/DMSubject"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[DmPub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatDM-MainAppSvr-P"
+topic = "StatDM-T"
+action = "pub"
+name = "archive-job/dmpub"
+proto = "tcp"
+addr = "172.16.33.158:6205"
+idle = 100
+active = 100
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[VideoupSub]
+key = "0QNB0ZgFozbKUCQhbTq8"
+secret = "0QNB0ZgFozbKUCQhbTq9"
+group = "ArchiveJob-Videoup2Bvc-S"
+topic = "Videoup2Bvc"
+action = "sub"
+name = "archive-job/videoupsub"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[CacheSub]
+key = "8e27ab7e39270b59"
+secret = "477df6a068d7332a163f95abbad2079c"
+group = "ArchiveUserCache-MainAppSvr-S"
+topic = "ArchiveUserCache-T"
+action = "sub"
+name = "archive-job/userCache"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "80s"
+
+[AccountNotifySub]
+key = "8e27ab7e39270b59"
+secret = "477df6a068d7332a163f95abbad2079c"
+group = "AccountNotify-MainAppSvr-S"
+topic = "AccountNotify-T"
+action = "sub"
+name = "archive-job/userCache"
+proto = "tcp"
+addr = "172.18.33.50:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "80s"
+[AccountNotifySub.discovery]
+    domain = "api.bilibili.co"
+    key = "53e2fa226f5ad348"
+    secret = "3cf6bd1b0ff671021da5f424fea4b04a"
+    region = "sh"
+    zone = "sh001"
+    env = "prod"
+
+[mail]
+host = "smtp.exmail.qq.com"
+port = 465
+username = "2333@bilibili.com"
+password = "2333"
+bangumi = ["233@bilibili.com"]
+movie = ["233@bilibili.com"]
+
+[[ArchiveServices]]
+cluster="group1"
+[[ArchiveServices]]
+cluster="group2"

app/job/main/archive/cmd/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/archive/conf"
+	"go-common/app/job/main/archive/http"
+	"go-common/app/job/main/archive/service"
+	"go-common/library/log"
+)
+
+var (
+	srv *service.Service
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	// init log
+	log.Init(conf.Conf.XLog)
+	defer log.Close()
+	log.Info("archive-job start")
+	srv = service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	signalHandler()
+}
+
+func signalHandler() {
+	var (
+		err error
+		ch  = make(chan os.Signal, 1)
+	)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		si := <-ch
+		switch si {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("get a signal %s, stop the consume process", si.String())
+			if err = srv.Close(); err != nil {
+				log.Error("srv close consumer error(%v)", err)
+			}
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/archive/conf/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/archive/conf",
+    tags = ["automanaged"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/conf/conf.go

@@ -0,0 +1,129 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc"
+	"go-common/library/queue/databus"
+
+	"github.com/BurntSushi/toml"
+)
+
+// is
+var (
+	confPath string
+	Conf     = &Config{}
+	client   *conf.Client
+)
+
+// Config is
+type Config struct {
+	// Env
+	Env string
+	// interface XLog
+	XLog *log.Config
+	// host
+	Host *Host
+	// httpClinet
+	HTTPClient *bm.ClientConfig
+	// databus
+	VideoupSub       *databus.Config
+	DmSub            *databus.Config
+	ArchiveResultPub *databus.Config
+	DmPub            *databus.Config
+	CacheSub         *databus.Config
+	AccountNotifySub *databus.Config
+	// rpc
+	ArchiveServices []*rpc.ClientConfig
+	Dm2RPC          *rpc.ClientConfig
+	// mail
+	Mail *Mail
+	// DB
+	DB *DB
+	// BM
+	BM *bm.ServerConfig
+	// Redis
+	Redis *redis.Config
+	// ChanSize aid%ChanSize
+	ChanSize    int
+	PGCAsync    int
+	UGCAsync    int
+	MonitorSize int
+	// qiye wechat
+	WeChatToken  string
+	WeChatSecret string
+	WeChantUsers string
+}
+
+// Mail is
+type Mail struct {
+	Host               string
+	Port               int
+	Username, Password string
+	Bangumi, Movie     []string
+}
+
+// Host is
+type Host struct {
+	APICo string
+}
+
+// DB is db config.
+type DB struct {
+	Archive *sql.Config
+	Result  *sql.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init config.
+func Init() (err error) {
+	if confPath != "" {
+		_, err = toml.DecodeFile(confPath, &Conf)
+		return
+	}
+	err = remote()
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf = &Config{}
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/archive/dao/archive/BUILD

@@ -0,0 +1,69 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "addit_test.go",
+        "archive_test.go",
+        "config_test.go",
+        "dao_test.go",
+        "first_pass_test.go",
+        "oper_test.go",
+        "staff_test.go",
+        "type_test.go",
+        "video_test.go",
+        "video_track_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "addit.go",
+        "archive.go",
+        "config.go",
+        "dao.go",
+        "first_pass.go",
+        "oper.go",
+        "staff.go",
+        "type.go",
+        "video.go",
+        "video_track.go",
+    ],
+    importpath = "go-common/app/job/main/archive/dao/archive",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/model/archive:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/dao/archive/addit.go

@@ -0,0 +1,28 @@
+package archive
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_additSQL = "SELECT id,aid,source,redirect_url,mission_id,up_from,order_id,dynamic FROM archive_addit WHERE aid=?"
+)
+
+// Addit get archive addit.
+func (d *Dao) Addit(c context.Context, aid int64) (addit *archive.Addit, err error) {
+	row := d.db.QueryRow(c, _additSQL, aid)
+	addit = &archive.Addit{}
+	if err = row.Scan(&addit.ID, &addit.Aid, &addit.Source, &addit.RedirectURL, &addit.MissionID, &addit.UpFrom, &addit.OrderID, &addit.Dynamic); err != nil {
+		if err == sql.ErrNoRows {
+			addit = nil
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}

app/job/main/archive/dao/archive/addit_test.go

@@ -0,0 +1,16 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Addit(t *testing.T) {
+	Convey("Addit", t, func() {
+		addit, err := d.Addit(context.TODO(), 1)
+		So(err, ShouldBeNil)
+		Println(addit)
+	})
+}

app/job/main/archive/dao/archive/archive.go

@@ -0,0 +1,32 @@
+package archive
+
+import (
+	"context"
+	"database/sql"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/library/log"
+)
+
+const (
+	_arcSQL = "SELECT id,mid,typeid,copyright,author,title,cover,content,tag,duration,round,attribute,access,state,reject_reason,pubtime,ctime,mtime,forward FROM archive WHERE id=?"
+)
+
+// Archive get a archive by avid.
+func (d *Dao) Archive(c context.Context, aid int64) (a *archive.Archive, err error) {
+	var reason sql.NullString
+	row := d.db.QueryRow(c, _arcSQL, aid)
+	a = &archive.Archive{}
+	if err = row.Scan(&a.ID, &a.Mid, &a.TypeID, &a.Copyright, &a.Author, &a.Title, &a.Cover, &a.Content, &a.Tag, &a.Duration,
+		&a.Round, &a.Attribute, &a.Access, &a.State, &reason, &a.PubTime, &a.CTime, &a.MTime, &a.Forward); err != nil {
+		if err == sql.ErrNoRows {
+			a = nil
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+		return
+	}
+	a.Reason = reason.String
+	return
+}

app/job/main/archive/dao/archive/archive_test.go

@@ -0,0 +1,16 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Archive(t *testing.T) {
+	Convey("Archive", t, func() {
+		archive, err := d.Archive(context.TODO(), 1)
+		So(err, ShouldBeNil)
+		Println(archive)
+	})
+}

app/job/main/archive/dao/archive/config.go

@@ -0,0 +1,41 @@
+package archive
+
+import (
+	"context"
+	"database/sql"
+
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_confSQL          = "SELECT value FROM archive_config WHERE state=0 AND name=?"
+	_confForAuditType = "wait_audit_arctype"
+)
+
+// AuditTypesConf get audit conf
+func (d *Dao) AuditTypesConf(c context.Context) (atps map[int16]struct{}, err error) {
+	row := d.db.QueryRow(c, _confSQL, _confForAuditType)
+	var (
+		value   string
+		typeIDs []int64
+	)
+	if err = row.Scan(&value); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+		return
+	}
+	typeIDs, err = xstr.SplitInts(value)
+	if err != nil {
+		log.Error("archive_config value(%s) xstr.SplitInts error(%v)", value, err)
+		return
+	}
+	atps = map[int16]struct{}{}
+	for _, typeid := range typeIDs {
+		atps[int16(typeid)] = struct{}{}
+	}
+	return
+}

app/job/main/archive/dao/archive/config_test.go

@@ -0,0 +1,16 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_AuditTypesConf(t *testing.T) {
+	Convey("AuditTypesConf", t, func() {
+		configs, err := d.AuditTypesConf(context.TODO())
+		So(err, ShouldBeNil)
+		Println(configs)
+	})
+}

app/job/main/archive/dao/archive/dao.go

@@ -0,0 +1,28 @@
+package archive
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/conf"
+	"go-common/library/database/sql"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	c  *conf.Config
+	db *sql.DB
+}
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:  c,
+		db: sql.NewMySQL(c.DB.Archive),
+	}
+	return d
+}
+
+// BeginTran begin transcation.
+func (d *Dao) BeginTran(c context.Context) (tx *sql.Tx, err error) {
+	return d.db.Begin(c)
+}

app/job/main/archive/dao/archive/dao_test.go

@@ -0,0 +1,34 @@
+package archive
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/job/main/archive/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.app-svr.archive-job")
+		flag.Set("conf_token", "MmQwIqWAyIaIu8CKb7MKcNSYlGGhoudN")
+		flag.Set("tree_id", "2301")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/job/main/archive/dao/archive/first_pass.go

@@ -0,0 +1,25 @@
+package archive
+
+import (
+	"context"
+
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_slFirstPassByAID = "SELECT `id` FROM `archive_first_pass` WHERE `aid`=? LIMIT 1"
+)
+
+// GetFirstPassByAID is
+func (d *Dao) GetFirstPassByAID(c context.Context, aid int64) (id int64, err error) {
+	row := d.db.QueryRow(c, _slFirstPassByAID, aid)
+	if err = row.Scan(&id); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("GetFirstPassByAID error(%v) aid(%d)", err, aid)
+		}
+	}
+	return
+}

app/job/main/archive/dao/archive/first_pass_test.go

@@ -0,0 +1,21 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestGetFirstPassByAID(t *testing.T) {
+	var (
+		c   = context.TODO()
+		aid = int64(4052032)
+	)
+	convey.Convey("GetFirstPassByAID", t, func(ctx convey.C) {
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			_, err := d.GetFirstPassByAID(c, aid)
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/job/main/archive/dao/archive/oper.go

@@ -0,0 +1,26 @@
+package archive
+
+import (
+	"context"
+	"database/sql"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/library/log"
+)
+
+const (
+	_arcPassedOperSQL = "SELECT id FROM archive_oper WHERE aid=? AND state>=? LIMIT 1"
+)
+
+// PassedOper check archive passed
+func (d *Dao) PassedOper(c context.Context, aid int64) (id int64, err error) {
+	row := d.db.QueryRow(c, _arcPassedOperSQL, aid, archive.StateOpen)
+	if err = row.Scan(&id); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}

app/job/main/archive/dao/archive/oper_test.go

@@ -0,0 +1,15 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_PassedOper(t *testing.T) {
+	Convey("PassedOper", t, func() {
+		_, err := d.PassedOper(context.TODO(), 1)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/archive/dao/archive/staff.go

@@ -0,0 +1,32 @@
+package archive
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/library/log"
+)
+
+const (
+	_staffSQL = "SELECT aid,staff_mid,staff_title,ctime,mtime FROM archive_staff WHERE aid=? AND state=1"
+)
+
+// Staff get Staff by aid.
+func (d *Dao) Staff(c context.Context, aid int64) (res []*archive.Staff, err error) {
+	rows, err := d.db.Query(c, _staffSQL, aid)
+	if err != nil {
+		log.Error("d.db.Query(%d) error(%v)", aid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		s := &archive.Staff{}
+		if err = rows.Scan(&s.Aid, &s.Mid, &s.Title, &s.Ctime, &s.Mtime); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		res = append(res, s)
+	}
+	err = rows.Err()
+	return
+}

app/job/main/archive/dao/archive/staff_test.go

@@ -0,0 +1,21 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestStaff(t *testing.T) {
+	var (
+		c   = context.TODO()
+		aid = int64(4052032)
+	)
+	convey.Convey("Staff", t, func(ctx convey.C) {
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			_, err := d.Staff(c, aid)
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/job/main/archive/dao/archive/type.go

@@ -0,0 +1,31 @@
+package archive
+
+import (
+	"context"
+
+	"go-common/library/log"
+)
+
+const (
+	_tpsSQL = "SELECT id,pid FROM archive_type WHERE pid !=0"
+)
+
+// TypeMapping is second types opposite first types.
+func (d *Dao) TypeMapping(c context.Context) (rmap map[int16]int16, err error) {
+	rows, err := d.db.Query(c, _tpsSQL)
+	if err != nil {
+		log.Error("d.tpsStmt.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	rmap = map[int16]int16{}
+	for rows.Next() {
+		var id, pid int16
+		if err = rows.Scan(&id, &pid); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		rmap[id] = pid
+	}
+	return
+}

app/job/main/archive/dao/archive/type_test.go

@@ -0,0 +1,15 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_TypeMapping(t *testing.T) {
+	Convey("TypeMapping", t, func() {
+		_, err := d.TypeMapping(context.TODO())
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/archive/dao/archive/video.go

@@ -0,0 +1,74 @@
+package archive
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/library/log"
+)
+
+const (
+	_videosSQL = `SELECT id,filename,cid,aid,eptitle,description,src_type,duration,filesize,resolutions,playurl,failinfo,
+	index_order,attribute,xcode_state,status,ctime,mtime FROM archive_video WHERE aid=? ORDER BY index_order`
+	_videos2SQL = `SELECT vr.id,v.filename,vr.cid,vr.aid,vr.title,vr.description,v.src_type,v.duration,v.filesize,v.resolutions,
+	v.playurl,v.failcode,vr.index_order,v.attribute,v.xcode_state,v.status,vr.state,vr.ctime,vr.mtime,v.dimensions FROM archive_video_relation AS vr JOIN video AS v ON vr.cid=v.id WHERE vr.aid=? ORDER BY vr.index_order`
+	_aidsSQL = "SELECT aid FROM archive_video_relation WHERE cid=? AND state=0"
+)
+
+// Videos get videos by aid
+func (d *Dao) Videos(c context.Context, aid int64) (vs []*archive.Video, err error) {
+	rows, err := d.db.Query(c, _videosSQL, aid)
+	if err != nil {
+		log.Error("d.db.Query(%s, %d) error(%v)", _videosSQL, aid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		v := &archive.Video{}
+		if err = rows.Scan(&v.ID, &v.Filename, &v.Cid, &v.Aid, &v.Title, &v.Desc, &v.SrcType, &v.Duration, &v.Filesize, &v.Resolutions,
+			&v.Playurl, &v.FailCode, &v.Index, &v.Attribute, &v.XcodeState, &v.Status, &v.CTime, &v.MTime); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		vs = append(vs, v)
+	}
+	return
+}
+
+// Videos2 get videos by 2 table em.......
+func (d *Dao) Videos2(c context.Context, aid int64) (vs []*archive.Video, err error) {
+	rows, err := d.db.Query(c, _videos2SQL, aid)
+	if err != nil {
+		log.Error("d.db.Query(%s, %d) error(%v)", _videos2SQL, aid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		v := &archive.Video{}
+		if err = rows.Scan(&v.ID, &v.Filename, &v.Cid, &v.Aid, &v.Title, &v.Desc, &v.SrcType, &v.Duration, &v.Filesize, &v.Resolutions,
+			&v.Playurl, &v.FailCode, &v.Index, &v.Attribute, &v.XcodeState, &v.Status, &v.State, &v.CTime, &v.MTime, &v.Dimensions); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		vs = append(vs, v)
+	}
+	return
+}
+
+// Aids get aids by cid
+func (d *Dao) Aids(c context.Context, cid int64) (aids []int64, err error) {
+	rows, err := d.db.Query(c, _aidsSQL, cid)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var aid int64
+		if err = rows.Scan(&aid); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		aids = append(aids, aid)
+	}
+	return
+}

app/job/main/archive/dao/archive/video_test.go

@@ -0,0 +1,25 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Videos(t *testing.T) {
+	Convey("Videos", t, func() {
+		_, err := d.Videos(context.TODO(), 1)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_Videos2(t *testing.T) {
+	Convey("Videos2", t, func() {
+		vs, err := d.Videos2(context.TODO(), 10098500)
+		So(err, ShouldBeNil)
+		for _, v := range vs {
+			Printf("%+v", v)
+		}
+	})
+}

app/job/main/archive/dao/archive/video_track.go

@@ -0,0 +1,22 @@
+package archive
+
+import (
+	"context"
+
+	"go-common/library/log"
+)
+
+const (
+	_inVideoHis = "INSERT INTO archive_video_track(aid,filename,status,xcode_state,remark,ctime,mtime) VALUES(?,?,?,?,?,?,?)"
+)
+
+// InVideoHis insert video track history
+func (d *Dao) InVideoHis(c context.Context, aid int64, filename string, status int16, xcodeState int8, remark string, ctime, mtime string) (rows int64, err error) {
+	rs, err := d.db.Exec(c, _inVideoHis, aid, filename, status, xcodeState, remark, ctime, mtime)
+	if err != nil {
+		log.Error("d.inVideoHisStmt.Exec(%d, %s, %d, %d, %s, %s, %s) error(%v)", aid, filename, status, xcodeState, remark, ctime, mtime, err)
+		return
+	}
+	rows, err = rs.RowsAffected()
+	return
+}

app/job/main/archive/dao/archive/video_track_test.go

@@ -0,0 +1,15 @@
+package archive
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_InVideoHis(t *testing.T) {
+	Convey("InVideoHis", t, func() {
+		_, err := d.InVideoHis(context.TODO(), 1, "Test_InVideoHis", 0, 0, "", "", "")
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/archive/dao/email/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "email_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/model/result:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "email.go",
+    ],
+    importpath = "go-common/app/job/main/archive/dao/email",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/model/result:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/dao/email/dao.go

@@ -0,0 +1,22 @@
+package email
+
+import (
+	gomail "gopkg.in/gomail.v2"
+
+	"go-common/app/job/main/archive/conf"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	c     *conf.Config
+	email *gomail.Dialer
+}
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:     c,
+		email: gomail.NewDialer(c.Mail.Host, c.Mail.Port, c.Mail.Username, c.Mail.Password),
+	}
+	return d
+}

app/job/main/archive/dao/email/dao_test.go

@@ -0,0 +1,19 @@
+package email
+
+import (
+	"flag"
+	"path/filepath"
+
+	"go-common/app/job/main/archive/conf"
+)
+
+var (
+	d *Dao
+)
+
+func init() {
+	dir, _ := filepath.Abs("../../cmd/archive-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	d = New(conf.Conf)
+}

app/job/main/archive/dao/email/email.go

@@ -0,0 +1,87 @@
+package email
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/main/archive/model/result"
+	"go-common/app/service/main/archive/api"
+	"go-common/app/service/main/archive/model/archive"
+	"go-common/library/log"
+
+	gomail "gopkg.in/gomail.v2"
+)
+
+var (
+	_states = map[int]string{
+		0:  "开放浏览",
+		-1: "待审",
+		-2: "打回稿件回收站",
+		-3: "网警锁定删除",
+		-4: "锁定稿件",
+		// -5:   "锁定稿件开放浏览",
+		-6: "修复待审",
+		-7: "暂缓审核",
+		// -8:   "补档待审",
+		-9:  "等待转码",
+		-10: "延迟发布",
+		-11: "视频源待修",
+		// -12:  "上传失败",
+		-13: "允许评论待审",
+		// -14:  "临时回收站",
+		-15:  "分发中",
+		-16:  "转码失败",
+		-30:  "创建已提交",
+		-40:  "UP主定时发布",
+		-100: "UP主删除",
+	}
+)
+
+func stateDescribe(state int) string {
+	des, ok := _states[state]
+	if ok {
+		return des
+	}
+	return strconv.Itoa(state)
+}
+
+const (
+	_bangumiMailSub = "番剧稿件《%s》状态变更"
+	_movieMailSub   = "电影稿件《%s》状态变更"
+	_mailBody       = `
+稿件标题：%s
+稿件状态：%s => %s
+其他变更：%s => %s
+稿件地址：http://www.bilibili.com/video/av%d
+审核后台：http://manager.bilibili.co/#!/archive/modify/%d`
+)
+
+// PGCNotifyMail notify pgc mail
+func (d *Dao) PGCNotifyMail(a *api.Arc, nw *result.Archive, old *result.Archive) {
+	msg := gomail.NewMessage()
+	msg.SetHeader("From", d.c.Mail.Username)
+	switch {
+	case a.AttrVal(archive.AttrBitIsBangumi) == archive.AttrYes:
+		msg.SetHeader("To", d.c.Mail.Bangumi...)
+		msg.SetHeader("Subject", fmt.Sprintf(_bangumiMailSub, a.Title))
+	case a.AttrVal(archive.AttrBitIsMovie) == archive.AttrYes:
+		msg.SetHeader("To", d.c.Mail.Movie...)
+		msg.SetHeader("Subject", fmt.Sprintf(_movieMailSub, a.Title))
+	default:
+		return
+	}
+	obs, _ := json.Marshal(old)
+	nbs, _ := json.Marshal(nw)
+
+	if old.State != nw.State {
+		oldState, newState := stateDescribe(old.State), stateDescribe(nw.State)
+		msg.SetBody("text/plain", fmt.Sprintf(_mailBody, a.Title, oldState, newState, obs, nbs, a.Aid, a.Aid))
+	} else {
+		state := stateDescribe(int(a.State))
+		msg.SetBody("text/plain", fmt.Sprintf(_mailBody, a.Title, state, state, obs, nbs, a.Aid, a.Aid))
+	}
+	if err := d.email.DialAndSend(msg); err != nil {
+		log.Error("s.email.DialAndSend(aid: %d) error(%v)", a.Aid, err)
+	}
+}

app/job/main/archive/dao/email/email_test.go

@@ -0,0 +1,15 @@
+package email
+
+import (
+	"go-common/app/job/main/archive/model/result"
+	"go-common/app/service/main/archive/api"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_PGCNotifyMail(t *testing.T) {
+	Convey("PGCNotifyMail", t, func() {
+		d.PGCNotifyMail(&api.Arc{}, &result.Archive{}, &result.Archive{})
+	})
+}

app/job/main/archive/dao/monitor/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "monitor_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "monitor.go",
+    ],
+    importpath = "go-common/app/job/main/archive/dao/monitor",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/dao/monitor/dao.go

@@ -0,0 +1,21 @@
+package monitor
+
+import (
+	"go-common/app/job/main/archive/conf"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	c      *conf.Config
+	client *bm.Client
+}
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		client: bm.NewClient(c.HTTPClient),
+	}
+	return d
+}

app/job/main/archive/dao/monitor/dao_test.go

@@ -0,0 +1,19 @@
+package monitor
+
+import (
+	"flag"
+	"path/filepath"
+
+	"go-common/app/job/main/archive/conf"
+)
+
+var (
+	d *Dao
+)
+
+func init() {
+	dir, _ := filepath.Abs("../../cmd/archive-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	d = New(conf.Conf)
+}

app/job/main/archive/dao/monitor/monitor.go

@@ -0,0 +1,55 @@
+package monitor
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/library/log"
+)
+
+type wxParams struct {
+	Username  string `json:"username"`
+	Content   string `json:"content"`
+	Token     string `json:"token"`
+	Timestamp int64  `json:"timestamp"`
+	Sign      string `json:"signature"`
+}
+type resp struct {
+	Status int64  `json:"status"`
+	Msg    string `json:"msg"`
+}
+
+// Send send message to phone
+func (d *Dao) Send(c context.Context, users, msg, token, secret string) (err error) {
+	params := url.Values{}
+	params.Set("username", users)
+	params.Set("content", msg)
+	params.Set("token", token)
+	params.Set("timestamp", strconv.FormatInt(time.Now().Unix(), 10))
+	mh := md5.Sum([]byte(params.Encode() + secret))
+	params.Set("signature", hex.EncodeToString(mh[:]))
+	p := &wxParams{
+		Username: params.Get("username"),
+		Content:  params.Get("content"),
+		Token:    params.Get("token"),
+		Sign:     params.Get("signature"),
+	}
+	p.Timestamp, _ = strconv.ParseInt(params.Get("timestamp"), 10, 64)
+	bs, _ := json.Marshal(p)
+	payload := strings.NewReader(string(bs))
+	req, _ := http.NewRequest("POST", "http://bap.bilibili.co/api/v1/message/add", payload)
+	req.Header.Add("content-type", "application/json; charset=utf-8")
+	v := &resp{}
+	if err = d.client.Do(context.TODO(), req, v); err != nil {
+		log.Error("s.client.Do error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/archive/dao/monitor/monitor_test.go

@@ -0,0 +1,17 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/archive/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Send(t *testing.T) {
+	Convey("Send", t, func() {
+		err := d.Send(context.TODO(), conf.Conf.WeChantUsers, "报警短信test", conf.Conf.WeChatToken, conf.Conf.WeChatSecret)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/archive/dao/reply/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "reply_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "reply.go",
+    ],
+    importpath = "go-common/app/job/main/archive/dao/reply",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/dao/reply/dao.go

@@ -0,0 +1,22 @@
+package reply
+
+import (
+	"go-common/app/job/main/archive/conf"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	client       *bm.Client
+	changeSubMid string
+}
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		client: bm.NewClient(c.HTTPClient),
+		// path
+		changeSubMid: c.Host.APICo + _changeSubjectMid,
+	}
+	return d
+}

app/job/main/archive/dao/reply/dao_test.go

@@ -0,0 +1,19 @@
+package reply
+
+import (
+	"flag"
+	"path/filepath"
+
+	"go-common/app/job/main/archive/conf"
+)
+
+var (
+	d *Dao
+)
+
+func init() {
+	dir, _ := filepath.Abs("../../cmd/archive-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	d = New(conf.Conf)
+}

app/job/main/archive/dao/reply/reply.go

@@ -0,0 +1,36 @@
+package reply
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"strconv"
+
+	"go-common/library/log"
+)
+
+const (
+	_changeSubjectMid = "/x/internal/v2/reply/subject/mid"
+)
+
+// ChangeSubjectMid change av's owner
+func (d *Dao) ChangeSubjectMid(oid, mid int64) (err error) {
+	params := url.Values{}
+	params.Set("adid", "0")
+	params.Set("oid", strconv.FormatInt(oid, 10))
+	params.Set("type", "1")
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	var res struct {
+		Code int64 `json:"code"`
+	}
+	if err = d.client.Post(context.TODO(), d.changeSubMid, "", params, &res); err != nil {
+		log.Error("d.client.Post(%s) error(%v)", d.changeSubMid+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 0 {
+		err = errors.New(strconv.FormatInt(res.Code, 10))
+		log.Error("d.client.Post(%s) code(%v)", d.changeSubMid+"?"+params.Encode(), res.Code)
+		return
+	}
+	return
+}

app/job/main/archive/dao/reply/reply_test.go

@@ -0,0 +1,13 @@
+package reply
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_ChangeSubjectMid(t *testing.T) {
+	Convey("ChangeSubjectMid", t, func() {
+		d.ChangeSubjectMid(0, 1684013)
+	})
+}

app/job/main/archive/dao/result/BUILD

@@ -0,0 +1,58 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "archive_test.go",
+        "dao_test.go",
+        "staff_test.go",
+        "video_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/model/archive:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "archive.go",
+        "dao.go",
+        "staff.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/main/archive/dao/result",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/model/archive:go_default_library",
+        "//app/job/main/archive/model/result:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/dao/result/archive.go

@@ -0,0 +1,86 @@
+package result
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/app/job/main/archive/model/result"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_arcSQL       = "SELECT aid,mid,typeid,videos,copyright,title,cover,content,duration,attribute,state,access,pubtime,ctime,mission_id,order_id,redirect_url,forward,dynamic FROM archive WHERE aid=?"
+	_inArchiveSQL = `INSERT IGNORE INTO archive (aid,mid,typeid,videos,title,cover,content,duration,attribute,copyright,access,pubtime,state,mission_id,order_id,redirect_url,forward,dynamic,cid,dimensions)
+			VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`
+	_upArchiveSQL  = "UPDATE archive SET mid=?,typeid=?,videos=?,title=?,cover=?,content=?,duration=?,attribute=?,copyright=?,access=?,pubtime=?,state=?,mission_id=?,order_id=?,redirect_url=?,mtime=?,forward=?,dynamic=?,cid=?,dimensions=? WHERE aid=?"
+	_delArchiveSQL = "UPDATE archive SET state=? WHERE aid=?"
+	_upPassedSQL   = "SELECT aid FROM archive WHERE mid=? AND state>=0 ORDER BY pubtime DESC"
+)
+
+// UpPassed is
+func (d *Dao) UpPassed(c context.Context, mid int64) (aids []int64, err error) {
+	rows, err := d.db.Query(c, _upPassedSQL, mid)
+	if err != nil {
+		log.Error("d.db.Query(%s, %d) error(%v)", _upPassedSQL, mid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var aid int64
+		if err = rows.Scan(&aid); err != nil {
+			log.Error("rows.Scan(%d) error(%v)", aid, err)
+			return
+		}
+		aids = append(aids, aid)
+	}
+	err = rows.Err()
+	return
+}
+
+// Archive get a archive by aid.
+func (d *Dao) Archive(c context.Context, aid int64) (a *result.Archive, err error) {
+	row := d.db.QueryRow(c, _arcSQL, aid)
+	a = &result.Archive{}
+	if err = row.Scan(&a.AID, &a.Mid, &a.TypeID, &a.Videos, &a.Copyright, &a.Title, &a.Cover, &a.Content, &a.Duration,
+		&a.Attribute, &a.State, &a.Access, &a.PubTime, &a.CTime, &a.MissionID, &a.OrderID, &a.RedirectURL, &a.Forward, &a.Dynamic); err != nil {
+		if err == sql.ErrNoRows {
+			a = nil
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// TxAddArchive add archive result
+func (d *Dao) TxAddArchive(c context.Context, tx *sql.Tx, a *archive.Archive, ad *archive.Addit, videoCnt int, firstCid int64, dimensions string) (rows int64, err error) {
+	res, err := tx.Exec(_inArchiveSQL, a.ID, a.Mid, a.TypeID, videoCnt, a.Title, a.Cover, a.Content, a.Duration, a.Attribute, a.Copyright, a.Access, a.PubTime, a.State, ad.MissionID, ad.OrderID, ad.RedirectURL, a.Forward, ad.Dynamic, firstCid, dimensions)
+	if err != nil {
+		log.Error("tx.Exec(%s) error(%v)", _inArchiveSQL, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// TxUpArchive update archive result
+func (d *Dao) TxUpArchive(c context.Context, tx *sql.Tx, a *archive.Archive, ad *archive.Addit, videoCnt int, firstCid int64, dimensions string) (rows int64, err error) {
+	res, err := tx.Exec(_upArchiveSQL, a.Mid, a.TypeID, videoCnt, a.Title, a.Cover, a.Content, a.Duration, a.Attribute, a.Copyright, a.Access, a.PubTime, a.State, ad.MissionID, ad.OrderID, ad.RedirectURL, time.Now(), a.Forward, ad.Dynamic, firstCid, dimensions, a.ID)
+	if err != nil {
+		log.Error("tx.Exec(%s) error(%v)", _upArchiveSQL, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// TxDelArchive delete archive
+func (d *Dao) TxDelArchive(c context.Context, tx *sql.Tx, aid int64) (rows int64, err error) {
+	res, err := tx.Exec(_delArchiveSQL, archive.StateForbidUpDelete, aid)
+	if err != nil {
+		log.Error("tx.Execerror(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/archive/dao/result/archive_test.go

@@ -0,0 +1,59 @@
+package result
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/archive/model/archive"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_UpPassed(t *testing.T) {
+	Convey("UpPassed", t, func() {
+		a, err := d.UpPassed(context.TODO(), 1684013)
+		So(err, ShouldBeNil)
+		Println(a)
+	})
+}
+
+func Test_Archive(t *testing.T) {
+	Convey("Archive", t, func() {
+		a, err := d.Archive(context.TODO(), 1684013)
+		So(err, ShouldBeNil)
+		Println(a)
+	})
+}
+
+func Test_TxAddArchive(t *testing.T) {
+	Convey("TxAddArchive", t, func() {
+		tx, err := d.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		_, err = d.TxAddArchive(context.TODO(), tx, &archive.Archive{}, &archive.Addit{}, 0, 0, "")
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxUpArchive(t *testing.T) {
+	Convey("TxUpArchive", t, func() {
+		tx, err := d.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		_, err = d.TxUpArchive(context.TODO(), tx, &archive.Archive{ID: 0}, &archive.Addit{}, 0, 0, "")
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxDelArchive(t *testing.T) {
+	Convey("TxDelArchive", t, func() {
+		tx, err := d.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		_, err = d.TxDelArchive(context.TODO(), tx, 0)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/archive/dao/result/dao.go

@@ -0,0 +1,28 @@
+package result
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/conf"
+	"go-common/library/database/sql"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	c  *conf.Config
+	db *sql.DB
+}
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:  c,
+		db: sql.NewMySQL(c.DB.Result),
+	}
+	return d
+}
+
+// BeginTran begin transcation.
+func (d *Dao) BeginTran(c context.Context) (tx *sql.Tx, err error) {
+	return d.db.Begin(c)
+}

app/job/main/archive/dao/result/dao_test.go

@@ -0,0 +1,34 @@
+package result
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/job/main/archive/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.app-svr.archive-job")
+		flag.Set("conf_token", "MmQwIqWAyIaIu8CKb7MKcNSYlGGhoudN")
+		flag.Set("tree_id", "2301")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/job/main/archive/dao/result/staff.go

@@ -0,0 +1,41 @@
+package result
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_addStaffSQL = "INSERT INTO archive_staff (aid,mid,title,ctime,mtime) VALUES "
+	_delStaffSQL = "DELETE FROM archive_staff WHERE aid=?"
+)
+
+// TxDelStaff del archive staff
+func (d *Dao) TxDelStaff(c context.Context, tx *sql.Tx, aid int64) (err error) {
+	_, err = tx.Exec(_delStaffSQL, aid)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return
+}
+
+// TxAddStaff add archive staff
+func (d *Dao) TxAddStaff(c context.Context, tx *sql.Tx, aid int64, staff []*archive.Staff) (err error) {
+	var valSQL []string
+	for _, s := range staff {
+		valSQL = append(valSQL, fmt.Sprintf("(%d,%d,'%s','%s','%s')", s.Aid, s.Mid, s.Title, s.Ctime, s.Mtime))
+	}
+	valSQLStr := strings.Join(valSQL, ",")
+	_, err = tx.Exec(_addStaffSQL + valSQLStr)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/archive/dao/result/staff_test.go

@@ -0,0 +1,47 @@
+package result
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/archive/model/archive"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestTxDelStaff(t *testing.T) {
+	var (
+		c   = context.TODO()
+		aid = int64(4052032)
+	)
+	convey.Convey("TxDelStaff", t, func(ctx convey.C) {
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			tx, err := d.BeginTran(c)
+			ctx.So(err, convey.ShouldBeNil)
+			err = d.TxDelStaff(c, tx, aid)
+			ctx.So(err, convey.ShouldBeNil)
+			err = tx.Commit()
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestTxAddStaff(t *testing.T) {
+	var (
+		c     = context.TODO()
+		aid   = int64(4052032)
+		staff []*archive.Staff
+	)
+	convey.Convey("TxAddStaff", t, func(ctx convey.C) {
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			staff = append(staff, &archive.Staff{Aid: aid, Mid: 3333, Title: "哈哈", Ctime: "2018-11-28T16:50:14+08:00", Mtime: "2018-12-21T11:41:37+08:00"})
+			staff = append(staff, &archive.Staff{Aid: aid, Mid: 4444, Title: "2223", Ctime: "2018-11-28T16:50:14+08:00", Mtime: "2018-12-21T11:41:38+08:00"})
+			tx, err := d.BeginTran(c)
+			ctx.So(err, convey.ShouldBeNil)
+			err = d.TxAddStaff(c, tx, aid, staff)
+			ctx.So(err, convey.ShouldBeNil)
+			err = tx.Commit()
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/job/main/archive/dao/result/video.go

@@ -0,0 +1,47 @@
+package result
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_inVideoSQL = `INSERT INTO archive_video (aid,eptitle,description,filename,src_type,cid,duration,index_order,attribute,weblink,dimensions) VALUES(?,?,?,?,?,?,?,?,?,?,?)
+				ON DUPLICATE KEY UPDATE eptitle=?,description=?,filename=?,src_type=?,duration=?,index_order=?,attribute=?,weblink=?,dimensions=?`
+	_delVideoByCidSQL = "DELETE FROM archive_video where aid=? and cid=?"
+	_delVideosSQL     = "DELETE FROM archive_video WHERE aid=?"
+)
+
+// TxAddVideo add videos result
+func (d *Dao) TxAddVideo(c context.Context, tx *sql.Tx, v *archive.Video) (rows int64, err error) {
+	res, err := tx.Exec(_inVideoSQL, v.Aid, v.Title, v.Desc, v.Filename, v.SrcType, v.Cid, v.Duration, v.Index, v.Attribute, v.WebLink, v.Dimensions,
+		v.Title, v.Desc, v.Filename, v.SrcType, v.Duration, v.Index, v.Attribute, v.WebLink, v.Dimensions)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// TxDelVideoByCid del videos by aid and cid
+func (d *Dao) TxDelVideoByCid(c context.Context, tx *sql.Tx, aid, cid int64) (rows int64, err error) {
+	res, err := tx.Exec(_delVideoByCidSQL, aid, cid)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// TxDelVideos del videos
+func (d *Dao) TxDelVideos(c context.Context, tx *sql.Tx, aid int64) (rows int64, err error) {
+	res, err := tx.Exec(_delVideosSQL, aid)
+	if err != nil {
+		log.Error("tx.Exec(%s, %d) error(%v)", _delVideosSQL, aid, err)
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/archive/dao/result/video_test.go

@@ -0,0 +1,43 @@
+package result
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/archive/model/archive"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_TxAddVideo(t *testing.T) {
+	Convey("Archive", t, func() {
+		tx, err := d.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		_, err = d.TxAddVideo(context.TODO(), tx, &archive.Video{Aid: 1, Cid: 1})
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxDelVideoByCid(t *testing.T) {
+	Convey("TxDelVideoByCid", t, func() {
+		tx, err := d.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		_, err = d.TxDelVideoByCid(context.TODO(), tx, 1, 1)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_TxDelVideos(t *testing.T) {
+	Convey("TxDelVideos", t, func() {
+		tx, err := d.BeginTran(context.TODO())
+		So(err, ShouldBeNil)
+		_, err = d.TxDelVideos(context.TODO(), tx, 0)
+		So(err, ShouldBeNil)
+		err = tx.Commit()
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/archive/http/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/archive/http",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/http/http.go

@@ -0,0 +1,27 @@
+package http
+
+import (
+	"go-common/app/job/main/archive/conf"
+	"go-common/app/job/main/archive/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Init init http router.
+func Init(c *conf.Config, s *service.Service) {
+	e := bm.DefaultServer(c.BM)
+	innerRouter(e)
+	// init internal server
+	if err := e.Start(); err != nil {
+		log.Error("xhttp.Serve error(%v)", err)
+		panic(err)
+	}
+}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {}

app/job/main/archive/model/archive/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "addit.go",
+        "archive.go",
+        "track.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/main/archive/model/archive",
+    tags = ["automanaged"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/model/archive/addit.go

@@ -0,0 +1,18 @@
+package archive
+
+const (
+	UpFromPGC       = 1
+	UpFromPGCSecret = 5
+)
+
+// Addit addit struct
+type Addit struct {
+	ID          int64
+	Aid         int64
+	Source      string
+	RedirectURL string
+	MissionID   int64
+	UpFrom      int8
+	OrderID     int
+	Dynamic     string
+}

app/job/main/archive/model/archive/archive.go

@@ -0,0 +1,117 @@
+package archive
+
+const (
+	StateOpen   = 0
+	StateOrange = 1
+
+	StateForbidWait      = -1
+	StateForbidRecicle   = -2
+	StateForbidPolice    = -3
+	StateForbidLock      = -4
+	StateForbidFixed     = -6
+	StateForbidLater     = -7
+	StateForbidSubmit    = -30
+	StateForbidUserDelay = -40
+	StateForbidUpDelete  = -100
+	// round
+	RoundBegin                  = 0
+	RoundAuditSecond            = 10 // 二审：选定分区的多P稿件 及 PGC/活动的单P多P稿件
+	RoundAuditThird             = 20 // 三审：选定分区/PGC/活动 的单P多P稿件
+	RoundReviewFirst            = 30 // 分区回查：粉丝小于配置阈值 如 5000 且 指定分区
+	RoundReviewFirstWaitTrigger = 31 // 点击/粉丝 等待触发中间状态，7天内达到阈值进列表，未达到自动变99
+	RoundReviewSecond           = 40 // 社区回查：粉丝大于配置阈值 如 5000 或 优质高危up
+	RoundTriggerFans            = 80 // 粉丝回查：粉丝量达到配置阈值
+	RoundTriggerClick           = 90 // 点击回查：点击量达到配置阈值
+	RoundEnd                    = 99
+	// access
+	AccessDefault = int16(0)
+	AccessMember  = int16(10000)
+	// copyright
+	CopyrightUnknow   = 0
+	CopyrightOriginal = 1
+	CopyrightCopy     = 2
+
+	// attribute yes and no
+	AttrYes = int32(1)
+	AttrNo  = int32(0)
+	// attribute bit
+	AttrBitNoRank      = uint(0)
+	AttrBitNoIndex     = uint(1)
+	AttrBitNoWeb       = uint(2)
+	AttrBitNoMobile    = uint(3)
+	AttrBitNoSearch    = uint(4)
+	AttrBitOverseaLock = uint(5)
+	AttrBitNoRecommend = uint(6)
+	// AttrBitHideCoins     = uint(7)
+	AttrBitHasHD5 = uint(8)
+	// AttrBitVisitorDm     = uint(9)
+	AttrBitAllowBp   = uint(10)
+	AttrBitIsBangumi = uint(11)
+	// AttrBitAllowDownload = uint(12)
+	AttrBitLimitArea = uint(13)
+	AttrBitAllowTag  = uint(14)
+	// AttrBitIsFromArcApi = uint(15)
+	AttrBitJumpUrl       = uint(16)
+	AttrBitIsMovie       = uint(17)
+	AttrBitBadgepay      = uint(18)
+	AttrBitIsCooperation = uint(24)
+)
+
+type UpInfo struct {
+	Nw  *Archive
+	Old *Archive
+}
+
+// archive
+type Archive struct {
+	ID        int64  `json:"id"`
+	Mid       int64  `json:"mid"`
+	TypeID    int16  `json:"typeid"`
+	HumanRank int    `json:"humanrank"`
+	Duration  int    `json:"duration"`
+	Title     string `json:"title"`
+	Cover     string `json:"cover"`
+	Content   string `json:"content"`
+	Tag       string `json:"tag"`
+	Attribute int32  `json:"attribute"`
+	Copyright int8   `json:"copyright"`
+	AreaLimit int8   `json:"arealimit"`
+	State     int    `json:"state"`
+	Author    string `json:"author"`
+	Access    int    `json:"access"`
+	Forward   int    `json:"forward"`
+	PubTime   string `json:"pubtime"`
+	Reason    string `json:"reject_reason"`
+	Round     int8   `json:"round"`
+	CTime     string `json:"ctime"`
+	MTime     string `json:"mtime"`
+}
+
+func (a *Archive) IsSyncState() bool {
+	if a.State >= 0 || a.State == StateForbidUserDelay || a.State == StateForbidUpDelete || a.State == StateForbidRecicle || a.State == StateForbidPolice ||
+		a.State == StateForbidLock {
+		return true
+	}
+	return false
+}
+
+type ArgStat struct {
+	Aid    int64
+	Field  int
+	Value  int
+	RealIP string
+}
+
+// AttrVal get attribute value.
+func (a *Archive) AttrVal(bit uint) int32 {
+	return (a.Attribute >> bit) & int32(1)
+}
+
+// Staff is
+type Staff struct {
+	Aid   int64  `json:"aid"`
+	Mid   int64  `json:"mid"`
+	Title string `json:"title"`
+	Ctime string `json:"ctime"`
+	Mtime string `json:"mtime"`
+}

app/job/main/archive/model/archive/track.go

@@ -0,0 +1,26 @@
+package archive
+
+import (
+	"go-common/library/time"
+)
+
+type Track struct {
+	// common values
+	Timestamp time.Time `json:"timestamp"`
+	// archive stat
+	State int `json:"state"`
+	Round int `json:"round"`
+	// AID    int64  `json:"aid,omitempty"`
+	Remark    string `json:"remark,omitempty"`
+	Attribute int32  `json:"attribute"`
+}
+
+type VideoTrack struct {
+	// common values
+	Timestamp  time.Time `json:"timestamp"`
+	XCodeState int8      `json:"xcode_state"`
+	// video status
+	Status int16  `json:"status"`
+	AID    int64  `json:"aid,omitempty"`
+	Remark string `json:"remark,omitempty"`
+}

app/job/main/archive/model/archive/video.go

@@ -0,0 +1,56 @@
+package archive
+
+const (
+	// video xcode and dispatch state.
+	VideoUploadInfo      = 0
+	VideoXcodeSDFail     = 1
+	VideoXcodeSDFinish   = 2
+	VideoXcodeHDFail     = 3
+	VideoXcodeHDFinish   = 4
+	VideoDispatchRunning = 5
+	VideoDispatchFinish  = 6
+
+	XcodeFailZero = 0
+
+	// video status.
+	VideoStatusOpen      = int16(0)
+	VideoStatusAccess    = int16(10000)
+	VideoStatusWait      = int16(-1)
+	VideoStatusRecicle   = int16(-2)
+	VideoStatusLock      = int16(-4)
+	VideoStatusXcodeFail = int16(-16)
+	VideoStatusSubmit    = int16(-30)
+	VideoStatusDelete    = int16(-100)
+
+	// video relation state
+	VideoRelationBind = int16(0)
+)
+
+type VideoUpInfo struct {
+	Nw  *Video
+	Old *Video
+}
+
+type Video struct {
+	ID          int64  `json:"id"`
+	Aid         int64  `json:"aid"`
+	Title       string `json:"eptitle"`
+	Desc        string `json:"description"`
+	Filename    string `json:"filename"`
+	SrcType     string `json:"src_type"`
+	Cid         int64  `json:"cid"`
+	Duration    int64  `json:"duration"`
+	Filesize    int64  `json:"filesize"`
+	Resolutions string `json:"resolutions"`
+	Index       int    `json:"index_order"`
+	CTime       string `json:"ctime"`
+	MTime       string `json:"mtime"`
+	Status      int16  `json:"status"`
+	State       int16  `json:"state"`
+	Playurl     string `json:"playurl"`
+	Attribute   int32  `json:"attribute"`
+	FailCode    int8   `json:"failinfo"`
+	XcodeState  int8   `json:"xcode_state"`
+	WebLink     string `json:"weblink"`
+	Dimensions  string `json:"dimensions"`
+}

app/job/main/archive/model/databus/BUILD

@@ -0,0 +1,27 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["message.go"],
+    importpath = "go-common/app/job/main/archive/model/databus",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/model/databus/message.go

@@ -0,0 +1,39 @@
+package databus
+
+import "encoding/json"
+
+// Message databus
+type Message struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+// const is
+const (
+	RouteFirstRoundForbid = "first_round_forbid"
+	RouteSecondRound      = "second_round"
+	RouteAutoOpen         = "auto_open"
+	RouteDelayOpen        = "delay_open"
+	RouteDeleteArchive    = "delete_archive"
+	RouteForceSync        = "force_sync"
+)
+
+// Videoup message for videoup2BVC
+type Videoup struct {
+	Route     string `json:"route"`
+	Timestamp int64  `json:"timestamp"`
+	Aid       int64  `json:"aid"`
+}
+
+// Rebuild is
+type Rebuild struct {
+	Aid int64 `json:"aid"`
+}
+
+// AccountNotify is
+type AccountNotify struct {
+	Mid    int64  `json:"mid"`
+	Action string `json:"action"`
+}

app/job/main/archive/model/dm/BUILD

@@ -0,0 +1,27 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dm.go"],
+    importpath = "go-common/app/job/main/archive/model/dm",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/model/dm/dm.go

@@ -0,0 +1,28 @@
+package dm
+
+import "encoding/json"
+
+// Canal canal message struct
+type Canal struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+// Subject for table dm_subject_[0-9]+
+type Subject struct {
+	ID    int64 `json:"id"`
+	Type  int64 `json:"type"`
+	AID   int64 `json:"pid"`
+	CID   int64 `json:"oid"`
+	Count int64 `json:"count"`
+}
+
+// Count dm count
+type Count struct {
+	Type      string `json:"type"`
+	ID        int64  `json:"id"`
+	Count     int64  `json:"count"`
+	Timestamp int64  `json:"timestamp"`
+}

app/job/main/archive/model/pgc/BUILD

@@ -0,0 +1,27 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/main/archive/model/pgc",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/model/pgc/model.go

@@ -0,0 +1,40 @@
+package pgc
+
+const (
+	TypeForBangumi     = 1
+	TypeForMovie       = 2
+	TypeForDocumentary = 3
+)
+
+type MovieEpisode struct {
+	EpID     int64 `json:"id"`
+	SeasonID int64 `json:"movieSeasonId"`
+	AID      int64 `json:"aid"`
+	CID      int64 `json:"cid"`
+	Status   int8  `json:"status"`
+}
+
+type BangumiEpisode struct {
+	EpID     int64 `json:"episodeId"`
+	AID      int64 `json:"avId"`
+	SeasonID int64 `json:"seasonId"`
+	CID      int64 `json:"danmaku"`
+	Status   int8  `json:"isDelete"`
+}
+
+type Documentary struct {
+	EpID     int64 `json:"id"`
+	SeasonID int64
+	AID      int64 `json:"aid"`
+	CID      int64 `json:"cid"`
+	Status   int8  `json:"is_delete"`
+}
+
+type Archive struct {
+	AID      int64
+	SeasonID int64
+	Tp       int8
+	Status   int8
+	EpID     int64
+	CID      int64
+}

app/job/main/archive/model/result/BUILD

@@ -0,0 +1,30 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "archive.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/main/archive/model/result",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/model/result/archive.go

@@ -0,0 +1,79 @@
+package result
+
+import (
+	"database/sql/driver"
+	"sync"
+	"time"
+)
+
+const (
+	AttrYes          = int32(1)
+	AttrNo           = int32(0)
+	AttrBitIsPGC     = uint(9)
+	AttrBitIsBangumi = uint(11)
+)
+
+type ArchiveUpInfo struct {
+	Table  string
+	Action string
+	Nw     *Archive
+	Old    *Archive
+}
+
+type ResultDelay struct {
+	Lock sync.RWMutex
+	AIDs map[int64]struct{}
+}
+
+// Result archive result
+type Archive struct {
+	ID          int64     `json:"id"`
+	AID         int64     `json:"aid"`
+	Mid         int64     `json:"mid"`
+	TypeID      int16     `json:"typeid"`
+	Videos      int       `json:"videos"`
+	Title       string    `json:"title"`
+	Cover       string    `json:"cover"`
+	Content     string    `json:"content"`
+	Duration    int       `json:"duration"`
+	Attribute   int32     `json:"attribute"`
+	Copyright   int8      `json:"copyright"`
+	Access      int       `json:"access"`
+	PubTime     wocaoTime `json:"pubtime"`
+	CTime       wocaoTime `json:"ctime"`
+	MTime       wocaoTime `json:"mtime"`
+	State       int       `json:"state"`
+	MissionID   int64     `json:"mission_id"`
+	OrderID     int64     `json:"order_id"`
+	RedirectURL string    `json:"redirect_url"`
+	Forward     int64     `json:"forward"`
+	Dynamic     string    `json:"dynamic"`
+}
+
+type wocaoTime string
+
+// Scan scan time.
+func (jt *wocaoTime) Scan(src interface{}) (err error) {
+	switch sc := src.(type) {
+	case time.Time:
+		*jt = wocaoTime(sc.Format("2006-01-02 15:04:05"))
+	case string:
+		*jt = wocaoTime(sc)
+	}
+	return
+}
+
+// Value get time value.
+func (jt wocaoTime) Value() (driver.Value, error) {
+	return time.Parse("2006-01-02 15:04:05", string(jt))
+}
+
+// AttrSet set attribute.
+func (a *Archive) AttrSet(v int32, bit uint) {
+	a.Attribute = a.Attribute&(^(1 << bit)) | (v << bit)
+}
+
+// AttrVal get attribute.
+func (a *Archive) AttrVal(bit uint) int32 {
+	return (a.Attribute >> bit) & int32(1)
+}

app/job/main/archive/model/result/video.go

@@ -0,0 +1,52 @@
+package result
+
+const (
+	// video xcode and dispatch state.
+	VideoUploadInfo      = 0
+	VideoXcodeSDFail     = 1
+	VideoXcodeSDFinish   = 2
+	VideoXcodeHDFail     = 3
+	VideoXcodeHDFinish   = 4
+	VideoDispatchRunning = 5
+	VideoDispatchFinish  = 6
+
+	XcodeFailZero = 0
+
+	// video status.
+	VideoStatusOpen      = int16(0)
+	VideoStatusAccess    = int16(10000)
+	VideoStatusWait      = int16(-1)
+	VideoStatusRecicle   = int16(-2)
+	VideoStatusLock      = int16(-4)
+	VideoStatusXcodeFail = int16(-16)
+	VideoStatusSubmit    = int16(-30)
+	VideoStatusDelete    = int16(-100)
+)
+
+type VideoUpInfo struct {
+	Table  string
+	Action string
+	Nw     *Video
+	Old    *Video
+}
+
+type Video struct {
+	ID          int64  `json:"id"`
+	Filename    string `json:"filename"`
+	Cid         int64  `json:"cid"`
+	Aid         int64  `json:"aid"`
+	Title       string `json:"eptitle"`
+	Desc        string `json:"description"`
+	SrcType     string `json:"src_type"`
+	Duration    int64  `json:"duration"`
+	Filesize    int64  `json:"filesize"`
+	Resolutions string `json:"resolutions"`
+	Playurl     string `json:"playurl"`
+	FailCode    int8   `json:"failinfo"`
+	Index       int    `json:"index_order"`
+	Attribute   int32  `json:"attribute"`
+	XcodeState  int8   `json:"xcode_state"`
+	Status      int16  `json:"status"`
+	CTime       string `json:"ctime"`
+	MTime       string `json:"mtime"`
+}

app/job/main/archive/model/retry/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/main/archive/model/retry",
+    tags = ["automanaged"],
+    deps = ["//app/job/main/archive/model/result:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/model/retry/model.go

@@ -0,0 +1,25 @@
+package retry
+
+import (
+	"go-common/app/job/main/archive/model/result"
+)
+
+const (
+	FailList          = "arc_job_fail_list"
+	FailUpCache       = "up_cache"
+	FailUpVideoCache  = "up_video_cache"
+	FailDelVideoCache = "del_video_cache"
+	FailDatabus       = "up_databus"
+	FailResultAdd     = "result_add"
+)
+
+// Info retry data
+type Info struct {
+	Action string `json:"action"`
+	Data   struct {
+		Aid        int64                 `json:"aid"`
+		State      int                   `json:"state"`
+		DatabusMsg *result.ArchiveUpInfo `json:"dbus_msg"`
+		Cids       []int64               `json:"cid"`
+	} `json:"data"`
+}

app/job/main/archive/service/BUILD

@@ -0,0 +1,72 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "archive.go",
+        "archive_result.go",
+        "cache.go",
+        "dm.go",
+        "retry.go",
+        "service.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/main/archive/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/dm2/model:go_default_library",
+        "//app/interface/main/dm2/rpc/client:go_default_library",
+        "//app/job/main/archive/conf:go_default_library",
+        "//app/job/main/archive/dao/archive:go_default_library",
+        "//app/job/main/archive/dao/email:go_default_library",
+        "//app/job/main/archive/dao/monitor:go_default_library",
+        "//app/job/main/archive/dao/reply:go_default_library",
+        "//app/job/main/archive/dao/result:go_default_library",
+        "//app/job/main/archive/model/archive:go_default_library",
+        "//app/job/main/archive/model/databus:go_default_library",
+        "//app/job/main/archive/model/dm:go_default_library",
+        "//app/job/main/archive/model/result:go_default_library",
+        "//app/job/main/archive/model/retry:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive/service/archive.go

@@ -0,0 +1,131 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/app/job/main/archive/model/result"
+	"go-common/app/job/main/archive/model/retry"
+	arcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/log"
+)
+
+func (s *Service) isPGC(aid int64) bool {
+	if addit, _ := s.archiveDao.Addit(context.TODO(), aid); addit != nil && (addit.UpFrom == archive.UpFromPGC || addit.UpFrom == archive.UpFromPGCSecret) {
+		return true
+	}
+	return false
+}
+
+func (s *Service) consumerVideoup(i int) {
+	defer s.waiter.Done()
+	for {
+		var (
+			aid int64
+			ok  bool
+		)
+		if aid, ok = <-s.videoupAids[i]; !ok {
+			log.Error("s.videoupAids chan closed")
+			return
+		}
+		arc, _ := s.arcServices[0].Archive3(context.TODO(), &arcmdl.ArgAid2{Aid: aid})
+		if arc != nil && (arc.AttrVal(arcmdl.AttrBitIsPGC) == arcmdl.AttrYes || arc.AttrVal(arcmdl.AttrBitIsBangumi) == arcmdl.AttrYes) {
+			if s.c.PGCAsync == 1 {
+				rt := &retry.Info{Action: retry.FailResultAdd}
+				rt.Data.Aid = aid
+				s.PushFail(context.TODO(), rt)
+				log.Warn("async PGC archive(%d)", aid)
+				continue
+			}
+			s.pgcAids <- aid
+			log.Info("aid(%d) title(%s) is PGC", aid, arc.Title)
+			continue
+		}
+		if s.c.UGCAsync == 1 {
+			rt := &retry.Info{Action: retry.FailResultAdd}
+			rt.Data.Aid = aid
+			s.PushFail(context.TODO(), rt)
+			log.Warn("async UGC archive(%d)", aid)
+			continue
+		}
+		s.arcUpdate(aid)
+	}
+}
+
+func (s *Service) pgcConsumer() {
+	defer s.waiter.Done()
+	for {
+		var (
+			aid int64
+			ok  bool
+		)
+		if aid, ok = <-s.pgcAids; !ok {
+			log.Error("s.pgcAids closed")
+			return
+		}
+		s.arcUpdate(aid)
+	}
+}
+
+func (s *Service) arcUpdate(aid int64) {
+	var (
+		oldResult *result.Archive
+		newResult *result.Archive
+		c         = context.TODO()
+		upCids    []int64
+		delCids   []int64
+		err       error
+		changed   bool
+	)
+	log.Info("sync resultDB archive(%d) start", aid)
+	defer func() {
+		if err != nil {
+			if oldResult != nil && (oldResult.AttrVal(result.AttrBitIsBangumi) == result.AttrYes || oldResult.AttrVal(result.AttrBitIsPGC) == result.AttrYes) {
+				s.pgcAids <- aid
+			} else {
+				s.videoupAids[aid%int64(s.c.ChanSize)] <- aid
+			}
+			log.Error("s.arcUpdate(%d) error(%v)", aid, err)
+		}
+	}()
+	if oldResult, err = s.resultDao.Archive(c, aid); err != nil {
+		log.Error("s.resultDao.Archive(%d) error(%v)", aid, err)
+	}
+	if changed, upCids, delCids, err = s.tranResult(c, aid); err != nil || !changed {
+		log.Error("aid(%d) nothing changed err(%+v)", aid, err)
+		err = nil
+		return
+	}
+	s.upVideoCache(aid, upCids)
+	s.delVideoCache(aid, delCids)
+	if newResult, err = s.resultDao.Archive(c, aid); err != nil {
+		log.Error("s.resultDao.Archive(%d) error(%v)", aid, err)
+		return
+	}
+	err = s.updateResultCache(newResult, oldResult)
+	if oldResult != nil {
+		s.updateResultField(newResult, oldResult)
+		s.updateSubjectMid(newResult, oldResult)
+		s.sendMail(newResult, oldResult)
+	}
+	action := "update"
+	if oldResult == nil {
+		action = "insert"
+	}
+	s.sendNotify(&result.ArchiveUpInfo{Table: "archive", Action: action, Nw: newResult, Old: oldResult})
+	if oldResult != nil {
+		log.Info("sync resultDB archive(%d) sync old(%+v) new(%+v) updated", aid, oldResult, newResult)
+		return
+	}
+	log.Info("sync resultDB archive(%d) new(%+v) inserted", aid, newResult)
+}
+
+func (s *Service) hadPassed(c context.Context, aid int64) (had bool) {
+	id, err := s.archiveDao.GetFirstPassByAID(c, aid)
+	if err != nil {
+		log.Error("hadPassed s.arc.GetFirstPassByAID error(%v) aid(%d)", err, aid)
+		return
+	}
+	had = id > 0
+	return
+}

app/job/main/archive/service/archive_result.go

@@ -0,0 +1,237 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/archive/model/archive"
+	"go-common/app/job/main/archive/model/result"
+	"go-common/app/job/main/archive/model/retry"
+	arcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+func (s *Service) updateSubjectMid(nw *result.Archive, old *result.Archive) {
+	if nw.Mid == old.Mid {
+		return
+	}
+	if err := s.replyDao.ChangeSubjectMid(nw.AID, nw.Mid); err == nil {
+		log.Info("/x/v2/reply/admin/subject/mid called")
+	}
+}
+
+// sendMail send e-mail after state change
+func (s *Service) sendMail(nw *result.Archive, old *result.Archive) {
+	// nw.Attribute == old.Attribute tmp remove
+	if nw.Mid == old.Mid && nw.TypeID == old.TypeID && nw.Duration == old.Duration && nw.Title == old.Title &&
+		nw.Cover == old.Cover && nw.Content == old.Content && nw.Copyright == old.Copyright &&
+		nw.State == old.State && nw.Access == old.Access && nw.Forward == old.Forward && nw.PubTime == old.PubTime && nw.CTime == old.CTime { // all field
+		log.Info("archive(%d) change only mtime no email", nw.ID)
+		return
+	}
+	a, err := s.arcServices[0].Archive3(context.TODO(), &arcmdl.ArgAid2{Aid: nw.AID})
+	if err != nil || a == nil {
+		log.Error("s.arcRPC.Archive3(%d) error(%v) or arc is nil", nw.AID, err)
+		return
+	}
+	// not bangumi or movie
+	if a.AttrVal(arcmdl.AttrBitIsBangumi) == arcmdl.AttrNo && a.AttrVal(arcmdl.AttrBitIsMovie) == arcmdl.AttrNo {
+		return
+	}
+	s.emailDao.PGCNotifyMail(a, nw, old)
+	log.Info("pgc notify(%d) mail send success", nw.AID)
+}
+
+func (s *Service) updateResultCache(nw *result.Archive, old *result.Archive) (err error) {
+	var (
+		c      = context.TODO()
+		oldMid int64
+	)
+	if old != nil && old.Mid != nw.Mid {
+		oldMid = old.Mid
+	}
+	for k, rpc := range s.arcServices {
+		if err = rpc.ArcCache2(c, &arcmdl.ArgCache2{Aid: nw.AID, Tp: arcmdl.CacheUpdate}); err != nil {
+			log.Error("s.arcRPC(%d).ArcCache2(%d, %s) error(%v)", k, nw.AID, arcmdl.CacheUpdate, err)
+		}
+		if nw.State >= 0 || nw.State == -6 {
+			if err = rpc.ArcCache2(c, &arcmdl.ArgCache2{Aid: nw.AID, OldMid: oldMid, Tp: arcmdl.CacheAdd}); err != nil {
+				log.Error("s.arcRpc(%d).ArcCache2(%d, %s) error(%v)", k, nw.AID, arcmdl.CacheAdd, err)
+			}
+		} else {
+			if err = rpc.ArcCache2(c, &arcmdl.ArgCache2{Aid: nw.AID, OldMid: oldMid, Tp: arcmdl.CacheDelete}); err != nil {
+				log.Error("s.arcRpc(%d).ArcCache2(%d, %s) error(%v)", k, nw.AID, arcmdl.CacheDelete, err)
+			}
+		}
+		if err != nil {
+			rt := &retry.Info{Action: retry.FailUpCache}
+			rt.Data.Aid = nw.AID
+			rt.Data.State = nw.State
+			s.PushFail(c, rt)
+			log.Error("updateResultCache error(%v)", err)
+		}
+	}
+	return
+}
+
+func (s *Service) updateResultField(nw *result.Archive, old *result.Archive) {
+	var (
+		c   = context.TODO()
+		err error
+	)
+	if nw.TypeID != old.TypeID {
+		for k, rpc := range s.arcServices {
+			if err = rpc.ArcFieldCache2(c, &arcmdl.ArgFieldCache2{Aid: nw.AID, TypeID: nw.TypeID, OldTypeID: old.TypeID}); err != nil {
+				log.Error("s.arcRPC(%d).ArcFieldCache2(%d, %d, %d) error(%v)", k, nw.AID, nw.TypeID, old.TypeID, err)
+			}
+		}
+	}
+}
+
+func (s *Service) tranResult(c context.Context, aid int64) (changed bool, upCids []int64, delCids []int64, err error) {
+	var (
+		tx        *sql.Tx
+		rows      int64
+		a         *archive.Archive
+		aResult   *result.Archive
+		ad        *archive.Addit
+		vs        []*archive.Video
+		videosCnt int
+		staff     []*archive.Staff
+	)
+	defer func() {
+		if err != nil {
+			rt := &retry.Info{Action: retry.FailResultAdd}
+			rt.Data.Aid = aid
+			s.PushFail(c, rt)
+			log.Error("tranResult error(%v)", err)
+		}
+	}()
+	if a, err = s.archiveDao.Archive(c, aid); err != nil || a == nil {
+		log.Error("s.arc.Archive(%d) error(%v)", aid, err)
+		return
+	}
+	if !a.IsSyncState() {
+		log.Info("archive(%d) state(%d) cant change", aid, a.State)
+		// FIXME: eeeee
+		if s.isPGC(aid) && !s.hadPassed(c, aid) {
+			log.Info("archive(%d) is PGC first change", aid)
+		} else {
+			return
+		}
+	}
+	if ad, err = s.archiveDao.Addit(c, aid); err != nil {
+		log.Error("s.arc.Addit(%d) error(%v)", aid, err)
+		return
+	}
+	if ad == nil {
+		ad = &archive.Addit{Aid: aid}
+	}
+	// if aid%10 == 0 || aid%10 == 1 || aid%10 == 2 {
+	if vs, err = s.archiveDao.Videos2(c, aid); err != nil {
+		log.Error("s.arc.Videos2(%d) error(%v)", aid, err)
+		return
+	}
+	// } else {
+	// if vs, err = s.archiveDao.Videos(c, aid); err != nil {
+	// 	log.Error("s.arc.Videos(%d) error(%v)", aid, err)
+	// 	return
+	// }
+	// }
+	for _, v := range vs {
+		if (v.Status == archive.VideoStatusAccess || v.Status == archive.VideoStatusOpen) && v.State >= 0 {
+			videosCnt++
+		}
+	}
+	// 辣鸡dede, check cid
+	for _, v := range vs {
+		if v.Cid == 0 && v.Status == archive.VideoStatusSubmit {
+			// NOTE: 刚上传，没必要同步去
+			log.Error("aid(%d) vid(%d) cid(%d) videoStatus(%d) return", v.Aid, v.ID, v.Cid, v.Status)
+			return
+		}
+	}
+	if aResult, err = s.resultDao.Archive(c, aid); err != nil {
+		log.Error("s.resultDao.Archive error(%+v)", err)
+		return
+	}
+	if tx, err = s.resultDao.BeginTran(c); err != nil {
+		log.Error("s.result.BeginTran error(%v)", err)
+		return
+	}
+	var (
+		duration   int
+		firstCid   int64
+		dimensions string
+	)
+	for _, v := range vs {
+		if (v.Status == archive.VideoStatusAccess || v.Status == archive.VideoStatusOpen) && v.State == archive.VideoRelationBind {
+			if _, err = s.resultDao.TxAddVideo(c, tx, v); err != nil {
+				tx.Rollback()
+				log.Error("s.result.TxAddVideo error(%v)", err)
+				break
+			}
+			duration += int(v.Duration)
+			upCids = append(upCids, v.Cid)
+			if v.Index == 1 && v.SrcType == "vupload" {
+				firstCid = v.Cid
+				dimensions = v.Dimensions
+			}
+		} else {
+			if _, err = s.resultDao.TxDelVideoByCid(c, tx, aid, v.Cid); err != nil {
+				tx.Rollback()
+				log.Error("s.result.TxDelVideoByCid error(%v)")
+				break
+			}
+			delCids = append(delCids, v.Cid)
+		}
+	}
+	a.Duration = duration
+	if rows, err = s.resultDao.TxAddArchive(c, tx, a, ad, videosCnt, firstCid, dimensions); err != nil {
+		tx.Rollback()
+		log.Error("s.result.TxAddArchive error(%v)", err)
+		return
+	}
+	if rows == 0 {
+		if _, err = s.resultDao.TxUpArchive(c, tx, a, ad, videosCnt, firstCid, dimensions); err != nil {
+			tx.Rollback()
+			log.Error("s.result.TxUpArchive error(%v)")
+			return
+		}
+	}
+	// 更新联合投稿人
+	if a.AttrVal(archive.AttrBitIsCooperation) == archive.AttrYes {
+		if staff, err = s.archiveDao.Staff(c, aid); err != nil {
+			tx.Rollback()
+			log.Error("s.archiveDao.Staff aid(%d) error(%v)", aid, err)
+			return
+		}
+		if err = s.resultDao.TxDelStaff(c, tx, aid); err != nil {
+			tx.Rollback()
+			log.Error("s.result.TxDelStaff aid(%d) error(%v)", aid, err)
+			return
+		}
+		if staff != nil {
+			if err = s.resultDao.TxAddStaff(c, tx, aid, staff); err != nil {
+				tx.Rollback()
+				log.Error("s.result.TxAddStaff aid(%d) error(%v)", aid, err)
+				return
+			}
+		}
+	} else { //从联合投稿改为非联合投稿的 删除staff数据
+		if aResult != nil && aResult.AttrVal(archive.AttrBitIsCooperation) == archive.AttrYes {
+			if err = s.resultDao.TxDelStaff(c, tx, aid); err != nil {
+				tx.Rollback()
+				log.Error("s.result.TxDelStaff aid(%d) error(%v)", aid, err)
+				return
+			}
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("tx.Commit error(%v)")
+		return
+	}
+	log.Info("aid(%d) upCids(%d) delCids(%d) db updated", aid, len(upCids), len(delCids))
+	changed = true
+	return
+}

app/job/main/archive/service/cache.go

@@ -0,0 +1,203 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	jobmdl "go-common/app/job/main/archive/model/databus"
+	"go-common/app/job/main/archive/model/result"
+	accgrpc "go-common/app/service/main/account/api"
+	"go-common/app/service/main/archive/api"
+	arcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_actForUname = "updateUname"
+	_actForFace  = "updateFace"
+	_actForAdmin = "updateByAdmin"
+)
+
+func (s *Service) cachesubproc() {
+	defer s.waiter.Done()
+	var msgs = s.cacheSub.Messages()
+	for {
+		var (
+			msg *databus.Message
+			ok  bool
+			err error
+		)
+		if msg, ok = <-msgs; !ok {
+			log.Error("s.cachesub.messages closed")
+			return
+		}
+		if s.closeSub {
+			return
+		}
+		m := &jobmdl.Rebuild{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
+			continue
+		}
+		log.Info("cacheSub key(%s) value(%s) start", msg.Key, msg.Value)
+		var retryError error
+		for {
+			var (
+				a         *result.Archive
+				infoReply *accgrpc.InfoReply
+				c         = context.TODO()
+			)
+			if retryError != nil {
+				time.Sleep(10 * time.Millisecond)
+			}
+			a, retryError = s.resultDao.Archive(c, m.Aid)
+			if retryError != nil {
+				log.Error("s.resultDao.Archive(%d) error(%v)", m.Aid, retryError)
+				continue
+			}
+			if a == nil || a.Mid == 0 {
+				log.Info("cache break archive(%d) not exist or mid==0", m.Aid)
+				break
+			}
+			infoReply, retryError = s.accGRPC.Info3(c, &accgrpc.MidReq{Mid: a.Mid})
+			if retryError != nil {
+				if ecode.Cause(retryError).Equal(ecode.MemberNotExist) {
+					log.Info("archive(%d) mid(%d) not exist", m.Aid, a.Mid)
+					break
+				}
+				log.Error("s.acc.RPC.Info3(%d) error(%v)", m.Aid, retryError)
+				continue
+			}
+			if infoReply == nil {
+				log.Error("infoReply mid(%d) err is nil,but info is nil too", a.Mid)
+				break
+			}
+			if infoReply.Info.Name == "" || infoReply.Info.Face == "" {
+				log.Error("empty info mid(%d) info(%+v)", infoReply.Info.Mid, infoReply.Info)
+				break
+			}
+			for k, arcRPC := range s.arcServices {
+				if retryError = arcRPC.ArcCache2(c, &arcmdl.ArgCache2{Aid: m.Aid, Tp: arcmdl.CacheUpdate}); retryError != nil {
+					log.Error("s.arcRPC(%d).ArcCache2(%d) error(%v)", k, m.Aid, retryError)
+					continue
+				}
+			}
+			log.Info("archive(%d) mid(%d) uname(%s) update success", m.Aid, infoReply.Info.Mid, infoReply.Info.Name)
+			break
+		}
+		msg.Commit()
+	}
+}
+
+func (s *Service) accountNotifyproc() {
+	defer s.waiter.Done()
+	var msgs = s.accountNotifySub.Messages()
+	for {
+		var (
+			msg *databus.Message
+			ok  bool
+			err error
+			c   = context.TODO()
+		)
+		if msg, ok = <-msgs; !ok {
+			log.Error("s.cachesub.messages closed")
+			return
+		}
+		if s.closeSub {
+			return
+		}
+		msg.Commit()
+		m := &jobmdl.AccountNotify{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		log.Info("accountNotify got key(%s) value(%s)", msg.Key, msg.Value)
+		if m.Action != _actForAdmin && m.Action != _actForFace && m.Action != _actForUname {
+			log.Warn("accountNotify skip action(%s) values(%s)", m.Action, msg.Value)
+			continue
+		}
+		var count int
+		if count, err = s.arcServices[0].UpCount2(c, &arcmdl.ArgUpCount2{Mid: m.Mid}); err != nil {
+			log.Error("s.arcRPC.UpCount2(%d) error(%v)", m.Mid, err)
+			continue
+		}
+		if count == 0 {
+			log.Info("accountNotify mid(%d) passed(%d)", m.Mid, count)
+			continue
+		}
+		if m.Action == _actForAdmin {
+			// check uname or face is updated
+			var am []*api.Arc
+			if am, err = s.arcServices[0].UpArcs3(c, &arcmdl.ArgUpArcs2{Mid: m.Mid, Ps: 2, Pn: 1}); err != nil {
+				if ecode.Cause(err).Equal(ecode.NothingFound) {
+					err = nil
+					log.Info("accountNotify mid(%d) no passed archive", m.Mid)
+					continue
+				}
+				log.Error("accountNotify mid(%d) error(%v)", m.Mid, err)
+				continue
+			}
+			if len(am) == 0 {
+				log.Info("accountNotify mid(%d) no passed archive", m.Mid)
+				continue
+			}
+			var reply *accgrpc.InfoReply
+			if reply, err = s.accGRPC.Info3(c, &accgrpc.MidReq{Mid: m.Mid}); err != nil || reply == nil {
+				log.Error("accountNotify accRPC.info3(%d) error(%v)", m.Mid, err)
+				continue
+			}
+			if reply.Info.Name == am[0].Author.Name && reply.Info.Face == am[0].Author.Face {
+				log.Info("accountNotify face(%s) name(%s) not change", reply.Info.Face, reply.Info.Name)
+				continue
+			}
+		}
+		s.notifyMu.Lock()
+		s.notifyMid[m.Mid] = struct{}{}
+		s.notifyMu.Unlock()
+	}
+}
+
+func (s *Service) clearMidCache() {
+	defer s.waiter.Done()
+	for {
+		time.Sleep(5 * time.Second)
+		s.notifyMu.Lock()
+		mids := s.notifyMid
+		s.notifyMid = make(map[int64]struct{})
+		s.notifyMu.Unlock()
+		for mid := range mids {
+			s.updateUpperCache(context.TODO(), mid)
+		}
+		if s.closeSub && len(s.notifyMid) == 0 {
+			return
+		}
+	}
+}
+
+func (s *Service) updateUpperCache(c context.Context, mid int64) (err error) {
+	// update archive cache
+	var aids []int64
+	if aids, err = s.resultDao.UpPassed(c, mid); err != nil {
+		log.Error("s.resultDao.UpPassed(%d) error(%v)", mid, err)
+		return
+	}
+	failedCnt := 0
+	for _, aid := range aids {
+		for k, rpc := range s.arcServices {
+			if err = rpc.ArcCache2(c, &arcmdl.ArgCache2{Aid: aid}); err != nil {
+				log.Error("s.arcRPC(%d).ArcCache2(%d) mid(%d) error(%v)", k, aid, mid, err)
+				failedCnt++
+			}
+		}
+	}
+	if failedCnt > 0 {
+		log.Error("accountNotify updateUpperCache mid(%d) failed(%d)", mid, failedCnt)
+		return
+	}
+	log.Info("accountNofity updateUpperCache mid(%d) successed(%d)", mid, len(aids))
+	return
+}

app/job/main/archive/service/dm.go

@@ -0,0 +1,155 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+	"strings"
+	"time"
+
+	dmmdl "go-common/app/interface/main/dm2/model"
+	"go-common/app/job/main/archive/model/dm"
+	"go-common/app/service/main/archive/api"
+	"go-common/app/service/main/archive/model/archive"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_type             = "archive"
+	_table            = "dm_subject_"
+	_update           = "update"
+	_subjectTypeForAv = 1
+)
+
+func (s *Service) dmConsumer() {
+	defer s.waiter.Done()
+	for {
+		var (
+			msg   *databus.Message
+			ok    bool
+			err   error
+			canal = &dm.Canal{}
+		)
+		if msg, ok = <-s.dmSub.Messages(); !ok || s.closeSub {
+			log.Error("s.dmSub Closed")
+			return
+		}
+		msg.Commit()
+		if err = json.Unmarshal(msg.Value, canal); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, canal)
+			continue
+		}
+		// not dm_subject_
+		if !strings.HasPrefix(canal.Table, _table) {
+			log.Warn("table(%s) message(%s) skiped", canal.Table, msg.Value)
+			continue
+		}
+		// not update
+		if canal.Action != _update {
+			log.Warn("action(%s) message(%s) skiped", canal.Action, msg.Value)
+			continue
+		}
+		var subject *dm.Subject
+		if err = json.Unmarshal(canal.New, &subject); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", canal.New, err)
+			continue
+		}
+		// type must be av
+		if subject.Type != _subjectTypeForAv {
+			log.Warn("subject type(%s) is not av message(%+v)", subject.Type, subject)
+			continue
+		}
+		s.dmMu.Lock()
+		s.dmCids[subject.CID] = struct{}{}
+		s.dmMu.Unlock()
+	}
+}
+
+func (s *Service) dmCounter() {
+	defer s.waiter.Done()
+	for {
+		time.Sleep(5 * time.Second)
+		s.dmMu.Lock()
+		cm := s.dmCids
+		s.dmCids = make(map[int64]struct{})
+		s.dmMu.Unlock()
+		var (
+			aids    []int64
+			err     error
+			c       = context.TODO()
+			am      = make(map[int64][]int64)
+			allCids []int64
+		)
+		for cid := range cm {
+			if aids, err = s.archiveDao.Aids(c, cid); err != nil {
+				log.Error("s.archiveDao.Aids(%d) error(%v)", err)
+				continue
+			}
+			for _, aid := range aids {
+				if _, ok := am[aid]; ok {
+					continue
+				}
+				var pages []*api.Page
+				if pages, err = s.arcServices[0].Page3(c, &archive.ArgAid2{Aid: aid}); err != nil {
+					log.Error("s.arcServices[0].Page3(%d) error(%v)", aid, err)
+					continue
+				}
+				for _, p := range pages {
+					am[aid] = append(am[aid], p.Cid)
+					allCids = append(allCids, p.Cid)
+				}
+			}
+		}
+		var (
+			times    int
+			argCount = 100
+			cids     []int64
+			cDmCount = make(map[int64]int64)
+		)
+		if len(allCids)%argCount == 0 {
+			times = len(allCids) / argCount
+		} else {
+			times = len(allCids)/argCount + 1
+		}
+		for i := 0; i < times; i++ {
+			if i == times-1 {
+				cids = allCids[i*argCount:]
+			} else {
+				cids = allCids[i*argCount : (i+1)*argCount]
+			}
+			var sm map[int64]*dmmdl.SubjectInfo
+			if sm, err = s.dm2RPC.SubjectInfos(c, &dmmdl.ArgOids{Type: 1, Oids: cids}); err != nil {
+				log.Error("s.dm2RPC.SubjectInfos(%v) error(%v)", cids, err)
+				continue
+			}
+			for cid, s := range sm {
+				cDmCount[cid] = int64(s.Count)
+			}
+		}
+	L:
+		for aid, cids := range am {
+			var sum int64
+			for _, cid := range cids {
+				var (
+					cnt int64
+					ok  bool
+				)
+				if cnt, ok = cDmCount[cid]; !ok {
+					log.Error("dm cid(%d) no count", cid)
+					break L
+				}
+				sum += cnt
+			}
+			dMsg := &dm.Count{ID: aid, Count: sum, Type: _type, Timestamp: time.Now().Unix()}
+			if err = s.dmPub.Send(c, strconv.FormatInt(aid, 10), dMsg); err != nil {
+				log.Error("s.dmPub.Send error(%v)", err)
+				continue
+			}
+			log.Info("s.dmPub.Send(%+v) success", dMsg)
+		}
+		if s.closeSub {
+			return
+		}
+	}
+}

app/job/main/archive/service/retry.go

@@ -0,0 +1,80 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/archive/model/result"
+	"go-common/app/job/main/archive/model/retry"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+func (s *Service) retryproc() {
+	defer s.waiter.Done()
+	for {
+		if s.closeRetry {
+			return
+		}
+		var (
+			c   = context.TODO()
+			bs  []byte
+			err error
+		)
+		bs, err = s.PopFail(c)
+		if err != nil || bs == nil {
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		msg := &retry.Info{}
+		if err = json.Unmarshal(bs, msg); err != nil {
+			log.Error("json.Unretry dedeSyncmarshal(%s) error(%v)", bs, err)
+			continue
+		}
+		log.Info("retry %s %s", retry.FailUpCache, bs)
+		switch msg.Action {
+		case retry.FailUpCache:
+			s.updateResultCache(&result.Archive{AID: msg.Data.Aid, State: msg.Data.State}, nil)
+		case retry.FailDatabus:
+			var upInfo = &result.ArchiveUpInfo{Table: msg.Data.DatabusMsg.Table, Action: msg.Data.DatabusMsg.Table, Nw: msg.Data.DatabusMsg.Nw, Old: msg.Data.DatabusMsg.Old}
+			s.sendNotify(upInfo)
+		case retry.FailUpVideoCache:
+			s.upVideoCache(msg.Data.Aid, msg.Data.Cids)
+		case retry.FailDelVideoCache:
+			s.delVideoCache(msg.Data.Aid, msg.Data.Cids)
+		case retry.FailResultAdd:
+			s.arcUpdate(msg.Data.Aid)
+		default:
+			continue
+		}
+	}
+}
+
+// PushFail rpush fail item to redis
+func (s *Service) PushFail(c context.Context, a interface{}) (err error) {
+	var (
+		conn = s.redis.Get(c)
+		bs   []byte
+	)
+	defer conn.Close()
+	if bs, err = json.Marshal(a); err != nil {
+		log.Error("json.Marshal(%v) error(%v)", a, err)
+		return
+	}
+	if _, err = conn.Do("RPUSH", retry.FailList, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s, %s) error(%v)")
+	}
+	return
+}
+
+// PopFail lpop fail item from redis
+func (s *Service) PopFail(c context.Context) (bs []byte, err error) {
+	var conn = s.redis.Get(c)
+	defer conn.Close()
+	if bs, err = redis.Bytes(conn.Do("LPOP", retry.FailList)); err != nil && err != redis.ErrNil {
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%v)", retry.FailList, err)
+		return
+	}
+	return
+}

app/job/main/archive/service/service.go

@@ -0,0 +1,198 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+
+	dm2rpc "go-common/app/interface/main/dm2/rpc/client"
+	"go-common/app/job/main/archive/conf"
+	"go-common/app/job/main/archive/dao/archive"
+	"go-common/app/job/main/archive/dao/email"
+	"go-common/app/job/main/archive/dao/monitor"
+	"go-common/app/job/main/archive/dao/reply"
+	"go-common/app/job/main/archive/dao/result"
+	dbusmdl "go-common/app/job/main/archive/model/databus"
+	resmdl "go-common/app/job/main/archive/model/result"
+	"go-common/app/job/main/archive/model/retry"
+	accgrpc "go-common/app/service/main/account/api"
+	arcrpc "go-common/app/service/main/archive/api/gorpc"
+	xredis "go-common/library/cache/redis"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+// Service service
+type Service struct {
+	c                *conf.Config
+	closeRetry       bool
+	closeSub         bool
+	archiveDao       *archive.Dao
+	emailDao         *email.Dao
+	monitorDao       *monitor.Dao
+	replyDao         *reply.Dao
+	resultDao        *result.Dao
+	redis            *xredis.Pool
+	waiter           sync.WaitGroup
+	videoupSub       *databus.Databus
+	archiveResultPub *databus.Databus
+	dmPub            *databus.Databus
+	dmSub            *databus.Databus
+	cacheSub         *databus.Databus
+	accountNotifySub *databus.Databus
+	sfTpsCache       map[int16]int16
+	adtTpsCache      map[int16]struct{}
+	arcServices      []*arcrpc.Service2
+	accGRPC          accgrpc.AccountClient
+	dm2RPC           *dm2rpc.Service
+	// databus channel
+	videoupAids []chan int64
+	pgcAids     chan int64
+	// dm count
+	dmCids    map[int64]struct{}
+	dmMu      sync.Mutex
+	notifyMid map[int64]struct{}
+	notifyMu  sync.Mutex
+}
+
+// New is archive service implementation.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:                c,
+		archiveDao:       archive.New(c),
+		emailDao:         email.New(c),
+		monitorDao:       monitor.New(c),
+		replyDao:         reply.New(c),
+		resultDao:        result.New(c),
+		dm2RPC:           dm2rpc.New(c.Dm2RPC),
+		videoupSub:       databus.New(c.VideoupSub),
+		dmSub:            databus.New(c.DmSub),
+		dmPub:            databus.New(c.DmPub),
+		archiveResultPub: databus.New(c.ArchiveResultPub),
+		cacheSub:         databus.New(c.CacheSub),
+		accountNotifySub: databus.New(c.AccountNotifySub),
+		redis:            xredis.NewPool(c.Redis),
+		pgcAids:          make(chan int64, 1024),
+		dmCids:           make(map[int64]struct{}),
+		notifyMid:        make(map[int64]struct{}),
+		arcServices:      make([]*arcrpc.Service2, 0),
+	}
+	var err error
+	if s.accGRPC, err = accgrpc.NewClient(nil); err != nil {
+		panic(fmt.Sprintf("account.service grpc not found!!!!!!!!!!!! error(%v)", err))
+	}
+	for _, sc := range s.c.ArchiveServices {
+		s.arcServices = append(s.arcServices, arcrpc.New2(sc))
+	}
+	for i := 0; i < s.c.ChanSize; i++ {
+		s.videoupAids = append(s.videoupAids, make(chan int64, 1024))
+		s.waiter.Add(1)
+		go s.consumerVideoup(i)
+		s.waiter.Add(1)
+		go s.pgcConsumer()
+	}
+	s.loadType()
+	go s.cacheproc()
+	// sync archive_result db!!!!!!!
+	s.waiter.Add(1)
+	go s.videoupConsumer()
+	s.waiter.Add(1)
+	go s.dmConsumer()
+	// check consumer
+	go s.checkConsume()
+	s.waiter.Add(1)
+	go s.retryproc()
+	s.waiter.Add(1)
+	go s.dmCounter()
+	s.waiter.Add(1)
+	go s.cachesubproc()
+	s.waiter.Add(1)
+	go s.accountNotifyproc()
+	s.waiter.Add(1)
+	go s.clearMidCache()
+	return s
+}
+
+func (s *Service) sendNotify(upInfo *resmdl.ArchiveUpInfo) {
+	var (
+		nw  []byte
+		old []byte
+		err error
+		msg *dbusmdl.Message
+		c   = context.TODO()
+		rt  = &retry.Info{}
+	)
+	if nw, err = json.Marshal(upInfo.Nw); err != nil {
+		log.Error("json.Marshal(%+v) error(%v)", upInfo.Nw, err)
+		return
+	}
+	if old, err = json.Marshal(upInfo.Old); err != nil {
+		log.Error("json.Marshal(%+v) error(%v)", upInfo.Old, err)
+		return
+	}
+	msg = &dbusmdl.Message{Action: upInfo.Action, Table: upInfo.Table, New: nw, Old: old}
+	if err = s.archiveResultPub.Send(c, strconv.FormatInt(upInfo.Nw.AID, 10), msg); err != nil {
+		log.Error("s.archiveResultPub.Send(%+v) error(%v)", msg, err)
+		rt.Action = retry.FailDatabus
+		rt.Data.Aid = upInfo.Nw.AID
+		rt.Data.DatabusMsg = upInfo
+		s.PushFail(c, rt)
+		return
+	}
+	msgStr, _ := json.Marshal(msg)
+	log.Info("sendNotify(%s) successed", msgStr)
+}
+
+func (s *Service) loadType() {
+	tpm, err := s.archiveDao.TypeMapping(context.TODO())
+	if err != nil {
+		log.Error("s.dede.TypeMapping error(%v)", err)
+		return
+	}
+	s.sfTpsCache = tpm
+	// audit types
+	adt, err := s.archiveDao.AuditTypesConf(context.TODO())
+	if err != nil {
+		log.Error("s.dede.AuditTypesConf error(%v)", err)
+		return
+	}
+	s.adtTpsCache = adt
+}
+
+func (s *Service) cacheproc() {
+	for {
+		time.Sleep(1 * time.Minute)
+		s.loadType()
+	}
+}
+
+// check consumer stat
+func (s *Service) checkConsume() {
+	if s.c.Env != "pro" {
+		return
+	}
+	for {
+		time.Sleep(1 * time.Minute)
+		for i := 0; i < s.c.ChanSize; i++ {
+			if l := len(s.videoupAids[i]); l > s.c.MonitorSize {
+				s.monitorDao.Send(context.TODO(), s.c.WeChantUsers, fmt.Sprintf("archive-job报警了啊\n UGC的chan太大了！！！\n s.videoupAids[%d] size(%d) is too large\n 是不是有人在刷数据！！！！", i, l), s.c.WeChatToken, s.c.WeChatSecret)
+			}
+		}
+		if l := len(s.pgcAids); l > s.c.MonitorSize {
+			s.monitorDao.Send(context.TODO(), s.c.WeChantUsers, fmt.Sprintf("archive-job报警了啊\n PGC的chan太大了！！！\n chan size(%d) is too large \n 是不是有人在刷数据！！！！", l), s.c.WeChatToken, s.c.WeChatSecret)
+		}
+	}
+}
+
+// Close kafaka consumer close.
+func (s *Service) Close() (err error) {
+	s.closeSub = true
+	time.Sleep(2 * time.Second)
+	s.videoupSub.Close()
+	s.closeRetry = true
+	s.waiter.Wait()
+	return
+}

app/job/main/archive/service/service_test.go

@@ -0,0 +1,42 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/archive/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/archive-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+func Test_loadType(t *testing.T) {
+	Convey("loadType", t, func() {
+		s.loadType()
+	})
+}
+
+func Test_PopFail(t *testing.T) {
+	Convey("PopFail", t, func() {
+		s.PopFail(context.TODO())
+	})
+}
+
+func Test_TranResult(t *testing.T) {
+	Convey("tranResult", t, func() {
+		_, _, _, err := s.tranResult(context.TODO(), 10098500)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/archive/service/video.go

@@ -0,0 +1,91 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	jobmdl "go-common/app/job/main/archive/model/databus"
+	"go-common/app/job/main/archive/model/retry"
+	"go-common/app/service/main/archive/model/archive"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+func (s *Service) videoupConsumer() {
+	defer s.waiter.Done()
+	var msgs = s.videoupSub.Messages()
+	for {
+		var (
+			msg *databus.Message
+			ok  bool
+			err error
+		)
+		if msg, ok = <-msgs; !ok {
+			log.Error("s.videoupSub.messages closed")
+			return
+		}
+		if s.closeSub {
+			return
+		}
+		msg.Commit()
+		m := &jobmdl.Videoup{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
+			continue
+		}
+		log.Info("videoupMessage key(%s) value(%s) start", msg.Key, msg.Value)
+		if m.Aid <= 0 {
+			log.Warn("aid(%d) <= 0 WTF(%s)", m.Aid, msg.Value)
+			continue
+		}
+		switch m.Route {
+		case jobmdl.RouteAutoOpen, jobmdl.RouteDelayOpen, jobmdl.RouteDeleteArchive, jobmdl.RouteSecondRound, jobmdl.RouteFirstRoundForbid, jobmdl.RouteForceSync:
+			select {
+			case s.videoupAids[m.Aid%int64(s.c.ChanSize)] <- m.Aid:
+			default:
+				rt := &retry.Info{Action: retry.FailResultAdd}
+				rt.Data.Aid = m.Aid
+				s.PushFail(context.TODO(), rt)
+				log.Warn("s.videoupAids is full!!! async databus archive(%d)", m.Aid)
+			}
+		}
+		log.Info("videoupMessage key(%s) value(%s) finish", msg.Key, msg.Value)
+	}
+}
+
+func (s *Service) delVideoCache(aid int64, cids []int64) (err error) {
+	for _, cid := range cids {
+		for k, rpc := range s.arcServices {
+			if err = rpc.DelVideo2(context.TODO(), &archive.ArgVideo2{Aid: aid, Cid: cid}); err != nil {
+				log.Error("s.arcRpc(%d).DelVideo2(%d, %d) error(%v)", k, aid, cid, err)
+				if ecode.Cause(err) != ecode.NothingFound {
+					rt := &retry.Info{Action: retry.FailDelVideoCache}
+					rt.Data.Aid = aid
+					rt.Data.Cids = []int64{cid}
+					s.PushFail(context.TODO(), rt)
+					log.Error("delVideoCache error(%v)", err)
+				}
+			}
+		}
+	}
+	return
+}
+
+func (s *Service) upVideoCache(aid int64, cids []int64) (err error) {
+	for _, cid := range cids {
+		for k, rpc := range s.arcServices {
+			if err = rpc.UpVideo2(context.TODO(), &archive.ArgVideo2{Aid: aid, Cid: cid}); err != nil {
+				log.Error("s.arcRpc(%d).UpVideo2(%d, %d) error(%v)", k, aid, cid, err)
+				if ecode.Cause(err) != ecode.NothingFound {
+					rt := &retry.Info{Action: retry.FailUpVideoCache}
+					rt.Data.Aid = aid
+					rt.Data.Cids = []int64{cid}
+					s.PushFail(context.TODO(), rt)
+					log.Error("upVideoCache error(%v)", err)
+				}
+			}
+		}
+	}
+	return
+}