openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
app/service/main/archive/BUILD Normal file
View File

@@ -0,0 +1,24 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/archive/api:all-srcs",
"//app/service/main/archive/cmd:all-srcs",
"//app/service/main/archive/conf:all-srcs",
"//app/service/main/archive/dao:all-srcs",
"//app/service/main/archive/model:all-srcs",
"//app/service/main/archive/server/gorpc:all-srcs",
"//app/service/main/archive/server/grpc:all-srcs",
"//app/service/main/archive/server/http:all-srcs",
"//app/service/main/archive/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

926
app/service/main/archive/CHANGELOG.md Normal file
View File

@@ -0,0 +1,926 @@
#### archive rpc service
##### Version 6.47.6
> 1.iPhone 5.36版本不吐拜年祭单品稿件的秒开地址
##### Version 6.47.5
> 1.拜年祭单品视频不吐秒开地址
##### Version 6.47.4
> 1.view接口只对archive做强判断
##### Version 6.47.3
> 1.fix view
##### Version 6.47.2
> 1.迁移gorpc方法到gpc
##### Version 6.47.1
> 1.view接口增加staff信息
##### Version 6.47.0
> 1.grpc增加注释
##### Version 6.46.6
> 1.更新稿件缓存增加联合投稿部分
##### Version 6.46.5
> 1.Dislike强制0
##### Version 6.46.4
> 1.增加高能看点、bgm、联合投稿attribute
##### Version 6.46.3
> 1.删除RPC中的addShare方法
##### Version 6.46.2
> 1.分享不发databus消息
##### Version 6.46.1
> 1.fix package
##### Version 6.46.0
> 1.issues #403 大仓库项目目录结构改进
##### Version 6.45.2
> 1.加参数控制pgc吐playurl
##### Version 6.45.1
> 1.pgc不吐playurl
##### Version 6.45.0
> 1.拦截archive miss不存在的稿件
##### Version 6.44.4
> 1.接入account grpc
##### Version 6.44.3
> 1.秒开拦截辣鸡参数
##### Version 6.44.2
> 1.秒开qn白名单+大会员清晰度降级
##### Version 6.44.1
> 1.计数默认返回值修改
##### Version 6.44.0
> 1.增加UGCPay标识
##### Version 6.43.9
> 1.dash格式加codecid
##### Version 6.43.8
> 1.修复冷门up主投稿列表稿件不全的bug
##### Version 6.43.7
> 1.初始化缓存日志
##### Version 6.43.6
> 1.初始化分区缓存的时候使用context.Background
##### Version 6.43.5
> 1.修改重置up信息的缓存逻辑
##### Version 6.43.4
> 1.增加日志观察job异步databus消息是否发送成功
##### Version 6.43.3
> 1.增加日志观察账号RPC服务的返回是否正常
##### Version 6.43.2
> 1.增加日志观察账号昵称头像是否为空
##### Version 6.43.1
> 1.秒开接口空字段不吐
##### Version 6.43.0
> 1.秒开接口增加dash字段
##### Version 6.42.2
> 1.同步IsNormal,AttrVal方法
##### Version 6.42.1
> 1.接入grpc
##### Version 6.42.0
> 1.添加dao层ut
##### Version 6.42.0
> 1.调整目录
##### Version 6.41.1
> 1.fix cache ctx
##### Version 6.41.0
> 1.issue 249 metadata ip
##### Version 6.40.10
> 1.增加视频云的fnver，fnval字段返回
##### Version 6.40.9
> 1.更新bvc pb文件
##### Version 6.40.8
> 1.优化重新生成账号缓存的逻辑
##### Version 6.40.7
> 1.账号接口请求失败时，走databus慢慢更新
##### Version 6.40.6
> 1.透传视频云fnval,fnver字段
##### Version 6.40.5
> 1.账号老是不知道刷什么东西
##### Version 6.40.4
> 1.增加是否可以投屏
##### Version 6.40.3
> 1.透传投屏信息
##### Version 6.40.2
> 1.支持地区限制
##### Version 6.40.1
> 1.稿件描述走缓存
##### Version 6.40.0
> 1.稿件更新缓存bugfix
> 2.重置retag
##### Version 6.39.3
> 1.增加autoplay字段
##### Version 6.39.2
> 1.HTTP接口增加稿件分辨率字段
##### Version 6.39.1
> 1.PB接口体增加json字段输出
##### Version 6.39.0
> 1.批量MC接口优化
##### Version 6.38.1
> 1.分辨率0,0,0不做处理
##### Version 6.38.0
> 1.稿件增加分辨率字段
##### Version 6.37.19
> 1.批量稿件接口代码优化
##### Version 6.37.18
> 1.conn close fix
##### Version 6.37.17
> 1.增加缓存容错
##### Version 6.37.16
> 1.bvc灰度
##### Version 6.37.15
> 1.redis set expire -> setex
##### Version 6.37.14
> 1.无投稿的用户只缓存10分钟
##### Version 6.37.13
> 1.分享行为的databus key从aid改为mid
##### Version 6.37.12
> 1.fix row close
##### Version 6.37.11
> 1.清理share代码
##### Version 6.37.10
> 1.删除videoshot add接口
##### Version 6.37.9
> 1.RPC不需要token了 By 郝冠伟确认
##### Version 6.37.8
> 1.增加register
##### Version 6.37.7
> 1.删除冗余代码
##### Version 6.37.6
> 1.使用bm
##### Version 6.37.5
> 1.删除多余配置
##### Version 6.37.4
> 1.增加批量获取up投稿数量的http接口
##### Version 6.37.3
> 1.删除limit模块
##### Version 6.37.2
> 1.第一次分享改发databus
##### Version 6.37.1
> 1.使用account-service v7
##### Version 6.37.0
> 1.迁移到主站目录下
##### Version 6.36.16
> 1.取消强制开关
##### Version 6.36.15
> 1.提供给B+的秒开接口强行不返回playurl
##### Version 6.36.14
> 1.参数长度调整为200
##### Version 6.36.13
> 1.补充UnitTest
##### Version 6.36.12
> 1.配置文件增加开关选项，控制是否请求视频云获取播放信息
##### Version 6.36.11
> 1.firstCid只用vupload，外部源不缓存
##### Version 6.36.10
> 1.优化秒开代码
##### Version 6.36.9
> 1.Archive3结构体增加第一P的cid，供后续业务扩展使用
##### Version 6.36.8
> 1.attr增加地区限制
##### Version 6.36.7
> 1.share数双写新databus
##### Version 6.36.6
> 1.接bvc的pb接口
##### Version 6.36.5
> 1.Convey test
##### Version 6.36.4
> 1.BFS改回来
##### Version 6.36.3
> 1.BFS的封面图强制返回https
##### Version 6.36.2
> 1.删除attrbithideclick相关代码
##### Version 6.36.1
> 1.删除like的相关代码与配置
##### Version 6.36.0
> 1.删除like的相关代码与配置
##### Version 6.35.1
> 1.修改透传的player字段名
##### Version 6.35.0
> 1.attr的第十二位改成 IsPorder 私单标记
##### Version 6.34.0
> 1.增加player接口
##### Version 6.33.1
> 1.批量接口增加参数日志
##### Version 6.33.0
> 1.增加maxAID的接口
##### Version 6.32.9
> 1.删除废弃的代码
##### Version 6.32.8
> 1.删除废弃的RPC server端代码
##### Version 6.32.7
> 1.删除冗余代码
##### Version 6.32.6
> 1.增加prom db
##### Version 6.32.5
> 1.内置prom
##### Version 6.32.4
> 1.使用内置prom
##### Version 6.32.3
> 1.修复缓存miss时少吐数据的问题
##### Version 6.32.1
> 1.Video3 RPC
##### Version 6.32.0
> 1.兼容客户端传多次点赞
##### Version 6.31.3
> 1.统一修改errgroup包路径
##### Version 6.31.2
> 1.attr的第九位改成 isPGC
##### Version 6.31.1
> 1.修改Views3返回值
##### Version 6.31.0
> 1.删除非internal的对外http接口
##### Version 6.30.0
> 1.Archive3结构体改为非指针
##### Version 6.29.1
> 1.补全RPC PB接口，video3
##### Version 6.29.0
> 1.补全RPC PB接口
##### Version 6.28.1
> 1.archive增加dynamic字段
##### Version 6.28.0
> 1.增加up主推荐视频的RPC接口
##### Version 6.27.0
> 1.删除pgc相关逻辑
##### Version 6.26.0
> 1.delete Movie2 AidByCid
##### Version 6.25.0
> 1.add Page3 pb rpc
##### Version 6.24.2
> 1.upArcs & upsArcs pb
##### Version 6.24.2
> 1.rpc删除likes2接口
##### Version 6.24.1
> 1.rpc增加likes3的pb接口
##### Version 6.24.0
> 1.rpc增加stat，stats的pb接口
##### Version 6.23.0
> 1.rpc 增加archive3,archives3的pb接口
> 2.rpc 删除废弃的videos2,videosByCids2,CidByEpIDs2等方法
> 3.pgc接口只吐电影信息
##### Version 6.22.8
> 1.rpc 增加view3的pb接口
##### Version 6.22.7
> 1.http video接口走PB
##### Version 6.22.6
> 1.http archive、archives、page全量开放，异步更新page缓存
##### Version 6.22.5
> 1.http/view接口全量pb缓存预热
##### Version 6.22.4
> 1.流量扩大得到aid%10<5走PB
> 2.分P的http接口也走pb
##### Version 6.22.3
> 1.aid%10<3走pb
##### Version 6.22.2
> 1.http stat/stats接口全量走pb
##### Version 6.22.1
> 1.pb的func/model/struct/service等全面改名为数字3结尾
##### Version 6.22.0
> 1.pb bugfix
##### Version 6.21.1
> 1.archive http 接口 aid%10=1的走pb
##### Version 6.21.0
> 1.增加limiter限流
##### Version 6.20.0
> 1.增加批量views接口，aids限制为20个
##### Version 6.19.0
> 1.直播限制50个
##### Version 6.18.0
> 1.cids接口不直接return
##### Version 6.17.0
> 1.videoshot接口试水pkg/errors
##### Version 6.16.0
> 1.增加全区7天内最新稿件
##### Version 6.15.0
> 1.redis errnil return
##### Version 6.14.0
> 1.likes相关数据落库
> 2.增加likes列表的RPC接口
##### Version 6.13.0
> 1.bilibili_archive库全都读写分离
##### Version 6.12.0
> 1.upspass score bugfix
##### Version 6.11.0
> 1.upsPass接口增加copyright
##### Version 6.10.0
> 1.添加获取单P信息的http接口(包含description字段)
> 2.修改原获取单P信息的service层逻辑
> 2.添加获取长简介的http和rpc接口
##### Version 6.9.0
> 1.升级go-common
##### Version 6.8.0
> 1.升级go-common
> 2.迁移model到项目中
##### Version 6.7.0
> 1.增加主站排行榜专用接口
##### Version 6.6.1
> 1.memcache json
##### Version 6.6.0
> 1.memcache gob
##### Version 6.5.0
> 1.增加点赞相关RPC接口
##### Version 6.4.0
> 1.升级go-common&go-business
> 2.videoshot rpc 增加aid参数
##### Version 6.3.0
> 1.videosho接口增加aid参数
##### Version 6.2.4
> 1.http context fix
##### Version 6.2.3
> 1.升级go-business
> 2.manager后台变更稿件归属mid时,变更相应缓存
##### Version 6.2.2
> 1.增加upspassed rpc方法
##### Version 6.2.1
> 1.rpc video2 nil fix
##### Version 6.2.0
> 1.所有稿件&视频走新archive_result数据库
> 2.升级go-common&go-business
##### Version 6.1.21
> 1.删除SetStatCache2接口
##### Version 6.1.20
> 1.修复可能导致panic的问题
##### Version 6.1.19
> 1.增加http的typelist接口
>>>>>>> develop
##### Version 6.1.18
> 1.修改ci配置
##### Version 6.1.17
> 1.增加account清楚缓存时的参数
##### Version 6.1.16
> 1.增加昵称&头像更新后的缓存清理逻辑
##### Version 6.1.15
> 1.增加无脑生成view&click缓存
##### Version 6.1.14
> 1.ci配置分支
##### Version 6.1.13
> 1.删掉zlimit相关残留代码
> 2.archive和archives接口返回archive_report_result中is_show等于1的result
##### Version 6.1.12
> 1.去掉老的dede
##### Version 6.1.11
> 1.分区表走新分区
> 2.增加RPC获取所有type的方法
> 3.升级go-common和go-business
##### Version 6.1.10
> 1.升级go-common和go-business
> 2.修改prom写法
##### Version 6.1.9
> 1.修复闭包缓存
##### Version 6.1.8
> 1.重发ci
##### Version 6.1.7
> 1.修复分类缓存
##### Version 6.1.6
> 1.修复videoshot nil 导致panic
##### Version 6.1.5
> 1.修复videoshot nil 导致panic
##### Version 6.1.4
> 1.page字段走自增形式
##### Version 6.1.3
> 1.增加auth
##### Version 6.1.2
> 1.增加prom
##### Version 6.1.1
> 1.批量大小改为60
##### Version 6.1.0
> 1.计数闭包问题修复
##### Version 6.0.16
> 1.rows close bug fix
##### Version 6.0.13
> 1.mc改成永不过期
##### Version 6.0.12
> 1.计数加aid在json
##### Version 6.0.11
> 1.增加cache出错不回写
> 2.增加prom回源统计
##### Version 6.0.10
> 1.修复点击计数panic
##### Version 6.0.9
> 1.修复prom参数个数
##### Version 6.0.8
> 1.增加prom包
##### Version 6.0.7
> 1.增加memcache随机过期时间
##### Version 6.0.6
> 1.增加upcount缓存逻辑
##### Version 6.0.5
> 1.修复chan未设置长度的bug
##### Version 6.0.4
> 1.修复view2和views2
##### Version 6.0.3
> 1.修复DB prepare配置
##### Version 6.0.2
> 1.批量没有默认返回空map
##### Version 6.0.1
> 1.去掉重复的view接口
##### Version 6.0.0
> 1.重构-删除无用方法(dede等)
> 2.重构-优化批量查询
> 3.重构-优化计数信息缓存
> 4.增加批量aids获取View信息
> 5.增加单aid获取view的http接口
> 6.增加SetStat rpc方法(mc)
##### Version 5.6.14
> 1.增加rpc接口,全量更新stat数值(redis)
##### Version 5.6.13
> 1.增加internal/view
##### Version 5.6.12
> 1.cache回写逻辑
##### Version 5.6.11
> 1.去除404的header
##### Version 5.6.10
> 1.click走mysql
##### Version 5.6.9
> 1.批量计数查不到不设置空值
##### Version 5.6.8
> 1.修改identity为verify
##### Version 5.6.7
> 1.修复stat panic
##### Version 5.6.6
> 1.修复rows.next()
##### Version 5.6.5
> 1.接入新配置中心
> 2.rpc接口参数校验
> 3.去除hbase
##### Version 5.6.4
> 1.升级go-common
##### Version 5.6.3
> 1.rpc接口支持缓存的修改
##### Version 5.6.2
> 1.日志错误修复
##### Version 5.6.1
> 1.archive/page走新表,修改sql
##### Version 5.6.0
> 1.archive/page走新表
##### Version 5.5.1
> 1.videos接口增加Ptitle
##### Version 5.5.0
> 1.RPC增加根据aids获取stat接口
##### Version 5.4.0
> 1.paas发布占用
##### Version 5.3.6
> 1.RPC增加一级分区最新视频与数量接口
> 2.RPC增加Upcount方法 获取用户投稿总数
> 3.内部http接口改名
> 4.升级go-common
##### Version 5.3.5
> 1.PGC只查status=开放的
##### Version 5.3.4
> 1.ArcsNoCheck2接口校验,aid为空则直接返回参数错误
##### Version 5.3.3
> 1.monitor挪到内部接口
##### Version 5.3.2
> 1.修复批量用户动态panic的bug
> 2.增加field数量
##### Version 5.3.1
> 1.统一monitor ping接口
> 2.修复批量用户动态panic的bug
> 3.增加field数量
> 4.分页接口增加兼容性处理
##### Version 5.3.0
> 1.修复redis cache
##### Version 5.2.8
> 1.增加根据aids获取seasonid接口 rpc
> 2.更改up过审稿件sql的排序字段
##### Version 5.2.7
> 1.up主过审稿件改为pubtime排序
##### Version 5.2.6
> 1.增加RPC方法,根据mids获取最新投稿
> 2.支持attribute参数,在列表中去除展示
> 3.升级go-common
##### Version 5.2.5
> 1.增加RPC方法,根据aids获取archive聚合信息
##### Version 5.2.4
> 1.增加RPC方法根据EpID获取cid
> 2.增加RPC方法根据CID获取video信息
##### Version 5.2.3
> 1.注释PGCproc方法
##### Version 5.2.2
> 1.增加RPC的分区信息接口
##### Version 5.2.1
> 1.注释pgc方法
##### Version 5.2.0
> 1.升级go-common新版本
> 2.fix view接口，多次查单个请求改为批量请求
> 3.conf支持优先从本地加载配置
##### Version 5.1.3
> 1.archives/nocheck接口新增返回返回archive_video和archive_video_audit表数据
> 2.去掉moment逻辑
##### Version 5.1.2
> 1.router加入rpcCloser
##### Version 5.1.1
> 1.忽略video计数错误
##### Version 5.1.0
> 1.升级配置中心
> 2.使用公用identify
> 3.使用统一参数开关
##### Version 5.0.0
> 1.net/rpc升级为golang/rpcx
##### Version 4.3.0
> 1.新增rpc获取稿件点击数量
> 2.新增rpc通过cid查aid
> 3.更新go-business
##### Version 4.2.5
> 1.分享计数增加databus双写
##### Version 4.2.4
> 1.新增videoshot接口供管理后台访问
##### Version 4.2.3
> 1.videoshot接口增加稿件状态校验
##### Version 4.2.2
> 1.依赖包升级
##### Version 4.2.1
> 1.修复db使用错误
##### Version 4.2.0
> 1.添加获取视频详情rpc接口
##### Version 4.1.4
> 1.fix len(attens) == 0 不能被除
##### Version 4.1.3
> 1.更新所有匿名rpc client为默认user
##### Version 4.1.2
> 1.修改syslog日志和上报
##### Version 4.1.1
> 1.更新go-business为1.3.1
##### Version 4.1.0
> 1.支持查询pgc信息
> 2.支持查询用户关注的up主的过审稿件
##### Version 4.0.0
> 1.go vendor支持
> 2.go-common/business换成go-business包
> 3.获取本机ip注册到zk
> 4.memcache批量获取支持多连接并发
> 5.新增rpc日志
##### Version 3.6.1
> 1.修复第一次分享的topic
##### Version 3.6.0
> 1.新增稿件page信息接口
##### Version 3.5.1
> 1.修复批量获取cache出错还加入cache问题
##### Version 3.5.0
> 1.获取稿件列表不检测权限
> 2.修复稿件分区变更后二级分区最新视频转移分区
##### Version 3.4.3
> 1.修复二级分区最新视频安装pubdate排序
##### Version 3.4.2
> 1.新增update稿件cache
##### Version 3.4.1
> 1.数组越界bug
##### Version 3.4.0
> 1.增加获取up主投稿列表接口
> 2.修复增加全量分区视频时变量没有重新初始化bug
> 3.优化缓存key使均匀分布
##### Version 3.3.1
> 1.修复最新视频bug：新增视频可见过滤条件：access、attrBitNoWeb、attrBitNoMobile
##### Version 3.3.0
> 1.增加分区的视频按投稿时间排序
> 2.新增查询过审记录接口
##### Version 3.2.3
> 1.修复回复的稿件置首bug
##### Version 3.2.2
> 1.修复Archive接口cache bug
##### Version 3.2.1
> 1.修复elk日志
##### Version 3.2.0
> 1.稿件添加字段reject_reason
> 2.修改share接口
> 3.新增set_tag接口
> 4.支持trace v2
##### Version 3.1.0
> 1.新增获取stat接口
> 2.新增获取多条stat接口
> 3.新增stat更新redis接口
> 4.修改稿件获取stat的方法
> 5.增加或修改ping方法
> 6.优化部分代码
##### Version 3.0.0
> 1.context使用官方接口
> 2.添加share计数
> 3.优化部分代码
##### Version 2.5.0
> 1.新增视频缩略图版本号
> 2.支持视频缩略图更新cid
> 3.添加up主视频动态接口
##### Version 2.4.0
> 1.添加获取用户最新评论稿件以及后台job
> 2.优化配置
> 3.添加服务发现
##### Version 2.3.0
> 1.添加获取videoshot接口
> 2.rpc调用bug
##### Version 2.2.0
> 1.优化
> 2.add elk
> 3.add trace id
> 4.add haiwai api
> 5.remove noused code
> 6.add mid recommend
> 7.fix some bug
##### Version 2.1.0
> 1.add tracer
##### Version 1.1.0
> 1.基于go-common重构
##### Version 1.0.0
> 1.初始化完成稿件基础查询功能

13
app/service/main/archive/CONTRIBUTORS.md Normal file
View File

@@ -0,0 +1,13 @@
# Owner
haoguanwei
peiyifei
liweijia
# Author
peiyifei
haoguanwei
yujia
# Reviewer
haoguanwei
peiyifei

17
app/service/main/archive/OWNERS Normal file
View File

@@ -0,0 +1,17 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- haoguanwei
- liweijia
- peiyifei
- yujia
labels:
- main
- service
- service/main/archive
options:
no_parent_owners: true
reviewers:
- haoguanwei
- peiyifei
- yujia

13
app/service/main/archive/README.md Normal file
View File

@@ -0,0 +1,13 @@
#### archive-service
##### 项目简介
> 1.提供对内稿件服务
##### 编译环境
> 请只用golang v1.8.x以上版本编译执行。
##### 依赖包
> 1.公共包go-common
##### 特别说明
> 1.model目录可能会被其他项目引用，请谨慎请改并通知各方。

79
app/service/main/archive/api/BUILD Normal file
View File

@@ -0,0 +1,79 @@
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
proto_library(
name = "api_proto",
srcs = ["api.proto"],
tags = ["automanaged"],
deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
)
go_proto_library(
name = "api_go_proto",
compilers = ["@io_bazel_rules_go//proto:gogofast_grpc"],
importpath = "go-common/app/service/main/archive/api",
proto = ":api_proto",
tags = ["automanaged"],
deps = [
"//library/time:go_default_library",
"@com_github_gogo_protobuf//gogoproto:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["client_test.go"],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//library/ecode:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"client.go",
"model.go",
],
embed = [":api_go_proto"],
importpath = "go-common/app/service/main/archive/api",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/net/rpc/warden:go_default_library",
"//library/time:go_default_library",
"@com_github_gogo_protobuf//gogoproto:go_default_library",
"@com_github_gogo_protobuf//proto:go_default_library",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_x_net//context:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/archive/api/gorpc:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

11710
app/service/main/archive/api/api.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

405
app/service/main/archive/api/api.proto Normal file
View File

@@ -0,0 +1,405 @@
syntax = "proto3";
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
// use {app_id}.{version} as package name
package archive.service.v1;
option go_package = "api";
// Archive grpc
service Archive {
// Types 获取所有分区列表
rpc Types(NoArgRequest) returns(TypesReply);
// 获取单个稿件信息
rpc Arc(ArcRequest) returns(ArcReply);
// 批量获取稿件信息
rpc Arcs(ArcsRequest) returns(ArcsReply);
// 获取单个稿件计数信息
rpc Stat(StatRequest) returns(StatReply);
// 批量获取稿件计数信息
rpc Stats(StatsRequest) returns(StatsReply);
// 获取单个稿件+分P信息
rpc View(ViewRequest) returns(ViewReply);
// 批量获取稿件+分P信息
rpc Views(ViewsRequest) returns(ViewsReply);
// 获取稿件在各平台的点击数
rpc Click(ClickRequest) returns(ClickReply);
// 获取稿件的所有分P信息
rpc Page(PageRequest) returns(PageReply);
// 获取Up主绑定的相关推荐 *几乎废弃
rpc Recommend(RecommendRequest) returns(RecommendReply);
// 获取aid&cid对应的视频信息
rpc Video(VideoRequest) returns(VideoReply);
// 获取当前最大的aid，各子业务防刷、防脏数据时使用
rpc MaxAid(NoArgRequest) returns(MaxAidReply);
// 获取稿件的长简介信息
rpc Description(DescriptionRequest) returns(DescriptionReply);
// 内部方法，管理稿件服务缓存，请勿使用-----------------------------------------------------------------------
rpc ArcFieldCache(ArcFieldCacheRequest) returns(NoReply);
rpc ArcCache(ArcCacheRequest) returns(NoReply);
rpc UpVideoCache(UpVideoCacheRequest) returns(NoReply);
rpc DelVideoCache(DelVideoCacheRequest) returns(NoReply);
rpc SetStat(SetStatRequest) returns(NoReply);
// 内部方法，管理稿件服务缓存，请勿使用-----------------------------------------------------------------------
}
// NoArgReq
message NoArgRequest{}
// NoReply
message NoReply{}
message DescriptionRequest {
int64 aid = 1;
}
message DescriptionReply {
string desc = 1;
}
message MaxAidReply {
int64 aid = 1;
}
message ArcRequest {
int64 aid = 1;
}
message ArcReply {
Arc arc = 1;
}
message ArcsRequest {
repeated int64 aids = 1;
}
message ArcsReply {
map<int64, Arc> arcs = 1;
}
message ViewRequest {
int64 aid = 1;
}
message StatRequest {
int64 aid = 1;
}
message StatReply {
Stat Stat = 1;
}
message StatsRequest {
repeated int64 aids = 1;
}
message StatsReply {
map<int64, Stat> Stats = 1;
}
message ViewReply {
Arc arc = 1 [(gogoproto.embed) = true, (gogoproto.jsontag) = ""];
repeated Page pages = 2;
}
message ViewsRequest {
repeated int64 aids = 1;
}
message ViewsReply {
map<int64, ViewReply> views = 1;
}
message TypesReply {
map<int32, Tp> types = 1;
}
message ClickRequest {
int64 aid = 1;
}
message ClickReply {
Click click = 1;
}
message PageRequest {
int64 aid = 1;
}
message PageReply {
repeated Page pages = 1;
}
message RecommendRequest {
int64 aid = 1;
}
message RecommendReply {
repeated Arc arcs = 1;
}
message VideoRequest {
int64 aid = 1;
int64 cid = 2;
}
message VideoReply {
Page page = 1;
}
message Tp {
// 分区id
int32 ID = 1 [(gogoproto.jsontag) = "id"];
// 该分区的父级id
int32 pid = 2 [(gogoproto.jsontag) = "pid"];
// 该分区对应的名称
string name = 3 [(gogoproto.jsontag) = "name"];
}
message AidVideos {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
// 分P信息
repeated Page Pages = 2 [(gogoproto.jsontag) = "pages"];
}
message Arc {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
// 稿件一共有多少分P
int64 Videos = 2 [(gogoproto.jsontag) = "videos"];
// 分区ID
int32 TypeID = 3 [(gogoproto.jsontag) = "tid"];
// 分区名
string TypeName = 4 [(gogoproto.jsontag) = "tname"];
// 是否转载 1=原创 2=转载 0=历史上可能遗留的脏数据
int32 Copyright = 5 [(gogoproto.jsontag) = "copyright"];
// 稿件封面图地址，绝对地址 如 http://i0.hdslb.com/bfs/xxx.jpg
string Pic = 6 [(gogoproto.jsontag) = "pic"];
// 稿件标题
string Title = 7 [(gogoproto.jsontag) = "title"];
// 稿件发布时间 对外展示的标准时间 时间戳格式
int64 PubDate = 8 [(gogoproto.jsontag) = "pubdate", (gogoproto.casttype) = "go-common/library/time.Time"];
// 用户提交稿件的时间 一般不对外展示 时间戳格式
int64 Ctime = 9 [(gogoproto.jsontag) = "ctime", (gogoproto.casttype) = "go-common/library/time.Time"];
// 稿件简介
string Desc = 10 [(gogoproto.jsontag) = "desc"];
// 稿件状态，>=0为前台用户可见状态，其他值详见 http://info.bilibili.co/pages/viewpage.action?pageId=3686597#id-%E7%A8%BF%E4%BB%B6%E5%AD%97%E6%AE%B5%E6%9E%9A%E4%B8%BE%E5%80%BC%E5%B1%9E%E6%80%A7%E8%AF%B4%E6%98%8E-%E7%A8%BF%E4%BB%B6%E7%8A%B6%E6%80%81state&access
int32 State = 11 [(gogoproto.jsontag) = "state"];
// 稿件访问属性 0=全员可见 10000=登录用户可见（极少)
int32 Access = 12 [(gogoproto.jsontag) = "access,omitempty"];
// 稿件属性 详见State字段的文档
int32 Attribute = 13 [(gogoproto.jsontag) = "attribute,omitempty"];
// 废弃字段
string Tag = 14 [(gogoproto.jsontag) = "-"];
// 废弃字段
repeated string Tags = 15 [(gogoproto.jsontag) = "tags,omitempty"];
// 稿件总时长（所有分P加起来的） 单位=秒
int64 Duration = 16 [(gogoproto.jsontag) = "duration"];
// 稿件绑定的活动id
int64 MissionID = 17 [(gogoproto.jsontag) = "mission_id,omitempty"];
// 稿件绑定的商单id
int64 OrderID = 18 [(gogoproto.jsontag) = "order_id,omitempty"];
// 稿件后台设置的强制跳转地址，如果该值不为空，则必须调转到它对应的页面上
string RedirectURL = 19 [(gogoproto.jsontag) = "redirect_url,omitempty"];
int64 Forward = 20 [(gogoproto.jsontag) = "forward,omitempty"];
// 见Rights字段说明
Rights Rights = 21 [(gogoproto.jsontag) = "rights", (gogoproto.nullable) = false];
// 稿件作者信息，详见Author字段说明
Author Author = 22 [(gogoproto.jsontag) = "owner", (gogoproto.nullable) = false];
// 稿件计数信息，详见Stat字段说明
Stat Stat = 23 [(gogoproto.jsontag) = "stat", (gogoproto.nullable) = false];
string ReportResult = 24 [(gogoproto.jsontag) = "report_result,omitempty"];
// 发布稿件时Up主设置的动态描述（仅在动态展示）
string Dynamic = 25 [(gogoproto.jsontag) = "dynamic"];
// 稿件第一P的cid，供秒开使用
int64 FirstCid = 26 [(gogoproto.jsontag) = "cid,omitempty"];
// 稿件第一P的分辨率，详见Dimession字段说明
Dimension Dimension = 27 [(gogoproto.jsontag) = "dimension,omitempty", (gogoproto.nullable) = false];
// 联合投稿信息，内部使用，详见StaffInfo说明
repeated StaffInfo StaffInfo = 28 [(gogoproto.jsontag) = "-"];
}
message StaffInfo {
// 联合投稿的成员Up主id
int64 Mid = 1 [(gogoproto.jsontag) = "mid"];
// 联合投稿的成员角色（如 声优、字幕）
string Title = 2 [(gogoproto.jsontag) = "title"];
}
// Dimension 视频分辨率
message Dimension {
// 宽 如 1920
int64 Width = 1 [(gogoproto.jsontag) = "width"];
// 高 如 1080
int64 Height = 2 [(gogoproto.jsontag) = "height"];
// 是否竖屏 0=否 1=是
int64 Rotate = 3 [(gogoproto.jsontag) = "rotate"];
}
// Rights 稿件各常用属性 0=否 1=是
message Rights {
// 老版是否付费
int32 Bp = 1 [(gogoproto.jsontag) = "bp"];
// 是否支持充电 （不可直接使用，网关层请求业务方后才有值）
int32 Elec = 2 [(gogoproto.jsontag) = "elec"];
// 是否下载（不可直接使用，网关层请求业务方后才有值）
int32 Download = 3 [(gogoproto.jsontag) = "download"];
// 是否电影
int32 Movie = 4 [(gogoproto.jsontag) = "movie"];
// 是否PGC付费
int32 Pay = 5 [(gogoproto.jsontag) = "pay"];
// 无用
int32 HD5 = 6 [(gogoproto.jsontag) = "hd5"];
// 是否允许转发
int32 NoReprint = 7 [(gogoproto.jsontag) = "no_reprint"];
// 是否可以自动播放
int32 Autoplay = 8 [(gogoproto.jsontag) = "autoplay"];
// 是否UGC付费
int32 UGCPay = 9 [(gogoproto.jsontag) = "ugc_pay"];
// 是否联合投稿
int32 IsCooperation = 10 [(gogoproto.jsontag) = "is_cooperation"];
}
// Author 稿件作者信息
message Author {
// Up主mid
int64 Mid = 1 [(gogoproto.jsontag) = "mid"];
// Up主名称
string Name = 2 [(gogoproto.jsontag) = "name"];
// Up主头像地址 绝对地址
string Face = 3 [(gogoproto.jsontag) = "face"];
}
// Stat 稿件的所有计数信息
message Stat {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
// 播放数
int32 View = 2 [(gogoproto.jsontag) = "view"];
// 弹幕数
int32 Danmaku = 3 [(gogoproto.jsontag) = "danmaku"];
// 评论数
int32 Reply = 4 [(gogoproto.jsontag) = "reply"];
// 收藏数
int32 Fav = 5 [(gogoproto.jsontag) = "favorite"];
// 投币数
int32 Coin = 6 [(gogoproto.jsontag) = "coin"];
// 分享数
int32 Share = 7 [(gogoproto.jsontag) = "share"];
// 当前排名
int32 NowRank = 8 [(gogoproto.jsontag) = "now_rank"];
// 历史最高排名
int32 HisRank = 9 [(gogoproto.jsontag) = "his_rank"];
// 点赞数
int32 Like = 10 [(gogoproto.jsontag) = "like"];
// 点踩数 已取消前台展示，现在均返回0
int32 DisLike = 11 [(gogoproto.jsontag) = "dislike"];
}
// Click 点击数详情，老版本播放器使用
message Click {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
// web端播放数
int32 Web = 2 [(gogoproto.jsontag) = "web"];
// iOS端播放数
int32 Ios = 3 [(gogoproto.jsontag) = "ios"];
// android端播放数
int32 Android = 4 [(gogoproto.jsontag) = "android"];
// h5端播放数
int32 H5 = 5 [(gogoproto.jsontag) = "h5"];
// 站外播放数
int32 Outter = 6 [(gogoproto.jsontag) = "outter"];
}
// Page 分P信息
message Page {
// 视频的cid
int64 Cid = 1 [(gogoproto.jsontag) = "cid"];
// 视频的序号顺序，从小到大
int32 Page = 2 [(gogoproto.jsontag) = "page"];
// 视频的来源，99%为vupload=B站自己的，少部分为腾讯（qq）、湖南tv(hunan)
string From = 3 [(gogoproto.jsontag) = "from"];
// 视频的标题
string Part = 4 [(gogoproto.jsontag) = "part"];
// 视频的时长 单位=秒
int64 Duration = 5 [(gogoproto.jsontag) = "duration"];
// 站外视频才有vid属性
string Vid = 6 [(gogoproto.jsontag) = "vid"];
// 视频的简介，多P视频才可填写简介
string Desc = 7 [(gogoproto.jsontag) = "description,omitempty"];
// 站外视频跳转地址
string WebLink = 8 [(gogoproto.jsontag) = "weblink"];
// 视频分辨率信息
Dimension Dimension = 9 [(gogoproto.jsontag) = "dimension", (gogoproto.nullable) = false];
}
// !!!Note: only response message defined, since we put pb into http response body
message VideoFileInfo {
uint64 filesize = 1 [(gogoproto.jsontag) = "filesize", (gogoproto.casttype) = "int64"];
uint64 timelength = 2 [(gogoproto.jsontag) = "timelength", (gogoproto.casttype) ="int64"];
string ahead = 3 [(gogoproto.jsontag) = "ahead"];
string vhead = 4 [(gogoproto.jsontag) = "vhead"];
}
message VideoFormatFileInfo {
repeated VideoFileInfo infos = 1 [(gogoproto.jsontag) = "infos"];
}
message DashItem {
uint32 id = 1 [(gogoproto.jsontag) = "id", (gogoproto.casttype) = "int64"];
string baseUrl = 2 [(gogoproto.jsontag) = "base_url"];
uint32 bandwidth = 3 [(gogoproto.jsontag) = "bandwidth", (gogoproto.casttype) = "int64"];
uint32 codecid = 4 [(gogoproto.jsontag) = "codecid", (gogoproto.casttype) = "int64"];
}
message ResponseDash {
repeated DashItem video = 1 [(gogoproto.jsontag) = "video"];
repeated DashItem audio = 2 [(gogoproto.jsontag) = "audio"];
}
message BvcVideoItem {
uint64 expire_time = 1 [(gogoproto.jsontag) = "expire_time", (gogoproto.casttype) = "int64"];
uint32 cid = 2 [(gogoproto.jsontag) = "cid", (gogoproto.casttype) ="int64"];
repeated uint32 support_quality = 3 [(gogoproto.jsontag) = "support_quality", (gogoproto.casttype) = "int"];
repeated string support_formats = 4 [(gogoproto.jsontag) = "support_formats"];
repeated string support_description = 5 [(gogoproto.jsontag) = "support_description"];
uint32 quality = 6 [(gogoproto.jsontag) = "quality", (gogoproto.casttype) = "int"];
string url = 7 [(gogoproto.jsontag) = "url"];
map<uint32, VideoFormatFileInfo> file_info = 8 [(gogoproto.jsontag) = "file_info"];
uint32 video_codecid = 9 [(gogoproto.jsontag) = "video_codecid"];
bool video_project = 10 [(gogoproto.jsontag) = "video_project"];
uint32 fnver = 11 [(gogoproto.jsontag) = "fnver", (gogoproto.casttype) = "int"];
uint32 fnval = 12 [(gogoproto.jsontag) = "fnval", (gogoproto.casttype) = "int"];
ResponseDash dash = 13 [(gogoproto.jsontag) = "dash"];
}
message BvcResponseMsg {
uint32 code = 1 [(gogoproto.jsontag) = "code", (gogoproto.casttype) = "int64"];
map<uint32, BvcVideoItem> data = 2 [(gogoproto.jsontag) = "data"];
}
// 内部方法 请勿使用--------------------------------------------------------------------------------------------
message ArcCacheRequest {
int64 aid = 1;
string tp = 2;
int64 OldMid = 3;
}
message ArcFieldCacheRequest {
int64 aid = 1;
int32 typeID = 2;
int32 oldTypeID = 3;
}
message UpVideoCacheRequest {
int64 aid = 1;
int64 cid = 2;
}
message DelVideoCacheRequest {
int64 aid = 1;
int64 cid = 2;
}
message SetStatRequest {
Stat stat = 1;
}
// 内部方法 请勿使用--------------------------------------------------------------------------------------------

25
app/service/main/archive/api/client.go Normal file
View File

@@ -0,0 +1,25 @@
package api
import (
"context"
"fmt"
"go-common/library/net/rpc/warden"
"google.golang.org/grpc"
)
// AppID .
const AppID = "archive.service"
// NewClient new grpc client
func NewClient(cfg *warden.ClientConfig, opts ...grpc.DialOption) (ArchiveClient, error) {
client := warden.NewClient(cfg, opts...)
cc, err := client.Dial(context.Background(), fmt.Sprintf("discovery://default/%s", AppID))
if err != nil {
return nil, err
}
return NewArchiveClient(cc), nil
}
//go:generate $GOPATH/src/go-common/app/tool/warden/protoc.sh

99
app/service/main/archive/api/client_test.go Normal file
View File

@@ -0,0 +1,99 @@
package api
import (
"context"
"testing"
"go-common/library/ecode"
"github.com/smartystreets/goconvey/convey"
)
var client ArchiveClient
func init() {
var err error
client, err = NewClient(nil)
if err != nil {
panic(err)
}
}
func TestTypes(t *testing.T) {
convey.Convey("Types", t, func(ctx convey.C) {
var c = context.Background()
ctx.Convey("When everything is correct", func(ctx convey.C) {
reply, err := client.Types(c, &NoArgRequest{})
ctx.So(err, convey.ShouldBeNil)
for k, v := range reply.Types {
ctx.Printf("key:%d id:%d name:%s pid:%d\n", k, v.ID, v.Name, v.Pid)
}
})
})
}
func TestArc(t *testing.T) {
convey.Convey("TestArc", t, func(ctx convey.C) {
var c = context.Background()
ctx.Convey("When everything is correct", func(ctx convey.C) {
reply, err := client.Arc(c, &ArcRequest{Aid: 10100696})
ctx.So(err, convey.ShouldBeNil)
ctx.Printf("%+v\n", reply.Arc)
})
ctx.Convey("When error", func(ctx convey.C) {
reply, err := client.Arc(context.TODO(), &ArcRequest{Aid: 99999999999})
ctx.So(err, convey.ShouldEqual, ecode.NothingFound)
ctx.So(reply, convey.ShouldBeNil)
})
})
}
func TestArcs(t *testing.T) {
convey.Convey("TestArcs", t, func(ctx convey.C) {
var c = context.Background()
ctx.Convey("When everything is correct", func(ctx convey.C) {
reply, err := client.Arcs(c, &ArcsRequest{Aids: []int64{10100696}})
ctx.So(err, convey.ShouldBeNil)
ctx.Printf("%+v\n", reply.Arcs)
})
ctx.Convey("When empty", func(ctx convey.C) {
reply, err := client.Arcs(c, &ArcsRequest{Aids: []int64{99999999999}})
// 批量接口 err=nil arcs=nil
ctx.So(err, convey.ShouldBeNil)
ctx.So(reply.Arcs, convey.ShouldBeNil)
})
})
}
func TestView(t *testing.T) {
convey.Convey("TestView", t, func(ctx convey.C) {
var c = context.Background()
ctx.Convey("When everything is correct", func(ctx convey.C) {
reply, err := client.View(c, &ViewRequest{Aid: 10100696})
ctx.So(err, convey.ShouldBeNil)
ctx.Printf("arc:%+v\n", reply.Arc)
ctx.Printf("pages:%+v\n", reply.Pages)
})
ctx.Convey("When empty", func(ctx convey.C) {
reply, err := client.View(c, &ViewRequest{Aid: 99999999999})
ctx.So(err, convey.ShouldNotBeNil)
ctx.So(reply, convey.ShouldBeNil)
})
})
}
func TestViews(t *testing.T) {
convey.Convey("TestViews", t, func(ctx convey.C) {
var c = context.Background()
ctx.Convey("When everything is correct", func(ctx convey.C) {
reply, err := client.Views(c, &ViewsRequest{Aids: []int64{10100696}})
ctx.So(err, convey.ShouldBeNil)
ctx.Printf("%+v\n", reply.Views)
})
ctx.Convey("When empty", func(ctx convey.C) {
arcs, err := client.Views(c, &ViewsRequest{Aids: []int64{99999999999}})
ctx.So(err, convey.ShouldBeNil)
ctx.So(arcs.Views, convey.ShouldBeNil)
})
})
}

50
app/service/main/archive/api/gorpc/BUILD Normal file
View File

@@ -0,0 +1,50 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["archive2_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/model/archive:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"archive1.go",
"archive2.go",
"archive3.go",
],
importpath = "go-common/app/service/main/archive/api/gorpc",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/model/archive:go_default_library",
"//library/net/rpc:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

25
app/service/main/archive/api/gorpc/archive1.go Normal file
View File

@@ -0,0 +1,25 @@
package gorpc
import (
"go-common/library/net/rpc"
)
const (
_appid = "archive.service"
)
var (
_noArg = &struct{}{}
)
// Service2 is archive rpc client.
type Service2 struct {
client *rpc.Client2
}
// New2 new a archive rpc client.
func New2(c *rpc.ClientConfig) (s *Service2) {
s = &Service2{}
s.client = rpc.NewDiscoveryCli(_appid, c)
return
}

96
app/service/main/archive/api/gorpc/archive2.go Normal file
View File

@@ -0,0 +1,96 @@
package gorpc
import (
"context"
"go-common/app/service/main/archive/api"
model "go-common/app/service/main/archive/model/archive"
)
const (
_types2 = "RPC.Types2"
_videoshot2 = "RPC.Videoshot2"
_upCount2 = "RPC.UpCount2"
_upsPassed2 = "RPC.UpsPassed2"
_upVideo2 = "RPC.UpVideo2"
_delVideo2 = "RPC.DelVideo2"
_description2 = "RPC.Description2"
_ranksTopCount2 = "RPC.RanksTopCount2"
_arcCache2 = "RPC.ArcCache2"
_arcFieldCache2 = "RPC.ArcFieldCache2"
_setStat2 = "RPC.SetStat2"
_setStatCache2 = "RPC.SetStatCache2"
)
// Types2 get all archive types
func (s *Service2) Types2(c context.Context) (res map[int16]*model.ArcType, err error) {
err = s.client.Call(c, _types2, _noArg, &res)
return
}
// Videoshot2 get videoshot.
func (s *Service2) Videoshot2(c context.Context, arg *model.ArgCid2) (res *model.Videoshot, err error) {
res = new(model.Videoshot)
err = s.client.Call(c, _videoshot2, arg, res)
return
}
// UpCount2 up count2
func (s *Service2) UpCount2(c context.Context, arg *model.ArgUpCount2) (count int, err error) {
err = s.client.Call(c, _upCount2, arg, &count)
return
}
// UpsPassed2 get UpsPassed aid and ptime
func (s *Service2) UpsPassed2(c context.Context, arg *model.ArgUpsArcs2) (res map[int64][]*model.AidPubTime, err error) {
err = s.client.Call(c, _upsPassed2, arg, &res)
return
}
// UpVideo2 update video cache by aid & cid
func (s *Service2) UpVideo2(c context.Context, arg *model.ArgVideo2) (err error) {
err = s.client.Call(c, _upVideo2, arg, _noArg)
return
}
// DelVideo2 delete video cache by aid & cid
func (s *Service2) DelVideo2(c context.Context, arg *model.ArgVideo2) (err error) {
err = s.client.Call(c, _delVideo2, arg, _noArg)
return
}
// Description2 add share.
func (s *Service2) Description2(c context.Context, arg *model.ArgAid) (des string, err error) {
err = s.client.Call(c, _description2, arg, &des)
return
}
// RanksTopCount2 get top region count.
func (s *Service2) RanksTopCount2(c context.Context, arg *model.ArgRankTopsCount2) (res map[int16]int, err error) {
err = s.client.Call(c, _ranksTopCount2, arg, &res)
return
}
// ArcCache2 add/update archive cache
func (s *Service2) ArcCache2(c context.Context, arg *model.ArgCache2) (err error) {
err = s.client.Call(c, _arcCache2, arg, _noArg)
return
}
// ArcFieldCache2 update archive field cache
func (s *Service2) ArcFieldCache2(c context.Context, arg *model.ArgFieldCache2) (err error) {
err = s.client.Call(c, _arcFieldCache2, arg, _noArg)
return
}
// SetStat2 set all stat info.
func (s *Service2) SetStat2(c context.Context, arg *api.Stat) (err error) {
err = s.client.Call(c, _setStat2, arg, _noArg)
return
}
// SetStatCache2 up stat.
func (s *Service2) SetStatCache2(c context.Context, arg *model.ArgStat2) (err error) {
err = s.client.Call(c, _setStatCache2, arg, _noArg)
return
}

252
app/service/main/archive/api/gorpc/archive2_test.go Normal file
View File

@@ -0,0 +1,252 @@
package gorpc
import (
"context"
"fmt"
"testing"
"time"
"go-common/app/service/main/archive/api"
"go-common/app/service/main/archive/model/archive"
)
func TestDynamic(t *testing.T) {
s := New2(nil)
time.Sleep(5 * time.Second)
testArchivesWithPlayer(t, s)
testMaxAID(t, s)
testVideo3(t, s)
testUpsArcs3(t, s)
testUpArcs3(t, s)
testRecommend3(t, s)
testArchive3(t, s)
testArchives3(t, s)
testAidPTime(t, s)
testTypes2(t, s)
testVideoshot2(t, s)
testUpCount2(t, s)
testView3(t, s)
testViews3(t, s)
testClick3(t, s)
testRankArcs3(t, s)
testRanksArcs3(t, s)
testRankTopArcs3(t, s)
testRankAllArcs3(t, s)
testPage3(t, s)
testRanksTopCount2(t, s)
testStat3(t, s)
testStats3(t, s)
testArcCache2(t, s)
testArcFieldCache2(t, s)
testSetStat2(t, s)
testUpVideo2(t, s)
testDelVideo2(t, s)
testDescription2(t, s)
}
func testViews3(t *testing.T, s *Service2) {
var vs, err = s.Views3(context.TODO(), &archive.ArgAids2{Aids: []int64{10097450, 10097454}})
if err != nil {
t.Log(err)
return
}
for _, v := range vs {
t.Log(v.Archive3)
}
}
func testClick3(t *testing.T, s *Service2) {
t.Log(s.Click3(context.TODO(), &archive.ArgAid2{Aid: 10097454}))
}
func testRankArcs3(t *testing.T, s *Service2) {
t.Log(s.RankArcs3(context.TODO(), &archive.ArgRank2{Rid: 20, Type: 0, Ps: 20, Pn: 1}))
}
func testRanksArcs3(t *testing.T, s *Service2) {
t.Log(s.RanksArcs3(context.TODO(), &archive.ArgRanks2{Rids: []int16{20, 24}, Type: 0, Ps: 20, Pn: 1}))
}
func testRankTopArcs3(t *testing.T, s *Service2) {
t.Log(s.RankTopArcs3(context.TODO(), &archive.ArgRankTop2{ReID: 1, Pn: 1, Ps: 10}))
}
func testRankAllArcs3(t *testing.T, s *Service2) {
am, err := s.RankAllArcs3(context.TODO(), &archive.ArgRankAll2{Pn: 1, Ps: 40})
if err != nil {
t.Log(err)
return
}
t.Logf("allRankTop(%d)", am.Count)
for _, arc := range am.Archives {
t.Logf("archive(%+v)", arc)
}
}
func testAidPTime(t *testing.T, s *Service2) {
res, err := s.UpsPassed2(context.TODO(), &archive.ArgUpsArcs2{Mids: []int64{15555180, 27515256}})
fmt.Println(len(res))
if err != nil {
t.Logf("s.UpsPassed2 error(%v)", err)
return
}
for mid, v := range res {
t.Logf("mid:%d", mid)
for _, aidAndPtime := range v {
t.Logf("mid:%d, aid:%d, ptime:%d, copyright:%d", mid, aidAndPtime.Aid, aidAndPtime.PubDate, aidAndPtime.Copyright)
}
}
}
func testArchivesWithPlayer(t *testing.T, s *Service2) {
arg := &archive.ArgPlayer{Aids: []int64{10097272, 1}, Qn: 16, Platform: "html5", RealIP: "222.73.196.18"}
res, err := s.ArchivesWithPlayer(context.TODO(), arg)
if err != nil {
t.Logf("s.ArchivesWithPlayer(%+v) error(%v)", arg, err)
return
}
for k, v := range res {
t.Logf("aid:%d, arc:%+v, player:%+v", k, v, v.PlayerInfo)
}
}
func testVideo3(t *testing.T, s *Service2) {
t.Log(s.Video3(context.TODO(), &archive.ArgVideo2{Aid: 10097760, Cid: 10107336}))
}
func testRecommend3(t *testing.T, s *Service2) {
for i := 0; i < 50; i++ {
time.Sleep(100 * time.Millisecond)
res, err := s.Recommend3(context.TODO(), &archive.ArgAid2{Aid: 4052562})
if err != nil {
t.Logf("s.Recommend3 error(%v)", err)
continue
}
t.Log(res)
}
}
func testArchive3(t *testing.T, s *Service2) {
res, err := s.Archive3(context.TODO(), &archive.ArgAid2{Aid: 5463609})
if err != nil {
t.Logf("s.Archive3 error(%v)", err)
return
}
t.Log(res)
}
func testArchives3(t *testing.T, s *Service2) {
res, err := s.Archives3(context.TODO(), &archive.ArgAids2{Aids: []int64{5463609, 5463608, 10097657}})
if err != nil {
t.Logf("s.Archives3 error(%v)", err)
return
}
for _, a := range res {
t.Log(a)
}
}
func testMaxAID(t *testing.T, s *Service2) {
res, err := s.MaxAID(context.TODO())
if err != nil {
t.Error(err)
return
}
t.Logf("maxAID is %d", res)
}
func testTypes2(t *testing.T, s *Service2) {
res, err := s.Types2(context.TODO())
if err != nil {
t.Error(err)
return
}
for _, tp := range res {
t.Log(tp)
}
}
func testVideoshot2(t *testing.T, s *Service2) {
res, err := s.Videoshot2(context.TODO(), &archive.ArgCid2{Cid: 10108203})
t.Log(res, err)
}
func testUpArcs3(t *testing.T, s *Service2) {
res, err := s.UpArcs3(context.TODO(), &archive.ArgUpArcs2{Mid: 27515615, Pn: 1, Ps: 20})
if err != nil {
t.Error(err)
return
}
for _, a := range res {
t.Log(a)
}
}
func testUpCount2(t *testing.T, s *Service2) {
count, err := s.UpCount2(context.TODO(), &archive.ArgUpCount2{Mid: 27515615})
if err != nil {
t.Error(err)
return
}
t.Log(count)
}
func testUpsArcs3(t *testing.T, s *Service2) {
res, err := s.UpsArcs3(context.TODO(), &archive.ArgUpsArcs2{Mids: []int64{27515615, 1684013}})
if err != nil {
t.Error(err)
return
}
t.Log(res)
}
func testView3(t *testing.T, s *Service2) {
t.Log(s.View3(context.TODO(), &archive.ArgAid2{Aid: 10098067}))
}
func testPage3(t *testing.T, s *Service2) {
t.Log(s.Page3(context.TODO(), &archive.ArgAid2{Aid: 10097454}))
}
func testRanksTopCount2(t *testing.T, s *Service2) {
t.Log(s.RanksTopCount2(context.TODO(), &archive.ArgRankTopsCount2{ReIDs: []int16{1, 2}}))
}
func testStat3(t *testing.T, s *Service2) {
res, err := s.Stat3(context.TODO(), &archive.ArgAid2{Aid: 10989901})
if err != nil {
t.Errorf("testStats3 error(%v)", err)
return
}
t.Log(res, 1)
}
func testStats3(t *testing.T, s *Service2) {
res, err := s.Stats3(context.TODO(), &archive.ArgAids2{Aids: []int64{10989901, 10097453}})
if err != nil {
t.Logf("testStats3 error(%v)", err)
return
}
for aid, r := range res {
t.Logf("%d:%+v\n", aid, r)
}
}
func testArcCache2(t *testing.T, s *Service2) {
t.Log(s.ArcCache2(context.TODO(), &archive.ArgCache2{Aid: 10097454, Tp: archive.CacheUpdate}))
}
func testArcFieldCache2(t *testing.T, s *Service2) {
t.Log(s.ArcFieldCache2(context.TODO(), &archive.ArgFieldCache2{Aid: 10097454, TypeID: 20, OldTypeID: 10}))
}
func testSetStat2(t *testing.T, s *Service2) {
t.Log(s.SetStat2(context.TODO(), &api.Stat{Aid: 10097454, View: 0, Danmaku: 0, Reply: 10, Fav: 10, Coin: 10, Share: 10, HisRank: 10, NowRank: 10}))
}
func testUpVideo2(t *testing.T, s *Service2) {
t.Log(s.UpVideo2(context.TODO(), &archive.ArgVideo2{Aid: 10097760, Cid: 10107336}))
}
func testDelVideo2(t *testing.T, s *Service2) {
t.Log(s.DelVideo2(context.TODO(), &archive.ArgVideo2{Aid: 10097760, Cid: 10107336}))
}
func testDescription2(t *testing.T, s *Service2) {
t.Log(s.Description2(context.TODO(), &archive.ArgAid{Aid: 10097454}))
}

141
app/service/main/archive/api/gorpc/archive3.go Normal file
View File

@@ -0,0 +1,141 @@
package gorpc
import (
"context"
"go-common/app/service/main/archive/api"
model "go-common/app/service/main/archive/model/archive"
)
const (
_archive3 = "RPC.Archive3"
_archives3 = "RPC.Archives3"
_view3 = "RPC.View3"
_views3 = "RPC.Views3"
_stat3 = "RPC.Stat3"
_stats3 = "RPC.Stats3"
_click3 = "RPC.Click3"
_upArcs3 = "RPC.UpArcs3"
_upsArcs3 = "RPC.UpsArcs3"
_page3 = "RPC.Page3"
_recommend3 = "RPC.Recommend3"
_rankArcs3 = "RPC.RankArcs3"
_ranksArcs3 = "RPC.RanksArcs3"
_rankTopArcs3 = "RPC.RankTopArcs3"
_rankAllArcs3 = "RPC.RankAllArcs3"
_video3 = "RPC.Video3"
_archivesWithPlayer = "RPC.ArchivesWithPlayer"
_maxAid = "RPC.MaxAID"
)
// MaxAID get max aid
func (s *Service2) MaxAID(c context.Context) (id int64, err error) {
err = s.client.Call(c, _maxAid, _noArg, &id)
return
}
// Archive3 Get receive aid, then init archive info.
func (s *Service2) Archive3(c context.Context, arg *model.ArgAid2) (res *api.Arc, err error) {
res = new(api.Arc)
err = s.client.Call(c, _archive3, arg, res)
return
}
// Archives3 receive aids, then init archives info.
func (s *Service2) Archives3(c context.Context, arg *model.ArgAids2) (res map[int64]*api.Arc, err error) {
err = s.client.Call(c, _archives3, arg, &res)
return
}
// View3 get archive info and view pages.
func (s *Service2) View3(c context.Context, arg *model.ArgAid2) (res *model.View3, err error) {
res = new(model.View3)
err = s.client.Call(c, _view3, arg, res)
return
}
// Views3 get archives info and view pages.
func (s *Service2) Views3(c context.Context, arg *model.ArgAids2) (res map[int64]*model.View3, err error) {
err = s.client.Call(c, _views3, arg, &res)
return
}
// Stat3 get archive stat
func (s *Service2) Stat3(c context.Context, arg *model.ArgAid2) (res *api.Stat, err error) {
err = s.client.Call(c, _stat3, arg, &res)
return
}
// ArchivesWithPlayer archives witch player
func (s *Service2) ArchivesWithPlayer(c context.Context, arg *model.ArgPlayer) (res map[int64]*model.ArchiveWithPlayer, err error) {
err = s.client.Call(c, _archivesWithPlayer, arg, &res)
return
}
// Stats3 get archive stat
func (s *Service2) Stats3(c context.Context, arg *model.ArgAids2) (res map[int64]*api.Stat, err error) {
err = s.client.Call(c, _stats3, arg, &res)
return
}
// Click3 get archive click
func (s *Service2) Click3(c context.Context, arg *model.ArgAid2) (res *api.Click, err error) {
err = s.client.Call(c, _click3, arg, &res)
return
}
// UpsArcs3 get archives of upper.
func (s *Service2) UpsArcs3(c context.Context, arg *model.ArgUpsArcs2) (res map[int64][]*api.Arc, err error) {
err = s.client.Call(c, _upsArcs3, arg, &res)
return
}
// UpArcs3 get archives of upper.
func (s *Service2) UpArcs3(c context.Context, arg *model.ArgUpArcs2) (res []*api.Arc, err error) {
err = s.client.Call(c, _upArcs3, arg, &res)
return
}
// Page3 get videos by aid
func (s *Service2) Page3(c context.Context, arg *model.ArgAid2) (res []*api.Page, err error) {
err = s.client.Call(c, _page3, arg, &res)
return
}
// Recommend3 from archive_recommend by aid
func (s *Service2) Recommend3(c context.Context, arg *model.ArgAid2) (res []*api.Arc, err error) {
err = s.client.Call(c, _recommend3, arg, &res)
return
}
// RankArcs3 get rank archives by type.
func (s *Service2) RankArcs3(c context.Context, arg *model.ArgRank2) (res *model.RankArchives3, err error) {
res = new(model.RankArchives3)
err = s.client.Call(c, _rankArcs3, arg, res)
return
}
// RanksArcs3 get rank archives by types.
func (s *Service2) RanksArcs3(c context.Context, arg *model.ArgRanks2) (res map[int16]*model.RankArchives3, err error) {
err = s.client.Call(c, _ranksArcs3, arg, &res)
return
}
// RankTopArcs3 get top region archives by reid
func (s *Service2) RankTopArcs3(c context.Context, arg *model.ArgRankTop2) (res []*api.Arc, err error) {
err = s.client.Call(c, _rankTopArcs3, arg, &res)
return
}
// RankAllArcs3 get left 7 days all archives
func (s *Service2) RankAllArcs3(c context.Context, arg *model.ArgRankAll2) (res *model.RankArchives3, err error) {
err = s.client.Call(c, _rankAllArcs3, arg, &res)
return
}
// Video3 get video by aid & cid.
func (s *Service2) Video3(c context.Context, arg *model.ArgVideo2) (res *api.Page, err error) {
res = new(api.Page)
err = s.client.Call(c, _video3, arg, res)
return
}

3
app/service/main/archive/api/http/api.md Normal file
View File

@@ -0,0 +1,3 @@
# API文档
http://info.bilibili.co/display/coding/archive-service

235
app/service/main/archive/api/model.go Normal file
View File

@@ -0,0 +1,235 @@
package api
import (
"hash/crc32"
"strconv"
"strings"
"go-common/library/time"
)
// 各属性地址见 http://syncsvn.bilibili.co/platform/doc/blob/master/archive/field/state.md
// all const
const (
// open state
StateOpen = 0
StateOrange = 1
AccessMember = int32(10000)
// forbid state
StateForbidWait = -1
StateForbidRecicle = -2
StateForbidPolice = -3
StateForbidLock = -4
StateForbidFixed = -6
StateForbidLater = -7
StateForbidAdminDelay = -10
StateForbidXcodeFail = -16
StateForbidSubmit = -30
StateForbidUserDelay = -40
StateForbidUpDelete = -100
// copyright
CopyrightUnknow = int8(0)
CopyrightOriginal = int8(1)
CopyrightCopy = int8(2)
// attribute yes and no
AttrYes = int32(1)
AttrNo = int32(0)
// attribute bit
AttrBitNoRank = uint(0)
AttrBitNoDynamic = uint(1)
AttrBitNoWeb = uint(2)
AttrBitNoMobile = uint(3)
// AttrBitNoSearch = uint(4)
AttrBitOverseaLock = uint(5)
AttrBitNoRecommend = uint(6)
AttrBitNoReprint = uint(7)
AttrBitHasHD5 = uint(8)
AttrBitIsPGC = uint(9)
AttrBitAllowBp = uint(10)
AttrBitIsBangumi = uint(11)
AttrBitIsPorder = uint(12)
AttrBitLimitArea = uint(13)
AttrBitAllowTag = uint(14)
// AttrBitIsFromArcApi = uint(15)
AttrBitJumpUrl = uint(16)
AttrBitIsMovie = uint(17)
AttrBitBadgepay = uint(18)
AttrBitUGCPay = uint(22)
AttrBitHasBGM = uint(23)
AttrBitIsCooperation = uint(24)
AttrBitHasViewpoint = uint(25)
AttrBitHasArgument = uint(26)
)
var (
_emptyTags = []string{}
)
// IsNormal is
func (a *Arc) IsNormal() bool {
return a.State >= StateOpen || a.State == StateForbidFixed
}
// RegionArc RegionArc
type RegionArc struct {
Aid int64
Attribute int32
Copyright int8
PubDate time.Time
}
// AllowShow AllowShow
func (ra *RegionArc) AllowShow() bool {
return ra.attrVal(AttrBitNoWeb) == AttrNo && ra.attrVal(AttrBitNoMobile) == AttrNo
}
func (ra *RegionArc) attrVal(bit uint) int32 {
return (ra.Attribute >> bit) & int32(1)
}
// AttrVal get attr val by bit.
func (a *Arc) AttrVal(bit uint) int32 {
return (a.Attribute >> bit) & int32(1)
}
// FillDimension is
func (a *Arc) FillDimension(d string) {
if d == "" || d == "0,0,0" {
return
}
ds := strings.Split(d, ",")
if len(ds) != 3 {
return
}
var (
width, height, rotate int64
err error
)
if width, err = strconv.ParseInt(ds[0], 10, 64); err != nil {
return
}
if height, err = strconv.ParseInt(ds[1], 10, 64); err != nil {
return
}
if rotate, err = strconv.ParseInt(ds[2], 10, 64); err != nil {
return
}
a.Dimension = Dimension{
Width: width,
Height: height,
Rotate: rotate,
}
}
// FillDimension is
func (v *Page) FillDimension(d string) {
if d == "" || d == "0,0,0" {
return
}
ds := strings.Split(d, ",")
if len(ds) != 3 {
return
}
var (
width, height, rotate int64
err error
)
if width, err = strconv.ParseInt(ds[0], 10, 64); err != nil {
return
}
if height, err = strconv.ParseInt(ds[1], 10, 64); err != nil {
return
}
if rotate, err = strconv.ParseInt(ds[2], 10, 64); err != nil {
return
}
v.Dimension = Dimension{
Width: width,
Height: height,
Rotate: rotate,
}
}
// Fill file archive some field.
func (a *Arc) Fill() {
a.Tags = _emptyTags
a.Pic = coverURL(a.Pic)
a.Rights.Bp = a.AttrVal(AttrBitAllowBp)
a.Rights.Movie = a.AttrVal(AttrBitIsMovie)
a.Rights.Pay = a.AttrVal(AttrBitBadgepay)
a.Rights.HD5 = a.AttrVal(AttrBitHasHD5)
a.Rights.NoReprint = a.AttrVal(AttrBitNoReprint)
a.Rights.UGCPay = a.AttrVal(AttrBitUGCPay)
a.Rights.IsCooperation = a.AttrVal(AttrBitIsCooperation)
if a.FirstCid == 0 ||
a.Access == AccessMember ||
a.AttrVal(AttrBitIsPGC) == AttrYes ||
a.AttrVal(AttrBitAllowBp) == AttrYes ||
a.AttrVal(AttrBitBadgepay) == AttrYes ||
a.AttrVal(AttrBitOverseaLock) == AttrYes ||
a.AttrVal(AttrBitUGCPay) == AttrYes ||
a.AttrVal(AttrBitLimitArea) == AttrYes {
return
}
a.Rights.Autoplay = 1
}
// coverURL convert cover url to full url.
func coverURL(uri string) (cover string) {
if uri == "" {
cover = "http://static.hdslb.com/images/transparent.gif"
return
}
cover = uri
if strings.Index(uri, "http://") == 0 {
return
}
if len(uri) >= 10 && uri[:10] == "/templets/" {
return
}
if strings.HasPrefix(uri, "group1") {
cover = "http://i0.hdslb.com/" + uri
return
}
if pos := strings.Index(uri, "/uploads/"); pos != -1 && (pos == 0 || pos == 3) {
cover = uri[pos+8:]
}
cover = strings.Replace(cover, "{IMG}", "", -1)
cover = "http://i" + strconv.FormatInt(int64(crc32.ChecksumIEEE([]byte(cover)))%3, 10) + ".hdslb.com" + cover
return
}
// FillStat file stat, check access.
func (a *Arc) FillStat() {
if a.Access > 0 {
a.Stat.View = 0
}
}
// PlayerInfo player info
type PlayerInfo struct {
Cid int64 `json:"cid"`
ExpireTime int64 `json:"expire_time,omitempty"`
FileInfo map[int][]*PlayerFileInfo `json:"file_info"`
SupportQuality []int `json:"support_quality"`
SupportFormats []string `json:"support_formats"`
SupportDescription []string `json:"support_description"`
Quality int `json:"quality"`
URL string `json:"url,omitempty"`
VideoCodecid uint32 `json:"video_codecid"`
VideoProject bool `json:"video_project"`
Fnver int `json:"fnver"`
Fnval int `json:"fnval"`
Dash *ResponseDash `json:"dash,omitempty"`
}
// PlayerFileInfo is
type PlayerFileInfo struct {
TimeLength int64 `json:"timelength"`
FileSize int64 `json:"filesize"`
Ahead string `json:"ahead,omitempty"`
Vhead string `json:"vhead,omitempty"`
URL string `json:"url,omitempty"`
Order int64 `json:"order,omitempty"`
}

48
app/service/main/archive/cmd/BUILD Normal file
View File

@@ -0,0 +1,48 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = [
"archive-service-example.toml",
"archive-service-test.toml",
],
importpath = "go-common/app/service/main/archive/cmd",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/conf:go_default_library",
"//app/service/main/archive/server/gorpc:go_default_library",
"//app/service/main/archive/server/grpc:go_default_library",
"//app/service/main/archive/server/http:go_default_library",
"//app/service/main/archive/service:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

261
app/service/main/archive/cmd/archive-service-example.toml Normal file
View File

@@ -0,0 +1,261 @@
# This is a TOML document. Boom.
version = "3.0.0"
user = "nobody"
pid = "/tmp/archive-service.pid"
dir = "./"
perf = "127.0.0.1:6080"
checkFile = "/data/www/archive.html"
family = "archive-service"
videoshotURI = "http://i3.hdslb.com/bfs/videoshot/"
videoshotKey = "f555a0f529152dc8c6b1ae60c2af4164"
tick = "5m"
[xlog]
dir = "/data/log/archive-service/"
[xlog.elk]
project = "archive-service"
addr = "172.18.20.17:8520"
chanSize = 1024
[ecode]
service = "archive-service"
[ecode.mysql]
name = "[archive]tcp@172.16.0.148:3306"
dsn = "test:test@tcp(172.16.0.148:3306)/bilibili_apm?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 5
[statsd]
project = "archive-service"
addr = "172.18.20.15:8200"
chanSize = 10240
[app]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
[multiHTTP]
[multiHTTP.inner]
addrs = ["0.0.0.0:6081"]
maxListen = 1000
[multiHTTP.local]
addrs = ["0.0.0.0:6082"]
maxListen = 100
[rpcServer2]
[[rpcServer2.servers]]
proto = "tcp"
addr = "127.0.0.1:6089"
weight = 10
[rpcServer2.zookeeper]
root = "/microservice/archive-service/"
addrs = ["127.0.0.1:2181"]
timeout = "1s"
[db]
[db.arc]
name = "[archive]tcp@172.16.0.148:3306"
dsn = "test:test@tcp(172.16.0.148:3306)/bilibili_archive?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 5
[db.dede]
name = "[dede]tcp@172.16.0.5:3306"
dsn = "bilibili:5Fq2M4FbPZK4fhtE@tcp(172.16.0.5:3306)/bilibili?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 5
[identify]
whiteAccessKey = ""
whiteMid = 0
[identify.app]
key = "6a29f8ed87407c11"
secret = "d3c5a85f5b895a03735b5d20a273bc57"
[identify.memcache]
name = "go-business/identify"
proto = "tcp"
addr = "172.16.33.54:11211"
active = 5
idle = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
[identify.host]
auth = "http://passport.bilibili.com"
secret = "http://open.bilibili.com"
[identify.authHTTPClient]
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[identify.secretHTTPClient]
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[memcache]
[memcache.archive]
name = "archive-service"
proto = "tcp"
addr = "172.16.0.148:11211"
active = 100
idle = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
expire = "10s"
[memcache.video]
name = "archive-service"
proto = "tcp"
addr = "172.16.0.148:11211"
active = 100
idle = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
expire = "10s"
[redis]
[redis.videoshot]
name = "archive-service/videoshot"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[redis.archive]
name = "archive-service/archive"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "8h"
[redis.region]
name = "archive-service/region"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
[redis.moment]
name = "archive-service/moment"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[redis.relation]
name = "archive-service/relation"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
[redis.share]
name = "archive-service/share"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[redis.stat]
name = "archive-service/stat"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[hbase]
master = ""
meta = ""
dialTimeout = "1s"
readTimeout = "200ms"
readsTimeout = "1s"
writeTimeout = "200ms"
writesTimeout = "200ms"
[hbase.zookeeper]
root = ""
addrs = ["172.16.33.45:2181"]
timeout = "30s"
[accountRPC]
pullInterval = "10s"
[accountRPC.client]
proto = "tcp"
addr = "127.0.0.1:6079"
timeout = "1s"
timer = 1000
[[accountRPC.backup]]
proto = "tcp"
addr = "127.0.0.1:6079"
timeout = "1s"
timer = 1000
[accountRPC.zookeeper]
root = "/microservice/account-service/"
addrs = ["127.0.0.1:2181"]
timeout = "1s"
[tracer]
proto = "udp"
addr = "172.16.33.46:5140"
tag = "platform/archive-service"
[kafkaProducer]
statTopic = "stat"
firstShareTopic = "share_first"
brokers = ["172.16.0.148:9092"]
sync = false
[kafkaProducer.zookeeper]
root = "/kafka"
addrs = ["172.16.0.148:2181"]
timeout = "200ms"
[databus]
key = "0PtNTguCX35XCtPpjUGC"
secret= "0PtNTguCX35XCtPpjUGD"
group= "Stat-UGC-P"
topic= "Stat-T"
action="pub"
[databus.redis]
name = "archive-service/stat"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"

343
app/service/main/archive/cmd/archive-service-test.toml Normal file
View File

@@ -0,0 +1,343 @@
tick = "5m"
playerQn = [15, 16, 32, 48, 64, 80]
playerVipQn = [74,112,116]
PlayerAPI = "http://172.18.33.143:13500/playurl/batch"
PGCPlayerAPI = "http://api.bilibili.co/pgc/internal/dynamic/aid/list"
PlayerSwitch = true
[videoshot]
uri = "http://i3.hdslb.com/bfs/videoshot/"
key = "f555a0f529152dc8c6b1ae60c2af4164"
[xlog]
dir = "/data/log/archive-service/"
[statsd]
project = "archive-service"
addr = "172.18.20.15:8200"
chanSize = 10240
[app]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
[bm]
[bm.inner]
addr = "0.0.0.0:6081"
timeout = "1s"
[bm.local]
addr = "0.0.0.0:6082"
timeout = "1s"
[rpcServer2]
discoverOff = false
token = "123456"
[[rpcServer2.servers]]
proto = "tcp"
addr = "0.0.0.0:6089"
weight = 10
group = "test"
[rpcServer2.zookeeper]
root = "/microservice/archive-service/"
addrs = ["172.18.33.172:2181"]
timeout = "30s"
[db]
[db.arc]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_archive?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 1
idleTimeout ="4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.arc.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.arcRead]
addr = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_archive?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 5
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.arcRead.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.arcResult]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_result?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 1
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.arcResult.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.stat]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_stat?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 1
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.stat.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.click]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_click?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 1
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.click.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.like]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_like?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 10
idle = 5
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.like.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[ecode]
domain = "172.16.33.248:6401"
all = "1h"
diff = "5m"
[ecode.clientconfig]
key = "test"
secret = "e6c4c252dc7e3d8a90805eecd7c73396"
dial = "2000ms"
timeout = "2s"
keepAlive = "10s"
timer = 128
[ecode.clientconfig.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[ecode.app]
key = "test"
secret = "e6c4c252dc7e3d8a90805eecd7c73396"
[identify]
whiteAccessKey = ""
whiteMid = 0
[identify.app]
key = "6a29f8ed87407c11"
secret = "d3c5a85f5b895a03735b5d20a273bc57"
[identify.host]
auth = "http://passport.bilibili.com"
secret = "http://open.bilibili.com"
[identify.httpClient]
key = "6a29f8ed87407c11"
secret = "d3c5a85f5b895a03735b5d20a273bc57"
dial = "30ms"
timeout = "100ms"
keepAlive = "60s"
[identify.httpClient.breaker]
window = "10s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[identify.httpClient.url]
"http://passport.bilibili.co/intranet/auth/tokenInfo" = {timeout = "100ms"}
"http://passport.bilibili.co/intranet/auth/cookieInfo" = {timeout = "100ms"}
"http://open.bilibili.co/api/getsecret" = {timeout = "500ms"}
[memcache]
[memcache.archive]
name = "archive-service"
proto = "tcp"
addr = "172.18.33.61:11211"
active = 50
idle = 10
dialTimeout = "30ms"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
archiveExpire = "10h"
videoExpire = "10h"
[redis]
[redis.archive]
name = "archive-service/archive"
proto = "tcp"
addr = "172.16.33.54:6379"
idle = 50
active = 100
dialTimeout = "100ms"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
expire = "8h"
[redis.relation]
name = "archive-service/relation"
proto = "tcp"
addr = "172.16.33.54:6379"
idle = 10
active = 10
dialTimeout = "5s"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
[redis.like]
name = "archive-service/like"
proto = "tcp"
addr = "172.16.33.54:6379"
idle = 10
active = 10
dialTimeout = "30ms"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
[kafkaProducer]
firstShareTopic = "share_first"
brokers = ["172.16.33.54:9092"]
sync = false
[kafkaProducer.zookeeper]
root = "/kafka"
addrs = ["172.18.33.172:2181"]
timeout = "30s"
[databus]
key = "0PtNTguCX35XCtPpjUGC"
secret= "0PtNTguCX35XCtPpjUGD"
group= "Stat-UGC-P"
topic= "Stat-T"
action="pub"
name = "archive-service/stat"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[CacheDatabus]
key = "8e27ab7e39270b59"
secret= "477df6a068d7332a163f95abbad2079c"
group= "ArchiveUserCache-MainAppSvr-P"
topic= "ArchiveUserCache-T"
action="pub"
name = "archive-service/stat"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 5
active = 50
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[StatDatabus]
key = "4c76cbb7a985ac90"
secret= "8c22a196bf00d623b69cba4e61b1f7dc"
group= "StatShare-MainArchive-P"
topic= "StatShare-T"
action="pub"
offset="old"
buffer=1024
name = "archive-service/StatShare"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 5
active = 50
dialTimeout = "80ms"
readTimeout = "80ms"
writeTimeout = "80ms"
idleTimeout = "80s"
[shareDatabus]
key = "8e27ab7e39270b59"
secret= "e5d94158a1118c14e1449481023edacf"
group= "AccountExp-MainAppSvr-P"
topic= "AccountExp-T"
action="pub"
name = "archive-service/share"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[limiter]
domain = "api.bilibili.co"
family = "archive-service"
all = "2h"
diff = "2s"
[limiter.clientconfig]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
dial = "1s"
timeout = "2s"
keepAlive = "10s"
timer = 128
[limiter.clientconfig.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[limiter.app]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
[playerClient]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
dial = "50ms"
timeout = "500ms"
keepAlive = "60s"
[playerClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100

63
app/service/main/archive/cmd/main.go Normal file
View File

@@ -0,0 +1,63 @@
package main
import (
"context"
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/service/main/archive/conf"
rpc "go-common/app/service/main/archive/server/gorpc"
"go-common/app/service/main/archive/server/grpc"
"go-common/app/service/main/archive/server/http"
"go-common/app/service/main/archive/service"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
// init ecode
ecode.Init(nil)
// init log
log.Init(conf.Conf.Xlog)
trace.Init(conf.Conf.Tracer)
defer trace.Close()
defer log.Close()
log.Info("archive-service start")
// service init
svr := service.New(conf.Conf)
// statsd init
rpcSvr := rpc.New(conf.Conf, svr)
grpcSvr, err := grpc.New(nil, svr)
if err != nil {
panic(err)
}
http.Init(conf.Conf, svr)
// init signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("archive-service get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
rpcSvr.Close()
grpcSvr.Shutdown(context.TODO())
time.Sleep(time.Second * 2)
log.Info("archive-service exit")
return
case syscall.SIGHUP:
// TODO reload
default:
return
}
}
}

42
app/service/main/archive/conf/BUILD Normal file
View File

@@ -0,0 +1,42 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/service/main/archive/conf",
tags = ["automanaged"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/conf:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/rpc:go_default_library",
"//library/net/rpc/warden:go_default_library",
"//library/net/trace:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

159
app/service/main/archive/conf/conf.go Normal file
View File

@@ -0,0 +1,159 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/conf"
"go-common/library/database/sql"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/rpc"
"go-common/library/net/rpc/warden"
"go-common/library/net/trace"
"go-common/library/queue/databus"
"go-common/library/time"
"github.com/BurntSushi/toml"
)
// config
var (
confPath string
Conf = &Config{}
client *conf.Client
)
// Config struct
type Config struct {
// base
Tick time.Duration
Videoshot *Videoshot
// xlog
Xlog *log.Config
// tracer
Tracer *trace.Config
// http
BM *BM
// http client
PlayerClient *bm.ClientConfig
// switch get player
PlayerSwitch bool
PlayerNum int64
// player qn config
PlayerQn []int
PlayerVipQn []int
BnjList []int64
// PlayerAPI path
PlayerAPI string
PGCPlayerAPI string
RPCServer *rpc.ServerConfig
// db
DB *DB
// ecode
Ecode *ecode.Config
// rpc client
AccountRPC *rpc.ClientConfig
// grpc client
AccClient *warden.ClientConfig
// mc
Memcache *Memcache
// redis
Redis *Redis
// databus
Databus *databus.Config
StatDatabus *databus.Config
ShareDatabus *databus.Config
CacheDatabus *databus.Config
}
// BM http
type BM struct {
Inner *bm.ServerConfig
Local *bm.ServerConfig
}
// Videoshot videoshot uri and key
type Videoshot struct {
URI string
Key string
}
// DB db config
type DB struct {
// archive db
Arc *sql.Config
ArcRead *sql.Config
ArcResult *sql.Config
Stat *sql.Config
Click *sql.Config
}
// Memcache memcache config
type Memcache struct {
Archive *struct {
*memcache.Config
ArchiveExpire time.Duration
VideoExpire time.Duration
StatExpire time.Duration
}
}
// Redis redis config
type Redis struct {
Archive *struct {
*redis.Config
}
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init config.
func Init() (err error) {
if confPath != "" {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
err = remote()
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
client.Watch("archive-service.toml")
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf = &Config{}
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

18
app/service/main/archive/dao/BUILD Normal file
View File

@@ -0,0 +1,18 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/archive/dao/archive:all-srcs",
"//app/service/main/archive/dao/share:all-srcs",
"//app/service/main/archive/dao/videoshot:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

95
app/service/main/archive/dao/archive/BUILD Normal file
View File

@@ -0,0 +1,95 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"addit_test.go",
"archive_mc_pb_test.go",
"archive_test.go",
"dao_test.go",
"player_test.go",
"region_rds_test.go",
"region_test.go",
"report_result_test.go",
"stat_mc_pb_test.go",
"stat_pb_test.go",
"stat_test.go",
"type_test.go",
"upper_rds_test.go",
"upper_test.go",
"video_mc_pb_test.go",
"video_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/conf:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"addit.go",
"archive.go",
"archive_mc_pb.go",
"dao.go",
"player.go",
"region.go",
"region_rds.go",
"report_result.go",
"stat.go",
"stat_mc_pb.go",
"stat_pb.go",
"type.go",
"upper.go",
"upper_rds.go",
"video.go",
"video_mc_pb.go",
],
importpath = "go-common/app/service/main/archive/dao/archive",
tags = ["automanaged"],
deps = [
"//app/job/main/archive/model/databus:go_default_library",
"//app/service/main/account/api:go_default_library",
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/conf:go_default_library",
"//app/service/main/archive/model/archive:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/queue/databus:go_default_library",
"//library/stat/prom:go_default_library",
"//library/sync/errgroup:go_default_library",
"//library/time:go_default_library",
"//library/xstr:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

30
app/service/main/archive/dao/archive/addit.go Normal file
View File

@@ -0,0 +1,30 @@
package archive
import (
"context"
"go-common/app/service/main/archive/model/archive"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_additSQL = "SELECT id,aid,source,redirect_url,mission_id,up_from,order_id,description FROM archive_addit WHERE aid=?"
)
// Addit get archive addit.
func (d *Dao) Addit(c context.Context, aid int64) (addit *archive.Addit, err error) {
d.infoProm.Incr("Addit")
row := d.additStmt.QueryRow(c, aid)
addit = &archive.Addit{}
if err = row.Scan(&addit.ID, &addit.Aid, &addit.Source, &addit.RedirectURL, &addit.MissionID, &addit.UpFrom, &addit.OrderID, &addit.Description); err != nil {
if err == sql.ErrNoRows {
addit = nil
err = nil
} else {
log.Error("row.Scan error(%v)", err)
}
}
return
}

21
app/service/main/archive/dao/archive/addit_test.go Normal file
View File

@@ -0,0 +1,21 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveAddit(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
)
convey.Convey("Addit", t, func(ctx convey.C) {
_, err := d.Addit(c, aid)
ctx.Convey("Then err should be nil.addit should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

127
app/service/main/archive/dao/archive/archive.go Normal file
View File

@@ -0,0 +1,127 @@
package archive
import (
"context"
"database/sql"
"fmt"
"go-common/app/service/main/archive/api"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_maxAIDSQL = "SELECT max(aid) FROM archive"
_arcSQL = "SELECT aid,mid,typeid,videos,copyright,title,cover,content,duration,attribute,state,access,pubtime,ctime,mission_id,order_id,redirect_url,forward,dynamic,cid,dimensions FROM archive WHERE aid=?"
_arcsSQL = "SELECT aid,mid,typeid,videos,copyright,title,cover,content,duration,attribute,state,access,pubtime,ctime,mission_id,order_id,redirect_url,forward,dynamic,cid,dimensions FROM archive WHERE aid IN (%s)"
_arcStaffSQL = "SELECT mid,title FROM archive_staff WHERE aid=?"
_arcsStaffSQL = "SELECT aid,mid,title FROM archive_staff WHERE aid IN(%s)"
)
// MaxAID get max aid
func (d *Dao) MaxAID(c context.Context) (id int64, err error) {
row := d.resultDB.QueryRow(c, _maxAIDSQL)
if err = row.Scan(&id); err != nil {
log.Error("row.Scan error(%v)", err)
return
}
return
}
// archivePB get a archive by aid.
func (d *Dao) archive3(c context.Context, aid int64) (a *api.Arc, err error) {
d.infoProm.Incr("archive3")
row := d.resultDB.QueryRow(c, _arcSQL, aid)
a = &api.Arc{}
var dimension string
if err = row.Scan(&a.Aid, &(a.Author.Mid), &a.TypeID, &a.Videos, &a.Copyright, &a.Title, &a.Pic, &a.Desc, &a.Duration,
&a.Attribute, &a.State, &a.Access, &a.PubDate, &a.Ctime, &a.MissionID, &a.OrderID, &a.RedirectURL, &a.Forward, &a.Dynamic, &a.FirstCid, &dimension); err != nil {
if err == sql.ErrNoRows {
a = nil
err = nil
} else {
log.Error("row.Scan error(%v)", err)
d.errProm.Incr("result_db")
}
return
}
a.FillDimension(dimension)
return
}
// archives3 multi get archives by avids.
func (d *Dao) archives3(c context.Context, aids []int64) (res map[int64]*api.Arc, err error) {
d.infoProm.Incr("archives3")
query := fmt.Sprintf(_arcsSQL, xstr.JoinInts(aids))
rows, err := d.resultDB.Query(c, query)
if err != nil {
log.Error("db.Query(%s) error(%v)", query, err)
d.errProm.Incr("result_db")
return
}
defer rows.Close()
res = make(map[int64]*api.Arc, len(aids))
for rows.Next() {
a := &api.Arc{}
var dimension string
if err = rows.Scan(&a.Aid, &(a.Author.Mid), &a.TypeID, &a.Videos, &a.Copyright, &a.Title, &a.Pic, &a.Desc, &a.Duration,
&a.Attribute, &a.State, &a.Access, &a.PubDate, &a.Ctime, &a.MissionID, &a.OrderID, &a.RedirectURL, &a.Forward, &a.Dynamic, &a.FirstCid, &dimension); err != nil {
log.Error("rows.Scan error(%v)", err)
d.errProm.Incr("result_db")
return
}
a.FillDimension(dimension)
res[a.Aid] = a
}
err = rows.Err()
return
}
// staff get archives staff by avid.
func (d *Dao) staff(c context.Context, aid int64) (res []*api.StaffInfo, err error) {
d.infoProm.Incr("archive_staff")
rows, err := d.resultDB.Query(c, _arcStaffSQL, aid)
if err != nil {
log.Error("d.resultDB.Query(%d) error(%v)", aid, err)
d.errProm.Incr("result_db")
return
}
defer rows.Close()
for rows.Next() {
as := &api.StaffInfo{}
if err = rows.Scan(&as.Mid, &as.Title); err != nil {
log.Error("rows.Scan error(%v)", err)
d.errProm.Incr("result_db")
return
}
res = append(res, as)
}
err = rows.Err()
return
}
// staffs get archives staff by avids.
func (d *Dao) staffs(c context.Context, aids []int64) (res map[int64][]*api.StaffInfo, err error) {
d.infoProm.Incr("archives_staff")
query := fmt.Sprintf(_arcsStaffSQL, xstr.JoinInts(aids))
rows, err := d.resultDB.Query(c, query)
if err != nil {
log.Error("d.resultDB.Query(%s) error(%v)", query, err)
d.errProm.Incr("result_db")
return
}
defer rows.Close()
res = make(map[int64][]*api.StaffInfo)
for rows.Next() {
as := &api.StaffInfo{}
var aid int64
if err = rows.Scan(&aid, &as.Mid, &as.Title); err != nil {
log.Error("rows.Scan error(%v)", err)
d.errProm.Incr("result_db")
return
}
res[aid] = append(res[aid], as)
}
err = rows.Err()
return
}

161
app/service/main/archive/dao/archive/archive_mc_pb.go Normal file
View File

@@ -0,0 +1,161 @@
package archive
import (
"context"
"strconv"
"sync"
"go-common/app/service/main/archive/api"
"go-common/library/cache/memcache"
"go-common/library/log"
"go-common/library/sync/errgroup"
)
const (
_prefixArc3 = "a3p_"
_prefixDesc = "desc_"
)
func descKey(aid int64) string {
return _prefixDesc + strconv.FormatInt(aid, 10)
}
func arcPBKey(aid int64) string {
return _prefixArc3 + strconv.FormatInt(aid, 10)
}
// archive3Cache get a archive info from cache.
func (d *Dao) archive3Cache(c context.Context, aid int64) (a *api.Arc, err error) {
var (
key = arcPBKey(aid)
rp *memcache.Item
conn = d.mc.Get(c)
)
defer conn.Close()
if rp, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
d.errProm.Incr("archive_mc")
log.Error("memcache.Get(%s) error(%v)", key, err)
}
return
}
a = &api.Arc{}
if err = conn.Scan(rp, a); err != nil {
log.Error("conn.Scan(%s) error(%v)", rp.Value, err)
}
return
}
// archive3Caches multi get archives, return map[aid]*Archive and missed aids.
func (d *Dao) archive3Caches(c context.Context, aids []int64) (cached map[int64]*api.Arc, err error) {
var (
keys = make([]string, 0)
keyMap = make(map[int64]struct{}, len(aids))
eg = errgroup.Group{}
mutex = sync.Mutex{}
)
cached = make(map[int64]*api.Arc)
for _, aid := range aids {
if _, ok := keyMap[aid]; ok {
continue
}
keyMap[aid] = struct{}{}
keys = append(keys, arcPBKey(aid))
if len(keys) == 50 {
egks := keys
eg.Go(func() (err error) {
as, _ := d._archiveCaches(c, egks)
mutex.Lock()
for _, a := range as {
cached[a.Aid] = a
}
mutex.Unlock()
return
})
keys = make([]string, 0)
}
}
if len(keys) > 0 {
eg.Go(func() (err error) {
as, _ := d._archiveCaches(c, keys)
mutex.Lock()
for _, a := range as {
cached[a.Aid] = a
}
mutex.Unlock()
return
})
}
eg.Wait()
return
}
// _archiveCaches is
func (d *Dao) _archiveCaches(c context.Context, keys []string) (as map[int64]*api.Arc, err error) {
conn := d.mc.Get(c)
defer conn.Close()
rs, err := conn.GetMulti(keys)
if err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
log.Error("conn.GetMulti(%v) error(%v)", keys, err)
return
}
as = make(map[int64]*api.Arc)
for _, r := range rs {
a := &api.Arc{}
if err = conn.Scan(r, a); err != nil {
log.Error("conn.Scan error(%v)", err)
continue
}
as[a.Aid] = a
}
return
}
// addArchivePBCache set archive into cache.
func (d *Dao) addArchive3Cache(c context.Context, a *api.Arc) (err error) {
key := arcPBKey(a.Aid)
conn := d.mc.Get(c)
if err = conn.Set(&memcache.Item{Key: key, Object: a, Flags: memcache.FlagProtobuf, Expiration: 0}); err != nil {
log.Error("memcache.Set(%s) error(%v)", key, err)
d.errProm.Incr("archive_mc")
}
conn.Close()
return
}
func (d *Dao) descCache(c context.Context, aid int64) (desc string, err error) {
key := descKey(aid)
conn := d.mc.Get(c)
defer conn.Close()
var item *memcache.Item
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
log.Error("conn.Get(%s) error(%v)", key, err)
return
}
if err = conn.Scan(item, &desc); err != nil {
log.Error("conn.Scan error(%v)", err)
return
}
return
}
func (d *Dao) addDescCache(c context.Context, aid int64, desc string) (err error) {
key := descKey(aid)
conn := d.mc.Get(c)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Value: []byte(desc), Flags: memcache.FlagRAW, Expiration: 0}); err != nil {
log.Error("conn.Set(%s, %s) error(%v)", key, desc, err)
return
}
return
}

98
app/service/main/archive/dao/archive/archive_mc_pb_test.go Normal file
View File

@@ -0,0 +1,98 @@
package archive
import (
"context"
"go-common/app/service/main/archive/api"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchivedescKey(t *testing.T) {
var (
aid = int64(1)
)
convey.Convey("descKey", t, func(ctx convey.C) {
p1 := descKey(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivearcPBKey(t *testing.T) {
var (
aid = int64(1)
)
convey.Convey("arcPBKey", t, func(ctx convey.C) {
p1 := arcPBKey(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivearchive3Cache(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
)
convey.Convey("archive3Cache", t, func(ctx convey.C) {
d.archive3Cache(c, aid)
})
}
func TestArchivearchive3Caches(t *testing.T) {
var (
c = context.TODO()
aids = []int64{1, 2}
)
convey.Convey("archive3Caches", t, func(ctx convey.C) {
cached, err := d.archive3Caches(c, aids)
ctx.Convey("Then err should be nil.cached should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(cached, convey.ShouldNotBeNil)
})
})
}
func TestArchiveaddArchive3Cache(t *testing.T) {
var (
c = context.TODO()
a = &api.Arc{}
)
convey.Convey("addArchive3Cache", t, func(ctx convey.C) {
err := d.addArchive3Cache(c, a)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivedescCache(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
)
convey.Convey("descCache", t, func(ctx convey.C) {
desc, err := d.descCache(c, aid)
ctx.Convey("Then err should be nil.desc should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(desc, convey.ShouldNotBeNil)
})
})
}
func TestArchiveaddDescCache(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
desc = ""
)
convey.Convey("addDescCache", t, func(ctx convey.C) {
err := d.addDescCache(c, aid, desc)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

48
app/service/main/archive/dao/archive/archive_test.go Normal file
View File

@@ -0,0 +1,48 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveMaxAID(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("MaxAID", t, func(ctx convey.C) {
id, err := d.MaxAID(c)
ctx.Convey("Then err should be nil.id should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(id, convey.ShouldNotBeNil)
})
})
}
func TestArchivearchive3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("archive3", t, func(ctx convey.C) {
_, err := d.archive3(c, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivearchives3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{10097272}
)
convey.Convey("archives3", t, func(ctx convey.C) {
res, err := d.archives3(c, aids)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}

655
app/service/main/archive/dao/archive/dao.go Normal file
View File

@@ -0,0 +1,655 @@
package archive
import (
"context"
"fmt"
"runtime"
"strconv"
"time"
ajbmdl "go-common/app/job/main/archive/model/databus"
accapi "go-common/app/service/main/account/api"
"go-common/app/service/main/archive/api"
"go-common/app/service/main/archive/conf"
"go-common/app/service/main/archive/model/archive"
arcmdl "go-common/app/service/main/archive/model/archive"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/database/sql"
"go-common/library/ecode"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/queue/databus"
"go-common/library/stat/prom"
"go-common/library/sync/errgroup"
)
const (
_multiInterval = 200
)
var (
_emptyPages3 = []*api.Page{}
)
// Dao is archive dao.
type Dao struct {
c *conf.Config
// db
db *sql.DB
arcReadDB *sql.DB
resultDB *sql.DB
statDB *sql.DB
clickDB *sql.DB
// acc rpc
acc accapi.AccountClient
// memcache
mc *memcache.Pool
// redis
upRds *redis.Pool
upExpire int32
// region
rgRds *redis.Pool
// archive stmt
rgnArcsStmt *sql.Stmt
upCntStmt *sql.Stmt
upPasStmt *sql.Stmt
reportResultStmt *sql.Stmt
additStmt *sql.Stmt
// video stmt
vdosStmt *sql.Stmt
// dede stmt
tpsStmt *sql.Stmt
// type cache
tNamem map[int16]string
// report_result cache
aidResult map[int64]string
// cache chan
cacheCh chan func()
hitProm *prom.Prom
missProm *prom.Prom
errProm *prom.Prom
infoProm *prom.Prom
// player http client
playerClient *bm.Client
// player qn map
playerQn map[int]struct{}
playerVipQn map[int]struct{}
cacheDatabus *databus.Databus
}
// New new a Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
c: c,
// db
db: sql.NewMySQL(c.DB.Arc),
arcReadDB: sql.NewMySQL(c.DB.ArcRead),
resultDB: sql.NewMySQL(c.DB.ArcResult),
statDB: sql.NewMySQL(c.DB.Stat),
clickDB: sql.NewMySQL(c.DB.Click),
// memcache
mc: memcache.NewPool(c.Memcache.Archive.Config),
// redis
upRds: redis.NewPool(c.Redis.Archive.Config),
upExpire: 480 * 60 * 60,
rgRds: redis.NewPool(c.Redis.Archive.Config),
// cache chan
cacheCh: make(chan func(), 1024),
hitProm: prom.CacheHit,
missProm: prom.CacheMiss,
errProm: prom.BusinessErrCount,
infoProm: prom.BusinessInfoCount,
playerClient: bm.NewClient(c.PlayerClient),
playerQn: make(map[int]struct{}),
playerVipQn: make(map[int]struct{}),
cacheDatabus: databus.New(c.CacheDatabus),
}
var err error
if d.acc, err = accapi.NewClient(c.AccClient); err != nil {
panic(fmt.Sprintf("account GRPC error(%v)!!!!!!!!!!!!!!!!!!!!!!", err))
}
d.rgnArcsStmt = d.resultDB.Prepared(_rgnArcsSQL)
d.upCntStmt = d.resultDB.Prepared(_upCntSQL)
d.upPasStmt = d.resultDB.Prepared(_upPasSQL)
d.tpsStmt = d.arcReadDB.Prepared(_tpsSQL)
d.additStmt = d.arcReadDB.Prepared(_additSQL)
// video stmt
d.vdosStmt = d.resultDB.Prepared(_vdosSQL)
// archive_report_resutl
d.reportResultStmt = d.arcReadDB.Prepared(_reportResultSQL)
// type cache
for _, pn := range d.c.PlayerQn {
d.playerQn[pn] = struct{}{}
}
for _, pvn := range d.c.PlayerVipQn {
d.playerVipQn[pvn] = struct{}{}
}
d.loadTypes()
d.loadReportResult()
for i := 0; i < runtime.NumCPU(); i++ {
go d.cacheproc()
}
go d.loadproc()
return
}
func (d *Dao) addCache(f func()) {
select {
case d.cacheCh <- f:
default:
log.Warn("d.cacheCh is full")
}
}
// Archive3 get archive by aid.
func (d *Dao) Archive3(c context.Context, aid int64) (a *api.Arc, err error) {
var cached = true
if a, err = d.archive3Cache(c, aid); err != nil {
log.Error("d.archivePBCache(%d) error(%v)", aid, err)
err = nil // NOTE ignore error use db
cached = false
}
if a != nil {
if st, _ := d.Stat3(c, aid); st != nil {
a.Stat = *st
a.FillStat()
}
a.ReportResult = d.aidResult[aid]
return
}
if a, err = d.archive3(c, aid); err != nil {
log.Error("d.archive(%d) error(%v)", aid, err)
return
}
if a == nil {
err = ecode.NothingFound
return
}
if st, _ := d.Stat3(c, aid); st != nil {
a.Stat = *st
a.FillStat()
}
d.fillArchive3(c, a, cached)
a.ReportResult = d.aidResult[aid]
return
}
// Archives3 get archives by aids.
func (d *Dao) Archives3(c context.Context, aids []int64) (am map[int64]*api.Arc, err error) {
var eg errgroup.Group
eg.Go(func() (err error) {
var (
missed []int64
missm map[int64]*api.Arc
cached = true
)
if am, err = d.archive3Caches(c, aids); err != nil {
log.Error("%+v", err)
am = make(map[int64]*api.Arc, len(aids))
err = nil // NOTE: ignore error
cached = false
}
for _, aid := range aids {
if _, ok := am[aid]; !ok {
missed = append(missed, aid)
}
}
if len(missed) == 0 {
return
}
if missm, err = d.archives3(c, missed); err != nil {
log.Error("d.archives(%v) error(%v)", missed, err)
return
}
if len(missm) == 0 {
log.Warn("archives miss(%+v)", missed)
return
}
d.fillArchives(c, missm, cached)
for _, a := range missm {
am[a.Aid] = a
}
return
})
var stm map[int64]*api.Stat
eg.Go(func() (err error) {
var (
missed []int64
missm map[int64]*api.Stat
cached = true
)
if stm, missed, err = d.statCaches3(c, aids); err != nil {
log.Error("d.statCaches(%d) error(%v)", aids, err)
missed = aids
stm = make(map[int64]*api.Stat, len(aids))
err = nil // NOTE: ignore error
cached = false
}
d.hitProm.Add("stat3", int64(len(stm)))
if stm != nil && len(missed) == 0 {
return
}
if missm, err = d.stats3(c, missed); err != nil {
log.Error("d.stats(%v) error(%v)", missed, err)
err = nil // NOTE: ignore error
}
for aid, st := range missm {
stm[aid] = st
if cached {
var cst = &api.Stat{}
*cst = *st
d.addCache(func() {
d.addStatCache3(context.TODO(), cst)
})
}
}
d.missProm.Add("stat3", int64(len(missm)))
return
})
if err = eg.Wait(); err != nil {
log.Error("eg.Wait(%v) error(%v)", aids, err)
return
}
for aid, a := range am {
if st, ok := stm[aid]; ok {
a.Stat = *st
a.FillStat()
}
a.ReportResult = d.aidResult[aid]
}
return
}
// Videos3 get archive videos by aid.
func (d *Dao) Videos3(c context.Context, aid int64) (ps []*api.Page, err error) {
var cached = true
if ps, err = d.pageCache3(c, aid); err != nil {
log.Error("d.pageCache3(%d) error(%v)", aid, err)
err = nil // NOTE ignore error use db
cached = false
}
if ps != nil {
d.hitProm.Add("videos3", 1)
return
}
if ps, err = d.videos3(c, aid); err != nil {
log.Error("d.videos(%d) error(%v)", aid, err)
return
}
if len(ps) == 0 {
log.Warn("archive(%d) have not passed video", aid)
ps = _emptyPages3
}
d.missProm.Add("videos3", 1)
if cached {
d.addCache(func() {
d.addPageCache3(context.TODO(), aid, ps)
})
}
return
}
// VideosByAids3 get videos by aids
func (d *Dao) VideosByAids3(c context.Context, aids []int64) (vs map[int64][]*api.Page, err error) {
var (
missed []int64
cached = true
)
if vs, missed, err = d.pagesCache3(c, aids); err != nil {
log.Error("d.pagesCache(%v) error(%v)", aids, err)
missed = aids
err = nil
cached = false
}
d.hitProm.Add("videos3", int64(len(vs)))
if len(missed) == 0 && vs != nil {
return
}
var missVs = make(map[int64][]*api.Page, len(missed))
if missVs, err = d.videosByAids3(c, missed); err != nil {
log.Error("d.videosByAids3(%v) error(%v)", missed, err)
return
}
d.missProm.Add("videos3", int64(len(missVs)))
for aid, v := range missVs {
vs[aid] = v
if cached {
var (
caid = aid
cv = v
)
d.addCache(func() {
d.addPageCache3(context.TODO(), caid, cv)
})
}
}
return
}
// Video3 get video by aid & cid.
func (d *Dao) Video3(c context.Context, aid, cid int64) (p *api.Page, err error) {
var cached = true
if p, err = d.videoCache3(c, aid, cid); err != nil {
log.Error("d.videoCache3(%d, %d) error(%v)", aid, cid, err)
err = nil // NOTE ignore error use db
cached = false
}
if p != nil {
d.hitProm.Add("video3", 1)
return
}
if p, err = d.video3(c, aid, cid); err != nil {
log.Error("d.video3(%d) error(%v)", aid, cid, err)
return
}
if p == nil {
log.Warn("archive(%d) cid(%d) no passed video", aid, cid)
err = ecode.NothingFound
cached = false
}
d.missProm.Add("video3", 1)
if cached {
d.addCache(func() {
d.addVideoCache3(context.TODO(), aid, cid, p)
})
}
return
}
// Description get Description from by aid || aid+cid.
func (d *Dao) Description(c context.Context, aid int64) (desc string, err error) {
var (
addit *archive.Addit
a *api.Arc
)
desc, _ = d.descCache(c, aid)
if desc != "" {
return
}
if addit, err = d.Addit(c, aid); err != nil {
log.Error("d.Addit(%d) error(%v)", aid, err)
err = nil
}
if addit != nil && addit.Description != "" {
desc = addit.Description
d.addCache(func() {
d.addDescCache(context.TODO(), aid, desc)
})
return
}
if a, err = d.archive3(c, aid); err != nil {
log.Error("d.archive(%d) error(%v)", aid, err)
return
}
if a != nil && a.Desc != "" {
desc = a.Desc
d.addCache(func() {
d.addDescCache(context.TODO(), aid, desc)
})
}
return
}
// UpVideo3 update video by aid & cid.
func (d *Dao) UpVideo3(c context.Context, aid, cid int64) (err error) {
var p *api.Page
if p, err = d.video3(c, aid, cid); err != nil {
log.Error("d.video2(%d) error(%v)", aid, cid, err)
return
}
if p == nil {
err = ecode.NothingFound
return
}
d.addCache(func() {
d.addVideoCache3(context.TODO(), aid, cid, p)
})
return
}
// UpperCache is
func (d *Dao) UpperCache(c context.Context, mid int64, action string, oldname string, oldface string) {
var (
aids []int64
err error
infoReply *accapi.InfoReply
)
if action == "updateByAdmin" {
if infoReply, err = d.acc.Info3(c, &accapi.MidReq{Mid: mid}); err != nil || infoReply == nil {
log.Error("d.acc.Info3(%d) error(%v)", mid, err)
return
}
if infoReply.Info.Face == oldface && infoReply.Info.Name == oldname {
log.Info("account updateByAdmin no need update")
return
}
}
if aids, _, _, err = d.UpperPassed(c, mid); err != nil {
log.Error("d.UpperPassed(%d) error(%v)", mid, err)
return
}
for _, aid := range aids {
d.UpArchiveCache(context.TODO(), aid)
}
}
// UpArchiveCache update archive cache by aid.
func (d *Dao) UpArchiveCache(c context.Context, aid int64) (err error) {
var (
a *api.Arc
addit *archive.Addit
desc string
)
if a, err = d.archive3(c, aid); a != nil && err == nil {
d.fillArchive3(c, a, true)
desc = a.Desc
}
if addit, err = d.Addit(c, aid); err == nil && addit != nil {
if addit.Description != "" {
desc = addit.Description
}
}
d.addCache(func() {
d.addDescCache(context.TODO(), aid, desc)
})
// pages3
var ps3 []*api.Page
if ps3, err = d.videos3(c, aid); err != nil {
log.Error("d.videos3(%d) error(%v)", aid, err)
return
}
d.addCache(func() {
d.addPageCache3(context.TODO(), aid, ps3)
})
return
}
// Ping ping success.
func (d *Dao) Ping(c context.Context) (err error) {
if err = d.db.Ping(c); err != nil {
log.Error("d.db error(%v)", err)
return
}
// mc
mconn := d.mc.Get(c)
defer mconn.Close()
if err = mconn.Set(&memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}); err != nil {
log.Error("mc.Store error(%v)", err)
return
}
// upper redis
rconn := d.upRds.Get(c)
if _, err = rconn.Do("SET", "ping", "pong"); err != nil {
rconn.Close()
log.Error("rds.Set error(%v)", err)
return
}
rconn.Close()
// region redis
rconn = d.rgRds.Get(c)
if _, err = rconn.Do("SET", "ping", "pong"); err != nil {
log.Error("rds.Set error(%v)", err)
}
rconn.Close()
return
}
// Close close resource.
func (d *Dao) Close() {
d.db.Close()
d.mc.Close()
d.upRds.Close()
d.rgRds.Close()
}
func (d *Dao) fillArchives(c context.Context, am map[int64]*api.Arc, cached bool) {
var (
mids []int64
aids []int64
staffs map[int64][]*api.StaffInfo
cooperationOK error
err error
)
for _, a := range am {
a.Fill()
a.TypeName = d.tNamem[int16(a.TypeID)]
if a.Author.Mid > 0 {
mids = append(mids, a.Author.Mid)
}
if a.AttrVal(archive.AttrBitIsCooperation) == archive.AttrYes {
aids = append(aids, a.Aid)
}
}
mi, err := d.acc.Infos3(c, &accapi.MidsReq{Mids: mids})
if err != nil || mi == nil {
log.Error("d.acc.Infos(%v) error(%v)", mids, err)
mi = new(accapi.InfosReply)
}
if len(aids) > 0 {
staffs, cooperationOK = d.staffs(c, aids)
}
for _, a := range am {
if m, ok := mi.Infos[a.Author.Mid]; ok {
a.Author.Name = m.Name
a.Author.Face = m.Face
}
if a.AttrVal(archive.AttrBitIsCooperation) == archive.AttrYes {
if staff, ok := staffs[a.Aid]; ok {
a.StaffInfo = staff
}
}
if cached {
var ca = &api.Arc{}
*ca = *a
d.addCache(func() {
d.addArchive3Cache(context.TODO(), ca)
if ca.Author.Name == "" || ca.Author.Face == "" || cooperationOK != nil {
log.Error("account empty aid(%d) name(%s) face(%s)", ca.Aid, ca.Author.Name, ca.Author.Face)
d.sendUpperCache(context.TODO(), ca.Aid)
}
})
}
}
}
func (d *Dao) fillArchive3(c context.Context, a *api.Arc, cached bool) {
a.Fill()
a.TypeName = d.tNamem[int16(a.TypeID)]
if a.Author.Mid > 0 {
reply, err := d.acc.Info3(c, &accapi.MidReq{Mid: a.Author.Mid})
if err != nil || reply == nil {
log.Error("d.acc.Info(%d) error(%v)", a.Author.Mid, err)
err = nil
} else {
a.Author.Name = reply.Info.Name
a.Author.Face = reply.Info.Face
}
}
var (
staffs []*api.StaffInfo
cooperationOK error
)
// cooperation
if a.AttrVal(archive.AttrBitIsCooperation) == archive.AttrYes {
if staffs, cooperationOK = d.staff(c, a.Aid); cooperationOK == nil {
a.StaffInfo = staffs
}
}
if cached {
var ca = &api.Arc{}
*ca = *a
d.addCache(func() {
d.addArchive3Cache(context.TODO(), ca)
if ca.Author.Name == "" || ca.Author.Face == "" || cooperationOK != nil {
log.Error("account empty aid(%d) name(%s) face(%s)", ca.Aid, ca.Author.Name, ca.Author.Face)
d.sendUpperCache(context.TODO(), ca.Aid)
}
})
}
}
func (d *Dao) sendUpperCache(c context.Context, aid int64) {
for i := 0; i < 10; i++ {
err := d.cacheDatabus.Send(context.Background(), strconv.FormatInt(aid, 10), &ajbmdl.Rebuild{Aid: aid})
if err != nil {
log.Error("d.cacheDatabus.Send(%d) error(%v)", aid, err)
time.Sleep(50 * time.Millisecond)
continue
}
log.Info("sendUpperCache(%d) ok", aid)
break
}
}
func (d *Dao) loadReportResult() {
var (
reportResults map[int64]*arcmdl.ReportResult
err error
ar = make(map[int64]string)
)
reportResults, err = d.ReportResults(context.TODO())
if err != nil {
log.Error("d.ReportResults error(%v)", err)
return
}
for _, rr := range reportResults {
ar[rr.Aid] = rr.Result
}
d.aidResult = ar
}
func (d *Dao) loadTypes() {
var (
types map[int16]*archive.ArcType
nm = make(map[int16]string)
err error
)
if types, err = d.Types(context.TODO()); err != nil {
log.Error("d.Types error(%v)", err)
return
}
for _, t := range types {
nm[t.ID] = t.Name
}
d.tNamem = nm
}
func (d *Dao) cacheproc() {
for {
f, ok := <-d.cacheCh
if !ok {
return
}
f()
}
}
func (d *Dao) loadproc() {
for {
time.Sleep(time.Duration(d.c.Tick))
d.loadTypes()
d.loadReportResult()
}
}

163
app/service/main/archive/dao/archive/dao_test.go Normal file
View File

@@ -0,0 +1,163 @@
package archive
import (
"context"
"flag"
"go-common/app/service/main/archive/conf"
"os"
"testing"
"github.com/smartystreets/goconvey/convey"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.app-svr.archive-service")
flag.Set("conf_token", "Y2LJhIsHx87nJaOBSxuG5TeZoLdBFlrE")
flag.Set("tree_id", "2302")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
m.Run()
os.Exit(0)
}
func TestArchiveArchive3(t *testing.T) {
var (
c = context.TODO()
aid = int64(-1)
)
convey.Convey("Archive3", t, func(ctx convey.C) {
_, err := d.Archive3(c, aid)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestArchiveArchives3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{1009799822}
)
convey.Convey("Archives3", t, func(ctx convey.C) {
am, err := d.Archives3(c, aids)
ctx.Convey("Then err should be nil.am should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(am, convey.ShouldNotBeNil)
})
})
}
func TestArchiveVideos3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("Videos3", t, func(ctx convey.C) {
vs, err := d.Videos3(c, aid)
ctx.Convey("Then err should be nil.vs should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(vs, convey.ShouldNotBeNil)
})
})
}
func TestArchiveVideosByAids3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{10097272, 10098500}
)
convey.Convey("VideosByAids3", t, func(ctx convey.C) {
vs, err := d.VideosByAids3(c, aids)
ctx.Convey("Then err should be nil.vs should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(vs, convey.ShouldNotBeNil)
})
})
}
func TestArchiveVideo3(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
cid = int64(2)
)
convey.Convey("Video3", t, func(ctx convey.C) {
v, err := d.Video3(c, aid, cid)
ctx.Convey("Then err should not be nil.v should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
ctx.So(v, convey.ShouldBeNil)
})
})
}
func TestArchiveDescription(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
desc = "我是个大描述我是个大描述"
)
convey.Convey("Description", t, func(ctx convey.C) {
err := d.addDescCache(c, aid, desc)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
desc, err := d.descCache(context.TODO(), aid)
ctx.Convey("Then err should be nil.desc should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(desc, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUpVideo3(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
cid = int64(2)
)
convey.Convey("UpVideo3", t, func(ctx convey.C) {
v, err := d.Video3(c, aid, cid)
ctx.Convey("Then err should not be nil.v should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
ctx.So(v, convey.ShouldBeNil)
})
})
}
func TestArchiveUpperCache(t *testing.T) {
var (
c = context.TODO()
aid = int64(10098500)
)
convey.Convey("UpperCache", t, func(ctx convey.C) {
err := d.UpArchiveCache(c, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func Test_Ping(t *testing.T) {
var c = context.TODO()
convey.Convey("Ping", t, func(ctx convey.C) {
err := d.Ping(c)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

82
app/service/main/archive/dao/archive/player.go Normal file
View File

@@ -0,0 +1,82 @@
package archive
import (
"context"
"net/http"
"net/url"
"strconv"
"go-common/app/service/main/archive/model/archive"
"go-common/library/ecode"
"go-common/library/xstr"
"github.com/pkg/errors"
)
// ValidateQn validate qn
func (d *Dao) ValidateQn(c context.Context, qn int) (allow bool) {
_, allow = d.playerQn[qn]
return
}
// VipQn vip qn
func (d *Dao) VipQn(c context.Context, qn int) (isVipQn bool) {
_, isVipQn = d.playerVipQn[qn]
return
}
// PlayerInfos cid with player info
func (d *Dao) PlayerInfos(c context.Context, cids []int64, qn int, platform, ip string, fnver, fnval, forceHost int) (pm map[uint32]*archive.BvcVideoItem, err error) {
params := url.Values{}
params.Set("cid", xstr.JoinInts(cids))
params.Set("qn", strconv.Itoa(qn))
params.Set("platform", platform)
params.Set("uip", ip)
params.Set("layout", "pb")
params.Set("fnver", strconv.Itoa(fnver))
params.Set("fnval", strconv.Itoa(fnval))
params.Set("force_host", strconv.Itoa(forceHost))
var req *http.Request
if req, err = d.playerClient.NewRequest("GET", d.c.PlayerAPI, ip, params); err != nil {
return
}
res := new(archive.BvcResponseMsg)
if err = d.playerClient.PB(c, req, res); err != nil {
return
}
if int(res.Code) != ecode.OK.Code() {
err = errors.Wrap(ecode.Int(int(res.Code)), d.c.PlayerAPI+params.Encode())
return
}
pm = res.Data
return
}
// PGCPlayerInfos cid with pgc player info
func (d *Dao) PGCPlayerInfos(c context.Context, aids []int64, platform, ip, session string, fnval, fnver int) (pgcm map[int64]*archive.PlayerInfo, err error) {
params := url.Values{}
params.Set("aids", xstr.JoinInts(aids))
params.Set("mobi_app", platform)
params.Set("ip", ip)
params.Set("fnver", strconv.Itoa(fnver))
params.Set("fnval", strconv.Itoa(fnval))
params.Set("session", session)
res := struct {
Code int `json:"code"`
Result map[int64]*archive.PGCPlayer `json:"result"`
}{}
if err = d.playerClient.Get(c, d.c.PGCPlayerAPI, ip, params, &res); err != nil {
return
}
if res.Code != 0 {
err = errors.Wrap(ecode.Int(res.Code), d.c.PGCPlayerAPI+params.Encode())
return
}
pgcm = make(map[int64]*archive.PlayerInfo)
for _, v := range res.Result {
if v.PlayerInfo != nil {
pgcm[v.PlayerInfo.Cid] = v.PlayerInfo
}
}
return
}

72
app/service/main/archive/dao/archive/player_test.go Normal file
View File

@@ -0,0 +1,72 @@
package archive
import (
"context"
"encoding/json"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveValidateQn(t *testing.T) {
var (
c = context.TODO()
qn = int(80)
)
convey.Convey("ValidateQn", t, func(ctx convey.C) {
allow := d.ValidateQn(c, qn)
ctx.Convey("Then allow should not be nil.", func(ctx convey.C) {
ctx.So(allow, convey.ShouldNotBeNil)
})
})
}
func TestArchiveVipQn(t *testing.T) {
var (
c = context.TODO()
qn = int(80)
)
convey.Convey("VipQn", t, func(ctx convey.C) {
isVipQn := d.VipQn(c, qn)
ctx.Convey("Then allow should not be nil.", func(ctx convey.C) {
ctx.So(isVipQn, convey.ShouldNotBeNil)
})
})
}
func TestArchivePlayerInfos(t *testing.T) {
var (
c = context.TODO()
cids = []int64{10133958}
qn = int(16)
platform = "ios"
ip = "127.0.0.1"
fnver = int(0)
fnval = int(8)
forceHost = int(0)
)
convey.Convey("PlayerInfos", t, func(ctx convey.C) {
d.PlayerInfos(c, cids, qn, platform, ip, fnver, fnval, forceHost)
})
}
func TestArchivePGCPlayerInfos(t *testing.T) {
var (
c = context.TODO()
aids = []int64{10111001}
platform = "iphone"
ip = "121.31.246.238"
session = ""
fnval = 16
fnver = 0
)
convey.Convey("PGCPlayerInfos", t, func(ctx convey.C) {
pm, err := d.PGCPlayerInfos(c, aids, platform, ip, session, fnval, fnver)
ctx.Convey("Then err should be nil.pm should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(pm, convey.ShouldNotBeNil)
p, _ := json.Marshal(pm)
convey.Printf("%s", p)
})
})
}

32
app/service/main/archive/dao/archive/region.go Normal file
View File

@@ -0,0 +1,32 @@
package archive
import (
"context"
"go-common/app/service/main/archive/api"
"go-common/library/log"
)
const (
_rgnArcsSQL = "SELECT aid,attribute,copyright,pubtime FROM archive WHERE typeid=? and (state>=0 or state=-6) LIMIT ?,?"
)
// RegionArcs multi get archvies by rid.
func (d *Dao) RegionArcs(c context.Context, rid int16, start, length int) (ras []*api.RegionArc, err error) {
d.infoProm.Incr("RegionArcs")
rows, err := d.rgnArcsStmt.Query(c, rid, start, length)
if err != nil {
log.Error("rgnArcsStmt.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
a := &api.RegionArc{}
if err = rows.Scan(&a.Aid, &a.Attribute, &a.Copyright, &a.PubDate); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
ras = append(ras, a)
}
return
}

257
app/service/main/archive/dao/archive/region_rds.go Normal file
View File

@@ -0,0 +1,257 @@
package archive
import (
"context"
"fmt"
"time"
"go-common/app/service/main/archive/api"
commarc "go-common/app/service/main/archive/model/archive"
"go-common/library/cache/redis"
"go-common/library/log"
)
const (
// all type archives
_allTypeKey = "bl_arc"
// all archives.
_suffixAll = "_a"
// origin archives copyright=1.
_suffixOrigin = "_o"
// top type archives
_suffixTopAll = "_t"
// top type expire
topTypeExpire = 3 * 24 * time.Hour
// all type expire
allTypeExpire = 7 * 24 * time.Hour
)
func rgAllKey(rid int16) string {
return fmt.Sprintf("%b", rid) + _suffixAll
}
func rgOriginKey(rid int16) string {
return fmt.Sprintf("%b", rid) + _suffixOrigin
}
func rgTopKey(rid int16) string {
return fmt.Sprintf("%b%s", rid, _suffixTopAll)
}
// AddRegionArcCache add archives into region cache.
func (d *Dao) AddRegionArcCache(c context.Context, rid, reid int16, as ...*api.RegionArc) (err error) {
var (
key = rgAllKey(rid)
okey = rgOriginKey(rid)
tpKey = rgTopKey(reid)
count int
conn = d.rgRds.Get(c)
)
defer conn.Close()
defer func() {
if err != nil {
d.errProm.Incr("regin_redis")
}
}()
for _, a := range as {
if !a.AllowShow() {
continue
}
if err = conn.Send("ZADD", tpKey, a.PubDate, a.Aid); err != nil {
log.Error("conn.Send(ZADD, %s, %d, %d) error(%v)")
return
}
count++
if err = conn.Send("ZREMRANGEBYSCORE", tpKey, "-inf", time.Now().Add(-time.Duration(topTypeExpire)).Unix()); err != nil {
log.Error("conn.Send(ZREMRANGEBYSCORE, %s, %d, %d) error(%v)", tpKey)
return
}
count++
if err = conn.Send("ZADD", key, a.PubDate, a.Aid); err != nil {
log.Error("conn.Send(ZADD, %s, %d) error(%v)", key, a.Aid, err)
return
}
count++
if a.Copyright == commarc.CopyrightOriginal {
if err = conn.Send("ZADD", okey, a.PubDate, a.Aid); err != nil {
log.Error("conn.Send(ZADD, %s, %d) error(%v)", okey, a.Aid, err)
return
}
count++
}
// 连载动画不记录
if rid != 33 {
if err = conn.Send("ZADD", _allTypeKey, a.PubDate, a.Aid); err != nil {
log.Error("conn.Send(ZADD, %s, %d, %d) error(%v)", _allTypeKey, a.PubDate, a.Aid, err)
return
}
count++
if err = conn.Send("ZREMRANGEBYSCORE", _allTypeKey, "-inf", time.Now().Add(-time.Duration(allTypeExpire)).Unix()); err != nil {
log.Error("conn.Send()")
return
}
count++
}
}
if count == 0 {
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
for i := 0; i < count; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
return
}
// RegionTopArcsCache top region archives.
func (d *Dao) RegionTopArcsCache(c context.Context, reid int16, start, end int) (aids []int64, err error) {
key := rgTopKey(reid)
if reid == 0 {
key = _allTypeKey
}
if aids, err = d.zrange(c, key, start, end); err != nil {
d.errProm.Incr("regin_redis")
log.Error("dao.zrange(%s, %d, %d) error(%v)", key, start, end, err)
}
return
}
// RegionArcsCache region archives.
func (d *Dao) RegionArcsCache(c context.Context, rid int16, start, end int) (aids []int64, err error) {
key := rgAllKey(rid)
aids, err = d.zrange(c, key, start, end)
return
}
// RegionOriginArcsCache region origin archives.
func (d *Dao) RegionOriginArcsCache(c context.Context, rid int16, start, end int) (aids []int64, err error) {
key := rgOriginKey(rid)
aids, err = d.zrange(c, key, start, end)
return
}
func (d *Dao) zrange(c context.Context, key string, start, end int) (aids []int64, err error) {
conn := d.rgRds.Get(c)
defer conn.Close()
values, err := redis.Values(conn.Do("ZREVRANGE", key, start, end))
if err != nil {
d.errProm.Incr("regin_redis")
log.Error("conn.Do(ZREVRANGE, %s) error(%v)", key, err)
return
}
if len(values) == 0 {
return
}
err = redis.ScanSlice(values, &aids)
return
}
// RegionTopCountCache top region count of archives.
func (d *Dao) RegionTopCountCache(c context.Context, reids []int16, min, max int64) (recm map[int16]int, err error) {
conn := d.rgRds.Get(c)
defer conn.Close()
for _, rid := range reids {
key := rgTopKey(rid)
if err = conn.Send("ZCOUNT", key, min, max); err != nil {
d.errProm.Incr("regin_redis")
log.Error("conn.Do(ZCOUNT, %s) error(%v)", key, err)
break
}
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
recm = make(map[int16]int, len(reids))
for _, rid := range reids {
if recm[rid], err = redis.Int(conn.Receive()); err != nil {
d.errProm.Incr("regin_redis")
log.Error("conn.Receive error(%v)")
return
}
}
return
}
// RegionAllCountCache count of all type
func (d *Dao) RegionAllCountCache(c context.Context) (count int, err error) {
conn := d.rgRds.Get(c)
defer conn.Close()
count, err = redis.Int(conn.Do("ZCARD", _allTypeKey))
return
}
// RegionCountCache count of arcs.
func (d *Dao) RegionCountCache(c context.Context, rid int16) (count int, err error) {
conn := d.rgRds.Get(c)
defer conn.Close()
key := rgAllKey(rid)
count, err = redis.Int(conn.Do("ZCARD", key))
return
}
// RegionOriginCountCache count of arcs.
func (d *Dao) RegionOriginCountCache(c context.Context, rid int16) (count int, err error) {
conn := d.rgRds.Get(c)
defer conn.Close()
key := rgOriginKey(rid)
count, err = redis.Int(conn.Do("ZCARD", key))
return
}
// DelRegionArcCache delete from zset.
func (d *Dao) DelRegionArcCache(c context.Context, rid, reid int16, aid int64) (err error) {
conn := d.rgRds.Get(c)
defer conn.Close()
defer func() {
if err != nil {
d.errProm.Incr("regin_redis")
}
}()
var (
rgAllKey = rgAllKey(rid)
oKey = rgOriginKey(rid)
rgTopKey = rgTopKey(reid)
)
if err = conn.Send("ZREM", rgAllKey, aid); err != nil {
log.Error("conn.Send error(%v)", err)
return
}
if err = conn.Send("ZREM", oKey, aid); err != nil {
log.Error("conn.Send error(%v)", err)
return
}
if err = conn.Send("ZREM", rgTopKey, aid); err != nil {
log.Error("conn.Send(ZREM, %s, %d) error(%v)", rgTopKey, aid, err)
return
}
if err = conn.Send("ZREM", _allTypeKey, aid); err != nil {
log.Error("conn.Send(ZREM, %s, %d) error(%v)", _allTypeKey, aid, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
return
}
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
return
}
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
}
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
}
return
}

193
app/service/main/archive/dao/archive/region_rds_test.go Normal file
View File

@@ -0,0 +1,193 @@
package archive
import (
"context"
"go-common/app/service/main/archive/api"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchivergAllKey(t *testing.T) {
var (
rid = int16(97)
)
convey.Convey("rgAllKey", t, func(ctx convey.C) {
p1 := rgAllKey(rid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivergOriginKey(t *testing.T) {
var (
rid = int16(97)
)
convey.Convey("rgOriginKey", t, func(ctx convey.C) {
p1 := rgOriginKey(rid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivergTopKey(t *testing.T) {
var (
rid = int16(97)
)
convey.Convey("rgTopKey", t, func(ctx convey.C) {
p1 := rgTopKey(rid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchiveAddRegionArcCache(t *testing.T) {
var (
c = context.TODO()
rid = int16(97)
reid = int16(1)
as = &api.RegionArc{}
)
convey.Convey("AddRegionArcCache", t, func(ctx convey.C) {
err := d.AddRegionArcCache(c, rid, reid, as)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveRegionTopArcsCache(t *testing.T) {
var (
c = context.TODO()
reid = int16(1)
start = int(0)
end = int(20)
)
convey.Convey("RegionTopArcsCache", t, func(ctx convey.C) {
_, err := d.RegionTopArcsCache(c, reid, start, end)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveRegionArcsCache(t *testing.T) {
var (
c = context.TODO()
rid = int16(97)
start = int(0)
end = int(20)
)
convey.Convey("RegionArcsCache", t, func(ctx convey.C) {
aids, err := d.RegionArcsCache(c, rid, start, end)
ctx.Convey("Then err should be nil.aids should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(aids, convey.ShouldNotBeNil)
})
})
}
func TestArchiveRegionOriginArcsCache(t *testing.T) {
var (
c = context.TODO()
rid = int16(97)
start = int(0)
end = int(20)
)
convey.Convey("RegionOriginArcsCache", t, func(ctx convey.C) {
_, err := d.RegionOriginArcsCache(c, rid, start, end)
ctx.Convey("Then err should be nil.aids should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivezrange(t *testing.T) {
var (
c = context.TODO()
key = "1_o"
start = int(1)
end = int(10)
)
convey.Convey("zrange", t, func(ctx convey.C) {
_, err := d.zrange(c, key, start, end)
ctx.Convey("Then err should be nil.aids should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveRegionTopCountCache(t *testing.T) {
var (
c = context.TODO()
reids = []int16{1}
min = int64(0)
max = int64(10)
)
convey.Convey("RegionTopCountCache", t, func(ctx convey.C) {
recm, err := d.RegionTopCountCache(c, reids, min, max)
ctx.Convey("Then err should be nil.recm should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(recm, convey.ShouldNotBeNil)
})
})
}
func TestArchiveRegionAllCountCache(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("RegionAllCountCache", t, func(ctx convey.C) {
count, err := d.RegionAllCountCache(c)
ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestArchiveRegionCountCache(t *testing.T) {
var (
c = context.TODO()
rid = int16(97)
)
convey.Convey("RegionCountCache", t, func(ctx convey.C) {
count, err := d.RegionCountCache(c, rid)
ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestArchiveRegionOriginCountCache(t *testing.T) {
var (
c = context.TODO()
rid = int16(97)
)
convey.Convey("RegionOriginCountCache", t, func(ctx convey.C) {
count, err := d.RegionOriginCountCache(c, rid)
ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestArchiveDelRegionArcCache(t *testing.T) {
var (
c = context.TODO()
rid = int16(97)
reid = int16(1)
aid = int64(1)
)
convey.Convey("DelRegionArcCache", t, func(ctx convey.C) {
err := d.DelRegionArcCache(c, rid, reid, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

23
app/service/main/archive/dao/archive/region_test.go Normal file
View File

@@ -0,0 +1,23 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveRegionArcs(t *testing.T) {
var (
c = context.TODO()
rid = int16(97)
start = int(0)
length = int(10)
)
convey.Convey("RegionArcs", t, func(ctx convey.C) {
_, err := d.RegionArcs(c, rid, start, length)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

34
app/service/main/archive/dao/archive/report_result.go Normal file
View File

@@ -0,0 +1,34 @@
package archive
import (
"context"
arcmdl "go-common/app/service/main/archive/model/archive"
"go-common/library/log"
)
const (
_reportResultSQL = "SELECT aid,result,state,adminid,after_deal FROM archive_report_result WHERE is_show=1"
)
// ReportResults get all report
func (d *Dao) ReportResults(c context.Context) (res map[int64]*arcmdl.ReportResult, err error) {
d.infoProm.Incr("ReportResults")
rows, err := d.reportResultStmt.Query(c)
if err != nil {
log.Error("d.reportResultStmt.Query() error(%v)", err)
return
}
defer rows.Close()
res = make(map[int64]*arcmdl.ReportResult)
for rows.Next() {
rr := &arcmdl.ReportResult{}
if err = rows.Scan(&rr.Aid, &rr.Result, &rr.State, &rr.AdminID, &rr.AfterDeal); err != nil {
log.Error("row.Scan error(%v)", err)
d.errProm.Incr("report_result_db")
return
}
res[rr.Aid] = rr
}
return
}

21
app/service/main/archive/dao/archive/report_result_test.go Normal file
View File

@@ -0,0 +1,21 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveReportResults(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("ReportResults", t, func(ctx convey.C) {
res, err := d.ReportResults(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}

89
app/service/main/archive/dao/archive/stat.go Normal file
View File

@@ -0,0 +1,89 @@
package archive
import (
"context"
"fmt"
"go-common/app/service/main/archive/api"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_statSharding = 100
// stat
_statSQL = "SELECT aid,fav,share,reply,coin,dm,click,now_rank,his_rank,likes FROM archive_stat_%02d WHERE aid=?"
_statsSQL = "SELECT aid,fav,share,reply,coin,dm,click,now_rank,his_rank,likes FROM archive_stat_%02d WHERE aid in (%s)"
// click
_clickSQL = "SELECT aid,web,h5,outside,ios,android FROM archive_click_%02d WHERE aid=?"
)
func statTbl(aid int64) int64 {
return aid % _statSharding
}
// stat3 archive stat.
func (d *Dao) stat3(c context.Context, aid int64) (st *api.Stat, err error) {
d.infoProm.Incr("stat3")
row := d.statDB.QueryRow(c, fmt.Sprintf(_statSQL, statTbl(aid)), aid)
st = &api.Stat{}
if err = row.Scan(&st.Aid, &st.Fav, &st.Share, &st.Reply, &st.Coin, &st.Danmaku, &st.View, &st.NowRank, &st.HisRank, &st.Like); err != nil {
if err == sql.ErrNoRows {
st = nil
err = nil
} else {
d.errProm.Incr("stat_db")
log.Error("row.Scan error(%v)", err)
}
return
}
return
}
// stats3 archive stats.
func (d *Dao) stats3(c context.Context, aids []int64) (sts map[int64]*api.Stat, err error) {
d.infoProm.Incr("stats3")
tbls := make(map[int64][]int64)
for _, aid := range aids {
tbls[statTbl(aid)] = append(tbls[statTbl(aid)], aid)
}
sts = make(map[int64]*api.Stat, len(aids))
for tbl, ids := range tbls {
var rows *sql.Rows
if rows, err = d.statDB.Query(c, fmt.Sprintf(_statsSQL, tbl, xstr.JoinInts(ids))); err != nil {
log.Error("d.statDB.Query(%s) error(%v)", fmt.Sprintf(_statsSQL, tbl, xstr.JoinInts(ids)), err)
d.errProm.Incr("stat_db")
return
}
for rows.Next() {
st := &api.Stat{}
if err = rows.Scan(&st.Aid, &st.Fav, &st.Share, &st.Reply, &st.Coin, &st.Danmaku, &st.View, &st.NowRank, &st.HisRank, &st.Like); err != nil {
log.Error("rows.Scan error(%v)", err)
d.errProm.Incr("stat_db")
rows.Close()
return
}
sts[st.Aid] = st
}
rows.Close()
}
return
}
// click3 archive click.
func (d *Dao) click3(c context.Context, aid int64) (cl *api.Click, err error) {
d.infoProm.Incr("click3")
row := d.clickDB.QueryRow(c, fmt.Sprintf(_clickSQL, statTbl(aid)), aid)
cl = &api.Click{}
if err = row.Scan(&cl.Aid, &cl.Web, &cl.H5, &cl.Outter, &cl.Ios, &cl.Android); err != nil {
if err == sql.ErrNoRows {
cl = nil
err = nil
} else {
log.Error("row.Scan error(%v)", err)
d.errProm.Incr("click_db")
}
}
return
}

188
app/service/main/archive/dao/archive/stat_mc_pb.go Normal file
View File

@@ -0,0 +1,188 @@
package archive
import (
"context"
"strconv"
"go-common/app/service/main/archive/api"
"go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_prefixStatPB = "stp_"
_prefixClickPB = "clkp_"
)
type batchStat3 struct {
cached map[int64]*api.Stat
missed []int64
err error
}
func statPBKey(aid int64) string {
return _prefixStatPB + strconv.FormatInt(aid, 10)
}
func clickPBKey(aid int64) string {
return _prefixClickPB + strconv.FormatInt(aid, 10)
}
// statCache3 get a archive stat from cache.
func (d *Dao) statCache3(c context.Context, aid int64) (st *api.Stat, err error) {
var (
key = statPBKey(aid)
rp *memcache.Item
conn = d.mc.Get(c)
)
defer conn.Close()
if rp, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
d.errProm.Incr("stat_mc")
log.Error("memcache.Get(%s) error(%v)", key, err)
}
return
}
st = new(api.Stat)
if err = conn.Scan(rp, st); err != nil {
log.Error("conn.Scan(%s) error(%v)", rp.Value, err)
return
}
st.DisLike = 0
return
}
// statCaches3 multi get archives stat, return map[aid]*Stat and missed aids.
func (d *Dao) statCaches3(c context.Context, aids []int64) (cached map[int64]*api.Stat, missed []int64, err error) {
var (
lt = len(aids)
times int
reqCh chan batchStat3
)
if lt%_multiInterval == 0 {
times = lt / _multiInterval
} else {
times = lt/_multiInterval + 1
}
reqCh = make(chan batchStat3, times)
for i := 0; i < times; i++ {
var tmps []int64
if i == times-1 {
tmps = aids[i*_multiInterval:]
} else {
tmps = aids[i*_multiInterval : (i+1)*_multiInterval]
}
go d._statCaches3(c, tmps, reqCh)
}
if times == 1 {
req := <-reqCh
cached = req.cached
missed = req.missed
err = req.err
return
}
cached = make(map[int64]*api.Stat, len(aids))
for i := 0; i < times; i++ {
req := <-reqCh
for k, v := range req.cached {
cached[k] = v
}
missed = append(missed, req.missed...)
}
return
}
func (d *Dao) _statCaches3(c context.Context, aids []int64, reqCh chan<- batchStat3) {
var (
req batchStat3
keys = make([]string, 0, len(aids))
am = make(map[int64]struct{}, len(aids))
rps map[string]*memcache.Item
err error
conn = d.mc.Get(c)
)
defer conn.Close()
for _, aid := range aids {
keys = append(keys, statPBKey(aid))
am[aid] = struct{}{}
}
if rps, err = conn.GetMulti(keys); err != nil {
req.missed = aids
req.err = err
reqCh <- req
log.Error("conn.Gets error(%v)", err)
d.errProm.Incr("stat_mc")
return
}
var (
cached = make(map[int64]*api.Stat, len(aids))
missed []int64
)
for _, rp := range rps {
var st = new(api.Stat)
if e := conn.Scan(rp, st); e != nil {
log.Error("conn.Scan(%s) error(%v)", rp.Value, e)
continue
}
st.DisLike = 0
cached[st.Aid] = st
delete(am, st.Aid)
}
for aid := range am {
missed = append(missed, aid)
}
req.cached = cached
req.missed = missed
reqCh <- req
}
// addStatCache set archive stat into cache.
func (d *Dao) addStatCache3(c context.Context, st *api.Stat) (err error) {
key := statPBKey(st.Aid)
conn := d.mc.Get(c)
st.DisLike = 0
if err = conn.Set(&memcache.Item{Key: key, Object: st, Flags: memcache.FlagProtobuf, Expiration: 0}); err != nil {
d.errProm.Incr("stat_mc")
log.Error("memcache.Set(%s) error(%v)", key, err)
}
conn.Close()
return
}
// clickCache get a archive click from cache.
func (d *Dao) clickCache3(c context.Context, aid int64) (clk *api.Click, err error) {
var (
key = clickPBKey(aid)
rp *memcache.Item
conn = d.mc.Get(c)
)
defer conn.Close()
if rp, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
d.errProm.Incr("stat_mc")
log.Error("memcache.Get(%s) error(%v)", key, err)
}
return
}
clk = new(api.Click)
if err = conn.Scan(rp, clk); err != nil {
log.Error("conn.Scan(%s) error(%v)", rp.Value, err)
}
return
}
// addClickCache set archive click into cache.
func (d *Dao) addClickCache3(c context.Context, clk *api.Click) (err error) {
key := clickPBKey(clk.Aid)
conn := d.mc.Get(c)
if err = conn.Set(&memcache.Item{Key: key, Object: clk, Flags: memcache.FlagProtobuf, Expiration: 0}); err != nil {
d.errProm.Incr("stat_mc")
log.Error("memcache.Set(%s) error(%v)", key, err)
}
conn.Close()
return
}

100
app/service/main/archive/dao/archive/stat_mc_pb_test.go Normal file
View File

@@ -0,0 +1,100 @@
package archive
import (
"context"
"go-common/app/service/main/archive/api"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchivestatPBKey(t *testing.T) {
var (
aid = int64(1)
)
convey.Convey("statPBKey", t, func(ctx convey.C) {
p1 := statPBKey(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchiveclickPBKey(t *testing.T) {
var (
aid = int64(1)
)
convey.Convey("clickPBKey", t, func(ctx convey.C) {
p1 := clickPBKey(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivestatCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
)
convey.Convey("statCache3", t, func(ctx convey.C) {
_, err := d.statCache3(c, aid)
ctx.Convey("Then err should be nil.st should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivestatCaches3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{1, 2}
)
convey.Convey("statCaches3", t, func(ctx convey.C) {
cached, missed, err := d.statCaches3(c, aids)
ctx.Convey("Then err should be nil.cached,missed should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(missed, convey.ShouldNotBeNil)
ctx.So(cached, convey.ShouldNotBeNil)
})
})
}
func TestArchiveaddStatCache3(t *testing.T) {
var (
c = context.TODO()
st = &api.Stat{}
)
convey.Convey("addStatCache3", t, func(ctx convey.C) {
err := d.addStatCache3(c, st)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveclickCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
)
convey.Convey("clickCache3", t, func(ctx convey.C) {
_, err := d.clickCache3(c, aid)
ctx.Convey("Then err should be nil.clk should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveaddClickCache3(t *testing.T) {
var (
c = context.TODO()
clk = &api.Click{}
)
convey.Convey("addClickCache3", t, func(ctx convey.C) {
err := d.addClickCache3(c, clk)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

139
app/service/main/archive/dao/archive/stat_pb.go Normal file
View File

@@ -0,0 +1,139 @@
package archive
import (
"context"
"go-common/app/service/main/archive/api"
"go-common/library/log"
)
// SetStat3 set all stat
func (d *Dao) SetStat3(c context.Context, st *api.Stat) (err error) {
d.addStatCache3(c, st)
var clk *api.Click
if clk, err = d.click3(c, st.Aid); err != nil {
log.Error("d.stat(%d) error(%v)", st.Aid, err)
return
}
if clk == nil {
clk = &api.Click{Aid: st.Aid}
}
d.addCache(func() {
d.addClickCache3(context.TODO(), clk)
})
return
}
// Stat3 get archive stat.
func (d *Dao) Stat3(c context.Context, aid int64) (st *api.Stat, err error) {
var cached = true
if st, err = d.statCache3(c, aid); err != nil {
log.Error("d.statCache(%d) error(%v)", aid, err)
cached = false
}
if st != nil {
return
}
if st, err = d.stat3(c, aid); err != nil {
log.Error("d.stat(%d) error(%v)", aid, err)
return
}
if st == nil {
st = &api.Stat{Aid: aid}
return
}
if cached {
d.addCache(func() {
d.addStatCache3(context.TODO(), st)
})
}
return
}
// Stats3 get archives stat.
func (d *Dao) Stats3(c context.Context, aids []int64) (stm map[int64]*api.Stat, err error) {
if len(aids) == 0 {
return
}
var (
missed []int64
missm map[int64]*api.Stat
cached = true
)
if stm, missed, err = d.statCaches3(c, aids); err != nil {
log.Error("d.statCaches(%d) error(%v)", aids, err)
missed = aids
stm = make(map[int64]*api.Stat, len(aids))
err = nil // ignore error
cached = false
}
if stm != nil && len(missed) == 0 {
return
}
if missm, err = d.stats3(c, missed); err != nil {
log.Error("d.stats(%v) error(%v)", missed, err)
err = nil // ignore error
}
for aid, st := range missm {
stm[aid] = st
if cached {
var cst = &api.Stat{}
*cst = *st
d.addCache(func() {
d.addStatCache3(context.TODO(), cst)
})
}
}
return
}
// Click3 get archive click.
func (d *Dao) Click3(c context.Context, aid int64) (clk *api.Click, err error) {
var cached = true
if clk, err = d.clickCache3(c, aid); err != nil {
log.Error("d.clickCache(%d) error(%v)", aid, err)
cached = false
}
if clk != nil {
return
}
if clk, err = d.click3(c, aid); err != nil {
log.Error("d.stat(%d) error(%v)", aid, err)
return
}
if clk == nil {
clk = &api.Click{Aid: aid}
return
}
if cached {
d.addCache(func() {
d.addClickCache3(context.TODO(), clk)
})
}
return
}
// InitStatCache3 if db is nil, set nil cache
func (d *Dao) InitStatCache3(c context.Context, aid int64) (err error) {
var st *api.Stat
if st, err = d.stat3(c, aid); err != nil {
log.Error("d.stat(%d) error(%v)", aid, err)
return
}
if st == nil {
d.addCache(func() {
d.addStatCache3(context.TODO(), &api.Stat{Aid: aid})
})
}
var clk *api.Click
if clk, err = d.click3(c, aid); err != nil {
log.Error("d.stat(%d) error(%v)", aid, err)
return
}
if clk == nil {
d.addCache(func() {
d.addClickCache3(context.TODO(), &api.Click{Aid: aid})
})
}
return
}

77
app/service/main/archive/dao/archive/stat_pb_test.go Normal file
View File

@@ -0,0 +1,77 @@
package archive
import (
"context"
"go-common/app/service/main/archive/api"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveSetStat3(t *testing.T) {
var (
c = context.TODO()
st = &api.Stat{}
)
convey.Convey("SetStat3", t, func(ctx convey.C) {
err := d.SetStat3(c, st)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveStat3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("Stat3", t, func(ctx convey.C) {
st, err := d.Stat3(c, aid)
ctx.Convey("Then err should be nil.st should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(st, convey.ShouldNotBeNil)
})
})
}
func TestArchiveStats3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{10097272}
)
convey.Convey("Stats3", t, func(ctx convey.C) {
stm, err := d.Stats3(c, aids)
ctx.Convey("Then err should be nil.stm should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(stm, convey.ShouldNotBeNil)
})
})
}
func TestArchiveClick3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("Click3", t, func(ctx convey.C) {
clk, err := d.Click3(c, aid)
ctx.Convey("Then err should be nil.clk should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(clk, convey.ShouldNotBeNil)
})
})
}
func TestArchiveInitStatCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("InitStatCache3", t, func(ctx convey.C) {
err := d.InitStatCache3(c, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

46
app/service/main/archive/dao/archive/stat_test.go Normal file
View File

@@ -0,0 +1,46 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchivestatTbl(t *testing.T) {
var (
aid = int64(1)
)
convey.Convey("statTbl", t, func(ctx convey.C) {
p1 := statTbl(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivestat3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("stat3", t, func(ctx convey.C) {
_, err := d.stat3(c, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivestats3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{10097272}
)
convey.Convey("stats3", t, func(ctx convey.C) {
_, err := d.stats3(c, aids)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

38
app/service/main/archive/dao/archive/type.go Normal file
View File

@@ -0,0 +1,38 @@
package archive
import (
"context"
"go-common/app/service/main/archive/model/archive"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_tpsSQL = "SELECT id,pid,name FROM archive_type"
)
// Types get type relation.
//func (d *Dao) Types(c context.Context) (nm map[int16]string, rids []int16, sf map[int16]int16, err error) {
func (d *Dao) Types(c context.Context) (types map[int16]*archive.ArcType, err error) {
d.infoProm.Incr("Types")
var rows *sql.Rows
if rows, err = d.tpsStmt.Query(c); err != nil {
log.Error("tpsStmt.Query error(%v)", err)
return
}
defer rows.Close()
types = make(map[int16]*archive.ArcType)
for rows.Next() {
var (
rid, pid int16
name string
)
if err = rows.Scan(&rid, &pid, &name); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
types[rid] = &archive.ArcType{ID: rid, Pid: pid, Name: name}
}
return
}

21
app/service/main/archive/dao/archive/type_test.go Normal file
View File

@@ -0,0 +1,21 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveTypes(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("Types", t, func(ctx convey.C) {
types, err := d.Types(c)
ctx.Convey("Then err should be nil.types should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(types, convey.ShouldNotBeNil)
})
})
}

128
app/service/main/archive/dao/archive/upper.go Normal file
View File

@@ -0,0 +1,128 @@
package archive
import (
"context"
"fmt"
"go-common/library/log"
"go-common/library/time"
"go-common/library/xstr"
"github.com/pkg/errors"
)
const (
_upCntSQL = "SELECT COUNT(*) FROM archive WHERE mid=? AND (state>=0 or state=-6)"
_upsCntSQL = "SELECT mid,COUNT(*) FROM archive WHERE mid IN(%s) AND (state>=0 or state=-6) GROUP BY mid"
_upPasSQL = "SELECT aid,pubtime,copyright FROM archive WHERE mid=? AND state>=0 ORDER BY pubtime DESC"
_upsPasSQL = "SELECT aid,mid,pubtime,copyright FROM archive WHERE mid IN (%s) AND state>=0 ORDER BY pubtime DESC"
_upRecommendSQL = "SELECT reco_aid FROM archive_recommend WHERE aid=? AND state=0"
)
// UppersCount get mids count
func (d *Dao) UppersCount(c context.Context, mids []int64) (uc map[int64]int, err error) {
rows, err := d.resultDB.Query(c, fmt.Sprintf(_upsCntSQL, xstr.JoinInts(mids)))
if err != nil {
err = errors.WithStack(err)
return
}
uc = make(map[int64]int, len(mids))
defer rows.Close()
for rows.Next() {
var (
mid int64
cnt int
)
if err = rows.Scan(&mid, &cnt); err != nil {
err = errors.WithStack(err)
return
}
uc[mid] = cnt
}
return
}
// UpperCount get the count of archives by mid of Up.
func (d *Dao) UpperCount(c context.Context, mid int64) (count int, err error) {
d.infoProm.Incr("UpperCount")
row := d.upCntStmt.QueryRow(c, mid)
if err = row.Scan(&count); err != nil {
log.Error("row.Scan error(%v)", err)
}
return
}
// UpperPassed get upper passed archives.
func (d *Dao) UpperPassed(c context.Context, mid int64) (aids []int64, ptimes []time.Time, copyrights []int8, err error) {
d.infoProm.Incr("UpperPassed")
rows, err := d.upPasStmt.Query(c, mid)
if err != nil {
log.Error("getUpPasStmt.Query(%d) error(%v)", mid, err)
return
}
defer rows.Close()
for rows.Next() {
var (
aid int64
ptime time.Time
copyright int8
)
if err = rows.Scan(&aid, &ptime, &copyright); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
aids = append(aids, aid)
ptimes = append(ptimes, ptime)
copyrights = append(copyrights, copyright)
}
return
}
// UppersPassed get uppers passed archives.
func (d *Dao) UppersPassed(c context.Context, mids []int64) (aidm map[int64][]int64, ptimes map[int64][]time.Time, copyrights map[int64][]int8, err error) {
d.infoProm.Incr("UppersPassed")
rows, err := d.resultDB.Query(c, fmt.Sprintf(_upsPasSQL, xstr.JoinInts(mids)))
if err != nil {
log.Error("UpsPassed error(%v)", err)
return
}
defer rows.Close()
aidm = make(map[int64][]int64, len(mids))
ptimes = make(map[int64][]time.Time, len(mids))
copyrights = make(map[int64][]int8, len(mids))
for rows.Next() {
var (
aid, mid int64
ptime time.Time
copyright int8
)
if err = rows.Scan(&aid, &mid, &ptime, &copyright); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
aidm[mid] = append(aidm[mid], aid)
ptimes[mid] = append(ptimes[mid], ptime)
copyrights[mid] = append(copyrights[mid], copyright)
}
return
}
// UpperReommend get up recommend
func (d *Dao) UpperReommend(c context.Context, aid int64) (reAids []int64, err error) {
d.infoProm.Incr("UpperRecommend")
rows, err := d.arcReadDB.Query(c, _upRecommendSQL, aid)
if err != nil {
log.Error("d.arcReadDB.Query(%s, %d) error(%v)", _upRecommendSQL, aid)
return
}
defer rows.Close()
for rows.Next() {
var reAid int64
if err = rows.Scan(&reAid); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
reAids = append(reAids, reAid)
}
return
}

316
app/service/main/archive/dao/archive/upper_rds.go Normal file
View File

@@ -0,0 +1,316 @@
package archive
import (
"context"
"strconv"
"go-common/app/service/main/archive/model/archive"
"go-common/library/cache/redis"
"go-common/library/log"
"go-common/library/time"
)
const (
_prefixUpCnt = "uc_"
_prefixUpPas = "up_"
)
func upCntKey(mid int64) string {
return _prefixUpCnt + strconv.FormatInt(mid, 10)
}
func upPasKey(mid int64) string {
return _prefixUpPas + strconv.FormatInt(mid, 10)
}
// AddUpperCountCache the count of up's archives
func (d *Dao) AddUpperCountCache(c context.Context, mid int64, count int) (err error) {
var (
key = upCntKey(mid)
conn = d.upRds.Get(c)
expireTime = d.upExpire
)
defer conn.Close()
if count == 0 {
expireTime = 600
}
if _, err = conn.Do("SETEX", key, expireTime, count); err != nil {
log.Error("conn.Do(SETEX, %s, %d, %d)", key, expireTime, count)
return
}
return
}
// UpperCountCache get up count from cache.
func (d *Dao) UpperCountCache(c context.Context, mid int64) (count int, err error) {
var (
key = upCntKey(mid)
conn = d.upRds.Get(c)
)
defer conn.Close()
if count, err = redis.Int(conn.Do("GET", key)); err != nil {
if err == redis.ErrNil {
count = -1
err = nil
} else {
d.errProm.Incr("upper_redis")
log.Error("conn.Do(GET, %s) error(%v)", key, err)
}
}
return
}
// UppersCountCache return uppers count cache
func (d *Dao) UppersCountCache(c context.Context, mids []int64) (cached map[int64]int, missed []int64, err error) {
conn := d.upRds.Get(c)
defer conn.Close()
cached = make(map[int64]int)
for _, mid := range mids {
key := upCntKey(mid)
if err = conn.Send("GET", key); err != nil {
missed = mids
continue
}
}
if err = conn.Flush(); err != nil {
missed = mids
return
}
for _, mid := range mids {
var cnt int
if cnt, err = redis.Int(conn.Receive()); err != nil {
if err == redis.ErrNil {
missed = append(missed, mid)
err = nil
continue
}
}
cached[mid] = cnt
}
return
}
// UpperPassedCache get upper passed archives from cache.
func (d *Dao) UpperPassedCache(c context.Context, mid int64, start, end int) (aids []int64, err error) {
var (
key = upPasKey(mid)
conn = d.upRds.Get(c)
)
defer conn.Close()
if aids, err = redis.Int64s(conn.Do("ZREVRANGE", key, start, end)); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Do(ZRANGE, %s, 0, -1) error(%v)", key, err)
}
return
}
// UppersPassedCacheWithScore get uppers passed archive from cache with score
func (d *Dao) UppersPassedCacheWithScore(c context.Context, mids []int64, start, end int) (aidm map[int64][]*archive.AidPubTime, err error) {
conn := d.upRds.Get(c)
defer conn.Close()
aidm = make(map[int64][]*archive.AidPubTime, len(mids))
for _, mid := range mids {
key := upPasKey(mid)
if err = conn.Send("ZREVRANGE", key, start, end, "WITHSCORES"); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Send(ZREVRANGE, %s) error(%v)", key, err)
return
}
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
for _, mid := range mids {
aidScores, err := redis.Int64s(conn.Receive())
if err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Do(GET, %d) error(%v)", mid, err)
continue
}
for i := 0; i < len(aidScores); i += 2 {
var (
score int64
ptime int64
copyright int8
)
score = aidScores[i+1]
if score > 1000000000 {
ptime = score >> 2
copyright = int8(score & 3)
aidm[mid] = append(aidm[mid], &archive.AidPubTime{Aid: aidScores[i], PubDate: time.Time(ptime), Copyright: copyright})
} else {
aidm[mid] = append(aidm[mid], &archive.AidPubTime{Aid: aidScores[i], PubDate: time.Time(score)})
}
}
}
return
}
// UppersPassedCache get uppers passed archives from cache.
func (d *Dao) UppersPassedCache(c context.Context, mids []int64, start, end int) (aidm map[int64][]int64, err error) {
conn := d.upRds.Get(c)
defer conn.Close()
aidm = make(map[int64][]int64, len(mids))
for _, mid := range mids {
key := upPasKey(mid)
if err = conn.Send("ZREVRANGE", key, start, end); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Send(ZREVRANGE, %s) error(%v)", key, err)
return
}
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
for _, mid := range mids {
aids, err := redis.Int64s(conn.Receive())
if err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Do(GET, %d) error(%v)", mid, err)
continue
}
aidm[mid] = aids
}
return
}
// ExpireUpperPassedCache expire up passed cache.
func (d *Dao) ExpireUpperPassedCache(c context.Context, mid int64) (ok bool, err error) {
var (
key = upPasKey(mid)
conn = d.upRds.Get(c)
)
defer conn.Close()
if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.upExpire)); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", key, d.upExpire, err)
}
return
}
// ExpireUppersCountCache expire ups count cache
func (d *Dao) ExpireUppersCountCache(c context.Context, mids []int64) (cachedUp, missed []int64, err error) {
var conn = d.upRds.Get(c)
defer conn.Close()
defer func() {
if err != nil {
d.errProm.Incr("upper_redis")
}
}()
for _, mid := range mids {
var key = upCntKey(mid)
if err = conn.Send("GET", key); err != nil {
log.Error("conn.Send(GET, %s) error(%v)", key, err)
return
}
}
for _, mid := range mids {
var key = upCntKey(mid)
if err = conn.Send("EXPIRE", key, d.upExpire); err != nil {
log.Error("conn.Send(GET, %s) error(%v)", key, err)
return
}
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
cachedUp = make([]int64, 0)
missed = make([]int64, 0)
for _, mid := range mids {
var cnt int
if cnt, err = redis.Int(conn.Receive()); err != nil {
if err == redis.ErrNil {
err = nil
missed = append(missed, mid)
} else {
log.Error("conn.Receive error(%v)", err)
return
}
} else if cnt > 0 {
cachedUp = append(cachedUp, mid)
}
}
for _, mid := range mids {
if _, err = redis.Bool(conn.Receive()); err != nil {
log.Error("conn.Receive mid(%d) error(%v)", mid, err)
return
}
}
return
}
// ExpireUppersPassedCache expire uppers passed cache.
func (d *Dao) ExpireUppersPassedCache(c context.Context, mids []int64) (res map[int64]bool, err error) {
conn := d.upRds.Get(c)
defer conn.Close()
res = make(map[int64]bool, len(mids))
for _, mid := range mids {
key := upPasKey(mid)
if err = conn.Send("EXPIRE", key, d.upExpire); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Send(%s) error(%v)", key, err)
return
}
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
var ok bool
for _, mid := range mids {
if ok, err = redis.Bool(conn.Receive()); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Receive() error(%v)", err)
return
}
res[mid] = ok
}
return
}
// AddUpperPassedCache add up paassed cache.
func (d *Dao) AddUpperPassedCache(c context.Context, mid int64, aids []int64, ptimes []time.Time, copyrights []int8) (err error) {
var (
key = upPasKey(mid)
conn = d.upRds.Get(c)
)
defer conn.Close()
for k, aid := range aids {
score := int64(ptimes[k]<<2) | int64(copyrights[k])
if err = conn.Send("ZADD", key, score, aid); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Send(ZADD, %s, %d, %d) error(%v)", key, aid, ptimes[k], err)
return
}
}
if err = conn.Flush(); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Flush error(%v)", err)
return
}
for i := 0; i < len(aids); i++ {
if _, err = conn.Receive(); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Receive error(%v)", err)
return
}
}
return
}
// DelUpperPassedCache delete up passed cache.
func (d *Dao) DelUpperPassedCache(c context.Context, mid, aid int64) (err error) {
var (
key = upPasKey(mid)
conn = d.upRds.Get(c)
)
defer conn.Close()
if _, err = conn.Do("ZREM", key, aid); err != nil {
d.errProm.Incr("upper_redis")
log.Error("conn.Do(ZERM, %s, %d) error(%v)", key, aid, err)
}
return
}

196
app/service/main/archive/dao/archive/upper_rds_test.go Normal file
View File

@@ -0,0 +1,196 @@
package archive
import (
"context"
"go-common/library/time"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveupCntKey(t *testing.T) {
var (
mid = int64(1)
)
convey.Convey("upCntKey", t, func(ctx convey.C) {
p1 := upCntKey(mid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchiveupPasKey(t *testing.T) {
var (
mid = int64(1)
)
convey.Convey("upPasKey", t, func(ctx convey.C) {
p1 := upPasKey(mid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchiveAddUpperCountCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
count = int(10)
)
convey.Convey("AddUpperCountCache", t, func(ctx convey.C) {
err := d.AddUpperCountCache(c, mid, count)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveUpperCountCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
)
convey.Convey("UpperCountCache", t, func(ctx convey.C) {
count, err := d.UpperCountCache(c, mid)
ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUppersCountCache(t *testing.T) {
var (
c = context.TODO()
mids = []int64{1, 2}
)
convey.Convey("UppersCountCache", t, func(ctx convey.C) {
cached, missed, err := d.UppersCountCache(c, mids)
ctx.Convey("Then err should be nil.cached,missed should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(missed, convey.ShouldNotBeNil)
ctx.So(cached, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUpperPassedCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
start = int(0)
end = int(10)
)
convey.Convey("UpperPassedCache", t, func(ctx convey.C) {
_, err := d.UpperPassedCache(c, mid, start, end)
ctx.Convey("Then err should be nil.aids should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveUppersPassedCacheWithScore(t *testing.T) {
var (
c = context.TODO()
mids = []int64{1, 2}
start = int(0)
end = int(10)
)
convey.Convey("UppersPassedCacheWithScore", t, func(ctx convey.C) {
aidm, err := d.UppersPassedCacheWithScore(c, mids, start, end)
ctx.Convey("Then err should be nil.aidm should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(aidm, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUppersPassedCache(t *testing.T) {
var (
c = context.TODO()
mids = []int64{1, 2}
start = int(0)
end = int(10)
)
convey.Convey("UppersPassedCache", t, func(ctx convey.C) {
aidm, err := d.UppersPassedCache(c, mids, start, end)
ctx.Convey("Then err should be nil.aidm should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(aidm, convey.ShouldNotBeNil)
})
})
}
func TestArchiveExpireUpperPassedCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
)
convey.Convey("ExpireUpperPassedCache", t, func(ctx convey.C) {
ok, err := d.ExpireUpperPassedCache(c, mid)
ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(ok, convey.ShouldNotBeNil)
})
})
}
func TestArchiveExpireUppersCountCache(t *testing.T) {
var (
c = context.TODO()
mids = []int64{1, 2}
)
convey.Convey("ExpireUppersCountCache", t, func(ctx convey.C) {
cachedUp, missed, err := d.ExpireUppersCountCache(c, mids)
ctx.Convey("Then err should be nil.cachedUp,missed should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(missed, convey.ShouldNotBeNil)
ctx.So(cachedUp, convey.ShouldNotBeNil)
})
})
}
func TestArchiveExpireUppersPassedCache(t *testing.T) {
var (
c = context.TODO()
mids = []int64{1, 2}
)
convey.Convey("ExpireUppersPassedCache", t, func(ctx convey.C) {
res, err := d.ExpireUppersPassedCache(c, mids)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestArchiveAddUpperPassedCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
aids = []int64{1}
ptimes = []time.Time{1531553274}
copyrights = []int8{1}
)
convey.Convey("AddUpperPassedCache", t, func(ctx convey.C) {
err := d.AddUpperPassedCache(c, mid, aids, ptimes, copyrights)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveDelUpperPassedCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
aid = int64(1)
)
convey.Convey("DelUpperPassedCache", t, func(ctx convey.C) {
err := d.DelUpperPassedCache(c, mid, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

78
app/service/main/archive/dao/archive/upper_test.go Normal file
View File

@@ -0,0 +1,78 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchiveUppersCount(t *testing.T) {
var (
c = context.TODO()
mids = []int64{1, 2}
)
convey.Convey("UppersCount", t, func(ctx convey.C) {
uc, err := d.UppersCount(c, mids)
ctx.Convey("Then err should be nil.uc should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(uc, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUpperCount(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
)
convey.Convey("UpperCount", t, func(ctx convey.C) {
count, err := d.UpperCount(c, mid)
ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUpperPassed(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
)
convey.Convey("UpperPassed", t, func(ctx convey.C) {
_, _, _, err := d.UpperPassed(c, mid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveUppersPassed(t *testing.T) {
var (
c = context.TODO()
mids = []int64{1}
)
convey.Convey("UppersPassed", t, func(ctx convey.C) {
aidm, ptimes, copyrights, err := d.UppersPassed(c, mids)
ctx.Convey("Then err should be nil.aidm,ptimes,copyrights should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(copyrights, convey.ShouldNotBeNil)
ctx.So(ptimes, convey.ShouldNotBeNil)
ctx.So(aidm, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUpperReommend(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("UpperReommend", t, func(ctx convey.C) {
_, err := d.UpperReommend(c, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

162
app/service/main/archive/dao/archive/video.go Normal file
View File

@@ -0,0 +1,162 @@
package archive
import (
"context"
"database/sql"
"fmt"
"go-common/app/service/main/archive/api"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_vdosSQL = "SELECT cid,src_type,index_order,eptitle,duration,filename,weblink,dimensions FROM archive_video WHERE aid=? ORDER BY index_order"
_vdosByAidsSQL = "SELECT aid,cid,src_type,index_order,eptitle,duration,filename,weblink,dimensions FROM archive_video WHERE aid in (%s) ORDER BY index_order"
_vdosByCidsSQL = "SELECT aid,cid,src_type,index_order,eptitle,duration,filename,weblink FROM archive_video WHERE cid IN (%s)"
_vdoSQL = "SELECT cid,src_type,index_order,eptitle,duration,filename,weblink,description,dimensions FROM archive_video WHERE aid=? AND cid=?"
_firstCidSQL = "SELECT cid,src_type,dimensions FROM archive_video WHERE aid=? ORDER BY index_order LIMIT 1"
)
// firstCid get first cid
func (d *Dao) firstCid(c context.Context, aid int64) (cid int64, dimensions string, err error) {
var srcType string
row := d.resultDB.QueryRow(c, _firstCidSQL, aid)
if err = row.Scan(&cid, &srcType, &dimensions); err != nil {
if err == sql.ErrNoRows {
err = nil
} else {
log.Error("row.Scan error(%v)", err)
}
return
}
if srcType != "vupload" {
cid = 0
}
return
}
// videos3 get videos by aid.
func (d *Dao) videos3(c context.Context, aid int64) (ps []*api.Page, err error) {
d.infoProm.Incr("videos3")
rows, err := d.vdosStmt.Query(c, aid)
if err != nil {
d.errProm.Incr("archive_db")
log.Error("d.vdosStmt.Query(%d) error(%v)", aid, err)
return
}
defer rows.Close()
var page = int32(0)
for rows.Next() {
var (
p = &api.Page{}
fn string
dimensions string
)
page++
if err = rows.Scan(&p.Cid, &p.From, &p.Page, &p.Part, &p.Duration, &fn, &p.WebLink, &dimensions); err != nil {
log.Error("rows.Scan error(%v)", err)
d.errProm.Incr("archive_db")
return
}
p.Page = page
if p.From != "vupload" {
p.Vid = fn
}
p.FillDimension(dimensions)
ps = append(ps, p)
}
return
}
// videosByAids3 get videos by aids
func (d *Dao) videosByAids3(c context.Context, aids []int64) (vs map[int64][]*api.Page, err error) {
d.infoProm.Incr("videosByAids3")
rows, err := d.resultDB.Query(c, fmt.Sprintf(_vdosByAidsSQL, xstr.JoinInts(aids)))
if err != nil {
d.errProm.Incr("archive_db")
log.Error("d.resultDB.Query(%s) error(%v)", fmt.Sprintf(_vdosByAidsSQL, xstr.JoinInts(aids)), err)
return
}
vs = make(map[int64][]*api.Page, len(aids))
var pages = make(map[int64]int32, len(aids))
defer rows.Close()
for rows.Next() {
var (
p = &api.Page{}
aid int64
fn string
dimensions string
)
if err = rows.Scan(&aid, &p.Cid, &p.From, &p.Page, &p.Part, &p.Duration, &fn, &p.WebLink, &dimensions); err != nil {
d.errProm.Incr("archive_db")
log.Error("rows.Scan error(%v)", err)
return
}
pages[aid]++
p.Page = pages[aid]
if p.From != "vupload" {
p.Vid = fn
}
p.FillDimension(dimensions)
vs[aid] = append(vs[aid], p)
}
return
}
// VideosByCids get videos by cids.
func (d *Dao) VideosByCids(c context.Context, cids []int64) (vs map[int64]map[int64]*api.Page, err error) {
if len(cids) == 0 {
return
}
d.infoProm.Incr("videosByCids")
rows, err := d.arcReadDB.Query(c, fmt.Sprintf(_vdosByCidsSQL, xstr.JoinInts(cids)))
if err != nil {
log.Error("d.arcReadDB.Query() error(%v)", err)
return
}
defer rows.Close()
vs = make(map[int64]map[int64]*api.Page, len(cids))
for rows.Next() {
var (
aid int64
p = &api.Page{}
fn string
)
if err = rows.Scan(&aid, &p.Cid, &p.From, &p.Page, &p.Part, &p.Duration, &fn, &p.WebLink); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
if p.From != "vupload" {
p.Vid = fn
}
if _, ok := vs[aid]; !ok {
vs[aid] = make(map[int64]*api.Page)
}
vs[aid][int64(p.Cid)] = p
}
return
}
// video3 get video by aid & cid.
func (d *Dao) video3(c context.Context, aid, cid int64) (p *api.Page, err error) {
d.infoProm.Incr("video3")
var fn, dimension string
row := d.resultDB.QueryRow(c, _vdoSQL, aid, cid)
p = &api.Page{}
if err = row.Scan(&p.Cid, &p.From, &p.Page, &p.Part, &p.Duration, &fn, &p.WebLink, &p.Desc, &dimension); err != nil {
if err == sql.ErrNoRows {
p = nil
err = nil
} else {
log.Error("row.Scan error(%v)", err)
d.errProm.Incr("result_db")
}
return
}
if p.From != "vupload" {
p.Vid = fn
}
p.FillDimension(dimension)
return
}

209
app/service/main/archive/dao/archive/video_mc_pb.go Normal file
View File

@@ -0,0 +1,209 @@
package archive
import (
"context"
"strconv"
"go-common/app/service/main/archive/api"
"go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_prefixPBPage = "psb_"
)
func pagePBKey(aid int64) string {
return _prefixPBPage + strconv.FormatInt(aid, 10)
}
func videoPBKey2(aid, cid int64) string {
return _prefixPBPage + strconv.FormatInt(aid, 10) + strconv.FormatInt(cid, 10)
}
type batchVdo3 struct {
cached map[int64][]*api.Page
missed []int64
err error
}
// addPageCache3 add pages cache by aid.
func (d *Dao) addPageCache3(c context.Context, aid int64, ps []*api.Page) (err error) {
var vs = &api.AidVideos{Aid: aid, Pages: ps}
key := pagePBKey(aid)
conn := d.mc.Get(c)
if err = conn.Set(&memcache.Item{Key: key, Object: vs, Expiration: 0, Flags: memcache.FlagProtobuf}); err != nil {
d.errProm.Incr("video_mc")
log.Error("memcache.Set(%s) error(%v)", key, err)
}
conn.Close()
return
}
// addVideoCache3 add video2 cache by aid & cid.
func (d *Dao) addVideoCache3(c context.Context, aid, cid int64, p *api.Page) (err error) {
key := videoPBKey2(aid, cid)
conn := d.mc.Get(c)
if err = conn.Set(&memcache.Item{Key: key, Object: p, Expiration: 0, Flags: memcache.FlagProtobuf}); err != nil {
d.errProm.Incr("video3_mc")
log.Error("conn.Set(%s) error(%v)", key, err)
}
conn.Close()
return
}
// DelVideoCache3 del video2 cache by aid & cid.
func (d *Dao) DelVideoCache3(c context.Context, aid, cid int64) (err error) {
key := videoPBKey2(aid, cid)
conn := d.mc.Get(c)
if err = conn.Delete(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
d.errProm.Incr("video3_mc")
log.Error("conn.Set(%s) error(%v)", key, err)
}
}
conn.Close()
return
}
// pageCache3 get page cache by aid.
func (d *Dao) pageCache3(c context.Context, aid int64) (ps []*api.Page, err error) {
var (
key = pagePBKey(aid)
rp *memcache.Item
conn = d.mc.Get(c)
)
defer conn.Close()
if rp, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
ps = nil
} else {
d.errProm.Incr("video_mc")
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
var vs = new(api.AidVideos)
if err = conn.Scan(rp, vs); err != nil {
log.Error("conn.Scan(%s) error(%v)", rp.Value, err)
ps = nil
return
}
ps = vs.Pages
return
}
// pagesCache3 get pages cache by aids
func (d *Dao) pagesCache3(c context.Context, aids []int64) (cached map[int64][]*api.Page, missed []int64, err error) {
var (
lt = len(aids)
times int
reqCh chan batchVdo3
)
if lt%_multiInterval == 0 {
times = lt / _multiInterval
} else {
times = lt/_multiInterval + 1
}
reqCh = make(chan batchVdo3, times)
for i := 0; i < times; i++ {
var tmps []int64
if i == times-1 {
tmps = aids[i*_multiInterval:]
} else {
tmps = aids[i*_multiInterval : (i+1)*_multiInterval]
}
go d._videoCaches3(c, tmps, reqCh)
}
if times == 1 {
req := <-reqCh
cached = req.cached
missed = req.missed
err = req.err
return
}
cached = make(map[int64][]*api.Page, len(aids))
for i := 0; i < times; i++ {
req := <-reqCh
for k, v := range req.cached {
cached[k] = v
}
missed = append(missed, req.missed...)
}
return
}
// videoCache3 get video cache by aid & cid.
func (d *Dao) videoCache3(c context.Context, aid, cid int64) (p *api.Page, err error) {
var (
key = videoPBKey2(aid, cid)
rp *memcache.Item
conn = d.mc.Get(c)
)
defer conn.Close()
if rp, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
p = nil
} else {
d.errProm.Incr("video2_mc")
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
p = new(api.Page)
if err = conn.Scan(rp, p); err != nil {
log.Error("conn.Scan(%s) error(%v)", rp.Value, err)
p = nil
return
}
return
}
func (d *Dao) _videoCaches3(c context.Context, aids []int64, reqCh chan<- batchVdo3) {
var (
req batchVdo3
keys = make([]string, 0, len(aids))
am = make(map[int64]struct{}, len(aids))
rps map[string]*memcache.Item
err error
conn = d.mc.Get(c)
)
defer conn.Close()
for _, aid := range aids {
keys = append(keys, pagePBKey(aid))
am[aid] = struct{}{}
}
if rps, err = conn.GetMulti(keys); err != nil {
req.missed = aids
req.cached = make(map[int64][]*api.Page)
req.err = err
reqCh <- req
log.Error("conn.Gets error(%v)", err)
d.errProm.Incr("video_mc")
return
}
var (
cached = make(map[int64][]*api.Page, len(aids))
missed []int64
)
for _, rp := range rps {
var v = new(api.AidVideos)
if e := conn.Scan(rp, v); e != nil {
log.Error("conn.Scan(%s) error(%v)", rp.Value, e)
d.errProm.Incr("video_mc")
continue
}
cached[v.Aid] = v.Pages
delete(am, v.Aid)
}
for aid := range am {
missed = append(missed, aid)
}
req.cached = cached
req.missed = missed
reqCh <- req
}

117
app/service/main/archive/dao/archive/video_mc_pb_test.go Normal file
View File

@@ -0,0 +1,117 @@
package archive
import (
"context"
"go-common/app/service/main/archive/api"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchivepagePBKey(t *testing.T) {
var (
aid = int64(1)
)
convey.Convey("pagePBKey", t, func(ctx convey.C) {
p1 := pagePBKey(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivevideoPBKey2(t *testing.T) {
var (
aid = int64(1)
cid = int64(1)
)
convey.Convey("videoPBKey2", t, func(ctx convey.C) {
p1 := videoPBKey2(aid, cid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchiveaddPageCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
ps = []*api.Page{}
)
convey.Convey("addPageCache3", t, func(ctx convey.C) {
err := d.addPageCache3(c, aid, ps)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveaddVideoCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
cid = int64(10097272)
p = &api.Page{}
)
convey.Convey("addVideoCache3", t, func(ctx convey.C) {
err := d.addVideoCache3(c, aid, cid, p)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveDelVideoCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
cid = int64(10097272)
)
convey.Convey("DelVideoCache3", t, func(ctx convey.C) {
err := d.DelVideoCache3(c, aid, cid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivepageCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("pageCache3", t, func(ctx convey.C) {
_, err := d.pageCache3(c, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivepagesCache3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{10097272}
)
convey.Convey("pagesCache3", t, func(ctx convey.C) {
_, _, err := d.pagesCache3(c, aids)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivevideoCache3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
cid = int64(10097272)
)
convey.Convey("videoCache3", t, func(ctx convey.C) {
_, err := d.videoCache3(c, aid, cid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

78
app/service/main/archive/dao/archive/video_test.go Normal file
View File

@@ -0,0 +1,78 @@
package archive
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestArchivefirstCid(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("firstCid", t, func(ctx convey.C) {
cid, dimensions, err := d.firstCid(c, aid)
ctx.Convey("Then err should be nil.cid,dimensions should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(dimensions, convey.ShouldNotBeNil)
ctx.So(cid, convey.ShouldNotBeNil)
})
})
}
func TestArchivevideos3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
)
convey.Convey("videos3", t, func(ctx convey.C) {
_, err := d.videos3(c, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchivevideosByAids3(t *testing.T) {
var (
c = context.TODO()
aids = []int64{10097272}
)
convey.Convey("videosByAids3", t, func(ctx convey.C) {
vs, err := d.videosByAids3(c, aids)
ctx.Convey("Then err should be nil.vs should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(vs, convey.ShouldNotBeNil)
})
})
}
func TestArchiveVideosByCids(t *testing.T) {
var (
c = context.TODO()
cids = []int64{10097272}
)
convey.Convey("VideosByCids", t, func(ctx convey.C) {
vs, err := d.VideosByCids(c, cids)
ctx.Convey("Then err should be nil.vs should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(vs, convey.ShouldNotBeNil)
})
})
}
func TestArchivevideo3(t *testing.T) {
var (
c = context.TODO()
aid = int64(10097272)
cid = int64(10097272)
)
convey.Convey("video3", t, func(ctx convey.C) {
_, err := d.video3(c, aid, cid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

55
app/service/main/archive/dao/share/BUILD Normal file
View File

@@ -0,0 +1,55 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"databus_test.go",
"redis_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"databus.go",
"redis.go",
],
importpath = "go-common/app/service/main/archive/dao/share",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/conf:go_default_library",
"//library/cache/redis:go_default_library",
"//library/log:go_default_library",
"//library/net/ip:go_default_library",
"//library/queue/databus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

48
app/service/main/archive/dao/share/dao.go Normal file
View File

@@ -0,0 +1,48 @@
package share
import (
"context"
"go-common/library/cache/redis"
"go-common/library/queue/databus"
"go-common/app/service/main/archive/conf"
)
// Dao is share dao.
type Dao struct {
c *conf.Config
// redis
rds *redis.Pool
expire int32
// databus
statDbus *databus.Databus
shareDbus *databus.Databus
}
// New new a share dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
c: c,
rds: redis.NewPool(c.Redis.Archive.Config),
expire: 168 * 60 * 60,
statDbus: databus.New(c.StatDatabus),
shareDbus: databus.New(c.ShareDatabus),
}
return d
}
// Close close resource.
func (d *Dao) Close() {
if d.rds != nil {
d.rds.Close()
}
}
// Ping ping success.
func (d *Dao) Ping(c context.Context) (err error) {
conn := d.rds.Get(c)
_, err = conn.Do("SET", "PING", "PONG")
conn.Close()
return
}

33
app/service/main/archive/dao/share/dao_test.go Normal file
View File

@@ -0,0 +1,33 @@
package share
import (
"flag"
"go-common/app/service/main/archive/conf"
"os"
"testing"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.app-svr.archive-service")
flag.Set("conf_token", "Y2LJhIsHx87nJaOBSxuG5TeZoLdBFlrE")
flag.Set("tree_id", "2302")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
m.Run()
os.Exit(0)
}

37
app/service/main/archive/dao/share/databus.go Normal file
View File

@@ -0,0 +1,37 @@
package share
import (
"context"
"strconv"
"time"
"go-common/library/log"
)
// PubStatDatabus pub share count into databus.
func (d *Dao) PubStatDatabus(c context.Context, aid int64, share int) (err error) {
type stat struct {
Type string `json:"type"`
ID int64 `json:"id"`
Count int `json:"count"`
Ts int64 `json:"timestamp"`
}
if err = d.statDbus.Send(c, strconv.FormatInt(aid, 10), &stat{Type: "archive", ID: aid, Count: share, Ts: time.Now().Unix()}); err != nil {
log.Error("d.databus.Send error(%v)", err)
}
return
}
// PubShare pub first share to databus
func (d *Dao) PubShare(c context.Context, aid int64, mid int64, ip string) (err error) {
type share struct {
Event string `json:"event"`
Mid int64 `json:"mid"`
IP string `json:"ip"`
Ts int64 `json:"ts"`
}
if err = d.shareDbus.Send(c, strconv.FormatInt(mid, 10), &share{Event: "share", Mid: mid, IP: ip, Ts: time.Now().Unix()}); err != nil {
log.Error("d.shareDbus.Send error(%v)", err)
}
return
}

37
app/service/main/archive/dao/share/databus_test.go Normal file
View File

@@ -0,0 +1,37 @@
package share
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestSharePubStatDatabus(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
share = int(1)
)
convey.Convey("PubStatDatabus", t, func(ctx convey.C) {
err := d.PubStatDatabus(c, aid, share)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestSharePubShare(t *testing.T) {
var (
c = context.TODO()
aid = int64(1)
mid = int64(1)
ip = "127.0.0.1"
)
convey.Convey("PubShare", t, func(ctx convey.C) {
err := d.PubShare(c, aid, mid, ip)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

75
app/service/main/archive/dao/share/redis.go Normal file
View File

@@ -0,0 +1,75 @@
package share
import (
"context"
"fmt"
"time"
"go-common/library/cache/redis"
"go-common/library/log"
xip "go-common/library/net/ip"
)
const (
_prefixShare = "as"
// the first shared every day
_prefixFirst = "afs"
)
func redisKey(aid int64) string {
return fmt.Sprintf("%s_%d", _prefixShare, aid)
}
func redisFKey(mid int64, date string) string {
return fmt.Sprintf("%s_%s_%d", _prefixFirst, date, mid%10000)
}
// AddShare add a share with mid and ip to redis.
func (d *Dao) AddShare(c context.Context, mid, aid int64, ip string) (ok bool, err error) {
var (
key = redisKey(aid)
value = (mid << 32) | int64(xip.InetAtoN(ip))
conn = d.rds.Get(c)
)
defer conn.Close()
if err = conn.Send("SADD", key, value); err != nil {
log.Error("conn.Send(SADD, %s, %d) error(%v)", key, value, err)
return
}
if err = conn.Send("EXPIRE", key, d.expire); err != nil {
log.Error("conn.Send(EXPIRE, %s, %d) error(%v)", key, value, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
}
if ok, err = redis.Bool(conn.Receive()); err != nil {
log.Error("conn.Receive error(%v)", err)
}
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
}
return
}
// HadFirstShare if key not exist need set expire to one day.
func (d *Dao) HadFirstShare(c context.Context, mid, aid int64, ip string) (had bool, err error) {
var (
date = time.Now().Format("0102")
key = redisFKey(mid, date)
addOk bool
conn = d.rds.Get(c)
)
defer conn.Close()
if addOk, err = redis.Bool(conn.Do("SADD", key, mid)); err != nil {
log.Error("conn.Send(SADD, %s, %d) error(%v)", key, mid, err)
return
}
if addOk {
if _, err = conn.Do("EXPIRE", key, 24*60*60); err != nil {
log.Error("conn.Send(EXPIRE, %s, 24*60*60) error(%v)", key, err)
}
}
had = !addOk
return
}

65
app/service/main/archive/dao/share/redis_test.go Normal file
View File

@@ -0,0 +1,65 @@
package share
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestShareredisKey(t *testing.T) {
var (
aid = int64(1)
)
convey.Convey("redisKey", t, func(ctx convey.C) {
p1 := redisKey(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestShareredisFKey(t *testing.T) {
var (
mid = int64(1)
date = ""
)
convey.Convey("redisFKey", t, func(ctx convey.C) {
p1 := redisFKey(mid, date)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestShareAddShare(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
aid = int64(1)
ip = ""
)
convey.Convey("AddShare", t, func(ctx convey.C) {
ok, err := d.AddShare(c, mid, aid, ip)
ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(ok, convey.ShouldNotBeNil)
})
})
}
func TestShareHadFirstShare(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
aid = int64(1)
ip = ""
)
convey.Convey("HadFirstShare", t, func(ctx convey.C) {
had, err := d.HadFirstShare(c, mid, aid, ip)
ctx.Convey("Then err should be nil.had should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(had, convey.ShouldNotBeNil)
})
})
}

56
app/service/main/archive/dao/videoshot/BUILD Normal file
View File

@@ -0,0 +1,56 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"mysql_test.go",
"redis_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"mysql.go",
"redis.go",
],
importpath = "go-common/app/service/main/archive/dao/videoshot",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/conf:go_default_library",
"//app/service/main/archive/model/videoshot:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/stat/prom:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

106
app/service/main/archive/dao/videoshot/dao.go Normal file
View File

@@ -0,0 +1,106 @@
package videoshot
import (
"context"
"go-common/library/cache/redis"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/stat/prom"
"go-common/app/service/main/archive/conf"
"go-common/app/service/main/archive/model/videoshot"
)
// Dao is videoshot dao.
type Dao struct {
// mysql
db *sql.DB
dbRead *sql.DB
getStmt *sql.Stmt
inStmt *sql.Stmt
// redis
rds *redis.Pool
// prom
infoProm *prom.Prom
// chan
cacheCh chan func()
}
// New new a videoshot dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
db: sql.NewMySQL(c.DB.Arc),
dbRead: sql.NewMySQL(c.DB.ArcRead),
rds: redis.NewPool(c.Redis.Archive.Config),
cacheCh: make(chan func(), 1024),
}
d.getStmt = d.dbRead.Prepared(_getSQL)
d.inStmt = d.db.Prepared(_inSQL)
d.infoProm = prom.BusinessInfoCount
go d.cacheproc()
return d
}
// Videoshot get videoshot.
func (d *Dao) Videoshot(c context.Context, cid int64) (v *videoshot.Videoshot, err error) {
var count, ver int
if count, ver, err = d.cache(c, cid); err != nil {
log.Error("d.cache(%d) error(%v)", cid, err)
err = nil // NOTE: ignore error use db
}
if count != 0 {
v = &videoshot.Videoshot{Cid: cid, Count: count}
v.SetVersion(ver)
return
}
if v, err = d.videoshot(c, cid); err != nil || v == nil {
log.Warn("d.videoshot(%d) error(%v) or v==nil", cid, err)
return
}
d.cacheCh <- func() {
d.addCache(context.TODO(), v.Cid, v.Version(), v.Count)
}
return
}
// AddVideoshot add videoshot.
func (d *Dao) AddVideoshot(c context.Context, v *videoshot.Videoshot) (err error) {
if _, err = d.addVideoshot(c, v); err != nil {
log.Error("d.addVideoshot(%v) error(%v)", v, err)
return
}
d.cacheCh <- func() {
d.addCache(context.TODO(), v.Cid, v.Version(), v.Count)
}
return
}
// Close close resource.
func (d *Dao) Close() {
if d.rds != nil {
d.rds.Close()
}
if d.db != nil {
d.db.Close()
}
close(d.cacheCh)
}
// Ping ping success.
func (d *Dao) Ping(c context.Context) (err error) {
conn := d.rds.Get(c)
_, err = conn.Do("SET", "PING", "PONG")
conn.Close()
return
}
func (d *Dao) cacheproc() {
for {
f, ok := <-d.cacheCh
if !ok {
return
}
f()
}
}

33
app/service/main/archive/dao/videoshot/dao_test.go Normal file
View File

@@ -0,0 +1,33 @@
package videoshot
import (
"flag"
"go-common/app/service/main/archive/conf"
"os"
"testing"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.app-svr.archive-service")
flag.Set("conf_token", "Y2LJhIsHx87nJaOBSxuG5TeZoLdBFlrE")
flag.Set("tree_id", "2302")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
m.Run()
os.Exit(0)
}

41
app/service/main/archive/dao/videoshot/mysql.go Normal file
View File

@@ -0,0 +1,41 @@
package videoshot
import (
"context"
"database/sql"
"go-common/app/service/main/archive/model/videoshot"
"go-common/library/log"
)
const (
_inSQL = "INSERT INTO archive_video_shot (id,count,ctime,mtime) VALUES (?,?,?,?) ON DUPLICATE KEY UPDATE count=?,mtime=? "
_getSQL = "SELECT id,count,ctime,mtime FROM archive_video_shot WHERE id=?"
)
// videoshot get a videoshot by id.
func (d *Dao) videoshot(c context.Context, cid int64) (shot *videoshot.Videoshot, err error) {
d.infoProm.Incr("videoshot")
row := d.getStmt.QueryRow(c, cid)
shot = &videoshot.Videoshot{}
if err = row.Scan(&shot.Cid, &shot.Count, &shot.CTime, &shot.MTime); err != nil {
if err == sql.ErrNoRows {
shot = nil
err = nil
} else {
log.Error("row.Scan() error(%v)", err)
}
}
return
}
// addVideoshot add a videoshot into mysql.
func (d *Dao) addVideoshot(c context.Context, shot *videoshot.Videoshot) (cid int64, err error) {
res, err := d.inStmt.Exec(c, shot.Cid, shot.Count, shot.CTime, shot.MTime, shot.Count, shot.MTime)
if err != nil {
log.Error("inStmt.Exec error(%v)", err)
return
}
cid, err = res.LastInsertId()
return
}

21
app/service/main/archive/dao/videoshot/mysql_test.go Normal file
View File

@@ -0,0 +1,21 @@
package videoshot
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestVideoshotvideoshot(t *testing.T) {
var (
c = context.TODO()
cid = int64(10097272)
)
convey.Convey("videoshot", t, func(ctx convey.C) {
_, err := d.videoshot(c, cid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

55
app/service/main/archive/dao/videoshot/redis.go Normal file
View File

@@ -0,0 +1,55 @@
package videoshot
import (
"context"
"fmt"
"go-common/library/cache/redis"
"go-common/library/log"
)
const (
// _hashKeyNum the max count of hash key
_hashKeyNum = int64(100000)
_keyPrefix = "vs_"
)
func hashKey(cid int64) string {
return fmt.Sprintf("%s%d", _keyPrefix, cid%_hashKeyNum)
}
// cache get videoshot's count by id.
func (d *Dao) cache(c context.Context, cid int64) (count, ver int, err error) {
var (
key = hashKey(cid)
conn = d.rds.Get(c)
out int64
)
defer conn.Close()
if out, err = redis.Int64(conn.Do("HGET", key, cid)); err != nil {
if err == redis.ErrNil {
err = nil
} else {
log.Error("HGET(%s, %d) error(%v)", key, cid, err)
}
return
}
ver = int(out >> 32)
count = int(int32(out))
return
}
// addCache set videoshot info into redis.
func (d *Dao) addCache(c context.Context, cid int64, ver, count int) (err error) {
var (
key = hashKey(cid)
conn = d.rds.Get(c)
in int64
)
in = int64(ver)<<32 | int64(count)
defer conn.Close()
if _, err = conn.Do("HSET", key, cid, in); err != nil {
log.Error("HSET(%s, %d, %d)", key, cid, count, err)
}
return
}

50
app/service/main/archive/dao/videoshot/redis_test.go Normal file
View File

@@ -0,0 +1,50 @@
package videoshot
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestVideoshothashKey(t *testing.T) {
var (
cid = int64(1)
)
convey.Convey("hashKey", t, func(ctx convey.C) {
p1 := hashKey(cid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestVideoshotcache(t *testing.T) {
var (
c = context.TODO()
cid = int64(1)
)
convey.Convey("cache", t, func(ctx convey.C) {
count, ver, err := d.cache(c, cid)
ctx.Convey("Then err should be nil.count,ver should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(ver, convey.ShouldNotBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestVideoshotaddCache(t *testing.T) {
var (
c = context.TODO()
cid = int64(1)
ver = int(0)
count = int(0)
)
convey.Convey("addCache", t, func(ctx convey.C) {
err := d.addCache(c, cid, ver, count)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

17
app/service/main/archive/model/BUILD Normal file
View File

@@ -0,0 +1,17 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/archive/model/archive:all-srcs",
"//app/service/main/archive/model/videoshot:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

69
app/service/main/archive/model/archive/BUILD Normal file
View File

@@ -0,0 +1,69 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
go_library(
name = "go_default_library",
srcs = [
"addit.go",
"archive.go",
"pgc.go",
"report_result.go",
"rpc.go",
"rpc2.go",
],
embed = [":archive_go_proto"],
importpath = "go-common/app/service/main/archive/model/archive",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//library/time:go_default_library",
"@com_github_gogo_protobuf//gogoproto:go_default_library",
"@com_github_golang_protobuf//proto:go_default_library",
],
)
proto_library(
name = "archive_proto",
srcs = [
"archive.proto",
"bvc.proto",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
)
go_proto_library(
name = "archive_go_proto",
compilers = ["@io_bazel_rules_go//proto:gogofast_proto"],
importpath = "go-common/app/service/main/archive/model/archive",
proto = ":archive_proto",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/time:go_default_library",
"@com_github_gogo_protobuf//gogoproto:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

13
app/service/main/archive/model/archive/addit.go Normal file
View File

@@ -0,0 +1,13 @@
package archive
// Addit id,aid,source,redirect_url,description
type Addit struct {
ID int64
Aid int64
Source string
RedirectURL string
MissionID int64
UpFrom int8
OrderID int
Description string
}

328
app/service/main/archive/model/archive/archive.go Normal file
View File

@@ -0,0 +1,328 @@
package archive
import (
"go-common/app/service/main/archive/api"
"go-common/library/time"
)
// 各属性地址见 http://syncsvn.bilibili.co/platform/doc/blob/master/archive/field/state.md
// all const
const (
// open state
StateOpen = 0
StateOrange = 1
AccessMember = int32(10000)
// forbid state
StateForbidWait = -1
StateForbidRecicle = -2
StateForbidPolice = -3
StateForbidLock = -4
StateForbidFixed = -6
StateForbidLater = -7
StateForbidAdminDelay = -10
StateForbidXcodeFail = -16
StateForbidSubmit = -30
StateForbidUserDelay = -40
StateForbidUpDelete = -100
// copyright
CopyrightUnknow = int8(0)
CopyrightOriginal = int8(1)
CopyrightCopy = int8(2)
// attribute yes and no
AttrYes = int32(1)
AttrNo = int32(0)
// attribute bit
AttrBitNoRank = uint(0)
AttrBitNoDynamic = uint(1)
AttrBitNoWeb = uint(2)
AttrBitNoMobile = uint(3)
// AttrBitNoSearch = uint(4)
AttrBitOverseaLock = uint(5)
AttrBitNoRecommend = uint(6)
AttrBitNoReprint = uint(7)
AttrBitHasHD5 = uint(8)
AttrBitIsPGC = uint(9)
AttrBitAllowBp = uint(10)
AttrBitIsBangumi = uint(11)
AttrBitIsPorder = uint(12)
AttrBitLimitArea = uint(13)
AttrBitAllowTag = uint(14)
// AttrBitIsFromArcApi = uint(15)
AttrBitJumpUrl = uint(16)
AttrBitIsMovie = uint(17)
AttrBitBadgepay = uint(18)
AttrBitUGCPay = uint(22)
AttrBitHasBGM = uint(23)
AttrBitIsCooperation = uint(24)
AttrBitHasViewpoint = uint(25)
AttrBitHasArgument = uint(26)
)
var (
_emptyTags = []string{}
)
// AidPubTime aid's pubdate and copyright
type AidPubTime struct {
Aid int64 `json:"aid"`
PubDate time.Time `json:"pubdate"`
Copyright int8 `json:"copyright"`
}
// ArchiveWithPlayer with first player info
type ArchiveWithPlayer struct {
*Archive3
PlayerInfo *PlayerInfo `json:"player_info,omitempty"`
}
// ArchiveWithBvc with first player info
type ArchiveWithBvc struct {
*Archive3
PlayerInfo *BvcVideoItem `json:"player_info,omitempty"`
}
// PlayerInfo player info
type PlayerInfo struct {
Cid int64 `json:"cid"`
ExpireTime int64 `json:"expire_time,omitempty"`
FileInfo map[int][]*PlayerFileInfo `json:"file_info"`
SupportQuality []int `json:"support_quality"`
SupportFormats []string `json:"support_formats"`
SupportDescription []string `json:"support_description"`
Quality int `json:"quality"`
URL string `json:"url,omitempty"`
VideoCodecid uint32 `json:"video_codecid"`
VideoProject bool `json:"video_project"`
Fnver int `json:"fnver"`
Fnval int `json:"fnval"`
Dash *ResponseDash `json:"dash,omitempty"`
}
// PlayerFileInfo is
type PlayerFileInfo struct {
TimeLength int64 `json:"timelength"`
FileSize int64 `json:"filesize"`
Ahead string `json:"ahead,omitempty"`
Vhead string `json:"vhead,omitempty"`
URL string `json:"url,omitempty"`
Order int64 `json:"order,omitempty"`
}
// ArcType arctype
type ArcType struct {
ID int16 `json:"id"`
Pid int16 `json:"pid"`
Name string `json:"name"`
}
// Videoshot videoshot
type Videoshot struct {
PvData string `json:"pvdata"`
XLen int `json:"img_x_len"`
YLen int `json:"img_y_len"`
XSize int `json:"img_x_size"`
YSize int `json:"img_y_size"`
Image []string `json:"image"`
Attr int32 `json:"-"`
}
// RankArchives3 RankArchives3
type RankArchives3 struct {
Archives []*api.Arc `json:"archives"`
Count int `json:"count"`
}
// LikedArchives3 LikedArchives3
type LikedArchives3 struct {
Archives []*Archive3 `json:"archives"`
Count int64
}
// RegionArc RegionArc
type RegionArc struct {
Aid int64
Attribute int32
Copyright int8
PubDate time.Time
}
// IsNormal is
func (a *Archive3) IsNormal() bool {
return a.State >= StateOpen || a.State == StateForbidFixed
}
// AttrVal get attr val by bit.
func (a *Archive3) AttrVal(bit uint) int32 {
return (a.Attribute >> bit) & int32(1)
}
func (ra *RegionArc) attrVal(bit uint) int32 {
return (ra.Attribute >> bit) & int32(1)
}
type PGCPlayer struct {
PlayerInfo *PlayerInfo `json:"player_info"`
Aid int64 `json:"aid"`
}
// BuildArchive3 build Archive3 with new proto
func BuildArchive3(a *api.Arc) (arc *Archive3) {
if a == nil {
return nil
}
arc = &Archive3{
Aid: a.Aid,
Videos: a.Videos,
TypeID: a.TypeID,
TypeName: a.TypeName,
Copyright: a.Copyright,
Pic: a.Pic,
Title: a.Title,
PubDate: a.PubDate,
Ctime: a.Ctime,
Desc: a.Desc,
State: a.State,
Access: a.Access,
Attribute: a.Attribute,
Duration: a.Duration,
MissionID: a.MissionID,
OrderID: a.OrderID,
RedirectURL: a.RedirectURL,
Forward: a.Forward,
Rights: Rights3{
Bp: a.Rights.Bp,
Elec: a.Rights.Elec,
Download: a.Rights.Download,
Movie: a.Rights.Movie,
Pay: a.Rights.Pay,
HD5: a.Rights.HD5,
NoReprint: a.Rights.NoReprint,
Autoplay: a.Rights.Autoplay,
UGCPay: a.Rights.UGCPay,
IsCooperation: a.Rights.IsCooperation,
},
Author: Author3{
Mid: a.Author.Mid,
Name: a.Author.Name,
Face: a.Author.Face,
},
Stat: Stat3{
Aid: a.Stat.Aid,
View: a.Stat.View,
Danmaku: a.Stat.Danmaku,
Reply: a.Stat.Reply,
Fav: a.Stat.Fav,
Coin: a.Stat.Coin,
Share: a.Stat.Share,
NowRank: a.Stat.NowRank,
HisRank: a.Stat.HisRank,
Like: a.Stat.Like,
DisLike: a.Stat.DisLike,
},
ReportResult: a.ReportResult,
Dynamic: a.Dynamic,
FirstCid: a.FirstCid,
Dimension: Dimension{
Width: a.Dimension.Width,
Height: a.Dimension.Height,
Rotate: a.Dimension.Rotate,
},
}
return
}
// BuildView3 build View3 with new proto.
func BuildView3(a *api.Arc, pages []*api.Page) (v *View3) {
if a == nil {
return nil
}
var arc = &Archive3{
Aid: a.Aid,
Videos: a.Videos,
TypeID: a.TypeID,
TypeName: a.TypeName,
Copyright: a.Copyright,
Pic: a.Pic,
Title: a.Title,
PubDate: a.PubDate,
Ctime: a.Ctime,
Desc: a.Desc,
State: a.State,
Access: a.Access,
Attribute: a.Attribute,
Duration: a.Duration,
MissionID: a.MissionID,
OrderID: a.OrderID,
RedirectURL: a.RedirectURL,
Forward: a.Forward,
Rights: Rights3{
Bp: a.Rights.Bp,
Elec: a.Rights.Elec,
Download: a.Rights.Download,
Movie: a.Rights.Movie,
Pay: a.Rights.Pay,
HD5: a.Rights.HD5,
NoReprint: a.Rights.NoReprint,
Autoplay: a.Rights.Autoplay,
UGCPay: a.Rights.UGCPay,
IsCooperation: a.Rights.IsCooperation,
},
Author: Author3{
Mid: a.Author.Mid,
Name: a.Author.Name,
Face: a.Author.Face,
},
Stat: Stat3{
Aid: a.Stat.Aid,
View: a.Stat.View,
Danmaku: a.Stat.Danmaku,
Reply: a.Stat.Reply,
Fav: a.Stat.Fav,
Coin: a.Stat.Coin,
Share: a.Stat.Share,
NowRank: a.Stat.NowRank,
HisRank: a.Stat.HisRank,
Like: a.Stat.Like,
DisLike: a.Stat.DisLike,
},
ReportResult: a.ReportResult,
Dynamic: a.Dynamic,
FirstCid: a.FirstCid,
Dimension: Dimension{
Width: a.Dimension.Width,
Height: a.Dimension.Height,
Rotate: a.Dimension.Rotate,
},
}
for _, s := range a.StaffInfo {
arc.StaffInfo = append(arc.StaffInfo, &StaffInfo{
Mid: s.Mid,
Title: s.Title,
})
}
var viewPages []*Page3
for _, v := range pages {
viewPages = append(viewPages, &Page3{
Cid: v.Cid,
Page: v.Page,
From: v.From,
Part: v.Part,
Duration: v.Duration,
Vid: v.Vid,
Desc: v.Desc,
WebLink: v.WebLink,
Dimension: Dimension{
Width: v.Dimension.Width,
Height: v.Dimension.Height,
Rotate: v.Dimension.Rotate,
},
})
}
v = &View3{
Archive3: arc,
Pages: viewPages,
}
return
}

3648
app/service/main/archive/model/archive/archive.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

115
app/service/main/archive/model/archive/archive.proto Normal file
View File

@@ -0,0 +1,115 @@
syntax = "proto3";
package archive;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_enum_prefix_all) = false;
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.unmarshaler_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.sizer_all) = true;
message Archive3 {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
int64 Videos = 2 [(gogoproto.jsontag) = "videos"];
int32 TypeID = 3 [(gogoproto.jsontag) = "tid"];
string TypeName = 4 [(gogoproto.jsontag) = "tname"];
int32 Copyright = 5 [(gogoproto.jsontag) = "copyright"];
string Pic = 6 [(gogoproto.jsontag) = "pic"];
string Title = 7 [(gogoproto.jsontag) = "title"];
int64 PubDate = 8 [(gogoproto.jsontag) = "pubdate", (gogoproto.casttype) = "go-common/library/time.Time"];
int64 Ctime = 9 [(gogoproto.jsontag) = "ctime", (gogoproto.casttype) = "go-common/library/time.Time"];
string Desc = 10 [(gogoproto.jsontag) = "desc"];
int32 State = 11 [(gogoproto.jsontag) = "state"];
int32 Access = 12 [(gogoproto.jsontag) = "access,omitempty"];
int32 Attribute = 13 [(gogoproto.jsontag) = "attribute,omitempty"];
string Tag = 14 [(gogoproto.jsontag) = "-"];
repeated string Tags = 15 [(gogoproto.jsontag) = "tags,omitempty"];
int64 Duration = 16 [(gogoproto.jsontag) = "duration"];
int64 MissionID = 17 [(gogoproto.jsontag) = "mission_id,omitempty"];
int64 OrderID = 18 [(gogoproto.jsontag) = "order_id,omitempty"];
string RedirectURL = 19 [(gogoproto.jsontag) = "redirect_url,omitempty"];
int64 Forward = 20 [(gogoproto.jsontag) = "forward,omitempty"];
Rights3 Rights = 21 [(gogoproto.jsontag) = "rights", (gogoproto.nullable) = false];
Author3 Author = 22 [(gogoproto.jsontag) = "owner", (gogoproto.nullable) = false];
Stat3 Stat = 23 [(gogoproto.jsontag) = "stat", (gogoproto.nullable) = false];
string ReportResult = 24 [(gogoproto.jsontag) = "report_result,omitempty"];
string Dynamic = 25 [(gogoproto.jsontag) = "dynamic"];
int64 FirstCid = 26 [(gogoproto.jsontag) = "cid,omitempty"];
Dimension Dimension = 27 [(gogoproto.jsontag) = "dimension,omitempty", (gogoproto.nullable) = false];
repeated StaffInfo StaffInfo = 28 [(gogoproto.jsontag) = "-"];
}
message StaffInfo {
int64 Mid = 1 [(gogoproto.jsontag) = "mid"];
string Title = 2 [(gogoproto.jsontag) = "title"];
}
message Dimension {
int64 Width = 1 [(gogoproto.jsontag) = "width"];
int64 Height = 2 [(gogoproto.jsontag) = "height"];
int64 Rotate = 3 [(gogoproto.jsontag) = "rotate"];
}
message Rights3 {
int32 Bp = 1 [(gogoproto.jsontag) = "bp"];
int32 Elec = 2 [(gogoproto.jsontag) = "elec"];
int32 Download = 3 [(gogoproto.jsontag) = "download"];
int32 Movie = 4 [(gogoproto.jsontag) = "movie"];
int32 Pay = 5 [(gogoproto.jsontag) = "pay"];
int32 HD5 = 6 [(gogoproto.jsontag) = "hd5"];
int32 NoReprint = 7 [(gogoproto.jsontag) = "no_reprint"];
int32 Autoplay = 8 [(gogoproto.jsontag) = "autoplay"];
int32 UGCPay = 9 [(gogoproto.jsontag) = "ugc_pay"];
int32 IsCooperation = 10 [(gogoproto.jsontag) = "is_cooperation"];
}
message Author3 {
int64 Mid = 1 [(gogoproto.jsontag) = "mid"];
string Name = 2 [(gogoproto.jsontag) = "name"];
string Face = 3 [(gogoproto.jsontag) = "face"];
}
message Stat3 {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
int32 View = 2 [(gogoproto.jsontag) = "view"];
int32 Danmaku = 3 [(gogoproto.jsontag) = "danmaku"];
int32 Reply = 4 [(gogoproto.jsontag) = "reply"];
int32 Fav = 5 [(gogoproto.jsontag) = "favorite"];
int32 Coin = 6 [(gogoproto.jsontag) = "coin"];
int32 Share = 7 [(gogoproto.jsontag) = "share"];
int32 NowRank = 8 [(gogoproto.jsontag) = "now_rank"];
int32 HisRank = 9 [(gogoproto.jsontag) = "his_rank"];
int32 Like = 10 [(gogoproto.jsontag) = "like"];
int32 DisLike = 11 [(gogoproto.jsontag) = "dislike"];
}
message Click3 {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
int32 Web = 2 [(gogoproto.jsontag) = "web"];
int32 Ios = 3 [(gogoproto.jsontag) = "ios"];
int32 Android = 4 [(gogoproto.jsontag) = "android"];
int32 H5 = 5 [(gogoproto.jsontag) = "h5"];
int32 Outter = 6 [(gogoproto.jsontag) = "outter"];
}
message AidVideos3 {
int64 Aid = 1 [(gogoproto.jsontag) = "aid"];
repeated Page3 Pages = 2 [(gogoproto.jsontag) = "pages"];
}
message View3 {
Archive3 Archive = 1 [(gogoproto.embed) = true, (gogoproto.jsontag) = ""];
repeated Page3 Pages = 2 [(gogoproto.jsontag) = "pages,omitempty"];
}
message Page3 {
int64 Cid = 1 [(gogoproto.jsontag) = "cid"];
int32 Page = 2 [(gogoproto.jsontag) = "page"];
string From = 3 [(gogoproto.jsontag) = "from"];
string Part = 4 [(gogoproto.jsontag) = "part"];
int64 Duration = 5 [(gogoproto.jsontag) = "duration"];
string Vid = 6 [(gogoproto.jsontag) = "vid"];
string Desc = 7 [(gogoproto.jsontag) = "description,omitempty"];
string WebLink = 8 [(gogoproto.jsontag) = "weblink"];
Dimension Dimension = 9 [(gogoproto.jsontag) = "dimension", (gogoproto.nullable) = false];
}

1944
app/service/main/archive/model/archive/bvc.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

54
app/service/main/archive/model/archive/bvc.proto Normal file
View File

@@ -0,0 +1,54 @@
syntax = "proto3";
package archive;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_enum_prefix_all) = false;
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.unmarshaler_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.sizer_all) = true;
// !!!Note: only response message defined, since we put pb into http response body
message VideoFileInfo {
uint64 filesize = 1 [(gogoproto.jsontag) = "filesize", (gogoproto.casttype) = "int64"];
uint64 timelength = 2 [(gogoproto.jsontag) = "timelength", (gogoproto.casttype) ="int64"];
string ahead = 3 [(gogoproto.jsontag) = "ahead"];
string vhead = 4 [(gogoproto.jsontag) = "vhead"];
}
message VideoFormatFileInfo {
repeated VideoFileInfo infos = 1 [(gogoproto.jsontag) = "infos"];
}
message DashItem {
uint32 id = 1 [(gogoproto.jsontag) = "id", (gogoproto.casttype) = "int64"];
string baseUrl = 2 [(gogoproto.jsontag) = "base_url"];
uint32 bandwidth = 3 [(gogoproto.jsontag) = "bandwidth", (gogoproto.casttype) = "int64"];
uint32 codecid = 4 [(gogoproto.jsontag) = "codecid", (gogoproto.casttype) = "int64"];
}
message ResponseDash {
repeated DashItem video = 1 [(gogoproto.jsontag) = "video"];
repeated DashItem audio = 2 [(gogoproto.jsontag) = "audio"];
}
message BvcVideoItem {
uint64 expire_time = 1 [(gogoproto.jsontag) = "expire_time", (gogoproto.casttype) = "int64"];
uint32 cid = 2 [(gogoproto.jsontag) = "cid", (gogoproto.casttype) ="int64"];
repeated uint32 support_quality = 3 [(gogoproto.jsontag) = "support_quality", (gogoproto.casttype) = "int"];
repeated string support_formats = 4 [(gogoproto.jsontag) = "support_formats"];
repeated string support_description = 5 [(gogoproto.jsontag) = "support_description"];
uint32 quality = 6 [(gogoproto.jsontag) = "quality", (gogoproto.casttype) = "int"];
string url = 7 [(gogoproto.jsontag) = "url"];
map<uint32, VideoFormatFileInfo> file_info = 8 [(gogoproto.jsontag) = "file_info"];
uint32 video_codecid = 9 [(gogoproto.jsontag) = "video_codecid"];
bool video_project = 10 [(gogoproto.jsontag) = "video_project"];
uint32 fnver = 11 [(gogoproto.jsontag) = "fnver", (gogoproto.casttype) = "int"];
uint32 fnval = 12 [(gogoproto.jsontag) = "fnval", (gogoproto.casttype) = "int"];
ResponseDash dash = 13 [(gogoproto.jsontag) = "dash"];
}
message BvcResponseMsg {
uint32 code = 1 [(gogoproto.jsontag) = "code", (gogoproto.casttype) = "int64"];
map<uint32, BvcVideoItem> data = 2 [(gogoproto.jsontag) = "data"];
}

7
app/service/main/archive/model/archive/pgc.go Normal file
View File

@@ -0,0 +1,7 @@
package archive
// type of archive. season or move.
const (
TypeSeason = 1
TypeMovie = 2
)

10
app/service/main/archive/model/archive/report_result.go Normal file
View File

@@ -0,0 +1,10 @@
package archive
// ReportResult ReportResult
type ReportResult struct {
Aid int64
Result string
State int
AdminID int64
AfterDeal int
}

63
app/service/main/archive/model/archive/rpc.go Normal file
View File

@@ -0,0 +1,63 @@
package archive
// ArgAid ArgAid
type ArgAid struct {
Aid int64
RealIP string
}
// ArgCid ArgCid
type ArgCid struct {
Cid int64
RealIP string
}
// ArgAids ArgAids
type ArgAids struct {
Aids []int64
RealIP string
}
// ArgPage ArgPage
type ArgPage struct {
Aid int64
Mid int64
AccessKey string
RealIP string
}
// ArgVideoshot ArgVideoshot
type ArgVideoshot struct {
Cid int64
Count int
RealIP string
}
// ArgStat ArgStat
type ArgStat struct {
Aid int64
Field int
Value int
RealIP string
}
// ArgTag ArgTag
type ArgTag struct {
Aid int64
Tag string
RealIP string
}
// ArgPlayer ArgPlayer
type ArgPlayer struct {
Aids []int64
Qn int
Platform string
RealIP string
Fnval int
Fnver int
Build int
// 非必传
Session string
ForceHost int
}

151
app/service/main/archive/model/archive/rpc2.go Normal file
View File

@@ -0,0 +1,151 @@
package archive
// ArgAid2 ArgAid2
type ArgAid2 struct {
Aid int64
RealIP string
}
// ArgCid2 ArgCid2
type ArgCid2 struct {
Aid int64
Cid int64
RealIP string
}
// ArgVideo2 ArgVideo2
type ArgVideo2 struct {
Aid, Cid int64
RealIP string
}
// ArgAids2 ArgAids2
type ArgAids2 struct {
Aids []int64
RealIP string
}
// ArgPage2 ArgPage2
type ArgPage2 struct {
Aid int64
Mid int64
AccessKey string
RealIP string
}
// ArgVideoshot2 ArgVideoshot2
type ArgVideoshot2 struct {
Cid int64
Count int
RealIP string
}
// ArgStat2 ArgStat2
type ArgStat2 struct {
Aid int64
Field int
Value int
RealIP string
}
// ArgAidMid2 ArgAidMid2
type ArgAidMid2 struct {
Aid int64
Mid int64
RealIP string
}
// ArgUpArcs2 ArgUpArcs2
type ArgUpArcs2 struct {
Mid int64
Pn, Ps int
RealIP string
}
// ArgUpCount2 ArgUpCount2
type ArgUpCount2 struct {
Mid int64
}
// ArgUpsArcs2 ArgUpsArcs2
type ArgUpsArcs2 struct {
Mids []int64
Pn, Ps int
RealIP string
}
// ArgMovie2 ArgMovie2
type ArgMovie2 struct {
MovieId int64
RealIP string
}
// ArgMid2 ArgMid2
type ArgMid2 struct {
Mid int64
RealIP string
}
// ArgRank2 ArgRank2
type ArgRank2 struct {
Rid int16
Type int8
Pn, Ps int
RealIP string
}
// ArgRanks2 ArgRanks2
type ArgRanks2 struct {
Rids []int16
Type int8
Pn, Ps int
RealIP string
}
// ArgRankTop2 ArgRankTop2
type ArgRankTop2 struct {
ReID int16
Pn, Ps int
}
// ArgRankAll2 ArgRankAll2
type ArgRankAll2 struct {
Pn, Ps int
}
// ArgRankTopsCount2 ArgRankTopsCount2
type ArgRankTopsCount2 struct {
ReIDs []int16
}
// ArgCIDs2 ArgCIDs2
type ArgCIDs2 struct {
Cids []int64
}
// ArgEp2 ArgEp2
type ArgEp2 struct {
EpIDs []int64
Tp int8
}
// const action type
const (
CacheAdd = "add"
CacheUpdate = "update"
CacheDelete = "delete"
)
// ArgCache2 ArgCache2
type ArgCache2 struct {
Aid int64
Tp string
OldMid int64
}
// ArgFieldCache2 ArgFieldCache2
type ArgFieldCache2 struct {
Aid int64
TypeID int16
OldTypeID int16
}

28
app/service/main/archive/model/videoshot/BUILD Normal file
View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["videoshot.go"],
importpath = "go-common/app/service/main/archive/model/videoshot",
tags = ["automanaged"],
deps = ["//library/time:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

33
app/service/main/archive/model/videoshot/videoshot.go Normal file
View File

@@ -0,0 +1,33 @@
package videoshot
import (
"time"
xtime "go-common/library/time"
)
var (
_verDate = time.Date(2015, 6, 1, 0, 0, 0, 0, time.Local)
)
// Videoshot is struct.
type Videoshot struct {
Cid int64
Count int
version int
CTime xtime.Time
MTime xtime.Time
}
// Version get version.
func (v *Videoshot) Version() int {
if v.version > 0 {
return v.version
}
return int(v.MTime.Time().Sub(_verDate) / time.Second)
}
// SetVersion set version from cache.
func (v *Videoshot) SetVersion(version int) {
v.version = version
}

41
app/service/main/archive/server/gorpc/BUILD Normal file
View File

@@ -0,0 +1,41 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"rpc.go",
"rpc2.go",
"rpc3.go",
],
importpath = "go-common/app/service/main/archive/server/gorpc",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/conf:go_default_library",
"//app/service/main/archive/model/archive:go_default_library",
"//app/service/main/archive/service:go_default_library",
"//library/log:go_default_library",
"//library/net/rpc:go_default_library",
"//library/net/rpc/context:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

33
app/service/main/archive/server/gorpc/rpc.go Normal file
View File

@@ -0,0 +1,33 @@
package server
import (
"go-common/app/service/main/archive/conf"
"go-common/app/service/main/archive/service"
"go-common/library/net/rpc"
"go-common/library/net/rpc/context"
)
// RPC rpc.
type RPC struct {
s *service.Service
}
// New new rpc server.
func New(c *conf.Config, s *service.Service) (svr *rpc.Server) {
r := &RPC{s: s}
svr = rpc.NewServer(c.RPCServer)
if err := svr.Register(r); err != nil {
panic(err)
}
return
}
// Auth check connection success.
func (r *RPC) Auth(c context.Context, arg *rpc.Auth, res *struct{}) (err error) {
return
}
// Ping check connection success.
func (r *RPC) Ping(c context.Context, arg *struct{}, res *struct{}) (err error) {
return
}

82
app/service/main/archive/server/gorpc/rpc2.go Normal file
View File

@@ -0,0 +1,82 @@
package server
import (
"go-common/app/service/main/archive/api"
"go-common/app/service/main/archive/model/archive"
"go-common/library/net/rpc/context"
)
// Types2 get all types
func (r *RPC) Types2(c context.Context, a *struct{}, res *map[int16]*archive.ArcType) (err error) {
*res = r.s.AllTypes(c)
return
}
// Videoshot2 get vidoshot info data.
func (r *RPC) Videoshot2(c context.Context, a *archive.ArgCid2, res *archive.Videoshot) (err error) {
var (
v *archive.Videoshot
)
if v, err = r.s.Videoshot(c, a.Aid, a.Cid); err == nil {
*res = *v
}
return
}
// UpCount2 up count.
func (r *RPC) UpCount2(c context.Context, a *archive.ArgUpCount2, count *int) (err error) {
*count, err = r.s.UpperCount(c, a.Mid)
return
}
// UpsPassed2 ups pass aid and ptime
func (r *RPC) UpsPassed2(c context.Context, a *archive.ArgUpsArcs2, res *map[int64][]*archive.AidPubTime) (err error) {
*res, err = r.s.UppersAidPubTime(c, a.Mids, a.Pn, a.Ps)
return
}
// UpVideo2 up video by aid & cid.
func (r *RPC) UpVideo2(c context.Context, a *archive.ArgVideo2, res *struct{}) (err error) {
return r.s.UpVideo(c, a.Aid, a.Cid)
}
// DelVideo2 delete video by aid & cid.
func (r *RPC) DelVideo2(c context.Context, a *archive.ArgVideo2, res *struct{}) (err error) {
return r.s.DelVideo(c, a.Aid, a.Cid)
}
// Description2 get description by aid.
func (r *RPC) Description2(c context.Context, a *archive.ArgAid, reDes *string) (err error) {
*reDes, err = r.s.Description(c, a.Aid)
return
}
// RanksTopCount2 top region count.
func (r *RPC) RanksTopCount2(c context.Context, a *archive.ArgRankTopsCount2, res *map[int16]int) (err error) {
*res, err = r.s.RegionTopCount(c, a.ReIDs)
return
}
// ArcsStat2 archive stat.
// func (r *RPC) ArcsStat2(c context.Context, a *archive.ArgAids2, res *map[int64]*archive.Stat) (err error) {
// *res, err = r.s.Stats(c, a.Aids)
// return
// }
// ArcCache2 update archive cache.
func (r *RPC) ArcCache2(c context.Context, a *archive.ArgCache2, res *struct{}) (err error) {
err = r.s.CacheUpdate(c, a.Aid, a.Tp, a.OldMid)
return
}
// ArcFieldCache2 update archive by field changed.
func (r *RPC) ArcFieldCache2(c context.Context, a *archive.ArgFieldCache2, res *struct{}) (err error) {
err = r.s.FieldCacheUpdate(c, a.Aid, a.OldTypeID, a.TypeID)
return
}
// SetStat2 set all stat cache(redis)
func (r *RPC) SetStat2(c context.Context, a *api.Stat, res *struct{}) (err error) {
err = r.s.SetStat(c, a)
return
}

183
app/service/main/archive/server/gorpc/rpc3.go Normal file
View File

@@ -0,0 +1,183 @@
package server
import (
"go-common/app/service/main/archive/api"
"go-common/app/service/main/archive/model/archive"
"go-common/library/log"
"go-common/library/net/rpc/context"
)
// 3结尾的方法全都是pb格式的memcache
// MaxAID get max aid
func (r *RPC) MaxAID(c context.Context, a *struct{}, res *int64) (err error) {
*res, err = r.s.MaxAID(c)
return
}
// Archive3 receive aid, then init archive info.
func (r *RPC) Archive3(c context.Context, a *archive.ArgAid2, res *api.Arc) (err error) {
var ar *api.Arc
if ar, err = r.s.Archive3(c, a.Aid); err == nil {
*res = *ar
}
return
}
// Archives3 receive aids, then init archives info.
func (r *RPC) Archives3(c context.Context, a *archive.ArgAids2, res *map[int64]*api.Arc) (err error) {
if len(a.Aids) > 300 {
log.Error("Too many Args aids(%d) caller(%s) arg(%v)", len(a.Aids), c.User(), a.Aids)
}
*res, err = r.s.Archives3(c, a.Aids)
return
}
// View3 view archive.
func (r *RPC) View3(c context.Context, a *archive.ArgAid2, av *archive.View3) (err error) {
var res *api.ViewReply
if res, err = r.s.View3(c, a.Aid); err == nil {
*av = *archive.BuildView3(res.Arc, res.Pages)
}
return
}
// Views3 view archive.
func (r *RPC) Views3(c context.Context, a *archive.ArgAids2, res *map[int64]*archive.View3) (err error) {
if len(a.Aids) > 300 {
log.Error("Too many Args aids(%d) caller(%s)", len(a.Aids), c.User())
}
var views map[int64]*api.ViewReply
if views, err = r.s.Views3(c, a.Aids); err == nil {
var resp = make(map[int64]*archive.View3)
for aid, view := range views {
v := archive.BuildView3(view.Arc, view.Pages)
if v != nil {
resp[aid] = v
}
}
*res = resp
}
return
}
// Stat3 archive stat.
func (r *RPC) Stat3(c context.Context, a *archive.ArgAid2, res *api.Stat) (err error) {
var st *api.Stat
if st, err = r.s.Stat3(c, a.Aid); err == nil {
*res = *st
}
return
}
// Page3 get videos by aid
func (r *RPC) Page3(c context.Context, a *archive.ArgAid2, res *[]*api.Page) (err error) {
*res, err = r.s.Page3(c, a.Aid)
return
}
// Stats3 archive stats.
func (r *RPC) Stats3(c context.Context, a *archive.ArgAids2, res *map[int64]*api.Stat) (err error) {
if len(a.Aids) > 200 {
log.Error("Too many Args aids(%d) caller(%s)", len(a.Aids), c.User())
log.Error("Too many Args aids(%d) caller(%s) arg(%v)", len(a.Aids), c.User(), a.Aids)
}
*res, err = r.s.Stats3(c, a.Aids)
return
}
// Click3 archive click.
func (r *RPC) Click3(c context.Context, a *archive.ArgAid2, res *api.Click) (err error) {
var clk *api.Click
if clk, err = r.s.Click3(c, a.Aid); err == nil {
*res = *clk
}
return
}
// UpArcs3 up archives.
func (r *RPC) UpArcs3(c context.Context, a *archive.ArgUpArcs2, res *[]*api.Arc) (err error) {
*res, err = r.s.UpperPassed3(c, a.Mid, a.Pn, a.Ps)
return
}
// UpsArcs3 ups archives.
func (r *RPC) UpsArcs3(c context.Context, a *archive.ArgUpsArcs2, res *map[int64][]*api.Arc) (err error) {
*res, err = r.s.UppersPassed3(c, a.Mids, a.Pn, a.Ps)
return
}
// Recommend3 from archive_recommend by aid
func (r *RPC) Recommend3(c context.Context, a *archive.ArgAid2, res *[]*api.Arc) (err error) {
*res, err = r.s.UpperReommend(c, a.Aid)
return
}
// RankArcs3 Arcs by rid
func (r *RPC) RankArcs3(c context.Context, a *archive.ArgRank2, res *archive.RankArchives3) (err error) {
if a.Type == 0 {
res.Archives, res.Count, err = r.s.RegionArcs3(c, a.Rid, a.Pn, a.Ps)
} else {
res.Archives, res.Count, err = r.s.RegionOriginArcs3(c, a.Rid, a.Pn, a.Ps)
}
return
}
// ArchivesWithPlayer archives with player info
func (r *RPC) ArchivesWithPlayer(c context.Context, a *archive.ArgPlayer, res *map[int64]*archive.ArchiveWithPlayer) (err error) {
var as map[int64]*archive.ArchiveWithPlayer
if as, err = r.s.ArchivesWithPlayer(c, a, false); err != nil {
return
}
*res = as
return
}
// RanksArcs3 Arcs by rids
func (r *RPC) RanksArcs3(c context.Context, a *archive.ArgRanks2, res *map[int16]*archive.RankArchives3) (err error) {
var (
as []*api.Arc
tmp map[int16]*archive.RankArchives3
count int
)
tmp = make(map[int16]*archive.RankArchives3)
for _, rid := range a.Rids {
if a.Type == 0 {
as, count, err = r.s.RegionArcs3(c, rid, a.Pn, a.Ps)
} else {
as, count, err = r.s.RegionOriginArcs3(c, rid, a.Pn, a.Ps)
}
if err != nil {
return
}
tmp[rid] = &archive.RankArchives3{Archives: as, Count: count}
}
*res = tmp
return
}
// RankTopArcs3 Arcs by reids
func (r *RPC) RankTopArcs3(c context.Context, a *archive.ArgRankTop2, res *[]*api.Arc) (err error) {
*res, err = r.s.RegionTopArcs3(c, a.ReID, a.Pn, a.Ps)
return
}
// RankAllArcs3 left 7 days all Arcs
func (r *RPC) RankAllArcs3(c context.Context, a *archive.ArgRankAll2, res *archive.RankArchives3) (err error) {
var data *archive.RankArchives3
data, err = r.s.RegionAllArcs3(c, a.Pn, a.Ps)
if err == nil {
*res = *data
}
return
}
// Video3 get video by aid & cid.
func (r *RPC) Video3(c context.Context, a *archive.ArgVideo2, res *api.Page) (err error) {
var p *api.Page
p, err = r.s.Video3(c, a.Aid, a.Cid)
if err == nil {
*res = *p
}
return
}

36
app/service/main/archive/server/grpc/BUILD Normal file
View File

@@ -0,0 +1,36 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"builder.go",
"server.go",
],
importpath = "go-common/app/service/main/archive/server/grpc",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/service:go_default_library",
"//library/net/rpc/warden:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

92
app/service/main/archive/server/grpc/builder.go Normal file
View File

@@ -0,0 +1,92 @@
package grpc
import (
v1 "go-common/app/service/main/archive/api"
)
func (s *server) archive3ToArc(a *v1.Arc) (arc *v1.Arc) {
arc = &v1.Arc{
Aid: a.Aid,
Videos: a.Videos,
TypeID: a.TypeID,
TypeName: a.TypeName,
Copyright: a.Copyright,
Pic: a.Pic,
Title: a.Title,
PubDate: a.PubDate,
Ctime: a.Ctime,
Desc: a.Desc,
State: a.State,
Access: a.Access,
Attribute: a.Attribute,
Duration: a.Duration,
MissionID: a.MissionID,
OrderID: a.OrderID,
RedirectURL: a.RedirectURL,
Forward: a.Forward,
Rights: v1.Rights{
Bp: a.Rights.Bp,
Elec: a.Rights.Elec,
Download: a.Rights.Download,
Movie: a.Rights.Movie,
Pay: a.Rights.Pay,
HD5: a.Rights.HD5,
NoReprint: a.Rights.NoReprint,
Autoplay: a.Rights.Autoplay,
UGCPay: a.Rights.UGCPay,
IsCooperation: a.Rights.IsCooperation,
},
Author: v1.Author{
Mid: a.Author.Mid,
Name: a.Author.Name,
Face: a.Author.Face,
},
Stat: v1.Stat{
Aid: a.Stat.Aid,
View: a.Stat.View,
Danmaku: a.Stat.Danmaku,
Reply: a.Stat.Reply,
Fav: a.Stat.Fav,
Coin: a.Stat.Coin,
Share: a.Stat.Share,
NowRank: a.Stat.NowRank,
HisRank: a.Stat.HisRank,
Like: a.Stat.Like,
DisLike: a.Stat.DisLike,
},
ReportResult: a.ReportResult,
Dynamic: a.Dynamic,
FirstCid: a.FirstCid,
Dimension: v1.Dimension{
Width: a.Dimension.Width,
Height: a.Dimension.Height,
Rotate: a.Dimension.Rotate,
},
}
for _, si := range a.StaffInfo {
if si != nil {
arc.StaffInfo = append(arc.StaffInfo, &v1.StaffInfo{Mid: si.Mid, Title: si.Title})
}
}
return
}
// ChangeToGrpc is
func (s *server) page3ToPage(v *v1.Page) (vg *v1.Page) {
vg = &v1.Page{
Cid: v.Cid,
Page: v.Page,
From: v.From,
Part: v.Part,
Duration: v.Duration,
Vid: v.Vid,
Desc: v.Desc,
WebLink: v.WebLink,
Dimension: v1.Dimension{
Width: v.Dimension.Width,
Height: v.Dimension.Height,
Rotate: v.Dimension.Rotate,
},
}
return
}

177
app/service/main/archive/server/grpc/server.go Normal file
View File

@@ -0,0 +1,177 @@
package grpc
import (
"context"
v1 "go-common/app/service/main/archive/api"
"go-common/app/service/main/archive/service"
"go-common/library/net/rpc/warden"
)
type server struct {
srv *service.Service
}
// New grpc server
func New(cfg *warden.ServerConfig, srv *service.Service) (wsvr *warden.Server, err error) {
wsvr = warden.NewServer(cfg)
v1.RegisterArchiveServer(wsvr.Server(), &server{srv: srv})
wsvr, err = wsvr.Start()
return
}
// Types get all types
func (s *server) Types(c context.Context, noArg *v1.NoArgRequest) (resp *v1.TypesReply, err error) {
types := s.srv.AllTypes(c)
resp = new(v1.TypesReply)
resp.Types = make(map[int32]*v1.Tp)
for _, tp := range types {
resp.Types[int32(tp.ID)] = &v1.Tp{
ID: int32(tp.ID),
Pid: int32(tp.Pid),
Name: tp.Name,
}
}
return
}
// Arc get archive
func (s *server) Arc(c context.Context, req *v1.ArcRequest) (resp *v1.ArcReply, err error) {
resp = new(v1.ArcReply)
a, err := s.srv.Archive3(c, req.Aid)
if err != nil {
return
}
resp.Arc = a
return
}
// Arcs get archives
func (s *server) Arcs(c context.Context, req *v1.ArcsRequest) (resp *v1.ArcsReply, err error) {
resp = new(v1.ArcsReply)
resp.Arcs = make(map[int64]*v1.Arc)
as, err := s.srv.Archives3(c, req.Aids)
if err != nil {
return
}
if len(as) == 0 {
return
}
for aid, a := range as {
resp.Arcs[aid] = a
}
return
}
// View get archive and page
func (s *server) View(c context.Context, req *v1.ViewRequest) (resp *v1.ViewReply, err error) {
resp = new(v1.ViewReply)
v, err := s.srv.View3(c, req.Aid)
if err != nil {
return
}
resp = v
return
}
// Views get archives and pages
func (s *server) Views(c context.Context, req *v1.ViewsRequest) (resp *v1.ViewsReply, err error) {
resp = new(v1.ViewsReply)
resp.Views = make(map[int64]*v1.ViewReply)
vs, err := s.srv.Views3(c, req.Aids)
if err != nil {
return
}
if len(vs) == 0 {
return
}
resp.Views = vs
return
}
func (s *server) Stat(c context.Context, req *v1.StatRequest) (resp *v1.StatReply, err error) {
resp = new(v1.StatReply)
stat, err := s.srv.Stat3(c, req.Aid)
if err != nil {
return
}
resp.Stat = stat
return
}
func (s *server) Stats(c context.Context, req *v1.StatsRequest) (resp *v1.StatsReply, err error) {
resp = new(v1.StatsReply)
resp.Stats = make(map[int64]*v1.Stat)
stats, err := s.srv.Stats3(c, req.Aids)
if err != nil {
return
}
resp.Stats = stats
return
}
func (s *server) Click(c context.Context, req *v1.ClickRequest) (resp *v1.ClickReply, err error) {
resp = new(v1.ClickReply)
resp.Click, err = s.srv.Click3(c, req.Aid)
return
}
func (s *server) Page(c context.Context, req *v1.PageRequest) (resp *v1.PageReply, err error) {
resp = new(v1.PageReply)
resp.Pages, err = s.srv.Page3(c, req.Aid)
return
}
func (s *server) Recommend(c context.Context, req *v1.RecommendRequest) (resp *v1.RecommendReply, err error) {
resp = new(v1.RecommendReply)
resp.Arcs, err = s.srv.UpperReommend(c, req.Aid)
return
}
func (s *server) Video(c context.Context, req *v1.VideoRequest) (resp *v1.VideoReply, err error) {
resp = new(v1.VideoReply)
resp.Page, err = s.srv.Video3(c, req.Aid, req.Cid)
return
}
func (s *server) MaxAid(c context.Context, req *v1.NoArgRequest) (resp *v1.MaxAidReply, err error) {
resp = new(v1.MaxAidReply)
resp.Aid, err = s.srv.MaxAID(c)
return
}
func (s *server) ArcFieldCache(c context.Context, req *v1.ArcFieldCacheRequest) (resp *v1.NoReply, err error) {
resp = new(v1.NoReply)
err = s.srv.FieldCacheUpdate(c, req.Aid, int16(req.OldTypeID), int16(req.TypeID))
return
}
func (s *server) ArcCache(c context.Context, req *v1.ArcCacheRequest) (resp *v1.NoReply, err error) {
resp = new(v1.NoReply)
err = s.srv.CacheUpdate(c, req.Aid, req.Tp, req.OldMid)
return
}
func (s *server) DelVideoCache(c context.Context, req *v1.DelVideoCacheRequest) (resp *v1.NoReply, err error) {
resp = new(v1.NoReply)
err = s.srv.DelVideo(c, req.Aid, req.Cid)
return
}
func (s *server) UpVideoCache(c context.Context, req *v1.UpVideoCacheRequest) (resp *v1.NoReply, err error) {
resp = new(v1.NoReply)
err = s.srv.UpVideo(c, req.Aid, req.Cid)
return
}
func (s *server) Description(c context.Context, req *v1.DescriptionRequest) (resp *v1.DescriptionReply, err error) {
resp = new(v1.DescriptionReply)
resp.Desc, err = s.srv.Description(c, req.Aid)
return
}
func (s *server) SetStat(c context.Context, req *v1.SetStatRequest) (resp *v1.NoReply, err error) {
resp = new(v1.NoReply)
err = s.srv.SetStat(c, req.Stat)
return
}

63
app/service/main/archive/server/http/BUILD Normal file
View File

@@ -0,0 +1,63 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["http_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/conf:go_default_library",
"//app/service/main/archive/service:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"archive.go",
"http.go",
"local.go",
"region.go",
"shot.go",
"stat.go",
"upper.go",
"video.go",
],
importpath = "go-common/app/service/main/archive/server/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/conf:go_default_library",
"//app/service/main/archive/model/archive:go_default_library",
"//app/service/main/archive/service:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/xstr:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

95
app/service/main/archive/server/http/archive.go Normal file
View File

@@ -0,0 +1,95 @@
package http
import (
"strconv"
"go-common/app/service/main/archive/model/archive"
"go-common/library/ecode"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/xstr"
)
// arcInfo write the archive data.
func arcInfo(c *bm.Context) {
var (
err error
aid int64
)
params := c.Request.Form
aidStr := params.Get("aid")
// check params
aid, err = strconv.ParseInt(aidStr, 10, 64)
if err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
c.JSON(arcSvc.Archive3(c, aid))
}
// archives write the archives data.
func archives(c *bm.Context) {
params := c.Request.Form
aidsStr := params.Get("aids")
// check params
aids, err := xstr.SplitInts(aidsStr)
if err != nil {
log.Error("query aids(%s) split error(%v)", aidsStr, err)
c.JSON(nil, ecode.RequestErr)
return
}
if params.Get("appkey") == "fb06a25c6338edbc" && len(aids) > 50 {
c.JSON(nil, ecode.RequestErr)
return
}
if len(aids) > 50 {
log.Error("Too many Args aids(%d) caller(%s)", len(aids), params.Get("appkey"))
}
c.JSON(arcSvc.Archives3(c, aids))
}
// archivesWithPlayer write the archives data.
func archivesWithPlayer(c *bm.Context) {
params := c.Request.Form
aidsStr := params.Get("aids")
qnStr := params.Get("qn")
pt := params.Get("platform")
ip := params.Get("ip")
fnver, _ := strconv.Atoi(params.Get("fnver"))
fnval, _ := strconv.Atoi(params.Get("fnval"))
forceHost, _ := strconv.Atoi(params.Get("force_host"))
session := params.Get("session")
containsPGC, _ := strconv.Atoi(params.Get("contains_pgc"))
build, _ := strconv.Atoi(params.Get("build"))
// check params
aids, err := xstr.SplitInts(aidsStr)
if err != nil {
log.Error("query aids(%s) split error(%v)", aidsStr, err)
c.JSON(nil, ecode.RequestErr)
return
}
if len(aids) > 50 {
c.JSON(nil, ecode.RequestErr)
return
}
qn, _ := strconv.Atoi(qnStr)
c.JSON(arcSvc.ArchivesWithPlayer(c, &archive.ArgPlayer{
Aids: aids,
Qn: qn,
Platform: pt,
Build: build,
RealIP: ip,
Fnval: fnval,
Fnver: fnver,
Session: session,
ForceHost: forceHost,
}, containsPGC == 1))
}
func typelist(c *bm.Context) {
c.JSON(arcSvc.AllTypes(c), nil)
}
func maxAID(c *bm.Context) {
c.JSON(arcSvc.MaxAID(c))
}

80
app/service/main/archive/server/http/http.go Normal file
View File

@@ -0,0 +1,80 @@
package http
import (
"go-common/app/service/main/archive/conf"
"go-common/app/service/main/archive/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
idfSvc *verify.Verify
arcSvc *service.Service
)
// Init init http router.
func Init(c *conf.Config, s *service.Service) {
arcSvc = s
idfSvc = verify.New(nil)
// init internal router
en := bm.DefaultServer(c.BM.Inner)
innerRouter(en)
// init internal server
if err := en.Start(); err != nil {
log.Error("xhttp.Serve error(%v)", err)
panic(err)
}
// init external router
enlocal := bm.DefaultServer(c.BM.Local)
localRouter(enlocal)
// init external server
if err := enlocal.Start(); err != nil {
log.Error("xhttp.Serve error(%v)", err)
panic(err)
}
}
// innerRouter init inner router.
func innerRouter(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
archive := e.Group("/x/internal/v2/archive", bm.CORS())
{
archive.GET("", idfSvc.Verify, arcInfo)
archive.GET("/view", idfSvc.Verify, arcView)
archive.GET("/views", idfSvc.Verify, arcViews)
archive.GET("/page", idfSvc.Verify, arcPage)
archive.GET("/video", idfSvc.Verify, video)
archive.GET("/archives", idfSvc.Verify, archives)
archive.GET("/archives/playurl", idfSvc.Verify, archivesWithPlayer)
archive.GET("/typelist", idfSvc.Verify, typelist)
archive.GET("/description", idfSvc.Verify, description)
archive.GET("/maxAid", idfSvc.Verify, maxAID)
regionGp := archive.Group("/region")
{
regionGp.GET("", idfSvc.Verify, regionArcs)
}
videoshotGp := archive.Group("/videoshot")
{
videoshotGp.GET("", idfSvc.Verify, videoshot)
}
statGp := archive.Group("/stat")
{
statGp.GET("", idfSvc.Verify, arcStat)
statGp.GET("/stats", idfSvc.Verify, arcStats)
}
upGp := archive.Group("/up")
{
upGp.GET("/count/batch", idfSvc.Verify, uppersCount)
upGp.GET("/count", idfSvc.Verify, upperCount)
upGp.GET("/passed", idfSvc.Verify, upperPassed)
upGp.GET("/cache", idfSvc.Verify, upperCache)
}
}
}
// localRouter init local router.
func localRouter(e *bm.Engine) {
e.GET("/archive-service/rank/init", addRegionArc)
}

123
app/service/main/archive/server/http/http_test.go Normal file
View File

@@ -0,0 +1,123 @@
package http
import (
"context"
"encoding/json"
"flag"
"net/url"
"path/filepath"
"testing"
"go-common/app/service/main/archive/conf"
"go-common/app/service/main/archive/service"
ghttp "go-common/library/net/http/blademaster"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *service.Service
client *ghttp.Client
)
func init() {
dir, _ := filepath.Abs("../cmd/archive-service-test.toml")
flag.Set("conf", dir)
conf.Init()
s = service.New(conf.Conf)
Init(conf.Conf, s)
client = ghttp.NewClient(conf.Conf.PlayerClient)
}
func Test_Archive(t *testing.T) {
Convey("/x/internal/v2/archive", t, func() {
p := url.Values{}
p.Set("aid", "10098813")
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
}
err := client.Get(context.TODO(), "http://0.0.0.0:6081/x/internal/v2/archive", "", p, &res)
So(err, ShouldBeNil)
So(res.Code, ShouldBeZeroValue)
Printf("code(%d) data(%s)\n", res.Code, res.Data)
})
}
func Test_ArchiveView(t *testing.T) {
Convey("/x/internal/v2/archive/view", t, func() {
p := url.Values{}
p.Set("aid", "10098813")
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
}
err := client.Get(context.TODO(), "http://0.0.0.0:6081/x/internal/v2/archive/view", "", p, &res)
So(err, ShouldBeNil)
So(res.Code, ShouldBeZeroValue)
Printf("code(%d) data(%s)\n", res.Code, res.Data)
})
}
func Test_ArchiveViews(t *testing.T) {
Convey("/x/internal/v2/archive/views", t, func() {
p := url.Values{}
p.Set("aids", "10098813,10098825,10098813")
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
}
err := client.Get(context.TODO(), "http://0.0.0.0:6081/x/internal/v2/archive/views", "", p, &res)
So(err, ShouldBeNil)
So(res.Code, ShouldBeZeroValue)
Printf("code(%d) data(%s)\n", res.Code, res.Data)
})
}
func Test_RegionArcs(t *testing.T) {
Convey("/x/internal/v2/archive/region", t, func() {
p := url.Values{}
p.Set("rid", "182")
p.Set("ps", "20")
p.Set("pn", "1")
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
}
err := client.Get(context.TODO(), "http://0.0.0.0:6081/x/internal/v2/archive/region", "", p, &res)
So(err, ShouldBeNil)
So(res.Code, ShouldBeZeroValue)
Printf("code(%d) data(%s)\n", res.Code, res.Data)
})
}
func Test_ShareAdd(t *testing.T) {
Convey("/x/internal/v2/archive/share/add", t, func() {
p := url.Values{}
p.Set("aid", "5463554")
p.Set("mid", "1684013")
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
}
err := client.Post(context.TODO(), "http://0.0.0.0:6081/x/internal/v2/archive/share/add", "", p, &res)
So(err, ShouldBeNil)
So(res.Code, ShouldBeZeroValue)
Printf("code(%d) data(%s)\n", res.Code, res.Data)
})
}
func Test_UpCount(t *testing.T) {
Convey("/x/internal/v2/archive/up/count", t, func() {
p := url.Values{}
p.Set("mid", "27515232")
var res struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
}
err := client.Get(context.TODO(), "http://0.0.0.0:6081/x/internal/v2/archive/up/count", "", p, &res)
So(err, ShouldBeNil)
So(res.Code, ShouldBeZeroValue)
Printf("code(%d) data(%s)\n", res.Code, res.Data)
})
}

20
app/service/main/archive/server/http/local.go Normal file
View File

@@ -0,0 +1,20 @@
package http
import (
"net/http"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
// ping check server ok.
func ping(c *bm.Context) {
if err := arcSvc.Ping(c); err != nil {
log.Error("archive service ping error(%v)", err)
c.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(nil, nil)
}

75
app/service/main/archive/server/http/region.go Normal file
View File

@@ -0,0 +1,75 @@
package http
import (
"strconv"
"go-common/app/service/main/archive/api"
"go-common/library/ecode"
bm "go-common/library/net/http/blademaster"
)
// regionArcs.
func regionArcs(c *bm.Context) {
params := c.Request.Form
ridStr := params.Get("rid")
tpStr := params.Get("type")
pnStr := params.Get("pn")
psStr := params.Get("ps")
// check params
rid, err := strconv.ParseInt(ridStr, 10, 64)
if err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
var tp int64
if tpStr != "" {
if tp, err = strconv.ParseInt(tpStr, 10, 64); err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
}
pn, err := strconv.Atoi(pnStr)
if err != nil || pn < 1 {
pn = 1
}
ps, err := strconv.Atoi(psStr)
if err != nil || ps < 1 || ps > 100 {
ps = 20
}
// service
var (
as []*api.Arc
count int
)
if tp == 0 {
as, count, err = arcSvc.RegionArcs3(c, int16(rid), pn, ps)
} else {
as, count, err = arcSvc.RegionOriginArcs3(c, int16(rid), pn, ps)
}
if err != nil {
c.JSON(nil, err)
return
}
var res struct {
Archives []*api.Arc `json:"archives"`
Page struct {
Count int `json:"count"`
Num int `json:"num"`
Size int `json:"size"`
} `json:"page"`
}
res.Archives = as
res.Page.Num = pn
res.Page.Size = ps
res.Page.Count = count
c.JSON(res, nil)
}
// addRegionArc.
func addRegionArc(c *bm.Context) {
params := c.Request.Form
ridStr := params.Get("rid")
// check params
rid, _ := strconv.ParseInt(ridStr, 10, 64)
c.JSON(nil, arcSvc.AddRegionArcs(c, int16(rid)))
}

27
app/service/main/archive/server/http/shot.go Normal file
View File

@@ -0,0 +1,27 @@
package http
import (
"strconv"
"go-common/library/ecode"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
func videoshot(c *bm.Context) {
params := c.Request.Form
cidStr := params.Get("cid")
aidStr := params.Get("aid")
// check params
cid, err := strconv.ParseInt(cidStr, 10, 64)
if err != nil || cid == 0 {
log.Warn("query (cid) must be number and > 0 but (%s) error(%v)", cidStr, err)
c.JSON(nil, ecode.RequestErr)
return
}
aid, err := strconv.ParseInt(aidStr, 10, 64)
if err != nil || aid == 0 {
log.Warn("videoshot aid(%s) error", aidStr)
}
c.JSON(arcSvc.Videoshot(c, aid, cid))
}

37
app/service/main/archive/server/http/stat.go Normal file
View File

@@ -0,0 +1,37 @@
package http
import (
"strconv"
"go-common/library/ecode"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/xstr"
)
// arcStat get archive stat.
func arcStat(c *bm.Context) {
params := c.Request.Form
aidStr := params.Get("aid")
// check params
aid, err := strconv.ParseInt(aidStr, 10, 64)
if err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
c.JSON(arcSvc.Stat3(c, aid))
}
// arcStats get archives stat.
func arcStats(c *bm.Context) {
params := c.Request.Form
aidsStr := params.Get("aids")
// check params
aids, err := xstr.SplitInts(aidsStr)
if err != nil {
log.Error("query aids(%s) split error(%v)", aidsStr, err)
c.JSON(nil, ecode.RequestErr)
return
}
c.JSON(arcSvc.Stats3(c, aids))
}

Some files were not shown because too many files have changed in this diff Show More