Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,22 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/dynamic/cmd:all-srcs",
"//app/service/main/dynamic/conf:all-srcs",
"//app/service/main/dynamic/dao:all-srcs",
"//app/service/main/dynamic/http:all-srcs",
"//app/service/main/dynamic/model:all-srcs",
"//app/service/main/dynamic/rpc:all-srcs",
"//app/service/main/dynamic/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,189 @@
### dynamic-service
#### Version 3.5.1
##### Features
> 1.dao ut
#### Version 3.5.0
##### Features
> 1.identify使用grpc
> 2.改用metadata.RemoteIP方法
#### Version 3.4.1
##### Features
> 1.common conf
#### Version 3.3.1
##### Features
> 1.HTTPServer
> 2.del default conf
#### Version 3.2.2
##### Features
> 1.live 替换新的API
#### Version 3.2.1
##### Features
> 1.使用discovery接入archive
#### Version 3.1.1
##### Features
> 1.test to archive3
#### Version 3.1.0
##### Features
> 1.use bm
> 2.move main
#### Version 3.0.0
##### Features
> 1.使用common mc 去掉redis
#### Version 2.5.5
##### Features
> 1.add archives3 log
#### Version 2.5.4
##### Features
> 1.rm config dentify-mc
#### Version 2.5.3
##### Features
> 1.rm archive2
> 2.config xlog to log
#### Version 2.5.2
##### Features
> 1.统一prom使用方法
#### Version 2.5.1
##### Features
> 1.切archive3 rid to int32
#### Version 2.5.0
##### Features
> 1.http切archive3
> 2.新增archive3 rpc方法
#### Version 2.4.0
##### Features
> 1.trace、httpClient修改
#### Version 2.3.0
##### Features
> 1.去掉DB依赖
#### Version 2.2.1
##### Bug Fixes
> 1.删除client下文件
#### Version 2.2.0
##### Features
> 1.合并大仓库
#### Version 2.1.8
##### Bug Fixes
> 1.fix 动态数空指针
#### Version 2.1.7
##### Bug Fixes
> 1.fix 批量动态空指针
#### Version 2.1.6
##### Features
> 1.remote cache 优化
> 2.返回结果加调用最新投稿
> 3.加单元测试
#### Version 2.1.5
##### Bug Fixes
> 1.添加禁止动态过滤
#### Version 2.1.4
##### Features
> 1.remote cache接入prometheus
> 2.错误码QPS接入prometheus
#### Version 2.1.3
##### Bug Fixes
> 1.db,redis接入prometheus
#### Version 2.1.2
##### Features
> 1.接入prometheus
#### Version 2.1.0
##### Features
> 1.添加remote cache
#### Version 2.0.0
##### Features
> 1.接入新的配制中心
#### Version 1.9.1
##### Features
> 1.动态加Archive IsNormal判断 and 升级ZKOff
#### Version 1.9.0
##### Features
> 1.增加分区动态总数rpc方法
#### Version 1.8.0
##### Features
> 1.更新最新的net/rpc
> 2.tag和分区最新动态轮训时间区分
#### Version 1.7.0
##### Features
> 1.分区动态总数,增加直播动态总数
#### Version 1.6.0
##### Features
> 1.批量获取分区动态接口
#### Version 1.5.0
##### Bug Fixes
> 1.fix 大数据接口容错的bug
#### Version 1.4.2
##### Bug Fixes
> 1.增加rpc ping方法
#### Version 1.4.1
##### Features
> 1.更新vendor
#### Version 1.4.0
##### Features
> 1.更新vendor支持rpcx
#### Version 1.3.2
##### Features
> 1.升级vendor
#### Version 1.3.1
##### Features
> 1.分区动态总数一次性返回所有分区
#### Version 1.3.0
##### Features
> 1.增加分区动态总数
#### Version 1.2.1
##### Features
> 1.初始化rpc
#### Version 1.2.0
##### Features
> 1.接入配置中心
> 2.升级vendor
#### Version 1.1.0
##### Features
> 1.支持一级分区动态
> 2.fix map 同时读写bug
> 3.更新vendor
#### Version 1.0.0
##### Features
> 1.支持分区和tag的最新视频以及最新动态

View File

@@ -0,0 +1,13 @@
# Owner
liweijia
zhapuyu
# Author
guanyanliang
wuhao02
liweijia
# Reviewer
zhapuyu
wuhao02
liweijia

View File

@@ -0,0 +1,18 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- guanyanliang
- liweijia
- wuhao02
- zhapuyu
labels:
- main
- service
- service/main/dynamic
options:
no_parent_owners: true
reviewers:
- guanyanliang
- liweijia
- wuhao02
- zhapuyu

View File

@@ -0,0 +1,18 @@
#### dynamic-service
##### 项目简介
> 1.提供稿件动态服务
##### 编译环境
> 请使用golang v1.7.x以上版本编译执行。
##### 依赖包
> 1.公共包go-common
##### 编译执行
> 在主目录执行go build。
> 编译后可执行 ./cmd -conf dynamic-service-test.toml 使用项目本地配置文件启动服务。
> 也可执行 ./dynamic-service -conf_appid=dynamic-service -conf_version=shsb-server-1 -conf_host=config.bilibili.co -conf_path=/data/conf/dynamic-service -conf_env=10 -conf_token=I4oCdH5cWAfP8wHqjEnbOu0qnB1miyEN 使用配置中心测试环境配置启动服务如无法启动可检查token是否正确。
##### 特别说明
> http接口文档可参考 http://info.bilibili.co/pages/viewpage.action?pageId=2493593

View File

@@ -0,0 +1,46 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = [
"dynamic-service-example.toml",
"dynamic-service-test.toml",
],
importpath = "go-common/app/service/main/dynamic/cmd",
tags = ["automanaged"],
deps = [
"//app/service/main/dynamic/conf:go_default_library",
"//app/service/main/dynamic/http:go_default_library",
"//app/service/main/dynamic/rpc/server:go_default_library",
"//app/service/main/dynamic/service:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,134 @@
# This is a TOML document. Boom.
version = "3.0.0"
user = "nobody"
pid = "/tmp/dynamic-service.pid"
dir = "./"
perf = "127.0.0.1:6239"
checkFile = "/data/www/dynamic.html"
family = "dynamic-service"
bigDataURI = "http://data.bilibili.co"
tickRegion = "5s"
tickTag = "20s"
numArcs = 5
numIndexArcs = 10
minRegionCount = 15
[xlog]
dir = "/data/log/dynamic-service/"
[tracer]
proto = "udp"
addr = "172.16.0.148:5640"
tag = "platform/dynamic-service"
[app]
key = "7c7ac0db1aa05587"
secret = "9a6d62d93290c5f771ad381e9ca23f26"
[httpClient]
[httpClient.read]
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[httpClient.read.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[httpClient.write]
dial = "1s"
timeout = "3s"
keepAlive = "60s"
timer = 1000
[httpClient.write.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[multiHTTP]
[multiHTTP.inner]
addrs = ["0.0.0.0:6231"]
maxListen = 1000
[multiHTTP.local]
addrs = ["0.0.0.0:6232"]
maxListen = 100
[identify]
whiteAccessKey = ""
whiteMid = 0
[identify.app]
key = "6a29f8ed87407c11"
secret = "d3c5a85f5b895a03735b5d20a273bc57"
[identify.memcache]
name = "go-business/identify"
proto = "tcp"
addr = "172.16.33.54:11211"
active = 5
idle = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
[identify.host]
auth = "http://passport.bilibili.com"
secret = "http://open.bilibili.com"
[identify.authHTTPClient]
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[identify.authHTTPClient.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[identify.secretHTTPClient]
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[identify.secretHTTPClient.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[rpcServer2]
[[rpcServer2.servers]]
proto = "tcp"
addr = ":6235"
weight = 10
[rpcServer2.zookeeper]
root = "/microservice/dynamic-service/"
addrs = ["127.0.0.1:2181"]
timeout = "1s"
[archiveRPC]
pullInterval = "10s"
group = "test"
[archiveRPC.client]
token = "123456"
proto = "tcp"
addr = "172.16.0.148:6089"
timeout = "1s"
timer = 1000
[archiveRPC.client.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[archiveRPC.zookeeper]
root = "/microservice/archive-service/"
addrs = ["172.16.0.148:2181"]
timeout = "30s"

View File

@@ -0,0 +1,84 @@
[rule]
tickRegion = "5s"
tickTag = "20s"
numArcs = 5
numIndexArcs = 10
minRegionCount = 15
[host]
liveURI = "http://api.live.bilibili.co"
apiURI = "http://api.bilibili.co"
bigDataURI = "http://data.bilibili.co"
[log]
family = "dynamic-service"
dir = "/data/log/dynamic-service/"
[app]
key = "7c7ac0db1aa05587"
secret = "9a6d62d93290c5f771ad381e9ca23f26"
[httpClient]
[httpClient.read]
key = "7c7ac0db1aa05587"
secret = "9a6d62d93290c5f771ad381e9ca23f26"
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[httpClient.write]
key = "7c7ac0db1aa05587"
secret = "9a6d62d93290c5f771ad381e9ca23f26"
dial = "1s"
timeout = "3s"
keepAlive = "60s"
timer = 1000
[HTTPServer]
addr = "0.0.0.0:6231"
timeout = "1s"
[identify]
whiteAccessKey = ""
whiteMid = 0
[identify.app]
key = "7c7ac0db1aa05587"
secret = "9a6d62d93290c5f771ad381e9ca23f26"
[identify.host]
auth = "http://passport.bilibili.com"
secret = "http://open.bilibili.com"
[identify.httpClient]
key = "7c7ac0db1aa05587"
secret = "9a6d62d93290c5f771ad381e9ca23f26"
dial = "30ms"
timeout = "150ms"
keepAlive = "60s"
[identify.httpClient.breaker]
window ="10s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[identify.httpClient.url]
"http://passport.bilibili.co/intranet/auth/tokenInfo" = {timeout = "100ms"}
"http://passport.bilibili.co/intranet/auth/cookieInfo" = {timeout = "100ms"}
"http://open.bilibili.co/api/getsecret" = {timeout = "500ms"}
[archiveRPC]
timeout = "1s"
[memcache]
name = "dynamic-service"
proto = "tcp"
addr = "172.16.33.54:11211"
idle = 5
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "24h"

View File

@@ -0,0 +1,6 @@
#!/bin/bash
command -v goconvey >/dev/null 2>&1 || { echo >&2 "required goconvey but it's not installed."; echo "Aborting."; echo "Please run commond: go get github.com/smartystreets/goconvey"; exit 1; }
cd ../
goconvey -excludedDirs "vendor" -packages 1

View File

@@ -0,0 +1,51 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/service/main/dynamic/conf"
"go-common/app/service/main/dynamic/http"
rpc "go-common/app/service/main/dynamic/rpc/server"
"go-common/app/service/main/dynamic/service"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
log.Init(conf.Conf.Log)
trace.Init(conf.Conf.Tracer)
defer trace.Close()
defer log.Close()
log.Info("dynamic-service start")
// service init
svr := service.New(conf.Conf)
rpcSvr := rpc.New(conf.Conf, svr)
http.Init(conf.Conf, svr)
// signal handler
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("dynamic-service get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
rpcSvr.Close()
log.Info("dynamic-service exit")
time.Sleep(time.Second)
return
case syscall.SIGHUP:
// TODO reload
default:
return
}
}
}

View File

@@ -0,0 +1,38 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/service/main/dynamic/conf",
tags = ["automanaged"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/conf:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/rpc:go_default_library",
"//library/net/trace:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,131 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/memcache"
"go-common/library/conf"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/rpc"
"go-common/library/net/trace"
"go-common/library/time"
"github.com/BurntSushi/toml"
)
// global var
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config .
type Config struct {
// elk
Log *log.Config
// Verify
Verify *verify.Config
// app
App *bm.App
// http client
HTTPClient httpClient
// rpc
ArchiveRPC *rpc.ClientConfig
// rpc server
RPCServer *rpc.ServerConfig
// tracer
Tracer *trace.Config
// mc
Memcache *Memcache
// Rule
Rule *Rule
// Host
Host *Host
// HTTPServer
HTTPServer *bm.ServerConfig
}
// Host hosts.
type Host struct {
BigDataURI string
LiveURI string
APIURI string
}
// Rule config.
type Rule struct {
// region tick.
TickRegion time.Duration
// tag tick.
TickTag time.Duration
// default num of dynamic archives.
NumArcs int
//default num of index dynamic archives.
NumIndexArcs int
//min region count.
MinRegionCount int
}
type httpClient struct {
Read *bm.ClientConfig
}
// Memcache memcache
type Memcache struct {
*memcache.Config
Expire time.Duration
}
func init() {
flag.StringVar(&confPath, "conf", "", "config path")
}
// Init init conf
func Init() (err error) {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,62 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"bigdata_test.go",
"dao_test.go",
"live_test.go",
"memcache_test.go",
"tag_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/dynamic/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
"//vendor/gopkg.in/h2non/gock.v1:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"bigdata.go",
"dao.go",
"live.go",
"memcache.go",
"tag.go",
],
importpath = "go-common/app/service/main/dynamic/dao",
tags = ["automanaged"],
deps = [
"//app/service/main/dynamic/conf:go_default_library",
"//app/service/main/dynamic/model:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/stat/prom:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,61 @@
package dao
import (
"context"
"net/url"
"strconv"
"go-common/app/service/main/dynamic/model"
"go-common/library/ecode"
"go-common/library/log"
)
// RegionArcs get new dynamic by bigData API.
func (d *Dao) RegionArcs(c context.Context, rid int32, remoteIP string) (aids []int64, total int, err error) {
params := url.Values{}
params.Set("rid", strconv.FormatInt(int64(rid), 10))
var res struct {
Code int `json:"code"`
Data model.AvidData `json:"data"`
}
if err = d.httpR.Get(c, d.regionURI, remoteIP, params, &res); err != nil {
PromError("大数据分区接口", "d.httpR.Get(%s) error(%+v)", d.regionURI+"?"+params.Encode(), err)
return
}
if res.Code != ecode.OK.Code() {
PromError("大数据分区接口", "dynamic url(%s) res code(%d) or res.data(%v)", d.regionURI+"?"+params.Encode(), res.Code, res.Data)
err = ecode.Int(res.Code)
return
}
if len(res.Data.Avid) == 0 {
log.Error("dynamic url(%s) res(%v)", d.regionURI+"?"+params.Encode(), res)
}
aids = res.Data.Avid
total = res.Data.Count
return
}
// RegionTagArcs get new dynamic by bigData API.
func (d *Dao) RegionTagArcs(c context.Context, rid int32, tagID int64, remoteIP string) (aids []int64, err error) {
params := url.Values{}
params.Set("rid", strconv.FormatInt(int64(rid), 10))
params.Set("tag_id", strconv.FormatInt(tagID, 10))
var res struct {
Code int `json:"code"`
Data model.AvidData `json:"data"`
}
if err = d.httpR.Get(c, d.regionTagURI, remoteIP, params, &res); err != nil {
PromError("大数据Tag接口", "d.httpR.Get(%s) error(%+v)", d.regionTagURI+"?"+params.Encode(), err)
return
}
if res.Code != ecode.OK.Code() {
PromError("大数据Tag接口", "dynamic url(%s) res code(%d) or res.data(%v)", d.regionTagURI+"?"+params.Encode(), res.Code, res.Data)
err = ecode.Int(res.Code)
return
}
if len(res.Data.Avid) == 0 {
log.Error("dynamic url(%s) res(%v)", d.regionTagURI+"?"+params.Encode(), res)
}
aids = res.Data.Avid
return
}

View File

@@ -0,0 +1,56 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoRegionArcs(t *testing.T) {
convey.Convey("RegionArcs", t, func() {
var (
c = context.Background()
rid = int32(1)
remoteIP = "0.0.0.0"
)
convey.Convey("When http request gets 404 error", func(ctx convey.C) {
httpMock("GET", d.regionURI).MatchParam("rid", "0").Reply(502).JSON(`{"code":-500}`)
_, _, err := d.RegionArcs(c, rid, remoteIP)
ctx.Convey("Then error should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
convey.Convey("When http request gets code == 0", func(ctx convey.C) {
httpMock("GET", d.regionURI).MatchParam("rid", "1").Reply(200).JSON(`{"code":0,"message":"ok","data":{"avid":[28868622,29365735,29621166],"count":4602378}}`)
aids, total, err := d.RegionArcs(c, rid, remoteIP)
ctx.Convey("Then error should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(total, convey.ShouldBeGreaterThan, 0)
ctx.So(len(aids), convey.ShouldBeGreaterThan, 0)
})
})
})
}
func TestDaoRegionTagArcs(t *testing.T) {
convey.Convey("RegionTagArcs", t, func() {
var (
c = context.Background()
rid = int32(136)
tagID = int64(4577)
remoteIP = "0.0.0.0"
params = map[string]string{"rid": "136", "tag_id": "4577"}
)
convey.Convey("When http request gets code == 0", func(ctx convey.C) {
httpMock("GET", d.regionTagURI).MatchParams(params).Reply(200).JSON(`{"code":0,"message":"ok","data":{"avid":[28868622,29365735,29621166],"count":4602378}}`)
aids, err := d.RegionTagArcs(c, rid, tagID, remoteIP)
ctx.Convey("Then error should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(len(aids), convey.ShouldBeGreaterThan, 0)
})
})
})
}

View File

@@ -0,0 +1,75 @@
package dao
import (
"context"
"time"
"go-common/app/service/main/dynamic/conf"
"go-common/library/cache/memcache"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/stat/prom"
)
const (
_regionURL = "/dynamic/region"
_regionTagURL = "/dynamic/tag"
_liveURL = "/room/v1/Area/dynamic"
_hotURL = "/x/internal/tag/hotmap"
_pridURL = "/x/internal/tag/prids"
)
// PromError stat and log.
func PromError(name string, format string, args ...interface{}) {
prom.BusinessErrCount.Incr(name)
log.Error(format, args...)
}
// Dao dao.
type Dao struct {
// http
httpR *bm.Client
// bigData api
regionURI string
regionTagURI string
// live api
liveURI string
// tag api
hotURI string
pridURI string
// memcache
mc *memcache.Pool
mcExpire int32
// cache Prom
cacheProm *prom.Prom
}
// New dao new.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
httpR: bm.NewClient(c.HTTPClient.Read),
regionURI: c.Host.BigDataURI + _regionURL,
regionTagURI: c.Host.BigDataURI + _regionTagURL,
liveURI: c.Host.LiveURI + _liveURL,
hotURI: c.Host.APIURI + _hotURL,
pridURI: c.Host.APIURI + _pridURL,
// memcache
mc: memcache.NewPool(c.Memcache.Config),
mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
}
d.cacheProm = prom.CacheHit
return
}
// Ping check connection success.
func (dao *Dao) Ping(c context.Context) (err error) {
err = dao.pingMC(c)
return
}
// Close close memcache resource.
func (dao *Dao) Close() {
if dao.mc != nil {
dao.mc.Close()
}
}

View File

@@ -0,0 +1,46 @@
package dao
import (
"flag"
"os"
"strings"
"testing"
"go-common/app/service/main/dynamic/conf"
"gopkg.in/h2non/gock.v1"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.web-svr.dynamic-service")
flag.Set("conf_token", "f7c6b053ef986c6d3c87b4fd69e64ddb")
flag.Set("tree_id", "2316")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../cmd/dynamic-service-test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
d.httpR.SetTransport(gock.DefaultTransport)
m.Run()
os.Exit(0)
}
func httpMock(method, url string) *gock.Request {
r := gock.New(url)
r.Method = strings.ToUpper(method)
return r
}

View File

@@ -0,0 +1,34 @@
package dao
import (
"context"
"net/http"
"go-common/library/ecode"
)
// Live gets live dynamic count
func (d *Dao) Live(c context.Context) (count int, err error) {
var req *http.Request
if req, err = d.httpR.NewRequest("GET", d.liveURI, "", nil); err != nil {
PromError("直播Live接口", "Live d.httpR.NewRequest(%s) error(%v)", d.liveURI, err)
return
}
var res struct {
Code int `json:"code"`
Msg string `json:"msg"`
Data struct{ Count int } `json:"data"`
}
err = d.httpR.Do(c, req, &res)
if err != nil {
PromError("直播Live接口", "Live d.httpR.Do(%s) error(%v)", d.liveURI, err)
return
}
if res.Code != ecode.OK.Code() {
PromError("直播Live接口", "Live dao.httpR.Do(%s) error(%v)", d.liveURI, err)
err = ecode.Int(res.Code)
return
}
count = res.Data.Count
return
}

View File

@@ -0,0 +1,24 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoLive(t *testing.T) {
convey.Convey("Live", t, func() {
var (
c = context.Background()
)
convey.Convey("When http request gets code == 0", func(ctx convey.C) {
httpMock("GET", d.liveURI).Reply(200).JSON(`{"code":0,"msg":"ok","message":"ok","data":{"count":770843}}`)
count, err := d.Live(c)
ctx.Convey("Then err should be nil.count should greater 0.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldBeGreaterThan, 0)
})
})
})
}

View File

@@ -0,0 +1,104 @@
package dao
import (
"context"
"go-common/app/service/main/dynamic/model"
gmc "go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_keyRegionArcs = "dyra" // key of region archives
_keyRegionTagArcs = "dyrta" // key of tag archives
)
// pingMC ping memcache.
func (d *Dao) pingMC(c context.Context) (err error) {
conn := d.mc.Get(c)
if err = conn.Set(&gmc.Item{Key: "ping", Value: []byte{1}, Expiration: d.mcExpire}); err != nil {
log.Error("conn.Store(set, ping, 1) error(%v)", err)
}
conn.Close()
return
}
// SetRegionCache set region archive to cache.
func (d *Dao) SetRegionCache(c context.Context, regionArcs map[int32][]int64) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
tmp := make(map[int32]*model.Aids)
for k, v := range regionArcs {
tmp[k] = &model.Aids{IDs: v}
}
item := &gmc.Item{Key: _keyRegionArcs, Object: &model.Region{Aids: tmp}, Expiration: d.mcExpire, Flags: gmc.FlagProtobuf}
if err = conn.Set(item); err != nil {
log.Error("SetRegionCache error(%v)", err)
}
return
}
// RegionCache get region archive from cache.
func (d *Dao) RegionCache(c context.Context) (rs map[int32][]int64) {
conn := d.mc.Get(c)
defer conn.Close()
res, err := conn.Get(_keyRegionArcs)
if err != nil {
if err == gmc.ErrNotFound {
err = nil
} else {
log.Error("conn.Get(%d) error(%v)", _keyRegionArcs, err)
}
return
}
rc := &model.Region{}
if err = conn.Scan(res, rc); err != nil {
log.Error("conn.Scan error(%v)", err)
return
}
rs = make(map[int32][]int64)
for k, v := range rc.Aids {
rs[k] = v.IDs
}
return
}
// SetTagCache set region tag archvie to cache.
func (d *Dao) SetTagCache(c context.Context, regionTagArcs map[string][]int64) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
tmp := make(map[string]*model.Aids)
for k, v := range regionTagArcs {
tmp[k] = &model.Aids{IDs: v}
}
item := &gmc.Item{Key: _keyRegionTagArcs, Object: &model.Tag{Aids: tmp}, Expiration: d.mcExpire, Flags: gmc.FlagProtobuf}
if err = conn.Set(item); err != nil {
log.Error("SetRegionCache error(%v)", err)
}
return
}
// TagCache get region tag archive from cache.
func (d *Dao) TagCache(c context.Context) (rs map[string][]int64) {
conn := d.mc.Get(c)
defer conn.Close()
res, err := conn.Get(_keyRegionTagArcs)
if err != nil {
if err == gmc.ErrNotFound {
err = nil
} else {
log.Error("conn.Get(%d) error(%v)", _keyRegionTagArcs, err)
}
return
}
tc := &model.Tag{}
if err = conn.Scan(res, tc); err != nil {
log.Error("conn.Scan error(%v)", err)
return
}
rs = make(map[string][]int64)
for k, v := range tc.Aids {
rs[k] = v.IDs
}
return
}

View File

@@ -0,0 +1,70 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaopingMC(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("pingMC", t, func(ctx convey.C) {
err := d.pingMC(c)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestDaoSetRegionCache(t *testing.T) {
var (
c = context.Background()
regionArcs map[int32][]int64
)
convey.Convey("SetRegionCache", t, func(ctx convey.C) {
err := d.SetRegionCache(c, regionArcs)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestDaoRegionCache(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("RegionCache", t, func(ctx convey.C) {
rs := d.RegionCache(c)
ctx.Convey("Then rs should not be nil.", func(ctx convey.C) {
ctx.So(rs, convey.ShouldNotBeNil)
})
})
}
func TestDaoSetTagCache(t *testing.T) {
var (
c = context.Background()
regionTagArcs map[string][]int64
)
convey.Convey("SetTagCache", t, func(ctx convey.C) {
err := d.SetTagCache(c, regionTagArcs)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestDaoTagCache(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("TagCache", t, func(ctx convey.C) {
rs := d.TagCache(c)
ctx.Convey("Then rs should not be nil.", func(ctx convey.C) {
ctx.So(rs, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,48 @@
package dao
import (
"context"
"net/url"
"go-common/library/ecode"
)
// Hot Second type hot tag ids.
func (d *Dao) Hot(c context.Context) (res map[int32][]int64, err error) {
params := url.Values{}
var rs struct {
Code int `json:"code"`
Data map[int32][]int64 `json:"data"`
}
if err = d.httpR.Get(c, d.hotURI, "", params, &rs); err != nil {
PromError("二级分区热门Tag接口", "d.httpR.Get(%s) error(%v)", d.hotURI, err)
return
}
if rs.Code != ecode.OK.Code() {
PromError("二级分区热门Tag接口", "tag hotmap url(%s) res code(%d) or res.data(%v)", d.hotURI, rs.Code, rs.Data)
err = ecode.Int(rs.Code)
return
}
res = rs.Data
return
}
// Rids first type ids.
func (d *Dao) Rids(c context.Context) (res []int32, err error) {
params := url.Values{}
var rs struct {
Code int `json:"code"`
Data []int32 `json:"data"`
}
if err = d.httpR.Get(c, d.pridURI, "", params, &rs); err != nil {
PromError("一级分区ID接口", "d.httpR.Get(%s) error(%v)", d.pridURI, err)
return
}
if rs.Code != ecode.OK.Code() {
PromError("一级分区ID接口", "tag prids url(%s) res code(%d) or res.data(%v)", d.pridURI, rs.Code, rs.Data)
err = ecode.Int(rs.Code)
return
}
res = rs.Data
return
}

View File

@@ -0,0 +1,40 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoHot(t *testing.T) {
convey.Convey("Hot", t, func(ctx convey.C) {
var (
c = context.Background()
)
convey.Convey("When http request gets code == 0", func(ctx convey.C) {
httpMock("GET", d.hotURI).Reply(200).JSON(`{"code":0,"message":"0","ttl":1,"data":{"121":[25638,830,147345]}}`)
res, err := d.Hot(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(len(res), convey.ShouldBeGreaterThan, 0)
})
})
})
}
func TestDaoRids(t *testing.T) {
convey.Convey("Rids", t, func(ctx convey.C) {
var (
c = context.Background()
)
convey.Convey("When http request gets code == 0", func(ctx convey.C) {
httpMock("GET", d.pridURI).Reply(200).JSON(`{"code":0,"message":"0","ttl":1,"data":[1,2,3,5]}`)
res, err := d.Rids(c)
ctx.Convey("Then err should be nil.res should greater 0.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(len(res), convey.ShouldBeGreaterThan, 0)
})
})
})
}

View File

@@ -0,0 +1,40 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"dynamic.go",
"http.go",
],
importpath = "go-common/app/service/main/dynamic/http",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/dynamic/conf:go_default_library",
"//app/service/main/dynamic/service:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/xstr:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,126 @@
package http
import (
"strconv"
"go-common/app/service/main/archive/api"
"go-common/app/service/main/dynamic/conf"
"go-common/library/ecode"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/xstr"
)
func regionTotal(c *bm.Context) {
c.JSON(dySvc.RegionTotal(c), nil)
}
// regionTagArcs get new arcs of region and hot tag
func regionTagArcs(c *bm.Context) {
var (
count int
rid, tagID int64
pn, ps int64
arcs []*api.Arc
err error
)
query := c.Request.Form
ridStr := query.Get("rid")
tagIDStr := query.Get("tag_id")
pnStr := query.Get("pn")
psStr := query.Get("ps")
if rid, err = strconv.ParseInt(ridStr, 10, 64); err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
if tagID, err = strconv.ParseInt(tagIDStr, 10, 64); err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
if pn, err = strconv.ParseInt(pnStr, 10, 64); err != nil || pn < 1 {
pn = 1
}
if ps, err = strconv.ParseInt(psStr, 10, 64); err != nil || ps < 1 {
ps = int64(conf.Conf.Rule.NumArcs)
}
if arcs, count, err = dySvc.RegionTagArcs3(c, int32(rid), tagID, int(pn), int(ps)); err != nil {
c.JSON(nil, err)
log.Error("dySvc.RegionTagArcs(%d,%d) error(%v)", rid, tagID, err)
return
}
data := make(map[string]interface{}, 2)
page := map[string]int{
"num": int(pn),
"size": int(ps),
"count": count,
}
data["page"] = page
data["archives"] = arcs
c.JSON(data, nil)
}
// regionArcs get new arcs of region.
func regionArcs(c *bm.Context) {
var (
count int
rid int64
pn, ps int64
arcs []*api.Arc
err error
)
query := c.Request.Form
ridStr := query.Get("rid")
pnStr := query.Get("pn")
psStr := query.Get("ps")
if rid, err = strconv.ParseInt(ridStr, 10, 64); err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
if pn, err = strconv.ParseInt(pnStr, 10, 64); err != nil || pn < 1 {
pn = 1
}
if ps, err = strconv.ParseInt(psStr, 10, 64); err != nil || ps < 1 {
ps = int64(conf.Conf.Rule.NumArcs)
}
if arcs, count, err = dySvc.RegionArcs3(c, int32(rid), int(pn), int(ps)); err != nil {
c.JSON(nil, err)
log.Error("dySvc.RegionArcs(%d) error(%v)", rid, err)
return
}
data := make(map[string]interface{}, 2)
page := map[string]int{
"num": int(pn),
"size": int(ps),
"count": count,
}
data["page"] = page
data["archives"] = arcs
c.JSON(data, nil)
}
// regionsArcs get batch new arcs of region.
func regionsArcs(c *bm.Context) {
var (
ridStr string
count int
rids []int32
ridsTmp []int64
err error
)
query := c.Request.Form
if count, err = strconv.Atoi(query.Get("count")); err != nil || count < 1 {
count = conf.Conf.Rule.NumIndexArcs
}
if ridStr = query.Get("rids"); ridStr == "" {
c.JSON(nil, ecode.RequestErr)
return
}
if ridsTmp, err = xstr.SplitInts(ridStr); err != nil {
c.JSON(nil, ecode.RequestErr)
return
}
for _, rid := range ridsTmp {
rids = append(rids, int32(rid))
}
c.JSON(dySvc.RegionsArcs3(c, rids, count))
}

View File

@@ -0,0 +1,54 @@
package http
import (
"net/http"
"go-common/app/service/main/dynamic/conf"
"go-common/app/service/main/dynamic/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
dySvc *service.Service
vfySvr *verify.Verify
)
// Init init.
func Init(c *conf.Config, s *service.Service) {
vfySvr = verify.New(c.Verify)
dySvc = s
engineInner := bm.DefaultServer(c.HTTPServer)
innerRouter(engineInner)
// init inner serve
if err := engineInner.Start(); err != nil {
log.Error("engineInner.Start error(%v)", err)
panic(err)
}
}
// innerRouter init inner router.
func innerRouter(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
group := e.Group("/x/dynamic", bm.CORS())
{
group.GET("/tag", vfySvr.Verify, regionTagArcs)
group.GET("/region", vfySvr.Verify, regionArcs)
group.GET("/regions", vfySvr.Verify, regionsArcs)
group.GET("/region/total", vfySvr.Verify, regionTotal)
}
}
// ping check server ok.
func ping(c *bm.Context) {
if err := dySvc.Ping(c); err != nil {
log.Error("dynamic service ping error")
c.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(nil, nil)
}

View File

@@ -0,0 +1,59 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
go_library(
name = "go_default_library",
srcs = [
"bigdata.go",
"dynamic.go",
"rpc.go",
],
embed = [":model_go_proto"],
importpath = "go-common/app/service/main/dynamic/model",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"@com_github_gogo_protobuf//gogoproto:go_default_library",
"@com_github_golang_protobuf//proto:go_default_library",
],
)
proto_library(
name = "model_proto",
srcs = ["dynamic.proto"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
)
go_proto_library(
name = "model_go_proto",
compilers = ["@io_bazel_rules_go//proto:gogofast_proto"],
importpath = "go-common/app/service/main/dynamic/model",
proto = ":model_proto",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["@com_github_gogo_protobuf//gogoproto:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,7 @@
package model
// AvidData .
type AvidData struct {
Avid []int64 `json:"avid"`
Count int `json:"count"`
}

View File

@@ -0,0 +1,16 @@
package model
import "go-common/app/service/main/archive/api"
// Page page.
type Page struct {
Num int `json:"num"`
Size int `json:"size"`
Count int `json:"count"`
}
// DynamicArcs3 dynamic archives3.
type DynamicArcs3 struct {
Page *Page `json:"page"`
Archives []*api.Arc `json:"archives"`
}

View File

@@ -0,0 +1,839 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: dynamic.proto
/*
Package model is a generated protocol buffer package.
It is generated from these files:
dynamic.proto
It has these top-level messages:
Aids
Region
Tag
*/
package model
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type Aids struct {
IDs []int64 `protobuf:"varint,1,rep,packed,name=IDs" json:"id"`
}
func (m *Aids) Reset() { *m = Aids{} }
func (m *Aids) String() string { return proto.CompactTextString(m) }
func (*Aids) ProtoMessage() {}
func (*Aids) Descriptor() ([]byte, []int) { return fileDescriptorDynamic, []int{0} }
type Region struct {
Aids map[int32]*Aids `protobuf:"bytes,1,rep,name=aids" json:"aids,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"`
}
func (m *Region) Reset() { *m = Region{} }
func (m *Region) String() string { return proto.CompactTextString(m) }
func (*Region) ProtoMessage() {}
func (*Region) Descriptor() ([]byte, []int) { return fileDescriptorDynamic, []int{1} }
type Tag struct {
Aids map[string]*Aids `protobuf:"bytes,1,rep,name=aids" json:"aids,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"`
}
func (m *Tag) Reset() { *m = Tag{} }
func (m *Tag) String() string { return proto.CompactTextString(m) }
func (*Tag) ProtoMessage() {}
func (*Tag) Descriptor() ([]byte, []int) { return fileDescriptorDynamic, []int{2} }
func init() {
proto.RegisterType((*Aids)(nil), "model.Aids")
proto.RegisterType((*Region)(nil), "model.Region")
proto.RegisterType((*Tag)(nil), "model.Tag")
}
func (m *Aids) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Aids) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.IDs) > 0 {
dAtA2 := make([]byte, len(m.IDs)*10)
var j1 int
for _, num1 := range m.IDs {
num := uint64(num1)
for num >= 1<<7 {
dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j1++
}
dAtA2[j1] = uint8(num)
j1++
}
dAtA[i] = 0xa
i++
i = encodeVarintDynamic(dAtA, i, uint64(j1))
i += copy(dAtA[i:], dAtA2[:j1])
}
return i, nil
}
func (m *Region) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Region) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Aids) > 0 {
for k, _ := range m.Aids {
dAtA[i] = 0xa
i++
v := m.Aids[k]
msgSize := 0
if v != nil {
msgSize = v.Size()
msgSize += 1 + sovDynamic(uint64(msgSize))
}
mapSize := 1 + sovDynamic(uint64(k)) + msgSize
i = encodeVarintDynamic(dAtA, i, uint64(mapSize))
dAtA[i] = 0x8
i++
i = encodeVarintDynamic(dAtA, i, uint64(k))
if v != nil {
dAtA[i] = 0x12
i++
i = encodeVarintDynamic(dAtA, i, uint64(v.Size()))
n3, err := v.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n3
}
}
}
return i, nil
}
func (m *Tag) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Tag) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Aids) > 0 {
for k, _ := range m.Aids {
dAtA[i] = 0xa
i++
v := m.Aids[k]
msgSize := 0
if v != nil {
msgSize = v.Size()
msgSize += 1 + sovDynamic(uint64(msgSize))
}
mapSize := 1 + len(k) + sovDynamic(uint64(len(k))) + msgSize
i = encodeVarintDynamic(dAtA, i, uint64(mapSize))
dAtA[i] = 0xa
i++
i = encodeVarintDynamic(dAtA, i, uint64(len(k)))
i += copy(dAtA[i:], k)
if v != nil {
dAtA[i] = 0x12
i++
i = encodeVarintDynamic(dAtA, i, uint64(v.Size()))
n4, err := v.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n4
}
}
}
return i, nil
}
func encodeVarintDynamic(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *Aids) Size() (n int) {
var l int
_ = l
if len(m.IDs) > 0 {
l = 0
for _, e := range m.IDs {
l += sovDynamic(uint64(e))
}
n += 1 + sovDynamic(uint64(l)) + l
}
return n
}
func (m *Region) Size() (n int) {
var l int
_ = l
if len(m.Aids) > 0 {
for k, v := range m.Aids {
_ = k
_ = v
l = 0
if v != nil {
l = v.Size()
l += 1 + sovDynamic(uint64(l))
}
mapEntrySize := 1 + sovDynamic(uint64(k)) + l
n += mapEntrySize + 1 + sovDynamic(uint64(mapEntrySize))
}
}
return n
}
func (m *Tag) Size() (n int) {
var l int
_ = l
if len(m.Aids) > 0 {
for k, v := range m.Aids {
_ = k
_ = v
l = 0
if v != nil {
l = v.Size()
l += 1 + sovDynamic(uint64(l))
}
mapEntrySize := 1 + len(k) + sovDynamic(uint64(len(k))) + l
n += mapEntrySize + 1 + sovDynamic(uint64(mapEntrySize))
}
}
return n
}
func sovDynamic(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozDynamic(x uint64) (n int) {
return sovDynamic(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *Aids) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Aids: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Aids: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType == 0 {
var v int64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.IDs = append(m.IDs, v)
} else if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
packedLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if packedLen < 0 {
return ErrInvalidLengthDynamic
}
postIndex := iNdEx + packedLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
for iNdEx < postIndex {
var v int64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.IDs = append(m.IDs, v)
}
} else {
return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType)
}
default:
iNdEx = preIndex
skippy, err := skipDynamic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDynamic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Region) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Region: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Region: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Aids", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDynamic
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Aids == nil {
m.Aids = make(map[int32]*Aids)
}
var mapkey int32
var mapvalue *Aids
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapkey |= (int32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
} else if fieldNum == 2 {
var mapmsglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapmsglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if mapmsglen < 0 {
return ErrInvalidLengthDynamic
}
postmsgIndex := iNdEx + mapmsglen
if mapmsglen < 0 {
return ErrInvalidLengthDynamic
}
if postmsgIndex > l {
return io.ErrUnexpectedEOF
}
mapvalue = &Aids{}
if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil {
return err
}
iNdEx = postmsgIndex
} else {
iNdEx = entryPreIndex
skippy, err := skipDynamic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDynamic
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
m.Aids[mapkey] = mapvalue
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipDynamic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDynamic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Tag) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Tag: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Tag: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Aids", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDynamic
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Aids == nil {
m.Aids = make(map[string]*Aids)
}
var mapkey string
var mapvalue *Aids
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthDynamic
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
} else if fieldNum == 2 {
var mapmsglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDynamic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapmsglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if mapmsglen < 0 {
return ErrInvalidLengthDynamic
}
postmsgIndex := iNdEx + mapmsglen
if mapmsglen < 0 {
return ErrInvalidLengthDynamic
}
if postmsgIndex > l {
return io.ErrUnexpectedEOF
}
mapvalue = &Aids{}
if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil {
return err
}
iNdEx = postmsgIndex
} else {
iNdEx = entryPreIndex
skippy, err := skipDynamic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDynamic
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
m.Aids[mapkey] = mapvalue
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipDynamic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDynamic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipDynamic(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDynamic
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDynamic
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDynamic
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthDynamic
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDynamic
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipDynamic(dAtA[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthDynamic = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowDynamic = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("dynamic.proto", fileDescriptorDynamic) }
var fileDescriptorDynamic = []byte{
// 253 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4d, 0xa9, 0xcc, 0x4b,
0xcc, 0xcd, 0x4c, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xcd, 0xcd, 0x4f, 0x49, 0xcd,
0x91, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f,
0xcf, 0xd7, 0x07, 0xcb, 0x26, 0x95, 0xa6, 0x81, 0x79, 0x60, 0x0e, 0x98, 0x05, 0xd1, 0xa5, 0xa4,
0xc0, 0xc5, 0xe2, 0x98, 0x99, 0x52, 0x2c, 0x24, 0xc1, 0xc5, 0xec, 0xe9, 0x52, 0x2c, 0xc1, 0xa8,
0xc0, 0xac, 0xc1, 0xec, 0xc4, 0xf6, 0xea, 0x9e, 0x3c, 0x53, 0x66, 0x4a, 0x10, 0x48, 0x48, 0xa9,
0x9a, 0x8b, 0x2d, 0x28, 0x35, 0x3d, 0x33, 0x3f, 0x4f, 0x48, 0x9b, 0x8b, 0x25, 0x31, 0x33, 0x05,
0xa2, 0x88, 0xdb, 0x48, 0x5c, 0x0f, 0x6c, 0xa1, 0x1e, 0x44, 0x52, 0x0f, 0x64, 0x8a, 0x6b, 0x5e,
0x49, 0x51, 0x65, 0x10, 0x58, 0x91, 0x94, 0x0b, 0x17, 0x27, 0x5c, 0x48, 0x48, 0x80, 0x8b, 0x39,
0x3b, 0xb5, 0x52, 0x82, 0x51, 0x81, 0x51, 0x83, 0x35, 0x08, 0xc4, 0x14, 0x52, 0xe4, 0x62, 0x2d,
0x4b, 0xcc, 0x29, 0x4d, 0x95, 0x60, 0x52, 0x60, 0xd4, 0xe0, 0x36, 0xe2, 0x86, 0x1a, 0x06, 0xd2,
0x12, 0x04, 0x91, 0xb1, 0x62, 0xb2, 0x60, 0x54, 0x2a, 0xe5, 0x62, 0x0e, 0x49, 0x4c, 0x17, 0xd2,
0x40, 0xb1, 0x59, 0x04, 0xaa, 0x38, 0x24, 0x31, 0x9d, 0x68, 0x6b, 0x39, 0x49, 0xb1, 0xd6, 0x49,
0xe2, 0xc4, 0x43, 0x39, 0x86, 0x0b, 0x0f, 0xe5, 0x18, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48,
0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x19, 0x8f, 0xe5, 0x18, 0x92, 0xd8, 0xc0, 0xc1, 0x66, 0x0c,
0x08, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x1b, 0x74, 0xd1, 0x7d, 0x01, 0x00, 0x00,
}

View File

@@ -0,0 +1,19 @@
syntax = "proto3";
package model;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_enum_prefix_all) = false;
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.unmarshaler_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.sizer_all) = true;
message Aids {
repeated int64 IDs = 1 [(gogoproto.jsontag) = "id"];
}
message Region {
map<int32,Aids> aids = 1;
}
message Tag {
map<string,Aids> aids = 1;
}

View File

@@ -0,0 +1,30 @@
package model
// ArgRegionTotal arg region total
type ArgRegionTotal struct {
RealIP string
}
// ArgRegion3 arg region
type ArgRegion3 struct {
RegionID int32
Pn int
Ps int
RealIP string
}
// ArgRegionTag3 arg region tag
type ArgRegionTag3 struct {
TagID int64
RegionID int32
Pn int
Ps int
RealIP string
}
// ArgRegions3 arg regions
type ArgRegions3 struct {
RegionIDs []int32
Count int
RealIP string
}

View File

@@ -0,0 +1,17 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/dynamic/rpc/client:all-srcs",
"//app/service/main/dynamic/rpc/server:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,32 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["dynamic.go"],
importpath = "go-common/app/service/main/dynamic/rpc/client",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/dynamic/model:go_default_library",
"//library/net/rpc:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,58 @@
package client
import (
"context"
"go-common/app/service/main/archive/api"
"go-common/app/service/main/dynamic/model"
"go-common/library/net/rpc"
)
const (
_regionArcs3 = "RPC.RegionArcs3"
_regionTagArcs3 = "RPC.RegionTagArcs3"
_regionsArcs3 = "RPC.RegionsArcs3"
_regionTotal = "RPC.RegionTotal"
)
const (
_appid = "archive.service.dynamic"
)
// Service struct info.
type Service struct {
client *rpc.Client2
}
// New new service instance and return.
func New(c *rpc.ClientConfig) (s *Service) {
s = &Service{}
s.client = rpc.NewDiscoveryCli(_appid, c)
return
}
// RegionTotal receive real ip,then return dynamic region total.
func (s *Service) RegionTotal(c context.Context, arg *model.ArgRegionTotal) (res map[string]int, err error) {
err = s.client.Call(c, _regionTotal, arg, &res)
return
}
// RegionArcs3 receive ArgRegion contains regionId and real ip, then return dynamic archives.
func (s *Service) RegionArcs3(c context.Context, arg *model.ArgRegion3) (res *model.DynamicArcs3, err error) {
res = new(model.DynamicArcs3)
err = s.client.Call(c, _regionArcs3, arg, res)
return
}
// RegionTagArcs3 receive ArgRegionTag contains tagId and regionId and real ip, then return dynamic archives3.
func (s *Service) RegionTagArcs3(c context.Context, arg *model.ArgRegionTag3) (res *model.DynamicArcs3, err error) {
res = new(model.DynamicArcs3)
err = s.client.Call(c, _regionTagArcs3, arg, res)
return
}
// RegionsArcs3 receive ArgRegion contains regionIds and real ip, then return dynamic archives3.
func (s *Service) RegionsArcs3(c context.Context, arg *model.ArgRegions3) (res map[int32][]*api.Arc, err error) {
err = s.client.Call(c, _regionsArcs3, arg, &res)
return
}

View File

@@ -0,0 +1,49 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["rpc_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/dynamic/model:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = ["rpc.go"],
importpath = "go-common/app/service/main/dynamic/rpc/server",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/dynamic/conf:go_default_library",
"//app/service/main/dynamic/model:go_default_library",
"//app/service/main/dynamic/service:go_default_library",
"//library/net/rpc:go_default_library",
"//library/net/rpc/context:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,68 @@
package server
import (
"go-common/app/service/main/archive/api"
"go-common/app/service/main/dynamic/conf"
"go-common/app/service/main/dynamic/model"
"go-common/app/service/main/dynamic/service"
"go-common/library/net/rpc"
"go-common/library/net/rpc/context"
)
// RPC struct info.
type RPC struct {
s *service.Service
}
// New new rpc server.
func New(c *conf.Config, s *service.Service) (svr *rpc.Server) {
r := &RPC{s: s}
svr = rpc.NewServer(c.RPCServer)
if err := svr.Register(r); err != nil {
panic(err)
}
return
}
// Ping check connection success.
func (r *RPC) Ping(c context.Context, arg *struct{}, res *struct{}) (err error) {
return
}
//RegionTotal return dynamic region total.
func (r *RPC) RegionTotal(c context.Context, a *model.ArgRegionTotal, res *map[string]int) (err error) {
*res = r.s.RegionTotal(c)
return
}
// RegionArcs3 receive aid, then init archive3 info.
func (r *RPC) RegionArcs3(c context.Context, a *model.ArgRegion3, res *model.DynamicArcs3) (err error) {
var (
count int
arcs []*api.Arc
)
if arcs, count, err = r.s.RegionArcs3(c, a.RegionID, a.Pn, a.Ps); err == nil {
res.Page = &model.Page{Num: a.Pn, Size: a.Ps, Count: count}
res.Archives = arcs
}
return
}
// RegionTagArcs3 receive aid, then init archive info.
func (r *RPC) RegionTagArcs3(c context.Context, a *model.ArgRegionTag3, res *model.DynamicArcs3) (err error) {
var (
count int
arcs []*api.Arc
)
if arcs, count, err = r.s.RegionTagArcs3(c, a.RegionID, a.TagID, a.Pn, a.Ps); err == nil {
res.Page = &model.Page{Num: a.Pn, Size: a.Ps, Count: count}
res.Archives = arcs
}
return
}
// RegionsArcs3 receive rids and return dynamic archives3.
func (r *RPC) RegionsArcs3(c context.Context, a *model.ArgRegions3, res *map[int32][]*api.Arc) (err error) {
*res, err = r.s.RegionsArcs3(c, a.RegionIDs, a.Count)
return
}

View File

@@ -0,0 +1,94 @@
package server
import (
"net/rpc"
"testing"
"go-common/app/service/main/archive/api"
"go-common/app/service/main/dynamic/model"
. "github.com/smartystreets/goconvey/convey"
)
const (
_addr = "127.0.0.1:6235"
_regionArcs3 = "RPC.RegionArcs3"
_regionTagArcs3 = "RPC.RegionTagArcs3"
_regionsArcs3 = "RPC.RegionsArcs3"
_regionTotal = "RPC.RegionTotal"
)
func Test_RPC(t *testing.T) {
Convey("RPC Region Archives", t, func() {
client, err := rpc.Dial("tcp", _addr)
So(err, ShouldBeNil)
defer client.Close()
testRPCRegionArcs(t, client)
testRPCRegionTagArcs(t, client)
testRPCRegionsArcs(t, client)
testRPCRegionTotal(t, client)
})
}
func testRPCRegionArcs(t *testing.T, client *rpc.Client) {
var (
rid int32 = 168
pn, ps int = 1, 10
ip = "127.0.0.1"
res *model.DynamicArcs3
)
Convey("RPC Region Archives", func() {
arg := &model.ArgRegion3{RegionID: rid, Pn: pn, Ps: ps, RealIP: ip}
res = new(model.DynamicArcs3)
err := client.Call(_regionArcs3, arg, &res)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
})
}
func testRPCRegionTagArcs(t *testing.T, client *rpc.Client) {
var (
rid int32 = 168
tid int64 = 123456
pn, ps int = 1, 10
ip = "127.0.0.1"
res *model.DynamicArcs3
)
Convey("RPC Region Tag Archives", func() {
arg := &model.ArgRegionTag3{TagID: tid, RegionID: rid, Pn: pn, Ps: ps, RealIP: ip}
res = new(model.DynamicArcs3)
err := client.Call(_regionTagArcs3, arg, &res)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
})
}
func testRPCRegionsArcs(t *testing.T, client *rpc.Client) {
var (
rids = []int32{1, 3, 4, 5, 13, 36, 129, 119, 23, 11, 155, 160, 165, 168}
count = 10
ip = "127.0.0.1"
res *map[int32][]*api.Arc
)
Convey("RPC Regions Archives", func() {
arg := &model.ArgRegions3{RegionIDs: rids, Count: count, RealIP: ip}
err := client.Call(_regionsArcs3, arg, &res)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
})
}
func testRPCRegionTotal(t *testing.T, client *rpc.Client) {
var (
ip = "127.0.0.1"
res map[string]int
)
Convey("RPC Region Total", func() {
arg := &model.ArgRegionTotal{RealIP: ip}
err := client.Call(_regionTotal, arg, &res)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
})
}

View File

@@ -0,0 +1,58 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dynamic_test.go",
"service_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/dynamic/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dynamic.go",
"service.go",
],
importpath = "go-common/app/service/main/dynamic/service",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/api:go_default_library",
"//app/service/main/archive/api/gorpc:go_default_library",
"//app/service/main/archive/model/archive:go_default_library",
"//app/service/main/dynamic/conf:go_default_library",
"//app/service/main/dynamic/dao:go_default_library",
"//library/cache:go_default_library",
"//library/log:go_default_library",
"//library/net/metadata:go_default_library",
"//library/sync/errgroup:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,246 @@
package service
import (
"context"
"strconv"
"sync"
"go-common/app/service/main/archive/api"
arcmdl "go-common/app/service/main/archive/model/archive"
"go-common/app/service/main/dynamic/conf"
"go-common/app/service/main/dynamic/dao"
"go-common/library/log"
"go-common/library/net/metadata"
"go-common/library/sync/errgroup"
)
var (
_emptyArcs3 = make([]*api.Arc, 0)
)
const (
_arcMuch = 5
_guoChuang = 168
)
// RegionTotal total dynamic of regin
func (s *Service) RegionTotal(c context.Context) (res map[string]int) {
res = map[string]int{}
for k, v := range s.regionTotal {
res[strconv.FormatInt(int64(k), 10)] = v
}
res["live"] = s.live
return
}
// regionNeedCache
func regionNeedCache(regionArcs map[int32][]int64) bool {
if len(regionArcs) <= 0 {
return false
}
for k, v := range regionArcs {
if len(v) < conf.Conf.Rule.MinRegionCount {
log.Error("arcNeedCache key(%d) len(%v)<=0", k, v)
return false
}
}
return true
}
// RegionArcs3 get new arcs of region.
func (s *Service) RegionArcs3(c context.Context, rid int32, pn, ps int) (arcs []*api.Arc, count int, err error) {
var (
ok bool
start, end int
aids []int64
)
if aids, ok = s.regionArcs[rid]; !ok {
arcs = _emptyArcs3
return
}
count = len(aids)
start = (pn - 1) * ps
end = start + ps + _arcMuch
if start > count {
arcs = _emptyArcs3
return
}
if end > count {
aids = aids[start:]
} else {
aids = aids[start:end]
}
if arcs, err = s.normalArcs3(c, aids, ps); err != nil {
log.Error("archives(%v) error(%v)", aids, err)
}
return
}
// normalArcs3 .
func (s *Service) normalArcs3(c context.Context, aids []int64, ps int) (res []*api.Arc, err error) {
var (
arcs []*api.Arc
tmpRes map[int64]*api.Arc
)
archivesLog("normalArcs3", aids)
if tmpRes, err = s.archives3(c, aids); err != nil {
return
}
for _, aid := range aids {
if arc, ok := tmpRes[aid]; ok {
arcs = append(arcs, arc)
} else {
log.Error("normalArcs s.archives aid(%d) nil", aid)
}
}
res = filterArc3(arcs, ps)
return
}
// archives3 .
func (s *Service) archives3(c context.Context, aids []int64) (res map[int64]*api.Arc, err error) {
arg := &arcmdl.ArgAids2{Aids: aids}
if res, err = s.arcRPC.Archives3(c, arg); err != nil {
dao.PromError("稿件rpc接口:Archives2", "archives s.arcRPC.Archives3(%v) error(%v)", aids, err)
}
return
}
func filterArc3(arcs []*api.Arc, count int) (res []*api.Arc) {
tmpPs := 1
for _, arc := range arcs {
if tmpPs <= count && isShow3(arc) {
res = append(res, arc)
tmpPs = tmpPs + 1
} else if tmpPs > count {
break
}
}
return
}
// isShow3
func isShow3(a *api.Arc) bool {
return a.IsNormal() && a.AttrVal(arcmdl.AttrBitNoDynamic) == arcmdl.AttrNo
}
// RegionTagArcs3 get new arcs of region and hot tag.
func (s *Service) RegionTagArcs3(c context.Context, rid int32, tagID int64, pn, ps int) (arcs []*api.Arc, count int, err error) {
var (
ok bool
start, end int
aids []int64
)
key := regionTagKey(rid, tagID)
if aids, ok = s.regionTagArcs[key]; !ok {
arcs = _emptyArcs3
return
}
count = len(aids)
start = (pn - 1) * ps
end = start + ps + _arcMuch
if start > count {
arcs = _emptyArcs3
return
}
if end > count {
aids = aids[start:]
} else {
aids = aids[start:end]
}
if arcs, err = s.normalArcs3(c, aids, ps); err != nil {
log.Error("archives(%v) error(%v)", aids, err)
}
return
}
// RegionsArcs3 get batch new arcs of regions.
func (s *Service) RegionsArcs3(c context.Context, rids []int32, count int) (mArcs map[int32][]*api.Arc, err error) {
var (
ok bool
noRids []int32
allAids []int64
aids []int64
mAids map[int32][]int64
res map[int64]*api.Arc
ip = metadata.String(c, metadata.RemoteIP)
)
mAids = make(map[int32][]int64, len(rids))
for _, rid := range rids {
end := count + _arcMuch
if aids, ok = s.regionArcs[rid]; !ok || len(aids) == 0 {
continue
}
if end > len(aids) {
end = len(aids)
}
allAids = append(allAids, aids[:end]...)
mAids[rid] = aids[:end]
}
archivesLog("RegionsArcs3", allAids)
if res, err = s.archives3(c, allAids); err != nil {
log.Error("archives(%v) error(%v)", allAids, err)
return
}
mArcs = make(map[int32][]*api.Arc, len(rids))
for _, rid := range rids {
var arcs []*api.Arc
for _, aid := range mAids[rid] {
if arc, ok := res[aid]; ok {
arcs = append(arcs, arc)
} else {
log.Error("RegionsArcs s.archives aid(%d) nil", aid)
}
}
mArcs[rid] = filterArc3(arcs, count)
if len(mArcs[rid]) < count {
dao.PromError("一级分区数据错误", "RegionsArcs rid(%d) len(mArcs[rid])(%d) count(%d)", rid, len(mArcs[rid]), count)
noRids = append(noRids, rid)
}
}
//last back up from rankIndexArc.
if len(noRids) > 0 {
err = s.rankIndexArc3(c, noRids, 1, count, ip, mArcs)
}
return
}
// rankIndexArc3 archives3.
func (s *Service) rankIndexArc3(c context.Context, rids []int32, pn, ps int, ip string, mArcs map[int32][]*api.Arc) (err error) {
var mutex = sync.Mutex{}
group, errCtx := errgroup.WithContext(c)
for _, rid := range rids {
trid := rid
group.Go(func() (err error) {
var (
topRes, tmpRes []*api.Arc
regionRes = &arcmdl.RankArchives3{}
)
if trid == _guoChuang {
arg := &arcmdl.ArgRank2{Rid: int16(trid), Type: 0, Pn: pn, Ps: ps + _arcMuch, RealIP: ip}
if regionRes, err = s.arcRPC.RankArcs3(errCtx, arg); err != nil {
dao.PromError("RankArcs2接口错误", "arcRPC.RankArcs2(%d,%d,%d,%s) error(%v)", trid, pn, ps, ip, err)
return nil
}
topRes = regionRes.Archives
} else {
arg := &arcmdl.ArgRankTop2{ReID: int16(trid), Pn: pn, Ps: ps + _arcMuch}
if topRes, err = s.arcRPC.RankTopArcs3(errCtx, arg); err != nil {
dao.PromError("RankTopArcs2接口错误", "arcRPC.RankTopArcs2(%d,%d,%d,%s) error(%v)", trid, pn, ps, ip, err)
return nil
}
}
tmpRes = filterArc3(topRes, ps)
mutex.Lock()
if len(tmpRes) == 0 {
mArcs[trid] = _emptyArcs3
} else {
mArcs[trid] = tmpRes
}
mutex.Unlock()
return nil
})
}
group.Wait()
return
}

View File

@@ -0,0 +1,71 @@
package service
import (
"context"
"go-common/app/service/main/archive/api"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestService_RegionTotal(t *testing.T) {
Convey("RegionTotal BigData", t, WithService(func(s *Service) {
for _, v := range s.regionTotal {
So(v, ShouldBeGreaterThan, 0)
}
}))
Convey("RegionTotal Live", t, WithService(func(s *Service) {
So(s.live, ShouldBeGreaterThan, 0)
}))
}
func TestService_RegionArcs(t *testing.T) {
var (
c = context.TODO()
rid int32 = 168
pn, ps, count int = 1, 10, 0
err error
rs []*api.Arc
)
Convey("Region Arcives", t, WithService(func(s *Service) {
rs, count, err = s.RegionArcs3(c, rid, pn, ps)
So(err, ShouldBeNil)
So(count, ShouldBeGreaterThan, 0)
Printf("%+v", rs)
}))
}
func TestService_RegionsArcs(t *testing.T) {
var (
c = context.TODO()
rids = []int32{1, 3, 4, 5, 13, 36, 129, 119, 23, 11, 155, 160, 165, 168}
count = 15
err error
rs map[int32][]*api.Arc
)
Convey("Regions Archives", t, WithService(func(s *Service) {
rs, err = s.RegionsArcs3(c, rids, count)
So(err, ShouldBeNil)
Printf("%+v", rs)
}))
}
func TestService_RegionTagArcs(t *testing.T) {
var (
c = context.TODO()
tid int64 = 123456
rid int32 = 168
pn, ps, count int = 1, 5, 0
err error
rs []*api.Arc
)
Convey("Region Tag Archives", t, WithService(func(s *Service) {
rs, count, err = s.RegionTagArcs3(c, rid, tid, pn, ps)
So(err, ShouldBeNil)
Printf("%+v", rs)
Printf("%d", count)
}))
}

View File

@@ -0,0 +1,198 @@
package service
import (
"context"
"fmt"
"time"
arcrpc "go-common/app/service/main/archive/api/gorpc"
"go-common/app/service/main/dynamic/conf"
"go-common/app/service/main/dynamic/dao"
"go-common/library/cache"
"go-common/library/log"
)
// Service service.
type Service struct {
dao *dao.Dao
c *conf.Config
// new dynamic arcs.
hotRidTids map[int32][]int64
regionTotal map[int32]int
regionArcs map[int32][]int64
regionTagArcs map[string][]int64
// live
live int
// rpc
arcRPC *arcrpc.Service2
// cache
cache *cache.Cache
}
// New service new.
func New(c *conf.Config) *Service {
s := &Service{
dao: dao.New(c),
c: c,
// rpc
arcRPC: arcrpc.New2(c.ArchiveRPC),
// new dynamic arcs
hotRidTids: make(map[int32][]int64),
regionTotal: make(map[int32]int),
regionArcs: make(map[int32][]int64),
regionTagArcs: make(map[string][]int64),
cache: cache.New(1, 1024),
}
go s.regionproc()
go s.tagproc()
return s
}
func regionTagKey(rid int32, tagID int64) string {
return fmt.Sprintf("%d_%d", rid, tagID)
}
// regionproc is a routine for pull region dynamic into cache.
func (s *Service) regionproc() {
var (
c = context.TODO()
res map[int32][]int64
rids []int32
cacheRegion map[int32][]int64
err error
)
for {
// load hot tags from tag api.
regionTotal := make(map[int32]int)
regionArcs := make(map[int32][]int64)
if res, err = s.dao.Hot(c); err != nil {
log.Error("dao.Hot() error(%v)", err)
time.Sleep(time.Second)
continue
}
if len(res) > 0 {
s.hotRidTids = res
}
if rids, err = s.dao.Rids(c); err != nil {
log.Error("dao.Rids() error(%v)", err)
time.Sleep(time.Second)
continue
}
//get region cache
cacheRegion = s.dao.RegionCache(c)
// init dynamic arcs from bigdata.
for rid := range s.hotRidTids {
// init region dynamic arcs.
if aids, total, err := s.dao.RegionArcs(c, rid, ""); err != nil || len(aids) < conf.Conf.Rule.MinRegionCount {
regionTotal[rid] = s.regionTotal[rid]
if len(s.regionArcs[rid]) >= conf.Conf.Rule.MinRegionCount {
regionArcs[rid] = s.regionArcs[rid]
} else if cacheRegion != nil && len(cacheRegion[rid]) >= conf.Conf.Rule.MinRegionCount {
regionArcs[rid] = cacheRegion[rid]
} else {
dao.PromError("热门动态数据错误", "dynamic data error rid(%d) bigdata(%d) memory(%d)", rid, len(aids), len(s.regionArcs[rid]))
if len(aids) > 0 {
regionArcs[rid] = aids
} else {
regionArcs[rid] = s.regionArcs[rid]
}
}
} else {
regionTotal[rid] = total
regionArcs[rid] = aids
}
}
for _, rid := range rids {
if aids, total, err := s.dao.RegionArcs(c, rid, ""); err != nil || len(aids) < conf.Conf.Rule.MinRegionCount {
regionTotal[rid] = s.regionTotal[rid]
if len(s.regionArcs[rid]) >= conf.Conf.Rule.MinRegionCount {
regionArcs[rid] = s.regionArcs[rid]
} else if cacheRegion != nil && len(cacheRegion[rid]) >= conf.Conf.Rule.MinRegionCount {
regionArcs[rid] = cacheRegion[rid]
} else {
dao.PromError("分区动态数据错误", "dynamic data error rid(%d) bigdata(%d) memory(%d)", rid, len(aids), len(s.regionArcs[rid]))
if len(aids) > 0 {
regionArcs[rid] = aids
} else {
regionArcs[rid] = s.regionArcs[rid]
}
}
} else {
regionTotal[rid] = total
regionArcs[rid] = aids
}
}
if count, err := s.dao.Live(c); err != nil {
log.Error("s.dao.Live() error(%v)", err)
} else {
s.live = count
}
s.regionTotal = regionTotal
s.regionArcs = regionArcs
if regionNeedCache(s.regionArcs) {
s.cache.Save(func() {
s.dao.SetRegionCache(context.TODO(), s.regionArcs)
})
}
time.Sleep(time.Duration(s.c.Rule.TickRegion))
}
}
// tagproc is a routine for pull tag dynamic into cache.
func (s *Service) tagproc() {
var (
c = context.TODO()
cacheTag map[string][]int64
tagNeedCache bool
)
for {
//get tag cache
cacheTag = s.dao.TagCache(c)
// load hot tags from tag api.
regionTagArcs := make(map[string][]int64)
// init dynamic arcs from bigdata.
for rid, tids := range s.hotRidTids {
// init region tag dynamic arcs.
for _, tid := range tids {
k := regionTagKey(rid, tid)
if aids, err := s.dao.RegionTagArcs(c, rid, tid, ""); err != nil || len(aids) == 0 {
if len(s.regionTagArcs[k]) == 0 {
if cacheTag != nil && len(cacheTag[k]) > 0 {
regionTagArcs[k] = cacheTag[k]
}
tagNeedCache = false || tagNeedCache
} else {
regionTagArcs[k] = s.regionTagArcs[k]
tagNeedCache = true
}
} else {
regionTagArcs[k] = aids
tagNeedCache = true
}
}
}
s.regionTagArcs = regionTagArcs
if tagNeedCache {
s.cache.Save(func() {
s.dao.SetTagCache(context.TODO(), s.regionTagArcs)
})
}
time.Sleep(time.Duration(s.c.Rule.TickTag))
}
}
// Ping check server ok
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close dao
func (s *Service) Close() {
s.dao.Close()
}
func archivesLog(name string, aids []int64) {
if aidLen := len(aids); aidLen >= 50 {
log.Info("s.archives3 func(%s) len(%d), arg(%v)", name, aidLen, aids)
}
}

View File

@@ -0,0 +1,27 @@
package service
import (
"flag"
"path/filepath"
"time"
"go-common/app/service/main/dynamic/conf"
)
var (
svr *Service
)
func init() {
dir, _ := filepath.Abs("../cmd/dynamic-service-test.toml")
flag.Set("conf", dir)
conf.Init()
svr = New(conf.Conf)
time.Sleep(time.Second)
}
func WithService(f func(s *Service)) func() {
return func() {
f(svr)
}
}