openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
app/service/main/archive/cmd/BUILD Normal file
View File

@@ -0,0 +1,48 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = [
"archive-service-example.toml",
"archive-service-test.toml",
],
importpath = "go-common/app/service/main/archive/cmd",
tags = ["automanaged"],
deps = [
"//app/service/main/archive/conf:go_default_library",
"//app/service/main/archive/server/gorpc:go_default_library",
"//app/service/main/archive/server/grpc:go_default_library",
"//app/service/main/archive/server/http:go_default_library",
"//app/service/main/archive/service:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

261
app/service/main/archive/cmd/archive-service-example.toml Normal file
View File

@@ -0,0 +1,261 @@
# This is a TOML document. Boom.
version = "3.0.0"
user = "nobody"
pid = "/tmp/archive-service.pid"
dir = "./"
perf = "127.0.0.1:6080"
checkFile = "/data/www/archive.html"
family = "archive-service"
videoshotURI = "http://i3.hdslb.com/bfs/videoshot/"
videoshotKey = "f555a0f529152dc8c6b1ae60c2af4164"
tick = "5m"
[xlog]
dir = "/data/log/archive-service/"
[xlog.elk]
project = "archive-service"
addr = "172.18.20.17:8520"
chanSize = 1024
[ecode]
service = "archive-service"
[ecode.mysql]
name = "[archive]tcp@172.16.0.148:3306"
dsn = "test:test@tcp(172.16.0.148:3306)/bilibili_apm?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 5
[statsd]
project = "archive-service"
addr = "172.18.20.15:8200"
chanSize = 10240
[app]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
[multiHTTP]
[multiHTTP.inner]
addrs = ["0.0.0.0:6081"]
maxListen = 1000
[multiHTTP.local]
addrs = ["0.0.0.0:6082"]
maxListen = 100
[rpcServer2]
[[rpcServer2.servers]]
proto = "tcp"
addr = "127.0.0.1:6089"
weight = 10
[rpcServer2.zookeeper]
root = "/microservice/archive-service/"
addrs = ["127.0.0.1:2181"]
timeout = "1s"
[db]
[db.arc]
name = "[archive]tcp@172.16.0.148:3306"
dsn = "test:test@tcp(172.16.0.148:3306)/bilibili_archive?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 5
[db.dede]
name = "[dede]tcp@172.16.0.5:3306"
dsn = "bilibili:5Fq2M4FbPZK4fhtE@tcp(172.16.0.5:3306)/bilibili?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 5
[identify]
whiteAccessKey = ""
whiteMid = 0
[identify.app]
key = "6a29f8ed87407c11"
secret = "d3c5a85f5b895a03735b5d20a273bc57"
[identify.memcache]
name = "go-business/identify"
proto = "tcp"
addr = "172.16.33.54:11211"
active = 5
idle = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
[identify.host]
auth = "http://passport.bilibili.com"
secret = "http://open.bilibili.com"
[identify.authHTTPClient]
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[identify.secretHTTPClient]
dial = "1s"
timeout = "1s"
keepAlive = "60s"
timer = 1000
[memcache]
[memcache.archive]
name = "archive-service"
proto = "tcp"
addr = "172.16.0.148:11211"
active = 100
idle = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
expire = "10s"
[memcache.video]
name = "archive-service"
proto = "tcp"
addr = "172.16.0.148:11211"
active = 100
idle = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
expire = "10s"
[redis]
[redis.videoshot]
name = "archive-service/videoshot"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[redis.archive]
name = "archive-service/archive"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "8h"
[redis.region]
name = "archive-service/region"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
[redis.moment]
name = "archive-service/moment"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[redis.relation]
name = "archive-service/relation"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
[redis.share]
name = "archive-service/share"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[redis.stat]
name = "archive-service/stat"
proto = "tcp"
addr = "172.16.0.148:6379"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "10s"
[hbase]
master = ""
meta = ""
dialTimeout = "1s"
readTimeout = "200ms"
readsTimeout = "1s"
writeTimeout = "200ms"
writesTimeout = "200ms"
[hbase.zookeeper]
root = ""
addrs = ["172.16.33.45:2181"]
timeout = "30s"
[accountRPC]
pullInterval = "10s"
[accountRPC.client]
proto = "tcp"
addr = "127.0.0.1:6079"
timeout = "1s"
timer = 1000
[[accountRPC.backup]]
proto = "tcp"
addr = "127.0.0.1:6079"
timeout = "1s"
timer = 1000
[accountRPC.zookeeper]
root = "/microservice/account-service/"
addrs = ["127.0.0.1:2181"]
timeout = "1s"
[tracer]
proto = "udp"
addr = "172.16.33.46:5140"
tag = "platform/archive-service"
[kafkaProducer]
statTopic = "stat"
firstShareTopic = "share_first"
brokers = ["172.16.0.148:9092"]
sync = false
[kafkaProducer.zookeeper]
root = "/kafka"
addrs = ["172.16.0.148:2181"]
timeout = "200ms"
[databus]
key = "0PtNTguCX35XCtPpjUGC"
secret= "0PtNTguCX35XCtPpjUGD"
group= "Stat-UGC-P"
topic= "Stat-T"
action="pub"
[databus.redis]
name = "archive-service/stat"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"

343
app/service/main/archive/cmd/archive-service-test.toml Normal file
View File

@@ -0,0 +1,343 @@
tick = "5m"
playerQn = [15, 16, 32, 48, 64, 80]
playerVipQn = [74,112,116]
PlayerAPI = "http://172.18.33.143:13500/playurl/batch"
PGCPlayerAPI = "http://api.bilibili.co/pgc/internal/dynamic/aid/list"
PlayerSwitch = true
[videoshot]
uri = "http://i3.hdslb.com/bfs/videoshot/"
key = "f555a0f529152dc8c6b1ae60c2af4164"
[xlog]
dir = "/data/log/archive-service/"
[statsd]
project = "archive-service"
addr = "172.18.20.15:8200"
chanSize = 10240
[app]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
[bm]
[bm.inner]
addr = "0.0.0.0:6081"
timeout = "1s"
[bm.local]
addr = "0.0.0.0:6082"
timeout = "1s"
[rpcServer2]
discoverOff = false
token = "123456"
[[rpcServer2.servers]]
proto = "tcp"
addr = "0.0.0.0:6089"
weight = 10
group = "test"
[rpcServer2.zookeeper]
root = "/microservice/archive-service/"
addrs = ["172.18.33.172:2181"]
timeout = "30s"
[db]
[db.arc]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_archive?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 1
idleTimeout ="4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.arc.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.arcRead]
addr = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_archive?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 5
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.arcRead.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.arcResult]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_result?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 1
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.arcResult.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.stat]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_stat?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 1
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.stat.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.click]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_click?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 5
idle = 1
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.click.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.like]
name = "172.16.33.205:3308"
dsn = "test:test@tcp(172.16.33.205:3308)/archive_like?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8"
active = 10
idle = 5
idleTimeout = "4h"
queryTimeout = "150ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[db.like.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[ecode]
domain = "172.16.33.248:6401"
all = "1h"
diff = "5m"
[ecode.clientconfig]
key = "test"
secret = "e6c4c252dc7e3d8a90805eecd7c73396"
dial = "2000ms"
timeout = "2s"
keepAlive = "10s"
timer = 128
[ecode.clientconfig.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[ecode.app]
key = "test"
secret = "e6c4c252dc7e3d8a90805eecd7c73396"
[identify]
whiteAccessKey = ""
whiteMid = 0
[identify.app]
key = "6a29f8ed87407c11"
secret = "d3c5a85f5b895a03735b5d20a273bc57"
[identify.host]
auth = "http://passport.bilibili.com"
secret = "http://open.bilibili.com"
[identify.httpClient]
key = "6a29f8ed87407c11"
secret = "d3c5a85f5b895a03735b5d20a273bc57"
dial = "30ms"
timeout = "100ms"
keepAlive = "60s"
[identify.httpClient.breaker]
window = "10s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[identify.httpClient.url]
"http://passport.bilibili.co/intranet/auth/tokenInfo" = {timeout = "100ms"}
"http://passport.bilibili.co/intranet/auth/cookieInfo" = {timeout = "100ms"}
"http://open.bilibili.co/api/getsecret" = {timeout = "500ms"}
[memcache]
[memcache.archive]
name = "archive-service"
proto = "tcp"
addr = "172.18.33.61:11211"
active = 50
idle = 10
dialTimeout = "30ms"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
archiveExpire = "10h"
videoExpire = "10h"
[redis]
[redis.archive]
name = "archive-service/archive"
proto = "tcp"
addr = "172.16.33.54:6379"
idle = 50
active = 100
dialTimeout = "100ms"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
expire = "8h"
[redis.relation]
name = "archive-service/relation"
proto = "tcp"
addr = "172.16.33.54:6379"
idle = 10
active = 10
dialTimeout = "5s"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
[redis.like]
name = "archive-service/like"
proto = "tcp"
addr = "172.16.33.54:6379"
idle = 10
active = 10
dialTimeout = "30ms"
readTimeout = "200ms"
writeTimeout = "200ms"
idleTimeout = "80s"
[kafkaProducer]
firstShareTopic = "share_first"
brokers = ["172.16.33.54:9092"]
sync = false
[kafkaProducer.zookeeper]
root = "/kafka"
addrs = ["172.18.33.172:2181"]
timeout = "30s"
[databus]
key = "0PtNTguCX35XCtPpjUGC"
secret= "0PtNTguCX35XCtPpjUGD"
group= "Stat-UGC-P"
topic= "Stat-T"
action="pub"
name = "archive-service/stat"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[CacheDatabus]
key = "8e27ab7e39270b59"
secret= "477df6a068d7332a163f95abbad2079c"
group= "ArchiveUserCache-MainAppSvr-P"
topic= "ArchiveUserCache-T"
action="pub"
name = "archive-service/stat"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 5
active = 50
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[StatDatabus]
key = "4c76cbb7a985ac90"
secret= "8c22a196bf00d623b69cba4e61b1f7dc"
group= "StatShare-MainArchive-P"
topic= "StatShare-T"
action="pub"
offset="old"
buffer=1024
name = "archive-service/StatShare"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 5
active = 50
dialTimeout = "80ms"
readTimeout = "80ms"
writeTimeout = "80ms"
idleTimeout = "80s"
[shareDatabus]
key = "8e27ab7e39270b59"
secret= "e5d94158a1118c14e1449481023edacf"
group= "AccountExp-MainAppSvr-P"
topic= "AccountExp-T"
action="pub"
name = "archive-service/share"
proto = "tcp"
addr = "172.16.33.56:6201"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[limiter]
domain = "api.bilibili.co"
family = "archive-service"
all = "2h"
diff = "2s"
[limiter.clientconfig]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
dial = "1s"
timeout = "2s"
keepAlive = "10s"
timer = 128
[limiter.clientconfig.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[limiter.app]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
[playerClient]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
dial = "50ms"
timeout = "500ms"
keepAlive = "60s"
[playerClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100

63
app/service/main/archive/cmd/main.go Normal file
View File

@@ -0,0 +1,63 @@
package main
import (
"context"
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/service/main/archive/conf"
rpc "go-common/app/service/main/archive/server/gorpc"
"go-common/app/service/main/archive/server/grpc"
"go-common/app/service/main/archive/server/http"
"go-common/app/service/main/archive/service"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
// init ecode
ecode.Init(nil)
// init log
log.Init(conf.Conf.Xlog)
trace.Init(conf.Conf.Tracer)
defer trace.Close()
defer log.Close()
log.Info("archive-service start")
// service init
svr := service.New(conf.Conf)
// statsd init
rpcSvr := rpc.New(conf.Conf, svr)
grpcSvr, err := grpc.New(nil, svr)
if err != nil {
panic(err)
}
http.Init(conf.Conf, svr)
// init signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("archive-service get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
rpcSvr.Close()
grpcSvr.Shutdown(context.TODO())
time.Sleep(time.Second * 2)
log.Info("archive-service exit")
return
case syscall.SIGHUP:
// TODO reload
default:
return
}
}
}