openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
app/job/main/member/cmd/BUILD Normal file
View File

@@ -0,0 +1,42 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["member-job-dev.toml"],
importpath = "go-common/app/job/main/member/cmd",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/http:go_default_library",
"//app/job/main/member/service:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus/report:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

146
app/job/main/member/cmd/databus.py Normal file
View File

@@ -0,0 +1,146 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date : 2017-12-25 下午2:43
# @Author : Hedan (hedan@bilibili.com)
# @file : databusTest
import json
import redis
# [databus]
# key = "4ba46ba31f9a44ef"
# secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
# group = "AccountLog-MainAccount-P"
# topic = "AccountLog-T"
# action = "pub"
# name = "member-service/databus"
# proto = "tcp"
# addr = "172.16.33.158:6205"
# idle = 100
# active = 100
# dialTimeout = "1s"
# readTimeout = "60s"
# writeTimeout = "1s"
# idleTimeout = "10s"
"""参考文档 http://info.bilibili.co/pages/viewpage.action?pageId=3670491
key 服务配置中databus的key
value 服务配置中databus的value
host和port 配置服务中databus的host和port
group-topic 一个group对应一个topic(开发申请)
"""
#每日登录 passport 获得5经验值
# data_passport ={
# 'mid':4780461,
# 'loginip':1726481463,
# "timestamp":1516517576,
# }
# auth_passport = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@PassportLog-MainAccount-P/topic=PassportLog-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_passport)
# msg = json.dumps(data_passport, ensure_ascii=False)
# rc.set("1023", msg)
# 每日登录 account 获得5经验值
# data_passport ={
# 'mid':110000092,
# 'loginip':1726481463,
# "timestamp":1516517576,
# }
# auth_passport = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@AccountLoginAward-MainAccount-P/topic=AccountLoginAward-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_passport)
# msg = json.dumps(data_passport, ensure_ascii=False)
# rc.set("1023", msg)
# 每日首次分享视频 archive 获得5经验值
# data_archive ={
# 'event':"share",
# 'mid':110000092,
# 'ip':"127.0.0.1",
# "ts":111,
# }
# auth_archive = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@AccountExp-MainAccount-P/topic=AccountExp-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_archive)
# msg = json.dumps(data_archive, ensure_ascii=False)
# rc.set("1023", msg)
#
#
# 每日首次分享视频 archive 获得5经验值
data_archive ={
'event':"view",
# 'mid':110000092,
'mid':4780461,
'ip':"127.0.0.1",
"ts":1521745000,
}
auth_archive = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@AccountExp-MainAccount-P/topic=AccountExp-T&role=pub&offset=new'
rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
print rc.execute_command('auth',auth_archive)
msg = json.dumps(data_archive, ensure_ascii=False)
rc.set("1023", msg)
# 每日首次观看 history 获得5经验值
# data_history ={
# "action":"insert",
# "table":"aso_account",
# "data":{"mid":110000193,
# "userid":"test_dan06",
# "uname":"test_dan08",
# "pwd":"6cfbf96b8f0eb2e0a82b46a4236e8883",
# "salt":"D8fd30Kj",
# "email":"169d9106a74d5e95de71be6cf373af04",
# "tel":"218cb4bf8762354eae473b3b612f707e",
# "country_id":1,
# "mobile_verified":0,
# "isleak":0,
# "mtime":"2018-01-02 17:18:58"},
# "flag":0
# }
# auth_history = '0QEO9F8JuuIxZzNDvklH:0QEO9F8JuuIxZzNDvklI@PassportGameTrans-ENCRYPT-P/topic=PassportGameTrans-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_history)
# msg = json.dumps(data_history, ensure_ascii=False)
# rc.set("110000193", msg)
# data_history ={
# "action":"insert",
# "table":"aso_account",
# "data":{"mid":110000194,
# "userid":"test_dan07",
# "uname":"test_dan07",
# "pwd":"d2c9d4acdfe9942979d7b4d3e711d499",
# "salt":"5Brw3JuP",
# "email":"169d9106a74d5e95de71be6cf373af04",
# "tel":"b65919197178b5db6ed5f85a229dfaf9",
# "country_id":1,
# "mobile_verified":0,
# "isleak":0,
# "mtime":"2018-01-02 17:18:58"},
# "flag":0
# }
# auth_history = '0QEO9F8JuuIxZzNDvklH:0QEO9F8JuuIxZzNDvklI@PassportGameTrans-ENCRYPT-P/topic=PassportGameTrans-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_history)
# msg = json.dumps(data_history, ensure_ascii=False)
# rc.set("110000193", msg)

56
app/job/main/member/cmd/main.go Normal file
View File

@@ -0,0 +1,56 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/http"
"go-common/app/job/main/member/service"
"go-common/library/log"
"go-common/library/queue/databus/report"
)
var (
svr *service.Service
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
log.Init(conf.Conf.Xlog)
defer log.Close()
report.InitUser(conf.Conf.UserReport)
report.InitManager(conf.Conf.ManagerReport)
log.Info("member-job start")
svr = service.New(conf.Conf)
http.Init(conf.Conf, svr)
signalHandler()
}
func signalHandler() {
var (
err error
ch = make(chan os.Signal, 1)
)
signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
si := <-ch
switch si {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
log.Info("get a signal %s, stop the consume process", si.String())
if err = svr.Close(); err != nil {
log.Error("srv close consumer error(%v)", err)
}
return
case syscall.SIGHUP:
default:
return
}
}
}

430
app/job/main/member/cmd/member-job-dev.toml Normal file
View File

@@ -0,0 +1,430 @@
#This is a TOML document. Boom.
version = "1.0.0"
user = "nobody"
pid = "/tmp/member-job.pid"
dir = "./"
perf = "0.0.0.0:6330"
family = "member-job"
address = "0.0.0.0"
[log]
dir = "/data/log/member-job"
stdout=true
[tracer]
proto = "udp"
addr = "172.16.33.46:5140"
tag = "platform/member-job"
[multiHTTP]
[multiHTTP.inner]
addrs = ["0.0.0.0:6688"]
maxListen = 10
[app]
key = "4699a07e59d7149e"
secret = "test"
[path]
accountInfo="http://uat-account.bilibili.co/api/internal/data/info"
[httpClient]
dial = "1s"
timeout = "2s"
keepAlive = "60s"
timer = 128
key = "4699a07e59d7149e"
secret = "test"
[httpClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[memcache]
name = "member"
proto = "tcp"
# addr = "172.16.33.54:11211"
addr = "172.18.33.60:11233"
idle = 5
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "24h"
[mysql]
addr = "172.16.33.205"
dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_member?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 2
IdleTimeout ="4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[mysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[accCheckMysql]
addr = "172.16.33.205:3306"
dsn = "account:wx2U1MwXRyWEuURw@tcp(172.16.33.205:3306)/account?timeout=50ms&readTimeout=300ms&writeTimeout=300ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 50
idle = 5
idleTimeout = "4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[accCheckMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[accMysql]
addr = "172.16.33.205:3306"
dsn = "account:wx2U1MwXRyWEuURw@tcp(172.16.33.205:3306)/account?timeout=50ms&readTimeout=300ms&writeTimeout=300ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 50
idle = 5
idleTimeout = "4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[accMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[asoMysql]
addr = "172.16.33.205:3306"
dsn = "aso:hA0DAnENNFz78kYB@tcp(172.16.33.205:3306)/aso?timeout=50ms&readTimeout=300ms&writeTimeout=300ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 50
idle = 5
idleTimeout = "4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[asoMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[PasslogMysql]
addr = "172.16.33.205"
dsn = "account:wx2U1MwXRyWEuURw@tcp(172.16.33.205:3306)/account?timeout=500ms&readTimeout=500ms&writeTimeout=500ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 20
idle = 10
idleTimeout ="4h"
queryTimeout = "1000ms"
execTimeout = "500ms"
tranTimeout = "2000ms"
[PasslogMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[PLogDatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountLog-MainAccount-S"
topic = "AccountLog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[logdatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountLog-MainAccount-S"
topic = "AccountLog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[expdatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountExp-MainAccount-S"
topic = "AccountExp-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[logindatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "PassportLog-MainAccount-S"
topic = "PassportLog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[awarddatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountLoginAward-MainAccount-S"
topic = "AccountLoginAward-T"
action = "sub"
offset = "new"
buffer = 2048
name = "coin-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[accountnotify]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountNotify-Notify-P"
topic = "AccountNotify-T"
action = "pub"
buffer = 2048
name = "account-notify-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
# addr = "172.18.33.61:6889"
# addr = "172.16.33.56:6205"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[sharemiddatabus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "ShareMid-MainAccount-S"
topic = "ShareMid-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[databus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "MemberBinlog-MainAccount-S"
topic = "MemberBinlog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[passortDataBus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "PassportPurgeCache-MainAccount-S"
topic = "PassportPurgeCache-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[accDataBus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "UserActive-MainAccount-S"
topic = "UserActive-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[databusutil]
size = 100
chan = 1024
num = 4
ticker="30s"
[redis]
name = "member-service"
proto = "tcp"
# addr = "172.16.33.54:6379"
addr = "172.18.33.60:6893"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "720h"
ansCountExpire = "12h"
ansAddFlagCountExpire = "1m"
[report]
taskID = "000293"
proto = "unixgram"
addr = "/var/run/lancer/collector.sock"
chan = 1024
timeout = "200ms"
[blockdb]
# addr = "172.22.34.101:3306"
# dsn = "block:fGCs8LbV6c0nSXoyA9kdKwOpIiHa5hBq@tcp(172.22.34.101:3306)/bilibili_block?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
addr = "127.0.0.1:3306"
dsn = "test:test@tcp(127.0.0.1:3306)/test?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 20
idle = 10
idleTimeout ="4h"
queryTimeout = "1s"
execTimeout = "1s"
tranTimeout = "2s"
[db.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[blockMemcache]
name = "block-service"
proto = "unix"
addr = "/tmp/uat-block-service-mc.sock"
idle = 5
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
[blockCreditDatabus]
key = "875d05ffa843e297"
secret= "1b58f18382f0e3662f64f8457c05c049"
group= "Credit-MainAccountLaw-S"
topic= "Credit-T"
action="sub"
name = "block-job/databus"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 2
active = 10
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[blockProperty]
limitExpireCheckLimit = 100
limitExpireCheckTick = "5m"
creditExpireCheckLimit = 100
creditExpireCheckTick = "5m"
msgURL = "http://message.bilibili.co/api/notify/send.user.notify.do"
[property.flag]
# 自动解禁检查
expireCheck = true
# 小黑屋答题状态订阅
creditSub = true
# 同步老库db
[property.msg]
[property.msg.blockRemove]
code = "2_3_6"
title = "账号封禁解除通知"
content = "你的账号已经解除封禁，封禁期间禁止使用的各项社区功能已经恢复。请遵守社区规范，共同维护良好的社区氛围。"
[biz]
realnameAlipayCheckTick = "5m"
realnameAlipayCheckLimit = 100
realnameAlipayAppID = "https://openapi.alipay.com/gateway.do"
realnameAlipayGateway = "2018062060353784"
[ParsedRealnameAgent]
TaskID="001826"
Proto="tcp"
Addr="dataflow.bilibili.co:5401"