Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,20 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/passport-game-local/cmd:all-srcs",
"//app/job/main/passport-game-local/conf:all-srcs",
"//app/job/main/passport-game-local/http:all-srcs",
"//app/job/main/passport-game-local/model:all-srcs",
"//app/job/main/passport-game-local/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,16 @@
## passport-game-local-job
#### Version 1.4.0
> 1.mv to main
#### Version 1.3.0
> 1.change to bm router
#### Version 1.2.0
> 1.add modify_time column trans
#### Version 1.1.0
> 1.move to kratos
#### Version 1.0.0
> 1.版本初始化

View File

@@ -0,0 +1,10 @@
# Owner
wanghuan01
# Author
wanghuan01
wutao
wucongyou
# Reviewer
wutao

View File

@@ -0,0 +1,16 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- wanghuan01
- wucongyou
- wutao
labels:
- job
- job/main/passport-game-local
- main
options:
no_parent_owners: true
reviewers:
- wanghuan01
- wucongyou
- wutao

View File

@@ -0,0 +1,12 @@
## passport-game-local-job
#### 项目简介
> 1.游戏云账号市北job主要用于订阅源站变更的数据
#### 编译环境
> 请只用golang v1.8.x以上版本编译执行。
#### 依赖包
> 1.公共包go-common
#### 特别说明

View File

@@ -0,0 +1,41 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_binary",
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["passport-game-local-job.toml"],
importpath = "go-common/app/job/main/passport-game-local/cmd",
tags = ["automanaged"],
visibility = ["//visibility:private"],
deps = [
"//app/job/main/passport-game-local/conf:go_default_library",
"//app/job/main/passport-game-local/http:go_default_library",
"//app/job/main/passport-game-local/service:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,48 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/job/main/passport-game-local/conf"
"go-common/app/job/main/passport-game-local/http"
"go-common/app/job/main/passport-game-local/service"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Xlog)
defer log.Close()
trace.Init(conf.Conf.Tracer)
defer trace.Close()
// service init
srv := service.New(conf.Conf)
http.Init(conf.Conf, srv)
// signal handler
log.Info("passport-game-local-job start")
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("passport-game-local-job get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
srv.Close()
time.Sleep(time.Second * 2)
log.Info("passport-game-local-job exit")
return
case syscall.SIGHUP:
// TODO reload
default:
return
}
}
}

View File

@@ -0,0 +1,65 @@
# This is a TOML document. Boom.
version = "1.0.0"
user = "noboby"
dir = "./"
perf = "0.0.0.0:6980"
family = "passport-game-local-job"
env = "qa"
[xlog]
dir = "/data/log/passport-game-local-job"
[tracer]
family = "passport-game-local-job"
proto = "unixgram"
addr = "/var/run/dapper-collect/dapper-collect.sock"
[multiHTTP]
[multiHTTP.inner]
addrs = ["0.0.0.0:6981"]
maxListen = 1000
[group]
[group.asoBinLog]
size = 200
num = 32
ticker = "1s"
chan = 1024
[databus]
[databus.asoBinLogSub]
key = "0QEO9F8JuuIxZzNDvklH"
secret = "0QEO9F8JuuIxZzNDvklI"
group = "PassportAccount-ASO-S"
topic = "PassportAccount-T"
action = "sub"
offset = "old"
buffer = 2048
name = "passport-game-local-job/databus"
proto = "tcp"
addr = "172.16.33.158:6205"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[databus.encryptTransPub]
key = "0QEO9F8JuuIxZzNDvklH"
secret = "0QEO9F8JuuIxZzNDvklI"
group = "PassportGameTrans-ENCRYPT-P"
topic = "PassportGameTrans-T"
action = "pub"
offset = "old"
buffer = 2048
name = "passport-game-local-job/databus"
proto = "tcp"
addr = "172.16.33.158:6205"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"

View File

@@ -0,0 +1,35 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/main/passport-game-local/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/conf:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/trace:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,111 @@
package conf
import (
"errors"
"flag"
"go-common/library/conf"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/trace"
"go-common/library/queue/databus"
"go-common/library/time"
"github.com/BurntSushi/toml"
)
var (
confPath string
// Conf conf.
Conf = &Config{}
client *conf.Client
)
// Config config.
type Config struct {
// log
Xlog *log.Config
// Tracer tracer
Tracer *trace.Config
// HTTP
BM *bm.ServerConfig
// Group group
Group *Group
//Databus databus
DataBus *DataBus
}
// Group multi group config collection.
type Group struct {
AsoBinLog *GroupConfig
}
// GroupConfig group config.
type GroupConfig struct {
// Size merge size
Size int
// Num merge goroutine num
Num int
// Ticker duration of submit merges when no new message
Ticker time.Duration
// Chan size of merge chan and done chan
Chan int
}
// DataBus databus.
type DataBus struct {
AsoBinLogSub *databus.Config
EncryptTransPub *databus.Config
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init config.
func Init() (err error) {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,32 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["http.go"],
importpath = "go-common/app/job/main/passport-game-local/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/passport-game-local/conf:go_default_library",
"//app/job/main/passport-game-local/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,36 @@
package http
import (
"go-common/app/job/main/passport-game-local/conf"
"go-common/app/job/main/passport-game-local/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
var (
srv *service.Service
)
// Init init http sever instance.
func Init(c *conf.Config, s *service.Service) {
srv = s
// init inner router
eng := bm.DefaultServer(c.BM)
initRouter(eng)
// init inner server
if err := eng.Start(); err != nil {
log.Error("bm.DefaultServer error(%v)", err)
panic(err)
}
}
// initRouter init inner router.
func initRouter(e *bm.Engine) {
// health check
e.Ping(ping)
}
// ping check server ok.
func ping(c *bm.Context) {
c.JSON(nil, srv.Ping(c))
}

View File

@@ -0,0 +1,29 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"message.go",
"passport.go",
],
importpath = "go-common/app/job/main/passport-game-local/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,19 @@
package model
import "encoding/json"
// BMsg databus binlog message.
type BMsg struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
}
// PMsg Push msg
type PMsg struct {
Action string `json:"action"`
Table string `json:"table"`
Data *AsoAccount `json:"data"`
Flag int `json:"flag"`
}

View File

@@ -0,0 +1,73 @@
package model
import (
"crypto/md5"
"encoding/hex"
)
const (
_cloudSalt = "bi_clould_tencent_01"
)
// AsoAccount aso account.
type AsoAccount struct {
Mid int64 `json:"mid"`
Userid string `json:"userid"`
Uname string `json:"uname"`
Pwd string `json:"pwd"`
Salt string `json:"salt"`
Email string `json:"email"`
Tel string `json:"tel"`
CountryID int64 `json:"country_id"`
MobileVerified int8 `json:"mobile_verified"`
Isleak int8 `json:"isleak"`
Mtime string `json:"mtime"`
}
// OriginAsoAccount origin aso account.
type OriginAsoAccount struct {
Mid int64 `json:"mid"`
Userid string `json:"userid"`
Uname string `json:"uname"`
Pwd string `json:"pwd"`
Salt string `json:"salt"`
Email string `json:"email"`
Tel string `json:"tel"`
CountryID int64 `json:"country_id"`
MobileVerified int8 `json:"mobile_verified"`
Isleak int8 `json:"isleak"`
Mtime string `json:"modify_time"`
}
// Default doHash aso account, including the followings fields: userid, uname, pwd, email, tel.
func Default(a *OriginAsoAccount) *AsoAccount {
return &AsoAccount{
Mid: a.Mid,
Userid: a.Userid,
Uname: a.Uname,
Pwd: doHash(a.Pwd, _cloudSalt),
Salt: a.Salt,
Email: doHash(a.Email, _cloudSalt),
Tel: doHash(a.Tel, _cloudSalt),
CountryID: a.CountryID,
MobileVerified: a.MobileVerified,
Isleak: a.Isleak,
Mtime: a.Mtime,
}
}
// DefaultHash hash a plain text using default salt.
func DefaultHash(plaintext string) string {
return doHash(plaintext, _cloudSalt)
}
func doHash(plaintext, salt string) string {
if plaintext == "" {
return ""
}
hash := md5.New()
hash.Write([]byte(plaintext))
hash.Write([]byte(salt))
md := hash.Sum(nil)
return hex.EncodeToString(md)
}

View File

@@ -0,0 +1,51 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"aso_bin_log.go",
"service.go",
],
importpath = "go-common/app/job/main/passport-game-local/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/passport-game-local/conf:go_default_library",
"//app/job/main/passport-game-local/model:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"aso_bin_log_test.go",
"service_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/passport-game-local/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

View File

@@ -0,0 +1,140 @@
package service
import (
"context"
"encoding/json"
"strconv"
"time"
"go-common/app/job/main/passport-game-local/model"
"go-common/library/log"
"go-common/library/queue/databus"
)
func (s *Service) asobinlogconsumeproc() {
mergeNum := int64(s.c.Group.AsoBinLog.Num)
for {
msg, ok := <-s.dsAsoBinLogSub.Messages()
if !ok {
log.Error("asobinlogconsumeproc closed")
return
}
// marked head to first commit
m := &message{data: msg}
s.mu.Lock()
if s.head == nil {
s.head = m
s.last = m
} else {
s.last.next = m
s.last = m
}
s.mu.Unlock()
bmsg := new(model.BMsg)
if err := json.Unmarshal(msg.Value, bmsg); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
continue
}
mid := int64(0)
if bmsg.Table == _asoAccountTable {
t := new(model.AsoAccount)
if err := json.Unmarshal(bmsg.New, t); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
}
mid = t.Mid
m.object = bmsg
log.Info("asobinlogconsumeproc table:%s key:%s partition:%d offset:%d", bmsg.Table, msg.Key, msg.Partition, msg.Offset)
} else {
continue
}
s.merges[mid%mergeNum] <- m
}
}
func (s *Service) asobinlogcommitproc() {
for {
done := <-s.done
commits := make(map[int32]*databus.Message)
for _, d := range done {
d.done = true
}
s.mu.Lock()
for ; s.head != nil && s.head.done; s.head = s.head.next {
commits[s.head.data.Partition] = s.head.data
}
s.mu.Unlock()
for _, m := range commits {
m.Commit()
}
}
}
func (s *Service) asobinlogmergeproc(c chan *message) {
var (
max = s.c.Group.AsoBinLog.Size
merges = make([]*model.BMsg, 0, max)
marked = make([]*message, 0, max)
ticker = time.NewTicker(time.Duration(s.c.Group.AsoBinLog.Ticker))
)
for {
select {
case msg, ok := <-c:
if !ok {
log.Error("asobinlogmergeproc closed")
return
}
p, assertOk := msg.object.(*model.BMsg)
if assertOk && p.Action != "" && (p.Table == _asoAccountTable) {
merges = append(merges, p)
}
marked = append(marked, msg)
if len(marked) < max && len(merges) < max {
continue
}
case <-ticker.C:
}
if len(merges) > 0 {
s.processAsoAccLogInfo(merges)
merges = make([]*model.BMsg, 0, max)
}
if len(marked) > 0 {
s.done <- marked
marked = make([]*message, 0, max)
}
}
}
func (s *Service) processAsoAccLogInfo(bmsgs []*model.BMsg) {
for _, msg := range bmsgs {
s.processAsoAccLog(msg)
}
}
func (s *Service) processAsoAccLog(msg *model.BMsg) {
aso := new(model.OriginAsoAccount)
if err := json.Unmarshal(msg.New, aso); err != nil {
log.Error("failed to parse binlog new, json.Unmarshal(%s) error(%v)", string(msg.New), err)
return
}
pmsg := new(model.PMsg)
if "update" == msg.Action {
old := new(model.AsoAccount)
if err := json.Unmarshal(msg.Old, old); err != nil {
log.Error("failed to parse binlog new, json.Unmarshal(%s) error(%v)", string(msg.New), err)
return
}
if old.Pwd != aso.Pwd {
pmsg.Flag = 1
}
}
pmsg.Action = msg.Action
pmsg.Table = msg.Table
pmsg.Data = model.Default(aso)
key := strconv.FormatInt(aso.Mid, 10)
for {
if err := s.dsAsoEncryptTransPub.Send(context.TODO(), key, pmsg); err == nil {
return
}
time.Sleep(time.Second)
}
}

View File

@@ -0,0 +1,14 @@
package service
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestService_Asobinlogconsumeproc(t *testing.T) {
once.Do(startService)
Convey("asobinlogcommitproc running", t, func() {
So(s.asobinlogcommitproc, ShouldBeNil)
})
}

View File

@@ -0,0 +1,71 @@
package service
import (
"context"
"sync"
"go-common/app/job/main/passport-game-local/conf"
"go-common/library/log"
"go-common/library/queue/databus"
)
const (
// table and duration
_asoAccountTable = "aso_account"
)
// Service service.
type Service struct {
c *conf.Config
// aso encrypt trans pub databus
dsAsoEncryptTransPub *databus.Databus
// aso binlog databus
dsAsoBinLogSub *databus.Databus
merges []chan *message
done chan []*message
// proc
head, last *message
mu sync.Mutex
}
type message struct {
next *message
data *databus.Message
object interface{}
done bool
}
// New new a service instance.
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dsAsoEncryptTransPub: databus.New(c.DataBus.EncryptTransPub),
dsAsoBinLogSub: databus.New(c.DataBus.AsoBinLogSub),
merges: make([]chan *message, c.Group.AsoBinLog.Num),
done: make(chan []*message, c.Group.AsoBinLog.Chan),
}
go s.asobinlogcommitproc()
for i := 0; i < c.Group.AsoBinLog.Num; i++ {
ch := make(chan *message, c.Group.AsoBinLog.Chan)
s.merges[i] = ch
go s.asobinlogmergeproc(ch)
}
go s.asobinlogconsumeproc()
return
}
// Ping check server ok.
func (s *Service) Ping(c context.Context) (err error) {
return
}
// Close close service, including databus and outer service.
func (s *Service) Close() (err error) {
if err = s.dsAsoBinLogSub.Close(); err != nil {
log.Error("srv.asoBinLog.Close() error(%v)", err)
}
if err = s.dsAsoEncryptTransPub.Close(); err != nil {
log.Error("srv.dsAsoEncryptTransPub.Close() error(%v)", err)
}
return
}

View File

@@ -0,0 +1,29 @@
package service
import (
"sync"
"testing"
"go-common/app/job/main/passport-game-local/conf"
. "github.com/smartystreets/goconvey/convey"
)
var (
once sync.Once
s *Service
)
func startService() {
if err := conf.Init(); err != nil {
panic(err)
}
s = New(conf.Conf)
}
func TestNew(t *testing.T) {
once.Do(startService)
Convey("ping should ok", t, func() {
So(s.Ping, ShouldBeNil)
})
}