Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

20
app/job/live/wallet/BUILD Normal file
View File

@@ -0,0 +1,20 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/live/wallet/cmd:all-srcs",
"//app/job/live/wallet/conf:all-srcs",
"//app/job/live/wallet/dao:all-srcs",
"//app/job/live/wallet/model:all-srcs",
"//app/job/live/wallet/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,13 @@
# v1.0.3
1. 修复wait group的bug
2. 取消merge逻辑
# v1.0.2
1. 更改日志,校验老数据库
# v1.0.1
1. 添加消费者日志
2. movedir
# v1.0.0
1. 新增

View File

@@ -0,0 +1,9 @@
# Owner
zhaohailin
zhouzhichao
# Author
zhouzhichao
# Reviewer
zhaohailin

View File

@@ -0,0 +1,14 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- zhaohailin
- zhouzhichao
labels:
- job
- job/live/wallet
- live
options:
no_parent_owners: true
reviewers:
- zhaohailin
- zhouzhichao

View File

@@ -0,0 +1,14 @@
# /live-wallet
# 项目简介
1.
# 编译环境
# 依赖包
# 编译执行

View File

@@ -0,0 +1,40 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["live-wallet-test.toml"],
importpath = "go-common/app/job/live/wallet/cmd",
tags = ["automanaged"],
deps = [
"//app/job/live/wallet/conf:go_default_library",
"//app/job/live/wallet/service:go_default_library",
"//library/log:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,59 @@
# This is a TOML document. Boom
version = "1.0.0"
user = "root"
pid = "/tmp/live-wallet-job.pid"
dir = "./"
family = "/live-wallet"
perf = "0.0.0.0:20202"
address = ""
env = "dev"
[log]
dir = "/data/log/live-wallet-job"
[db]
[db.wallet]
addr = "172.16.33.69:3306"
dsn = "live:oWni@ElNs0P0C(dphdj*F1y4@tcp(172.16.33.69:3306)/live_wallet?timeout=5s&readTimeout=30s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 5
queryTimeout = "1s"
execTimeout = "1s"
tranTimeout = "2s"
[db.wallet.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[memcache]
walletExpire = "3s"
[memcache.wallet]
name = "live-wallet-service/wallet"
proto = "tcp"
addr = "172.16.33.251:11211"
idle = 5
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
[userSub]
key = "ec4c0820d525d67b"
secret = "2bdf3bd4ecab041b5d5640a1da4f7f81"
group = "LiveUserBinlog-LiveLive-S"
topic = "LiveUserBinlog-T"
action = "sub"
offset = "new"
name = "liveuser-wallet/user-sub"
proto = "tcp"
addr = "172.16.33.154:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"

View File

@@ -0,0 +1,43 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"go-common/app/job/live/wallet/conf"
"go-common/app/job/live/wallet/service"
"go-common/library/log"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
// init log
log.Init(conf.Conf.Log)
defer log.Close()
log.Info("/live-wallet start")
// service init
srv := Service.New(conf.Conf)
// init pprof conf.Conf.Perf
// init signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("/live-wallet get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
log.Info("/live-wallet exit")
srv.Close()
return
case syscall.SIGHUP:
default:
return
}
}
}

View File

@@ -0,0 +1,36 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/live/wallet/conf",
tags = ["automanaged"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/conf:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,100 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/memcache"
"go-common/library/conf"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/queue/databus"
"go-common/library/time"
"github.com/BurntSushi/toml"
)
// global var
var (
ConfPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config config set
type Config struct {
// base
// log
Log *log.Config
// db
DB *DB
// databus
UserSub *databus.Config
// Env
Env string
// memcache
Memcache *Memcache
}
// DB db config.
type DB struct {
Wallet *sql.Config
}
// Memcache config
type Memcache struct {
Wallet *memcache.Config
WalletExpire time.Duration
}
func init() {
flag.StringVar(&ConfPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if ConfPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(ConfPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,56 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"mc_wallet_test.go",
"wallet_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/live/wallet/conf:go_default_library",
"//app/job/live/wallet/model:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"mc_wallet.go",
"wallet.go",
],
importpath = "go-common/app/job/live/wallet/dao",
tags = ["automanaged"],
deps = [
"//app/job/live/wallet/conf:go_default_library",
"//app/job/live/wallet/model:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,55 @@
package dao
import (
"context"
"go-common/app/job/live/wallet/conf"
"go-common/library/cache/memcache"
xsql "go-common/library/database/sql"
"go-common/library/log"
)
type Dao struct {
c *conf.Config
db *xsql.DB
mc *memcache.Pool
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
db: xsql.NewMySQL(c.DB.Wallet),
mc: memcache.NewPool(c.Memcache.Wallet),
}
return
}
// Close close the resource.
func (dao *Dao) Close() {
dao.db.Close()
}
// Ping dao ping
func (d *Dao) Ping(c context.Context) (err error) {
if err = d.db.Ping(c); err != nil {
log.Error("PingDb error(%v)", err)
return
}
return d.pingMC(c)
}
// pingMc ping
func (d *Dao) pingMC(c context.Context) (err error) {
item := &memcache.Item{
Key: "ping",
Value: []byte{1},
}
conn := d.mc.Get(c)
err = conn.Set(item)
conn.Close()
if err != nil {
log.Error("PingMemcache conn.Set(%v) error(%v)", item, err)
}
return
}

View File

@@ -0,0 +1,30 @@
package dao
import (
"context"
"fmt"
mc "go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_walletMcKey = "wu:%d"
)
func mcKey(uid int64) string {
return fmt.Sprintf(_walletMcKey, uid)
}
// DelWalletCache 删除等级缓存
func (d *Dao) DelWalletCache(c context.Context, uid int64) (err error) {
key := mcKey(uid)
conn := d.mc.Get(c)
defer conn.Close()
if err = conn.Delete(key); err == mc.ErrNotFound {
err = nil
} else if err != nil {
log.Error("[dao.mc_wallet|DelWalletCache] conn.Delete(%s) error(%v)", key, err)
}
return
}

View File

@@ -0,0 +1,14 @@
package dao
import (
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestDelWalletCache(t *testing.T) {
Convey("Test Del Memcache", t, func() {
once.Do(startService)
err := d.DelWalletCache(ctx, 10000)
So(err, ShouldBeNil)
})
}

View File

@@ -0,0 +1,36 @@
package dao
import (
"context"
"fmt"
"go-common/app/job/live/wallet/model"
"go-common/library/log"
)
const (
_shard = 10
_insWallet = "INSERT IGNORE INTO user_wallet_%d(uid,gold,iap_gold,silver) VALUES(?,?,?,?)"
_mergeWallet = "INSERT INTO user_wallet_%d(uid,gold,iap_gold,silver) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE gold=gold+%d,iap_gold=iap_gold+%d,silver=silver+%d"
)
func tableIndex(uid int64) int64 {
return uid % _shard
}
func (d *Dao) InitWallet(c context.Context, user *model.User) (row int64, err error) {
res, err := d.db.Exec(c, fmt.Sprintf(_insWallet, tableIndex(user.Uid)), user.Uid, user.Gold, user.IapGold, user.Silver)
if err != nil {
log.Error("[dao.wallet|InitWallet] d.db.Exec err: %v {uid:%d gold:%d iap_gold:%d silver:%d}", err, user.Uid, user.Gold, user.IapGold, user.Silver)
return
}
return res.RowsAffected()
}
func (d *Dao) MergeWallet(c context.Context, uid int64, gold int64, iap_gold int64, silver int64) (row int64, err error) {
res, err := d.db.Exec(c, fmt.Sprintf(_mergeWallet, tableIndex(uid), gold, iap_gold, silver), uid, gold, iap_gold, silver)
if err != nil {
log.Error("[dao.wallet|MergeWallet] d.db.Exec err: %v {uid:%d gold:%d iap_gold:%d silver:%d}", err, uid, gold, iap_gold, silver)
return
}
return res.RowsAffected()
}

View File

@@ -0,0 +1,53 @@
package dao
import (
"context"
"sync"
"testing"
"time"
"go-common/app/job/live/wallet/conf"
"go-common/library/log"
. "github.com/smartystreets/goconvey/convey"
"go-common/app/job/live/wallet/model"
)
var (
once sync.Once
d *Dao
ctx = context.TODO()
testUid int64
)
func initConf() {
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
defer log.Close()
}
func startService() {
initConf()
d = New(conf.Conf)
time.Sleep(time.Second * 2)
}
func TestInitWallet(t *testing.T) {
Convey("Init Wallet", t, func() {
once.Do(startService)
user := &model.User{}
user.Uid = testUid
_, err := d.InitWallet(ctx, user)
So(err, ShouldBeNil)
})
}
func TestMergeWallet(t *testing.T) {
Convey("Merge Wallet", t, func() {
once.Do(startService)
_, err := d.MergeWallet(ctx, testUid, 1, 2, 3)
So(err, ShouldBeNil)
})
}

View File

@@ -0,0 +1,27 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["model.go"],
importpath = "go-common/app/job/live/wallet/model",
tags = ["automanaged"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,19 @@
package model
import "encoding/json"
// Message canal standary message
type Message struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
}
// User canal base message
type User struct {
Uid int64 `json:"uid"`
Gold int64 `json:"gold"`
IapGold int64 `json:"iap_gold"`
Silver int64 `json:"silver"`
}

View File

@@ -0,0 +1,54 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"service_test.go",
"wallet_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/live/wallet/conf:go_default_library",
"//app/job/live/wallet/model:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"service.go",
"wallet.go",
],
importpath = "go-common/app/job/live/wallet/service",
tags = ["automanaged"],
deps = [
"//app/job/live/wallet/conf:go_default_library",
"//app/job/live/wallet/dao:go_default_library",
"//app/job/live/wallet/model:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,101 @@
package Service
import (
"context"
"encoding/json"
"strings"
"sync"
"time"
"go-common/app/job/live/wallet/conf"
"go-common/app/job/live/wallet/dao"
"go-common/app/job/live/wallet/model"
"go-common/library/log"
"go-common/library/queue/databus"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
userSub *databus.Databus
waiter *sync.WaitGroup
userUpMo int64
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
userSub: databus.New(c.UserSub),
waiter: new(sync.WaitGroup),
}
s.waiter.Add(1)
go s.userCanalConsumeproc()
go s.checkUserCanalConsumeproc()
return s
}
// Ping Service
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close Service
func (s *Service) Close() {
defer s.waiter.Wait()
s.userSub.Close()
s.dao.Close()
}
// Wait goroutinue to close
func (s *Service) Wait() {
s.waiter.Wait()
}
// expCanalConsumeproc consumer archive
func (s *Service) userCanalConsumeproc() {
var (
msgs = s.userSub.Messages()
err error
)
defer s.waiter.Done()
for {
msg, ok := <-msgs
if !ok {
log.Info("userCanal databus Consumer exit")
return
}
s.userUpMo++
msg.Commit()
m := &model.Message{}
//log.Info("canal message %s", msg.Value)
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
continue
}
if !strings.HasPrefix(m.Table, "user_") || (m.Action != "update" && m.Action != "insert") {
continue
}
s.mergeData(m.New, m.Old, m.Action)
}
}
// checkConsumeproc check consumer stat
func (s *Service) checkUserCanalConsumeproc() {
if s.c.Env != "pro" {
return
}
var userMo int64
for {
time.Sleep(1 * time.Minute)
if s.userUpMo-userMo == 0 {
msg := "live-wallet-job userCanal did not consume within a minute"
//s.dao.SendSMS(msg)
log.Warn(msg)
}
userMo = s.userUpMo
}
}

View File

@@ -0,0 +1,41 @@
package Service
import (
"context"
"go-common/app/job/live/wallet/conf"
"os"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
func TestMain(m *testing.M) {
conf.ConfPath = "../cmd/live-wallet-test.toml"
once.Do(startService)
os.Exit(m.Run())
}
func TestPing(t *testing.T) {
Convey("Ping", t, func() {
once.Do(startService)
time.Sleep(time.Second)
s.Ping(context.TODO())
})
}
func TestClose(t *testing.T) {
Convey("Close", t, func() {
once.Do(startService)
time.Sleep(time.Second)
s.Close()
})
}
func TestWait(t *testing.T) {
Convey("Wait", t, func() {
once.Do(startService)
time.Sleep(time.Second)
s.Wait()
})
}

View File

@@ -0,0 +1,24 @@
package Service
import (
"context"
"encoding/json"
"go-common/app/job/live/wallet/model"
"go-common/library/log"
)
func (s *Service) mergeData(nwMsg []byte, oldMsg []byte, action string) {
if action == "update" {
// do noting
} else if action == "insert" {
userNew := &model.User{}
if err := json.Unmarshal(nwMsg, userNew); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", string(nwMsg), err)
return
}
log.Info("new user %d", userNew.Uid)
s.dao.InitWallet(context.TODO(), userNew)
//s.dao.InitWallet(context.TODO(), userNew.Uid, userNew.Gold, userNew.IapGold, userNew.Silver)
s.dao.DelWalletCache(context.TODO(), userNew.Uid)
}
}

View File

@@ -0,0 +1,39 @@
package Service
import (
"fmt"
"sync"
"testing"
"time"
"encoding/json"
"go-common/app/job/live/wallet/conf"
"go-common/app/job/live/wallet/model"
. "github.com/smartystreets/goconvey/convey"
)
var (
once sync.Once
s *Service
)
func startService() {
if err := conf.Init(); err != nil {
panic(fmt.Sprintf("conf.Init() error(%v)", err))
}
s = New(conf.Conf)
}
func TestMergeData(t *testing.T) {
Convey("Cache update", t, func() {
once.Do(startService)
time.Sleep(time.Second)
m := &model.Message{}
user := &model.User{}
m.New, _ = json.Marshal(user)
m.Old, _ = json.Marshal(user)
s.mergeData(m.New, m.Old, "update")
})
}