Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

21
app/job/main/point/BUILD Normal file
View File

@@ -0,0 +1,21 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/point/cmd:all-srcs",
"//app/job/main/point/conf:all-srcs",
"//app/job/main/point/dao:all-srcs",
"//app/job/main/point/http:all-srcs",
"//app/job/main/point/model:all-srcs",
"//app/job/main/point/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,23 @@
# v1.1.0
1. 消费积分变更databus为用户加积分
# v1.0.6
1. table check
# v1.0.5
1. point notify
# v1.0.4
1. 开启消费,去除修复数据逻辑
# v1.0.3
1. 去除vip_point表binlog insert同步
# v1.0.2
1. 修改同步逻辑
2. 修复point data
# v1.0.1
1. 增加 vip_point_change_history 数据同步去重
2. vip_point 数据同步增加 version check
# v1.0.0
1. 初始化vip积分服务

View File

@@ -0,0 +1,10 @@
# Owner
yubaihai
zhaogangtao
# Author
yubaihai
# Reviewer
zhaogangtao
linmiao

15
app/job/main/point/OWNERS Normal file
View File

@@ -0,0 +1,15 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- yubaihai
- zhaogangtao
labels:
- job
- job/main/point
- main
options:
no_parent_owners: true
reviewers:
- linmiao
- yubaihai
- zhaogangtao

View File

@@ -0,0 +1,14 @@
# point
# 项目简介
1.
# 编译环境
# 依赖包
# 编译执行

View File

@@ -0,0 +1,42 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["point-job.toml"],
importpath = "go-common/app/job/main/point/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/point/conf:go_default_library",
"//app/job/main/point/http:go_default_library",
"//app/job/main/point/service:go_default_library",
"//library/log:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,53 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/job/main/point/conf"
"go-common/app/job/main/point/http"
"go-common/app/job/main/point/service"
"go-common/library/log"
)
var (
svc *service.Service
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
// init log
log.Init(conf.Conf.Log)
defer log.Close()
log.Info("point start")
// service init
svc = service.New(conf.Conf)
http.Init(conf.Conf, svc)
// init pprof conf.Conf.Perf
// init signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("point get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
log.Info("point exit")
if err := svc.Close(); err != nil {
log.Error("srv close consumer error(%v)", err)
}
time.Sleep(2 * time.Second)
return
case syscall.SIGHUP:
default:
return
}
}
}

View File

@@ -0,0 +1,118 @@
# This is a TOML document. Boom
version = "1.0.0"
user = "nobody"
pid = "/tmp/point.pid"
dir = "./"
perf = "0.0.0.0:6420"
[log]
dir = "/data/log/point"
[bm]
[bm.inner]
addr = "0.0.0.0:9002"
maxListen = 10
timeout = "1s"
[bm.local]
addr = "0.0.0.0:9003"
maxListen = 10
timeout = "1s"
[mysql]
addr = "127.0.0.1:3306"
dsn = "root:123456@tcp(127.0.0.1:3306)/bilibili_point?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 20
idle = 10
idleTimeout ="4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[mysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[memcache]
name = "point"
proto = "tcp"
addr = ""
idle = 5
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "24h"
[dataBus]
[dataBus.OldVipBinlog]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "VipBinlog-MainAccount-Point-S"
topic = "VipBinlog-T"
action = "sub"
offset = "old"
buffer = 1024
name = "vip-job"
proto = "tcp"
addr = "172.16.33.158:6205"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[dataBus.PointBinlog]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "PointBinlog-MainAccount-S"
topic = "PointBinlog-T"
action = "sub"
offset = "old"
buffer = 1024
name = "vip-job"
proto = "tcp"
addr = "172.16.33.158:6205"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[dataBus.PointUpdate]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "PointUpdate-MainAccount-S"
topic = "PointUpdate-T"
action = "sub"
offset = "old"
buffer = 1024
name = "vip-job"
proto = "tcp"
addr = "172.16.33.158:6205"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[httpClient]
key = "5245acedbb88a3e2"
secret = "66998d7f276f47b0fd31dc026c9f22cf"
dial = "500ms"
timeout = "1s"
keepAlive = "60s"
timer = 10
[httpClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[properties]
notifyCacheDelURL = ["http://account.bilibili.co/api/communicate/purge_cache"]

View File

@@ -0,0 +1,39 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/main/point/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/conf:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/netutil:go_default_library",
"//library/net/rpc:go_default_library",
"//library/queue/databus:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,119 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/memcache"
"go-common/library/conf"
"go-common/library/database/sql"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/netutil"
"go-common/library/net/rpc"
"go-common/library/queue/databus"
"github.com/BurntSushi/toml"
)
// global var
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config config set
type Config struct {
// base
// elk
Log *log.Config
// http
BM *bm.ServerConfig
// memcache
Memcache *memcache.Config
// MySQL
MySQL *sql.Config
// Databus
DataBus *DataSource
// ProPerties
Properties *Properties
// http client
HTTPClient *bm.ClientConfig
// Backoff retries config
Backoff *netutil.BackoffConfig
PointRPC *rpc.ClientConfig
}
// DataSource databus source
type DataSource struct {
OldVipBinlog *databus.Config
PointBinlog *databus.Config
PointUpdate *databus.Config
}
// Properties def.
type Properties struct {
MaxRetries int
PointConsumeNotify map[string]string
NotifyCacheDelURL []string
}
// HTTPServers Http Servers
type HTTPServers struct {
Outer *bm.ServerConfig
Inner *bm.ServerConfig
Local *bm.ServerConfig
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,57 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["dao_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/point/conf:go_default_library",
"//app/job/main/point/model:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"memcache.go",
"mysql.go",
"notify.go",
],
importpath = "go-common/app/job/main/point/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/point/conf:go_default_library",
"//app/job/main/point/model:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,52 @@
package dao
import (
"context"
"go-common/app/job/main/point/conf"
"go-common/library/cache/memcache"
xsql "go-common/library/database/sql"
bm "go-common/library/net/http/blademaster"
)
// Dao dao
type Dao struct {
c *conf.Config
mc *memcache.Pool
db *xsql.DB
client *bm.Client
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
mc: memcache.NewPool(c.Memcache),
db: xsql.NewMySQL(c.MySQL),
client: bm.NewClient(c.HTTPClient),
}
return
}
// Close close the resource.
func (dao *Dao) Close() {
dao.mc.Close()
dao.db.Close()
}
// Ping dao ping
func (dao *Dao) Ping(c context.Context) (err error) {
if err = dao.db.Ping(c); err != nil {
return
}
err = dao.pingMC(c)
return
}
// pingMc ping
func (dao *Dao) pingMC(c context.Context) (err error) {
conn := dao.mc.Get(c)
defer conn.Close()
item := memcache.Item{Key: "ping", Value: []byte{1}, Expiration: 60}
return conn.Set(&item)
}

View File

@@ -0,0 +1,81 @@
package dao
import (
"context"
"flag"
"path/filepath"
"testing"
"time"
"go-common/app/job/main/point/conf"
"go-common/app/job/main/point/model"
xtime "go-common/library/time"
. "github.com/smartystreets/goconvey/convey"
)
var (
mid int64 = 7593623
d *Dao
c = context.TODO()
)
func init() {
dir, _ := filepath.Abs("../cmd/point-job.toml")
flag.Set("conf", dir)
conf.Init()
d = New(conf.Conf)
}
func TestAddPoint(t *testing.T) {
Convey("TestAddPoint", t, func() {
p := &model.VipPoint{
Mid: 111,
PointBalance: 1,
Ver: 1,
}
_, err := d.AddPoint(c, p)
So(err, ShouldBeNil)
})
}
// go test -test.v -test.run TestUpdatePoint
func TestUpdatePoint(t *testing.T) {
Convey("TestUpdatePoint", t, func() {
p := &model.VipPoint{
Mid: 111,
PointBalance: 2,
Ver: 1,
}
oldp := &model.VipPoint{
Ver: 1,
}
_, err := d.UpdatePoint(c, p, oldp.Ver)
So(err, ShouldBeNil)
})
}
// go test -test.v -test.run TestChangeHistory
func TestChangeHistory(t *testing.T) {
Convey("TestChangeHistory", t, func() {
h := &model.VipPointChangeHistory{
Mid: 111,
Point: 2,
OrderID: "wqwqe22112",
ChangeType: 10,
ChangeTime: xtime.Time(time.Now().Unix()),
RelationID: "11",
PointBalance: 111,
}
_, err := d.AddPointHistory(c, h)
So(err, ShouldBeNil)
})
}
// go test -test.v -test.run TestDeletePointCache
func TestDeletePointCache(t *testing.T) {
Convey("TestDeletePointCache", t, func() {
err := d.DelPointInfoCache(c, mid)
So(err, ShouldBeNil)
})
}

View File

@@ -0,0 +1,36 @@
package dao
import (
"context"
"fmt"
"go-common/library/cache/memcache"
"github.com/pkg/errors"
)
const (
_pointInfo = "pt:%d"
)
func pointKey(mid int64) string {
return fmt.Sprintf(_pointInfo, mid)
}
//DelPointInfoCache .
func (d *Dao) DelPointInfoCache(c context.Context, mid int64) (err error) {
return d.delCache(c, pointKey(mid))
}
func (d *Dao) delCache(c context.Context, key string) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
if err = conn.Delete(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
err = errors.Wrapf(err, "conn.Delete(%s)", key)
}
}
return
}

View File

@@ -0,0 +1,186 @@
package dao
import (
"context"
xsql "database/sql"
"time"
"go-common/app/job/main/point/model"
"go-common/library/database/sql"
"github.com/pkg/errors"
)
const (
_insertPoint = "INSERT INTO point_info(mid,point_balance,ver) VALUES(?,?,?);"
_updatePoint = "UPDATE point_info SET point_balance=?,ver=? WHERE mid=? AND ver=?;"
_insertPointHistory = "INSERT INTO point_change_history(mid,point,order_id,change_type,change_time,relation_id,point_balance,remark,operator) VALUES(?,?,?,?,?,?,?,?,?);"
_checkHistoryCount = "SELECT COUNT(1) FROM point_change_history WHERE mid = ? AND order_id = ?;"
_insertPointSQL = "INSERT INTO point_info(mid,point_balance,ver) VALUES(?,?,?)"
_updatePointSQL = "UPDATE point_info SET point_balance = ?,ver=? WHERE mid=? AND ver=?"
_pointInfoSQL = "SELECT mid,point_balance,ver FROM point_info WHERE mid=?"
_InsertPointHistorySQL = "INSERT INTO point_change_history(mid,point,order_id,change_type,change_time,relation_id,point_balance,remark,operator) VALUES(?,?,?,?,?,?,?,?,?)"
_midByMtime = "SELECT mid, point_balance FROM point_info where mtime > ?;"
_lastOneHistory = "SELECT `point_balance` FROM `point_change_history` WHERE mid =? ORDER BY id DESC LIMIT 1;"
_fixUpdatePointSQL = "UPDATE point_info SET point_balance = ? WHERE mid=?"
)
// BeginTran begin transaction.
func (d *Dao) BeginTran(c context.Context) (*sql.Tx, error) {
return d.db.Begin(c)
}
//TxPointInfo .
func (d *Dao) TxPointInfo(c context.Context, tx *sql.Tx, mid int64) (pi *model.PointInfo, err error) {
row := tx.QueryRow(_pointInfoSQL, mid)
pi = new(model.PointInfo)
if err = row.Scan(&pi.Mid, &pi.PointBalance, &pi.Ver); err != nil {
if err == sql.ErrNoRows {
err = nil
pi = nil
} else {
err = errors.WithStack(err)
}
}
return
}
//InsertPoint .
func (d *Dao) InsertPoint(c context.Context, tx *sql.Tx, pi *model.PointInfo) (a int64, err error) {
var res xsql.Result
if res, err = tx.Exec(_insertPointSQL, pi.Mid, pi.PointBalance, pi.Ver); err != nil {
err = errors.WithStack(err)
return
}
if a, err = res.RowsAffected(); err != nil {
err = errors.WithStack(err)
return
}
return
}
//UpdatePointInfo .
func (d *Dao) UpdatePointInfo(c context.Context, tx *sql.Tx, pi *model.PointInfo, ver int64) (a int64, err error) {
var res xsql.Result
if res, err = tx.Exec(_updatePointSQL, pi.PointBalance, pi.Ver, pi.Mid, ver); err != nil {
err = errors.WithStack(err)
return
}
if a, err = res.RowsAffected(); err != nil {
err = errors.WithStack(err)
return
}
return
}
//InsertPointHistory .
func (d *Dao) InsertPointHistory(c context.Context, tx *sql.Tx, ph *model.PointHistory) (a int64, err error) {
var res xsql.Result
if res, err = tx.Exec(_InsertPointHistorySQL, ph.Mid, ph.Point, ph.OrderID, ph.ChangeType, ph.ChangeTime, ph.RelationID, ph.PointBalance, ph.Remark, ph.Operator); err != nil {
err = errors.WithStack(err)
return
}
if a, err = res.RowsAffected(); err != nil {
err = errors.WithStack(err)
}
return
}
//AddPoint addPoint
func (d *Dao) AddPoint(c context.Context, p *model.VipPoint) (a int64, err error) {
var res xsql.Result
if res, err = d.db.Exec(c, _insertPoint, &p.Mid, &p.PointBalance, &p.Ver); err != nil {
err = errors.WithStack(err)
return
}
if a, err = res.RowsAffected(); err != nil {
err = errors.WithStack(err)
return
}
return
}
//UpdatePoint UpdatePoint row
func (d *Dao) UpdatePoint(c context.Context, p *model.VipPoint, oldver int64) (a int64, err error) {
var res xsql.Result
if res, err = d.db.Exec(c, _updatePoint, &p.PointBalance, &p.Ver, &p.Mid, oldver); err != nil {
err = errors.WithStack(err)
return
}
if a, err = res.RowsAffected(); err != nil {
err = errors.WithStack(err)
}
return
}
//AddPointHistory add point history
func (d *Dao) AddPointHistory(c context.Context, ph *model.VipPointChangeHistory) (a int64, err error) {
var res xsql.Result
if res, err = d.db.Exec(c, _insertPointHistory, &ph.Mid, &ph.Point, &ph.OrderID, &ph.ChangeType, &ph.ChangeTime, &ph.RelationID, &ph.PointBalance, &ph.Remark, &ph.Operator); err != nil {
err = errors.WithStack(err)
return
}
if a, err = res.RowsAffected(); err != nil {
err = errors.WithStack(err)
}
return
}
//HistoryCount check if have repeat record.
func (d *Dao) HistoryCount(c context.Context, mid int64, orderID string) (count int64, err error) {
row := d.db.QueryRow(c, _checkHistoryCount, mid, orderID)
if err = row.Scan(&count); err != nil {
if err == xsql.ErrNoRows {
err = nil
} else {
err = errors.WithStack(err)
}
}
return
}
//MidsByMtime point mids by mtime.
func (d *Dao) MidsByMtime(c context.Context, mtime time.Time) (pis []*model.PointInfo, err error) {
var rows *sql.Rows
if rows, err = d.db.Query(c, _midByMtime, mtime); err != nil {
err = errors.WithStack(err)
return
}
for rows.Next() {
pi := new(model.PointInfo)
if err = rows.Scan(&pi.Mid, &pi.PointBalance); err != nil {
pis = nil
err = errors.WithStack(err)
return
}
pis = append(pis, pi)
}
return
}
//LastOneHistory last one history.
func (d *Dao) LastOneHistory(c context.Context, mid int64) (point int64, err error) {
row := d.db.QueryRow(c, _lastOneHistory, mid)
if err = row.Scan(&point); err != nil {
if err == xsql.ErrNoRows {
err = nil
} else {
err = errors.WithStack(err)
}
}
return
}
//FixPointInfo fix point data .
func (d *Dao) FixPointInfo(c context.Context, mid int64, pointBalance int64) (a int64, err error) {
var res xsql.Result
if res, err = d.db.Exec(c, _fixUpdatePointSQL, pointBalance, mid); err != nil {
err = errors.WithStack(err)
return
}
if a, err = res.RowsAffected(); err != nil {
err = errors.WithStack(err)
return
}
return
}

View File

@@ -0,0 +1,65 @@
package dao
import (
"context"
"go-common/library/log"
"net/http"
"net/url"
"strconv"
"github.com/pkg/errors"
)
//notify status.
const (
_notifySuccess = "1"
)
// Notify point change .
func (d *Dao) Notify(c context.Context, notifyURL string, mid int64, orderID string, ip string) (err error) {
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("order_id", orderID)
params.Set("status", _notifySuccess)
req, err := d.client.NewRequest(http.MethodPost, notifyURL, ip, params)
if err != nil {
err = errors.Wrapf(err, "Notify NewRequest(%s)", notifyURL+"?"+params.Encode())
return
}
var res struct {
Code int `json:"code"`
}
if err = d.client.Do(c, req, &res); err != nil {
err = errors.Wrapf(err, "Notify d.client.Do(%s)", notifyURL+"?"+params.Encode())
return
}
if res.Code != 0 {
err = errors.Wrapf(err, "Notify code != 0 (%s)", notifyURL+"?"+params.Encode())
return
}
return
}
// NotifyCacheDel notify cache del.
func (d *Dao) NotifyCacheDel(c context.Context, notifyURL string, mid int64, ip string) (err error) {
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
req, err := d.client.NewRequest(http.MethodGet, notifyURL, ip, params)
if err != nil {
err = errors.Wrapf(err, "NotifyCacheDel NewRequest(%s)", notifyURL+"?"+params.Encode())
return
}
var res struct {
Code int `json:"code"`
}
if err = d.client.Do(c, req, &res); err != nil {
err = errors.Wrapf(err, "NotifyCacheDel d.client.Do(%s)", notifyURL+"?"+params.Encode())
return
}
if res.Code != 0 {
err = errors.Wrapf(err, "NotifyCacheDel code != 0 (%s)", notifyURL+"?"+params.Encode())
return
}
log.Info("notify suc(%d) url(%s)", mid, notifyURL)
return
}

View File

@@ -0,0 +1,34 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["http.go"],
importpath = "go-common/app/job/main/point/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/point/conf:go_default_library",
"//app/job/main/point/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,36 @@
package http
import (
"go-common/app/job/main/point/conf"
"go-common/app/job/main/point/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
var (
svc *service.Service
)
// Init init
func Init(c *conf.Config, s *service.Service) {
// service init
svc = s
// init router
engineInner := bm.DefaultServer(c.BM)
innerRouter(engineInner)
if err := engineInner.Start(); err != nil {
log.Error("engineInner.Start error(%v)", err)
panic(err)
}
}
// outerRouter init outer router api path.
func innerRouter(e *bm.Engine) {
//init api
e.Ping(ping)
}
// ping check server ok.
func ping(c *bm.Context) {
svc.Ping(c)
}

View File

@@ -0,0 +1,32 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"msg.go",
"point.go",
],
importpath = "go-common/app/job/main/point/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["//library/time:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,11 @@
package model
import "encoding/json"
// MsgCanal canal message struct
type MsgCanal struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
}

View File

@@ -0,0 +1,62 @@
package model
import (
"go-common/library/time"
)
// VipPoint vip_point table
type VipPoint struct {
ID int64 `json:"id"`
Mid int64 `json:"mid"`
PointBalance int64 `json:"point_balance"`
Ver int64 `json:"ver"`
}
//VipPointChangeHistory vip_point_change_history table
type VipPointChangeHistory struct {
ID int64 `json:"id"`
Mid int64 `json:"mid"`
Point int64 `json:"point"`
OrderID string `json:"order_id"`
ChangeType int8 `json:"change_type"`
ChangeTime time.Time `json:"change_time"`
RelationID string `json:"relation_id"`
PointBalance int64 `json:"point_balance"`
Remark string `json:"remark"`
Operator string `json:"operator"`
}
//VipPointChangeHistoryMsg get databus json data
type VipPointChangeHistoryMsg struct {
ID int64 `json:"id"`
Mid int64 `json:"mid"`
Point int64 `json:"point"`
OrderID string `json:"order_id"`
ChangeType int8 `json:"change_type"`
ChangeTime string `json:"change_time"`
RelationID string `json:"relation_id"`
PointBalance int64 `json:"point_balance"`
Remark string `json:"remark"`
Operator string `json:"operator"`
}
// PointInfo def.
type PointInfo struct {
Mid int64 `protobuf:"varint,1,opt,name=Mid,proto3" json:"mid"`
PointBalance int64 `protobuf:"varint,2,opt,name=PointBalance,proto3" json:"pointBalance"`
Ver int64 `protobuf:"varint,3,opt,name=Ver,proto3" json:"ver"`
}
//PointHistory point history
type PointHistory struct {
ID int64 `json:"id"`
Mid int64 `json:"mid"`
Point int64 `json:"point"`
OrderID string `json:"orderID"`
ChangeType int `json:"changeType"`
ChangeTime time.Time `json:"changeTime"`
RelationID string `json:"relationID"`
PointBalance int64 `json:"pointBalance"`
Remark string `json:"remark"`
Operator string `json:"operator"`
}

View File

@@ -0,0 +1,58 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["service_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/point/conf:go_default_library",
"//app/job/main/point/model:go_default_library",
"//library/log:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"point.go",
"service.go",
],
importpath = "go-common/app/job/main/point/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/point/conf:go_default_library",
"//app/job/main/point/dao:go_default_library",
"//app/job/main/point/model:go_default_library",
"//app/service/main/point/model:go_default_library",
"//app/service/main/point/rpc/client:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,70 @@
package service
import (
"context"
"time"
"go-common/app/job/main/point/model"
"go-common/library/log"
xtime "go-common/library/time"
)
const (
_timeFormat = "2006-01-02 15:04:05"
)
// AddPoint add point.
func (s *Service) AddPoint(c context.Context, p *model.VipPoint) (err error) {
if _, err = s.dao.AddPoint(c, p); err != nil {
log.Error("%+v", err)
return
}
s.dao.DelPointInfoCache(c, p.Mid)
return
}
// UpdatePoint update point.
func (s *Service) UpdatePoint(c context.Context, p *model.VipPoint, oldpoint *model.VipPoint) (err error) {
if _, err = s.dao.UpdatePoint(c, p, oldpoint.Ver); err != nil {
log.Error("%+v", err)
return
}
s.dao.DelPointInfoCache(c, p.Mid)
return
}
// AddPointHistory add point history.
func (s *Service) AddPointHistory(c context.Context, h *model.VipPointChangeHistoryMsg) (err error) {
var (
history = new(model.VipPointChangeHistory)
changeTime time.Time
)
history.ChangeType = h.ChangeType
history.Mid = h.Mid
history.Operator = h.Operator
history.OrderID = h.OrderID
history.Point = h.Point
history.PointBalance = h.PointBalance
history.RelationID = h.RelationID
history.Remark = h.Remark
if changeTime, err = time.ParseInLocation(_timeFormat, h.ChangeTime, time.Local); err != nil {
log.Error("time.ParseInLocation error %+v", err)
return
}
history.ChangeTime = xtime.Time(changeTime.Unix())
if _, err = s.dao.AddPointHistory(c, history); err != nil {
log.Error("%+v", err)
return
}
return
}
// Notify notify.
func (s *Service) Notify(c context.Context, msg *model.VipPointChangeHistoryMsg) (err error) {
for _, url := range s.c.Properties.NotifyCacheDelURL {
if err = s.dao.NotifyCacheDel(c, url, msg.Mid, "127.0.0.1"); err != nil {
log.Error("NotifyCacheDel fail(%d) %+v", msg.Mid, err)
}
}
return
}

View File

@@ -0,0 +1,334 @@
package service
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"go-common/app/job/main/point/conf"
"go-common/app/job/main/point/dao"
"go-common/app/job/main/point/model"
rpcmdl "go-common/app/service/main/point/model"
"go-common/app/service/main/point/rpc/client"
xsql "go-common/library/database/sql"
"go-common/library/log"
"go-common/library/queue/databus"
xtime "go-common/library/time"
)
const (
_vipPointChange = "vip_point_change_history"
_pointChange = "point_change_history"
_insert = "insert"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
oldVipPointDatabus *databus.Databus
pointDatabus *databus.Databus
pointUpdate *databus.Databus
waiter sync.WaitGroup
closed bool
pointRPC *client.Service
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
pointRPC: client.New(c.PointRPC),
}
if c.DataBus.OldVipBinlog != nil {
s.oldVipPointDatabus = databus.New(c.DataBus.OldVipBinlog)
s.waiter.Add(1)
go s.syncoldpointdataproc()
}
if c.DataBus.PointBinlog != nil {
s.pointDatabus = databus.New(c.DataBus.PointBinlog)
s.waiter.Add(1)
go s.pointbinlogproc()
}
if c.DataBus.PointUpdate != nil {
s.pointUpdate = databus.New(c.DataBus.PointUpdate)
s.waiter.Add(1)
go s.pointupdateproc()
}
return s
}
// Ping Service
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close Service
func (s *Service) Close() (err error) {
defer s.waiter.Wait()
s.closed = true
s.dao.Close()
if err = s.oldVipPointDatabus.Close(); err != nil {
log.Error("s.oldVipPointDatabus.Close() error(%v)", err)
return
}
if err = s.pointDatabus.Close(); err != nil {
log.Error("s.pointDatabus.Close() error(%v)", err)
return
}
if err = s.pointUpdate.Close(); err != nil {
log.Error("s.pointUpdate.Close() error(%v)", err)
return
}
return
}
func (s *Service) syncoldpointdataproc() {
defer s.waiter.Done()
var (
err error
msg *databus.Message
msgChan = s.oldVipPointDatabus.Messages()
ok bool
c = context.Background()
)
for {
msg, ok = <-msgChan
if s.closed || !ok {
log.Info("syncoldpointdataproc msgChan closed")
return
}
if err = msg.Commit(); err != nil {
log.Error("msg.Commit err(%v)", err)
}
message := &model.MsgCanal{}
if err = json.Unmarshal([]byte(msg.Value), message); err != nil {
log.Error("json.Unmarshal(%v) err(%v)", message, err)
continue
}
if message.Table == _vipPointChange {
history := new(model.VipPointChangeHistoryMsg)
if err = json.Unmarshal(message.New, history); err != nil {
log.Error("json.Unmarshal val(%v) error(%v)", string(message.New), err)
continue
}
if message.Action == _insert {
var count int64
if count, err = s.dao.HistoryCount(c, history.Mid, history.OrderID); err != nil {
log.Error("update point(%v) history(%v)", err, history)
continue
}
if count > 0 {
log.Error("update point change history had repeat record(%v)", history)
continue
}
changeTime, err := time.ParseInLocation("2006-01-02 15:04:05", history.ChangeTime, time.Local)
if err != nil {
log.Error("update point time.ParseInLocation(%s) error(%v)", history.ChangeTime, err)
continue
}
ph := &model.PointHistory{
Mid: history.Mid,
Point: history.Point,
OrderID: history.OrderID,
ChangeType: int(history.ChangeType),
ChangeTime: xtime.Time(changeTime.Unix()),
RelationID: history.RelationID,
PointBalance: history.PointBalance,
Remark: history.Remark,
Operator: history.Operator,
}
s.updatePointWithHistory(c, ph)
}
}
}
}
func (s *Service) pointbinlogproc() {
defer s.waiter.Done()
var (
err error
msg *databus.Message
msgChan = s.pointDatabus.Messages()
ok bool
c = context.Background()
)
for {
msg, ok = <-msgChan
if s.closed || !ok {
log.Info("pointbinlogproc msgChan closed")
return
}
if err = msg.Commit(); err != nil {
log.Error("msg.Commit err(%v)", err)
}
v := &model.MsgCanal{}
if err = json.Unmarshal([]byte(msg.Value), v); err != nil {
log.Error("json.Unmarshal(%v) err(%v)", v, err)
continue
}
if v.Table != _pointChange || v.Action != _insert {
continue
}
h := new(model.VipPointChangeHistoryMsg)
if err = json.Unmarshal(v.New, h); err != nil {
log.Error("json.Unmarshal val(%v) error(%v)", string(v.New), err)
continue
}
log.Info("point change log %+v", h)
s.Notify(c, h)
}
}
func (s *Service) updatePointWithHistory(c context.Context, ph *model.PointHistory) (pointBalance int64, activePoint int64, err error) {
var (
tx *xsql.Tx
)
if tx, err = s.dao.BeginTran(c); err != nil {
log.Error("update point mid(%d) %+v record(%v)", ph.Mid, err, ph)
return
}
defer func() {
if err == nil {
if err = tx.Commit(); err != nil {
log.Error("update point tx.Commit %+v record(%v)", err, ph)
tx.Rollback()
}
} else {
tx.Rollback()
}
}()
if pointBalance, err = s.updatePoint(c, tx, ph.Mid, ph.Point, ph); err != nil {
log.Error("update point mid(%d) %+v record(%v)", ph.Mid, err, ph)
return
}
if _, err = s.dao.InsertPointHistory(c, tx, ph); err != nil {
log.Error("update point mid(%d) %+v record(%v)", ph.Mid, err, ph)
return
}
return
}
func (s *Service) updatePoint(c context.Context, tx *xsql.Tx, mid, point int64, ph *model.PointHistory) (pb int64, err error) {
var (
pi *model.PointInfo
ver int64
a int64
)
if pi, err = s.dao.TxPointInfo(c, tx, mid); err != nil {
log.Error("%+v", err)
return
}
if point == 0 {
return
}
if pi == nil {
if point < 0 {
point = 0
log.Error("update point<0 mid(%d) (%v)", mid, ph)
}
pb = point
pi = new(model.PointInfo)
pi.Ver = 1
pi.PointBalance = point
pi.Mid = mid
if a, err = s.dao.InsertPoint(c, tx, pi); err != nil {
log.Error("%v", err)
return
}
if a != 1 {
err = fmt.Errorf("operation failed")
return
}
} else {
pb = pi.PointBalance + point
if pb < 0 {
pb = 0
log.Error("update point<0 mid(%d)(%v)", mid, ph)
}
pi.PointBalance = pb
ver = pi.Ver
pi.Ver++
if a, err = s.dao.UpdatePointInfo(c, tx, pi, ver); err != nil {
log.Error("%v", err)
return
}
if a != 1 {
err = fmt.Errorf("operation failed")
return
}
}
return
}
func (s *Service) fixdata(mtimeStr string) (err error) {
var (
pis []*model.PointInfo
c = context.TODO()
mtime time.Time
)
log.Info("fixdata start ")
if mtime, err = time.ParseInLocation("2006-01-02 15:04:05", mtimeStr, time.Local); err != nil {
log.Error("fixdata ParseInLocation error(%v)", err)
return
}
if pis, err = s.dao.MidsByMtime(c, mtime); err != nil {
log.Error("fixdata err %+v ", err)
return
}
log.Info("fixdata count %d ", len(pis))
for _, pi := range pis {
log.Info("fixdata ing %v ", pi)
var point int64
if point, err = s.dao.LastOneHistory(c, pi.Mid); err != nil {
log.Error("fixdata history err %+v ", err)
return
}
if point == pi.PointBalance {
continue
}
log.Info("fixdata mid(%d) %+v ", pi.Mid, err)
var af int64
if af, err = s.dao.FixPointInfo(c, pi.Mid, point); err != nil {
log.Error("fixdata FixPointInfo mid(%d) %+v ", pi.Mid, err)
return
}
if af != 1 {
log.Error("fixdata af!=1 mid(%d)", pi.Mid)
return
}
}
log.Info("fixdata end ")
return
}
func (s *Service) pointupdateproc() {
defer s.waiter.Done()
var (
err error
msg *databus.Message
msgChan = s.pointUpdate.Messages()
ok bool
c = context.Background()
)
for {
msg, ok = <-msgChan
if !ok {
log.Info("pointupdateproc msgChan closed")
return
}
if err = msg.Commit(); err != nil {
log.Error("msg.Commit err(%v)", err)
}
v := &rpcmdl.ArgPointAdd{}
if err = json.Unmarshal([]byte(msg.Value), v); err != nil {
log.Error("json.Unmarshal(%v) err(%v)", v, err)
continue
}
s.pointRPC.PointAddByBp(c, v)
}
}

View File

@@ -0,0 +1,159 @@
package service
import (
"context"
"flag"
"testing"
"time"
"go-common/app/job/main/point/conf"
"go-common/app/job/main/point/model"
"go-common/library/log"
xtime "go-common/library/time"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *Service
c = context.TODO()
)
func init() {
var (
err error
)
flag.Set("conf", "../cmd/point-job.toml")
if err = conf.Init(); err != nil {
panic(err)
}
c = context.Background()
if s == nil {
s = New(conf.Conf)
}
time.Sleep(time.Second)
}
// go test -test.v -test.run TestAddPoint
func TestAddPoint(t *testing.T) {
Convey("TestAddPoint", t, func() {
p := &model.VipPoint{
Mid: 111,
PointBalance: 1,
Ver: 1,
}
err := s.AddPoint(c, p)
So(err, ShouldBeNil)
})
}
// go test -test.v -test.run TestUpdatePoint
func TestUpdatePoint(t *testing.T) {
Convey("TestUpdatePoint", t, func() {
p := &model.VipPoint{
Mid: 27515415,
PointBalance: 2,
Ver: 260,
}
oldp := &model.VipPoint{
Ver: 259,
}
err := s.UpdatePoint(c, p, oldp)
So(err, ShouldBeNil)
})
}
// go test -test.v -test.run TestChangeHistory
func TestChangeHistory(t *testing.T) {
Convey("TestChangeHistory", t, func() {
h := &model.VipPointChangeHistoryMsg{
Mid: 111,
Point: 2,
OrderID: "wqwqe22112",
ChangeType: 10,
ChangeTime: "2018-03-14 18:30:57",
RelationID: "11",
PointBalance: 111,
}
err := s.AddPointHistory(c, h)
So(err, ShouldBeNil)
})
}
// go test -test.v -test.run TestUpdatePointByHistory
func TestUpdatePointByHistory(t *testing.T) {
Convey("TestUpdatePointByHistory", t, func() {
var (
count int64
err error
hs []*model.VipPointChangeHistoryMsg
history *model.VipPointChangeHistoryMsg
)
history = &model.VipPointChangeHistoryMsg{
Mid: int64(26645632),
Point: int64(150),
OrderID: "201804121752494042821812",
ChangeType: 3,
ChangeTime: "2018-04-12 17:52:49",
RelationID: "seasonId:6339",
PointBalance: int64(150),
Remark: "欢迎来到实力至上主义的教室",
Operator: "",
}
hs = append(hs, history)
history = &model.VipPointChangeHistoryMsg{
Mid: int64(26645632),
Point: int64(100),
OrderID: "201804121755088867934612",
ChangeType: 3,
ChangeTime: "2018-04-12 17:55:09",
RelationID: "seasonId:1699",
PointBalance: int64(250),
Remark: "四月是你的谎言",
Operator: "",
}
hs = append(hs, history)
for _, history := range hs {
if count, err = s.dao.HistoryCount(c, history.Mid, history.OrderID); err != nil {
log.Error("update point(%v) history(%v)", err, history)
continue
}
So(err, ShouldBeNil)
So(count == 0, ShouldBeTrue)
changeTime, err := time.ParseInLocation("2006-01-02 15:04:05", history.ChangeTime, time.Local)
So(err, ShouldBeNil)
ph := &model.PointHistory{
Mid: history.Mid,
Point: history.Point,
OrderID: history.OrderID,
ChangeType: int(history.ChangeType),
ChangeTime: xtime.Time(changeTime.Unix()),
RelationID: history.RelationID,
PointBalance: history.PointBalance,
Remark: history.Remark,
Operator: history.Operator,
}
s.updatePointWithHistory(c, ph)
So(err, ShouldBeNil)
}
})
}
// go test -test.v -test.run TestFixData
func TestFixData(t *testing.T) {
Convey("TestFixData", t, func() {
err := s.fixdata("2018-04-12 18:20:00")
So(err, ShouldBeNil)
})
}
// go test -test.v -test.run TestFixData
func TestNotify(t *testing.T) {
Convey("TestNotify", t, func() {
msg := &model.VipPointChangeHistoryMsg{
Mid: 1,
}
err := s.Notify(context.TODO(), msg)
So(err, ShouldBeNil)
})
}