openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
app/job/main/passport-game-data/BUILD Normal file
View File

@@ -0,0 +1,21 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/passport-game-data/cmd:all-srcs",
"//app/job/main/passport-game-data/conf:all-srcs",
"//app/job/main/passport-game-data/dao:all-srcs",
"//app/job/main/passport-game-data/http:all-srcs",
"//app/job/main/passport-game-data/model:all-srcs",
"//app/job/main/passport-game-data/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

7
app/job/main/passport-game-data/CHANGELOG.md Normal file
View File

@@ -0,0 +1,7 @@
## passport-game-data-job
#### Version 1.1.0
> 1.support bm.
#### Version 1.0.0
> 1.版本初始化

12
app/job/main/passport-game-data/CONTRIBUTORS.md Normal file
View File

@@ -0,0 +1,12 @@
# Owner
wanghuan01
# Author
wanghuan01
wutao
wucongyou
# Reviewer
linmiao
wanghuan01
wutao

17
app/job/main/passport-game-data/OWNERS Normal file
View File

@@ -0,0 +1,17 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- wanghuan01
- wucongyou
- wutao
labels:
- job
- job/main/passport-game-data
- main
options:
no_parent_owners: true
reviewers:
- linmiao
- wanghuan01
- wucongyou
- wutao

12
app/job/main/passport-game-data/README.md Normal file
View File

@@ -0,0 +1,12 @@
## passport-game-data-job
#### 项目简介
> 1.游戏云账号数据校验任务。
#### 编译环境
> 请只用golang v1.8.x以上版本编译执行。
#### 依赖包
> 1.公共包go-common
#### 特别说明

43
app/job/main/passport-game-data/cmd/BUILD Normal file
View File

@@ -0,0 +1,43 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["passport-game-data-job.toml"],
importpath = "go-common/app/job/main/passport-game-data/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/passport-game-data/conf:go_default_library",
"//app/job/main/passport-game-data/http:go_default_library",
"//app/job/main/passport-game-data/service:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

97
app/job/main/passport-game-data/cmd/main.go Normal file
View File

@@ -0,0 +1,97 @@
package main
import (
"context"
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/job/main/passport-game-data/conf"
"go-common/app/job/main/passport-game-data/http"
"go-common/app/job/main/passport-game-data/service"
"go-common/library/log"
"go-common/library/net/trace"
)
var (
mode int
compareMidListFile string
diffLogFile string
diffParseResFile string
)
const (
_modeNormal = 0
_modeCompareOnly = 1
_modeParseDiffLog = 2
_modeInitCloud = 3
)
func init() {
flag.IntVar(&mode, "mode", _modeNormal, "mode for starting this job, 0 for normal, 1 for compare only, 2 for parse diff log")
flag.StringVar(&compareMidListFile, "compare_mid_list_file", "/tmp/mids.txt", "compare mid list file path")
flag.StringVar(&diffLogFile, "diff_log_file", "/tmp/diff.txt", "diff log file path")
flag.StringVar(&diffParseResFile, "diff_parse_res_file", "/tmp/diff_parse_res.txt", "diff parse result file path")
}
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Xlog)
defer log.Close()
trace.Init(conf.Conf.Tracer)
defer trace.Close()
var srv *service.Service
switch mode {
case _modeCompareOnly:
s := service.NewCompareOnly(conf.Conf)
if err := s.CompareFromMidListFile(context.TODO(), compareMidListFile); err != nil {
log.Error("service.CompareFromMidListFile(%s) error(%v)", compareMidListFile, err)
}
case _modeParseDiffLog:
log.Info("parse diff log from %s", diffLogFile)
if err := service.ParseDiffLog(diffLogFile, diffParseResFile); err != nil {
log.Error("service.ParseDiffLog(%s) error(%v)", diffLogFile, err)
}
time.Sleep(time.Second * 2)
return
case _modeInitCloud:
s := service.NewInitCloud(conf.Conf)
s.InitCloud(context.TODO())
time.Sleep(time.Second * 2)
case _modeNormal:
// service init
srv = service.New(conf.Conf)
http.Init(conf.Conf, srv)
// signal handler
log.Info("passport-game-data-job start")
}
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("passport-game-data-job get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
if srv != nil {
srv.Close()
}
log.Info("passport-game-data-job exit")
return
case syscall.SIGHUP:
// TODO reload
default:
return
}
}
}

84
app/job/main/passport-game-data/cmd/passport-game-data-job.toml Normal file
View File

@@ -0,0 +1,84 @@
# This is a TOML document. Boom.
[compare]
# NOTE:
# startTime the mtime for compareproc to check from
# endTime the mtime for compareproc to check to
# stepDuration the duration step of compareproc to increase mtime,
# need to be greater than zero,
# the default value is 15m.
# loopDuration the duration between two adjacent loops of compareproc,
# need to be greater than or equal to zero,
# the default value is 3s.
# batchSize the batch size of selecting aso accounts from local aso_account table,
# the default value is 1000.
# batchMissRetryCount batch miss retry count of retrive aso accounts from local aso_account table,
# the default value is 3.
# debug if show debug messages,
# if switch this to on, will log additional compareproc info,
# such as the start time and end time of every loop.
[compare.cloud2Local]
on = false
offsetFilePath = "/data/passport-game-data-job.c2l.offset"
useOldOffset = true
end = true
startTime = "2017-11-15 22:24:45"
endTime = "2018-01-25 12:00:00"
delayDuration="15m"
stepDuration = "10m"
loopDuration = "1s"
batchSize = 1001
batchMissRetryCount = 4
fix = true
[compare.Local2Cloud]
on = true
offsetFilePath = "/data/passport-game-data-job.l2c.offset"
useOldOffset = false
end = true
startTime = "2018-02-06 11:36:09"
endTime = "2018-02-06 12:00:00"
delayDuration="10m"
stepDuration = "10m"
loopDuration = "1s"
batchSize = 1001
batchMissRetryCount = 4
fix = true
[db]
[db.local]
addr = "172.16.33.205:3306"
dsn = "aso:hA0DAnENNFz78kYB@tcp(172.16.33.205:3306)/aso?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 2
queryTimeout = "1s"
execTimeout = "2s"
tranTimeout = "2s"
[db.local.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[db.cloud]
addr = "172.16.33.205:3306"
dsn = "account:wx2U1MwXRyWEuURw@tcp(172.16.33.205:3306)/member_app1?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 2
queryTimeout = "1s"
execTimeout = "2s"
tranTimeout = "2s"
[db.cloud.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100

3
app/job/main/passport-game-data/cmd/start_init_cloud.sh Normal file
View File

@@ -0,0 +1,3 @@
#!/usr/bin/env bash
go run main.go -conf ./passport-game-data-job-init-cloud.toml -mode 3

37
app/job/main/passport-game-data/conf/BUILD Normal file
View File

@@ -0,0 +1,37 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/main/passport-game-data/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/conf:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/trace:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

136
app/job/main/passport-game-data/conf/conf.go Normal file
View File

@@ -0,0 +1,136 @@
package conf
import (
"errors"
"flag"
"go-common/library/conf"
"go-common/library/database/sql"
"go-common/library/log"
httpx "go-common/library/net/http/blademaster"
"go-common/library/net/trace"
"go-common/library/time"
"github.com/BurntSushi/toml"
)
var (
confPath string
// Conf conf.
Conf = &Config{}
client *conf.Client
)
// Config config.
type Config struct {
// log
Xlog *log.Config
// Tracer tracer
Tracer *trace.Config
// DB db
DB *DB
// Compare compare
Compare *Compare
// InitCloud init cloud.
InitCloud *InitCloud
// BM
BM *httpx.ServerConfig
}
// InitCloud init cloud conf.
type InitCloud struct {
OffsetFilePath string
UseOldOffset bool
Start, End int64
Batch int
Sleep time.Duration
}
// Compare compare
type Compare struct {
Cloud2Local *CompareConfig
Local2Cloud *CompareConfig
}
// CompareConfig compare proc config.
type CompareConfig struct {
On bool
Debug bool
OffsetFilePath string
UseOldOffset bool
End bool
StartTime string
EndTime string
DelayDuration time.Duration
StepDuration time.Duration
LoopDuration time.Duration
BatchSize int
BatchMissRetryCount int
Fix bool
}
// DB db config.
type DB struct {
Local *sql.Config
Cloud *sql.Config
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init config.
func Init() (err error) {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

54
app/job/main/passport-game-data/dao/BUILD Normal file
View File

@@ -0,0 +1,54 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"mysql_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/passport-game-data/conf:go_default_library",
"//app/job/main/passport-game-data/model:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"mysql.go",
],
importpath = "go-common/app/job/main/passport-game-data/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/passport-game-data/conf:go_default_library",
"//app/job/main/passport-game-data/model:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

48
app/job/main/passport-game-data/dao/dao.go Normal file
View File

@@ -0,0 +1,48 @@
package dao
import (
"context"
"go-common/app/job/main/passport-game-data/conf"
"go-common/library/database/sql"
"go-common/library/log"
)
// Dao dao
type Dao struct {
c *conf.Config
localDB *sql.DB
cloudDB *sql.DB
}
// New new dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
c: c,
localDB: sql.NewMySQL(c.DB.Local),
cloudDB: sql.NewMySQL(c.DB.Cloud),
}
return
}
// Ping check dao ok.
func (d *Dao) Ping(c context.Context) (err error) {
if err = d.localDB.Ping(c); err != nil {
log.Info("dao.localDB.Ping() error(%v)", err)
}
if err = d.cloudDB.Ping(c); err != nil {
log.Info("dao.cloudDB.Ping() error(%v)", err)
}
return
}
// Close close connections.
func (d *Dao) Close() (err error) {
if d.localDB != nil {
d.localDB.Close()
}
if d.cloudDB != nil {
d.cloudDB.Close()
}
return
}

20
app/job/main/passport-game-data/dao/dao_test.go Normal file
View File

@@ -0,0 +1,20 @@
package dao
import (
"fmt"
"sync"
"go-common/app/job/main/passport-game-data/conf"
)
var (
once sync.Once
d *Dao
)
func startDao() {
if err := conf.Init(); err != nil {
panic(fmt.Sprintf("conf.Init() error(%v)", err))
}
d = New(conf.Conf)
}

240
app/job/main/passport-game-data/dao/mysql.go Normal file
View File

@@ -0,0 +1,240 @@
package dao
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"go-common/app/job/main/passport-game-data/model"
xsql "go-common/library/database/sql"
"go-common/library/log"
)
const (
_getAsoAccountRangeCloudSQL = "SELECT mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak,ctime,mtime FROM aso_account WHERE mtime>=? AND mtime<?"
_getAsoAccountsCloudSQL = "SELECT mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak,ctime,mtime FROM aso_account WHERE mid in(%s)"
_getRangeAsoAccountsLocalSQL = "SELECT mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak,modify_time FROM aso_account WHERE modify_time>=? AND modify_time<?"
_getAsoAccountsLocalSQL = "SELECT mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak,modify_time FROM aso_account WHERE mid in(%s)"
_updateAsoAccountCloudSQL = "UPDATE aso_account SET userid=?,uname=?,pwd=?,salt=?,email=?,tel=?,country_id=?,mobile_verified=?,isleak=? WHERE mid=? AND mtime=?"
_addIgnoreAsoAccountCloudSQL = "INSERT IGNORE INTO aso_account (mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak) VALUES(?,?,?,?,?,?,?,?,?,?)"
_addIgnoreAsoAccountsCloudSQL = "INSERT IGNORE INTO aso_account (mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak) VALUES %s"
)
// AddAsoAccountsCloud batch add aso account to cloud.
func (d *Dao) AddAsoAccountsCloud(c context.Context, vs []*model.AsoAccount) (err error) {
if len(vs) == 0 {
return
}
var args = make([]string, 0)
for _, v := range vs {
args = append(args, getValues(v))
}
s := fmt.Sprintf(_addIgnoreAsoAccountsCloudSQL, strings.Join(args, ","))
if _, err = d.cloudDB.Exec(c, s); err != nil {
log.Error("d.cloudDB.Exec(%s) error(%v)", s, err)
}
return
}
func getValues(a *model.AsoAccount) string {
email := "NULL"
tel := "NULL"
if len(a.Email) > 0 {
email = "'" + a.Email + "'"
}
if len(a.Tel) > 0 {
tel = "'" + a.Tel + "'"
}
return fmt.Sprintf(`(%d,'%s','%s','%s','%s',%s,%s,%d,%d,%d)`, a.Mid, a.UserID, a.Uname, a.Pwd, a.Salt, email, tel, a.CountryID, a.MobileVerified, a.Isleak)
}
// AsoAccountRangeCloud get aso account from cloud.
func (d *Dao) AsoAccountRangeCloud(c context.Context, st, ed time.Time) (res []*model.AsoAccount, err error) {
var rows *xsql.Rows
if rows, err = d.cloudDB.Query(c, _getAsoAccountRangeCloudSQL, st, ed); err != nil {
log.Error("get aso account range cloud, dao.cloudDB.Query(%s) error(%v)", _getAsoAccountRangeCloudSQL, err)
return
}
for rows.Next() {
a := new(model.AsoAccount)
var telPtr, emailPtr *string
if err = rows.Scan(&a.Mid, &a.UserID, &a.Uname, &a.Pwd, &a.Salt, &emailPtr, &telPtr, &a.CountryID, &a.MobileVerified, &a.Isleak, &a.Ctime, &a.Mtime); err != nil {
if err == xsql.ErrNoRows {
err = nil
res = nil
return
}
log.Error("row.Scan() error(%v)", err)
return
}
if telPtr != nil {
a.Tel = *telPtr
}
if emailPtr != nil {
a.Email = *emailPtr
}
res = append(res, a)
}
err = rows.Err()
return
}
// AsoAccountsCloud get aso accounts from cloud.
func (d *Dao) AsoAccountsCloud(c context.Context, vs []int64) (res []*model.AsoAccount, err error) {
if len(vs) == 0 {
return
}
var args = make([]string, 0, len(vs))
for _, v := range vs {
args = append(args, fmt.Sprintf(`'%d'`, v))
}
if len(args) == 0 {
return
}
s := fmt.Sprintf(_getAsoAccountsCloudSQL, strings.Join(args, ","))
var rows *xsql.Rows
if rows, err = d.cloudDB.Query(c, s); err != nil {
log.Error("get aso accounts cloud, dao.cloudDB.Query(%s) error(%v)", s, err)
return
}
for rows.Next() {
a := new(model.AsoAccount)
var telPtr, emailPtr *string
if err = rows.Scan(&a.Mid, &a.UserID, &a.Uname, &a.Pwd, &a.Salt, &emailPtr, &telPtr, &a.CountryID, &a.MobileVerified, &a.Isleak, &a.Ctime, &a.Mtime); err != nil {
if err == xsql.ErrNoRows {
err = nil
res = nil
return
}
log.Error("row.Scan() error(%v)", err)
return
}
if telPtr != nil {
a.Tel = *telPtr
}
if emailPtr != nil {
a.Email = *emailPtr
}
res = append(res, a)
}
err = rows.Err()
return
}
// AsoAccountRangeLocal get aso account from local range start and end time.
func (d *Dao) AsoAccountRangeLocal(c context.Context, st, ed time.Time) (res []*model.OriginAsoAccount, err error) {
var rows *xsql.Rows
if rows, err = d.localDB.Query(c, _getRangeAsoAccountsLocalSQL, st, ed); err != nil {
log.Error("get aso account range local, dao.localDB.Query(%s) error(%v)", _getRangeAsoAccountsLocalSQL, err)
return
}
for rows.Next() {
a := new(model.OriginAsoAccount)
var telPtr, emailPtr *string
if err = rows.Scan(&a.Mid, &a.UserID, &a.Uname, &a.Pwd, &a.Salt, &emailPtr, &telPtr, &a.CountryID, &a.MobileVerified, &a.Isleak, &a.Mtime); err != nil {
if err == xsql.ErrNoRows {
err = nil
res = nil
return
}
log.Error("row.Scan() error(%v)", err)
return
}
if telPtr != nil {
a.Tel = *telPtr
}
if emailPtr != nil {
a.Email = *emailPtr
}
res = append(res, a)
}
err = rows.Err()
return
}
// AsoAccountsLocal get aso accounts from origin.
func (d *Dao) AsoAccountsLocal(c context.Context, vs []int64) (res []*model.OriginAsoAccount, err error) {
if len(vs) == 0 {
return
}
var args = make([]string, 0, len(vs))
for _, v := range vs {
args = append(args, fmt.Sprintf(`'%d'`, v))
}
if len(args) == 0 {
return
}
s := fmt.Sprintf(_getAsoAccountsLocalSQL, strings.Join(args, ","))
var rows *xsql.Rows
if rows, err = d.localDB.Query(c, s); err != nil {
log.Error("get aso accounts local, dao.localDB.Query(%s) error(%v)", s, err)
return
}
for rows.Next() {
a := new(model.OriginAsoAccount)
var telPtr, emailPtr *string
if err = rows.Scan(&a.Mid, &a.UserID, &a.Uname, &a.Pwd, &a.Salt, &emailPtr, &telPtr, &a.CountryID, &a.MobileVerified, &a.Isleak, &a.Mtime); err != nil {
if err == xsql.ErrNoRows {
err = nil
res = nil
return
}
log.Error("row.Scan() error(%v)", err)
return
}
if telPtr != nil {
a.Tel = *telPtr
}
if emailPtr != nil {
a.Email = *emailPtr
}
res = append(res, a)
}
err = rows.Err()
return
}
// UpdateAsoAccountCloud update aso account.
func (d *Dao) UpdateAsoAccountCloud(c context.Context, a *model.AsoAccount, mt time.Time) (affected int64, err error) {
var telPtr, emailPtr *string
if a.Tel != "" {
telPtr = &a.Tel
}
if a.Email != "" {
emailPtr = &a.Email
}
var res sql.Result
if res, err = d.cloudDB.Exec(c, _updateAsoAccountCloudSQL, a.UserID, a.Uname, a.Pwd, a.Salt, emailPtr, telPtr, a.CountryID, a.MobileVerified, a.Isleak, a.Mid, mt); err != nil {
log.Error("failed to update aso account, dao.cloudDB.Exec(%s) email(%s) tel(%s) error(%v)", _updateAsoAccountCloudSQL, emailPtr, telPtr, err)
return
}
return res.RowsAffected()
}
// AddIgnoreAsoAccount add ignore aso account.
func (d *Dao) AddIgnoreAsoAccount(c context.Context, a *model.AsoAccount) (affected int64, err error) {
var res sql.Result
var telPtr, emailPtr *string
if a.Tel != "" {
telPtr = &a.Tel
}
if a.Email != "" {
emailPtr = &a.Email
}
if res, err = d.cloudDB.Exec(c, _addIgnoreAsoAccountCloudSQL, a.Mid, a.UserID, a.Uname, a.Pwd, a.Salt, emailPtr, telPtr, a.CountryID, a.MobileVerified, a.Isleak); err != nil {
log.Error("failed to add ignore aso account, dao.cloudDB.Exec(%s) email(%s) tel(%s) error(%s)", _addIgnoreAsoAccountCloudSQL, a.Email, a.Tel, err)
return
}
return res.RowsAffected()
}

418
app/job/main/passport-game-data/dao/mysql_test.go Normal file
View File

@@ -0,0 +1,418 @@
package dao
import (
"context"
"encoding/json"
"fmt"
"testing"
"time"
"go-common/app/job/main/passport-game-data/model"
. "github.com/smartystreets/goconvey/convey"
)
const (
_timeFormat = "2006-01-02 15:04:05"
)
var (
_loc = time.Now().Location()
)
func TestDao_AddAsoAccountsCloud(t *testing.T) {
once.Do(startDao)
Convey("batch add aso account to cloud", t, func() {
Convey("single", func() {
as := make([]*model.AsoAccount, 0)
a := &model.AsoAccount{
Mid: 12047569,
UserID: "bili_1710676855",
Uname: "Bili_12047569",
Pwd: "3686c9d96ae6896fe117319ba6c07087",
Salt: "pdMXF856",
Email: "62fe0d616162f56ecab3e12a2de83ea6",
Tel: "bdb27b0300e3984e48e7aea5c672a243",
CountryID: 1,
MobileVerified: 1,
Isleak: 0,
}
as = append(as, a)
err := d.AddAsoAccountsCloud(context.TODO(), as)
So(err, ShouldBeNil)
})
Convey("multiple", func() {
as := make([]*model.AsoAccount, 0)
a := &model.AsoAccount{
Mid: 12047569,
UserID: "bili_1710676855",
Uname: "Bili_12047569",
Pwd: "3686c9d96ae6896fe117319ba6c07087",
Salt: "pdMXF856",
Email: "62fe0d616162f56ecab3e12a2de83ea6",
Tel: "bdb27b0300e3984e48e7aea5c672a243",
CountryID: 1,
MobileVerified: 1,
Isleak: 0,
}
as = append(as, a)
as = append(as, a)
err := d.AddAsoAccountsCloud(context.TODO(), as)
So(err, ShouldBeNil)
})
})
}
func TestDao_AsoAccountRangeCloud(t *testing.T) {
once.Do(startDao)
Convey("get a aso account from cloud range start time and end time", t, func() {
Convey("when start time after end time", func() {
st, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:50:41", _loc)
ed, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:49:41", _loc)
So(err, ShouldBeNil)
res, err := d.AsoAccountRangeCloud(context.TODO(), st, ed)
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 0)
})
Convey("when res is not empty", func() {
st, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:50:41", _loc)
ed, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:51:41", _loc)
So(err, ShouldBeNil)
res, err := d.AsoAccountRangeCloud(context.TODO(), st, ed)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
mid := int64(88888970)
ok := false
var target *model.AsoAccount
for _, a := range res {
if a.Mid == mid {
target = a
ok = true
break
}
}
So(ok, ShouldBeTrue)
So(target.Email, ShouldNotBeNil)
So(target.Tel, ShouldBeEmpty)
str, _ := json.Marshal(target)
t.Logf("res: %s", str)
})
})
}
func TestDao_AsoAccountRangeLocal(t *testing.T) {
once.Do(startDao)
Convey("get a aso account from local range start time and end time", t, func() {
Convey("when start time is after end time", func() {
st, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:50:41", _loc)
ed, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:49:41", _loc)
So(err, ShouldBeNil)
res, err := d.AsoAccountRangeLocal(context.TODO(), st, ed)
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 0)
})
Convey("when res is not empty", func() {
st, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:50:41", _loc)
ed, err := time.ParseInLocation(_timeFormat, "2018-01-22 12:51:41", _loc)
So(err, ShouldBeNil)
res, err := d.AsoAccountRangeLocal(context.TODO(), st, ed)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
mid := int64(88888970)
ok := false
var target *model.OriginAsoAccount
for _, a := range res {
if a.Mid == mid {
target = a
ok = true
break
}
}
So(ok, ShouldBeTrue)
So(target.Email, ShouldNotBeNil)
So(target.Tel, ShouldNotBeEmpty)
str, _ := json.Marshal(target)
t.Logf("res: %s", str)
})
})
}
func TestDao_AsoAccountsCloud(t *testing.T) {
once.Do(startDao)
Convey("get aso accounts from cloud", t, func() {
Convey("when res is empty", func() {
res, err := d.AsoAccountsCloud(context.TODO(), []int64{10000000000})
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 0)
})
Convey("when res has single item", func() {
mids := []int64{88888970}
res, err := d.AsoAccountsCloud(context.TODO(), mids)
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 1)
m := make(map[int64]*model.AsoAccount)
for _, a := range res {
m[a.Mid] = a
}
for _, mid := range mids {
a, ok := m[mid]
So(ok, ShouldBeTrue)
So(a.Email, ShouldBeEmpty)
So(a.Tel, ShouldNotBeEmpty)
str, _ := json.Marshal(a)
t.Logf("a: %s", str)
}
})
Convey("when res has multiple items", func() {
mids := []int64{88888970, 110000784}
res, err := d.AsoAccountsCloud(context.TODO(), mids)
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 2)
m := make(map[int64]*model.AsoAccount)
for _, a := range res {
m[a.Mid] = a
}
for _, mid := range mids {
a, ok := m[mid]
So(ok, ShouldBeTrue)
So(a.Email, ShouldBeEmpty)
So(a.Tel, ShouldNotBeEmpty)
str, _ := json.Marshal(a)
t.Logf("a: %s", str)
}
})
})
}
func TestDao_AsoAccountsLocal(t *testing.T) {
once.Do(startDao)
Convey("get aso accounts from local", t, func() {
Convey("when res is empty", func() {
res, err := d.AsoAccountsCloud(context.TODO(), []int64{10000000000})
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 0)
})
Convey("when res has single item", func() {
mids := []int64{88888970}
res, err := d.AsoAccountsCloud(context.TODO(), mids)
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 1)
m := make(map[int64]*model.AsoAccount)
for _, a := range res {
m[a.Mid] = a
}
for _, mid := range mids {
a, ok := m[mid]
So(ok, ShouldBeTrue)
So(a.Email, ShouldNotBeNil)
So(a.Tel, ShouldBeEmpty)
str, _ := json.Marshal(a)
t.Logf("a: %s", str)
}
})
Convey("when res has multiple items", func() {
mids := []int64{88888970, 110000784}
res, err := d.AsoAccountsCloud(context.TODO(), mids)
So(err, ShouldBeNil)
So(len(res), ShouldEqual, 2)
m := make(map[int64]*model.AsoAccount)
for _, a := range res {
m[a.Mid] = a
}
for _, mid := range mids {
a, ok := m[mid]
So(ok, ShouldBeTrue)
So(a.Email, ShouldNotBeNil)
So(a.Tel, ShouldBeEmpty)
str, _ := json.Marshal(a)
t.Logf("a: %s", str)
}
})
})
}
func TestDao_UpdateAsoAccountCloud(t *testing.T) {
once.Do(startDao)
Convey("update aso account", t, func() {
Convey("when mtime matches", func() {
mid := int64(12047569)
as, err := d.AsoAccountsCloud(context.TODO(), []int64{mid})
So(err, ShouldBeNil)
So(len(as), ShouldEqual, 1)
account := as[0]
account.MobileVerified = 1 - account.MobileVerified
affected, err := d.UpdateAsoAccountCloud(context.TODO(), account, account.Mtime)
So(err, ShouldBeNil)
So(affected, ShouldEqual, 1)
})
Convey("when mtime not matches", func() {
mid := int64(12047569)
as, err := d.AsoAccountsCloud(context.TODO(), []int64{mid})
So(err, ShouldBeNil)
So(len(as), ShouldEqual, 1)
account := as[0]
account.MobileVerified = 1 - account.MobileVerified
affected, err := d.UpdateAsoAccountCloud(context.TODO(), account, time.Now())
So(err, ShouldBeNil)
So(affected, ShouldEqual, 0)
})
})
}
func TestDao_AddIgnoreAsoAccount(t *testing.T) {
once.Do(startDao)
Convey("add ignore a aso account when not exist", t, func() {
//Convey("when not exists", func() {
// account := &model.AsoAccount{
// Mid: 12047569,
// UserID: "bili_1710676855",
// Uname: "Bili_12047569",
// Pwd: "3686c9d96ae6896fe117319ba6c07087",
// Salt: "pdMXF856",
// Email: "62fe0d616162f56ecab3e12a2de83ea6",
// Tel: "bdb27b0300e3984e48e7aea5c672a243",
// CountryID: 1,
// MobileVerified: 1,
// Isleak: 0,
// }
// affected, err := d.AddIgnoreAsoAccount(context.TODO(), account)
// So(err, ShouldBeNil)
// So(affected, ShouldEqual, 1)
//})
Convey("when not exists", func() {
account := &model.AsoAccount{
Mid: 12047569,
UserID: "bili_1710676855",
Uname: "Bili_12047569",
Pwd: "3686c9d96ae6896fe117319ba6c07087",
Salt: "pdMXF856",
Email: "62fe0d616162f56ecab3e12a2de83ea6",
Tel: "bdb27b0300e3984e48e7aea5c672a243",
CountryID: 1,
MobileVerified: 1,
Isleak: 0,
}
affected, err := d.AddIgnoreAsoAccount(context.TODO(), account)
So(err, ShouldBeNil)
So(affected, ShouldEqual, 0)
})
})
}
const (
_pattern = "INSERT INTO aso_account (mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak) VALUES(%d,'%s','%s','%s','%s',%s,%s,%d,%d,%d) ON DUPLICATE KEY UPDATE userid='%s',uname='%s',pwd='%s',salt='%s',email=%s,tel=%s,country_id=%d,mobile_verified=%d,isleak=%d;"
)
func TestDao_AddAsoAccount(t *testing.T) {
oldStr := `{
"mid": 255554277,
"userid": "bili_93079136999",
"uname": "白又寻",
"pwd": "8489c2cbddb7ee1438698a4f21ee1d78",
"salt": "r0MHcs5M",
"email": "",
"tel": "ca6d0469ca340f67f4635425dcd11581",
"country_id": 1,
"mobile_verified": 2,
"isleak": 0,
"ctime": "2017-11-25T12:03:38+08:00",
"mtime": "2017-12-04T14:19:59+08:00"
}`
old := new(model.AsoAccount)
err := json.Unmarshal([]byte(oldStr), &old)
if err != nil {
t.Error(err)
t.FailNow()
}
sql := getSQL(old)
t.Logf("sql: %s", sql)
afterStr := `{
"mid": 255554277,
"userid": "bili_93079136999",
"uname": "白又寻",
"pwd": "5f064b2ddb4d8cd5f9e01507ab1d34c6",
"salt": "ggr58PEs",
"email": "",
"tel": "ca6d0469ca340f67f4635425dcd11581",
"country_id": 1,
"mobile_verified": 2,
"isleak": 0,
"ctime": "0001-01-01T00:00:00Z",
"mtime": "2017-11-25T19:14:20+08:00"
}`
after := new(model.AsoAccount)
err = json.Unmarshal([]byte(afterStr), &after)
if err != nil {
t.Error(err)
t.FailNow()
}
afterSQL := getSQL(after)
t.Logf("after sql: %s", afterSQL)
}
func getSQL(a *model.AsoAccount) string {
email := "NULL"
tel := "NULL"
if len(a.Email) > 0 {
email = "'" + a.Email + "'"
}
if len(a.Tel) > 0 {
tel = "'" + a.Tel + "'"
}
return fmt.Sprintf(_pattern, a.Mid, a.UserID, a.Uname, a.Pwd, a.Salt, email, tel, a.CountryID, a.MobileVerified, a.Isleak, a.UserID, a.Uname, a.Pwd, a.Salt, email, tel, a.CountryID, a.MobileVerified, a.Isleak)
}

37
app/job/main/passport-game-data/http/BUILD Normal file
View File

@@ -0,0 +1,37 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"http.go",
"stat.go",
],
importpath = "go-common/app/job/main/passport-game-data/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/passport-game-data/conf:go_default_library",
"//app/job/main/passport-game-data/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

41
app/job/main/passport-game-data/http/http.go Normal file
View File

@@ -0,0 +1,41 @@
package http
import (
"net/http"
"go-common/app/job/main/passport-game-data/conf"
"go-common/app/job/main/passport-game-data/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
var (
srv *service.Service
)
// Init init http sever instance.
func Init(c *conf.Config, s *service.Service) {
srv = s
// init inner router
// engine
engIn := bm.DefaultServer(c.BM)
innerRouter(engIn)
// init inner server
if err := engIn.Start(); err != nil {
log.Error("bm.DefaultServer error(%v)", err)
panic(err)
}
}
// innerRouter init inner router.
func innerRouter(e *bm.Engine) {
e.Ping(ping)
}
// ping check server ok.
func ping(c *bm.Context) {
if err := srv.Ping(c); err != nil {
log.Error("ping error(%v)", err)
c.AbortWithStatus(http.StatusServiceUnavailable)
}
}

10
app/job/main/passport-game-data/http/stat.go Normal file
View File

@@ -0,0 +1,10 @@
package http
// import (
// wctx "go-common/library/net/http/context"
// )
// func compareProcStat(c wctx.Context) {
// res := c.Result()
// res["data"] = srv.CompareProcStat(c)
// }

33
app/job/main/passport-game-data/model/BUILD Normal file
View File

@@ -0,0 +1,33 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"compare.go",
"model.go",
"parse_log.go",
"stat.go",
],
importpath = "go-common/app/job/main/passport-game-data/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

33
app/job/main/passport-game-data/model/compare.go Normal file
View File

@@ -0,0 +1,33 @@
package model
// CompareRes the result of comparing aso account between local and cloud.
type CompareRes struct {
Flags uint8 `json:"flag"`
FlagsDesc string `json:"flags_desc"`
Seq int64 `json:"seq"`
Local *OriginAsoAccount `json:"local"`
LocalEncrypted *AsoAccount `json:"local_encrypted"`
Cloud *AsoAccount `json:"cloud"`
}
// DiffParseResp diff parse resp.
type DiffParseResp struct {
Total int `json:"total"`
SeqAndPercents []*SeqCountAndPercent `json:"seq_and_percents"`
CountAndPercents []*CountAndPercent `json:"count_and_percents"`
CompareResList []*CompareRes `json:"compare_res_list"`
}
// CountAndPercent count and percent.
type CountAndPercent struct {
DiffType string `json:"diff_type"`
Count int `json:"count"`
Percent string `json:"percent"`
}
// SeqCountAndPercent process goroutine seq count and percent.
type SeqCountAndPercent struct {
Seq int64 `json:"seq"`
Count int `json:"count"`
Percent string `json:"percent"`
}

85
app/job/main/passport-game-data/model/model.go Normal file
View File

@@ -0,0 +1,85 @@
package model
import (
"crypto/md5"
"encoding/hex"
"time"
)
const (
_cloudSalt = "bi_clould_tencent_01"
)
// AsoAccount aso account.
type AsoAccount struct {
Mid int64 `json:"mid"`
UserID string `json:"userid"`
Uname string `json:"uname"`
Pwd string `json:"pwd"`
Salt string `json:"salt"`
Email string `json:"email"`
Tel string `json:"tel"`
CountryID int64 `json:"country_id"`
MobileVerified int8 `json:"mobile_verified"`
Isleak int8 `json:"isleak"`
Ctime time.Time `json:"ctime"`
Mtime time.Time `json:"mtime"`
}
// Equals check equals.
func (a *AsoAccount) Equals(b *AsoAccount) bool {
if b == nil {
return false
}
return a.Mid == b.Mid && a.UserID == b.UserID && a.Uname == b.Uname && a.Pwd == b.Pwd &&
a.Salt == b.Salt && a.Email == b.Email && a.Tel == b.Tel && a.CountryID == b.CountryID &&
a.MobileVerified == b.MobileVerified && a.Isleak == b.Isleak
}
// OriginAsoAccount origin aso account.
type OriginAsoAccount struct {
Mid int64 `json:"mid"`
UserID string `json:"userid"`
Uname string `json:"uname"`
Pwd string `json:"pwd"`
Salt string `json:"salt"`
Email string `json:"email"`
Tel string `json:"tel"`
CountryID int64 `json:"country_id"`
MobileVerified int8 `json:"mobile_verified"`
Isleak int8 `json:"isleak"`
Mtime time.Time `json:"modify_time"`
}
// Default doHash aso account, including the followings fields: userid, uname, pwd, email, tel.
func Default(a *OriginAsoAccount) *AsoAccount {
return &AsoAccount{
Mid: a.Mid,
UserID: a.UserID,
Uname: a.Uname,
Pwd: doHash(a.Pwd, _cloudSalt),
Salt: a.Salt,
Email: doHash(a.Email, _cloudSalt),
Tel: doHash(a.Tel, _cloudSalt),
CountryID: a.CountryID,
MobileVerified: a.MobileVerified,
Isleak: a.Isleak,
Mtime: a.Mtime,
}
}
// DefaultHash hash a plain text using default salt.
func DefaultHash(plaintext string) string {
return doHash(plaintext, _cloudSalt)
}
func doHash(plaintext, salt string) string {
if plaintext == "" {
return ""
}
hash := md5.New()
hash.Write([]byte(plaintext))
hash.Write([]byte(salt))
md := hash.Sum(nil)
return hex.EncodeToString(md)
}

6
app/job/main/passport-game-data/model/parse_log.go Normal file
View File

@@ -0,0 +1,6 @@
package model
// Log log.
type Log struct {
Log string `json:"log"`
}

51
app/job/main/passport-game-data/model/stat.go Normal file
View File

@@ -0,0 +1,51 @@
package model
import (
"fmt"
"time"
)
type ProcStat struct {
Cloud2Local *CompareProcStat `json:"cloud_2_local"`
Local2Cloud *CompareProcStat `json:"local_2_cloud"`
}
// CompareProcStat status of compare proc.
type CompareProcStat struct {
StartTime string `json:"start_time"`
EndTime string `json:"end_time"`
StepDuration JsonDuration `json:"step_duration"`
LoopDuration JsonDuration `json:"loop_duration"`
DelayDuration JsonDuration `json:"delay_duration"`
BatchSize int `json:"batch_size"`
BatchMissRetryCount int `json:"batch_miss_retry_count"`
Debug bool `json:"debug"`
Fix bool `json:"fix"`
CurrentRangeStart JSONTime `json:"current_range_start"`
CurrentRangeEnd JSONTime `json:"current_range_end"`
CurrentRangeRecordsCount int `json:"current_range_records_count"`
TotalRangeRecordsCount int `json:"total_range_records_count"`
DiffCount int `json:"diff_count"`
Sleeping bool `json:"sleeping"`
SleepFrom string `json:"sleep_from,omitempty"`
SleepSeconds int64 `json:"sleep_seconds,omitempty"`
SleepRemainSeconds int64 `json:"sleep_remain_seconds,omitempty"`
}
type JSONTime time.Time
func (t JSONTime) MarshalJSON() ([]byte, error) {
s := fmt.Sprintf(`"%s"`, time.Time(t).Format("2006-01-02 15:04:05"))
return []byte(s), nil
}
type JsonDuration time.Duration
func (t JsonDuration) MarshalJSON() ([]byte, error) {
s := fmt.Sprintf(`"%v"`, time.Duration(t))
return []byte(s), nil
}

64
app/job/main/passport-game-data/service/BUILD Normal file
View File

@@ -0,0 +1,64 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"batch_query_test.go",
"cloud2local_test.go",
"parse_diff_log_test.go",
"service_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/passport-game-data/conf:go_default_library",
"//app/job/main/passport-game-data/model:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"batch_query.go",
"cloud2local.go",
"compare_only.go",
"diff.go",
"init_cloud.go",
"local2cloud.go",
"log_and_fix.go",
"parse_diff_log.go",
"service.go",
"stat.go",
],
importpath = "go-common/app/job/main/passport-game-data/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/passport-game-data/conf:go_default_library",
"//app/job/main/passport-game-data/dao:go_default_library",
"//app/job/main/passport-game-data/model:go_default_library",
"//library/log:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

106
app/job/main/passport-game-data/service/batch_query.go Normal file
View File

@@ -0,0 +1,106 @@
package service
import (
"context"
"go-common/app/job/main/passport-game-data/model"
"go-common/library/log"
)
func (s *Service) batchQueryLocalNonMiss(c context.Context, mids []int64, batchSize, batchMissRetryCount int) (res []*model.OriginAsoAccount) {
as, miss := s.batchQueryLocalByMid(c, mids, batchSize)
if len(miss) == 0 {
return as
}
res = as
for i := 0; i < batchMissRetryCount; i++ {
log.Info("try for the %dth retry, miss mids: %v", miss)
as, miss = s.batchQueryLocalByMid(c, miss, batchSize)
res = append(res, as...)
if len(miss) == 0 {
return
}
if i == batchMissRetryCount-1 {
log.Error("still miss those mids: %v after %d tries", miss, batchMissRetryCount)
}
}
return
}
func (s *Service) batchQueryLocalByMid(c context.Context, mids []int64, batchSize int) (res []*model.OriginAsoAccount, miss []int64) {
if len(mids) == 0 {
return
}
res = make([]*model.OriginAsoAccount, 0)
miss = make([]int64, 0)
bc := len(mids)/batchSize + 1
for i := 0; i < bc; i++ {
start := i * batchSize
end := (i + 1) * batchSize
if end > len(mids) {
end = len(mids)
}
partMids := mids[start:end]
as, err := s.d.AsoAccountsLocal(c, partMids)
if err != nil {
miss = append(miss, partMids...)
continue
}
res = append(res, as...)
}
return
}
func (s *Service) batchQueryCloudNonMiss(c context.Context, mids []int64, batchSize, batchMissRetryCount int) (res []*model.AsoAccount) {
if len(mids) == 0 {
return
}
as, miss := s.batchQueryCloudByMid(c, mids, batchSize)
if len(miss) == 0 {
return as
}
res = as
for i := 0; i < batchMissRetryCount; i++ {
log.Info("try for the %dth times, miss mids: %v", i, miss)
as, miss = s.batchQueryCloudByMid(c, miss, batchSize)
res = append(res, as...)
if len(miss) == 0 {
return
}
if i == batchMissRetryCount-1 {
log.Error("still miss those mids: %v after %d tries", miss, batchMissRetryCount)
}
}
return
}
func (s *Service) batchQueryCloudByMid(c context.Context, mids []int64, batchSize int) (res []*model.AsoAccount, miss []int64) {
if len(mids) == 0 {
return
}
res = make([]*model.AsoAccount, 0)
miss = make([]int64, 0)
bc := len(mids)/batchSize + 1
for i := 0; i < bc; i++ {
start := i * batchSize
end := (i + 1) * batchSize
if end > len(mids) {
end = len(mids)
}
partMids := mids[start:end]
as, err := s.d.AsoAccountsCloud(c, partMids)
if err != nil {
miss = append(miss, partMids...)
continue
}
res = append(res, as...)
}
return
}

36
app/job/main/passport-game-data/service/batch_query_test.go Normal file
View File

@@ -0,0 +1,36 @@
package service
import (
"context"
"encoding/json"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestService_batchQueryCloudByMid(t *testing.T) {
once.Do(startService)
Convey("batch query cloud by mid", t, func() {
mids := []int64{88888970, 88894849}
res, miss := s.batchQueryCloudByMid(context.TODO(), mids, 1)
str, _ := json.Marshal(res)
t.Logf("res: %s", str)
So(len(miss), ShouldEqual, 0)
So(len(res), ShouldEqual, 2)
})
}
func TestService_batchQueryCloudByMidNonMiss(t *testing.T) {
once.Do(startService)
Convey("batch query cloud by mid", t, func() {
mids := []int64{88888970, 88894849}
res := s.batchQueryCloudNonMiss(context.TODO(), mids, 1001, 1)
str, _ := json.Marshal(res)
t.Logf("res: %s", str)
So(len(res), ShouldEqual, 2)
})
}

156
app/job/main/passport-game-data/service/cloud2local.go Normal file
View File

@@ -0,0 +1,156 @@
package service
import (
"context"
"io/ioutil"
"os"
"time"
"go-common/app/job/main/passport-game-data/model"
"go-common/library/log"
)
// cloud2localcompareproc compare aso accounts between cloud and local.
// select last modified from cloud
// load batchSize from origin
// compare:
// if cloud_mtime >= local_mtime, directly compare
// if cloud_mtime < local_mtime, sleep and reload from cloud, then do compare again
func (s *Service) cloud2localcompareproc() {
var (
err error
cloudRes []*model.AsoAccount
ack = false
)
cc := s.c2lC
delay := cc.DelayDuration
cc.st = cc.StartTime
cc.ed = cc.st.Add(cc.StepDuration)
offsetFile, err := os.Create(cc.OffsetFilePath)
if err != nil {
log.Error("failed to create offset file, os.Create(%s) error(%v)", cc.OffsetFilePath, err)
return
}
defer offsetFile.Close()
log.Info("created offset file %s", cc.OffsetFilePath)
for {
time.Sleep(cc.LoopDuration)
cc.sleeping = false
if ack {
cc.st = cc.st.Add(cc.StepDuration)
cc.ed = cc.ed.Add(cc.StepDuration)
}
st, ed := cc.st, cc.ed
if err = ioutil.WriteFile(cc.OffsetFilePath, []byte(st.Format(_timeFormat)), os.ModeAppend); err != nil {
log.Error("failed to write offset, ioutil.WriteFile(%s, %s, os.ModeAppend), error(%v)", cc.OffsetFilePath, st.Format(_timeFormat), err)
continue
}
if cc.Debug {
log.Info("st: %s, ed: %s", st.Format(_timeFormat), ed.Format(_timeFormat))
}
if cc.End && st.After(cc.EndTime) {
log.Info("st:%s is after endTime:%s, all data compares ok, cloud2localcompareproc exit", st.Format(_timeFormat), cc.EndTime.Format(_timeFormat))
return
}
now := time.Now()
if now.Sub(st) <= delay {
delta := int64(delay/time.Second) - (now.Unix() - st.Unix())
log.Info("now time is just after st by %d seconds, not greater than delay duration: %v, will sleep %d seconds", int64(delay/time.Second)-delta, delay, delta)
cc.sleeping = true
cc.sleepingSeconds = delta
cc.sleepFromTs = now.Unix()
time.Sleep(time.Duration(int64(time.Second) * delta))
continue
}
if cloudRes, err = s.d.AsoAccountRangeCloud(context.TODO(), st, ed); err != nil {
continue
}
cc.rangeCount = len(cloudRes)
cc.totalCount += len(cloudRes)
if err = s.cloud2LocalCompare(context.TODO(), cloudRes); err != nil {
continue
}
ack = true
}
}
func (s *Service) cloud2LocalCompare(c context.Context, cloudRes []*model.AsoAccount) (err error) {
mids := make([]int64, 0)
for _, item := range cloudRes {
mids = append(mids, item.Mid)
}
cc := s.c2lC
localRes := s.batchQueryLocalNonMiss(context.TODO(), mids, cc.BatchSize, cc.BatchMissRetryCount)
m := make(map[int64]*model.OriginAsoAccount)
for _, item := range localRes {
m[item.Mid] = item
}
// compare
pendingMids := make([]int64, 0)
for _, item := range cloudRes {
cloud := item
local := m[item.Mid]
status := doCompare(cloud, local, true)
switch status {
case _statusOK:
// do nothing
case _statusNo:
cc.diffCount++
s.doLog(cloud, local, false)
if cc.Fix {
s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
}
case _statusPending:
pendingMids = append(pendingMids, item.Mid)
}
}
if len(pendingMids) == 0 {
return
}
// reload pending mids from cloud
var pendingRes []*model.AsoAccount
if pendingRes, err = s.d.AsoAccountsCloud(context.TODO(), pendingMids); err != nil {
return
}
// compare
for _, item := range pendingRes {
cloud := item
local := m[item.Mid]
status := doCompare(item, m[item.Mid], false)
switch status {
case _statusOK:
case _statusNo:
cc.diffCount++
s.doLog(cloud, local, true)
if cc.Fix {
s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
}
}
}
return
}

14
app/job/main/passport-game-data/service/cloud2local_test.go Normal file
View File

@@ -0,0 +1,14 @@
package service
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestService_DoCompare(t *testing.T) {
once.Do(startService)
Convey("compare", t, func() {
})
}

72
app/job/main/passport-game-data/service/compare_only.go Normal file
View File

@@ -0,0 +1,72 @@
package service
import (
"bufio"
"context"
"io"
"os"
"strconv"
"time"
"go-common/app/job/main/passport-game-data/conf"
"go-common/app/job/main/passport-game-data/dao"
"go-common/library/log"
)
// NewCompareOnly new a service for compare only.
func NewCompareOnly(c *conf.Config) (s *Service) {
s = &Service{
c: c,
d: dao.New(c),
l2cC: newCompareConfigFrom(c.Compare.Local2Cloud),
}
return
}
// CompareFromMidListFile load mid list from file and compare.
func (s *Service) CompareFromMidListFile(c context.Context, fn string) (err error) {
var f *os.File
if f, err = os.Open(fn); err != nil {
log.Error("failed to open file %s, error(%v)", fn, err)
return
}
defer f.Close()
cc := s.l2cC
rd := bufio.NewReader(f)
skippedCount := 0
var (
mid int64
mids = make([]int64, 0)
line []byte
isPrefix bool
)
for {
line, isPrefix, err = rd.ReadLine()
if isPrefix || err != nil || err == io.EOF {
break
}
mid, err = strconv.ParseInt(string(line), 10, 64)
if err != nil {
log.Error("failed to parse mid, strconv.ParseInt(%s, 10 ,64) error(%v), skip", line, err)
skippedCount++
continue
}
mids = append(mids, mid)
}
log.Info("mid list len: %d, total skipped count: %d", len(mids), skippedCount)
if len(mids) == 0 {
return
}
for {
time.Sleep(cc.LoopDuration)
if err = s.local2CloudCompare(context.TODO(), s.batchQueryLocalNonMiss(context.TODO(), mids, cc.BatchSize, cc.BatchMissRetryCount)); err == nil {
break
}
log.Error("failed to compare mids %v, retrying", mids)
}
return
}

29
app/job/main/passport-game-data/service/diff.go Normal file
View File

@@ -0,0 +1,29 @@
package service
import (
"go-common/app/job/main/passport-game-data/model"
"go-common/library/log"
)
const (
_statusOK = 0
_statusPending = 1
_statusNo = 2
)
func doCompare(cloud *model.AsoAccount, local *model.OriginAsoAccount, pending bool) int {
if cloud == nil || local == nil {
log.Info("either cloud or local aso account is nil, cloud %+v, local: %+v", cloud, local)
return _statusNo
}
if cloud.Mtime.After(local.Mtime) {
if model.Default(local).Equals(cloud) {
return _statusOK
}
return _statusNo
}
if pending {
return _statusPending
}
return _statusNo
}

146
app/job/main/passport-game-data/service/init_cloud.go Normal file
View File

@@ -0,0 +1,146 @@
package service
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"strconv"
"time"
"go-common/app/job/main/passport-game-data/conf"
"go-common/app/job/main/passport-game-data/dao"
"go-common/app/job/main/passport-game-data/model"
"go-common/library/log"
)
const (
_defaultInitCloudOffsetFilePath = "/data/passport-game-data-job.initcloud.offset"
_defaultInitCloudSleep = time.Second
)
type initCloudConfig struct {
OffsetFilePath string
UseOldOffset bool
Start, End int64
Batch int
Sleep time.Duration
}
func newInitCloudConfigFrom(c *conf.Config) (ic *initCloudConfig) {
ic = &initCloudConfig{
OffsetFilePath: c.InitCloud.OffsetFilePath,
UseOldOffset: c.InitCloud.UseOldOffset,
Start: c.InitCloud.Start,
End: c.InitCloud.End,
Batch: c.InitCloud.Batch,
Sleep: time.Duration(c.InitCloud.Sleep),
}
ic.fix()
if ic.UseOldOffset {
data, err := ioutil.ReadFile(ic.OffsetFilePath)
if err != nil {
log.Error("failed to read old offset, skip")
return
}
oldOffset, err := strconv.ParseInt(string(data), 10, 64)
if err != nil {
log.Error("failed to parse offset, strconv.ParseInt(%s, 10, 64)", string(data), err)
return
}
if oldOffset > 0 {
ic.Start = oldOffset
}
}
return
}
func (ic *initCloudConfig) fix() {
if len(ic.OffsetFilePath) == 0 {
ic.OffsetFilePath = _defaultInitCloudOffsetFilePath
}
if ic.Start < 0 {
ic.Start = 0
}
if ic.End < 0 {
ic.End = 0
}
if ic.Batch <= 0 {
ic.Batch = _defaultBatchSize
}
if int64(ic.Sleep) < 0 {
ic.Sleep = _defaultInitCloudSleep
}
}
// NewInitCloud new a service for initiating cloud.
func NewInitCloud(c *conf.Config) (s *Service) {
s = &Service{
c: c,
d: dao.New(c),
ic: newInitCloudConfigFrom(c),
}
return
}
// InitCloud init cloud.
func (s *Service) InitCloud(c context.Context) {
var err error
ic := s.ic
dstFile, err := os.Create(ic.OffsetFilePath)
if err != nil {
log.Error("failed to open file %s, error(%v)", ic.OffsetFilePath, err)
return
}
defer dstFile.Close()
for i := ic.Start; i <= ic.End; {
time.Sleep(ic.Sleep)
if err = ioutil.WriteFile(ic.OffsetFilePath, []byte(strconv.FormatInt(i, 10)), os.ModeAppend); err != nil {
log.Error("failed to record offset, offsetFilePath: %s, offset: %d, error(%v)", ic.OffsetFilePath, i, err)
continue
}
st := i
ed := i + int64(ic.Batch)
if ed > ic.End {
ed = ic.End
}
mids := make([]int64, 0)
for j := st; j <= ed; j++ {
mids = append(mids, j)
}
var as []*model.OriginAsoAccount
if as, err = s.d.AsoAccountsLocal(c, mids); err != nil {
log.Error("failed to get local aso accounts by mids, service.dao.AsoAccountsLocal(%v) error(%v)", mids, err)
continue
}
cloudAs := make([]*model.AsoAccount, 0)
for _, a := range as {
cloudAs = append(cloudAs, model.Default(a))
}
if err = s.d.AddAsoAccountsCloud(c, cloudAs); err != nil {
str, _ := json.Marshal(cloudAs)
log.Error("failed to add aso accounts to cloud, service.dao.AddAsoAccountsCloud(%v) error(%v)", str, err)
continue
}
i += int64(ic.Batch)
}
}

162
app/job/main/passport-game-data/service/local2cloud.go Normal file
View File

@@ -0,0 +1,162 @@
package service
import (
"context"
"io/ioutil"
"os"
"time"
"go-common/app/job/main/passport-game-data/model"
"go-common/library/log"
)
// local2cloudcompareproc compare aso accounts between local and cloud, delay for the given duration.
// select last modified from cloud
// load batchSize from origin
// compare:
// if cloud_mtime >= local_mtime, directly compare
// if cloud_mtime < local_mtime, sleep and reload from cloud, then do compare again
func (s *Service) local2cloudcompareproc() {
var (
err error
localRes []*model.OriginAsoAccount
ack = false
)
cc := s.l2cC
delay := cc.DelayDuration
cc.st = cc.StartTime
cc.ed = cc.st.Add(cc.StepDuration)
offsetFile, err := os.Create(cc.OffsetFilePath)
if err != nil {
log.Error("failed to create offset file, os.Create(%s) error(%v)", cc.OffsetFilePath, err)
return
}
defer offsetFile.Close()
log.Info("created offset file %s", cc.OffsetFilePath)
for {
time.Sleep(cc.LoopDuration)
cc.sleeping = false
if ack {
cc.st = cc.st.Add(cc.StepDuration)
cc.ed = cc.ed.Add(cc.StepDuration)
}
st, ed := cc.st, cc.ed
if err = ioutil.WriteFile(cc.OffsetFilePath, []byte(st.Format(_timeFormat)), os.ModeAppend); err != nil {
log.Error("failed to write offset, ioutil.WriteFile(%s, %s, os.ModeAppend), error(%v)", cc.OffsetFilePath, st.Format(_timeFormat), err)
continue
}
if cc.Debug {
log.Info("st: %s, ed: %s", st.Format(_timeFormat), ed.Format(_timeFormat))
}
if cc.End && st.After(cc.EndTime) {
log.Info("st:%s is after endTime:%s, all data compares ok, local2cloudcompareproc exit", st.Format(_timeFormat), cc.EndTime.Format(_timeFormat))
return
}
now := time.Now()
if now.Sub(st) <= delay {
delta := int64(delay/time.Second) - (now.Unix() - st.Unix())
log.Info("now time is just after st by %d seconds, not greater than delay duration: %v, will sleep %d seconds", int64(delay/time.Second)-delta, delay, delta)
cc.sleeping = true
cc.sleepingSeconds = delta
cc.sleepFromTs = now.Unix()
time.Sleep(time.Duration(int64(time.Second) * delta))
continue
}
if localRes, err = s.d.AsoAccountRangeLocal(context.TODO(), st, ed); err != nil {
continue
}
cc.rangeCount = len(localRes)
cc.totalCount += len(localRes)
if err = s.local2CloudCompare(context.TODO(), localRes); err != nil {
continue
}
ack = true
}
}
func (s *Service) local2CloudCompare(c context.Context, lRes []*model.OriginAsoAccount) (err error) {
mids := make([]int64, 0)
for _, item := range lRes {
mids = append(mids, item.Mid)
}
cc := s.l2cC
// query from cloud
cRes := s.batchQueryCloudNonMiss(context.TODO(), mids, cc.BatchSize, cc.BatchMissRetryCount)
cM := make(map[int64]*model.AsoAccount)
for _, item := range cRes {
cM[item.Mid] = item
}
// compare cloud with local
pendingMids := make([]int64, 0)
for _, item := range lRes {
local := item
cloud := cM[item.Mid]
status := doCompare(cloud, local, true)
switch status {
case _statusOK:
// do nothing
case _statusNo:
cc.diffCount++
s.doLog(cloud, local, false)
if cc.Fix {
s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
}
case _statusPending:
pendingMids = append(pendingMids, item.Mid)
}
}
if len(pendingMids) == 0 {
return
}
// reload pending mids from cloud
var pendingRes []*model.AsoAccount
if pendingRes, err = s.d.AsoAccountsCloud(context.TODO(), pendingMids); err != nil {
return
}
lM := make(map[int64]*model.OriginAsoAccount)
for _, item := range lRes {
lM[item.Mid] = item
}
// compare
for _, item := range pendingRes {
cloud := item
local := lM[item.Mid]
status := doCompare(item, local, false)
switch status {
case _statusOK:
case _statusNo:
cc.diffCount++
s.doLog(cloud, local, true)
if cc.Fix {
s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
}
}
}
return
}

67
app/job/main/passport-game-data/service/log_and_fix.go Normal file
View File

@@ -0,0 +1,67 @@
package service
import (
"context"
"encoding/json"
"go-common/app/job/main/passport-game-data/model"
"go-common/library/log"
)
func (s *Service) fixCloudRecord(c context.Context, newRecord *model.AsoAccount, old *model.AsoAccount) {
if old == nil {
affected, err := s.d.AddIgnoreAsoAccount(c, newRecord)
if err != nil {
oldStr, _ := json.Marshal(old)
newStr, _ := json.Marshal(newRecord)
log.Error("failed to fix cloud record by adding, old(%s) new(%s) error(%v)", oldStr, newStr, err)
return
}
if affected == 0 {
oldStr, _ := json.Marshal(old)
newStr, _ := json.Marshal(newRecord)
log.Error("failed to fix cloud record by adding because of concurrent update, old(%s) new(%s)", oldStr, newStr)
return
}
oldStr, _ := json.Marshal(old)
newStr, _ := json.Marshal(newRecord)
log.Info("fix cloud record by adding ok, old(%s) new(%s)", oldStr, newStr)
return
}
affected, err := s.d.UpdateAsoAccountCloud(c, newRecord, old.Mtime)
if err != nil {
oldStr, _ := json.Marshal(old)
newStr, _ := json.Marshal(newRecord)
log.Error("failed to fix cloud record by updating, old(%s) new(%s) error(%v)", oldStr, newStr, err)
return
}
if affected == 0 {
oldStr, _ := json.Marshal(old)
newStr, _ := json.Marshal(newRecord)
log.Error("failed to fix cloud record by updating because of concurrent update, old(%s) new(%s)", oldStr, newStr)
return
}
oldStr, _ := json.Marshal(old)
newStr, _ := json.Marshal(newRecord)
log.Info("fix cloud record by updating ok, old(%s) new(%s)", oldStr, newStr)
}
func (s *Service) doLog(cloud *model.AsoAccount, local *model.OriginAsoAccount, afterPending bool) {
localStr := []byte("nil")
localEncStr := []byte("nil")
if local != nil {
localStr, _ = json.Marshal(local)
localEncStr, _ = json.Marshal(model.Default(local))
}
cloudStr := []byte("nil")
if cloud != nil {
cloudStr, _ = json.Marshal(cloud)
}
if afterPending {
log.Info("failed to compare, because cloud record is not updated in time, local(%s) local_encrypted(%s) cloud(%s)", localStr, localEncStr, cloudStr)
return
}
log.Info("compare diff, local(%s) local_encrypted(%s) cloud(%s)", localStr, localEncStr, cloudStr)
}

255
app/job/main/passport-game-data/service/parse_diff_log.go Normal file
View File

@@ -0,0 +1,255 @@
package service
import (
"bufio"
"encoding/json"
"fmt"
"io"
"os"
"sort"
"strings"
"go-common/app/job/main/passport-game-data/model"
"go-common/library/log"
)
const (
_cloudJobGoroutineNum = 32
)
// ParseDiffLog parse diff log printed by compare proc.
func ParseDiffLog(src, dst string) (err error) {
f, err := os.Open(src)
if err != nil {
log.Error("failed to open file %s, error(%v)", src, err)
return
}
defer f.Close()
dstFile, err := os.Create(dst)
if err != nil {
log.Error("failed to open file %s, error(%v)", dst, err)
return
}
defer dstFile.Close()
var (
line string
skippedCount = 0
res = make([]*model.CompareRes, 0)
rd = bufio.NewReader(f)
)
for {
line, err = rd.ReadString('\n')
if err != nil || io.EOF == err {
break
}
idx := strings.LastIndex(line, "]")
if idx == -1 {
log.Error("failed to parse log, expected have ] in string but not")
skippedCount++
continue
}
logJSON := line[idx+1:]
l := new(model.Log)
if err = json.Unmarshal([]byte(logJSON), &l); err != nil {
log.Error("failed to parse log, json.Unmarshal(%s) error(%v), skip", logJSON, err)
skippedCount++
continue
}
var cRes *model.CompareRes
if cRes, err = diffLog2CompareRes(l.Log); err != nil {
log.Error("diffLog2CompareRes(%s) error(%v), skip", l.Log, err)
skippedCount++
continue
}
// compare local encrypted and cloud, parse diff flags
flags := diff(cRes.Cloud, cRes.LocalEncrypted)
if flags == _diffTypeNon {
continue
}
cRes.Flags = flags
cRes.FlagsDesc = formatFlags(flags)
cRes.Seq = cRes.Local.Mid % _cloudJobGoroutineNum
res = append(res, cRes)
}
percentMap := make(map[uint8]*model.CountAndPercent)
seqMap := make(map[int64]*model.SeqCountAndPercent)
for _, v := range res {
percent, ok := percentMap[v.Flags]
if !ok {
percent = &model.CountAndPercent{
DiffType: v.FlagsDesc,
}
percentMap[v.Flags] = percent
}
percent.Count++
seq, ok := seqMap[v.Seq]
if !ok {
seq = &model.SeqCountAndPercent{
Seq: v.Seq,
}
seqMap[v.Seq] = seq
}
seq.Count++
}
sort.Slice(res, func(i, j int) bool {
return res[i].Cloud.Mtime.After(res[j].Cloud.Mtime)
})
percentList := make([]*model.CountAndPercent, 0)
for _, v := range percentMap {
v.Percent = fmt.Sprintf("%0.2f", 100*float64(v.Count)/float64(len(res))) + "%"
percentList = append(percentList, v)
}
sort.Slice(percentList, func(i, j int) bool {
return percentList[i].Count > percentList[j].Count
})
seqList := make([]*model.SeqCountAndPercent, 0)
for _, v := range seqMap {
v.Percent = fmt.Sprintf("%0.2f", 100*float64(v.Count)/float64(_cloudJobGoroutineNum)) + "%"
seqList = append(seqList, v)
}
sort.Slice(seqList, func(i, j int) bool {
return seqList[i].Count > seqList[j].Count
})
stat := &model.DiffParseResp{
Total: len(res),
SeqAndPercents: seqList,
CompareResList: res,
CountAndPercents: percentList,
}
str, _ := json.Marshal(stat)
_, err = dstFile.WriteString(string(str))
if err != nil {
log.Info("failed to write parse diff log result to file %s, error(%v)", dst, err)
}
log.Info("len res: %d, write ok", len(res))
return
}
func diffLog2CompareRes(str string) (*model.CompareRes, error) {
idx := strings.Index(str, "local")
if idx == -1 {
return nil, fmt.Errorf("failed to parse diff log, expected have local in string but not")
}
res := replace(str[idx:])
cRes := new(model.CompareRes)
err := json.Unmarshal([]byte(res), &cRes)
return cRes, err
}
// parse string like "local({\"mid\":1}) local_encrypted({\"mid\":1}) cloud({\"mid\":1})" to json string {"local":{},"local_encrypted":{},"cloud":{}}
func replace(str string) string {
res := strings.Replace(str, "local(", `{"local":`, -1)
res = strings.Replace(res, "local_encrypted(", `"local_encrypted":`, -1)
res = strings.Replace(res, "cloud(", `"cloud":`, -1)
res = strings.Replace(res, ")", ",", -1)
res = strings.Replace(res, "\\", "", -1)
if strings.HasSuffix(res, ",") {
res = res[:len(res)-1]
}
res = res + "}"
return res
}
const (
_diffTypeNon = uint8(0) // 0x00000000
_diffTypePwd = uint8(1) // 0x00000001
_diffTypeEmail = uint8(2) // 0x00000010
_diffTypeTel = uint8(4) // 0x00000100
_diffTypeCountryID = uint8(16) // 0x00001000
_diffTypeMobileVerified = uint8(32) // 0x00010000
_diffTypeIsLeak = uint8(64) // 0x00100000
)
func formatFlags(flags uint8) string {
fs := make([]string, 0)
if flags&_diffTypePwd > 0 {
fs = append(fs, "pwd")
}
if flags&_diffTypeEmail > 0 {
fs = append(fs, "email")
}
if flags&_diffTypeTel > 0 {
fs = append(fs, "tel")
}
if flags&_diffTypeCountryID > 0 {
fs = append(fs, "country_id")
}
if flags&_diffTypeMobileVerified > 0 {
fs = append(fs, "mobile_verified")
}
if flags&_diffTypeIsLeak > 0 {
fs = append(fs, "is_leak")
}
if len(fs) == 0 {
return "non"
}
return strings.Join(fs, ",")
}
func diff(cloud, localEncrypted *model.AsoAccount) uint8 {
if localEncrypted == cloud {
return _diffTypeNon
}
if localEncrypted == nil || cloud == nil {
return _diffTypePwd | _diffTypeEmail | _diffTypeTel
}
res := _diffTypeNon
if cloud.Salt != localEncrypted.Salt || cloud.Pwd != localEncrypted.Pwd {
res = res | _diffTypePwd
}
if cloud.Email != localEncrypted.Email {
res = res | _diffTypeEmail
}
if cloud.Tel != localEncrypted.Tel {
res = res | _diffTypeTel
}
if cloud.CountryID != localEncrypted.CountryID {
res = res | _diffTypeCountryID
}
if cloud.MobileVerified != localEncrypted.MobileVerified {
res = res | _diffTypeMobileVerified
}
if cloud.Isleak != localEncrypted.Isleak {
res = res | _diffTypeIsLeak
}
return res
}

28
app/job/main/passport-game-data/service/parse_diff_log_test.go Normal file
View File

@@ -0,0 +1,28 @@
package service
import (
"testing"
"encoding/json"
. "github.com/smartystreets/goconvey/convey"
"go-common/app/job/main/passport-game-data/model"
)
func TestParseDiffLog(t *testing.T) {
Convey("parse log text", t, func() {
str := `local({\"mid\":80793085,\"userid\":\"adeqdiffer\",\"uname\":\"adeqdiffer\",\"pwd\":\"33a5fd6290550b88cc229275e9f790f7\",\"salt\":\"SFhrkmK3\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"modify_time\":\"2018-01-21T21:36:50+08:00\"}) local_encrypted({\"mid\":80793085,\"userid\":\"adeqdiffer\",\"uname\":\"adeqdiffer\",\"pwd\":\"7f0aa1b3dadda0c483aa78c3f3b048cf\",\"salt\":\"SFhrkmK3\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"0001-01-01T00:00:00Z\",\"mtime\":\"2018-01-21T21:36:50+08:00\"}) cloud({\"mid\":80793085,\"userid\":\"adeqdiffer\",\"uname\":\"adeqdiffer\",\"pwd\":\"7f0aa1b3dadda0c483aa78c3f3b048cf\",\"salt\":\"SFhrkmK3\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"2017-11-16T19:47:16+08:00\",\"mtime\":\"2018-01-21T21:36:50+08:00\"}`
str = `local({\"mid\":83768597,\"userid\":\"difficenemy\",\"uname\":\"difficenemy\",\"pwd\":\"7e9f9a98269eb6fcc717f2d6e3a25fc2\",\"salt\":\"8pscksH6\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"modify_time\":\"2018-01-10T18:18:05+08:00\"}) local_encrypted({\"mid\":83768597,\"userid\":\"difficenemy\",\"uname\":\"difficenemy\",\"pwd\":\"08fa599f4497e876f7b4c7861f748361\",\"salt\":\"8pscksH6\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"0001-01-01T00:00:00Z\",\"mtime\":\"2018-01-10T18:18:05+08:00\"}) cloud({\"mid\":83768597,\"userid\":\"difficenemy\",\"uname\":\"difficenemy\",\"pwd\":\"08fa599f4497e876f7b4c7861f748361\",\"salt\":\"8pscksH6\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"2017-11-16T23:59:21+08:00\",\"mtime\":\"2018-01-10T18:18:05+08:00\"})`
res := replace(str)
t.Logf("res: %s", res)
cRes := new(model.CompareRes)
err := json.Unmarshal([]byte(res), &cRes)
So(err, ShouldBeNil)
rStr, _ := json.Marshal(cRes)
t.Logf("res: %s, cRes: %s", res, rStr)
})
}

175
app/job/main/passport-game-data/service/service.go Normal file
View File

@@ -0,0 +1,175 @@
package service
import (
"context"
"io/ioutil"
"time"
"go-common/app/job/main/passport-game-data/conf"
"go-common/app/job/main/passport-game-data/dao"
"go-common/library/log"
)
const (
_defaultDelayDuration = time.Minute * 0
_defaultStepDuration = time.Minute * 15
_defaultLoopDuration = time.Second * 3
_defaultBatchSize = 1000
_defaultBatchMissRetryCount = 3
_timeFormat = "2006-01-02 15:04:05"
)
var (
_loc = time.Now().Location()
)
// Service service.
type Service struct {
c *conf.Config
d *dao.Dao
// init cloud
ic *initCloudConfig
// c2l
c2lC *compareConfig
// l2c
l2cC *compareConfig
}
type compareConfig struct {
On bool
OffsetFilePath string
UseOldOffset bool
End bool
StartTime time.Time
EndTime time.Time
DelayDuration time.Duration
StepDuration time.Duration
LoopDuration time.Duration
BatchSize int
BatchMissRetryCount int
Debug bool
Fix bool
// runtime
st, ed time.Time
rangeCount int
totalCount int
diffCount int
sleeping bool
sleepingSeconds int64
sleepFromTs int64
}
func newCompareConfigFrom(c *conf.CompareConfig) (cc *compareConfig) {
st, err := time.ParseInLocation(_timeFormat, c.StartTime, _loc)
if err != nil {
log.Error("failed to parse end time, time.ParseInLocation(%s, %s, %v), error(%v)", _timeFormat, c.StartTime, _loc, err)
return
}
ed, err := time.ParseInLocation(_timeFormat, c.EndTime, _loc)
if err != nil {
log.Error("failed to parse end time, time.ParseInLocation(%s, %s, %v), error(%v)", _timeFormat, c.EndTime, _loc, err)
return
}
cc = &compareConfig{
On: c.On,
Debug: c.Debug,
OffsetFilePath: c.OffsetFilePath,
UseOldOffset: c.UseOldOffset,
End: c.End,
StartTime: st,
EndTime: ed,
DelayDuration: time.Duration(c.DelayDuration),
StepDuration: time.Duration(c.StepDuration),
LoopDuration: time.Duration(c.LoopDuration),
BatchSize: c.BatchSize,
BatchMissRetryCount: c.BatchMissRetryCount,
Fix: c.Fix,
}
if cc.UseOldOffset {
if oldOffset, err := parseOldOffset(cc.OffsetFilePath); err == nil {
cc.StartTime = oldOffset
}
}
cc.fix()
return
}
func parseOldOffset(path string) (oldOffset time.Time, err error) {
data, err := ioutil.ReadFile(path)
if err != nil {
log.Error("failed to read old offset, ioutil.ReadFile(%s) error(%v) skip", path, err)
return
}
if oldOffset, err = time.ParseInLocation(_timeFormat, string(data), _loc); err != nil {
log.Error("failed to parse offset, time.ParseInLocation(%s, %s, %v) error(%v)", _timeFormat, string(data), _loc, err)
}
return
}
func (cc *compareConfig) fix() {
if int64(cc.DelayDuration) < 0 {
cc.DelayDuration = _defaultDelayDuration
}
if int64(cc.StepDuration) < 0 {
cc.StepDuration = _defaultStepDuration
}
if int64(cc.LoopDuration) < 0 {
cc.LoopDuration = _defaultLoopDuration
}
if cc.BatchSize <= 0 {
cc.BatchSize = _defaultBatchSize
}
if cc.BatchMissRetryCount < 0 {
cc.BatchMissRetryCount = _defaultBatchMissRetryCount
}
}
// New new a service instance.
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
d: dao.New(c),
}
if c.Compare.Cloud2Local.On {
s.c2lC = newCompareConfigFrom(c.Compare.Cloud2Local)
go s.cloud2localcompareproc()
}
if c.Compare.Local2Cloud.On {
s.l2cC = newCompareConfigFrom(c.Compare.Local2Cloud)
go s.local2cloudcompareproc()
}
return
}
// Ping check server ok.
func (s *Service) Ping(c context.Context) (err error) {
err = s.d.Ping(c)
return
}
// Close close service.
func (s *Service) Close() (err error) {
s.d.Close()
return
}

22
app/job/main/passport-game-data/service/service_test.go Normal file
View File

@@ -0,0 +1,22 @@
package service
import (
"flag"
"go-common/app/job/main/passport-game-data/conf"
"sync"
)
var (
once sync.Once
s *Service
)
func startService() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
// service init
s = New(conf.Conf)
}

78
app/job/main/passport-game-data/service/stat.go Normal file
View File

@@ -0,0 +1,78 @@
package service
import (
"context"
"time"
"go-common/app/job/main/passport-game-data/model"
)
// CompareProcStat get actual compare proc stat.
func (s *Service) CompareProcStat(c context.Context) *model.ProcStat {
var c2lStat *model.CompareProcStat
if s.c.Compare.Cloud2Local.On {
cc := s.c2lC
c2lStat = &model.CompareProcStat{
StartTime: s.c.Compare.Cloud2Local.StartTime,
EndTime: s.c.Compare.Cloud2Local.EndTime,
StepDuration: model.JsonDuration(cc.StepDuration),
LoopDuration: model.JsonDuration(cc.LoopDuration),
DelayDuration: model.JsonDuration(cc.DelayDuration),
BatchSize: cc.BatchSize,
BatchMissRetryCount: cc.BatchMissRetryCount,
Debug: cc.Debug,
Fix: cc.Fix,
CurrentRangeStart: model.JSONTime(cc.st),
CurrentRangeEnd: model.JSONTime(cc.ed),
CurrentRangeRecordsCount: cc.rangeCount,
TotalRangeRecordsCount: cc.totalCount,
DiffCount: cc.diffCount,
Sleeping: cc.sleeping,
}
if cc.sleeping {
c2lStat.SleepSeconds = cc.sleepingSeconds
c2lStat.SleepFrom = time.Unix(cc.sleepFromTs, 0).Format(_timeFormat)
c2lStat.SleepRemainSeconds = cc.sleepingSeconds - (time.Now().Unix() - cc.sleepFromTs)
}
}
var l2cStat *model.CompareProcStat
if s.c.Compare.Local2Cloud.On {
cc := s.l2cC
l2cStat = &model.CompareProcStat{
StartTime: s.c.Compare.Local2Cloud.StartTime,
EndTime: s.c.Compare.Local2Cloud.EndTime,
StepDuration: model.JsonDuration(cc.StepDuration),
LoopDuration: model.JsonDuration(cc.LoopDuration),
DelayDuration: model.JsonDuration(cc.DelayDuration),
BatchSize: cc.BatchSize,
BatchMissRetryCount: cc.BatchMissRetryCount,
Debug: cc.Debug,
Fix: cc.Fix,
CurrentRangeStart: model.JSONTime(cc.st),
CurrentRangeEnd: model.JSONTime(cc.ed),
CurrentRangeRecordsCount: cc.rangeCount,
TotalRangeRecordsCount: cc.totalCount,
DiffCount: cc.diffCount,
Sleeping: cc.sleeping,
}
if cc.sleeping {
l2cStat.SleepSeconds = cc.sleepingSeconds
l2cStat.SleepFrom = time.Unix(cc.sleepFromTs, 0).Format(_timeFormat)
l2cStat.SleepRemainSeconds = cc.sleepingSeconds - (time.Now().Unix() - cc.sleepFromTs)
}
}
return &model.ProcStat{
Cloud2Local: c2lStat,
Local2Cloud: l2cStat,
}
}