Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,68 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"account_test.go",
"dao_test.go",
"hbase_test.go",
"mysql_test.go",
"redis_test.go",
"vip_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/figure/conf:go_default_library",
"//app/job/main/figure/model:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"account.go",
"blocked.go",
"danmaku.go",
"dao.go",
"hbase.go",
"mysql.go",
"redis.go",
"vip.go",
],
importpath = "go-common/app/job/main/figure/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/figure/conf:go_default_library",
"//app/job/main/figure/model:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/hbase.v2:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,42 @@
package dao
import (
"context"
"encoding/binary"
"time"
"go-common/app/job/main/figure/conf"
"go-common/app/job/main/figure/model"
"github.com/pkg/errors"
)
// UpdateAccountExp update user exp
func (d *Dao) UpdateAccountExp(c context.Context, mid, exp int64) (err error) {
var (
expByte = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
binary.BigEndian.PutUint64(expByte, uint64(exp))
values := map[string]map[string][]byte{model.USFamilyUser: map[string][]byte{model.USColumnExp: expByte}}
if _, err = d.hbase.PutStr(ctx, model.UserInfoTable, d.rowKey(mid), values); err != nil {
err = errors.Wrapf(err, "mid(%d), hbase.Put(key: %s, values: %v)", d.rowKey(mid), values)
}
return
}
// IncArchiveViews .
func (d *Dao) IncArchiveViews(c context.Context, mid int64) (err error) {
var (
incrByte = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
binary.BigEndian.PutUint64(incrByte, uint64(1))
values := map[string]map[string][]byte{model.USFamilyUser: map[string][]byte{model.USColumnArchiveViews: incrByte}}
if _, err = d.hbase.Increment(ctx, model.UserInfoTable, d.rowKey(mid), values); err != nil {
err = errors.Wrapf(err, "msg(%v), hbase.Increment(key: %s values: %v)", mid, d.rowKey(mid), values)
}
return
}

View File

@@ -0,0 +1,20 @@
package dao
import (
"context"
"testing"
)
func TestPubStat(t *testing.T) {
once.Do(startService)
if err := d.UpdateAccountExp(context.TODO(), 7593623, 120.00); err != nil {
t.Errorf("d.UpdateAccountExp(%d) error(%v)", 7593623, err)
}
}
func TestIncArchiveViews(t *testing.T) {
once.Do(startService)
if err := d.IncArchiveViews(context.TODO(), 7593623); err != nil {
t.Errorf("error(%v)", err)
}
}

View File

@@ -0,0 +1,27 @@
package dao
import (
"context"
"encoding/binary"
"time"
"go-common/app/job/main/figure/conf"
"go-common/app/job/main/figure/model"
"github.com/pkg/errors"
)
// BlockedRage .
func (d *Dao) BlockedRage(c context.Context, mid int64, vs int16) (err error) {
var (
rageByte = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
binary.BigEndian.PutUint16(rageByte, uint16(vs))
values := map[string]map[string][]byte{model.USFamilyUser: map[string][]byte{model.USColumnBlockedRage: rageByte}}
if _, err = d.hbase.PutStr(ctx, model.UserInfoTable, d.rowKey(mid), values); err != nil {
err = errors.Wrapf(err, "mid(%v), hbase.Put(key: %s values: %v)", mid, mid, values)
}
return
}

View File

@@ -0,0 +1,31 @@
package dao
import (
"bytes"
"context"
"encoding/binary"
"time"
"go-common/app/job/main/figure/conf"
"go-common/app/job/main/figure/model"
"go-common/library/log"
"github.com/pkg/errors"
)
// DanmakuReport .
func (d *Dao) DanmakuReport(c context.Context, mid int64, column string, incr int64) (err error) {
var (
key = d.rowVerKey(mid, time.Now())
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
log.Info("Put danmaku act key [%s] c [%s] incr [%d]", key, column, incr)
bytesBuffer := bytes.NewBuffer([]byte{})
binary.Write(bytesBuffer, binary.BigEndian, incr)
values := map[string]map[string][]byte{model.ACFamilyUser: map[string][]byte{column: bytesBuffer.Bytes()}}
if _, err = d.hbase.Increment(ctx, model.ActionCounterTable, key, values); err != nil {
err = errors.Wrapf(err, "msg(%d,%s,%d), hbase.Increment(key: %s values: %v)", mid, column, incr, key, values)
}
return
}

View File

@@ -0,0 +1,63 @@
package dao
import (
"context"
"time"
"go-common/app/job/main/figure/conf"
"go-common/library/cache/redis"
"go-common/library/database/hbase.v2"
"go-common/library/database/sql"
)
// Dao figure job dao
type Dao struct {
c *conf.Config
hbase *hbase.Client
db *sql.DB
redis *redis.Pool
redisExpire int
waiteMidExpire int
}
// New new a figure DAO
func New(c *conf.Config) (d *Dao) {
d = &Dao{
c: c,
hbase: hbase.NewClient(c.HBase.Config),
db: sql.NewMySQL(c.Mysql),
redis: redis.NewPool(c.Redis.Config),
redisExpire: int(time.Duration(c.Redis.Expire) / time.Second),
waiteMidExpire: int(time.Duration(c.Redis.WaiteMidExpire) / time.Second),
}
return
}
// Ping check service health
func (d *Dao) Ping(c context.Context) (err error) {
return d.PingRedis(c)
}
// Close close all dao.
func (d *Dao) Close() {
if d.hbase != nil {
d.hbase.Close()
}
}
//Version get ever monday start time ts.
func (d *Dao) Version(now time.Time) (ts int64) {
var (
n int8
)
y, m, day := now.Date()
w := now.Weekday()
switch w {
case time.Sunday:
n = 6
default:
n = int8(w) - 1
}
t := time.Date(y, m, day, 0, 0, 0, 0, time.Local).Add(-time.Duration(n) * 24 * time.Hour)
return t.Unix()
}

View File

@@ -0,0 +1,54 @@
package dao
import (
"flag"
"path/filepath"
"sync"
"time"
"go-common/app/job/main/figure/conf"
"go-common/library/log"
. "github.com/smartystreets/goconvey/convey"
)
var (
once sync.Once
d *Dao
)
func initConf() {
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
defer log.Close()
}
func startService() {
initConf()
if d == nil {
d = New(conf.Conf)
}
time.Sleep(time.Second * 2)
}
func CleanDB() {
}
func init() {
dir, _ := filepath.Abs("../cmd/figure-job-test.toml")
flag.Set("conf", dir)
conf.Init()
if d == nil {
d = New(conf.Conf)
}
}
func WithDao(f func(d *Dao)) func() {
return func() {
Reset(func() { CleanDB() })
f(d)
}
}

View File

@@ -0,0 +1,110 @@
package dao
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"time"
"go-common/app/job/main/figure/conf"
"go-common/app/job/main/figure/model"
"go-common/library/log"
"github.com/pkg/errors"
)
func (d *Dao) rowKey(mid int64) (res string) {
res = fmt.Sprintf("%d", mid)
return
}
func (d *Dao) rowVerKey(mid int64, now time.Time) (res string) {
res = fmt.Sprintf("%d_%d", mid, d.Version(now))
return
}
// PutSpyScore add spy score info.
func (d *Dao) PutSpyScore(c context.Context, mid int64, score int8) (err error) {
var (
key = d.rowKey(mid)
scoreB = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
log.Info("Put spy score key [%s] score [%d]", key, score)
binary.BigEndian.PutUint64(scoreB, uint64(score))
values := map[string]map[string][]byte{model.USFamilyUser: map[string][]byte{model.USColumnSpyScore: scoreB}}
if _, err = d.hbase.PutStr(ctx, model.UserInfoTable, key, values); err != nil {
log.Error("hbase.Put error(%v)", err)
}
return
}
// PutReplyAct add spy score info.
func (d *Dao) PutReplyAct(c context.Context, mid int64, column string, incr int64) (err error) {
var (
key = d.rowVerKey(mid, time.Now())
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
log.Info("Put reply act key [%s] c [%s] incr [%d]", key, column, incr)
bytesBuffer := bytes.NewBuffer([]byte{})
binary.Write(bytesBuffer, binary.BigEndian, incr)
values := map[string]map[string][]byte{model.ACFamilyUser: map[string][]byte{column: bytesBuffer.Bytes()}}
if _, err = d.hbase.Increment(ctx, model.ActionCounterTable, key, values); err != nil {
err = errors.Wrapf(err, "msg(%d,%s,%d), hbase.Increment(key: %s values: %v)", mid, column, incr, key, values)
}
return
}
// PutCoinUnusual coin unusual.
func (d *Dao) PutCoinUnusual(c context.Context, mid int64, column string) (err error) {
var (
key = d.rowVerKey(mid, time.Now())
incrBytes = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
log.Info("Put coin unusual key [%s] c [%s]", key, column)
binary.BigEndian.PutUint64(incrBytes, uint64(1))
values := map[string]map[string][]byte{model.ACFamilyUser: map[string][]byte{column: incrBytes}}
if _, err = d.hbase.Increment(ctx, model.ActionCounterTable, key, values); err != nil {
err = errors.Wrapf(err, "msg(%d,%s), hbase.Increment(key: %s values: %v)", mid, column, key, values)
}
return
}
// PutCoinCount coin count.
func (d *Dao) PutCoinCount(c context.Context, mid int64) (err error) {
var (
key = d.rowVerKey(mid, time.Now())
incrBytes = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
log.Info("Put coin count key [%s]", key)
binary.BigEndian.PutUint64(incrBytes, uint64(1))
values := map[string]map[string][]byte{model.ACFamilyUser: map[string][]byte{model.ACColumnCoins: incrBytes}}
if _, err = d.hbase.Increment(ctx, model.ActionCounterTable, key, values); err != nil {
err = errors.Wrapf(err, "msg(%d), hbase.Increment(key: %s values: %v)", mid, key, values)
}
return
}
// PayOrderInfo user pay order info.
func (d *Dao) PayOrderInfo(c context.Context, column string, mid, money int64) (err error) {
var (
key = d.rowVerKey(mid, time.Now())
incrBytes = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
log.Info("Pay Order key [%s]", key)
binary.BigEndian.PutUint64(incrBytes, uint64(money))
values := map[string]map[string][]byte{model.ACFamilyUser: map[string][]byte{column: incrBytes}}
if _, err = d.hbase.Increment(ctx, model.ActionCounterTable, key, values); err != nil {
err = errors.Wrapf(err, "msg(%d,%s,%d), hbase.Increment(key: %s values: %v)", mid, column, key, values)
}
return
}

View File

@@ -0,0 +1,54 @@
package dao
import (
"context"
"go-common/app/job/main/figure/model"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
var (
mid int64 = 15555180
score int8 = 60
)
//go test -test.v -test.run TestPutSpyScore
func TestPutSpyScore(t *testing.T) {
Convey("TestPutSpyScore no err", t, WithDao(func(d *Dao) {
err := d.PutSpyScore(context.TODO(), mid, score)
So(err, ShouldBeNil)
}))
}
//go test -test.v -test.run TestPutReplyAct
func TestPutReplyAct(t *testing.T) {
Convey("TestPutReplyAct no err", t, WithDao(func(d *Dao) {
err := d.PutReplyAct(context.TODO(), mid, model.ACColumnReplyLiked, int64(-1))
So(err, ShouldBeNil)
}))
}
//go test -test.v -test.run TestPutCoinUnusual
func TestPutCoinUnusual(t *testing.T) {
Convey("TestPutCoinUnusual no err", t, WithDao(func(d *Dao) {
err := d.PutCoinUnusual(context.TODO(), mid, model.ACColumnLowRisk)
So(err, ShouldBeNil)
}))
}
//go test -test.v -test.run TestPutCoinCount
func TestPutCoinCount(t *testing.T) {
Convey("TestPutCoinCount no err", t, WithDao(func(d *Dao) {
err := d.PutCoinCount(context.TODO(), mid)
So(err, ShouldBeNil)
}))
}
//go test -test.v -test.run TestPayOrderInfo
func TestPayOrderInfo(t *testing.T) {
Convey("PayOrderInfo no err", t, WithDao(func(d *Dao) {
err := d.PayOrderInfo(context.TODO(), "", mid, 1253)
So(err, ShouldBeNil)
}))
}

View File

@@ -0,0 +1,49 @@
package dao
import (
"context"
"fmt"
"go-common/app/job/main/figure/model"
"go-common/library/database/sql"
"go-common/library/log"
"github.com/pkg/errors"
)
const (
_shard = 100
_insertFigure = "INSERT INTO figure_user_%02d (mid,score,lawful_score,wide_score,friendly_score,bounty_score,creativity_score,ver,ctime,mtime) VALUES (?,?,?,?,?,?,?,?,?,?)"
_existFigure = "SELECT id FROM figure_user_%02d WHERE mid=? LIMIT 1"
)
func hit(mid int64) int64 {
return mid % _shard
}
// ExistFigure exist user figure info
func (d *Dao) ExistFigure(c context.Context, mid int64) (id int64, err error) {
res := d.db.QueryRow(c, fmt.Sprintf(_existFigure, hit(mid)), mid)
if err = res.Scan(&id); err != nil {
if err == sql.ErrNoRows {
err = nil
return
}
errors.Wrapf(err, "row.Scan(%d) error", mid)
}
return
}
// SaveFigure init user figure info
func (d *Dao) SaveFigure(c context.Context, f *model.Figure) (id int64, err error) {
res, err := d.db.Exec(c, fmt.Sprintf(_insertFigure, hit(f.Mid)), f.Mid, f.Score, f.LawfulScore, f.WideScore, f.FriendlyScore, f.BountyScore, f.CreativityScore, f.Ver, f.Ctime, f.Mtime)
if err != nil {
errors.Wrapf(err, "init user(%d) Figure info error(%v)", f.Mid, err)
return
}
if id, err = res.LastInsertId(); err != nil {
log.Error("res.LastInsertId() error(%v)", err)
}
return
}

View File

@@ -0,0 +1,37 @@
package dao
import (
"context"
"testing"
"time"
"go-common/app/job/main/figure/model"
)
func TestSaveFigure(t *testing.T) {
once.Do(startService)
f := &model.Figure{
Mid: 761223,
Score: 100,
LawfulScore: d.c.Figure.Lawful,
WideScore: d.c.Figure.Wide,
FriendlyScore: d.c.Figure.Friendly,
BountyScore: d.c.Figure.Bounty,
CreativityScore: d.c.Figure.Creativity,
Ver: 1,
Ctime: time.Now(),
Mtime: time.Now(),
}
if id, err := d.SaveFigure(context.TODO(), f); err != nil {
t.Errorf("figure err (%v)", err)
} else {
t.Logf("id(%d)", id)
}
}
func TestGetFigure(t *testing.T) {
once.Do(startService)
var mid int64 = 7593623
id, _ := d.ExistFigure(context.TODO(), mid)
t.Logf("id:%d", id)
}

View File

@@ -0,0 +1,86 @@
package dao
import (
"context"
"encoding/json"
"fmt"
"go-common/app/job/main/figure/model"
"go-common/library/log"
"github.com/pkg/errors"
)
const (
_keyWaitDealUser = "w:u" // b_batch_no wait block
_figureKey = "f:%d"
)
func figureKey(mid int64) string {
return fmt.Sprintf(_figureKey, mid)
}
// keyWaitBlock return block cache key.
func keyWaitBlock(version int64, mid int64) string {
return fmt.Sprintf("%s%d%d", _keyWaitDealUser, version, mid%10000)
}
// SetWaiteUserCache set waite deal user cache.
func (d *Dao) SetWaiteUserCache(c context.Context, mid int64, ver int64) (err error) {
if mid <= 0 {
log.Error("%+v", errors.Errorf("SetWaiteUserCache mid [%d] ver [%d] error", mid, ver))
return
}
var (
key = keyWaitBlock(ver, mid)
conn = d.redis.Get(c)
)
defer conn.Close()
if err = conn.Send("SADD", key, mid); err != nil {
log.Error("conn.Send error(%v)", err)
return
}
if err = conn.Send("EXPIRE", key, d.waiteMidExpire); err != nil {
log.Error("conn.Send error(%v)", err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
return
}
for i := 0; i < 2; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
return
}
}
return
}
// AddFigureInfoCache put figure to redis
func (d *Dao) AddFigureInfoCache(c context.Context, f *model.Figure) (err error) {
key := figureKey(f.Mid)
conn := d.redis.Get(c)
defer conn.Close()
values, err := json.Marshal(f)
if err != nil {
return
}
if err = conn.Send("SET", key, values); err != nil {
log.Error("conn.Send(SET, %s, %d) error(%v)", key, values, err)
return
}
if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
log.Error("conn.Send(Expire, %s, %d) error(%v)", key, d.redisExpire, err)
return
}
return
}
// PingRedis check redis connection
func (d *Dao) PingRedis(c context.Context) (err error) {
conn := d.redis.Get(c)
_, err = conn.Do("SET", "PING", "PONG")
conn.Close()
return
}

View File

@@ -0,0 +1,32 @@
package dao
import (
"context"
"go-common/app/job/main/figure/model"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
var (
testRedisMid int64 = 15555180
)
func Test_PingRedis(t *testing.T) {
Convey("ping redis", t, WithDao(func(d *Dao) {
So(d.PingRedis(context.TODO()), ShouldBeNil)
}))
}
func Test_SetWaiteUserCache(t *testing.T) {
Convey("set waite user cache", t, WithDao(func(d *Dao) {
So(d.SetWaiteUserCache(context.TODO(), testRedisMid, 111), ShouldBeNil)
}))
}
func Test_AddFigureInfoCache(t *testing.T) {
Convey("add figure info cache", t, WithDao(func(d *Dao) {
f := &model.Figure{Mid: testRedisMid}
So(d.AddFigureInfoCache(context.TODO(), f), ShouldBeNil)
}))
}

View File

@@ -0,0 +1,27 @@
package dao
import (
"context"
"encoding/binary"
"time"
"go-common/app/job/main/figure/conf"
"go-common/app/job/main/figure/model"
"github.com/pkg/errors"
)
// UpdateVipStatus .
func (d *Dao) UpdateVipStatus(c context.Context, mid int64, vs int32) (err error) {
var (
vipByte = make([]byte, 8)
ctx, cancel = context.WithTimeout(c, time.Duration(conf.Conf.HBase.WriteTimeout))
)
defer cancel()
binary.BigEndian.PutUint64(vipByte, uint64(vs))
values := map[string]map[string][]byte{model.USFamilyUser: map[string][]byte{model.USColumnVipStatus: vipByte}}
if _, err = d.hbase.PutStr(ctx, model.UserInfoTable, d.rowKey(mid), values); err != nil {
err = errors.Wrapf(err, "mid(%v), hbase.Put(key: %s values: %v)", mid, d.rowKey(mid), values)
}
return
}

View File

@@ -0,0 +1,11 @@
package dao
import (
"context"
"testing"
)
func TestUpdateVipStatus(t *testing.T) {
once.Do(startService)
d.UpdateVipStatus(context.TODO(), 7593623, 1)
}