Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,57 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"db.go",
"redis.go",
],
importpath = "go-common/app/service/main/reply-feed/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/reply-feed/conf:go_default_library",
"//app/service/main/reply-feed/model:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/xstr:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"db_test.go",
"redis_test.go",
],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/service/main/reply-feed/conf:go_default_library",
"//app/service/main/reply-feed/model:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

View File

@@ -0,0 +1,44 @@
package dao
import (
"context"
"time"
"go-common/app/service/main/reply-feed/conf"
"go-common/library/cache/redis"
xsql "go-common/library/database/sql"
)
// Dao dao
type Dao struct {
c *conf.Config
redis *redis.Pool
redisReplyZSetExpire int
db *xsql.DB
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
redis: redis.NewPool(c.Redis),
redisReplyZSetExpire: int(time.Duration(c.RedisExpire.RedisReplyZSetExpire) / time.Second),
db: xsql.NewMySQL(c.MySQL),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.redis.Close()
d.db.Close()
}
// Ping dao ping
func (d *Dao) Ping(c context.Context) error {
if err := d.PingRedis(c); err != nil {
return err
}
return d.db.Ping(c)
}

View File

@@ -0,0 +1,35 @@
package dao
import (
"flag"
"os"
"testing"
"go-common/app/service/main/reply-feed/conf"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.community.reply-feed-service")
flag.Set("conf_token", "b54e90375c747f48f477939fe272eefe")
flag.Set("tree_id", "71072")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../cmd/test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
os.Exit(m.Run())
}

View File

@@ -0,0 +1,242 @@
package dao
import (
"context"
"encoding/json"
"errors"
"fmt"
"go-common/app/service/main/reply-feed/model"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_getSlotsStat = "SELECT name,slot,state FROM reply_abtest_strategy"
_getSlotsStatManager = "SELECT name,slot,algorithm,weight,state FROM reply_abtest_strategy"
_setSlot = "UPDATE reply_abtest_strategy SET name=?,algorithm=?,weight=?,state=? WHERE slot IN (%s)"
_setWeight = "UPDATE reply_abtest_strategy SET weight=? WHERE name=?"
_modifyState = "UPDATE reply_abtest_strategy SET state=? WHERE name=?"
_getSlotsStatByName = "SELECT slot,algorithm,weight FROM reply_abtest_strategy WHERE name=?"
_getStatisticsDate = "SELECT name,date,hour,view,total_view,hot_view,hot_click,hot_like,hot_hate,hot_child,hot_report,total_like,total_hate,total_report,total_root,total_child,hot_like_uv,hot_hate_uv,hot_report_uv,hot_child_uv,total_like_uv,total_hate_uv,total_report_uv,total_child_uv,total_root_uv" +
" FROM reply_abtest_statistics WHERE date>=? AND date<=? AND name!='default'"
_upsertLog = "INSERT INTO reply_abtest_statistics (name,date,hour,view,hot_click,hot_view,total_view) VALUES(?,?,?,?,?,?,?)" +
" ON DUPLICATE KEY UPDATE view=view+?,hot_click=hot_click+?,hot_view=hot_view+?,total_view=total_view+?"
)
var (
_countIdleSlot = fmt.Sprintf("SELECT COUNT(*) FROM reply_abtest_strategy WHERE state=1 AND name='%s'", model.DefaultSlotName)
_getIdelSlots = fmt.Sprintf("SELECT slot FROM reply_abtest_strategy WHERE state=1 AND name='%s' ORDER BY slot ASC LIMIT ?", model.DefaultSlotName)
)
/*
SlotsStat
*/
// SlotsMapping get slot name stat from database.
func (d *Dao) SlotsMapping(ctx context.Context) (slotsMap map[string]*model.SlotsMapping, err error) {
slotsMap = make(map[string]*model.SlotsMapping)
rows, err := d.db.Query(ctx, _getSlotsStat)
if err != nil {
log.Error("db.Query(%s) args(%s) error(%v)", _getSlotsStat, err)
return
}
defer rows.Close()
for rows.Next() {
var (
name string
slot int
state int
)
if err = rows.Scan(&name, &slot, &state); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
mapping, ok := slotsMap[name]
if ok {
mapping.Slots = append(mapping.Slots, slot)
} else {
mapping = &model.SlotsMapping{
Name: name,
Slots: []int{slot},
State: state,
}
}
slotsMap[name] = mapping
}
if err = rows.Err(); err != nil {
log.Error("rows.Err() error(%v)", err)
}
return
}
// SlotsStatManager get slots stat from database, used by manager.
func (d *Dao) SlotsStatManager(ctx context.Context) (s []*model.SlotsStat, err error) {
slotsMap := make(map[string]*model.SlotsStat)
rows, err := d.db.Query(ctx, _getSlotsStatManager)
if err != nil {
log.Error("db.Query(%s) error(%v)", _getSlotsStatManager, err)
return
}
defer rows.Close()
for rows.Next() {
var (
name, algorithm, weight string
slot, state int
)
if err = rows.Scan(&name, &slot, &algorithm, &weight, &state); err != nil {
log.Error("rows.Scan error(%v)", err)
return
}
if mapping, ok := slotsMap[name]; ok {
mapping.Slots = append(mapping.Slots, slot)
} else {
slotsMap[name] = &model.SlotsStat{
Name: name,
Slots: []int{slot},
Algorithm: algorithm,
Weight: weight,
State: state,
}
}
}
if err = rows.Err(); err != nil {
log.Error("rows.Err() error(%v)", err)
return
}
for _, stat := range slotsMap {
s = append(s, stat)
}
return
}
// CountIdleSlot count idle slot which name="default" and state=1
func (d *Dao) CountIdleSlot(ctx context.Context) (count int, err error) {
if err = d.db.QueryRow(ctx, _countIdleSlot).Scan(&count); err != nil {
log.Error("db.QueryRow() error(%v)", err)
}
return
}
// IdleSlots get idle slots
func (d *Dao) IdleSlots(ctx context.Context, count int) (slots []int64, err error) {
rows, err := d.db.Query(ctx, _getIdelSlots, count)
if err != nil {
log.Error("db.Query(%s) args(%d) error(%v)", _getIdelSlots, count, err)
return
}
defer rows.Close()
for rows.Next() {
var slot int64
if err = rows.Scan(&slot); err != nil {
log.Error("rows.Scan() error(%v)", err)
return
}
slots = append(slots, slot)
}
// 槽位不够新创建实验组
if len(slots) < count {
slots = nil
err = errors.New("out of slot")
return
}
if err = rows.Err(); err != nil {
log.Error("rows.Err() error(%v)", err)
}
return
}
// ModifyState ModifyState
func (d *Dao) ModifyState(ctx context.Context, name string, state int) (err error) {
if _, err = d.db.Exec(ctx, _modifyState, state, name); err != nil {
log.Error("db.Exec(%s) args(%d, %s) error(%v)", _modifyState, state, name, err)
}
return
}
// UpdateSlotsStat UpdateSlotStat and set state inactive.
func (d *Dao) UpdateSlotsStat(ctx context.Context, name, algorithm, weight string, slots []int64, state int) (err error) {
if _, err = d.db.Exec(ctx, fmt.Sprintf(_setSlot, xstr.JoinInts(slots)), name, algorithm, weight, state); err != nil {
log.Error("db.Exec() error(%v)", err)
}
return
}
// SlotsStatByName get slots stat by name.
func (d *Dao) SlotsStatByName(ctx context.Context, name string) (slots []int64, algorithm, weight string, err error) {
rows, err := d.db.Query(ctx, _getSlotsStatByName, name)
if err != nil {
log.Error("db.Query(%s) args(%s) error(%v)", _getSlotsStatByName, name, err)
return
}
defer rows.Close()
for rows.Next() {
var (
slot int64
)
if err = rows.Scan(&slot, &algorithm, &weight); err != nil {
log.Error("rows.Scan() error(%v)", err)
return
}
slots = append(slots, slot)
}
if err = rows.Err(); err != nil {
log.Error("rows.Err() error(%v)", err)
}
return
}
// UpdateWeight update a test set weight by name and algorithm.
func (d *Dao) UpdateWeight(ctx context.Context, name string, weight interface{}) (err error) {
b, err := json.Marshal(weight)
if err != nil {
return
}
if _, err = d.db.Exec(ctx, _setWeight, string(b), name); err != nil {
log.Error("db.Exec(%s), error(%v)", _setWeight, err)
}
return
}
/*
Statistics
*/
// UpsertStatistics insert or update statistics from database, if err != nil, retry
func (d *Dao) UpsertStatistics(ctx context.Context, name string, date int, hour int, s *model.StatisticsStat) (err error) {
if _, err = d.db.Exec(ctx, _upsertLog,
name, date, hour,
s.View, s.HotClick, s.HotView, s.TotalView,
s.View, s.HotClick, s.HotView, s.TotalView); err != nil {
return
}
return
}
// StatisticsByDate StatisticsByDate
func (d *Dao) StatisticsByDate(ctx context.Context, begin, end int64) (stats model.StatisticsStats, err error) {
rows, err := d.db.Query(ctx, _getStatisticsDate, begin, end)
if err != nil {
log.Error("db.Query(%s) args(%d, %d) error(%v)", _getStatisticsDate, begin, end, err)
return
}
defer rows.Close()
for rows.Next() {
var s = new(model.StatisticsStat)
err = rows.Scan(&s.Name, &s.Date, &s.Hour, &s.View, &s.TotalView, &s.HotView, &s.HotClick, &s.HotLike, &s.HotHate, &s.HotChildReply,
&s.HotReport, &s.TotalLike, &s.TotalHate, &s.TotalReport, &s.TotalRootReply, &s.TotalChildReply,
&s.HotLikeUV, &s.HotHateUV, &s.HotReportUV, &s.HotChildUV, &s.TotalLikeUV, &s.TotalHateUV, &s.TotalReportUV, &s.TotalChildUV, &s.TotalRootUV)
if err != nil {
log.Error("rows.Scan() error(%v)", err)
return
}
stats = append(stats, s)
}
if err = rows.Err(); err != nil {
log.Error("rows.Err() error(%v)", err)
return
}
return
}

View File

@@ -0,0 +1,158 @@
package dao
import (
"context"
"testing"
"go-common/app/service/main/reply-feed/model"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoSlotsMapping(t *testing.T) {
convey.Convey("SlotsMapping", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
slotsMap, err := d.SlotsMapping(context.Background())
ctx.Convey("Then err should be nil.slotsMap should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(slotsMap, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoSlotsStatManager(t *testing.T) {
convey.Convey("SlotsStatManager", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
s, err := d.SlotsStatManager(context.Background())
ctx.Convey("Then err should be nil.s should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(s, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoIdleSlot(t *testing.T) {
convey.Convey("IdleSlot", t, func(ctx convey.C) {
var (
count = int(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
slots, err := d.IdleSlots(context.Background(), count)
ctx.Convey("Then err should be nil.slots should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(slots, convey.ShouldBeNil)
})
})
})
}
func TestDaoCountIdleSlot(t *testing.T) {
convey.Convey("CountIdleSlot", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
count, err := d.CountIdleSlot(context.Background())
ctx.Convey("Then err should be nil. count should greater than 0.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldBeGreaterThan, 0)
})
})
})
}
func TestDaoModifyState(t *testing.T) {
convey.Convey("ModifyState", t, func(ctx convey.C) {
var (
name = ""
state = int(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.ModifyState(context.Background(), name, state)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoUpdateSlotsStat(t *testing.T) {
convey.Convey("UpdateSlotsStat", t, func(ctx convey.C) {
var (
name = ""
algorithm = ""
weight = ""
slots = []int64{-1}
state = int(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.UpdateSlotsStat(context.Background(), name, algorithm, weight, slots, state)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestSlotsStatByName(t *testing.T) {
convey.Convey("SlotsStatByName", t, func(ctx convey.C) {
var (
name = ""
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
slots, algorithm, weight, err := d.SlotsStatByName(context.Background(), name)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(slots, convey.ShouldBeNil)
ctx.So(algorithm, convey.ShouldEqual, "")
ctx.So(weight, convey.ShouldEqual, "")
})
})
})
}
func TestDaoUpdateWeight(t *testing.T) {
convey.Convey("UpdateWeight", t, func(ctx convey.C) {
var (
name = ""
weight = interface{}(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.UpdateWeight(context.Background(), name, weight)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoUpsertStatistics(t *testing.T) {
convey.Convey("UpsertStatistics", t, func(ctx convey.C) {
var (
name = ""
date = int(0)
hour = int(0)
s = &model.StatisticsStat{}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.UpsertStatistics(context.Background(), name, date, hour, s)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoStatisticsByDate(t *testing.T) {
convey.Convey("StatisticsByDate", t, func(ctx convey.C) {
var (
begin = int64(0)
end = int64(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
stats, err := d.StatisticsByDate(context.Background(), begin, end)
ctx.Convey("Then err should be nil.stats should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(stats, convey.ShouldNotBeNil)
})
})
})
}

View File

@@ -0,0 +1,68 @@
package dao
import (
"context"
"fmt"
"go-common/library/cache/redis"
"go-common/library/log"
)
const (
// r_<实验组名>_<oid>_<type>
// 用redis ZSet存储热门评论列表score为热评分数member为rpID
_replyZSetFormat = "r_%s_%d_%d"
)
func keyReplyZSet(name string, oid int64, tp int) string {
return fmt.Sprintf(_replyZSetFormat, name, oid, tp)
}
// PingRedis redis health check.
func (d *Dao) PingRedis(ctx context.Context) (err error) {
conn := d.redis.Get(ctx)
defer conn.Close()
_, err = conn.Do("SET", "ping", "pong")
return
}
// ExpireReplyZSetRds expire reply list.
func (d *Dao) ExpireReplyZSetRds(ctx context.Context, name string, oid int64, tp int) (ok bool, err error) {
conn := d.redis.Get(ctx)
defer conn.Close()
key := keyReplyZSet(name, oid, tp)
if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.redisReplyZSetExpire)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis EXPIRE key(%s) error(%v)", key, err)
}
return
}
// ReplyZSetRds get reply list from redis sorted set.
func (d *Dao) ReplyZSetRds(ctx context.Context, name string, oid int64, tp, start, end int) (rpIDs []int64, err error) {
conn := d.redis.Get(ctx)
defer conn.Close()
key := keyReplyZSet(name, oid, tp)
values, err := redis.Values(conn.Do("ZREVRANGE", key, start, end))
if err != nil {
log.Error("redis ZREVRANGE(%s, %d, %d) error(%v)", key, start, end, err)
return
}
if err = redis.ScanSlice(values, &rpIDs); err != nil {
log.Error("redis ScanSlice(%v) error(%v)", values, err)
}
return
}
// CountReplyZSetRds count reply num.
func (d *Dao) CountReplyZSetRds(ctx context.Context, name string, oid int64, tp int) (count int, err error) {
conn := d.redis.Get(ctx)
defer conn.Close()
key := keyReplyZSet(name, oid, tp)
if count, err = redis.Int(conn.Do("ZCARD", key)); err != nil {
log.Error("conn.Do(ZCARD, %s) error(%v)", key, err)
}
return
}

View File

@@ -0,0 +1,88 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaokeyReplyZSet(t *testing.T) {
convey.Convey("keyReplyZSet", t, func(ctx convey.C) {
var (
name = ""
oid = int64(0)
tp = int(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
p1 := keyReplyZSet(name, oid, tp)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoPingRedis(t *testing.T) {
convey.Convey("PingRedis", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.PingRedis(context.Background())
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoExpireReplyZSetRds(t *testing.T) {
convey.Convey("ExpireReplyZSetRds", t, func(ctx convey.C) {
var (
name = ""
oid = int64(0)
tp = int(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
ok, err := d.ExpireReplyZSetRds(context.Background(), name, oid, tp)
ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(ok, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoGetReplyZSetRds(t *testing.T) {
convey.Convey("ReplyZSetRds", t, func(ctx convey.C) {
var (
name = ""
oid = int64(0)
tp = int(0)
start = int(0)
end = int(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
rpIDs, err := d.ReplyZSetRds(context.Background(), name, oid, tp, start, end)
ctx.Convey("Then err should be nil.rpIDs should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(rpIDs, convey.ShouldBeNil)
})
})
})
}
func TestDaoCountReplyZSetRds(t *testing.T) {
convey.Convey("CountReplyZSetRds", t, func(ctx convey.C) {
var (
name = ""
oid = int64(-1)
tp = int(0)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
count, err := d.CountReplyZSetRds(context.Background(), name, oid, tp)
ctx.Convey("Then err should be nil.rpIDs should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldEqual, 0)
})
})
})
}