Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,57 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"hbase_test.go",
"redis_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/interface/main/history/conf:go_default_library",
"//app/interface/main/history/model:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"hbase.go",
"redis.go",
],
importpath = "go-common/app/interface/main/history/dao/toview",
tags = ["automanaged"],
deps = [
"//app/interface/main/history/conf:go_default_library",
"//app/interface/main/history/model:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/hbase.v2:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,41 @@
package toview
import (
"context"
"time"
"go-common/app/interface/main/history/conf"
"go-common/library/cache/redis"
"go-common/library/database/hbase.v2"
)
// Dao dao.
type Dao struct {
conf *conf.Config
info *hbase.Client
redis *redis.Pool
expire int
}
// New new history dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
info: hbase.NewClient(c.Info.Config),
redis: redis.NewPool(c.Toview.Config),
expire: int(time.Duration(c.Toview.Expire) / time.Second),
}
return
}
// Ping check connection success.
func (d *Dao) Ping(c context.Context) (err error) {
return d.PingRedis(c)
}
// Close close the redis and kafka resource.
func (d *Dao) Close() {
if d.redis != nil {
d.redis.Close()
}
}

View File

@@ -0,0 +1,33 @@
package toview
import (
"flag"
"os"
"testing"
"go-common/app/interface/main/history/conf"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.community.history")
flag.Set("conf_token", "01423f5b752144440e01cb8ff4432e96")
flag.Set("tree_id", "2298")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
os.Exit(m.Run())
}

View File

@@ -0,0 +1,191 @@
package toview
import (
"bytes"
"context"
"crypto/md5"
"encoding/binary"
"fmt"
"strconv"
"time"
"go-common/app/interface/main/history/model"
"go-common/library/log"
"github.com/tsuna/gohbase/hrpc"
)
var (
tableInfo = "ugc:history_to_view"
familyAid = "aid"
familyAidB = []byte(familyAid)
)
// hashRowKey create rowkey(md5(mid)[:2]+mid) for histroy by mid .
func hashRowKey(mid int64) string {
var bs = make([]byte, 8)
binary.LittleEndian.PutUint64(bs, uint64(mid))
rk := md5.Sum(bs)
return fmt.Sprintf("%x%d", rk[:2], mid)
}
// Add add one toview.
func (d *Dao) Add(ctx context.Context, mid, aid, now int64) (err error) {
var (
timeB = make([]byte, 8)
key = hashRowKey(mid)
column = strconv.FormatInt(aid, 10)
)
ctx, cancel := context.WithTimeout(ctx, time.Duration(d.conf.Info.WriteTimeout))
defer cancel()
binary.BigEndian.PutUint64(timeB, uint64(now))
values := map[string]map[string][]byte{familyAid: map[string][]byte{column: timeB}}
if _, err = d.info.PutStr(ctx, tableInfo, key, values); err != nil {
log.Error("toview info.Put error(%v)", err)
}
return
}
// Adds add some toview.
func (d *Dao) Adds(ctx context.Context, mid int64, aids []int64, now int64) (err error) {
var (
timeB = make([]byte, 8)
key = hashRowKey(mid)
)
ctx, cancel := context.WithTimeout(ctx, time.Duration(d.conf.Info.WriteTimeout))
defer cancel()
binary.BigEndian.PutUint64(timeB, uint64(now))
aidValues := make(map[string][]byte, len(aids))
for _, aid := range aids {
aidValues[strconv.FormatInt(aid, 10)] = timeB
}
values := map[string]map[string][]byte{familyAid: aidValues}
if _, err = d.info.PutStr(ctx, tableInfo, key, values); err != nil {
log.Error("toview info.Put error(%v)", err)
}
return
}
// AddMap add some toview.
func (d *Dao) AddMap(ctx context.Context, mid int64, views map[int64]*model.ToView) (err error) {
var (
timeB = make([]byte, 8)
key = hashRowKey(mid)
)
ctx, cancel := context.WithTimeout(ctx, time.Duration(d.conf.Info.WriteTimeout))
defer cancel()
aidValues := make(map[string][]byte, len(views))
for _, v := range views {
binary.BigEndian.PutUint64(timeB, uint64(v.Unix))
aidValues[strconv.FormatInt(v.Aid, 10)] = timeB
}
values := map[string]map[string][]byte{familyAid: aidValues}
if _, err = d.info.PutStr(ctx, tableInfo, key, values); err != nil {
log.Error("toview info.Put error(%v)", err)
}
return
}
// ListInfo get all ToViews from hbase.
func (d *Dao) ListInfo(ctx context.Context, mid int64, aids []int64) (res []*model.ToView, err error) {
var (
result *hrpc.Result
key = hashRowKey(mid)
)
ctx, cancel := context.WithTimeout(ctx, time.Duration(d.conf.Info.ReadTimeout))
defer cancel()
var options []func(hrpc.Call) error
if len(aids) != 0 {
colunms := make([]string, 0, len(aids))
for _, aid := range aids {
colunms = append(colunms, fmt.Sprintf("%d", aid))
}
options = append(options, hrpc.Families(map[string][]string{familyAid: colunms}))
}
result, err = d.info.GetStr(ctx, tableInfo, key, options...)
if err != nil && result == nil {
log.Error("d.info.Get error(%v)", err)
return
}
res = make([]*model.ToView, 0)
for _, c := range result.Cells {
if c != nil && len(c.Value) == 8 && bytes.Equal(c.Family, familyAidB) {
aid, err := strconv.ParseInt(string(c.Qualifier), 10, 64)
if err != nil {
log.Error("strconv.ParseInt error(%v)", err)
continue
}
t := &model.ToView{Aid: aid}
t.Unix = int64(binary.BigEndian.Uint64(c.Value))
res = append(res, t)
}
}
return
}
// MapInfo get all ToViews from hbase.
func (d *Dao) MapInfo(ctx context.Context, mid int64, aids []int64) (res map[int64]*model.ToView, err error) {
var (
result *hrpc.Result
key = hashRowKey(mid)
)
ctx, cancel := context.WithTimeout(ctx, time.Duration(d.conf.Info.ReadTimeout))
defer cancel()
var options []func(hrpc.Call) error
if len(aids) != 0 {
colunms := make([]string, 0, len(aids))
for _, aid := range aids {
colunms = append(colunms, fmt.Sprintf("%d", aid))
}
options = append(options, hrpc.Families(map[string][]string{familyAid: colunms}))
}
result, err = d.info.GetStr(ctx, tableInfo, key, options...)
if err != nil {
log.Error("d.info.Get error(%v)", err)
return
}
res = make(map[int64]*model.ToView, len(aids))
if result == nil {
return
}
for _, c := range result.Cells {
if c != nil && len(c.Value) == 8 && bytes.Equal(c.Family, familyAidB) {
aid, err := strconv.ParseInt(string(c.Qualifier), 10, 64)
if err != nil {
log.Error("strconv.ParseInt error(%v)", err)
continue
}
t := &model.ToView{Aid: aid}
t.Unix = int64(binary.BigEndian.Uint64(c.Value))
res[aid] = t
}
}
return
}
// Del delete more toview.
func (d *Dao) Del(ctx context.Context, mid int64, aids []int64) (err error) {
key := hashRowKey(mid)
ctx, cancel := context.WithTimeout(ctx, time.Duration(d.conf.Info.WriteTimeout))
defer cancel()
columns := make(map[string][]byte, len(aids))
for _, aid := range aids {
columns[strconv.FormatInt(aid, 10)] = []byte{}
}
values := map[string]map[string][]byte{familyAid: columns}
if _, err = d.info.Delete(ctx, tableInfo, key, values); err != nil {
log.Error("toview info.Delete error(%v)", err)
}
return
}
// Clear clear ToView.
func (d *Dao) Clear(ctx context.Context, mid int64) (err error) {
key := hashRowKey(mid)
ctx, cancel := context.WithTimeout(ctx, time.Duration(d.conf.Info.WriteTimeout))
defer cancel()
if _, err = d.info.Delete(ctx, tableInfo, key, nil); err != nil {
log.Error("toview info.Delete error(%v)", err)
}
return
}

View File

@@ -0,0 +1,140 @@
package toview
import (
"context"
"testing"
"time"
"go-common/app/interface/main/history/model"
"github.com/smartystreets/goconvey/convey"
)
func TestToviewhashRowKey(t *testing.T) {
convey.Convey("hashRowKey", t, func(ctx convey.C) {
var (
mid = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
p1 := hashRowKey(mid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestToviewAdd(t *testing.T) {
convey.Convey("Add", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
aid = int64(141787)
now = time.Now().Unix()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.Add(c, mid, aid, now)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewAdds(t *testing.T) {
convey.Convey("Adds", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
aids = []int64{147717, 147787}
now = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.Adds(c, mid, aids, now)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewAddMap(t *testing.T) {
convey.Convey("AddMap", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
views map[int64]*model.ToView
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.AddMap(c, mid, views)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewListInfo(t *testing.T) {
convey.Convey("ListInfo", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
aids = []int64{147717, 147787}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := d.ListInfo(c, mid, aids)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}
func TestToviewMapInfo(t *testing.T) {
convey.Convey("MapInfo", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
aids = []int64{147717, 147787}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := d.MapInfo(c, mid, aids)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}
func TestToviewDel(t *testing.T) {
convey.Convey("Del", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
aids = []int64{147717, 147787}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.Del(c, mid, aids)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewClear(t *testing.T) {
convey.Convey("Clear", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.Clear(c, mid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}

View File

@@ -0,0 +1,160 @@
package toview
import (
"context"
"strconv"
"go-common/app/interface/main/history/model"
"go-common/library/cache/redis"
"go-common/library/log"
)
const _key = "v_" // mid -> score:time member:aid
// keyToView return key string
func key(mid int64) string {
return _key + strconv.FormatInt(mid, 10)
}
// Expire expire toview by mid.
func (d *Dao) Expire(c context.Context, mid int64) (ok bool, err error) {
conn := d.redis.Get(c)
if ok, err = redis.Bool(conn.Do("EXPIRE", key(mid), d.expire)); err != nil {
log.Error("conn.Do(EXPIRE, %s) error(%v)", key(mid), err)
}
conn.Close()
return
}
// Cache return the user all toview from redis.
func (d *Dao) Cache(c context.Context, mid int64, start, end int) (res []*model.ToView, err error) {
conn := d.redis.Get(c)
defer conn.Close()
values, err := redis.Values(conn.Do("ZREVRANGE", key(mid), start, end, "WITHSCORES"))
if err != nil {
log.Error("dao.Do(ZREVRANGE %v) error(%v)", key(mid), err)
return
}
if len(values) == 0 {
return
}
res = make([]*model.ToView, 0, len(values)/2)
for len(values) > 0 {
t := &model.ToView{}
if values, err = redis.Scan(values, &t.Aid, &t.Unix); err != nil {
log.Error("redis.Scan(%v) error(%v)", values, err)
return
}
res = append(res, t)
}
return
}
// CacheMap return the user all toview map from redis.
func (d *Dao) CacheMap(c context.Context, mid int64) (res map[int64]*model.ToView, err error) {
conn := d.redis.Get(c)
defer conn.Close()
values, err := redis.Values(conn.Do("ZREVRANGE", key(mid), 0, -1, "WITHSCORES"))
if err != nil {
log.Error("dao.Do(ZREVRANGE %v) error(%v)", key(mid), err)
return
}
if len(values) == 0 {
return
}
res = make(map[int64]*model.ToView, len(values)/2)
for len(values) > 0 {
t := &model.ToView{}
if values, err = redis.Scan(values, &t.Aid, &t.Unix); err != nil {
log.Error("redis.Scan(%v) error(%v)", values, err)
return
}
res[t.Aid] = t
}
return
}
// CntCache return the user toview count from redis.
func (d *Dao) CntCache(c context.Context, mid int64) (count int, err error) {
conn := d.redis.Get(c)
if count, err = redis.Int(conn.Do("ZCARD", key(mid))); err != nil {
log.Error("dao.Do(ZCARD,%s) err(%v)", key(mid), err)
}
conn.Close()
return
}
// ClearCache delete the user toview redis.
func (d *Dao) ClearCache(c context.Context, mid int64) (err error) {
conn := d.redis.Get(c)
if _, err = conn.Do("DEL", key(mid)); err != nil {
log.Error("conn.Do(DEL %s) error(%v)", key(mid), err)
}
conn.Close()
return
}
// DelCaches delete the user toview redis.
func (d *Dao) DelCaches(c context.Context, mid int64, aids []int64) (err error) {
conn := d.redis.Get(c)
defer conn.Close()
for _, aid := range aids {
if err = conn.Send("ZREM", key(mid), aid); err != nil {
log.Error("conn.Send(ZREM %s,%d) error(%v)", key(mid), aid, err)
return
}
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for i := 0; i < len(aids); i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
return
}
// AddCache add user toview to redis.
func (d *Dao) AddCache(c context.Context, mid, aid, now int64) error {
return d.addCache(c, key(mid), []*model.ToView{&model.ToView{Aid: aid, Unix: now}})
}
// AddCacheList add user toview to redis.
func (d *Dao) AddCacheList(c context.Context, mid int64, views []*model.ToView) error {
return d.addCache(c, key(mid), views)
}
// addCache add user toview to redis.
func (d *Dao) addCache(c context.Context, key string, views []*model.ToView) (err error) {
conn := d.redis.Get(c)
defer conn.Close()
for _, v := range views {
if err = conn.Send("ZADD", key, v.Unix, v.Aid); err != nil {
log.Error("conn.Send(ZREM %s,%d) error(%v)", key, v.Aid, err)
return
}
}
if err = conn.Send("EXPIRE", key, d.expire); err != nil {
log.Error("conn.Send(EXPIRE) error(%v)", err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for i := 0; i < len(views)+1; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
return
}
// PingRedis check redis connection
func (d *Dao) PingRedis(c context.Context) (err error) {
return
}

View File

@@ -0,0 +1,184 @@
package toview
import (
"context"
"testing"
"time"
"go-common/app/interface/main/history/model"
"github.com/smartystreets/goconvey/convey"
)
func TestToviewkey(t *testing.T) {
convey.Convey("key", t, func(ctx convey.C) {
var (
mid = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
p1 := key(mid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestToviewExpire(t *testing.T) {
convey.Convey("Expire", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
ok, err := d.Expire(c, mid)
ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(ok, convey.ShouldNotBeNil)
})
})
})
}
func TestToviewCache(t *testing.T) {
convey.Convey("Cache", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
start = int(1)
end = int(2)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
_, err := d.Cache(c, mid, start, end)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewCacheMap(t *testing.T) {
convey.Convey("CacheMap", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := d.CacheMap(c, mid)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}
func TestToviewCntCache(t *testing.T) {
convey.Convey("CntCache", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
count, err := d.CntCache(c, mid)
ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
})
}
func TestToviewClearCache(t *testing.T) {
convey.Convey("ClearCache", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.ClearCache(c, mid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewDelCaches(t *testing.T) {
convey.Convey("DelCaches", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
aids = []int64{}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.DelCaches(c, mid, aids)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewAddCache(t *testing.T) {
convey.Convey("AddCache", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
aid = int64(14771787)
now = time.Now().Unix()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.AddCache(c, mid, aid, now)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewAddCacheList(t *testing.T) {
convey.Convey("AddCacheList", t, func(ctx convey.C) {
var (
c = context.Background()
mid = int64(14771787)
views = []*model.ToView{}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.AddCacheList(c, mid, views)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewaddCache(t *testing.T) {
convey.Convey("addCache", t, func(ctx convey.C) {
var (
c = context.Background()
key = ""
views = []*model.ToView{{Aid: 1477, Unix: 11}}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.addCache(c, key, views)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestToviewPingRedis(t *testing.T) {
convey.Convey("PingRedis", t, func(ctx convey.C) {
var (
c = context.Background()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.PingRedis(c)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}