Create & Init Project...

app/admin/main/relation/dao/BUILD

@@ -0,0 +1,66 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "account_test.go",
+        "dao_test.go",
+        "relation_log_test.go",
+        "relation_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/relation/conf:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//vendor/github.com/bouk/monkey:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "account.go",
+        "dao.go",
+        "relation.go",
+        "relation_log.go",
+    ],
+    importpath = "go-common/app/admin/main/relation/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/relation/conf:go_default_library",
+        "//app/admin/main/relation/model:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//app/service/main/account/model:go_default_library",
+        "//app/service/main/relation/api:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/jinzhu/gorm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/relation/dao/account.go

@@ -0,0 +1,76 @@
+package dao
+
+import (
+	"context"
+	"sync"
+
+	accountApi "go-common/app/service/main/account/api"
+	account "go-common/app/service/main/account/model"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+)
+
+const (
+	_perCall = 50
+)
+
+func chain(ids ...[]int64) []int64 {
+	res := make([]int64, 0, len(ids))
+	for _, l := range ids {
+		res = append(res, l...)
+	}
+	return res
+}
+
+func uniq(ids ...[]int64) []int64 {
+	hm := make(map[int64]struct{})
+	for _, i := range chain(ids...) {
+		hm[i] = struct{}{}
+	}
+	res := make([]int64, 0, len(ids))
+	for i := range hm {
+		res = append(res, i)
+	}
+	return res
+}
+
+// RPCInfos rpc info get by  muti mid .
+func (d *Dao) RPCInfos(c context.Context, mids []int64) (res map[int64]*account.Info, err error) {
+	var (
+		g errgroup.Group
+		l sync.RWMutex
+	)
+	mids = uniq(mids)
+	total := len(mids)
+	pageNum := total / _perCall
+	if total%_perCall != 0 {
+		pageNum++
+	}
+	res = make(map[int64]*account.Info, total)
+	for i := 0; i < pageNum; i++ {
+		start := i * _perCall
+		end := (i + 1) * _perCall
+		if end > total {
+			end = total
+		}
+		g.Go(func() (err error) {
+			midsReq := &accountApi.MidsReq{Mids: mids[start:end]}
+			infosReply, err := d.accountClient.Infos3(c, midsReq)
+			if err != nil {
+				log.Error("d.accountClient.Infos3(%+v) error(%v)", midsReq, err)
+				err = nil
+				return
+			}
+			for mid, info := range infosReply.Infos {
+				l.Lock()
+				res[mid] = info
+				l.Unlock()
+			}
+			return
+		})
+	}
+	if err = g.Wait(); err != nil {
+		log.Error("g.Wait error(%v)", err)
+	}
+	return
+}

app/admin/main/relation/dao/account_test.go

@@ -0,0 +1,16 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestRPCInfos(t *testing.T) {
+	convey.Convey("RPCInfos", t, func() {
+		infos, err := d.RPCInfos(context.TODO(), []int64{1, 2, 3})
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(infos, convey.ShouldNotBeNil)
+	})
+}

app/admin/main/relation/dao/dao.go

@@ -0,0 +1,58 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/admin/main/relation/conf"
+	accountGRPC "go-common/app/service/main/account/api"
+	relationGRPC "go-common/app/service/main/relation/api"
+	"go-common/library/database/hbase.v2"
+	"go-common/library/database/orm"
+
+	"github.com/jinzhu/gorm"
+)
+
+// Dao dao
+type Dao struct {
+	c              *conf.Config
+	ReadORM        *gorm.DB
+	accountClient  accountGRPC.AccountClient
+	relationClient relationGRPC.RelationClient
+	hbase          *hbase.Client
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:       c,
+		ReadORM: orm.NewMySQL(c.ORM.Read),
+		hbase:   hbase.NewClient(c.LogHBase),
+	}
+	relationGRPCClient, err := relationGRPC.NewClient(c.RelationGRPC)
+	if err != nil {
+		panic(err)
+	}
+
+	accountGRPCClient, err := accountGRPC.NewClient(c.AccountGRPC)
+	if err != nil {
+		panic(err)
+	}
+	dao.relationClient = relationGRPCClient
+	dao.accountClient = accountGRPCClient
+	dao.initORM()
+	return
+}
+
+func (dao *Dao) initORM() {
+	dao.ReadORM.LogMode(true)
+}
+
+// Close close the resource.
+func (dao *Dao) Close() {
+	dao.ReadORM.Close()
+}
+
+// Ping dao ping
+func (dao *Dao) Ping(c context.Context) error {
+	return dao.ReadORM.DB().PingContext(c)
+}

app/admin/main/relation/dao/dao_test.go

@@ -0,0 +1,35 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/admin/main/relation/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.account.relation-admin")
+		flag.Set("conf_token", "d79d38fa305b01c1ce59099cb0385ebf")
+		flag.Set("tree_id", "21423")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/relation-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/admin/main/relation/dao/relation.go

@@ -0,0 +1,85 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/admin/main/relation/model"
+	relationPB "go-common/app/service/main/relation/api"
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+)
+
+const (
+	_shard   = 500
+	_maxSize = 20000
+)
+
+func midTable(mid int64) string {
+	return fmt.Sprintf("user_relation_mid_%03d", mid%_shard)
+}
+
+func fidTable(fid int64) string {
+	return fmt.Sprintf("user_relation_fid_%03d", fid%_shard)
+}
+
+// Followers is
+func (d *Dao) Followers(ctx context.Context, fid int64, mid int64) (model.RelationList, error) {
+	list := model.RelationList{}
+	db := d.ReadORM.Table(fidTable(fid)).Where("status=?", 0).Where("fid=?", fid).Limit(_maxSize).Order("id desc")
+	if mid > 0 {
+		db = db.Where("mid=?", mid).Limit(1)
+	}
+	if err := db.Find(&list).Error; err != nil {
+		return nil, err
+	}
+	for _, f := range list {
+		f.ParseRelation()
+	}
+	return list, nil
+}
+
+// Followings is
+func (d *Dao) Followings(ctx context.Context, mid int64, fid int64) (model.RelationList, error) {
+	list := model.RelationList{}
+	db := d.ReadORM.Table(midTable(mid)).Where("status=?", 0).Where("mid=?", mid).Limit(_maxSize).Order("id desc")
+	if fid > 0 {
+		db = db.Where("fid=?", fid).Limit(1)
+	}
+	if err := db.Find(&list).Error; err != nil {
+		return nil, err
+	}
+	for _, f := range list {
+		f.ParseRelation()
+	}
+	return list, nil
+}
+
+// Stat is
+func (d *Dao) Stat(ctx context.Context, mid int64) (*relationPB.StatReply, error) {
+	stat, err := d.relationClient.Stat(ctx, &relationPB.MidReq{
+		Mid:    mid,
+		RealIp: metadata.String(ctx, metadata.RemoteIP),
+	})
+	if err != nil {
+		log.Error("d.relationRPC.Stat err(%+v)", err)
+		return nil, err
+	}
+	return stat, nil
+}
+
+// Stats is
+func (d *Dao) Stats(ctx context.Context, mids []int64) (map[int64]*relationPB.StatReply, error) {
+	statReply, err := d.relationClient.Stats(ctx, &relationPB.MidsReq{
+		Mids:   mids,
+		RealIp: metadata.String(ctx, metadata.RemoteIP),
+	})
+	if err != nil {
+		log.Error("d.relationRPC.Stats err(%+v)", err)
+		return nil, err
+	}
+	if len(statReply.StatReplyMap) == 0 {
+		return nil, nil
+	}
+	return statReply.StatReplyMap, nil
+}

app/admin/main/relation/dao/relation_log.go

@@ -0,0 +1,84 @@
+package dao
+
+import (
+	"context"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/admin/main/relation/model"
+	"go-common/library/log"
+)
+
+// reverse returns its argument string reversed rune-wise left to right.
+func reverse(s string) string {
+	r := []rune(s)
+	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
+		r[i], r[j] = r[j], r[i]
+	}
+	return string(r)
+}
+
+func rpad(s string, l int) string {
+	dt := l - len(s)
+	if dt <= 0 {
+		return s
+	}
+	return s + strings.Repeat("0", dt)
+}
+
+func logKey(mid, fid, ts int64) string {
+	midStr := rpad(reverse(strconv.FormatInt(mid, 10)), 10)
+	fidStr := rpad(reverse(strconv.FormatInt(fid, 10)), 10)
+	tsStr := strconv.FormatInt(ts, 10)
+	return midStr + fidStr + tsStr
+}
+
+// RelationLogs is used to retriev relation log.
+func (d *Dao) RelationLogs(ctx context.Context, mid, fid int64, from time.Time, to time.Time) (model.RelationLogList, error) {
+	scanner, err := d.hbase.ScanRangeStr(ctx, d.c.LogTable, logKey(mid, fid, from.Unix()), logKey(mid, fid, to.Unix()))
+	if err != nil {
+		log.Error("Failed to d.hbase.Scan(): %+v", err)
+		return nil, err
+	}
+
+	logs := make(model.RelationLogList, 0)
+
+	for {
+		r, err := scanner.Next()
+		if err != nil {
+			if err != io.EOF {
+				return nil, err
+			}
+			break
+		}
+		l := &model.RelationLog{
+			Mid: mid,
+			Fid: fid,
+		}
+		for _, c := range r.Cells {
+			key := string(c.Row)
+			qf := string(c.Qualifier)
+			v := string(c.Value)
+			log.Info("Retrieving relation log: mid(%d) fid(%d) key(%s) qualifier(%s) value(%s)", mid, fid, key, qf, v)
+
+			// fill fields
+			switch qf {
+			case "att":
+				l.Attention = model.ParseAction(v)
+			case "bl":
+				l.Black = model.ParseAction(v)
+			case "wh":
+				l.Whisper = model.ParseAction(v)
+			case "src":
+				l.Source = model.ParseSource(v)
+			case "mt":
+				l.MTime, _ = model.ParseLogTime(v)
+			}
+		}
+		l.FillAttrField()
+		logs = append(logs, l)
+	}
+	return logs, nil
+}

app/admin/main/relation/dao/relation_log_test.go

@@ -0,0 +1,85 @@
+package dao
+
+import (
+	"context"
+
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	"go-common/library/database/hbase.v2"
+
+	"github.com/bouk/monkey"
+	"github.com/smartystreets/goconvey/convey"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+func TestDaoreverse(t *testing.T) {
+	convey.Convey("reverse", t, func(convCtx convey.C) {
+		var (
+			s = ""
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			p1 := reverse(s)
+			convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
+				convCtx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaorpad(t *testing.T) {
+	convey.Convey("rpad", t, func(convCtx convey.C) {
+		var (
+			s = ""
+			l = int(0)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			p1 := rpad(s, l)
+			convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
+				convCtx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaologKey(t *testing.T) {
+	convey.Convey("logKey", t, func(convCtx convey.C) {
+		var (
+			mid = int64(0)
+			fid = int64(0)
+			ts  = int64(0)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			p1 := logKey(mid, fid, ts)
+			convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
+				convCtx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRelationLogs(t *testing.T) {
+	convey.Convey("RelationLogs", t, func(convCtx convey.C) {
+		var (
+			ctx  = context.Background()
+			mid  = int64(0)
+			fid  = int64(0)
+			from = time.Now()
+			to   = time.Now()
+		)
+		convCtx.Convey("RelationLogs failed", func(convCtx convey.C) {
+			monkey.PatchInstanceMethod(reflect.TypeOf(d.hbase), "ScanRangeStr", func(_ *hbase.Client, _ context.Context, _ string, _ string, _ string, _ ...func(hrpc.Call) error) (hrpc.Scanner, error) {
+				return nil, fmt.Errorf("hbase scan err")
+			})
+			defer monkey.UnpatchAll()
+			p1, err := d.RelationLogs(ctx, mid, fid, from, to)
+			convCtx.Convey("Then err should be nil.p1 should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldNotBeNil)
+				convCtx.So(p1, convey.ShouldBeNil)
+			})
+		})
+
+	})
+}

app/admin/main/relation/dao/relation_test.go

@@ -0,0 +1,49 @@
+package dao
+
+import (
+	"context"
+
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestFollowers(t *testing.T) {
+	convey.Convey("Followers", t, func() {
+		rl, err := d.Followers(context.TODO(), 1, 2)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(rl, convey.ShouldNotBeNil)
+	})
+}
+
+func TestFollowings(t *testing.T) {
+	convey.Convey("Followings", t, func() {
+		rl, err := d.Followings(context.TODO(), 1, 2)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(rl, convey.ShouldNotBeNil)
+	})
+}
+
+func TestStat(t *testing.T) {
+	var (
+		c         = context.Background()
+		mid int64 = 2
+	)
+	convey.Convey("stat", t, func() {
+		reply, err := d.Stat(c, mid)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(reply, convey.ShouldNotBeNil)
+	})
+}
+
+func TestStats(t *testing.T) {
+	var (
+		c    = context.Background()
+		mids = []int64{2, 3}
+	)
+	convey.Convey("stats", t, func() {
+		replys, err := d.Stats(c, mids)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(replys, convey.ShouldNotBeNil)
+	})
+}