Create & Init Project...

app/service/main/up/dao/data/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "field_test.go",
+        "hbase_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/up/conf:go_default_library",
+        "//app/service/main/up/dao:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//vendor/github.com/bouk/monkey:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "field.go",
+        "hbase.go",
+    ],
+    importpath = "go-common/app/service/main/up/dao/data",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/main/up/conf:go_default_library",
+        "//app/service/main/up/model/data:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/up/dao/data/dao.go

@@ -0,0 +1,24 @@
+package data
+
+import (
+	"go-common/app/service/main/up/conf"
+	"go-common/library/database/hbase.v2"
+	"time"
+)
+
+//Dao hbase dao
+type Dao struct {
+	c            *conf.Config
+	hbase        *hbase.Client
+	hbaseTimeOut time.Duration
+}
+
+//New create dao
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:            c,
+		hbase:        hbase.NewClient(&c.HBase.Config),
+		hbaseTimeOut: time.Millisecond * 500,
+	}
+	return
+}

app/service/main/up/dao/data/dao_test.go

@@ -0,0 +1,40 @@
+package data
+
+import (
+	"flag"
+	"go-common/app/service/main/up/conf"
+	"os"
+	"testing"
+
+	"go-common/app/service/main/up/dao"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", dao.AppID)
+		flag.Set("conf_token", dao.UatToken)
+		flag.Set("tree_id", dao.TreeID)
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/up-service.toml")
+	}
+	if os.Getenv("UT_LOCAL_TEST") != "" {
+		flag.Set("conf", "../../cmd/up-service.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/service/main/up/dao/data/field.go

@@ -0,0 +1,94 @@
+package data
+
+var (
+	//HBaseVideoTablePrefix 播放流失分布
+	HBaseVideoTablePrefix = "video_play_churn_"
+	//HBaseArchiveTablePrefix 分类分端播放
+	HBaseArchiveTablePrefix = "video_play_category_"
+	//HBaseAreaTablePrefix 地区播放
+	HBaseAreaTablePrefix = "video_play_area_"
+	//HBaseUpStatTablePrefix up主概况
+	HBaseUpStatTablePrefix = "up_stats_"
+	//HBaseUpViewerBase 观众数据,性别年龄分布 + 设备分布
+	HBaseUpViewerBase = "up_viewer_base_"
+	//HBaseUpViewerArea 地区分布
+	HBaseUpViewerArea = "up_viewer_area_"
+	//HBaseUpViewerTrend 内容倾向
+	HBaseUpViewerTrend = "up_viewer_trend_"
+	//HBaseUpViewerActionHour 行为时间分布
+	HBaseUpViewerActionHour = "up_viewer_action_hour_"
+	//HBaseUpRelationFansDay 日维度 最近30天 只保留31天
+	HBaseUpRelationFansDay = "up_relation_fans_day"
+	// HBaseUpRelationFansHistory 日维度 各月份每日数据,日更,永久保存
+	HBaseUpRelationFansHistory = "up_relation_fans_history"
+	//HBaseUpRelationFansMonth 年维度 2017.8月以后的数据永久保存
+	HBaseUpRelationFansMonth = "up_relation_fans_month"
+	//HBaseUpPlayInc 我的概况 播放相关
+	HBaseUpPlayInc = "up_play_inc_"
+	//HBaseUpDmInc 弹幕相关
+	HBaseUpDmInc = "up_dm_inc_"
+	//HBaseUpReplyInc 评论相关
+	HBaseUpReplyInc = "up_reply_inc_"
+	//HBaseUpShareInc 分享相关
+	HBaseUpShareInc = "up_share_inc_"
+	//HBaseUpCoinInc 投币相关
+	HBaseUpCoinInc = "up_coin_inc_"
+	//HBaseUpFavInc 收藏相关
+	HBaseUpFavInc = "up_fav_inc_"
+	//HBaseUpElecInc 充电相关
+	HBaseUpElecInc = "up_elec_inc_"
+	//HBaseUpFansAnalysis  粉丝管理
+	HBaseUpFansAnalysis = "up_fans_analysis"
+	//HBaseUpPlaySourceAnalysis  播放来源
+	HBaseUpPlaySourceAnalysis = "up_play_analysis"
+	//HBaseUpArcPlayAnalysis  平均观看时长、播放用户数、留存率
+	HBaseUpArcPlayAnalysis = "up_archive_play_analysis"
+	//HBaseUpArcQuery  稿件索引表
+	HBaseUpArcQuery = "up_archive_query"
+
+	//HBasePlayArc 播放相关 archive for 30 days
+	HBasePlayArc = "up_play_trend"
+	//HBaseDmArc 弹幕相关
+	HBaseDmArc = "up_dm_trend"
+	//HBaseReplyArc 评论相关
+	HBaseReplyArc = "up_reply_trend"
+	//HBaseShareArc 分享相关
+	HBaseShareArc = "up_share_trend"
+	//HBaseCoinArc 投币相关
+	HBaseCoinArc = "up_coin_trend"
+	//HBaseFavArc 收藏相关
+	HBaseFavArc = "up_fav_trend"
+	//HBaseElecArc 充电相关
+	HBaseElecArc = "up_elec_trend"
+
+	//HBaseFamilyPlat  family
+	HBaseFamilyPlat = []byte("v")
+	//HBaseColumnAid aid
+	HBaseColumnAid = []byte("avid")
+	//HBaseColumnWebPC pc
+	HBaseColumnWebPC = []byte("plat0")
+	//HBaseColumnWebH5 h5
+	HBaseColumnWebH5 = []byte("plat1")
+	//HBaseColumnOutsite out
+	HBaseColumnOutsite = []byte("plat2")
+	//HBaseColumnIOS ios
+	HBaseColumnIOS = []byte("plat3")
+	//HBaseColumnAndroid android
+	HBaseColumnAndroid = []byte("plat4")
+	//HBaseColumnElse else
+	HBaseColumnElse = []byte("else")
+	//HBaseColumnFans fans
+	HBaseColumnFans = []byte("fans")
+	//HBaseColumnGuest guest
+	HBaseColumnGuest = []byte("guest")
+	//HBaseColumnAll all
+	HBaseColumnAll = []byte("all")
+	//HBaseColumnCoin coin
+	HBaseColumnCoin = []byte("coin")
+	//HBaseColumnElec elec
+	HBaseColumnElec = []byte("elec")
+	//HBaseColumnFav fav
+	HBaseColumnFav = []byte("fav")
+	//HBaseColumnShare share
+	HBaseColumnShare = []byte("share")
+)

app/service/main/up/dao/data/field_test.go

@@ -0,0 +1 @@
+package data

app/service/main/up/dao/data/hbase.go

@@ -0,0 +1,62 @@
+package data
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"go-common/app/service/main/up/model/data"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"strconv"
+)
+
+func hbaseMd5Key(aid int64) []byte {
+	hasher := md5.New()
+	hasher.Write([]byte(strconv.Itoa(int(aid))))
+	return []byte(hex.EncodeToString(hasher.Sum(nil)))
+}
+
+// BaseUpStat get base up stat.
+func (d *Dao) BaseUpStat(c context.Context, mid int64, date string) (stat *data.UpBaseStat, err error) {
+	var (
+		ctx, cancel = context.WithTimeout(c, d.hbaseTimeOut)
+		tableName   = HBaseUpStatTablePrefix + date // change table at 12:00am
+	)
+	defer cancel()
+	result, err := d.hbase.Get(ctx, []byte(tableName), hbaseMd5Key(mid))
+	if err != nil {
+		log.Error("BaseUpStat d.hbase.Get BackupTable(%s, %d) error(%v)", tableName, mid, err)
+		err = ecode.ServerErr
+		return
+	}
+	if result == nil {
+		log.Error("BaseUpStat d.hbase.Get BackupTable(%s, %d) result nil", tableName, mid)
+		return
+	}
+	stat = &data.UpBaseStat{}
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+		if !bytes.Equal(c.Family, []byte("u")) {
+			continue
+		}
+		switch {
+		case bytes.Equal(c.Qualifier, []byte("play")):
+			stat.View = v
+		case bytes.Equal(c.Qualifier, []byte("dm")):
+			stat.Dm = v
+		case bytes.Equal(c.Qualifier, []byte("reply")):
+			stat.Reply = v
+		case bytes.Equal(c.Qualifier, []byte("fans")):
+			stat.Fans = v
+		case bytes.Equal(c.Qualifier, []byte("fav")):
+			stat.Fav = v
+		case bytes.Equal(c.Qualifier, []byte("like")):
+			stat.Like = v
+		}
+	}
+	return
+}

app/service/main/up/dao/data/hbase_test.go

@@ -0,0 +1,53 @@
+package data
+
+import (
+	"context"
+	"github.com/bouk/monkey"
+	"github.com/tsuna/gohbase/hrpc"
+	"go-common/library/database/hbase.v2"
+	"reflect"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDatahbaseMd5Key(t *testing.T) {
+	var (
+		aid = int64(0)
+	)
+	convey.Convey("hbaseMd5Key", t, func(ctx convey.C) {
+		p1 := hbaseMd5Key(aid)
+		ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDataBaseUpStat(t *testing.T) {
+	var (
+		c    = context.TODO()
+		mid  = int64(0)
+		date = ""
+	)
+	// Hbase never ok
+	convey.Convey("BaseUpStat", t, func(ctx convey.C) {
+		guard := monkey.PatchInstanceMethod(reflect.TypeOf(d.hbase), "Get", func(_ *hbase.Client, _ context.Context, _ []byte, _ []byte, _ ...func(hrpc.Call) error) (*hrpc.Result, error) {
+			cells := make([]*hrpc.Cell, 5)
+			for i := range cells {
+				cell := new(hrpc.Cell)
+				cell.Value = []byte("test")
+				cells[i] = cell
+			}
+			res := &hrpc.Result{
+				Cells: cells,
+			}
+			return res, nil
+		})
+		defer guard.Unpatch()
+		stat, err := d.BaseUpStat(c, mid, date)
+		ctx.Convey("Then err should be nil.stat should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(stat, convey.ShouldNotBeNil)
+		})
+	})
+}