Create & Init Project...

app/admin/main/up/dao/data/BUILD

@@ -0,0 +1,78 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "api_test.go",
+        "center_test.go",
+        "dao_test.go",
+        "field_test.go",
+        "hbase_set2_test.go",
+        "hbase_test.go",
+        "phase_three_test.go",
+        "utils_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/up/conf:go_default_library",
+        "//app/admin/main/up/model/datamodel:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//vendor/github.com/bouk/monkey:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "center.go",
+        "dao.go",
+        "field.go",
+        "hbase.go",
+        "hbase_set2.go",
+        "phase_three.go",
+        "utils.go",
+    ],
+    importpath = "go-common/app/admin/main/up/dao/data",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/up/conf:go_default_library",
+        "//app/admin/main/up/model/datamodel:go_default_library",
+        "//app/admin/main/up/util/hbaseutil:go_default_library",
+        "//app/interface/main/creative/model/data:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+        "@org_golang_x_net//context:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/up/dao/data/api.go

@@ -0,0 +1,101 @@
+package data
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"go-common/app/interface/main/creative/model/data"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_statURL  = "/data/member/upinfo/%d.json"
+	_tagURL   = "/tag_predict"
+	_tagv2URL = "/tag_predict/v2"
+	_coverURL = "/cover_recomm"
+)
+
+// Tags get predict tag.
+func (d *Dao) Tags(c context.Context, mid int64, tid uint16, title, filename, desc, cover string) (t *data.Tags, err error) {
+	params := url.Values{}
+	params.Set("typeid", strconv.Itoa(int(tid)))
+	params.Set("title", title)
+	params.Set("filename", filename)
+	params.Set("desc", desc)
+	params.Set("cover", cover)
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	var res struct {
+		Code int        `json:"code"`
+		Data *data.Tags `json:"data"`
+	}
+	if err = d.client.Get(c, d.tagURI, "", params, &res); err != nil {
+		log.Error("data url(%s) error(%v)", d.tagURI+"?"+params.Encode(), err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if res.Code != 0 {
+		log.Error("data url(%s) res(%v)", d.tagURI+"?"+params.Encode(), res)
+		err = ecode.CreativeDataErr
+		return
+	}
+	t = res.Data
+	return
+}
+
+// TagsWithChecked get predict tag with checked mark.
+func (d *Dao) TagsWithChecked(c context.Context, mid int64, tid uint16, title, filename, desc, cover string, tagFrom int8) (t []*data.CheckedTag, err error) {
+	params := url.Values{}
+	t = make([]*data.CheckedTag, 0)
+	params.Set("client_type", strconv.Itoa(int(tagFrom)))
+	params.Set("typeid", strconv.Itoa(int(tid)))
+	params.Set("title", title)
+	params.Set("filename", filename)
+	params.Set("desc", desc)
+	params.Set("cover", cover)
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	var res struct {
+		Code int `json:"code"`
+		Data struct {
+			Tags []*data.CheckedTag `json:"tags"`
+		} `json:"data"`
+	}
+	log.Info("TagsWithChecked url(%s)", d.tagV2URI+"?"+params.Encode())
+	if err = d.client.Get(c, d.tagV2URI, "", params, &res); err != nil {
+		log.Error("data url(%s) error(%v)", d.tagV2URI+"?"+params.Encode(), err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if res.Code != 0 {
+		log.Error("data url(%s) res(%v)", d.tagV2URI+"?"+params.Encode(), res)
+		err = ecode.CreativeDataErr
+		return
+	}
+	t = res.Data.Tags
+	return
+}
+
+// RecommendCovers get recommend covers from AI.
+func (d *Dao) RecommendCovers(c context.Context, mid int64, fns []string) (cvs []string, err error) {
+	params := url.Values{}
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	params.Set("filename", strings.Join(fns, ","))
+	var res struct {
+		Code int      `json:"code"`
+		Data []string `json:"data"`
+	}
+	if err = d.client.Get(c, d.coverBFSURI, "", params, &res); err != nil {
+		log.Error("Covers url(%s) error(%v)", d.coverBFSURI+"?"+params.Encode(), err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if res.Code != 0 {
+		log.Error("Covers url(%s) res(%v)", d.coverBFSURI+"?"+params.Encode(), res)
+		err = ecode.CreativeDataErr
+		return
+	}
+	cvs = res.Data
+	return
+}

app/admin/main/up/dao/data/api_test.go

@@ -0,0 +1,87 @@
+package data
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+	"gopkg.in/h2non/gock.v1"
+)
+
+func TestDataTags(t *testing.T) {
+	convey.Convey("Tags", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			mid      = int64(0)
+			tid      = uint16(0)
+			title    = ""
+			filename = ""
+			desc     = ""
+			cover    = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			defer gock.OffAll()
+			httpMock("GET", d.tagURI).Reply(200).BodyString(`
+{
+	"code":0, 
+	"data": {"tags" : ["1"]}
+}`)
+			no, err := d.Tags(c, mid, tid, title, filename, desc, cover)
+			ctx.Convey("Then err should be nil.no should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(no, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataTagsWithChecked(t *testing.T) {
+	convey.Convey("TagsWithChecked", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			mid      = int64(0)
+			tid      = uint16(0)
+			title    = ""
+			filename = ""
+			desc     = ""
+			cover    = ""
+			tagFrom  = int8(0)
+		)
+		defer gock.OffAll()
+		httpMock("GET", d.tagV2URI).Reply(200).BodyString(`
+{
+	"code":0, 
+	"data": {"tags" : []}
+}`)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			no, err := d.TagsWithChecked(c, mid, tid, title, filename, desc, cover, tagFrom)
+			ctx.Convey("Then err should be nil.no should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(no, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataRecommendCovers(t *testing.T) {
+	convey.Convey("RecommendCovers", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			fns = []string{}
+		)
+		defer gock.OffAll()
+		httpMock("GET", d.coverBFSURI).Reply(200).BodyString(`
+{
+	"code":0, 
+	"data": ["1"]
+}`)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			cvs, err := d.RecommendCovers(c, mid, fns)
+			ctx.Convey("Then err should be nil.cvs should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(cvs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/admin/main/up/dao/data/center.go

@@ -0,0 +1,556 @@
+package data
+
+import (
+	"bytes"
+	"strconv"
+	"time"
+
+	"go-common/app/admin/main/up/model/datamodel"
+	"go-common/app/admin/main/up/util/hbaseutil"
+	"go-common/app/interface/main/creative/model/data"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/tsuna/gohbase/hrpc"
+	"golang.org/x/net/context"
+)
+
+// ViewerBase visitor data analysis.
+func (d *Dao) ViewerBase(c context.Context, mid int64, dt time.Time) (res *datamodel.ViewerBaseInfo, err error) {
+	var (
+		result       *hrpc.Result
+		ctx, cancel  = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		key          = hbaseMd5Key(mid)
+		genTableName = generateTableNameFunc(HBaseUpViewerBase, dt, -7)
+	)
+	defer cancel()
+
+	if result, err = d.getDataWithBackup(ctx, d.hbase, genTableName, 2, key); err != nil {
+		log.Error("ViewerBase d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", HBaseUpViewerBase, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("ViewerBase no data tableName(%s)|mid(%d)|key(%v)", HBaseUpViewerBase, mid, key)
+		return
+	}
+	res = &datamodel.ViewerBaseInfo{}
+	families := make(map[string]string, 2)
+	families["f"] = ""
+	families["g"] = ""
+	fb := &datamodel.ViewerBaseData{}
+	gb := &datamodel.ViewerBaseData{}
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		if _, ok := families[string(c.Family)]; ok {
+			var data = gb
+			if string(c.Family) == "f" {
+				data = fb
+			}
+			v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+			/*
+				f:plat0	web-pc播放
+				f:plat1	web-h5播放
+				f:plat2	站外播放
+				f:plat3	ios播放
+				f:plat4	android播放
+			*/
+			switch {
+			case bytes.Equal(c.Qualifier, []byte("male")):
+				data.Male = v
+			case bytes.Equal(c.Qualifier, []byte("female")):
+				data.Female = v
+			case bytes.Equal(c.Qualifier, []byte("age1")):
+				data.Age1 = v
+			case bytes.Equal(c.Qualifier, []byte("age2")):
+				data.Age2 = v
+			case bytes.Equal(c.Qualifier, []byte("age3")):
+				data.Age3 = v
+			case bytes.Equal(c.Qualifier, []byte("age4")):
+				data.Age4 = v
+			case bytes.Equal(c.Qualifier, []byte("plat0")):
+				data.PlatPC = v
+			case bytes.Equal(c.Qualifier, []byte("plat1")):
+				data.PlatH5 = v
+			case bytes.Equal(c.Qualifier, []byte("plat2")):
+				data.PlatOut = v
+			case bytes.Equal(c.Qualifier, []byte("plat3")):
+				data.PlatIOS = v
+			case bytes.Equal(c.Qualifier, []byte("plat4")):
+				data.PlatAndroid = v
+			case bytes.Equal(c.Qualifier, []byte("else")):
+				data.PlatOtherApp = v
+			}
+		}
+	}
+	res.Fans = fb
+	res.Guest = gb
+	return
+}
+
+// ViewerArea visitor area data analysis.
+func (d *Dao) ViewerArea(c context.Context, mid int64, dt time.Time) (res *datamodel.ViewerAreaInfo, err error) {
+	var (
+		result       *hrpc.Result
+		ctx, cancel  = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		key          = hbaseMd5Key(mid)
+		genTableName = generateTableNameFunc(HBaseUpViewerArea, dt, -7)
+	)
+	defer cancel()
+	if result, err = d.getDataWithBackup(ctx, d.hbase, genTableName, 2, key); err != nil {
+		log.Error("ViewerArea d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", HBaseUpViewerArea, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("ViewerArea no data tableName(%s)|mid(%d)|key(%v)", HBaseUpViewerArea, mid, key)
+		return
+	}
+	res = &datamodel.ViewerAreaInfo{}
+	families := make(map[string]string, 2)
+	families["f"] = ""
+	families["g"] = ""
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		if _, ok := families[string(c.Family)]; ok {
+			a := string(c.Qualifier[:])
+			var list = &res.Guest
+			v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+			if string(c.Family) == "f" {
+				list = &res.Fans
+			}
+			*list = append(*list, datamodel.ViewerAreaData{Area: a, Viewers: v})
+		}
+	}
+
+	return
+}
+
+// ViewerTrend visitor trend data analysis.
+func (d *Dao) ViewerTrend(c context.Context, mid int64, dt time.Time) (res map[string]*data.Trend, err error) {
+	var (
+		result       *hrpc.Result
+		ctx, cancel  = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		genTableName = generateTableNameFunc(HBaseUpViewerTrend, dt, -7)
+		key          = hbaseMd5Key(mid)
+	)
+	defer cancel()
+	if result, err = d.getDataWithBackup(ctx, d.hbase, genTableName, 2, key); err != nil {
+		log.Error("ViewerTrend d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", HBaseUpViewerTrend, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("ViewerTrend no data tableName(%s)|mid(%d)|key(%v)", HBaseUpViewerTrend, mid, key)
+		return
+	}
+	families := make(map[string]string, 4)
+	families["fs"] = ""
+	families["ft"] = ""
+	families["gs"] = ""
+	families["gt"] = ""
+	res = make(map[string]*data.Trend)
+	ftd := &data.Trend{}
+	gtd := &data.Trend{}
+	ty := make(map[int]int64)
+	tg := make(map[int]int64)
+	nty := make(map[int]int64)
+	ntg := make(map[int]int64)
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		if _, ok := families[string(c.Family)]; ok {
+			if string(c.Family) == "fs" {
+				tid, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+				ty[tid] = v
+				ftd.Ty = ty
+			} else if string(c.Family) == "gs" {
+				tid, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+				nty[tid] = v
+				gtd.Ty = nty
+			}
+			if string(c.Family) == "ft" {
+				o, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+				tg[o] = v
+				ftd.Tag = tg
+			} else if string(c.Family) == "gt" {
+				o, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+				ntg[o] = v
+				gtd.Tag = ntg
+			}
+		}
+	}
+	res["fan"] = ftd
+	res["guest"] = gtd
+	return
+}
+
+// RelationFansDay up relation 30 days analysis.
+func (d *Dao) RelationFansDay(c context.Context, mid int64) (res datamodel.RelationFanHistoryData, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName   = HBaseUpRelationFansDay
+		key         = hbaseMd5Key(mid)
+		parser      = hbaseutil.Parser{
+			ParseIntFunc: hbaseutil.StringToUint,
+		}
+	)
+	defer cancel()
+	if result, err = d.hbase.GetStr(ctx, tableName, key); err != nil {
+		log.Error("RelationFansDay d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", tableName, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("RelationFansDay no data tableName(%s)|mid(%d)|key(%v)", tableName, mid, key)
+		return
+	}
+
+	if err = parser.Parse(result.Cells, &res); err != nil {
+		log.Error("parse cell fail, err=%v", err)
+		return
+	}
+	/*
+		families := make(map[string]string, 2)
+		families["a"] = "" //某日新增关注量
+		families["u"] = "" //某日取关量
+		fd := make(map[string]int)
+		nfd := make(map[string]int)
+		for _, c := range result.Cells {
+			if c == nil {
+				continue
+			}
+			if _, ok := families[string(c.Family)]; ok {
+				if string(c.Family) == "a" {
+					k := string(c.Qualifier[:])
+					v, _ := strconv.Atoi(string(c.Value[:]))
+					fd[k] = v
+				} else if string(c.Family) == "u" {
+					k := string(c.Qualifier[:])
+					v, _ := strconv.Atoi(string(c.Value[:]))
+					nfd[k] = v
+				}
+			}
+		}
+		res["follow"] = fd
+		res["unfollow"] = nfd
+	*/
+	return
+}
+
+// RelationFansHistory up relation history.
+func (d *Dao) RelationFansHistory(c context.Context, mid int64, month string) (res datamodel.RelationFanHistoryData, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName   = HBaseUpRelationFansHistory
+		key         = string(hbaseMd5Key(mid)) + "_" + month
+		parser      = hbaseutil.Parser{ParseIntFunc: hbaseutil.StringToUint}
+	)
+	defer cancel()
+	if result, err = d.hbase.GetStr(ctx, tableName, key); err != nil {
+		log.Error("RelationFansHistory d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", tableName, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("RelationFansHistory no data tableName(%s)|mid(%d)|key(%v)", tableName, mid, key)
+		return
+	}
+
+	if err = parser.Parse(result.Cells, &res); err != nil {
+		log.Error("parse cell fail, err=%v", err)
+		return
+	}
+
+	return
+}
+
+// RelationFansMonth up relation 400 days analysis.
+func (d *Dao) RelationFansMonth(c context.Context, mid int64) (res datamodel.RelationFanHistoryData, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName   = HBaseUpRelationFansMonth
+		key         = hbaseMd5Key(mid)
+		parser      = hbaseutil.Parser{ParseIntFunc: hbaseutil.StringToUint}
+	)
+	defer cancel()
+	if result, err = d.hbase.GetStr(ctx, tableName, key); err != nil {
+		log.Error("RelationFansMonth d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", tableName, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("RelationFansMonth no data tableName(%s)|mid(%d)|key(%v)", tableName, mid, key)
+		return
+	}
+	if err = parser.Parse(result.Cells, &res); err != nil {
+		log.Error("parse cell fail, err=%v", err)
+		return
+	}
+	return
+}
+
+// ViewerActionHour visitor action hour analysis.
+func (d *Dao) ViewerActionHour(c context.Context, mid int64, dt string) (res map[string]*data.ViewerActionHour, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName   = HBaseUpViewerActionHour + dt
+		key         = hbaseMd5Key(mid)
+	)
+	defer cancel()
+	if result, err = d.hbase.GetStr(ctx, tableName, key); err != nil {
+		log.Error("ViewerActionHour d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", tableName, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("ViewerActionHour no data tableName(%s)|mid(%d)|key(%v)", tableName, mid, key)
+		return
+	}
+	families := make(map[string]string, 4)
+	families["fp"] = "" //播放数
+	families["fr"] = "" //评论数
+	families["fd"] = "" //弹幕数
+	families["fe"] = "" //充电数
+	families["fs"] = "" //承包数
+	res = make(map[string]*data.ViewerActionHour)
+	view := make(map[int]int)
+	reply := make(map[int]int)
+	danmu := make(map[int]int)
+	elec := make(map[int]int)
+	con := make(map[int]int)
+	fah := &data.ViewerActionHour{}
+	gah := &data.ViewerActionHour{}
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		if _, ok := families[string(c.Family)]; ok {
+			if string(c.Family) == "fp" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				view[k] = v
+				fah.View = view
+			} else if string(c.Family) == "gp" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				view[k] = v
+				gah.View = view
+			}
+			if string(c.Family) == "fr" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				reply[k] = v
+				fah.Reply = reply
+			} else if string(c.Family) == "gr" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				reply[k] = v
+				gah.Reply = reply
+			}
+			if string(c.Family) == "fd" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				danmu[k] = v
+				fah.Dm = danmu
+			} else if string(c.Family) == "gd" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				danmu[k] = v
+				gah.Dm = danmu
+			}
+			if string(c.Family) == "fe" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				elec[k] = v
+				fah.Elec = elec
+			} else if string(c.Family) == "ge" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				elec[k] = v
+				gah.Elec = elec
+			}
+			if string(c.Family) == "fs" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				con[k] = v
+				fah.Contract = con
+			} else if string(c.Family) == "gs" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				con[k] = v
+				gah.Contract = con
+			}
+		}
+	}
+	res["fan"] = fah
+	res["not_fan"] = gah
+	return
+}
+
+// UpIncr for Play/Dm/Reply/Fav/Share/Elec/Coin incr.
+func (d *Dao) UpIncr(c context.Context, mid int64, ty int8, now string) (res *data.UpDataIncrMeta, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, d.hbaseTimeOut)
+		tableName   string
+		IncrKey     string
+		key         = hbaseMd5Key(mid)
+	)
+	defer cancel()
+	switch {
+	case ty == data.Play:
+		tableName = HBaseUpPlayInc + now
+	case ty == data.Dm:
+		tableName = HBaseUpDmInc + now
+	case ty == data.Reply:
+		tableName = HBaseUpReplyInc + now
+	case ty == data.Share:
+		tableName = HBaseUpShareInc + now
+	case ty == data.Coin:
+		tableName = HBaseUpCoinInc + now
+	case ty == data.Fav:
+		tableName = HBaseUpFavInc + now
+	case ty == data.Elec:
+		tableName = HBaseUpElecInc + now
+	}
+	IncrKey, _ = data.IncrTy(ty)
+	if result, err = d.hbase.GetStr(ctx, tableName, key); err != nil {
+		log.Error("UpIncr d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", tableName, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("UpIncr no data tableName(%s)|mid(%d)|key(%v)", tableName, mid, key)
+		return
+	}
+	families := make(map[string]string, 4)
+	families["u"] = ""  //单日播放增量 u:play ...
+	families["av"] = "" //播放top稿件 av:1
+	families["v"] = ""  //top稿件播放增量 v:1
+	families["rk"] = "" //up主播放量排名 rk: [tid]
+	aids := make(map[int]int64)
+	incs := make(map[int]int)
+	rk := make(map[int]int)
+	var incr int
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		if _, ok := families[string(c.Family)]; ok {
+			if string(c.Family) == "av" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+				aids[k] = v
+			} else if string(c.Family) == "v" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				incs[k] = v
+			} else if string(c.Family) == "rk" {
+				k, _ := strconv.Atoi(string(c.Qualifier[:]))
+				v, _ := strconv.Atoi(string(c.Value[:]))
+				rk[k] = v
+			} else if string(c.Family) == "u" {
+				if bytes.Equal(c.Qualifier, []byte(IncrKey)) {
+					v, _ := strconv.Atoi(string(c.Value[:]))
+					if v < 0 {
+						v = 0
+					}
+					incr = v
+				}
+			}
+
+		}
+	}
+	res = &data.UpDataIncrMeta{}
+	res.Incr = incr
+	res.TopAIDList = aids
+	res.TopIncrList = incs
+	res.Rank = rk
+	return
+}
+
+// ThirtyDayArchive for Play/Dm/Reply/Fav/Share/Elec/Coin for archive 30 days.
+func (d *Dao) ThirtyDayArchive(c context.Context, mid int64, ty int8) (res []*data.ThirtyDay, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName   string
+		key         = hbaseMd5Key(mid)
+	)
+	defer cancel()
+	switch {
+	case ty == data.Play:
+		tableName = HBasePlayArc
+	case ty == data.Dm:
+		tableName = HBaseDmArc
+	case ty == data.Reply:
+		tableName = HBaseReplyArc
+	case ty == data.Share:
+		tableName = HBaseShareArc
+	case ty == data.Coin:
+		tableName = HBaseCoinArc
+	case ty == data.Fav:
+		tableName = HBaseFavArc
+	case ty == data.Elec:
+		tableName = HBaseElecArc
+	case ty == data.Like:
+		tableName = HBaseLikeArc
+	}
+	if result, err = d.hbase.GetStr(ctx, tableName, key); err != nil {
+		log.Error("ThirtyDayArchive d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", tableName, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("ThirtyDay no data tableName(%s)|mid(%d)|key(%v)", tableName, mid, key)
+		return
+	}
+	res = make([]*data.ThirtyDay, 0, len(result.Cells))
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		qual := string(c.Qualifier[:])
+		val := string(c.Value[:])
+		if string(c.Family) == "u" {
+			t, v, err := parseKeyValue(qual, val)
+			if err != nil {
+				break
+			}
+			td := &data.ThirtyDay{}
+			td.DateKey = t
+			td.TotalIncr = v
+			res = append(res, td)
+		}
+	}
+	log.Info("ThirtyDayArchive mid(%d) type(%d) return data(%+v)", mid, ty, res)
+	return
+}
+
+func parseKeyValue(k string, v string) (timestamp, value int64, err error) {
+	tm, err := time.Parse("20060102", k)
+	if err != nil {
+		log.Error("time.Parse error(%v)", err)
+		return
+	}
+	timestamp = tm.Unix()
+	value, err = strconv.ParseInt(v, 10, 64)
+	if err != nil {
+		log.Error("strconv.ParseInt error(%v)", err)
+	}
+	return
+}

app/admin/main/up/dao/data/center_test.go

@@ -0,0 +1,178 @@
+package data
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDataViewerBase(t *testing.T) {
+	convey.Convey("ViewerBase", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			dt  = time.Now()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.ViewerBase(c, mid, dt)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataViewerArea(t *testing.T) {
+	convey.Convey("ViewerArea", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			dt  = time.Now()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.ViewerArea(c, mid, dt)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataViewerTrend(t *testing.T) {
+	convey.Convey("ViewerTrend", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			dt  = time.Now()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.ViewerTrend(c, mid, dt)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataRelationFansDay(t *testing.T) {
+	convey.Convey("RelationFansDay", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.RelationFansDay(c, mid)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataRelationFansHistory(t *testing.T) {
+	convey.Convey("RelationFansHistory", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			mid   = int64(0)
+			month = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.RelationFansHistory(c, mid, month)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataRelationFansMonth(t *testing.T) {
+	convey.Convey("RelationFansMonth", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.RelationFansMonth(c, mid)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataViewerActionHour(t *testing.T) {
+	convey.Convey("ViewerActionHour", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			dt  = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.ViewerActionHour(c, mid, dt)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataUpIncr(t *testing.T) {
+	convey.Convey("UpIncr", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			ty  = int8(0)
+			now = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.UpIncr(c, mid, ty, now)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataThirtyDayArchive(t *testing.T) {
+	convey.Convey("ThirtyDayArchive", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			ty  = int8(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.ThirtyDayArchive(c, mid, ty)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataparseKeyValue(t *testing.T) {
+	convey.Convey("parseKeyValue", t, func(ctx convey.C) {
+		var (
+			k = "20060102"
+			v = "123"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			timestamp, value, err := parseKeyValue(k, v)
+			ctx.Convey("Then err should be nil.timestamp,value should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(value, convey.ShouldNotBeNil)
+				ctx.So(timestamp, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/admin/main/up/dao/data/dao.go

@@ -0,0 +1,115 @@
+package data
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/admin/main/up/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+
+	"go-common/library/database/hbase.v2"
+)
+
+// Dao is data dao.
+type Dao struct {
+	c *conf.Config
+	// http
+	client     *bm.Client
+	statClient *bm.Client
+	// uri
+	statURI     string
+	tagURI      string
+	tagV2URI    string
+	coverBFSURI string
+	// mc
+	mc *memcache.Pool
+	//mcExpire    int32
+	//mcIdxExpire int32
+	//statCacheOn bool
+	// hbase
+	hbase        *hbase.Client
+	hbaseTimeOut time.Duration
+	// chan
+	missch chan func()
+}
+
+// New init dao
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c: c,
+		// http
+		client:      bm.NewClient(c.HTTPClient.Slow),
+		statClient:  bm.NewClient(c.HTTPClient.Slow),
+		statURI:     c.Host.Data + _statURL,
+		tagURI:      c.Host.Tag + _tagURL,
+		tagV2URI:    c.Host.Tag + _tagv2URL,
+		coverBFSURI: c.Host.Coverrec + _coverURL,
+		// memcache
+		//mc:          memcache.NewPool(c.Memcache.Data.Config),
+		//mcExpire:    int32(time.Duration(c.Memcache.Data.DataExpire) / time.Second),
+		//mcIdxExpire: int32(time.Duration(c.Memcache.Data.IndexExpire) / time.Second),
+		//statCacheOn: c.StatCacheOn,
+		// hbase
+		hbase: hbase.NewClient(c.HBase.Config),
+		// chan
+		missch:       make(chan func(), 1024),
+		hbaseTimeOut: time.Duration(time.Millisecond * 200),
+	}
+	return
+}
+
+// Stat get user stat play/fans/...
+//func (d *Dao) Stat(c context.Context, ip string, mid int64) (st *data.Stat, err error) {
+//	// try cache
+//	if st, _ = d.statCache(c, mid); st != nil {
+//		return
+//	}
+//	// from api
+//	if st, err = d.stat(c, mid, ip); st != nil {
+//		d.AddCache(func() {
+//			d.addStatCache(context.Background(), mid, st)
+//		})
+//	}
+//	return
+//}
+
+// UpStat get up stat from hbase
+//func (d *Dao) UpStat(c context.Context, mid int64, dt string) (st *data.UpBaseStat, err error) {
+//	// try cache
+//	if d.statCacheOn {
+//		if st, _ = d.upBaseStatCache(c, mid, dt); st != nil {
+//			log.Info("upBaseStatCache cache found mid(%d)", mid)
+//			return
+//		}
+//	}
+//	// from hbase
+//	if st, err = d.BaseUpStat(c, mid, dt); st != nil {
+//		d.AddCache(func() {
+//			d.addUpBaseStatCache(context.Background(), mid, dt, st)
+//		})
+//	}
+//	return
+//}
+
+// Ping ping success.
+func (d *Dao) Ping(c context.Context) (err error) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}); err != nil {
+		log.Error("mc.ping.Store error(%v)", err)
+	}
+	return
+}
+
+// Close mc close
+func (d *Dao) Close() (err error) {
+	if d.mc != nil {
+		d.mc.Close()
+	}
+	if d.hbase != nil {
+		d.hbase.Close()
+	}
+	return
+}

app/admin/main/up/dao/data/dao_test.go

@@ -0,0 +1,49 @@
+package data
+
+import (
+	"flag"
+	"os"
+	"strings"
+	"testing"
+
+	"go-common/app/admin/main/up/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.archive.up-admin")
+		flag.Set("conf_token", "930697bb7def4df0713ef8080596b863")
+		flag.Set("tree_id", "36438")
+		flag.Set("conf_version", "1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/up-admin.toml")
+	}
+	if os.Getenv("UT_LOCAL_TEST") != "" {
+		flag.Set("conf", "../../cmd/up-admin.toml")
+	}
+
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.client.SetTransport(gock.DefaultTransport)
+	os.Exit(m.Run())
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/admin/main/up/dao/data/field.go

@@ -0,0 +1,96 @@
+package data
+
+var (
+	//HBaseVideoTablePrefix 播放流失分布
+	HBaseVideoTablePrefix = "video_play_churn_"
+	//HBaseArchiveTablePrefix 分类分端播放
+	HBaseArchiveTablePrefix = "video_play_category_"
+	//HBaseAreaTablePrefix 地区播放
+	HBaseAreaTablePrefix = "video_play_area_"
+	//HBaseUpStatTablePrefix up主概况
+	HBaseUpStatTablePrefix = "up_stats_"
+	//HBaseUpViewerBase 观众数据,性别年龄分布 + 设备分布
+	HBaseUpViewerBase = "up_viewer_base_"
+	//HBaseUpViewerArea 地区分布
+	HBaseUpViewerArea = "up_viewer_area_"
+	//HBaseUpViewerTrend 内容倾向
+	HBaseUpViewerTrend = "up_viewer_trend_"
+	//HBaseUpViewerActionHour 行为时间分布
+	HBaseUpViewerActionHour = "up_viewer_action_hour_"
+	//HBaseUpRelationFansDay 日维度 最近30天 只保留31天
+	HBaseUpRelationFansDay = "up_relation_fans_day"
+	// HBaseUpRelationFansHistory 日维度 各月份每日数据,日更,永久保存
+	HBaseUpRelationFansHistory = "up_relation_fans_history"
+	//HBaseUpRelationFansMonth 年维度 2017.8月以后的数据永久保存
+	HBaseUpRelationFansMonth = "up_relation_fans_month"
+	//HBaseUpPlayInc 我的概况 播放相关
+	HBaseUpPlayInc = "up_play_inc_"
+	//HBaseUpDmInc 弹幕相关
+	HBaseUpDmInc = "up_dm_inc_"
+	//HBaseUpReplyInc 评论相关
+	HBaseUpReplyInc = "up_reply_inc_"
+	//HBaseUpShareInc 分享相关
+	HBaseUpShareInc = "up_share_inc_"
+	//HBaseUpCoinInc 投币相关
+	HBaseUpCoinInc = "up_coin_inc_"
+	//HBaseUpFavInc 收藏相关
+	HBaseUpFavInc = "up_fav_inc_"
+	//HBaseUpElecInc 充电相关
+	HBaseUpElecInc = "up_elec_inc_"
+	//HBaseUpFansAnalysis  粉丝管理
+	HBaseUpFansAnalysis = "up_fans_analysis"
+	//HBaseUpPlaySourceAnalysis  播放来源
+	HBaseUpPlaySourceAnalysis = "up_play_analysis"
+	//HBaseUpArcPlayAnalysis  平均观看时长、播放用户数、留存率
+	HBaseUpArcPlayAnalysis = "up_archive_play_analysis"
+	//HBaseUpArcQuery  稿件索引表
+	HBaseUpArcQuery = "up_archive_query"
+
+	//HBasePlayArc 播放相关 archive for 30 days
+	HBasePlayArc = "up_play_trend"
+	//HBaseDmArc 弹幕相关
+	HBaseDmArc = "up_dm_trend"
+	//HBaseReplyArc 评论相关
+	HBaseReplyArc = "up_reply_trend"
+	//HBaseShareArc 分享相关
+	HBaseShareArc = "up_share_trend"
+	//HBaseCoinArc 投币相关
+	HBaseCoinArc = "up_coin_trend"
+	//HBaseFavArc 收藏相关
+	HBaseFavArc = "up_fav_trend"
+	//HBaseElecArc 充电相关
+	HBaseElecArc = "up_elec_trend"
+	//HBaseLikeArc 点赞相关
+	HBaseLikeArc = "up_like_trend"
+
+	//HBaseFamilyPlat  family
+	HBaseFamilyPlat = []byte("v")
+	//HBaseColumnAid aid
+	HBaseColumnAid = []byte("avid")
+	//HBaseColumnWebPC pc
+	HBaseColumnWebPC = []byte("plat0")
+	//HBaseColumnWebH5 h5
+	HBaseColumnWebH5 = []byte("plat1")
+	//HBaseColumnOutsite out
+	HBaseColumnOutsite = []byte("plat2")
+	//HBaseColumnIOS ios
+	HBaseColumnIOS = []byte("plat3")
+	//HBaseColumnAndroid android
+	HBaseColumnAndroid = []byte("plat4")
+	//HBaseColumnElse else
+	HBaseColumnElse = []byte("else")
+	//HBaseColumnFans fans
+	HBaseColumnFans = []byte("fans")
+	//HBaseColumnGuest guest
+	HBaseColumnGuest = []byte("guest")
+	//HBaseColumnAll all
+	HBaseColumnAll = []byte("all")
+	//HBaseColumnCoin coin
+	HBaseColumnCoin = []byte("coin")
+	//HBaseColumnElec elec
+	HBaseColumnElec = []byte("elec")
+	//HBaseColumnFav fav
+	HBaseColumnFav = []byte("fav")
+	//HBaseColumnShare share
+	HBaseColumnShare = []byte("share")
+)

app/admin/main/up/dao/data/field_test.go

@@ -0,0 +1 @@
+package data

app/admin/main/up/dao/data/hbase.go

@@ -0,0 +1,226 @@
+package data
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"sort"
+	"strconv"
+	"time"
+
+	"go-common/app/interface/main/creative/model/data"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/tsuna/gohbase/hrpc"
+	"golang.org/x/net/context"
+)
+
+func hbaseMd5Key(aid int64) string {
+	hasher := md5.New()
+	hasher.Write([]byte(strconv.Itoa(int(aid))))
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+// VideoQuitPoints get video quit points.
+func (d *Dao) VideoQuitPoints(c context.Context, cid int64) (res []int64, err error) {
+	var (
+		ctx, cancel     = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName       = HBaseVideoTablePrefix + time.Now().AddDate(0, 0, -1).Add(-12*time.Hour).Format("20060102") // change table at 12:00am
+		backupTableName = HBaseVideoTablePrefix + time.Now().AddDate(0, 0, -2).Add(-12*time.Hour).Format("20060102") // change table at 12:00am
+		key             = hbaseMd5Key(cid)
+	)
+	defer cancel()
+	result, err := d.hbase.GetStr(ctx, tableName, key)
+	if err != nil {
+		if result, err = d.hbase.GetStr(ctx, backupTableName, key); err != nil {
+			log.Error("VideoQuitPoints d.hbase.GetStr backupTableName(%s)|cid(%d)|key(%v)|error(%v)", backupTableName, cid, key, err)
+			err = ecode.CreativeDataErr
+			return
+		}
+	}
+	if result == nil {
+		return
+	}
+	// get parts and max part for fill res
+	partMap := make(map[int]int64)
+	maxPart := 0
+	for _, c := range result.Cells {
+		if c != nil {
+			part, _ := strconv.Atoi(string(c.Qualifier[:]))
+			per, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+			partMap[part] = per
+			if part > maxPart {
+				maxPart = part
+			}
+		}
+	}
+	var restPercent int64 = 10000 // start from 100%
+	for i := 1; i <= maxPart; i++ {
+		if _, ok := partMap[i]; ok {
+			restPercent = restPercent - partMap[i]
+		}
+		res = append(res, restPercent)
+	}
+	return
+}
+
+// ArchiveStat get the stat of archive.
+func (d *Dao) ArchiveStat(c context.Context, aid int64) (stat *data.ArchiveData, err error) {
+	var (
+		ctx, cancel     = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName       = HBaseArchiveTablePrefix + time.Now().AddDate(0, 0, -1).Add(-12*time.Hour).Format("20060102") // change table at 12:00am
+		backupTableName = HBaseArchiveTablePrefix + time.Now().AddDate(0, 0, -2).Add(-12*time.Hour).Format("20060102") // change table at 12:00am
+		key             = hbaseMd5Key(aid)
+	)
+	defer cancel()
+	result, err := d.hbase.GetStr(ctx, tableName, key)
+	if err != nil {
+		if result, err = d.hbase.GetStr(ctx, backupTableName, key); err != nil {
+			log.Error("ArchiveStat d.hbase.GetStr backupTableName(%s)|aid(%d)|key(%v)|error(%v)", backupTableName, aid, key, err)
+			err = ecode.CreativeDataErr
+			return
+		}
+	}
+	if result == nil {
+		return
+	}
+	stat = &data.ArchiveData{}
+	stat.ArchiveSource = &data.ArchiveSource{}
+	stat.ArchiveGroup = &data.ArchiveGroup{}
+	stat.ArchiveStat = &data.ArchiveStat{}
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+		if !bytes.Equal(c.Family, HBaseFamilyPlat) {
+			continue
+		}
+		switch {
+		case bytes.Equal(c.Qualifier, HBaseColumnWebPC):
+			stat.ArchiveSource.WebPC = v
+		case bytes.Equal(c.Qualifier, HBaseColumnWebH5):
+			stat.ArchiveSource.WebH5 = v
+		case bytes.Equal(c.Qualifier, HBaseColumnOutsite):
+			stat.ArchiveSource.Outsite = v
+		case bytes.Equal(c.Qualifier, HBaseColumnIOS):
+			stat.ArchiveSource.IOS = v
+		case bytes.Equal(c.Qualifier, HBaseColumnAndroid):
+			stat.ArchiveSource.Android = v
+		case bytes.Equal(c.Qualifier, HBaseColumnElse):
+			stat.ArchiveSource.Others = v
+		case bytes.Equal(c.Qualifier, HBaseColumnFans):
+			stat.ArchiveGroup.Fans = v
+		case bytes.Equal(c.Qualifier, HBaseColumnGuest):
+			stat.ArchiveGroup.Guest = v
+		case bytes.Equal(c.Qualifier, HBaseColumnAll):
+			stat.ArchiveStat.Play = v
+		case bytes.Equal(c.Qualifier, HBaseColumnCoin):
+			stat.ArchiveStat.Coin = v
+		case bytes.Equal(c.Qualifier, HBaseColumnElec):
+			stat.ArchiveStat.Elec = v
+		case bytes.Equal(c.Qualifier, HBaseColumnFav):
+			stat.ArchiveStat.Fav = v
+		case bytes.Equal(c.Qualifier, HBaseColumnShare):
+			stat.ArchiveStat.Share = v
+		}
+	}
+	stat.ArchiveSource.Mainsite = stat.ArchiveSource.WebPC + stat.ArchiveSource.WebH5
+	stat.ArchiveSource.Mobile = stat.ArchiveSource.Android + stat.ArchiveSource.IOS
+	return
+}
+
+// ArchiveArea get the count of area.
+func (d *Dao) ArchiveArea(c context.Context, aid int64) (res []*data.ArchiveArea, err error) {
+	var (
+		ctx, cancel     = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName       = HBaseAreaTablePrefix + time.Now().AddDate(0, 0, -1).Add(-12*time.Hour).Format("20060102") // change table at 12:00am
+		backupTableName = HBaseAreaTablePrefix + time.Now().AddDate(0, 0, -2).Add(-12*time.Hour).Format("20060102") // change table at 12:00am
+		key             = hbaseMd5Key(aid)
+	)
+	defer cancel()
+	result, err := d.hbase.GetStr(ctx, tableName, key)
+	if err != nil {
+		if result, err = d.hbase.GetStr(ctx, backupTableName, key); err != nil {
+			log.Error("ArchiveArea d.hbase.GetStr backupTableName(%s)|aid(%d)|key(%v)|error(%v)", backupTableName, aid, key, err)
+			err = ecode.CreativeDataErr
+			return
+		}
+	}
+	if result == nil {
+		return
+	}
+	var countArr []int
+	countMap := make(map[int64][]*data.ArchiveArea)
+	countSet := make(map[int]struct{}) // empty struct{} for saving memory
+	for _, c := range result.Cells {
+		if c != nil {
+			area := &data.ArchiveArea{}
+			area.Location = string(c.Qualifier[:])
+			area.Count, _ = strconv.ParseInt(string(c.Value[:]), 10, 64)
+			countMap[area.Count] = append(countMap[area.Count], area)
+			countSet[int(area.Count)] = struct{}{}
+		}
+	}
+	for key := range countSet {
+		countArr = append(countArr, key)
+	}
+	sort.Sort(sort.Reverse(sort.IntSlice(countArr)))
+	for _, c := range countArr {
+		if _, ok := countMap[int64(c)]; ok {
+			res = append(res, countMap[int64(c)]...)
+		}
+		if len(res) >= 10 {
+			res = res[:10] // exact 10 item
+			break
+		}
+	}
+	return
+}
+
+// BaseUpStat get base up stat.
+func (d *Dao) BaseUpStat(c context.Context, mid int64, date string) (stat *data.UpBaseStat, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, d.hbaseTimeOut)
+		tableName   = HBaseUpStatTablePrefix + date // change table at 12:00am
+		key         = hbaseMd5Key(mid)
+	)
+	defer cancel()
+	if result, err = d.hbase.GetStr(ctx, tableName, key); err != nil {
+		log.Error("BaseUpStat d.hbase.GetStr tableName(%s)|mid(%d)|key(%v)|error(%v)", tableName, mid, key, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil {
+		return
+	}
+	stat = &data.UpBaseStat{}
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		v, _ := strconv.ParseInt(string(c.Value[:]), 10, 64)
+		if !bytes.Equal(c.Family, []byte("u")) {
+			continue
+		}
+		switch {
+		case bytes.Equal(c.Qualifier, []byte("play")):
+			stat.View = v
+		case bytes.Equal(c.Qualifier, []byte("dm")):
+			stat.Dm = v
+		case bytes.Equal(c.Qualifier, []byte("reply")):
+			stat.Reply = v
+		case bytes.Equal(c.Qualifier, []byte("fans")):
+			stat.Fans = v
+		case bytes.Equal(c.Qualifier, []byte("fav")):
+			stat.Fav = v
+		case bytes.Equal(c.Qualifier, []byte("like")):
+			stat.Like = v
+		case bytes.Equal(c.Qualifier, []byte("sh")):
+			stat.Share = v
+		}
+	}
+	return
+}

app/admin/main/up/dao/data/hbase_set2.go

@@ -0,0 +1,148 @@
+package data
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"go-common/app/admin/main/up/model/datamodel"
+	"go-common/app/admin/main/up/util/hbaseutil"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+// 这里是集群2的数据请求
+const (
+	// HbaseUpArchiveInfoPrefix archive info表
+	HbaseUpArchiveInfoPrefix = "upcrm:up_influence"
+	// HbaseUpArchiveTagInfoPrefix tag表
+	HbaseUpArchiveTagInfoPrefix = "upcrm:up_archive_tag_info"
+	// HbaseUpArchiveTypeInfoPrefix 分区表
+	HbaseUpArchiveTypeInfoPrefix = "upcrm:up_archive_type_info"
+)
+
+var (
+	//ErrInvalidDataType invalid data type
+	ErrInvalidDataType = fmt.Errorf("invalid data type")
+)
+
+//UpArchiveDataType data type
+type UpArchiveDataType int
+
+const (
+	//DataTypeDay7 7 day
+	DataTypeDay7 UpArchiveDataType = 1
+	//DataTypeDay30 30 day
+	DataTypeDay30 UpArchiveDataType = 2
+	//DataTypeDay90 90 day
+	DataTypeDay90 UpArchiveDataType = 3
+	//DataTypeDay180 180 day
+	DataTypeDay180 UpArchiveDataType = 4
+	//DataTypeDayAll accumulated
+	DataTypeDayAll UpArchiveDataType = 5
+)
+
+var (
+	dataType2FamilyMap = map[UpArchiveDataType]string{
+		DataTypeDay7:   "d7",
+		DataTypeDay30:  "d30",
+		DataTypeDay90:  "d90",
+		DataTypeDay180: "d180",
+		DataTypeDayAll: "dall",
+	}
+)
+
+//UpArchiveInfo get up archive info
+func (d *Dao) UpArchiveInfo(c context.Context, mids []int64, dataType UpArchiveDataType) (result map[int64]*datamodel.UpArchiveData, err error) {
+	var now = time.Now()
+	var date = now.AddDate(0, 0, -1).Add(-12 * time.Hour).Format("20060102")
+	var tableName = generateTableName(HbaseUpArchiveInfoPrefix, date)
+
+	var family, ok = dataType2FamilyMap[dataType]
+	if !ok {
+		log.Error("invalid data type, type=%d", dataType)
+		err = ErrInvalidDataType
+		return
+	}
+	var group, ctx = errgroup.WithContext(c)
+	var lock sync.Mutex
+	result = make(map[int64]*datamodel.UpArchiveData)
+	for _, mid := range mids {
+		var copymid = mid
+		group.Go(
+			func() error {
+				var data = &datamodel.UpArchiveData{}
+				var key = hbaseMd5Key(copymid)
+				if e := d.getHbaseRowResult(ctx, tableName, key, data, hrpc.Families(map[string][]string{family: nil})); e != nil {
+					log.Error("get up archive info fail, mid=%d, err=%v", err)
+					return nil
+				}
+				lock.Lock()
+				result[copymid] = data
+				lock.Unlock()
+				return nil
+			})
+	}
+
+	if err = group.Wait(); err != nil {
+		log.Error("batch get fail, err=%v", err)
+		return
+	}
+	log.Info("get archive info ok, find result count=%d", len(result))
+	return
+}
+
+//UpArchiveTagInfo get up archive tag info
+func (d *Dao) UpArchiveTagInfo(c context.Context, mid int64) (result datamodel.UpArchiveTagData, err error) {
+	var now = time.Now()
+	var date = now.AddDate(0, 0, -1).Add(-12 * time.Hour).Format("20060102")
+	var tableName = generateTableName(HbaseUpArchiveTagInfoPrefix, date)
+	var key = hbaseMd5Key(mid)
+
+	if err = d.getHbaseRowResult(c, tableName, key, &result); err != nil {
+		log.Error("get up archive tag info fail, err=%v", err)
+	}
+	return
+}
+
+//UpArchiveTypeInfo get up archive type info
+func (d *Dao) UpArchiveTypeInfo(c context.Context, mid int64) (result datamodel.UpArchiveTypeData, err error) {
+	var now = time.Now()
+	var date = now.AddDate(0, 0, -1).Add(-12 * time.Hour).Format("20060102")
+	var tableName = generateTableName(HbaseUpArchiveTypeInfoPrefix, date)
+	var key = hbaseMd5Key(mid)
+	if err = d.getHbaseRowResult(c, tableName, key, &result); err != nil {
+		log.Error("get up archive type info fail, err=%v", err)
+	}
+	return
+}
+
+func (d *Dao) getHbaseRowResult(c context.Context, table, key string, result interface{}, options ...func(hrpc.Call) error) (err error) {
+	var ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+	defer cancel()
+	res, err := d.hbase.GetStr(ctx, table, key, options...)
+	if err != nil {
+		log.Error("fail to get data from hbase, table=%s, key=%s, err=%v", table, key, err)
+		return
+	}
+
+	if len(res.Cells) == 0 {
+		log.Warn("no cells get, table=%s, key=%s, err=%v", table, key, err)
+		return
+	}
+
+	var parser = hbaseutil.Parser{}
+	if err = parser.Parse(res.Cells, result); err != nil {
+		log.Error("parse data fail, table=%s, key=%s, err=%v", table, key, err)
+		return
+	}
+	return
+}
+
+// dont use date string
+func generateTableName(prefix, date string) string {
+	return prefix
+}

app/admin/main/up/dao/data/hbase_set2_test.go

@@ -0,0 +1,102 @@
+package data
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/bouk/monkey"
+	"github.com/tsuna/gohbase/hrpc"
+
+	"go-common/app/admin/main/up/model/datamodel"
+
+	hbase "go-common/library/database/hbase.v2"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDataUpArchiveInfo(t *testing.T) {
+	convey.Convey("UpArchiveInfo", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			mids     = []int64{}
+			dataType UpArchiveDataType
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			result, err := d.UpArchiveInfo(c, mids, dataType)
+			ctx.Convey("Then err should be nil.result should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(result, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataUpArchiveTagInfo(t *testing.T) {
+	convey.Convey("UpArchiveTagInfo", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			result, err := d.UpArchiveTagInfo(c, mid)
+			ctx.Convey("Then err should be nil.result should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(result, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataUpArchiveTypeInfo(t *testing.T) {
+	convey.Convey("UpArchiveTypeInfo", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			result, err := d.UpArchiveTypeInfo(c, mid)
+			ctx.Convey("Then err should be nil.result should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(result, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDatagetHbaseRowResult(t *testing.T) {
+	convey.Convey("getHbaseRowResult", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			table  = "crm"
+			key    = "123"
+			result = datamodel.UpArchiveTypeData{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			guard := monkey.PatchInstanceMethod(reflect.TypeOf(d.hbase), "GetStr", func(_ *hbase.Client, _ context.Context, _, _ string, _ ...func(hrpc.Call) error) (*hrpc.Result, error) {
+				res := &hrpc.Result{}
+				return res, nil
+			})
+			defer guard.Unpatch()
+			err := d.getHbaseRowResult(c, table, key, &result)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDatagenerateTableName(t *testing.T) {
+	convey.Convey("generateTableName", t, func(ctx convey.C) {
+		var (
+			prefix = ""
+			date   = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := generateTableName(prefix, date)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/admin/main/up/dao/data/hbase_test.go

@@ -0,0 +1,87 @@
+package data
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDatahbaseMd5Key(t *testing.T) {
+	convey.Convey("hbaseMd5Key", t, func(ctx convey.C) {
+		var (
+			aid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := hbaseMd5Key(aid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataVideoQuitPoints(t *testing.T) {
+	convey.Convey("VideoQuitPoints", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			cid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.VideoQuitPoints(c, cid)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataArchiveStat(t *testing.T) {
+	convey.Convey("ArchiveStat", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			aid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			stat, err := d.ArchiveStat(c, aid)
+			ctx.Convey("Then err should be nil.stat should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(stat, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataArchiveArea(t *testing.T) {
+	convey.Convey("ArchiveArea", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			aid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.ArchiveArea(c, aid)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataBaseUpStat(t *testing.T) {
+	convey.Convey("BaseUpStat", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			mid  = int64(0)
+			date = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			stat, err := d.BaseUpStat(c, mid, date)
+			ctx.Convey("Then err should be nil.stat should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(stat, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/admin/main/up/dao/data/phase_three.go

@@ -0,0 +1,227 @@
+package data
+
+import (
+	"context"
+	"encoding/binary"
+	"strconv"
+	"time"
+
+	"go-common/app/admin/main/up/model/datamodel"
+	"go-common/app/admin/main/up/util/hbaseutil"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+// reverse for string.
+func reverseString(s string) string {
+	rs := []rune(s)
+	l := len(rs)
+	for f, t := 0, l-1; f < t; f, t = f+1, t-1 {
+		rs[f], rs[t] = rs[t], rs[f]
+	}
+	ns := string(rs)
+	if l < 10 {
+		for i := 0; i < 10-l; i++ {
+			ns = ns + "0"
+		}
+	}
+	return ns
+}
+
+// 粉丝管理 - up_fans_analysis
+// mid倒置补0 （补满共10位）+0 +yyyyMMdd          – 累计数据
+// mid倒置补0 （补满共10位）+1 +yyyyMMdd          – 7日数据
+// mid倒置补0 （补满共10位）+2 +yyyyMMdd        – 30日数据
+// mid倒置补0 （补满共10位）+3 +yyyyMMdd        – 90日数据
+func fansRowKey(id int64, ty int) string {
+	idStr := strconv.FormatInt(id, 10)
+	s := reverseString(idStr)
+	s = s + strconv.Itoa(ty) + time.Now().AddDate(0, 0, -1).Add(-12*time.Hour).Format("20060102")
+	return s
+}
+
+// 稿件索引表 - up_archive_query
+// [mid倒置补0（10位）] + [投稿年月（4位）] + [原创/转载（1位）]
+func arcQueryKey(id int64, dt string, cp int) string {
+	idStr := strconv.FormatInt(id, 10)
+	cpStr := strconv.Itoa(cp)
+	s := reverseString(idStr)
+	s = s + dt + cpStr
+	return s
+}
+
+// UpFansAnalysis for web fans analysis.
+func (d *Dao) UpFansAnalysis(c context.Context, mid int64, ty int) (res *datamodel.FanInfo, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName   = HBaseUpFansAnalysis
+		//summary      = make(map[string]int32)            //total & classify proportion
+		//rankDr       = make(map[string]int32)            //播放时长 mid list
+		//rankVideoAct = make(map[string]int32)            //视频互动 mid list
+		//rankDyAct    = make(map[string]int32)            //动态互动 mid list
+		rankMap = make(map[string]map[string]int32) //top 10 rank list
+		source  = make(map[string]int32)            //source  proportion
+		rowkey  = fansRowKey(mid, ty)
+		parser  = hbaseutil.Parser{}
+	)
+	res = new(datamodel.FanInfo)
+	defer cancel()
+	log.Info("UpFansAnalysisForWeb mid(%d)|rowkey(%s)", mid, rowkey)
+	if result, err = d.hbase.GetStr(ctx, tableName, rowkey, hrpc.Families(map[string][]string{"f": nil})); err != nil {
+		log.Error("d.hbase.GetStr tableName(%s)|mid(%d)|rowkey(%v)|error(%v)", tableName, mid, rowkey, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("UpFansAnalysisForWeb no data tableName(%s)|mid(%d)|rowkey(%v)", tableName, mid, rowkey)
+		return
+	}
+	var summary datamodel.FanSummaryData
+	err = parser.Parse(result.Cells, &summary)
+	if err != nil {
+		log.Error("convert from hbase fail, err=%v", err)
+		return
+	}
+
+	/*
+		for _, c := range result.Cells {
+			if c == nil {
+				continue
+			}
+			if string(c.Family) == "f" {
+
+				k := string(c.Qualifier[:])
+				var v int32
+				if len(c.Value) == 4 {
+					v = byteToInt32(c.Value)
+					switch k {
+					case "all": //总粉丝
+						summary.Total = v
+						//summary["total"] = v
+					case "inc": //新增粉丝
+						summary.Inc = v
+					case "act": //活跃粉丝
+						summary.Active = v
+						//	summary["active"] = v
+						//case "mdl": //领取勋章粉丝
+						//	summary["medal"] = v
+						//case "elec": //充电粉丝
+						//	summary["elec"] = v
+						//case "act_diff": //活跃粉丝（增量）
+						//	summary["act_diff"] = v
+						//case "mdl_diff": //领取勋章粉丝（增量）
+						//	summary["mdl_diff"] = v
+						//case "elec_diff": //充电粉丝（增量）
+						//	summary["elec_diff"] = v
+						//case "v": //播放粉丝占比*10000
+						//	summary["v"] = v
+						//case "dm": //弹幕粉丝占比*10000
+						//	summary["dm"] = v
+						//case "r": //评论粉丝占比*10000
+						//	summary["r"] = v
+						//case "c": //投币粉丝占比*10000
+						//	summary["c"] = v
+						//	//新增粉丝活跃占比
+						//case "inter": //互动活跃度*10000
+						//	summary["inter"] = v
+						//case "vv": //观看活跃度*10000
+						//	summary["vv"] = v
+						//case "da": //弹幕粉丝占比*10000
+						//	summary["da"] = v
+						//case "re": //评论粉丝占比*10000
+						//	summary["re"] = v
+						//case "co": //投币粉丝占比*10000
+						//	summary["co"] = v
+						//case "fv": //收藏粉丝占比*10000
+						//	summary["fv"] = v
+						//case "sh": //分享粉丝占比*10000
+						//	summary["sh"] = v
+						//case "lk": //点赞粉丝占比*10000
+						//	summary["lk"] = v
+					}
+				} else {
+					log.Error("UpFansAnalysisForWeb family[f] get dirty tableName(%s)|mid(%d)|rowkey(%s)|value(%+v)", tableName, mid, rowkey, c.Value)
+				}
+			}
+			if string(c.Family) == "t" {
+				var v int32
+				if len(c.Value) == 4 {
+					v = int32(binary.BigEndian.Uint32(c.Value))
+				} else {
+					log.Error("UpFansAnalysisForWeb family t get dirty tableName(%s)|mid(%d)|rowkey(%s)|value(%+v)", tableName, mid, fansRowKey(mid, ty), c.Value)
+				}
+				if strings.Contains(string(c.Qualifier[:]), "dr") {
+					rankDr[string(c.Qualifier[:])] = v
+				}
+				if strings.Contains(string(c.Qualifier[:]), "act") {
+					rankVideoAct[string(c.Qualifier[:])] = v
+				}
+				if strings.Contains(string(c.Qualifier[:]), "dy") {
+					rankDyAct[string(c.Qualifier[:])] = v
+				}
+			}
+			if string(c.Family) == "s" {
+				var v int32
+				if len(c.Value) == 4 {
+					v = int32(binary.BigEndian.Uint32(c.Value))
+				} else {
+					log.Error("UpFansAnalysisForWeb family[t] get dirty data tableName(%s)|mid(%d)|rowkey(%s)|value(%+v)", tableName, mid, rowkey, c.Value)
+				}
+				if strings.Contains(string(c.Qualifier[:]), "pf") {
+					if pk, ok := fanSource[string(c.Qualifier[:])]; ok {
+						source[pk] = v
+					}
+				}
+			}
+		}
+		for _, k := range fanSource { //粉丝来源页面占比如果数据平台未返回,则设置对应的key
+			if _, ok := source[k]; !ok {
+				source[k] = 0
+			}
+		}
+	*/
+	log.Info("UpFansAnalysisForWebRankMap mid(%d)|rowkey(%s)|summary(%+v)|rankMap(%+v)|source(%+v)", mid, rowkey, summary, rankMap, source)
+	res.Summary = summary
+	return
+}
+
+// UpArcQuery for play aids by mid.
+func (d *Dao) UpArcQuery(c context.Context, mid int64, dt string, cp int) (res []int64, err error) {
+	var (
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.ReadTimeout))
+		tableName   = HBaseUpArcQuery
+		rowkey      = arcQueryKey(mid, dt, cp)
+	)
+	defer cancel()
+	log.Info("UpArcQuery mid(%d)|rowkey(%s)", mid, rowkey)
+	if result, err = d.hbase.GetStr(ctx, tableName, rowkey); err != nil {
+		log.Error("d.hbase.GetStr tableName(%s)|mid(%d)|rowkey(%s)|error(%v)", tableName, mid, rowkey, err)
+		err = ecode.CreativeDataErr
+		return
+	}
+	if result == nil || len(result.Cells) == 0 {
+		log.Warn("UpArcQuery no data tableName(%s)|mid(%d)|rowkey(%s)", tableName, mid, rowkey)
+		return
+	}
+	res = make([]int64, 0)
+	for _, c := range result.Cells {
+		if c == nil {
+			continue
+		}
+		if string(c.Family) == "av" {
+			var v int64
+			if len(c.Value) == 4 {
+				v = int64(binary.BigEndian.Uint32(c.Value))
+			} else {
+				log.Error("UpArcQuery family[av] get dirty value  tableName(%s)|rowkey(%s)", tableName, rowkey)
+			}
+			res = append(res, v)
+		}
+	}
+	log.Info("UpArcQuery mid(%d)|rowkey(%s)|res(%+v)", mid, rowkey, res)
+	return
+}

app/admin/main/up/dao/data/phase_three_test.go

@@ -0,0 +1,88 @@
+package data
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDatareverseString(t *testing.T) {
+	convey.Convey("reverseString", t, func(ctx convey.C) {
+		var (
+			s = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := reverseString(s)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDatafansRowKey(t *testing.T) {
+	convey.Convey("fansRowKey", t, func(ctx convey.C) {
+		var (
+			id = int64(0)
+			ty = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := fansRowKey(id, ty)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataarcQueryKey(t *testing.T) {
+	convey.Convey("arcQueryKey", t, func(ctx convey.C) {
+		var (
+			id = int64(0)
+			dt = ""
+			cp = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := arcQueryKey(id, dt, cp)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataUpFansAnalysis(t *testing.T) {
+	convey.Convey("UpFansAnalysis", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			ty  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.UpFansAnalysis(c, mid, ty)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataUpArcQuery(t *testing.T) {
+	convey.Convey("UpArcQuery", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			mid = int64(0)
+			dt  = ""
+			cp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.UpArcQuery(c, mid, dt, cp)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+				ctx.So(res, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/admin/main/up/dao/data/utils.go

@@ -0,0 +1,41 @@
+package data
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/library/log"
+
+	"go-common/library/database/hbase.v2"
+
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+func (d *Dao) getDataWithBackup(c context.Context, client *hbase.Client, tableNameFunc func(retryCount int) string, maxRetry int, key string, options ...func(hrpc.Call) error) (result *hrpc.Result, err error) {
+	if client == nil {
+		err = fmt.Errorf("hbase client is nil")
+		return
+	}
+
+	for i := 0; i < maxRetry; i++ {
+		var tableName = tableNameFunc(i)
+		if result, err = d.hbase.GetStr(c, tableName, key, options...); err != nil {
+			log.Error("hbase GetStr tableName(%s)|key(%v)|error(%v)", tableName, key, err)
+			continue
+		}
+		break
+	}
+	return
+}
+
+func getTableName(tablePrefix string, date time.Time) string {
+	return tablePrefix + date.Format("20060102")
+}
+
+func generateTableNameFunc(tablePrefix string, date time.Time, dayDiff int) func(retryCount int) string {
+	return func(retryCount int) string {
+		var backdate = date.AddDate(0, 0, dayDiff*retryCount)
+		return getTableName(tablePrefix, backdate)
+	}
+}

app/admin/main/up/dao/data/utils_test.go

@@ -0,0 +1,63 @@
+package data
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go-common/library/database/hbase.v2"
+
+	"github.com/smartystreets/goconvey/convey"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+func TestDatagetDataWithBackup(t *testing.T) {
+	convey.Convey("getDataWithBackup", t, func(ctx convey.C) {
+		var (
+			c             = context.Background()
+			client        = &hbase.Client{}
+			tableNameFunc func(retryCount int) string
+			maxRetry      = int(0)
+			key           = ""
+			options       func(hrpc.Call) error
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			result, err := d.getDataWithBackup(c, client, tableNameFunc, maxRetry, key, options)
+			ctx.Convey("Then err should be nil.result should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(result, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDatagetTableName(t *testing.T) {
+	convey.Convey("getTableName", t, func(ctx convey.C) {
+		var (
+			tablePrefix = ""
+			date        = time.Now()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := getTableName(tablePrefix, date)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDatagenerateTableNameFunc(t *testing.T) {
+	convey.Convey("generateTableNameFunc", t, func(ctx convey.C) {
+		var (
+			tablePrefix = ""
+			date        = time.Now()
+			dayDiff     = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := generateTableNameFunc(tablePrefix, date, dayDiff)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}