Create & Init Project...

app/service/main/dynamic/dao/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bigdata_test.go",
+        "dao_test.go",
+        "live_test.go",
+        "memcache_test.go",
+        "tag_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/dynamic/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bigdata.go",
+        "dao.go",
+        "live.go",
+        "memcache.go",
+        "tag.go",
+    ],
+    importpath = "go-common/app/service/main/dynamic/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/dynamic/conf:go_default_library",
+        "//app/service/main/dynamic/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/dynamic/dao/bigdata.go

@@ -0,0 +1,61 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+
+	"go-common/app/service/main/dynamic/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// RegionArcs get new dynamic by bigData API.
+func (d *Dao) RegionArcs(c context.Context, rid int32, remoteIP string) (aids []int64, total int, err error) {
+	params := url.Values{}
+	params.Set("rid", strconv.FormatInt(int64(rid), 10))
+	var res struct {
+		Code int            `json:"code"`
+		Data model.AvidData `json:"data"`
+	}
+	if err = d.httpR.Get(c, d.regionURI, remoteIP, params, &res); err != nil {
+		PromError("大数据分区接口", "d.httpR.Get(%s) error(%+v)", d.regionURI+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		PromError("大数据分区接口", "dynamic url(%s) res code(%d) or res.data(%v)", d.regionURI+"?"+params.Encode(), res.Code, res.Data)
+		err = ecode.Int(res.Code)
+		return
+	}
+	if len(res.Data.Avid) == 0 {
+		log.Error("dynamic url(%s) res(%v)", d.regionURI+"?"+params.Encode(), res)
+	}
+	aids = res.Data.Avid
+	total = res.Data.Count
+	return
+}
+
+// RegionTagArcs get new dynamic by bigData API.
+func (d *Dao) RegionTagArcs(c context.Context, rid int32, tagID int64, remoteIP string) (aids []int64, err error) {
+	params := url.Values{}
+	params.Set("rid", strconv.FormatInt(int64(rid), 10))
+	params.Set("tag_id", strconv.FormatInt(tagID, 10))
+	var res struct {
+		Code int            `json:"code"`
+		Data model.AvidData `json:"data"`
+	}
+	if err = d.httpR.Get(c, d.regionTagURI, remoteIP, params, &res); err != nil {
+		PromError("大数据Tag接口", "d.httpR.Get(%s) error(%+v)", d.regionTagURI+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		PromError("大数据Tag接口", "dynamic url(%s) res code(%d) or res.data(%v)", d.regionTagURI+"?"+params.Encode(), res.Code, res.Data)
+		err = ecode.Int(res.Code)
+		return
+	}
+	if len(res.Data.Avid) == 0 {
+		log.Error("dynamic url(%s) res(%v)", d.regionTagURI+"?"+params.Encode(), res)
+	}
+	aids = res.Data.Avid
+	return
+}

app/service/main/dynamic/dao/bigdata_test.go

@@ -0,0 +1,56 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoRegionArcs(t *testing.T) {
+	convey.Convey("RegionArcs", t, func() {
+		var (
+			c        = context.Background()
+			rid      = int32(1)
+			remoteIP = "0.0.0.0"
+		)
+		convey.Convey("When http request gets 404 error", func(ctx convey.C) {
+			httpMock("GET", d.regionURI).MatchParam("rid", "0").Reply(502).JSON(`{"code":-500}`)
+			_, _, err := d.RegionArcs(c, rid, remoteIP)
+			ctx.Convey("Then error should not be nil", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+			})
+		})
+
+		convey.Convey("When http request gets code == 0", func(ctx convey.C) {
+			httpMock("GET", d.regionURI).MatchParam("rid", "1").Reply(200).JSON(`{"code":0,"message":"ok","data":{"avid":[28868622,29365735,29621166],"count":4602378}}`)
+			aids, total, err := d.RegionArcs(c, rid, remoteIP)
+			ctx.Convey("Then error should not be nil", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(total, convey.ShouldBeGreaterThan, 0)
+				ctx.So(len(aids), convey.ShouldBeGreaterThan, 0)
+			})
+		})
+	})
+}
+
+func TestDaoRegionTagArcs(t *testing.T) {
+	convey.Convey("RegionTagArcs", t, func() {
+		var (
+			c        = context.Background()
+			rid      = int32(136)
+			tagID    = int64(4577)
+			remoteIP = "0.0.0.0"
+			params   = map[string]string{"rid": "136", "tag_id": "4577"}
+		)
+		convey.Convey("When http request gets code == 0", func(ctx convey.C) {
+			httpMock("GET", d.regionTagURI).MatchParams(params).Reply(200).JSON(`{"code":0,"message":"ok","data":{"avid":[28868622,29365735,29621166],"count":4602378}}`)
+			aids, err := d.RegionTagArcs(c, rid, tagID, remoteIP)
+			ctx.Convey("Then error should not be nil", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(len(aids), convey.ShouldBeGreaterThan, 0)
+			})
+		})
+	})
+
+}

app/service/main/dynamic/dao/dao.go

@@ -0,0 +1,75 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/service/main/dynamic/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/stat/prom"
+)
+
+const (
+	_regionURL    = "/dynamic/region"
+	_regionTagURL = "/dynamic/tag"
+	_liveURL      = "/room/v1/Area/dynamic"
+	_hotURL       = "/x/internal/tag/hotmap"
+	_pridURL      = "/x/internal/tag/prids"
+)
+
+// PromError stat and log.
+func PromError(name string, format string, args ...interface{}) {
+	prom.BusinessErrCount.Incr(name)
+	log.Error(format, args...)
+}
+
+// Dao dao.
+type Dao struct {
+	// http
+	httpR *bm.Client
+	// bigData api
+	regionURI    string
+	regionTagURI string
+	// live api
+	liveURI string
+	// tag api
+	hotURI  string
+	pridURI string
+	// memcache
+	mc       *memcache.Pool
+	mcExpire int32
+	// cache Prom
+	cacheProm *prom.Prom
+}
+
+// New dao new.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		httpR:        bm.NewClient(c.HTTPClient.Read),
+		regionURI:    c.Host.BigDataURI + _regionURL,
+		regionTagURI: c.Host.BigDataURI + _regionTagURL,
+		liveURI:      c.Host.LiveURI + _liveURL,
+		hotURI:       c.Host.APIURI + _hotURL,
+		pridURI:      c.Host.APIURI + _pridURL,
+		// memcache
+		mc:       memcache.NewPool(c.Memcache.Config),
+		mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
+	}
+	d.cacheProm = prom.CacheHit
+	return
+}
+
+// Ping check connection success.
+func (dao *Dao) Ping(c context.Context) (err error) {
+	err = dao.pingMC(c)
+	return
+}
+
+// Close close memcache resource.
+func (dao *Dao) Close() {
+	if dao.mc != nil {
+		dao.mc.Close()
+	}
+}

app/service/main/dynamic/dao/dao_test.go

@@ -0,0 +1,46 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"strings"
+	"testing"
+
+	"go-common/app/service/main/dynamic/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.dynamic-service")
+		flag.Set("conf_token", "f7c6b053ef986c6d3c87b4fd69e64ddb")
+		flag.Set("tree_id", "2316")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/dynamic-service-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.httpR.SetTransport(gock.DefaultTransport)
+	m.Run()
+	os.Exit(0)
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/service/main/dynamic/dao/live.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"context"
+	"net/http"
+
+	"go-common/library/ecode"
+)
+
+// Live gets live dynamic count
+func (d *Dao) Live(c context.Context) (count int, err error) {
+	var req *http.Request
+	if req, err = d.httpR.NewRequest("GET", d.liveURI, "", nil); err != nil {
+		PromError("直播Live接口", "Live d.httpR.NewRequest(%s) error(%v)", d.liveURI, err)
+		return
+	}
+	var res struct {
+		Code int                 `json:"code"`
+		Msg  string              `json:"msg"`
+		Data struct{ Count int } `json:"data"`
+	}
+	err = d.httpR.Do(c, req, &res)
+	if err != nil {
+		PromError("直播Live接口", "Live d.httpR.Do(%s) error(%v)", d.liveURI, err)
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		PromError("直播Live接口", "Live dao.httpR.Do(%s) error(%v)", d.liveURI, err)
+		err = ecode.Int(res.Code)
+		return
+	}
+	count = res.Data.Count
+	return
+}

app/service/main/dynamic/dao/live_test.go

@@ -0,0 +1,24 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoLive(t *testing.T) {
+	convey.Convey("Live", t, func() {
+		var (
+			c = context.Background()
+		)
+		convey.Convey("When http request gets code == 0", func(ctx convey.C) {
+			httpMock("GET", d.liveURI).Reply(200).JSON(`{"code":0,"msg":"ok","message":"ok","data":{"count":770843}}`)
+			count, err := d.Live(c)
+			ctx.Convey("Then err should be nil.count should greater 0.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(count, convey.ShouldBeGreaterThan, 0)
+			})
+		})
+	})
+}

app/service/main/dynamic/dao/memcache.go

@@ -0,0 +1,104 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/service/main/dynamic/model"
+	gmc "go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_keyRegionArcs    = "dyra"  // key of region archives
+	_keyRegionTagArcs = "dyrta" // key of tag archives
+)
+
+// pingMC ping memcache.
+func (d *Dao) pingMC(c context.Context) (err error) {
+	conn := d.mc.Get(c)
+	if err = conn.Set(&gmc.Item{Key: "ping", Value: []byte{1}, Expiration: d.mcExpire}); err != nil {
+		log.Error("conn.Store(set, ping, 1) error(%v)", err)
+	}
+	conn.Close()
+	return
+}
+
+// SetRegionCache set region archive to cache.
+func (d *Dao) SetRegionCache(c context.Context, regionArcs map[int32][]int64) (err error) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	tmp := make(map[int32]*model.Aids)
+	for k, v := range regionArcs {
+		tmp[k] = &model.Aids{IDs: v}
+	}
+	item := &gmc.Item{Key: _keyRegionArcs, Object: &model.Region{Aids: tmp}, Expiration: d.mcExpire, Flags: gmc.FlagProtobuf}
+	if err = conn.Set(item); err != nil {
+		log.Error("SetRegionCache error(%v)", err)
+	}
+	return
+}
+
+// RegionCache get region archive from cache.
+func (d *Dao) RegionCache(c context.Context) (rs map[int32][]int64) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	res, err := conn.Get(_keyRegionArcs)
+	if err != nil {
+		if err == gmc.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("conn.Get(%d) error(%v)", _keyRegionArcs, err)
+		}
+		return
+	}
+	rc := &model.Region{}
+	if err = conn.Scan(res, rc); err != nil {
+		log.Error("conn.Scan error(%v)", err)
+		return
+	}
+	rs = make(map[int32][]int64)
+	for k, v := range rc.Aids {
+		rs[k] = v.IDs
+	}
+	return
+}
+
+// SetTagCache set region tag archvie to cache.
+func (d *Dao) SetTagCache(c context.Context, regionTagArcs map[string][]int64) (err error) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	tmp := make(map[string]*model.Aids)
+	for k, v := range regionTagArcs {
+		tmp[k] = &model.Aids{IDs: v}
+	}
+	item := &gmc.Item{Key: _keyRegionTagArcs, Object: &model.Tag{Aids: tmp}, Expiration: d.mcExpire, Flags: gmc.FlagProtobuf}
+	if err = conn.Set(item); err != nil {
+		log.Error("SetRegionCache error(%v)", err)
+	}
+	return
+}
+
+// TagCache get region tag archive from cache.
+func (d *Dao) TagCache(c context.Context) (rs map[string][]int64) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	res, err := conn.Get(_keyRegionTagArcs)
+	if err != nil {
+		if err == gmc.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("conn.Get(%d) error(%v)", _keyRegionTagArcs, err)
+		}
+		return
+	}
+	tc := &model.Tag{}
+	if err = conn.Scan(res, tc); err != nil {
+		log.Error("conn.Scan error(%v)", err)
+		return
+	}
+	rs = make(map[string][]int64)
+	for k, v := range tc.Aids {
+		rs[k] = v.IDs
+	}
+	return
+}

app/service/main/dynamic/dao/memcache_test.go

@@ -0,0 +1,70 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaopingMC(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("pingMC", t, func(ctx convey.C) {
+		err := d.pingMC(c)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoSetRegionCache(t *testing.T) {
+	var (
+		c          = context.Background()
+		regionArcs map[int32][]int64
+	)
+	convey.Convey("SetRegionCache", t, func(ctx convey.C) {
+		err := d.SetRegionCache(c, regionArcs)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoRegionCache(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("RegionCache", t, func(ctx convey.C) {
+		rs := d.RegionCache(c)
+		ctx.Convey("Then rs should not be nil.", func(ctx convey.C) {
+			ctx.So(rs, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoSetTagCache(t *testing.T) {
+	var (
+		c             = context.Background()
+		regionTagArcs map[string][]int64
+	)
+	convey.Convey("SetTagCache", t, func(ctx convey.C) {
+		err := d.SetTagCache(c, regionTagArcs)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoTagCache(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("TagCache", t, func(ctx convey.C) {
+		rs := d.TagCache(c)
+		ctx.Convey("Then rs should not be nil.", func(ctx convey.C) {
+			ctx.So(rs, convey.ShouldNotBeNil)
+		})
+	})
+}

app/service/main/dynamic/dao/tag.go

@@ -0,0 +1,48 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+
+	"go-common/library/ecode"
+)
+
+// Hot Second type hot tag ids.
+func (d *Dao) Hot(c context.Context) (res map[int32][]int64, err error) {
+	params := url.Values{}
+	var rs struct {
+		Code int               `json:"code"`
+		Data map[int32][]int64 `json:"data"`
+	}
+	if err = d.httpR.Get(c, d.hotURI, "", params, &rs); err != nil {
+		PromError("二级分区热门Tag接口", "d.httpR.Get(%s) error(%v)", d.hotURI, err)
+		return
+	}
+	if rs.Code != ecode.OK.Code() {
+		PromError("二级分区热门Tag接口", "tag hotmap url(%s) res code(%d) or res.data(%v)", d.hotURI, rs.Code, rs.Data)
+		err = ecode.Int(rs.Code)
+		return
+	}
+	res = rs.Data
+	return
+}
+
+// Rids first type ids.
+func (d *Dao) Rids(c context.Context) (res []int32, err error) {
+	params := url.Values{}
+	var rs struct {
+		Code int     `json:"code"`
+		Data []int32 `json:"data"`
+	}
+	if err = d.httpR.Get(c, d.pridURI, "", params, &rs); err != nil {
+		PromError("一级分区ID接口", "d.httpR.Get(%s) error(%v)", d.pridURI, err)
+		return
+	}
+	if rs.Code != ecode.OK.Code() {
+		PromError("一级分区ID接口", "tag prids url(%s) res code(%d) or res.data(%v)", d.pridURI, rs.Code, rs.Data)
+		err = ecode.Int(rs.Code)
+		return
+	}
+	res = rs.Data
+	return
+}

app/service/main/dynamic/dao/tag_test.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoHot(t *testing.T) {
+	convey.Convey("Hot", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		convey.Convey("When http request gets code == 0", func(ctx convey.C) {
+			httpMock("GET", d.hotURI).Reply(200).JSON(`{"code":0,"message":"0","ttl":1,"data":{"121":[25638,830,147345]}}`)
+			res, err := d.Hot(c)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(len(res), convey.ShouldBeGreaterThan, 0)
+			})
+		})
+	})
+}
+
+func TestDaoRids(t *testing.T) {
+	convey.Convey("Rids", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		convey.Convey("When http request gets code == 0", func(ctx convey.C) {
+			httpMock("GET", d.pridURI).Reply(200).JSON(`{"code":0,"message":"0","ttl":1,"data":[1,2,3,5]}`)
+			res, err := d.Rids(c)
+			ctx.Convey("Then err should be nil.res should greater 0.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(len(res), convey.ShouldBeGreaterThan, 0)
+			})
+		})
+	})
+}