Create & Init Project...

app/interface/main/app-interface/dao/archive/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "memcache_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/app-interface/conf:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "memcache.go",
+    ],
+    importpath = "go-common/app/interface/main/app-interface/dao/archive",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/app-interface/conf:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/main/app-interface/dao/archive/dao.go

@@ -0,0 +1,167 @@
+package archive
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/interface/main/app-interface/conf"
+	"go-common/app/service/main/archive/api"
+	arcrpc "go-common/app/service/main/archive/api/gorpc"
+	"go-common/app/service/main/archive/model/archive"
+	"go-common/library/cache/memcache"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/sync/errgroup"
+
+	"github.com/pkg/errors"
+)
+
+// Dao is archive dao.
+type Dao struct {
+	// http client
+	client *bm.Client
+	// rpc
+	arcRPC *arcrpc.Service2
+	// memcache
+	arcMc     *memcache.Pool
+	expireArc int32
+}
+
+// New new a archive dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		// http client
+		client: bm.NewClient(c.HTTPWrite),
+		arcRPC: arcrpc.New2(c.ArchiveRPC),
+		// memcache
+		arcMc:     memcache.NewPool(c.Memcache.Archive.Config),
+		expireArc: int32(time.Duration(c.Memcache.Archive.ArchiveExpire) / time.Second),
+	}
+	return
+}
+
+// UpArcs3 get upper archives
+func (d *Dao) UpArcs3(c context.Context, mid int64, pn, ps int) (as []*api.Arc, err error) {
+	arg := &archive.ArgUpArcs2{Mid: mid, Pn: pn, Ps: ps}
+	if as, err = d.arcRPC.UpArcs3(c, arg); err != nil {
+		if ecode.Cause(err) == ecode.NothingFound {
+			err = nil
+		}
+	}
+	return
+}
+
+// UpCount2 get upper count.
+func (d *Dao) UpCount2(c context.Context, mid int64) (cnt int, err error) {
+	arg := &archive.ArgUpCount2{Mid: mid}
+	return d.arcRPC.UpCount2(c, arg)
+}
+
+// Archives multi get archives.
+func (d *Dao) Archives(c context.Context, aids []int64) (as map[int64]*api.Arc, err error) {
+	if len(aids) == 0 {
+		return
+	}
+	g, ctx := errgroup.WithContext(c)
+	g.Go(func() (err error) {
+		var missed []int64
+		if as, missed, err = d.arcsCache(ctx, aids); err != nil {
+			log.Error("%+v", err)
+			missed = aids
+			err = nil
+		}
+		if len(missed) == 0 {
+			return
+		}
+		var tmp map[int64]*api.Arc
+		arg := &archive.ArgAids2{Aids: missed}
+		if tmp, err = d.arcRPC.Archives3(ctx, arg); err != nil {
+			log.Error("d.arcRPC.Archives3(%v) error(%v)", arg, err)
+			return
+		}
+		for aid, a := range tmp {
+			as[aid] = a
+		}
+		return
+	})
+	var stm map[int64]*api.Stat
+	g.Go(func() (err error) {
+		var missed []int64
+		if stm, missed, err = d.statsCache(ctx, aids); err != nil {
+			log.Error("%+v", err)
+			missed = aids
+			err = nil
+		}
+		if len(missed) == 0 {
+			return
+		}
+		var tmp map[int64]*api.Stat
+		arg := &archive.ArgAids2{Aids: missed}
+		if tmp, err = d.arcRPC.Stats3(ctx, arg); err != nil {
+			log.Error("d.arcRPC.Stats3(%v) error(%v)", arg, err)
+			err = nil
+			return
+		}
+		for aid, st := range tmp {
+			stm[aid] = st
+		}
+		return
+	})
+	if err = g.Wait(); err != nil {
+		return
+	}
+	for aid, a := range as {
+		if st, ok := stm[aid]; ok {
+			a.Stat = *st
+		}
+	}
+	return
+}
+
+// Archives2 multi get archives.
+func (d *Dao) Archives2(c context.Context, aids []int64) (am map[int64]*api.Arc, err error) {
+	if len(aids) == 0 {
+		return
+	}
+	var avMissed, stMissed []int64
+	if am, avMissed, stMissed, err = d.avWithStCaches(c, aids); err != nil {
+		avMissed = aids
+		log.Error("%+v", err)
+		err = nil
+	}
+	g, ctx := errgroup.WithContext(c)
+	if len(avMissed) != 0 {
+		g.Go(func() (err error) {
+			arg := &archive.ArgAids2{Aids: avMissed}
+			avm, err := d.arcRPC.Archives3(ctx, arg)
+			if err != nil {
+				err = errors.Wrapf(err, "%v", arg)
+				return
+			}
+			for aid, a := range avm {
+				am[aid] = a
+			}
+			return
+		})
+	}
+	var stm map[int64]*api.Stat
+	if len(stMissed) != 0 {
+		g.Go(func() error {
+			arg := &archive.ArgAids2{Aids: stMissed}
+			if stm, err = d.arcRPC.Stats3(ctx, arg); err != nil {
+				log.Error("d.arcRPC.Stats3(%v) error(%v)", arg, err)
+			}
+			return nil
+		})
+	}
+	if err = g.Wait(); err != nil {
+		return
+	}
+	for aid, a := range am {
+		if st, ok := stm[aid]; ok {
+			a.Stat = *st
+		}
+	}
+	return
+}

app/interface/main/app-interface/dao/archive/dao_test.go

@@ -0,0 +1,93 @@
+package archive
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/interface/main/app-interface/conf"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.app-svr.app-interface")
+		flag.Set("conf_token", "1mWvdEwZHmCYGoXJCVIdszBOPVdtpXb3")
+		flag.Set("tree_id", "2688")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/app-interface-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+	// time.Sleep(time.Second)
+}
+
+func TestUpArcs3(t *testing.T) {
+	var (
+		c   = context.TODO()
+		mid = int64(1)
+		pn  = 1
+		ps  = 5
+	)
+	convey.Convey("UpArcs3", t, func(ctx convey.C) {
+		_, err := d.UpArcs3(c, mid, pn, ps)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestUpCount2(t *testing.T) {
+	var (
+		c   = context.TODO()
+		mid = int64(1)
+	)
+	convey.Convey("UpCount2", t, func(ctx convey.C) {
+		_, err := d.UpCount2(c, mid)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestArchive(t *testing.T) {
+	var (
+		c    = context.TODO()
+		aids = []int64{1}
+	)
+	convey.Convey("Archives", t, func(ctx convey.C) {
+		_, err := d.Archives(c, aids)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestArchive2(t *testing.T) {
+	var (
+		c    = context.TODO()
+		aids = []int64{1}
+	)
+	convey.Convey("Archives2", t, func(ctx convey.C) {
+		_, err := d.Archives2(c, aids)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/interface/main/app-interface/dao/archive/memcache.go

@@ -0,0 +1,164 @@
+package archive
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/app/service/main/archive/api"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_prefixArchive = "a3p_"
+	_prefixStat    = "stp_"
+)
+
+func keyArc(aid int64) string {
+	return _prefixArchive + strconv.FormatInt(aid, 10)
+}
+
+func keyStat(aid int64) string {
+	return _prefixStat + strconv.FormatInt(aid, 10)
+}
+
+// arcsCache get archives cache.
+func (d *Dao) arcsCache(c context.Context, aids []int64) (cached map[int64]*api.Arc, missed []int64, err error) {
+	cached = make(map[int64]*api.Arc, len(aids))
+	var rs map[string]*memcache.Item
+	conn := d.arcMc.Get(c)
+	keys := make([]string, 0, len(aids))
+	aidmap := make(map[string]int64, len(aids))
+	defer conn.Close()
+	for _, aid := range aids {
+		k := keyArc(aid)
+		if _, ok := aidmap[k]; !ok {
+			keys = append(keys, k)
+			aidmap[k] = aid
+		}
+	}
+	if rs, err = conn.GetMulti(keys); err != nil {
+		err = errors.Wrapf(err, "%v", keys)
+		return
+	}
+	for k, r := range rs {
+		a := &api.Arc{}
+		if err = conn.Scan(r, a); err != nil {
+			log.Error("conn.Scan(%s) error(%v)", r.Value, err)
+			err = nil
+			continue
+		}
+		cached[aidmap[k]] = a
+		// delete hit key
+		delete(aidmap, k)
+	}
+	// missed key
+	missed = make([]int64, 0, len(aidmap))
+	for _, aid := range aidmap {
+		missed = append(missed, aid)
+	}
+	return
+}
+
+// statsCache get stat cache by aids
+func (d *Dao) statsCache(c context.Context, aids []int64) (cached map[int64]*api.Stat, missed []int64, err error) {
+	cached = make(map[int64]*api.Stat, len(aids))
+	var rs map[string]*memcache.Item
+	conn := d.arcMc.Get(c)
+	keys := make([]string, 0, len(aids))
+	defer conn.Close()
+	for _, aid := range aids {
+		keys = append(keys, keyStat(aid))
+	}
+	if rs, err = conn.GetMulti(keys); err != nil {
+		err = errors.Wrapf(err, "%v", keys)
+		return
+	}
+	for _, r := range rs {
+		st := &api.Stat{}
+		if err = conn.Scan(r, st); err != nil {
+			log.Error("conn.Scan(%s) error(%v)", r.Value, err)
+			err = nil
+			continue
+		}
+		cached[st.Aid] = st
+	}
+	if len(cached) == len(aids) {
+		return
+	}
+	for _, aid := range aids {
+		if _, ok := cached[aid]; !ok {
+			missed = append(missed, aid)
+		}
+	}
+	return
+}
+
+func (d *Dao) avWithStCaches(c context.Context, aids []int64) (cached map[int64]*api.Arc, avMissed, stMissed []int64, err error) {
+	cached = make(map[int64]*api.Arc, len(aids))
+	conn := d.arcMc.Get(c)
+	defer conn.Close()
+	keys := make([]string, 0, len(aids)*2)
+	avm := make(map[string]int64, len(aids))
+	stm := make(map[string]int64, len(aids))
+	for _, aid := range aids {
+		ak := keyArc(aid)
+		if _, ok := avm[ak]; !ok {
+			keys = append(keys, ak)
+			avm[ak] = aid
+		}
+		sk := keyStat(aid)
+		if _, ok := stm[sk]; !ok {
+			keys = append(keys, sk)
+			stm[sk] = aid
+		}
+	}
+	rs, err := conn.GetMulti(keys)
+	if err != nil {
+		err = errors.Wrapf(err, "%v", keys)
+		return
+	}
+	stCached := make(map[string]*api.Stat, len(aids))
+	for k, r := range rs {
+		if aid, ok := avm[k]; ok {
+			a := &api.Arc{}
+			if err = conn.Scan(r, a); err != nil {
+				log.Error("conn.Scan(%s) error(%v)", r.Value, err)
+				err = nil
+				continue
+			}
+			cached[aid] = a
+			// delete hit key
+			delete(avm, k)
+		}
+		if _, ok := stm[k]; ok {
+			st := &api.Stat{}
+			if err = conn.Scan(r, st); err != nil {
+				log.Error("conn.Scan(%s) error(%v)", r.Value, err)
+				err = nil
+				continue
+			}
+			stCached[k] = st
+		}
+	}
+	for k, st := range stCached {
+		if a, ok := cached[st.Aid]; ok {
+			a.Stat = *st
+			// delete hit key
+			delete(stm, k)
+		}
+	}
+	// missed key
+	avMissed = make([]int64, 0, len(avm))
+	for _, aid := range avm {
+		avMissed = append(avMissed, aid)
+
+	}
+	stMissed = make([]int64, 0, len(stm))
+	for _, aid := range stm {
+		stMissed = append(stMissed, aid)
+	}
+	return
+}

app/interface/main/app-interface/dao/archive/memcache_test.go

@@ -0,0 +1,148 @@
+package archive
+
+import (
+	"context"
+	"go-common/app/service/main/archive/api"
+	"reflect"
+	"testing"
+)
+
+func Test_keyArc(t *testing.T) {
+	type args struct {
+		aid int64
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+	// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := keyArc(tt.args.aid); got != tt.want {
+				t.Errorf("keyArc() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_keyStat(t *testing.T) {
+	type args struct {
+		aid int64
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+	// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := keyStat(tt.args.aid); got != tt.want {
+				t.Errorf("keyStat() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestDao_arcsCache(t *testing.T) {
+	type args struct {
+		c    context.Context
+		aids []int64
+	}
+	tests := []struct {
+		name       string
+		d          *Dao
+		args       args
+		wantCached map[int64]*api.Arc
+		wantMissed []int64
+		wantErr    bool
+	}{
+	// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCached, gotMissed, err := tt.d.arcsCache(tt.args.c, tt.args.aids)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Dao.arcsCache() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotCached, tt.wantCached) {
+				t.Errorf("Dao.arcsCache() gotCached = %v, want %v", gotCached, tt.wantCached)
+			}
+			if !reflect.DeepEqual(gotMissed, tt.wantMissed) {
+				t.Errorf("Dao.arcsCache() gotMissed = %v, want %v", gotMissed, tt.wantMissed)
+			}
+		})
+	}
+}
+
+func TestDao_statsCache(t *testing.T) {
+	type args struct {
+		c    context.Context
+		aids []int64
+	}
+	tests := []struct {
+		name       string
+		d          *Dao
+		args       args
+		wantCached map[int64]*api.Stat
+		wantMissed []int64
+		wantErr    bool
+	}{
+	// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCached, gotMissed, err := tt.d.statsCache(tt.args.c, tt.args.aids)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Dao.statsCache() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotCached, tt.wantCached) {
+				t.Errorf("Dao.statsCache() gotCached = %v, want %v", gotCached, tt.wantCached)
+			}
+			if !reflect.DeepEqual(gotMissed, tt.wantMissed) {
+				t.Errorf("Dao.statsCache() gotMissed = %v, want %v", gotMissed, tt.wantMissed)
+			}
+		})
+	}
+}
+
+func TestDao_avWithStCaches(t *testing.T) {
+	type args struct {
+		c    context.Context
+		aids []int64
+	}
+	tests := []struct {
+		name         string
+		d            *Dao
+		args         args
+		wantCached   map[int64]*api.Arc
+		wantAvMissed []int64
+		wantStMissed []int64
+		wantErr      bool
+	}{
+	// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCached, gotAvMissed, gotStMissed, err := tt.d.avWithStCaches(tt.args.c, tt.args.aids)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Dao.avWithStCaches() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotCached, tt.wantCached) {
+				t.Errorf("Dao.avWithStCaches() gotCached = %v, want %v", gotCached, tt.wantCached)
+			}
+			if !reflect.DeepEqual(gotAvMissed, tt.wantAvMissed) {
+				t.Errorf("Dao.avWithStCaches() gotAvMissed = %v, want %v", gotAvMissed, tt.wantAvMissed)
+			}
+			if !reflect.DeepEqual(gotStMissed, tt.wantStMissed) {
+				t.Errorf("Dao.avWithStCaches() gotStMissed = %v, want %v", gotStMissed, tt.wantStMissed)
+			}
+		})
+	}
+}