Create & Init Project...

app/service/main/share/dao/BUILD

@@ -0,0 +1,68 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "databus_test.go",
+        "mysql_test.go",
+        "redis_test.go",
+        "share_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/share/conf:go_default_library",
+        "//app/service/main/share/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "databus.go",
+        "mysql.go",
+        "redis.go",
+        "share.go",
+    ],
+    importpath = "go-common/app/service/main/share/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/main/share/conf:go_default_library",
+        "//app/service/main/share/model:go_default_library",
+        "//library/cache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/ip:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/dgryski/go-farm:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/share/dao/dao.go

@@ -0,0 +1,73 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/service/main/share/conf"
+	"go-common/library/cache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	c   *conf.Config
+	db  *sql.DB
+	rds *redis.Pool
+	// cache chan
+	cacheCh chan func()
+	// databus
+	databus *databus.Databus
+	// archiveDatabus
+	archiveDatabus *databus.Databus
+	// sources
+	sources map[int64]struct{}
+	// asyncCache
+	asyncCache *cache.Cache
+}
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:              c,
+		db:             sql.NewMySQL(c.DB),
+		rds:            redis.NewPool(c.Redis),
+		cacheCh:        make(chan func(), 1024),
+		databus:        databus.New(c.Databus),
+		archiveDatabus: databus.New(c.ArchiveDatabus),
+		sources:        make(map[int64]struct{}, len(c.Sources)),
+		asyncCache:     cache.New(c.Worker, 1024),
+	}
+	for _, s := range c.Sources {
+		d.sources[s] = struct{}{}
+	}
+	return d
+}
+
+// BeginTran begin transcation.
+func (d *Dao) BeginTran(c context.Context) (tx *sql.Tx, err error) {
+	return d.db.Begin(c)
+}
+
+// Close close
+func (d *Dao) Close() {
+	d.db.Close()
+	d.rds.Close()
+}
+
+// Ping ping
+func (d *Dao) Ping() (err error) {
+	if err = d.db.Ping(context.Background()); err != nil {
+		log.Error("d.db.Ping error(%v)", err)
+		return
+	}
+	conn := d.rds.Get(context.Background())
+	defer conn.Close()
+	if _, err = conn.Do("SET", "ping", "pong"); err != nil {
+		log.Error("redis.Set error(%v)", err)
+		return
+	}
+	return
+}

app/service/main/share/dao/dao_test.go

@@ -0,0 +1,45 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/service/main/share/conf"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.app-svr.share-service")
+		flag.Set("conf_token", "120ed94e23b963fc0564fbdb662916c3")
+		flag.Set("tree_id", "25960")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/share-service-test.toml")
+	}
+
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}
+
+func TestPing(t *testing.T) {
+	convey.Convey("Ping", t, func() {
+		d.Ping()
+	})
+}

app/service/main/share/dao/databus.go

@@ -0,0 +1,31 @@
+package dao
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"go-common/app/service/main/share/model"
+)
+
+// PubShare .
+func (d *Dao) PubShare(c context.Context, p *model.ShareParams) (err error) {
+	msg := &model.MIDShare{
+		OID:  p.OID,
+		MID:  p.MID,
+		TP:   p.TP,
+		Time: time.Now().Unix(),
+	}
+	return d.databus.Send(c, strconv.FormatInt(p.MID, 10), &msg)
+}
+
+// PubStatShare .
+func (d *Dao) PubStatShare(c context.Context, typ string, oid, count int64) (err error) {
+	msg := &model.ArchiveShare{
+		Type:  typ,
+		ID:    oid,
+		Count: int(count),
+		Ts:    time.Now().Unix(),
+	}
+	return d.archiveDatabus.Send(c, strconv.FormatInt(oid, 10), &msg)
+}

app/service/main/share/dao/databus_test.go

@@ -0,0 +1,31 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/service/main/share/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoPubShare(t *testing.T) {
+	convey.Convey("PubShare", t, func(ctx convey.C) {
+		p := &model.ShareParams{
+			OID: int64(1),
+			MID: int64(1),
+			TP:  int(3),
+		}
+		err := d.PubShare(context.Background(), p)
+		convey.So(err, convey.ShouldBeNil)
+	})
+}
+
+func TestDaoPubStatShare(t *testing.T) {
+	convey.Convey("PubStatShare", t, func(ctx convey.C) {
+		oid := int64(1)
+		count := int64(666)
+		err := d.PubStatShare(context.Background(), model.ArchiveMsgTyp, oid, count)
+		convey.So(err, convey.ShouldBeNil)
+	})
+}

app/service/main/share/dao/mysql.go

@@ -0,0 +1,66 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/service/main/share/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_shareSQL    = "SELECT oid,tp,share FROM %s WHERE oid=? AND tp=?"
+	_addShareSQL = "INSERT INTO %s(oid,tp,share) VALUES(?,?,1) ON DUPLICATE KEY UPDATE share=share+1"
+)
+
+func table(oid int64) string {
+	return fmt.Sprintf("share_count_%02d", oid%100)
+}
+
+// Share get share
+func (d *Dao) Share(c context.Context, oid int64, tp int) (share *model.Share, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_shareSQL, table(oid)), oid, tp)
+	share = &model.Share{}
+	if err = row.Scan(&share.OID, &share.Tp, &share.Count); err != nil {
+		if err == sql.ErrNoRows {
+			share = nil
+			err = nil
+		} else {
+			err = errors.WithStack(err)
+		}
+		return
+	}
+	return
+}
+
+// AddShare add share
+func (d *Dao) AddShare(c context.Context, oid int64, tp int) (err error) {
+	if _, err = d.db.Exec(c, fmt.Sprintf(_addShareSQL, table(oid)), oid, tp); err != nil {
+		err = errors.Wrapf(err, "d.db.Exec(%s, %d, %d)", fmt.Sprintf(_addShareSQL, table(oid)), oid, tp)
+		return
+	}
+	if _, ok := d.sources[oid]; ok && tp == model.ArchiveTyp {
+		if _, err = d.db.Exec(c, fmt.Sprintf(_addShareSQL, table(d.c.Target)), d.c.Target, tp); err != nil {
+			return
+		}
+		var share *model.Share
+		if share, err = d.Share(c, d.c.Target, tp); err != nil {
+			return
+		}
+		// pub msg
+		if err = d.PubStatShare(c, model.ArchiveMsgTyp, d.c.Target, share.Count); err != nil {
+			log.Error("s.dao.PubArchiveShare error(%v)", err)
+			err = nil
+		}
+		d.asyncCache.Save(func() {
+			if err = d.SetShareCache(context.Background(), d.c.Target, tp, share.Count); err != nil {
+				log.Error("d.SetShareCache error(%v)", err)
+				return
+			}
+		})
+	}
+	return
+}

app/service/main/share/dao/mysql_test.go

@@ -0,0 +1,50 @@
+package dao
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoTable(t *testing.T) {
+	var (
+		oid = int64(0)
+	)
+	convey.Convey("table", t, func(ctx convey.C) {
+		p1 := table(oid)
+		ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoShare(t *testing.T) {
+	var (
+		c   = context.TODO()
+		oid = int64(0)
+		tp  = int(0)
+	)
+	convey.Convey("Share", t, func(ctx convey.C) {
+		share, err := d.Share(c, oid, tp)
+		ctx.Convey("Then err should be nil.share should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(share, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoAddShare(t *testing.T) {
+	var (
+		c   = context.TODO()
+		oid = int64(rand.Intn(10000000))
+		tp  = int(3)
+	)
+	convey.Convey("AddShare", t, func(ctx convey.C) {
+		err := d.AddShare(c, oid, tp)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/service/main/share/dao/redis.go

@@ -0,0 +1,121 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/service/main/share/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	xip "go-common/library/net/ip"
+
+	farm "github.com/dgryski/go-farm"
+	"github.com/pkg/errors"
+)
+
+func redisKey(oid int64, tp int) string {
+	return fmt.Sprintf("%d_%d", oid, tp)
+}
+
+func redisValue(p *model.ShareParams) int64 {
+	return int64(farm.Hash64([]byte(fmt.Sprintf("%d_%d_%d_%s", p.MID, p.OID, p.TP, p.IP))))
+}
+
+func shareKey(oid int64, tp int) string {
+	return fmt.Sprintf("c_%d_%d", oid, tp)
+}
+
+// AddShareMember add share
+func (d *Dao) AddShareMember(ctx context.Context, p *model.ShareParams) (ok bool, err error) {
+	var (
+		conn  = d.rds.Get(ctx)
+		key   = redisKey(p.OID, p.TP)
+		value = (p.MID << 32) | int64(xip.InetAtoN(p.IP))
+	)
+	log.Info("oid-%d mid-%d ip-%s tp-%d key-%s value-%d", p.OID, p.MID, p.IP, p.TP, key, value)
+	defer conn.Close()
+	if err = conn.Send("SADD", key, value); err != nil {
+		err = errors.Wrapf(err, "conn.Do(SADD, %s, %d)", key, value)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.c.RedisExpire); err != nil {
+		err = errors.Wrapf(err, "conn.Do(SADD, %s, %d)", key, value)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		err = errors.Wrap(err, "conn.Flush")
+		return
+	}
+	if ok, err = redis.Bool(conn.Receive()); err != nil {
+		log.Error("sadd failed mid(%d) oid(%d) type(%d) ip(%s) key(%s) value(%d)",
+			p.MID, p.OID, p.TP, p.IP, key, value)
+		err = errors.Wrap(err, "redis.Bool(conn.Receive)")
+		return
+	}
+	if _, err = conn.Receive(); err != nil {
+		err = errors.Wrap(err, "conn.Receive")
+		return
+	}
+	return
+}
+
+// SetShareCache set share cache
+func (d *Dao) SetShareCache(c context.Context, oid int64, tp int, shared int64) (err error) {
+	var (
+		conn = d.rds.Get(c)
+		key  = shareKey(oid, tp)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("SET", key, shared); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}
+
+// ShareCache return oid share count
+func (d *Dao) ShareCache(c context.Context, oid int64, tp int) (shared int64, err error) {
+	var (
+		conn = d.rds.Get(c)
+		key  = shareKey(oid, tp)
+	)
+	defer conn.Close()
+	if shared, err = redis.Int64(conn.Do("GET", key)); err != nil {
+		if err == redis.ErrNil {
+			shared = -1
+			err = nil
+		} else {
+			err = errors.WithStack(err)
+		}
+	}
+	return
+}
+
+// SharesCache return oids share
+func (d *Dao) SharesCache(c context.Context, oids []int64, tp int) (shares map[int64]int64, err error) {
+	conn := d.rds.Get(c)
+	defer conn.Close()
+	for _, oid := range oids {
+		if err = conn.Send("GET", shareKey(oid, tp)); err != nil {
+			log.Error("conn.Send(GET, %s) error(%v)", shareKey(oid, tp), err)
+			return
+		}
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		return
+	}
+	shares = make(map[int64]int64, len(oids))
+	for _, oid := range oids {
+		var cnt int64
+		if cnt, err = redis.Int64(conn.Receive()); err != nil {
+			if err == redis.ErrNil {
+				err = nil
+				continue
+			}
+			return
+		}
+		shares[oid] = cnt
+	}
+	return
+}

app/service/main/share/dao/redis_test.go

@@ -0,0 +1,111 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/service/main/share/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoRedisKey(t *testing.T) {
+	var (
+		oid = int64(0)
+		tp  = int(0)
+	)
+	convey.Convey("redisKey", t, func(ctx convey.C) {
+		p1 := redisKey(oid, tp)
+		ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoRedisValue(t *testing.T) {
+	convey.Convey("redisValue", t, func(ctx convey.C) {
+		p := &model.ShareParams{
+			OID: 22,
+			MID: 33,
+			TP:  2,
+			IP:  "",
+		}
+		p1 := redisValue(p)
+		ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoShareKey(t *testing.T) {
+	var (
+		oid = int64(0)
+		tp  = int(0)
+	)
+	convey.Convey("shareKey", t, func(ctx convey.C) {
+		p1 := shareKey(oid, tp)
+		ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoAddShareMember(t *testing.T) {
+	convey.Convey("AddShareMember", t, func(ctx convey.C) {
+		p := &model.ShareParams{
+			OID: int64(1),
+			MID: int64(1),
+			TP:  int(3),
+		}
+		ok, err := d.AddShareMember(context.Background(), p)
+		ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(ok, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoSetShareCache(t *testing.T) {
+	var (
+		c      = context.TODO()
+		oid    = int64(0)
+		tp     = int(0)
+		shared = int64(0)
+	)
+	convey.Convey("SetShareCache", t, func(ctx convey.C) {
+		err := d.SetShareCache(c, oid, tp, shared)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoShareCache(t *testing.T) {
+	var (
+		c   = context.TODO()
+		oid = int64(0)
+		tp  = int(0)
+	)
+	convey.Convey("ShareCache", t, func(ctx convey.C) {
+		shared, err := d.ShareCache(c, oid, tp)
+		ctx.Convey("Then err should be nil.shared should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(shared, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoSharesCache(t *testing.T) {
+	var (
+		c    = context.TODO()
+		oids = []int64{}
+		tp   = int(0)
+	)
+	convey.Convey("SharesCache", t, func(ctx convey.C) {
+		shares, err := d.SharesCache(c, oids, tp)
+		ctx.Convey("Then err should be nil.shares should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(shares, convey.ShouldNotBeNil)
+		})
+	})
+}

app/service/main/share/dao/share.go

@@ -0,0 +1,92 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/service/main/share/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+// Shares get shares
+func (d *Dao) Shares(ctx context.Context, oids []int64, tp int) (shares map[int64]int64, err error) {
+	shares, err = d.SharesCache(ctx, oids, tp)
+	if err != nil {
+		log.Error("d.SharesCache(%v) tp(%d) error(%v)", oids, tp, err)
+		err = nil
+		shares = make(map[int64]int64, len(oids))
+	}
+	var missed []int64
+	for _, oid := range oids {
+		if _, ok := shares[oid]; !ok {
+			missed = append(missed, oid)
+		}
+	}
+	if len(missed) == 0 {
+		return
+	}
+	// 最大30个id，并且分了100张表，用in的优化空间也不大，暂时循环单个查
+	for _, oid := range missed {
+		cnt, err := d.ShareCount(ctx, oid, tp)
+		if err != nil {
+			continue
+		}
+		shares[oid] = cnt
+	}
+	return
+}
+
+// ShareCount get share from cache/db
+func (d *Dao) ShareCount(ctx context.Context, oid int64, tp int) (count int64, err error) {
+	count, err = d.ShareCache(ctx, oid, tp)
+	if count != -1 && err == nil {
+		return
+	}
+	var share *model.Share
+	if share, err = d.Share(ctx, oid, tp); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	count = 0
+	if share != nil {
+		count = share.Count
+	}
+	d.asyncCache.Save(func() {
+		if err = d.SetShareCache(context.Background(), oid, tp, count); err != nil {
+			log.Error("%+v", err)
+			return
+		}
+	})
+	return
+}
+
+// Add add share
+func (d *Dao) Add(ctx context.Context, p *model.ShareParams) (shared int64, err error) {
+	var ok bool
+	if ok, err = d.AddShareMember(ctx, p); err != nil {
+		return
+	}
+	if !ok {
+		err = ecode.ShareAlreadyAdd
+		return
+	}
+	if err = d.AddShare(ctx, p.OID, p.TP); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	var share *model.Share
+	if share, err = d.Share(ctx, p.OID, p.TP); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	shared = share.Count
+	d.asyncCache.Save(func() {
+		if err = d.SetShareCache(context.Background(), p.OID, p.TP, shared); err != nil {
+			log.Error("%+v", err)
+			return
+		}
+	})
+	return
+}

app/service/main/share/dao/share_test.go

@@ -0,0 +1,63 @@
+package dao
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"go-common/app/service/main/share/model"
+	"go-common/library/ecode"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoShares(t *testing.T) {
+	var (
+		c    = context.TODO()
+		oids = []int64{1, 2}
+		tp   = int(0)
+	)
+	convey.Convey("Shares", t, func(ctx convey.C) {
+		shares, err := d.Shares(c, oids, tp)
+		ctx.Convey("Then err should be nil.shares should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(shares, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoShareCount(t *testing.T) {
+	var (
+		c   = context.TODO()
+		oid = int64(rand.Intn(100000000))
+		tp  = int(2)
+	)
+	convey.Convey("ShareCount", t, func(ctx convey.C) {
+		count, err := d.ShareCount(c, oid, tp)
+		ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(count, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoAdd(t *testing.T) {
+	convey.Convey("Add", t, func(ctx convey.C) {
+		oid := rand.Intn(1000000)
+		mid := rand.Intn(1000000)
+		p := &model.ShareParams{
+			OID: int64(oid),
+			MID: int64(mid),
+			TP:  int(3),
+			IP:  "",
+		}
+		shared, err := d.Add(context.Background(), p)
+		if err == ecode.ShareAlreadyAdd {
+			err = nil
+		}
+		ctx.Convey("Then err should be nil.shared should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(shared, convey.ShouldNotBeNil)
+		})
+	})
+}