Create & Init Project...

app/service/main/history/dao/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "databus_test.go",
+        "redis_test.go",
+        "tidb_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/history/api/grpc:go_default_library",
+        "//app/service/main/history/conf:go_default_library",
+        "//app/service/main/history/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "databus.go",
+        "redis.go",
+        "tidb.go",
+    ],
+    importpath = "go-common/app/service/main/history/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/main/history/api/grpc:go_default_library",
+        "//app/service/main/history/conf:go_default_library",
+        "//app/service/main/history/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/database/tidb:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/history/dao/dao.go

@@ -0,0 +1,96 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/service/main/history/conf"
+	"go-common/app/service/main/history/model"
+	"go-common/library/cache/redis"
+	"go-common/library/database/tidb"
+	"go-common/library/queue/databus"
+)
+
+// Dao dao
+type Dao struct {
+	c                     *conf.Config
+	tidb                  *tidb.DB
+	redis                 *redis.Pool
+	redisExpire           int32
+	mergeDbus             *databus.Databus
+	businessesStmt        *tidb.Stmts
+	historiesStmt         *tidb.Stmts
+	historyStmt           *tidb.Stmts
+	insertStmt            *tidb.Stmts
+	deleteHistoriesStmt   *tidb.Stmts
+	clearAllHistoriesStmt *tidb.Stmts
+	userHideStmt          *tidb.Stmts
+	updateUserHideStmt    *tidb.Stmts
+	Businesses            map[int64]*model.Business
+	BusinessNames         map[string]*model.Business
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:           c,
+		redis:       redis.NewPool(c.Redis.Config),
+		tidb:        tidb.NewTiDB(c.TiDB),
+		mergeDbus:   databus.New(c.DataBus.Merge),
+		redisExpire: int32(time.Duration(c.Redis.Expire) / time.Second),
+	}
+	dao.businessesStmt = dao.tidb.Prepared(_businessesSQL)
+	dao.historiesStmt = dao.tidb.Prepared(_historiesSQL)
+	dao.deleteHistoriesStmt = dao.tidb.Prepared(_deleteHistoriesSQL)
+	dao.clearAllHistoriesStmt = dao.tidb.Prepared(_clearAllHistoriesSQL)
+	dao.historyStmt = dao.tidb.Prepared(_historySQL)
+	dao.userHideStmt = dao.tidb.Prepared(_userHide)
+	dao.updateUserHideStmt = dao.tidb.Prepared(_updateUserHide)
+	dao.insertStmt = dao.tidb.Prepared(_addHistorySQL)
+
+	dao.loadBusiness()
+	go dao.loadBusinessproc()
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.tidb.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.tidb.Ping(c); err != nil {
+		return
+	}
+	return d.pingRedis(c)
+}
+
+// LoadBusiness .
+func (d *Dao) loadBusiness() {
+	var business []*model.Business
+	var err error
+	businessMap := make(map[string]*model.Business)
+	businessIDMap := make(map[int64]*model.Business)
+	for {
+		if business, err = d.QueryBusinesses(context.TODO()); err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		for _, b := range business {
+			businessMap[b.Name] = b
+			businessIDMap[b.ID] = b
+		}
+		d.BusinessNames = businessMap
+		d.Businesses = businessIDMap
+		return
+	}
+}
+
+func (d *Dao) loadBusinessproc() {
+	for {
+		time.Sleep(time.Minute * 5)
+		d.loadBusiness()
+	}
+}

app/service/main/history/dao/dao_test.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+
+	"go-common/app/service/main/history/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.community.history-service")
+		flag.Set("conf_token", "10f1bb6e589c42e7e1ee2560aff96b81")
+		flag.Set("tree_id", "56699")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		fmt.Println("") // 存在才能pass
+		flag.Set("conf", "../cmd/history-service-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}
+
+// INSERT INTO `bilibili_history`.`histories`(`mtime`, `ctime`, `mid`, `business_id`, `kid`, `aid`, `sid`, `epid`, `sub_type`, `cid`, `device`, `progress`, `view_at`) VALUES ('2018-08-27 03:03:50', '2018-08-27 03:01:29', 1, 4, 2, 3, 4, 5, 7, 6, 8, 9, 10);

app/service/main/history/dao/databus.go

@@ -0,0 +1,20 @@
+package dao
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/app/service/main/history/model"
+	"go-common/library/log"
+	"go-common/library/stat/prom"
+)
+
+// AddHistoryMessage .
+func (d *Dao) AddHistoryMessage(c context.Context, k int, msg []*model.Merge) (err error) {
+	key := strconv.Itoa(k)
+	prom.BusinessInfoCount.Add("dbus-"+key, int64(len(msg)))
+	if err = d.mergeDbus.Send(c, key, msg); err != nil {
+		log.Error("Pub(%s,%+v) error(%v)", key, msg, err)
+	}
+	return
+}

app/service/main/history/dao/databus_test.go

@@ -0,0 +1,28 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/service/main/history/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoAddHistoryMessage(t *testing.T) {
+	var (
+		c   = context.Background()
+		k   = int(1)
+		msg = []*model.Merge{{
+			Mid:  1,
+			Bid:  4,
+			Time: 10000,
+		}}
+	)
+	convey.Convey("AddHistoryMessage", t, func(ctx convey.C) {
+		err := d.AddHistoryMessage(c, k, msg)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/service/main/history/dao/redis.go

@@ -0,0 +1,394 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	pb "go-common/app/service/main/history/api/grpc"
+	"go-common/app/service/main/history/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_keySwitch = "s_" // mid -> bit(value)
+	_bucket    = 1000 // bit bucket
+)
+
+// keyHistory return history key.
+func keyHistory(business string, mid int64) string {
+	return fmt.Sprintf("h_%d_%s", mid, business)
+}
+
+// keyIndex return history index key.
+func keyIndex(business string, mid int64) string {
+	return fmt.Sprintf("i_%d_%s", mid, business)
+}
+
+// keySwitch return Switch key.
+func keySwitch(mid int64) string {
+	return _keySwitch + strconv.FormatInt(mid/_bucket, 10)
+}
+
+// ListCacheByTime get aids from redis where score.
+func (d *Dao) ListCacheByTime(c context.Context, business string, mid int64, start int64) (aids []int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	values, err := redis.Values(conn.Do("ZRANGEBYSCORE", keyIndex(business, mid), start, "INF", "WITHSCORES"))
+	if err != nil {
+		log.Error("conn.Do(ZRANGEBYSCORE %v) error(%v)", keyIndex(business, mid), err)
+		return
+	}
+	if len(values) == 0 {
+		return
+	}
+	var aid, unix int64
+	for len(values) > 0 {
+		if values, err = redis.Scan(values, &aid, &unix); err != nil {
+			log.Error("redis.Scan(%v) error(%v)", values, err)
+			return
+		}
+		aids = append(aids, aid)
+	}
+	return
+}
+
+// ListsCacheByTime get aids from redis where score.
+func (d *Dao) ListsCacheByTime(c context.Context, businesses []string, mid int64, viewAt, ps int64) (res map[string][]int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	var count int
+	for _, business := range businesses {
+		if err = conn.Send("ZREVRANGEBYSCORE", keyIndex(business, mid), viewAt, "-INF", "LIMIT", 0, ps); err != nil {
+			log.Error("conn.Do(ZRANGEBYSCORE %v) error(%v)", keyIndex(business, mid), err)
+			return
+		}
+		count++
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < count; i++ {
+		var values []int64
+		values, err = redis.Int64s(conn.Receive())
+		if err != nil {
+			if err == redis.ErrNil {
+				err = nil
+				continue
+			}
+			log.Error("receive error(%v)", err)
+			return
+		}
+		if len(values) == 0 {
+			continue
+		}
+		if res == nil {
+			res = make(map[string][]int64)
+		}
+		res[businesses[i]] = values
+	}
+	return
+}
+
+// SetUserHideCache set the user hide to redis.
+func (d *Dao) SetUserHideCache(c context.Context, mid, value int64) (err error) {
+	key := keySwitch(mid)
+	conn := d.redis.Get(c)
+	if _, err = conn.Do("HSET", key, mid%_bucket, value); err != nil {
+		log.Error("conn.Do(HSET %s,%d) error(%v)", key, value, err)
+	}
+	conn.Close()
+	return
+}
+
+// UserHideCache return user hide state from redis.
+func (d *Dao) UserHideCache(c context.Context, mid int64) (value int64, err error) {
+	key := keySwitch(mid)
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if value, err = redis.Int64(conn.Do("HGET", key, mid%_bucket)); err != nil {
+		if err == redis.ErrNil {
+			return model.HideStateNotFound, nil
+		}
+		log.Error("conn.Do(HGET %s) error(%v)", key, err)
+	}
+	return
+}
+
+// HistoriesCache return the user histories from redis.
+func (d *Dao) HistoriesCache(c context.Context, mid int64, hs map[string][]int64) (res map[string]map[int64]*model.History, err error) {
+	var (
+		values, businesses []string
+		aid                int64
+		k                  int
+	)
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	for business, aids := range hs {
+		businesses = append(businesses, business)
+		key := keyHistory(business, mid)
+		args := []interface{}{key}
+		for _, aid := range aids {
+			args = append(args, aid)
+		}
+		if err = conn.Send("HMGET", args...); err != nil {
+			log.Error("conn.Send(HMGET %v %v) error(%v)", key, aids, err)
+			return
+		}
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < len(hs); i++ {
+		if values, err = redis.Strings(conn.Receive()); err != nil {
+			log.Error("conn.Receive error(%v)", err)
+			if err == redis.ErrNil {
+				continue
+			}
+			return
+		}
+		if res == nil {
+			res = make(map[string]map[int64]*model.History)
+		}
+		business := businesses[i]
+		for k, aid = range hs[business] {
+			if values[k] == "" {
+				continue
+			}
+			h := &model.History{}
+			if err = json.Unmarshal([]byte(values[k]), h); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", values[k], err)
+				err = nil
+				continue
+			}
+			h.BusinessID = d.BusinessNames[h.Business].ID
+			if res[business] == nil {
+				res[business] = make(map[int64]*model.History)
+			}
+			res[business][aid] = h
+		}
+	}
+	return
+}
+
+// ClearHistoryCache clear the user redis.
+func (d *Dao) ClearHistoryCache(c context.Context, mid int64, businesses []string) (err error) {
+	var conn = d.redis.Get(c)
+	var count int
+	defer conn.Close()
+	for _, business := range businesses {
+		idxKey := keyIndex(business, mid)
+		key := keyHistory(business, mid)
+		if err = conn.Send("DEL", idxKey); err != nil {
+			log.Error("conn.Send(DEL %s) error(%v)", idxKey, err)
+			return
+		}
+		count++
+		if err = conn.Send("DEL", key); err != nil {
+			log.Error("conn.Send(DEL %s) error(%v)", key, err)
+			return
+		}
+		count++
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < count; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// DelHistoryCache delete the history redis.
+func (d *Dao) DelHistoryCache(c context.Context, arg *pb.DelHistoriesReq) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	var count int
+	for _, r := range arg.Records {
+		var (
+			indxKey = keyIndex(r.Business, arg.Mid)
+			key     = keyHistory(r.Business, arg.Mid)
+		)
+		if err = conn.Send("ZREM", indxKey, r.ID); err != nil {
+			log.Error("conn.Send(ZREM %s,%v) error(%v)", indxKey, r.ID, err)
+			return
+		}
+		count++
+		if err = conn.Send("HDEL", key, r.ID); err != nil {
+			log.Error("conn.Send(HDEL %s,%v) error(%v)", key, r.ID, err)
+			return
+		}
+		count++
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < count; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// AddHistoryCache add the history to redis.
+func (d *Dao) AddHistoryCache(c context.Context, h *pb.AddHistoryReq) (err error) {
+	var (
+		b           []byte
+		mid         = h.Mid
+		idxKey, key = keyIndex(h.Business, mid), keyHistory(h.Business, mid)
+	)
+	if b, err = json.Marshal(h); err != nil {
+		return
+	}
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if err = conn.Send("ZADD", idxKey, h.ViewAt, h.Kid); err != nil {
+		log.Error("conn.Send(ZADD %s,%d) error(%v)", key, h.Kid, err)
+		return
+	}
+	if err = conn.Send("HSET", key, h.Kid, string(b)); err != nil {
+		log.Error("conn.Send(HSET %s,%d) error(%v)", key, h.Kid, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", idxKey, d.redisExpire); err != nil {
+		log.Error("conn.Send(EXPIRE) error(%v)", err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
+		log.Error("conn.Send(EXPIRE) error(%v)", err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < 2+2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// AddHistoriesCache add the user to redis.
+func (d *Dao) AddHistoriesCache(c context.Context, hs []*pb.AddHistoryReq) (err error) {
+	if len(hs) == 0 {
+		return
+	}
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	var count int
+	for _, h := range hs {
+		var (
+			b           []byte
+			mid         = h.Mid
+			idxKey, key = keyIndex(h.Business, mid), keyHistory(h.Business, mid)
+		)
+		if b, err = json.Marshal(h); err != nil {
+			continue
+		}
+		if err = conn.Send("ZADD", idxKey, h.ViewAt, h.Kid); err != nil {
+			log.Error("conn.Send(ZADD %s,%d) error(%v)", key, h.Kid, err)
+			continue
+		}
+		count++
+		if err = conn.Send("HSET", key, h.Kid, string(b)); err != nil {
+			log.Error("conn.Send(HSET %s,%d) error(%v)", key, h.Kid, err)
+			continue
+		}
+		count++
+		if err = conn.Send("EXPIRE", idxKey, d.redisExpire); err != nil {
+			log.Error("conn.Send(EXPIRE) error(%v)", err)
+			continue
+		}
+		count++
+		if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
+			log.Error("conn.Send(EXPIRE) error(%v)", err)
+			continue
+		}
+		count++
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < count; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// TrimCache trim history.
+func (d *Dao) TrimCache(c context.Context, business string, mid int64, limit int) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	aids, err := redis.Int64s(conn.Do("ZRANGE", keyIndex(business, mid), 0, -limit-1))
+	if err != nil {
+		log.Error("conn.Do(ZRANGE %v) error(%v)", keyIndex(business, mid), err)
+		return
+	}
+	if len(aids) == 0 {
+		return
+	}
+	return d.DelCache(c, business, mid, aids)
+}
+
+// DelCache delete the history redis.
+func (d *Dao) DelCache(c context.Context, business string, mid int64, aids []int64) (err error) {
+	var (
+		key1  = keyIndex(business, mid)
+		key2  = keyHistory(business, mid)
+		args1 = []interface{}{key1}
+		args2 = []interface{}{key2}
+	)
+	for _, aid := range aids {
+		args1 = append(args1, aid)
+		args2 = append(args2, aid)
+	}
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if err = conn.Send("ZREM", args1...); err != nil {
+		log.Error("conn.Send(ZREM %s,%v) error(%v)", key1, aids, err)
+		return
+	}
+	if err = conn.Send("HDEL", args2...); err != nil {
+		log.Error("conn.Send(HDEL %s,%v) error(%v)", key2, aids, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// pingRedis ping redis.
+func (d *Dao) pingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	if _, err = conn.Do("SET", "PING", "PONG"); err != nil {
+		log.Error("redis: conn.Do(SET,PING,PONG) error(%v)", err)
+	}
+	conn.Close()
+	return
+}

app/service/main/history/dao/redis_test.go

@@ -0,0 +1,130 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	pb "go-common/app/service/main/history/api/grpc"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoListCache(t *testing.T) {
+	var (
+		c        = context.Background()
+		business = "pgc"
+		mid      = int64(1)
+		start    = int64(0)
+		his      = []*pb.AddHistoryReq{{
+			Mid:      1,
+			Business: "pgc",
+			Kid:      1,
+			Aid:      2,
+			Sid:      3,
+		},
+		}
+		h = &pb.AddHistoryReq{
+			Mid:      2,
+			Business: "pgc",
+			Kid:      1,
+			Aid:      2,
+			Sid:      3,
+		}
+	)
+	convey.Convey("add his", t, func() {
+		convey.So(d.AddHistoriesCache(c, his), convey.ShouldBeNil)
+		convey.So(d.AddHistoryCache(c, h), convey.ShouldBeNil)
+		convey.Convey("ListCacheByTime", func(ctx convey.C) {
+			aids, err := d.ListCacheByTime(c, business, mid, start)
+			ctx.Convey("Then err should be nil.aids should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(aids, convey.ShouldNotBeEmpty)
+			})
+		})
+		convey.Convey("ListsCacheByTime", func(ctx convey.C) {
+			var (
+				c          = context.Background()
+				businesses = []string{"pgc"}
+				viewAt     = int64(100)
+				ps         = int64(1)
+			)
+			res, err := d.ListsCacheByTime(c, businesses, mid, viewAt, ps)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+
+		convey.Convey("HistoriesCache", func(ctx convey.C) {
+			var hs = map[string][]int64{"pgc": {1}}
+			res, err := d.HistoriesCache(c, 2, hs)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+
+		convey.Convey("ClearHistoryCache", func(ctx convey.C) {
+			err := d.ClearHistoryCache(c, mid, []string{business})
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+		convey.Convey("DelHistoryCache", func(ctx convey.C) {
+			ctx.So(d.DelHistoryCache(c, &pb.DelHistoriesReq{
+				Mid: 1, Records: []*pb.DelHistoriesReq_Record{{ID: 1, Business: "pgc"}},
+			}), convey.ShouldBeNil)
+		})
+		convey.Convey("TrimCache", func(ctx convey.C) {
+			err := d.TrimCache(c, business, mid, 10)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+
+	})
+}
+
+func TestDaoDelCache(t *testing.T) {
+	var (
+		c        = context.Background()
+		business = "pgc"
+		mid      = int64(1)
+		aids     = []int64{1}
+	)
+	convey.Convey("DelCache", t, func(ctx convey.C) {
+		err := d.DelCache(c, business, mid, aids)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoSetUserHideCache(t *testing.T) {
+	var (
+		c     = context.Background()
+		mid   = int64(1)
+		value = int64(1)
+	)
+	convey.Convey("SetUserHideCache", t, func(ctx convey.C) {
+		err := d.SetUserHideCache(c, mid, value)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoUserHideCache(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(1)
+	)
+	convey.Convey("UserHideCache", t, func(ctx convey.C) {
+		value, err := d.UserHideCache(c, mid)
+		ctx.Convey("Then err should be nil.value should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(value, convey.ShouldNotBeNil)
+			// ctx.So(value, convey.ShouldEqual, 1)
+		})
+	})
+}

app/service/main/history/dao/tidb.go

@@ -0,0 +1,208 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	pb "go-common/app/service/main/history/api/grpc"
+	"go-common/app/service/main/history/model"
+	"go-common/library/database/sql"
+	"go-common/library/database/tidb"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+var (
+	_addHistorySQL = "INSERT INTO histories(mid, kid, business_id, aid, sid, epid, sub_type, cid, device, progress, view_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)" +
+		"ON DUPLICATE KEY UPDATE aid =?, sid=?, epid=?, sub_type=?, cid=?, device=?, progress=?, view_at=?"
+	_businessesSQL        = "SELECT id, name, ttl FROM business"
+	_historiesSQL         = "SELECT ctime, mtime, business_id, kid, aid, sid, epid, sub_type, cid, device, progress, view_at FROM histories WHERE mid=? AND view_at < ? ORDER BY view_at DESC LIMIT ?"
+	_partHistoriesSQL     = "SELECT ctime, mtime, business_id, kid, aid, sid, epid, sub_type, cid, device, progress, view_at FROM histories WHERE mid=? AND business_id in (%s) AND view_at < ? ORDER BY view_at DESC LIMIT ? "
+	_queryHistoriesSQL    = "SELECT ctime, mtime, business_id, kid, aid, sid, epid, sub_type, cid, device, progress, view_at FROM histories WHERE mid=? AND kid in (%s) AND business_id = ?"
+	_historySQL           = "SELECT ctime, mtime, business_id, kid, aid, sid, epid, sub_type, cid, device, progress, view_at FROM histories WHERE mid=? AND kid = ? AND business_id = ?"
+	_deleteHistoriesSQL   = "DELETE FROM histories WHERE mid = ? AND kid = ? AND business_id = ?"
+	_clearHistoriesSQL    = "DELETE FROM histories WHERE mid = ? AND business_id in (%s)"
+	_clearAllHistoriesSQL = "DELETE FROM histories WHERE mid = ?"
+	_userHide             = "SELECT hide FROM users WHERE mid = ?"
+	_updateUserHide       = "INSERT INTO users(mid, hide) VALUES(?,?) ON DUPLICATE KEY UPDATE hide =?"
+)
+
+// AddHistories add histories to db
+func (d *Dao) AddHistories(c context.Context, hs []*model.History) (err error) {
+	if len(hs) == 0 {
+		return
+	}
+	var tx *tidb.Tx
+	if tx, err = d.tidb.Begin(c); err != nil {
+		log.Error("tx.BeginTran() error(%v)", err)
+		return
+	}
+	for _, h := range hs {
+		if _, err = tx.Stmts(d.insertStmt).Exec(c, h.Mid, h.Kid, h.BusinessID, h.Aid, h.Sid, h.Epid, h.SubType, h.Cid, h.Device, h.Progress, h.ViewAt,
+			h.Aid, h.Sid, h.Epid, h.SubType, h.Cid, h.Device, h.Progress, h.ViewAt); err != nil {
+			log.Errorv(c, log.D{Key: "mid", Value: h.Mid}, log.D{Key: "err", Value: err}, log.D{Key: "detail", Value: h})
+			tx.Rollback()
+			return
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("add histories commit(%+v) err: %v", hs, err)
+	}
+	return
+}
+
+// QueryBusinesses business
+func (d *Dao) QueryBusinesses(c context.Context) (res []*model.Business, err error) {
+	var rows *tidb.Rows
+	if rows, err = d.businessesStmt.Query(c); err != nil {
+		log.Error("db.businessesStmt.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		b := &model.Business{}
+		if err = rows.Scan(&b.ID, &b.Name, &b.TTL); err != nil {
+			log.Error("rows.Business.Scan error(%v)", err)
+			return
+		}
+		res = append(res, b)
+	}
+	err = rows.Err()
+	return
+}
+
+// UserHistories get histories by time
+func (d *Dao) UserHistories(c context.Context, businesses []string, mid, viewAt, ps int64) (res map[string][]*model.History, err error) {
+	var rows *tidb.Rows
+	if len(businesses) == 0 {
+		if rows, err = d.historiesStmt.Query(c, mid, viewAt, ps); err != nil {
+			log.Error("db.Histories.Query error(%v)", err)
+			return
+		}
+	} else {
+		var ids []int64
+		for _, b := range businesses {
+			ids = append(ids, d.BusinessNames[b].ID)
+		}
+		sqlStr := fmt.Sprintf(_partHistoriesSQL, xstr.JoinInts(ids))
+		if rows, err = d.tidb.Query(c, sqlStr, mid, viewAt, ps); err != nil {
+			log.Error("UserHistories(%v,%d,%d,%d),db.Histories.Query error(%v)", businesses, mid, viewAt, ps, err)
+			return
+		}
+	}
+	defer rows.Close()
+	for rows.Next() {
+		b := &model.History{Mid: mid}
+		if err = rows.Scan(&b.Ctime, &b.Mtime, &b.BusinessID, &b.Kid, &b.Aid, &b.Sid, &b.Epid, &b.SubType, &b.Cid, &b.Device, &b.Progress, &b.ViewAt); err != nil {
+			log.Error("UserHistories(%v,%d,%d,%d),rows.Scan error(%v)", businesses, mid, viewAt, ps, err)
+			if strings.Contains(fmt.Sprintf("%v", err), "18446744073709551615") {
+				err = nil
+				continue
+			}
+			return
+		}
+		b.Business = d.Businesses[b.BusinessID].Name
+		if res == nil {
+			res = make(map[string][]*model.History)
+		}
+		res[b.Business] = append(res[b.Business], b)
+	}
+	err = rows.Err()
+	return
+}
+
+// Histories get histories by id
+func (d *Dao) Histories(c context.Context, business string, mid int64, ids []int64) (res map[int64]*model.History, err error) {
+	var rows *tidb.Rows
+	bid := d.BusinessNames[business].ID
+	if len(ids) == 1 {
+		if rows, err = d.historyStmt.Query(c, mid, ids[0], bid); err != nil {
+			log.Error("db.Histories.Query error(%v)", err)
+			return
+		}
+	} else {
+		sqlStr := fmt.Sprintf(_queryHistoriesSQL, xstr.JoinInts(ids))
+		if rows, err = d.tidb.Query(c, sqlStr, mid, bid); err != nil {
+			log.Error("tidb.Histories.Query error(%v)", err)
+			return
+		}
+	}
+	defer rows.Close()
+	for rows.Next() {
+		b := &model.History{Mid: mid}
+		if err = rows.Scan(&b.Ctime, &b.Mtime, &b.BusinessID, &b.Kid, &b.Aid, &b.Sid, &b.Epid, &b.SubType, &b.Cid, &b.Device, &b.Progress, &b.ViewAt); err != nil {
+			log.Error("rows.Business.Scan error(%v)", err)
+			return
+		}
+		b.Business = d.Businesses[b.BusinessID].Name
+		if res == nil {
+			res = make(map[int64]*model.History)
+		}
+		res[b.Kid] = b
+	}
+	err = rows.Err()
+	return
+}
+
+// DeleteHistories .
+func (d *Dao) DeleteHistories(c context.Context, h *pb.DelHistoriesReq) (err error) {
+	var tx *tidb.Tx
+	if tx, err = d.tidb.Begin(c); err != nil {
+		log.Error("tx.BeginTran() error(%v)", err)
+		return
+	}
+	for _, r := range h.Records {
+		_, err = tx.Stmts(d.deleteHistoriesStmt).Exec(c, h.Mid, r.ID, d.BusinessNames[r.Business].ID)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("DeleteHistories(%+v),commit err:%+v", h, err)
+	}
+	return
+}
+
+// ClearHistory clear histories
+func (d *Dao) ClearHistory(c context.Context, mid int64, businesses []string) (err error) {
+	var ids []string
+	for _, b := range businesses {
+		ids = append(ids, strconv.FormatInt(d.BusinessNames[b].ID, 10))
+	}
+	sqlStr := fmt.Sprintf(_clearHistoriesSQL, strings.Join(ids, ","))
+	if _, err = d.tidb.Exec(c, sqlStr, mid); err != nil {
+		log.Error("mid: %d clear(%v) err: %v", mid, businesses, err)
+	}
+	return
+}
+
+// ClearAllHistory clear all histories
+func (d *Dao) ClearAllHistory(c context.Context, mid int64) (err error) {
+	if _, err = d.clearAllHistoriesStmt.Exec(c, mid); err != nil {
+		log.Error("mid: %d clear all err: %v", mid, err)
+	}
+	return
+}
+
+// UpdateUserHide update user hide
+func (d *Dao) UpdateUserHide(c context.Context, mid int64, hide bool) (err error) {
+	if _, err = d.updateUserHideStmt.Exec(c, mid, hide, hide); err != nil {
+		log.Error("mid: %d updateUserHide(%v) err: %v", mid, hide, err)
+	}
+	return
+}
+
+// UserHide get user hide
+func (d *Dao) UserHide(c context.Context, mid int64) (hide bool, err error) {
+	if err = d.userHideStmt.QueryRow(c, mid).Scan(&hide); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+		log.Error("mid: %d UserHide err: %v", mid, err)
+	}
+	return
+}

app/service/main/history/dao/tidb_test.go

@@ -0,0 +1,183 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	pb "go-common/app/service/main/history/api/grpc"
+	"go-common/app/service/main/history/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+var _history = &model.History{
+	Mid:        1,
+	BusinessID: 4,
+	Business:   "pgc",
+	Kid:        2,
+	Aid:        3,
+	Sid:        4,
+	Epid:       5,
+	Cid:        6,
+	SubType:    7,
+	Device:     8,
+	Progress:   9,
+	ViewAt:     10,
+}
+var hs = []*model.History{_history}
+
+func TestDaoAddHistories(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("AddHistories", t, func(ctx convey.C) {
+		err := d.AddHistories(c, hs)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoQueryBusinesses(t *testing.T) {
+	var (
+		c = context.Background()
+	)
+	convey.Convey("QueryBusinesses", t, func(ctx convey.C) {
+		res, err := d.QueryBusinesses(c)
+		ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+		})
+	})
+}
+
+func TestDaoUserHistories(t *testing.T) {
+	var (
+		c      = context.Background()
+		mid    = int64(1)
+		viewAt = time.Now().Unix()
+		ps     = int64(1)
+	)
+	convey.Convey("UserHistories all business", t, func(ctx convey.C) {
+		var businesses []string
+		res, err := d.UserHistories(c, businesses, mid, viewAt, ps)
+		ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+			res["pgc"][0].Ctime = 0
+			res["pgc"][0].Mtime = 0
+			ctx.So(res, convey.ShouldResemble, map[string][]*model.History{
+				"pgc": {_history},
+			})
+		})
+	})
+
+	convey.Convey("UserHistories one business", t, func(ctx convey.C) {
+		var businesses = []string{"pgc"}
+		res, err := d.UserHistories(c, businesses, mid, viewAt, ps)
+		ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+			res["pgc"][0].Ctime = 0
+			res["pgc"][0].Mtime = 0
+			ctx.So(res, convey.ShouldResemble, map[string][]*model.History{
+				"pgc": {_history},
+			})
+		})
+	})
+}
+
+func TestDaoHistories(t *testing.T) {
+	var (
+		c        = context.Background()
+		business = "pgc"
+		mid      = int64(1)
+		ids      = []int64{2}
+	)
+	convey.Convey("Histories", t, func(ctx convey.C) {
+		res, err := d.Histories(c, business, mid, ids)
+		ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+		})
+	})
+}
+
+func TestDaoDeleteHistories(t *testing.T) {
+	var (
+		c = context.Background()
+		h = &pb.DelHistoriesReq{Mid: 1, Records: []*pb.DelHistoriesReq_Record{
+			{Business: "pgc", ID: 2},
+		},
+		}
+	)
+	convey.Convey("DeleteHistories", t, func(ctx convey.C) {
+		d.AddHistories(c, hs)
+		err := d.DeleteHistories(c, h)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoClearHistory(t *testing.T) {
+	var (
+		c          = context.Background()
+		mid        = int64(1)
+		businesses = []string{"pgc"}
+	)
+	convey.Convey("ClearHistory", t, func(ctx convey.C) {
+		d.AddHistories(c, hs)
+		err := d.ClearHistory(c, mid, businesses)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoClearAllHistory(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(1)
+	)
+	convey.Convey("ClearAllHistory", t, func(ctx convey.C) {
+		err := d.ClearAllHistory(c, mid)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoUpdateUserHide(t *testing.T) {
+	var (
+		c    = context.Background()
+		mid  = int64(1)
+		hide = true
+	)
+	convey.Convey("UpdateUserHide", t, func(ctx convey.C) {
+		err := d.UpdateUserHide(c, mid, hide)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoUserHide(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(1)
+	)
+	convey.Convey("UserHide", t, func(ctx convey.C) {
+		hide, err := d.UserHide(c, mid)
+		ctx.Convey("Then err should be nil.hide should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(hide, convey.ShouldEqual, true)
+		})
+		hide, err = d.UserHide(c, 200)
+		ctx.Convey("not found .Then err should be nil.hide should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(hide, convey.ShouldBeFalse)
+		})
+	})
+}