Create & Init Project...

app/job/main/favorite/dao/stat/BUILD

@@ -0,0 +1,59 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "memcache.go",
+        "mysql.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/favorite/dao/stat",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/favorite/conf:go_default_library",
+        "//app/service/main/favorite/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "memcache_test.go",
+        "mysql_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/favorite/conf:go_default_library",
+        "//app/service/main/favorite/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/favorite/dao/stat/dao.go

@@ -0,0 +1,45 @@
+package stat
+
+import (
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/favorite/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+)
+
+// Dao favorite dao.
+type Dao struct {
+	db          *sql.DB
+	redis       *redis.Pool
+	mc          *memcache.Pool
+	redisExpire int
+	ipExpire    int
+	buvidExpire int
+	mcExpire    int32
+}
+
+// New new a dao and return.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		db: sql.NewMySQL(c.DB.Fav),
+		// redis
+		redis:       redis.NewPool(c.Redis.Config),
+		redisExpire: int(time.Duration(c.Redis.Expire) / time.Second),
+		ipExpire:    int(time.Duration(c.Redis.IPExpire) / time.Second),
+		buvidExpire: int(time.Duration(c.Redis.BuvidExpire) / time.Second),
+		// memcache
+		mc:       memcache.NewPool(c.Memcache.Config),
+		mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
+	}
+	return
+}
+
+// hit .
+func hit(id int64) (fid int64, table string) {
+	fid = id / _folderStatSharding
+	table = fmt.Sprintf("%02d", id%_folderStatSharding)
+	return
+}

app/job/main/favorite/dao/stat/dao_test.go

@@ -0,0 +1,35 @@
+package stat
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/job/main/favorite/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.community.favorite-job")
+		flag.Set("conf_token", "29adace99d5c5be327ae4a6b70c6582d")
+		flag.Set("tree_id", "2295")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/favorite-job-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/favorite/dao/stat/memcache.go

@@ -0,0 +1,57 @@
+package stat
+
+import (
+	"context"
+	"fmt"
+
+	favmdl "go-common/app/service/main/favorite/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_folderStat = "ft_%s_%d" // key:ft_{mid%100}_{fid},value:{*FodlerStat}.pb
+)
+
+// folderStatMcKey
+func folderStatMcKey(table string, fid int64) string {
+	return fmt.Sprintf(_folderStat, table, fid)
+}
+
+// SetFolderStatMc add folder stat mc cache.
+func (d *Dao) SetFolderStatMc(c context.Context, id int64, s *favmdl.Folder) (err error) {
+	fid, table := hit(id)
+	conn := d.mc.Get(c)
+	defer conn.Close()
+
+	item := &memcache.Item{Key: folderStatMcKey(table, fid), Object: s, Flags: memcache.FlagProtobuf, Expiration: d.mcExpire}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%s) error(%v)", folderStatMcKey(table, fid), err)
+	}
+	return
+}
+
+// FolderStatMc return one folder stat from mc.
+func (d *Dao) FolderStatMc(c context.Context, id int64) (f *favmdl.Folder, err error) {
+	fid, table := hit(id)
+	var (
+		key  = folderStatMcKey(table, fid)
+		item *memcache.Item
+		conn = d.mc.Get(c)
+	)
+	defer conn.Close()
+	if item, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("memcache.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	f = new(favmdl.Folder)
+	if err = conn.Scan(item, f); err != nil {
+		log.Error("conn.Scan(%s) error(%v)", item.Value, err)
+		f = nil
+	}
+	return
+}

app/job/main/favorite/dao/stat/memcache_test.go

@@ -0,0 +1,57 @@
+package stat
+
+import (
+	"context"
+	"testing"
+
+	favmdl "go-common/app/service/main/favorite/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestStatfolderStatMcKey(t *testing.T) {
+	convey.Convey("folderStatMcKey", t, func(convCtx convey.C) {
+		var (
+			table = ""
+			fid   = int64(111)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			p1 := folderStatMcKey(table, fid)
+			convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
+				convCtx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestStatSetFolderStatMc(t *testing.T) {
+	convey.Convey("SetFolderStatMc", t, func(convCtx convey.C) {
+		var (
+			c  = context.Background()
+			id = int64(111)
+			s  = &favmdl.Folder{}
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			err := d.SetFolderStatMc(c, id, s)
+			convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestStatFolderStatMc(t *testing.T) {
+	convey.Convey("FolderStatMc", t, func(convCtx convey.C) {
+		var (
+			c  = context.Background()
+			id = int64(111)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			f, err := d.FolderStatMc(c, id)
+			convCtx.Convey("Then err should be nil.f should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+				convCtx.So(f, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/favorite/dao/stat/mysql.go

@@ -0,0 +1,73 @@
+package stat
+
+import (
+	"context"
+	"fmt"
+
+	favmdl "go-common/app/service/main/favorite/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_folderStatSharding int64 = 100
+
+	// stat
+	_statSQL = "SELECT play,fav,share from fav_folder_stat_%s WHERE fid=?"
+
+	_upsertPlaySQL  = "INSERT INTO fav_folder_stat_%s (fid,play) VALUES(?,?) ON DUPLICATE KEY UPDATE play=?"
+	_upsertFavSQL   = "INSERT INTO fav_folder_stat_%s (fid,fav) VALUES(?,?) ON DUPLICATE KEY UPDATE fav=?"
+	_upsertShareSQL = "INSERT INTO fav_folder_stat_%s (fid,share) VALUES(?,?) ON DUPLICATE KEY UPDATE share=?"
+)
+
+// UpdateFav updates stat in db.
+func (d *Dao) UpdateFav(c context.Context, oid, count int64) (rows int64, err error) {
+	fid, table := hit(oid)
+	res, err := d.db.Exec(c, fmt.Sprintf(_upsertFavSQL, table), fid, count, count)
+	if err != nil {
+		log.Error("UpdateFav(%d,%d) error(%+v)", oid, count, err)
+		return
+	}
+	rows, err = res.RowsAffected()
+	return
+}
+
+// UpdateShare updates stat in db.
+func (d *Dao) UpdateShare(c context.Context, oid, count int64) (rows int64, err error) {
+	fid, table := hit(oid)
+	res, err := d.db.Exec(c, fmt.Sprintf(_upsertShareSQL, table), fid, count, count)
+	if err != nil {
+		log.Error("UpdateShare(%d,%d) error(%+v)", oid, count, err)
+		return
+	}
+	rows, err = res.RowsAffected()
+	return
+}
+
+// UpdatePlay updates stat in db.
+func (d *Dao) UpdatePlay(c context.Context, oid, count int64) (rows int64, err error) {
+	fid, table := hit(oid)
+	res, err := d.db.Exec(c, fmt.Sprintf(_upsertPlaySQL, table), fid, count, count)
+	if err != nil {
+		log.Error("UpdatePlay(%d) error(%+v)", oid, err)
+		return
+	}
+	rows, err = res.RowsAffected()
+	return
+}
+
+// Stat return stat count from mysql.
+func (d *Dao) Stat(c context.Context, oid int64) (f *favmdl.Folder, err error) {
+	fid, table := hit(oid)
+	f = &favmdl.Folder{}
+	row := d.db.QueryRow(c, fmt.Sprintf(_statSQL, table), fid)
+	if err = row.Scan(&f.PlayCount, &f.FavedCount, &f.ShareCount); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			f = nil
+			return
+		}
+		log.Error("Stat(%v) error(%+v)", f, err)
+	}
+	return
+}

app/job/main/favorite/dao/stat/mysql_test.go

@@ -0,0 +1,75 @@
+package stat
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestStatUpdateFav(t *testing.T) {
+	convey.Convey("UpdateFav", t, func(convCtx convey.C) {
+		var (
+			c     = context.Background()
+			oid   = int64(111)
+			count = int64(1)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			rows, err := d.UpdateFav(c, oid, count)
+			convCtx.Convey("Then err should be nil.rows should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+				convCtx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestStatUpdateShare(t *testing.T) {
+	convey.Convey("UpdateShare", t, func(convCtx convey.C) {
+		var (
+			c     = context.Background()
+			oid   = int64(111)
+			count = int64(1)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			rows, err := d.UpdateShare(c, oid, count)
+			convCtx.Convey("Then err should be nil.rows should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+				convCtx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestStatUpdatePlay(t *testing.T) {
+	convey.Convey("UpdatePlay", t, func(convCtx convey.C) {
+		var (
+			c     = context.Background()
+			oid   = int64(111)
+			count = int64(1)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			rows, err := d.UpdatePlay(c, oid, count)
+			convCtx.Convey("Then err should be nil.rows should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+				convCtx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestStatStat(t *testing.T) {
+	convey.Convey("Stat", t, func(convCtx convey.C) {
+		var (
+			c   = context.Background()
+			oid = int64(111)
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			f, err := d.Stat(c, oid)
+			convCtx.Convey("Then err should be nil.f should not be nil.", func(convCtx convey.C) {
+				convCtx.So(err, convey.ShouldBeNil)
+				convCtx.So(f, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/favorite/dao/stat/redis.go

@@ -0,0 +1,103 @@
+package stat
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_ipBanKey    = "ipb_"
+	_buvidBanKey = "bvb_"
+)
+
+func ipBanKey(id int64, ip string) (key string) {
+	key = _ipBanKey + strconv.FormatInt(id, 10) + ip
+	return
+}
+
+func buvidBanKey(id, mid int64, ip, buvid string) (key string) {
+	key = _buvidBanKey + strconv.FormatInt(id, 10) + ip + buvid
+	if mid != 0 {
+		key += strconv.FormatInt(mid, 10)
+	}
+	return
+}
+
+// IPBan intercepts illegal views.
+func (d *Dao) IPBan(c context.Context, id int64, ip string) (ban bool) {
+	var (
+		err   error
+		exist bool
+		key   = ipBanKey(id, ip)
+		conn  = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
+		log.Error("conn.Do(EXISTS, %s) error(%+v)", key, err)
+		return
+	}
+	if exist {
+		ban = true
+		return
+	}
+	if err = conn.Send("SET", key, "1"); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.ipExpire); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%+v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%+v)", err)
+			return
+		}
+	}
+	return
+}
+
+// BuvidBan intercepts illegal views.
+func (d *Dao) BuvidBan(c context.Context, id, mid int64, ip, buvid string) (ban bool) {
+	var (
+		err   error
+		exist bool
+		key   = buvidBanKey(id, mid, ip, buvid)
+		conn  = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
+		log.Error("conn.Do(EXISTS, %s) error(%+v)", key, err)
+		return
+	}
+	if exist {
+		ban = true
+		return
+	}
+	if err = conn.Send("SET", key, "1"); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.buvidExpire); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%+v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%+v)", err)
+			return
+		}
+	}
+	return
+}

app/job/main/favorite/dao/stat/redis_test.go

@@ -0,0 +1,74 @@
+package stat
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestStatipBanKey(t *testing.T) {
+	convey.Convey("ipBanKey", t, func(convCtx convey.C) {
+		var (
+			id = int64(0)
+			ip = ""
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			key := ipBanKey(id, ip)
+			convCtx.Convey("Then key should not be nil.", func(convCtx convey.C) {
+				convCtx.So(key, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestStatbuvidBanKey(t *testing.T) {
+	convey.Convey("buvidBanKey", t, func(convCtx convey.C) {
+		var (
+			id    = int64(0)
+			mid   = int64(0)
+			ip    = ""
+			buvid = ""
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			key := buvidBanKey(id, mid, ip, buvid)
+			convCtx.Convey("Then key should not be nil.", func(convCtx convey.C) {
+				convCtx.So(key, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestStatIPBan(t *testing.T) {
+	convey.Convey("IPBan", t, func(convCtx convey.C) {
+		var (
+			c  = context.Background()
+			id = int64(0)
+			ip = ""
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			ban := d.IPBan(c, id, ip)
+			convCtx.Convey("Then ban should not be nil.", func(convCtx convey.C) {
+				convCtx.So(ban, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestStatBuvidBan(t *testing.T) {
+	convey.Convey("BuvidBan", t, func(convCtx convey.C) {
+		var (
+			c     = context.Background()
+			id    = int64(0)
+			mid   = int64(0)
+			buvid = ""
+			ip    = ""
+		)
+		convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
+			ban := d.BuvidBan(c, id, mid, ip, buvid)
+			convCtx.Convey("Then ban should not be nil.", func(convCtx convey.C) {
+				convCtx.So(ban, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}