Create & Init Project...

app/service/bbq/recsys-recall/dao/BUILD

@@ -0,0 +1,38 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bloomfilter.go",
+        "dao.go",
+    ],
+    importpath = "go-common/app/service/bbq/recsys-recall/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/bbq/recsys-recall/conf:go_default_library",
+        "//app/service/bbq/recsys-recall/dao/cache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//vendor/github.com/Dai0522/go-hash/bloomfilter:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/bbq/recsys-recall/dao/cache:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/bbq/recsys-recall/dao/bloomfilter.go

@@ -0,0 +1,23 @@
+package dao
+
+import (
+	"context"
+	"go-common/library/cache/redis"
+
+	"github.com/Dai0522/go-hash/bloomfilter"
+)
+
+// LoadBloomFilter .
+func (d *Dao) LoadBloomFilter(ctx *context.Context, key string) (*bloomfilter.BloomFilter, error) {
+	conn := d.bfredis.Get(*ctx)
+	defer conn.Close()
+
+	var bf *bloomfilter.BloomFilter
+	// 获取mid维度
+	raw, err := redis.Bytes(conn.Do("GET", key))
+	if err != redis.ErrNil && raw != nil {
+		bf, err = bloomfilter.Load(&raw)
+	}
+
+	return bf, err
+}

app/service/bbq/recsys-recall/dao/cache/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["local.go"],
+    importpath = "go-common/app/service/bbq/recsys-recall/dao/cache",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//app/service/bbq/recsys-recall/conf:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/bbq/recsys-recall/dao/cache/local.go

@@ -0,0 +1,88 @@
+package cache
+
+import (
+	"sync"
+	"time"
+
+	"go-common/app/service/bbq/recsys-recall/conf"
+)
+
+// Solt .
+type Solt struct {
+	data       []byte
+	ctime      int64
+	maxAge     int64
+	lastUsed   int64
+	keepAlived int64
+}
+
+// LocalCache .
+type LocalCache struct {
+	d      map[string]*Solt
+	l1tags map[string]byte
+	l2tags map[string]byte
+	c      *conf.LocalCacheConfig
+	lock   *sync.RWMutex
+}
+
+// NewLocalCache .
+func NewLocalCache(c *conf.LocalCacheConfig) *LocalCache {
+	l1 := make(map[string]byte)
+	l2 := make(map[string]byte)
+	for _, v := range c.L1Tags {
+		l1[v] = byte(1)
+	}
+	for _, v := range c.L2Tags {
+		l2[v] = byte(1)
+	}
+
+	return &LocalCache{
+		d:      make(map[string]*Solt),
+		l1tags: l1,
+		l2tags: l2,
+		c:      c,
+		lock:   &sync.RWMutex{},
+	}
+}
+
+// Set .
+func (lc *LocalCache) Set(key string, val []byte) bool {
+	lc.lock.Lock()
+	defer lc.lock.Unlock()
+	keep := lc.c.Level3
+	if _, ok := lc.l1tags[key]; ok {
+		keep = lc.c.Level1
+	} else if _, ok := lc.l2tags[key]; ok {
+		keep = lc.c.Level2
+	}
+
+	lc.d[key] = &Solt{
+		data:       val,
+		ctime:      time.Now().Unix(),
+		maxAge:     int64(lc.c.MaxAge),
+		lastUsed:   time.Now().Unix(),
+		keepAlived: int64(keep),
+	}
+	return true
+}
+
+// Get .
+func (lc *LocalCache) Get(key string) []byte {
+	lc.lock.RLock()
+	defer lc.lock.RUnlock()
+
+	current := time.Now().Unix()
+	s := lc.d[key]
+	if s == nil {
+		return nil
+	}
+
+	keepAlived := s.keepAlived / int64(time.Second)
+	maxAge := s.maxAge / int64(time.Second)
+	if keepAlived < (current-s.lastUsed) || maxAge < (current-s.ctime) {
+		return nil
+	}
+
+	s.lastUsed = current
+	return s.data
+}

app/service/bbq/recsys-recall/dao/dao.go

@@ -0,0 +1,73 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/service/bbq/recsys-recall/conf"
+	xcache "go-common/app/service/bbq/recsys-recall/dao/cache"
+	"go-common/library/cache/redis"
+)
+
+// Dao dao
+type Dao struct {
+	c       *conf.Config
+	redis   *redis.Pool
+	bfredis *redis.Pool
+	lcache  *xcache.LocalCache
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:       c,
+		redis:   redis.NewPool(c.Redis),
+		bfredis: redis.NewPool(c.BFRedis),
+		lcache:  xcache.NewLocalCache(c.LocalCache),
+	}
+	return
+}
+
+// GetInvertedIndex 获取倒排索引
+func (d *Dao) GetInvertedIndex(ctx context.Context, key string, force bool) (b []byte, err error) {
+	if b = d.lcache.Get(key); b != nil && !force {
+		return
+	}
+
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+
+	for retry := 0; retry < 3; retry++ {
+		b, err = redis.Bytes(conn.Do("GET", key))
+		if err == redis.ErrNil {
+			b = make([]byte, 0)
+			return
+		}
+		if b != nil {
+			d.lcache.Set(key, b)
+			return
+		}
+	}
+
+	return
+}
+
+// SetInvertedIndex 更新倒排索引
+func (d *Dao) SetInvertedIndex(c context.Context, key string, value []byte) error {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+
+	_, err := conn.Do("SETEX", key, 86400, value)
+
+	return err
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return nil
+}