Create & Init Project...

app/job/bbq/recall/internal/dao/BUILD

@@ -0,0 +1,49 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "bloomfilter.go",
+        "dao.go",
+        "hdfs.go",
+        "inverted_index.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/recall/internal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/recall/internal/conf:go_default_library",
+        "//app/job/bbq/recall/internal/model:go_default_library",
+        "//app/job/bbq/recall/proto:go_default_library",
+        "//app/job/bbq/recall/proto/quality:go_default_library",
+        "//app/service/bbq/recsys-recall/api/grpc/v1:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//vendor/github.com/Dai0522/go-hash/bloomfilter:go_default_library",
+        "//vendor/github.com/golang/snappy:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/internal/dao/berserker.go

@@ -0,0 +1,14 @@
+package dao
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+)
+
+func (d *Dao) berserkerSign(ak, sk, dt, ver string) string {
+	str := fmt.Sprintf("%sappKey%stimestamp%sversion%s%s", sk, ak, dt, ver, sk)
+	b := md5.Sum([]byte(str))
+	sign := hex.EncodeToString(b[:])
+	return sign
+}

app/job/bbq/recall/internal/dao/bloomfilter.go

@@ -0,0 +1,68 @@
+package dao
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/Dai0522/go-hash/bloomfilter"
+)
+
+// FetchMidView .
+func (d *Dao) FetchMidView(c context.Context) (result []string, err error) {
+	dt := time.Now().AddDate(0, 0, -1).Format("20060102")
+	hdfs, err := d.scanHDFSPath(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/mid/")
+	if err != nil {
+		return
+	}
+	for _, v := range hdfs.Result {
+		raw, err := d.loadHDFSFile(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/mid/"+v)
+		if err != nil {
+			break
+		}
+		lines := strings.Split(string(*raw), "\n")
+		result = append(result, lines...)
+	}
+	return
+}
+
+// FetchBuvidView .
+func (d *Dao) FetchBuvidView(c context.Context) (result []string, err error) {
+	dt := time.Now().AddDate(0, 0, -1).Format("20060102")
+	hdfs, err := d.scanHDFSPath(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/buvid/")
+	if err != nil {
+		return
+	}
+	for _, v := range hdfs.Result {
+		raw, err := d.loadHDFSFile(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/buvid/"+v)
+		if err != nil {
+			break
+		}
+		lines := strings.Split(string(*raw), "\n")
+		result = append(result, lines...)
+	}
+	return
+}
+
+// InsertBloomFilter 构建BF，插入redis
+func (d *Dao) InsertBloomFilter(c context.Context, key string, svidList []uint64) error {
+	bfK := "BBQ:BF:V1:" + key
+	bf, err := bloomfilter.New(uint64(len(svidList)), 0.0001)
+	if err != nil {
+		return err
+	}
+	for _, v := range svidList {
+		bf.PutUint64(v)
+	}
+
+	b := bf.Serialized()
+	return d.SetBloomFilter(c, bfK, b)
+}
+
+// SetBloomFilter .
+func (d *Dao) SetBloomFilter(c context.Context, key string, b *[]byte) error {
+	conn := d.bfredis.Get(c)
+	defer conn.Close()
+	_, err := conn.Do("SETEX", []byte(key), 86400, *b)
+	return err
+}

app/job/bbq/recall/internal/dao/dao.go

@@ -0,0 +1,56 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/bbq/recall/internal/conf"
+	recall "go-common/app/service/bbq/recsys-recall/api/grpc/v1"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/net/rpc/warden"
+)
+
+// Dao dao
+type Dao struct {
+	c            *conf.Config
+	redis        *redis.Pool
+	bfredis      *redis.Pool
+	db           *xsql.DB
+	dbOffline    *xsql.DB
+	dbCms        *xsql.DB
+	recallClient recall.RecsysRecallClient
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:            c,
+		redis:        redis.NewPool(c.Redis),
+		bfredis:      redis.NewPool(c.BfRedis),
+		db:           xsql.NewMySQL(c.MySQL),
+		dbOffline:    xsql.NewMySQL(c.OfflineMySQL),
+		dbCms:        xsql.NewMySQL(c.CmsMySQL),
+		recallClient: newRecallClient(c.GRPCClient["recall"]),
+	}
+	return
+}
+
+func newRecallClient(cfg *conf.GRPCConfig) recall.RecsysRecallClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return recall.NewRecsysRecallClient(cc)
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(ctx context.Context) error {
+	// TODO: add mc,redis... if you use
+	return d.db.Ping(ctx)
+}

app/job/bbq/recall/internal/dao/hdfs.go

@@ -0,0 +1,66 @@
+package dao
+
+import (
+	"context"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/json-iterator/go"
+
+	"go-common/app/job/bbq/recall/internal/conf"
+	"go-common/app/job/bbq/recall/internal/model"
+)
+
+func (d *Dao) queryHDFS(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) {
+	dt := time.Now().Format("2006-01-02 15:04:05")
+	sign := d.berserkerSign(key.AppKey, key.Secret, dt, "1.0")
+
+	params := &url.Values{}
+	params.Set("appKey", key.AppKey)
+	params.Set("timestamp", dt)
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	params.Set("sign", sign)
+
+	fileSuffix := struct {
+		FileSuffix string `json:"fileSuffix"`
+	}{
+		FileSuffix: suffix,
+	}
+	j, err := jsoniter.Marshal(fileSuffix)
+	params.Set("query", string(j))
+
+	for retry := 0; retry < 3; retry++ {
+		resp, err := http.DefaultClient.Get(api + "?" + params.Encode())
+		if err != nil {
+			continue
+		}
+
+		b, err := ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err == nil && len(b) > 0 {
+			result = &b
+			break
+		}
+		// sleep 5s berserker limit
+		time.Sleep(5 * time.Second)
+	}
+
+	return
+}
+
+func (d *Dao) scanHDFSPath(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *model.HDFSResult, err error) {
+	b, err := d.queryHDFS(c, api, key, suffix)
+	if err != nil {
+		return
+	}
+	result = &model.HDFSResult{}
+	err = jsoniter.Unmarshal(*b, result)
+	return
+}
+
+func (d *Dao) loadHDFSFile(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) {
+	return d.queryHDFS(c, api, key, suffix)
+}

app/job/bbq/recall/internal/dao/inverted_index.go

@@ -0,0 +1,16 @@
+package dao
+
+import (
+	"context"
+
+	recall "go-common/app/service/bbq/recsys-recall/api/grpc/v1"
+)
+
+// SetInvertedIndex 倒排写入redis
+func (d *Dao) SetInvertedIndex(c context.Context, key string, svids []int64) error {
+	_, err := d.recallClient.NewIncomeVideo(c, &recall.NewIncomeVideoRequest{
+		Key:   key,
+		SVIDs: svids,
+	})
+	return err
+}

app/job/bbq/recall/internal/dao/video.go

@@ -0,0 +1,143 @@
+package dao
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/job/bbq/recall/internal/model"
+	"go-common/app/job/bbq/recall/proto"
+	"go-common/app/job/bbq/recall/proto/quality"
+	"go-common/library/log"
+
+	"github.com/golang/snappy"
+)
+
+const (
+	// _fetchVideo    = "select `id`, `title`, `content`, `mid`, `avid`, `cid`, `pubtime`, `ctime`, `mtime`, `duration`, `state`, `tid`, `sub_tid` from video where pubtime > ? limit ?, ?;"
+	_fetchVideo          = "select `svid`, `title`, `content`, `mid`, `avid`, `cid`, `pubtime`, `ctime`, `mtime`, `duration`, `state`, `tid`, `sub_tid` from video limit ?, ?;"
+	_fetchVideoTag       = "select `id`, `name`, `type` from `tag` where `id` = ? and `status` = 1;"
+	_fetchVideoTagAll    = "select `id`, `name`, `type` from `tag` where `status` = 1;"
+	_fetchVideoTextTag   = "select `tag` from `video_repository` where `svid` = ? limit 1;"
+	_queryVideoQuality   = "select `stat_info` from `video_forward_index_stat_info` where `svid` = ?;"
+	_fetchNewIncomeVideo = "select `svid` from `video` where ctime > ? and state in (%s);"
+)
+
+// FetchVideoInfo .
+func (d *Dao) FetchVideoInfo(c context.Context, offset, size int) (result []*model.Video, err error) {
+	// rows, err := d.db.Query(c, _fetchVideo, ptime.Format("2006-01-02 15:04:05"), offset, size)
+	rows, err := d.db.Query(c, _fetchVideo, offset, size)
+	if err != nil {
+		return nil, err
+	}
+
+	for rows.Next() {
+		tmp := &model.Video{}
+		if err = rows.Scan(&tmp.SVID, &tmp.Title, &tmp.Content, &tmp.MID, &tmp.AVID, &tmp.CID, &tmp.PubTime, &tmp.CTime, &tmp.MTime, &tmp.Duration, &tmp.State, &tmp.TID, &tmp.SubTID); err != nil {
+			log.Error("FetchVideoInfo: %v", err)
+			return nil, err
+		}
+		result = append(result, tmp)
+	}
+
+	return result, nil
+}
+
+// FetchVideoTagAll .
+func (d *Dao) FetchVideoTagAll(c context.Context) (result []*proto.Tag, err error) {
+	result = make([]*proto.Tag, 0)
+	rows, err := d.db.Query(c, _fetchVideoTagAll)
+	if err != nil {
+		return
+	}
+
+	for rows.Next() {
+		tmp := new(proto.Tag)
+		if err = rows.Scan(&tmp.TagID, &tmp.TagName, &tmp.TagType); err != nil {
+			log.Error("FetchVideoTag: %v", err)
+			continue
+		}
+		result = append(result, tmp)
+	}
+
+	return
+}
+
+// FetchVideoTag .
+func (d *Dao) FetchVideoTag(c context.Context, tid int32) (result *proto.Tag, err error) {
+	row := d.db.QueryRow(c, _fetchVideoTag, tid)
+
+	result = new(proto.Tag)
+	if err = row.Scan(&result.TagID, &result.TagName, &result.TagType); err != nil {
+		log.Error("FetchVideoTag: %v", err)
+		return
+	}
+
+	return
+}
+
+// FetchVideoTextTag .
+func (d *Dao) FetchVideoTextTag(c context.Context, svid int64) (result []string, err error) {
+	row := d.dbCms.QueryRow(c, _fetchVideoTextTag, svid)
+	var tags string
+	if err = row.Scan(&tags); err != nil {
+		log.Errorv(c, log.KV("log", "_fetchVideoTextTag failed"), log.KV("error", err), log.KV("svid", svid))
+		return
+	}
+
+	result = strings.Split(tags, ",")
+	return
+}
+
+// FetchVideoQuality .
+func (d *Dao) FetchVideoQuality(c context.Context, svid uint64) (result *quality.VideoQuality, err error) {
+	var raw string
+	row := d.dbOffline.QueryRow(c, _queryVideoQuality, svid)
+	row.Scan(&raw)
+	if raw == "" {
+		return
+	}
+
+	trimed := strings.Trim(raw, "\n")
+	hexDst, err := hex.DecodeString(trimed)
+	if err != nil {
+		log.Error("FetchVideoQuality: %v src[%s] raw[%s]", err, trimed, trimed)
+		return
+	}
+
+	snappyDst, err := snappy.Decode(nil, hexDst)
+	if err != nil {
+		log.Error("FetchVideoQuality: %v src[%s] raw[%s]", err, string(hexDst), trimed)
+		return
+	}
+
+	result = &quality.VideoQuality{}
+	result.Unmarshal(snappyDst)
+	if err != nil {
+		log.Error("FetchVideoQuality: %v src[%s] raw[%s]", err, snappyDst, trimed)
+	}
+	return
+}
+
+// FetchNewincomeVideo .
+func (d *Dao) FetchNewincomeVideo() (res []int64, err error) {
+	duration, _ := time.ParseDuration("-24h")
+	today := time.Now().Add(duration).Format("2006-01-02")
+	_query := fmt.Sprintf(_fetchNewIncomeVideo, strings.Join(model.RecommendVideoState, ","))
+	row, err := d.db.Query(context.Background(), _query, today)
+	if err != nil {
+		return
+	}
+	res = make([]int64, 0)
+	for row.Next() {
+		var tmp int64
+		if err = row.Scan(&tmp); err != nil {
+			return
+		}
+		res = append(res, tmp)
+	}
+
+	return
+}