Create & Init Project...

app/job/bbq/recall/internal/service/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bloomfilter.go",
+        "forward_index.go",
+        "inverted_index.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/bbq/recall/internal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/recall/internal/conf:go_default_library",
+        "//app/job/bbq/recall/internal/dao:go_default_library",
+        "//app/job/bbq/recall/proto:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/golang/snappy:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/internal/service/bloomfilter.go

@@ -0,0 +1,52 @@
+package service
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"go-common/library/log"
+)
+
+// GenBloomFilter .
+func (s *Service) GenBloomFilter() {
+	log.Info("run [%s]", "GenBloomFilter")
+	result, err := s.dao.FetchMidView(context.Background())
+	if err != nil {
+		log.Error("FetchMidView: %v", err)
+		return
+	}
+	s.bloomFilter(result)
+
+	result, err = s.dao.FetchBuvidView(context.Background())
+	if err != nil {
+		log.Error("FetchBuvidView: %v", err)
+		return
+	}
+	s.bloomFilter(result)
+	log.Info("finish [%s]", "GenBloomFilter")
+}
+
+func (s *Service) bloomFilter(result []string) {
+	m := make(map[string][]uint64)
+	for _, v := range result {
+		items := strings.Split(v, "\u0001")
+		if len(items) != 2 {
+			continue
+		}
+		if _, ok := m[items[0]]; !ok {
+			m[items[0]] = []uint64{}
+		}
+		svid, _ := strconv.Atoi(items[1])
+		m[items[0]] = append(m[items[0]], uint64(svid))
+	}
+	for k, v := range m {
+		if k == "" {
+			continue
+		}
+		if err := s.dao.InsertBloomFilter(context.Background(), k, v); err != nil {
+			log.Error("InsertBloomFilter: %v", err)
+			continue
+		}
+	}
+}

app/job/bbq/recall/internal/service/forward_index.go

@@ -0,0 +1,135 @@
+package service
+
+import (
+	"context"
+	"encoding/hex"
+	"os"
+	"os/exec"
+
+	"go-common/app/job/bbq/recall/proto"
+	"go-common/library/log"
+
+	"github.com/golang/snappy"
+)
+
+// GenForwardIndex 生产正排索引
+func (s *Service) GenForwardIndex() {
+	log.Info("run [%s]", "GenForwardIndex")
+	c := context.Background()
+	vInfo, err := s.videoBasicInfo(c)
+	if err != nil {
+		log.Error("video info: %v", err)
+		return
+	}
+
+	outputFile, err := os.OpenFile(s.c.Job.ForwardIndex.Output, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		log.Error("open file: %v", err)
+		return
+	}
+	shadowFile, _ := os.OpenFile(s.c.Job.ForwardIndex.Output+".bak", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	defer outputFile.Close()
+	defer shadowFile.Close()
+
+	for _, v := range vInfo {
+		qu, _ := s.dao.FetchVideoQuality(c, v.SVID)
+		tmp := &proto.ForwardIndex{
+			SVID:         v.SVID,
+			BasicInfo:    v,
+			VideoQuality: qu,
+		}
+		raw, err := tmp.Marshal()
+		if err != nil {
+			log.Error("json marshal: %v", err)
+			continue
+		}
+		_, err = outputFile.WriteString(hex.EncodeToString(snappy.Encode(nil, raw)))
+		if err != nil {
+			log.Error("output: %v", err)
+		}
+		outputFile.Write([]byte("\n"))
+		if err != nil {
+			log.Error("output endline: %v", err)
+		}
+		shadowFile.WriteString(tmp.String())
+		if err != nil {
+			log.Error("shadow: %v", err)
+		}
+		shadowFile.Write([]byte("\n"))
+		if err != nil {
+			log.Error("shadow endline: %v", err)
+		}
+	}
+	exec.Command(s.c.Job.ForwardIndex.Output + ".sh").Run()
+
+	log.Info("finish [GenForwardIndex]")
+
+	s.GenRealTimeInvertedIndex()
+}
+
+func (s *Service) videoBasicInfo(c context.Context) (result []*proto.VideoInfo, err error) {
+	// fetch tag info from db
+	tags, err := s.dao.FetchVideoTagAll(c)
+	if err != nil {
+		return
+	}
+	tagIDMap := make(map[int32]*proto.Tag)
+	tagNameMap := make(map[string]*proto.Tag)
+	for _, v := range tags {
+		tagIDMap[v.TagID] = v
+		tagNameMap[v.TagName] = v
+	}
+
+	// fetch video info from db
+	offset := 0
+	size := 1000
+	basic, err := s.dao.FetchVideoInfo(c, offset, size)
+	if err != nil {
+		log.Error("FetchVideoInfo: %v", err)
+		return
+	}
+	for len(basic) > 0 && err == nil {
+		log.Info("FetchVideoInfo: %v", len(result))
+		for _, v := range basic {
+			vInfo := &proto.VideoInfo{
+				SVID:     uint64(v.SVID),
+				Title:    v.Title,
+				Content:  v.Content,
+				MID:      uint64(v.MID),
+				AVID:     uint64(v.AVID),
+				CID:      uint64(v.CID),
+				PubTime:  v.PubTime.Time().Unix(),
+				CTime:    v.CTime.Time().Unix(),
+				MTime:    v.MTime.Time().Unix(),
+				Duration: uint32(v.Duration),
+				State:    int32(v.State),
+			}
+			vTags := make([]*proto.Tag, 0)
+			// 一级标签
+			if tag, ok := tagIDMap[v.TID]; ok {
+				vTags = append(vTags, tag)
+			}
+			// 二级标签
+			if subTag, ok := tagIDMap[v.SubTID]; ok {
+				vTags = append(vTags, subTag)
+			}
+			// 三级标签
+			if textTags, e := s.dao.FetchVideoTextTag(c, v.SVID); e == nil {
+				for _, v := range textTags {
+					if tmp, ok := tagNameMap[v]; ok {
+						vTags = append(vTags, tmp)
+					}
+				}
+			}
+
+			vInfo.Tags = vTags
+			result = append(result, vInfo)
+		}
+		offset += size
+		basic, err = s.dao.FetchVideoInfo(c, offset, size)
+		if err != nil {
+			log.Error("FetchVideoInfo: %v", err)
+		}
+	}
+	return
+}

app/job/bbq/recall/internal/service/inverted_index.go

@@ -0,0 +1,57 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"go-common/library/log"
+	"math/rand"
+	"sort"
+	"time"
+)
+
+const (
+	_nblocks     = 5
+	_redisPrefix = "RECALL:NEWPUB:%d"
+)
+
+// GenRealTimeInvertedIndex 实时倒排标签
+func (s *Service) GenRealTimeInvertedIndex() {
+	svids, err := s.dao.FetchNewincomeVideo()
+	if err != nil || svids == nil || len(svids) == 0 {
+		log.Error("GenRealTimeInvertedIndex FetchNewincomeVideo err[%v] svids[%v]", err, svids)
+		return
+	}
+	// svid 乱序
+	rand.Seed(time.Now().Unix())
+	sort.Slice(svids, func(i int, j int) bool {
+		return rand.Float32() > 0.5
+	})
+
+	// 平均分为5份
+	offset := 0
+	blocks := len(svids) / _nblocks
+	invertedIndex := make([][]int64, _nblocks)
+	for i := 0; i < _nblocks; i++ {
+		invertedIndex[i] = make([]int64, 0)
+		for j := 0; j < blocks; j++ {
+			invertedIndex[i] = append(invertedIndex[i], svids[offset+j])
+		}
+		offset = offset + blocks
+	}
+	if blocks*_nblocks < len(svids) {
+		invertedIndex[_nblocks-1] = append(invertedIndex[_nblocks-1], svids[len(svids)-1])
+	}
+
+	log.Info("GenRealTimeInvertedIndex invertedIndex[%v]", invertedIndex)
+
+	// 序列化后写入redis
+	for i, v := range invertedIndex {
+		key := fmt.Sprintf(_redisPrefix, i)
+		err = s.dao.SetInvertedIndex(context.Background(), key, v)
+		if err != nil {
+			log.Error("GenRealTimeInvertedIndex SetInvertedIndex err[%v]", err)
+		}
+	}
+
+	log.Info("finish [GenRealTimeInvertedIndex]")
+}

app/job/bbq/recall/internal/service/service.go

@@ -0,0 +1,64 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/bbq/recall/internal/conf"
+	"go-common/app/job/bbq/recall/internal/dao"
+	"go-common/library/log"
+
+	"github.com/robfig/cron"
+)
+
+// Service struct
+type Service struct {
+	c    *conf.Config
+	dao  *dao.Dao
+	sche *cron.Cron
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:    c,
+		dao:  dao.New(c),
+		sche: cron.New(),
+	}
+	return s
+}
+
+// InitCron .
+func (s *Service) InitCron() {
+	s.sche.AddFunc("@every 3s", s.HeartBeat)
+	s.sche.AddFunc(s.c.Job.ForwardIndex.Schedule, s.GenForwardIndex)
+	s.sche.AddFunc(s.c.Job.BloomFilter.Schedule, s.GenBloomFilter)
+	s.sche.Start()
+}
+
+// RunSrv .
+func (s *Service) RunSrv(name string) {
+	log.Info("run job{%s}", name)
+	switch name {
+	case s.c.Job.ForwardIndex.JobName:
+		s.GenForwardIndex()
+	case s.c.Job.BloomFilter.JobName:
+		s.GenBloomFilter()
+	default:
+		s.HeartBeat()
+	}
+}
+
+// HeartBeat .
+func (s *Service) HeartBeat() {
+	log.Info("alive...")
+}
+
+// Ping Service
+func (s *Service) Ping(ctx context.Context) (err error) {
+	return s.dao.Ping(ctx)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}