136 lines
3.1 KiB
Go
136 lines
3.1 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/hex"
|
|
"os"
|
|
"os/exec"
|
|
|
|
"go-common/app/job/bbq/recall/proto"
|
|
"go-common/library/log"
|
|
|
|
"github.com/golang/snappy"
|
|
)
|
|
|
|
// GenForwardIndex 生产正排索引
|
|
func (s *Service) GenForwardIndex() {
|
|
log.Info("run [%s]", "GenForwardIndex")
|
|
c := context.Background()
|
|
vInfo, err := s.videoBasicInfo(c)
|
|
if err != nil {
|
|
log.Error("video info: %v", err)
|
|
return
|
|
}
|
|
|
|
outputFile, err := os.OpenFile(s.c.Job.ForwardIndex.Output, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
|
|
if err != nil {
|
|
log.Error("open file: %v", err)
|
|
return
|
|
}
|
|
shadowFile, _ := os.OpenFile(s.c.Job.ForwardIndex.Output+".bak", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
|
|
defer outputFile.Close()
|
|
defer shadowFile.Close()
|
|
|
|
for _, v := range vInfo {
|
|
qu, _ := s.dao.FetchVideoQuality(c, v.SVID)
|
|
tmp := &proto.ForwardIndex{
|
|
SVID: v.SVID,
|
|
BasicInfo: v,
|
|
VideoQuality: qu,
|
|
}
|
|
raw, err := tmp.Marshal()
|
|
if err != nil {
|
|
log.Error("json marshal: %v", err)
|
|
continue
|
|
}
|
|
_, err = outputFile.WriteString(hex.EncodeToString(snappy.Encode(nil, raw)))
|
|
if err != nil {
|
|
log.Error("output: %v", err)
|
|
}
|
|
outputFile.Write([]byte("\n"))
|
|
if err != nil {
|
|
log.Error("output endline: %v", err)
|
|
}
|
|
shadowFile.WriteString(tmp.String())
|
|
if err != nil {
|
|
log.Error("shadow: %v", err)
|
|
}
|
|
shadowFile.Write([]byte("\n"))
|
|
if err != nil {
|
|
log.Error("shadow endline: %v", err)
|
|
}
|
|
}
|
|
exec.Command(s.c.Job.ForwardIndex.Output + ".sh").Run()
|
|
|
|
log.Info("finish [GenForwardIndex]")
|
|
|
|
s.GenRealTimeInvertedIndex()
|
|
}
|
|
|
|
func (s *Service) videoBasicInfo(c context.Context) (result []*proto.VideoInfo, err error) {
|
|
// fetch tag info from db
|
|
tags, err := s.dao.FetchVideoTagAll(c)
|
|
if err != nil {
|
|
return
|
|
}
|
|
tagIDMap := make(map[int32]*proto.Tag)
|
|
tagNameMap := make(map[string]*proto.Tag)
|
|
for _, v := range tags {
|
|
tagIDMap[v.TagID] = v
|
|
tagNameMap[v.TagName] = v
|
|
}
|
|
|
|
// fetch video info from db
|
|
offset := 0
|
|
size := 1000
|
|
basic, err := s.dao.FetchVideoInfo(c, offset, size)
|
|
if err != nil {
|
|
log.Error("FetchVideoInfo: %v", err)
|
|
return
|
|
}
|
|
for len(basic) > 0 && err == nil {
|
|
log.Info("FetchVideoInfo: %v", len(result))
|
|
for _, v := range basic {
|
|
vInfo := &proto.VideoInfo{
|
|
SVID: uint64(v.SVID),
|
|
Title: v.Title,
|
|
Content: v.Content,
|
|
MID: uint64(v.MID),
|
|
AVID: uint64(v.AVID),
|
|
CID: uint64(v.CID),
|
|
PubTime: v.PubTime.Time().Unix(),
|
|
CTime: v.CTime.Time().Unix(),
|
|
MTime: v.MTime.Time().Unix(),
|
|
Duration: uint32(v.Duration),
|
|
State: int32(v.State),
|
|
}
|
|
vTags := make([]*proto.Tag, 0)
|
|
// 一级标签
|
|
if tag, ok := tagIDMap[v.TID]; ok {
|
|
vTags = append(vTags, tag)
|
|
}
|
|
// 二级标签
|
|
if subTag, ok := tagIDMap[v.SubTID]; ok {
|
|
vTags = append(vTags, subTag)
|
|
}
|
|
// 三级标签
|
|
if textTags, e := s.dao.FetchVideoTextTag(c, v.SVID); e == nil {
|
|
for _, v := range textTags {
|
|
if tmp, ok := tagNameMap[v]; ok {
|
|
vTags = append(vTags, tmp)
|
|
}
|
|
}
|
|
}
|
|
|
|
vInfo.Tags = vTags
|
|
result = append(result, vInfo)
|
|
}
|
|
offset += size
|
|
basic, err = s.dao.FetchVideoInfo(c, offset, size)
|
|
if err != nil {
|
|
log.Error("FetchVideoInfo: %v", err)
|
|
}
|
|
}
|
|
return
|
|
}
|