466 lines
11 KiB
Go
466 lines
11 KiB
Go
package dao
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/pkg/errors"
|
||
|
||
"go-common/app/service/live/recommend/internal/conf"
|
||
"go-common/app/service/live/recommend/recconst"
|
||
relation_api "go-common/app/service/live/relation/api/liverpc"
|
||
room_api "go-common/app/service/live/room/api/liverpc"
|
||
"go-common/library/cache/redis"
|
||
"go-common/library/log"
|
||
"go-common/library/net/rpc/liverpc"
|
||
)
|
||
|
||
var _userRecCandidateKey = "rec_candidate_%d"
|
||
var _recommendOffsetKey = "rec_offset_%d"
|
||
|
||
// 已经推荐过的池子，用户+日期
|
||
var _recommendedKey = "recommended_%d_%s"
|
||
|
||
// RoomAPI room liverpc client
|
||
var RoomAPI *room_api.Client
|
||
|
||
// RelationAPI relation liverpc client
|
||
var RelationAPI *relation_api.Client
|
||
|
||
// Dao dao
|
||
type Dao struct {
|
||
c *conf.Config
|
||
redis *redis.Pool
|
||
}
|
||
|
||
func init() {
|
||
RoomAPI = room_api.New(getConf("room"))
|
||
RelationAPI = relation_api.New(getConf("relation"))
|
||
}
|
||
|
||
func getConf(appName string) *liverpc.ClientConfig {
|
||
c := conf.Conf.LiveRpc
|
||
if c != nil {
|
||
return c[appName]
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ClearRecommend 清空该用户相关的推荐缓存
|
||
func (d *Dao) ClearRecommend(ctx context.Context, uid int64) error {
|
||
candidateKey := fmt.Sprintf(_userRecCandidateKey, uid)
|
||
recommendedKey := fmt.Sprintf(_recommendedKey, uid, time.Now().Format("20060102"))
|
||
offsetKey := fmt.Sprintf(_recommendOffsetKey, uid)
|
||
conn := d.redis.Get(ctx)
|
||
defer conn.Close()
|
||
_, err := conn.Do("DEL", candidateKey, recommendedKey, offsetKey)
|
||
return errors.WithStack(err)
|
||
}
|
||
|
||
// New init mysql db
|
||
func New(c *conf.Config) (dao *Dao) {
|
||
dao = &Dao{
|
||
c: c,
|
||
redis: redis.NewPool(c.Redis),
|
||
}
|
||
return
|
||
}
|
||
|
||
// Close close the resource.
|
||
func (d *Dao) Close() {
|
||
d.redis.Close()
|
||
}
|
||
|
||
func (d *Dao) saveOffset(conn redis.Conn, uid int64, offset int) {
|
||
conn.Do("SETEX", fmt.Sprintf(_recommendOffsetKey, uid), 86400, offset)
|
||
}
|
||
|
||
func (d *Dao) addToRecommended(conn redis.Conn, uid int64, ids []int64) {
|
||
if len(ids) == 0 {
|
||
return
|
||
}
|
||
day := time.Now().Format("20060102")
|
||
|
||
key := fmt.Sprintf(_recommendedKey, uid, day)
|
||
var is []interface{}
|
||
is = append(is, key)
|
||
for _, id := range ids {
|
||
is = append(is, id)
|
||
}
|
||
|
||
conn.Send("EXPIRE", key, 86400)
|
||
conn.Send("SADD", is...)
|
||
conn.Flush()
|
||
conn.Receive()
|
||
_, err := conn.Receive()
|
||
if err != nil {
|
||
log.Info("addToRecommended error +%v", err)
|
||
}
|
||
}
|
||
|
||
// GetRandomRoomIds 随机获取count个推荐
|
||
// 如果总数量total比count小，则返回total个
|
||
func (d *Dao) GetRandomRoomIds(ctx context.Context, uid int64, reqCount int, existRoomIDs []int64) (ret []int64, err error) {
|
||
if reqCount == 0 {
|
||
return
|
||
}
|
||
var (
|
||
candidateLen int
|
||
)
|
||
r := d.redis.Get(ctx)
|
||
defer r.Close()
|
||
|
||
candidateKey := fmt.Sprintf(_userRecCandidateKey, uid)
|
||
exists, err := redis.Int(r.Do("exists", candidateKey))
|
||
if err != nil {
|
||
err = errors.WithStack(err)
|
||
return
|
||
}
|
||
|
||
existMap := map[int64]struct{}{}
|
||
for _, id := range existRoomIDs {
|
||
existMap[id] = struct{}{}
|
||
}
|
||
if exists == 0 {
|
||
var candidate []int64
|
||
var currentOffset = 0
|
||
candidate, err = d.generateLrCandidateList(r, uid, candidateKey)
|
||
if err != nil {
|
||
return
|
||
}
|
||
Loop:
|
||
for len(ret) < reqCount && currentOffset < len(candidate) {
|
||
var tmp []int64
|
||
if len(candidate)-currentOffset < int(reqCount) {
|
||
tmp = candidate[currentOffset:]
|
||
} else {
|
||
tmp = candidate[currentOffset : currentOffset+reqCount]
|
||
}
|
||
//去重
|
||
for _, id := range tmp {
|
||
_, ok := existMap[id]
|
||
currentOffset += 1
|
||
if !ok {
|
||
ret = append(ret, id)
|
||
if len(ret) >= int(reqCount) {
|
||
break Loop
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
d.addToRecommended(r, uid, ret)
|
||
d.saveOffset(r, uid, currentOffset)
|
||
} else {
|
||
candidateLen, err = redis.Int(r.Do("LLEN", candidateKey))
|
||
if err != nil {
|
||
return
|
||
}
|
||
|
||
var offset int
|
||
offset, _ = redis.Int(r.Do("GET", fmt.Sprintf(_recommendOffsetKey, uid)))
|
||
if offset > (candidateLen - 1) {
|
||
return
|
||
}
|
||
var currentOffset = offset
|
||
Loop2:
|
||
for len(ret) < reqCount && currentOffset < candidateLen {
|
||
var ids []int64
|
||
ids, err = redis.Int64s(r.Do("LRANGE", candidateKey, currentOffset, currentOffset+reqCount-1))
|
||
if err != nil {
|
||
err = errors.WithStack(err)
|
||
return
|
||
}
|
||
// 去重
|
||
for _, id := range ids {
|
||
currentOffset++
|
||
_, ok := existMap[id]
|
||
if !ok {
|
||
ret = append(ret, id)
|
||
if len(ret) >= int(reqCount) {
|
||
break Loop2
|
||
}
|
||
}
|
||
}
|
||
if len(ids) == 0 {
|
||
log.Error("Cannot get recommend candidate, key=%s, offset=%d, count=%d", candidateKey, offset, reqCount)
|
||
break
|
||
}
|
||
}
|
||
|
||
d.addToRecommended(r, uid, ret)
|
||
d.saveOffset(r, uid, currentOffset)
|
||
}
|
||
return
|
||
}
|
||
|
||
// GetLrRecRoomIds 在GetRandomRoomIds的基础上进行LR计算并返回倒排的房间号列表
|
||
// 与GetRandomRoomIds有相同的输入输出结构
|
||
func (d *Dao) GetLrRecRoomIds(r redis.Conn, uid int64, candidateIds []int64) (ret []int64, err error) {
|
||
var areas string
|
||
areaIds := map[int64]struct{}{}
|
||
areas, err = redis.String(r.Do("GET", fmt.Sprintf(recconst.UserAreaKey, uid)))
|
||
if err != nil && err != redis.ErrNil {
|
||
log.Error("redis GET error: %v", err)
|
||
return
|
||
}
|
||
err = nil
|
||
if areas != "" {
|
||
split := strings.Split(areas, ";")
|
||
for _, areaIdStr := range split {
|
||
areaId, _ := strconv.ParseInt(areaIdStr, 10, 64)
|
||
areaIds[areaId] = struct{}{}
|
||
}
|
||
}
|
||
|
||
weightVector := makeWeightVec(d.c)
|
||
roomFeatures, ok := roomFeatureValue.Load().(map[int64][]int64)
|
||
if !ok {
|
||
ret = candidateIds
|
||
return
|
||
}
|
||
roomScoreSlice := ScoreSlice{}
|
||
for _, roomId := range candidateIds {
|
||
if fv, ok := roomFeatures[roomId]; ok {
|
||
featureVector := make([]int64, len(fv))
|
||
copy(featureVector, fv)
|
||
areaId := featureVector[0]
|
||
if _, ok := areaIds[areaId]; ok {
|
||
featureVector[0] = 1
|
||
} else {
|
||
featureVector[0] = 0
|
||
}
|
||
counter := Counter{roomId: roomId, score: calcScore(weightVector, featureVector)}
|
||
roomScoreSlice = append(roomScoreSlice, counter)
|
||
}
|
||
}
|
||
|
||
sort.Sort(roomScoreSlice)
|
||
for _, counter := range roomScoreSlice {
|
||
ret = append(ret, counter.roomId)
|
||
}
|
||
return
|
||
}
|
||
|
||
// generateCandidateList 得到候选集
|
||
func (d *Dao) generateCandidateList(r redis.Conn, uid int64, candidateKey string) (ret []int64, err error) {
|
||
|
||
// 第一步 itemcf，优先级最高。
|
||
itemCFKey := fmt.Sprintf(recconst.UserItemCFRecKey, uid)
|
||
var itemCFList []int64
|
||
itemCFList, err = redis.Int64s(r.Do("ZREVRANGE", itemCFKey, 0, -1))
|
||
if err != nil {
|
||
err = errors.WithStack(err)
|
||
return
|
||
}
|
||
itemCFOnlineIds := d.FilterOnlineRoomIds(itemCFList)
|
||
if len(itemCFOnlineIds) == 0 {
|
||
log.Info("No item-cf room online for user, uid=%d, before online filter room ids: %+v", uid, itemCFList)
|
||
}
|
||
|
||
// 第二步 取兴趣分区的房间 人气超过100的房间
|
||
var areas string
|
||
areas, err = redis.String(r.Do("GET", fmt.Sprintf(recconst.UserAreaKey, uid)))
|
||
if err != nil && err != redis.ErrNil {
|
||
err = errors.WithStack(err)
|
||
return
|
||
}
|
||
err = nil
|
||
var areaRoomIDs []int64
|
||
if areas != "" {
|
||
split := strings.Split(areas, ";")
|
||
for _, areaIdStr := range split {
|
||
areaId, _ := strconv.ParseInt(areaIdStr, 10, 64)
|
||
var ids = d.getAreaRoomIds(areaId)
|
||
areaRoomIDs = append(areaRoomIDs, ids...)
|
||
}
|
||
}
|
||
|
||
// 第三步 取兴趣分区大分区的100个 先不做
|
||
// 第四步 减去已经推荐过的
|
||
day := time.Now().Format("20060102")
|
||
var recommendedList []int64
|
||
edKey := fmt.Sprintf(_recommendedKey, uid, day)
|
||
recommendedList, err = redis.Int64s(r.Do("SMEMBERS", edKey))
|
||
if err != nil {
|
||
err = errors.WithStack(err)
|
||
return
|
||
}
|
||
recommended := map[int64]struct{}{}
|
||
for _, id := range recommendedList {
|
||
recommended[id] = struct{}{}
|
||
}
|
||
var itemCFFinalIDs []int64
|
||
for _, id := range itemCFOnlineIds {
|
||
_, exist := recommended[id]
|
||
if !exist {
|
||
itemCFFinalIDs = append(itemCFFinalIDs, id)
|
||
}
|
||
}
|
||
|
||
var areaRoomFinalIDs []int64
|
||
for _, id := range areaRoomIDs {
|
||
_, exist := recommended[id]
|
||
if !exist {
|
||
areaRoomFinalIDs = append(areaRoomFinalIDs, id)
|
||
}
|
||
}
|
||
|
||
ret = mergeArr(itemCFFinalIDs, areaRoomFinalIDs)
|
||
|
||
log.Info("UserRecommend : uid=%d total=%d, "+
|
||
"itemcf.original=%d, itemcf.online=%d, itemcf.noviewd=%d, "+
|
||
"areaRoom.original=%d, itemcf.noviewd=%d viewed=%d",
|
||
uid, len(ret), len(itemCFList), len(itemCFOnlineIds), len(itemCFFinalIDs),
|
||
len(areaRoomIDs), len(areaRoomFinalIDs), len(recommendedList))
|
||
return
|
||
}
|
||
|
||
// generateCandidateList 得到进过LR的候选集
|
||
func (d *Dao) generateLrCandidateList(r redis.Conn, uid int64, candidateKey string) (ret []int64, err error) {
|
||
roomIDs, err := d.generateCandidateList(r, uid, candidateKey)
|
||
if err != nil {
|
||
log.Error("generateLrCandidateList failed 1, error:%v", err)
|
||
return
|
||
}
|
||
|
||
if len(ret) > 0 {
|
||
ret, err = d.GetLrRecRoomIds(r, uid, roomIDs)
|
||
if err != nil {
|
||
log.Error("generateLrCandidateList failed 2, error:%v", err)
|
||
return
|
||
}
|
||
}
|
||
|
||
// 召回源不足的情况下补足推荐房间数
|
||
if len(ret) < 150 {
|
||
ids, ok := recDefaultRoomIds.Load().([]int64)
|
||
if !ok {
|
||
return
|
||
}
|
||
ret1, err1 := d.GetLrRecRoomIds(r, uid, ids)
|
||
if err1 != nil {
|
||
log.Error("generateLrCandidateList failed 3, error:%v", err1)
|
||
return
|
||
}
|
||
ret = mergeArrWithOrder(ret, ret1, 150) // TODO:当前ret1的结果是没有过滤掉今天看过的房间的, 看后面是否需要优化
|
||
}
|
||
{
|
||
for _, roomID := range ret {
|
||
r.Send("RPUSH", candidateKey, roomID)
|
||
}
|
||
r.Send("EXPIRE", candidateKey, 60*2)
|
||
err = r.Flush()
|
||
if err != nil {
|
||
err = errors.WithStack(err)
|
||
return
|
||
}
|
||
for i := 0; i < len(ret)+1; i++ {
|
||
r.Receive()
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
// Ping dao ping
|
||
func (d *Dao) Ping(ctx context.Context) (err error) {
|
||
conn := d.redis.Get(ctx)
|
||
defer conn.Close()
|
||
_, err = conn.Do("ping")
|
||
if err != nil {
|
||
err = errors.Wrap(err, "dao Ping err")
|
||
}
|
||
return err
|
||
}
|
||
|
||
// Counter 房间-分数结构体, 用于构建一个可排序的slice
|
||
type Counter struct {
|
||
roomId int64
|
||
score float32
|
||
}
|
||
|
||
// ScoreSlice Counter对象的slice
|
||
type ScoreSlice []Counter
|
||
|
||
func (s ScoreSlice) Len() int {
|
||
return len(s)
|
||
}
|
||
|
||
func (s ScoreSlice) Swap(i, j int) {
|
||
s[i], s[j] = s[j], s[i]
|
||
}
|
||
|
||
func (s ScoreSlice) Less(i, j int) bool {
|
||
return s[j].score < s[i].score
|
||
}
|
||
|
||
func calcScore(weightVector []float32, featureVector []int64) (score float32) {
|
||
if len(weightVector) != len(featureVector) {
|
||
panic(fmt.Sprintf("权重数量和特征数量不匹配, 请检查配置或逻辑, weight: %+v, feature: %+v", weightVector, featureVector))
|
||
}
|
||
for i := 0; i < min(len(weightVector), len(featureVector)); i++ {
|
||
score += weightVector[i] * float32(featureVector[i])
|
||
}
|
||
return
|
||
}
|
||
|
||
func min(x int, y int) int {
|
||
if x < y {
|
||
return x
|
||
}
|
||
return y
|
||
}
|
||
|
||
// 合并两个集合
|
||
func mergeArr(x []int64, y []int64) (ret []int64) {
|
||
tmpMap := map[int64]struct{}{}
|
||
for _, id := range x {
|
||
tmpMap[id] = struct{}{}
|
||
}
|
||
for _, id := range y {
|
||
tmpMap[id] = struct{}{}
|
||
}
|
||
for id := range tmpMap {
|
||
ret = append(ret, id)
|
||
}
|
||
return
|
||
}
|
||
|
||
// 按x, y的顺序合并两个集合, 当x的长度不小于limit则直接返回
|
||
func mergeArrWithOrder(x []int64, y []int64, limit int) (ret []int64) {
|
||
if len(x) >= limit {
|
||
ret = x
|
||
return
|
||
}
|
||
tmpMap := map[int64]struct{}{}
|
||
ret = append(ret, x...)
|
||
num := len(ret)
|
||
for _, id := range x {
|
||
tmpMap[id] = struct{}{}
|
||
}
|
||
for _, id := range y {
|
||
if _, ok := tmpMap[id]; ok {
|
||
continue
|
||
}
|
||
num += 1
|
||
tmpMap[id] = struct{}{}
|
||
ret = append(ret, id)
|
||
if num >= limit {
|
||
break
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func makeWeightVec(c *conf.Config) (ret []float32) {
|
||
ret = append(ret, c.CommonFeature.UserAreaInterest.Weights...)
|
||
ret = append(ret, c.CommonFeature.FansNum.Weights...)
|
||
ret = append(ret, c.CommonFeature.CornerSign.Weights...)
|
||
ret = append(ret, c.CommonFeature.Online.Weights...)
|
||
return
|
||
}
|