Create & Init Project...

app/service/live/recommend/internal/dao/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "online_filter.go",
+        "room_feature.go",
+    ],
+    importpath = "go-common/app/service/live/recommend/internal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//app/service/live/recommend/recconst:go_default_library",
+        "//app/service/live/relation/api/liverpc:go_default_library",
+        "//app/service/live/relation/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc:go_default_library",
+        "//app/service/live/room/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc/v2:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/liverpc:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "room_feature_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/service/live/recommend/internal/dao/dao.go

@@ -0,0 +1,465 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"go-common/app/service/live/recommend/internal/conf"
+	"go-common/app/service/live/recommend/recconst"
+	relation_api "go-common/app/service/live/relation/api/liverpc"
+	room_api "go-common/app/service/live/room/api/liverpc"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"go-common/library/net/rpc/liverpc"
+)
+
+var _userRecCandidateKey = "rec_candidate_%d"
+var _recommendOffsetKey = "rec_offset_%d"
+
+// 已经推荐过的池子，用户+日期
+var _recommendedKey = "recommended_%d_%s"
+
+// RoomAPI room liverpc client
+var RoomAPI *room_api.Client
+
+// RelationAPI relation liverpc client
+var RelationAPI *relation_api.Client
+
+// Dao dao
+type Dao struct {
+	c     *conf.Config
+	redis *redis.Pool
+}
+
+func init() {
+	RoomAPI = room_api.New(getConf("room"))
+	RelationAPI = relation_api.New(getConf("relation"))
+}
+
+func getConf(appName string) *liverpc.ClientConfig {
+	c := conf.Conf.LiveRpc
+	if c != nil {
+		return c[appName]
+	}
+	return nil
+}
+
+// ClearRecommend 清空该用户相关的推荐缓存
+func (d *Dao) ClearRecommend(ctx context.Context, uid int64) error {
+	candidateKey := fmt.Sprintf(_userRecCandidateKey, uid)
+	recommendedKey := fmt.Sprintf(_recommendedKey, uid, time.Now().Format("20060102"))
+	offsetKey := fmt.Sprintf(_recommendOffsetKey, uid)
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	_, err := conn.Do("DEL", candidateKey, recommendedKey, offsetKey)
+	return errors.WithStack(err)
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:     c,
+		redis: redis.NewPool(c.Redis),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+}
+
+func (d *Dao) saveOffset(conn redis.Conn, uid int64, offset int) {
+	conn.Do("SETEX", fmt.Sprintf(_recommendOffsetKey, uid), 86400, offset)
+}
+
+func (d *Dao) addToRecommended(conn redis.Conn, uid int64, ids []int64) {
+	if len(ids) == 0 {
+		return
+	}
+	day := time.Now().Format("20060102")
+
+	key := fmt.Sprintf(_recommendedKey, uid, day)
+	var is []interface{}
+	is = append(is, key)
+	for _, id := range ids {
+		is = append(is, id)
+	}
+
+	conn.Send("EXPIRE", key, 86400)
+	conn.Send("SADD", is...)
+	conn.Flush()
+	conn.Receive()
+	_, err := conn.Receive()
+	if err != nil {
+		log.Info("addToRecommended error +%v", err)
+	}
+}
+
+// GetRandomRoomIds 随机获取count个推荐
+// 如果总数量total比count小，则返回total个
+func (d *Dao) GetRandomRoomIds(ctx context.Context, uid int64, reqCount int, existRoomIDs []int64) (ret []int64, err error) {
+	if reqCount == 0 {
+		return
+	}
+	var (
+		candidateLen int
+	)
+	r := d.redis.Get(ctx)
+	defer r.Close()
+
+	candidateKey := fmt.Sprintf(_userRecCandidateKey, uid)
+	exists, err := redis.Int(r.Do("exists", candidateKey))
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+
+	existMap := map[int64]struct{}{}
+	for _, id := range existRoomIDs {
+		existMap[id] = struct{}{}
+	}
+	if exists == 0 {
+		var candidate []int64
+		var currentOffset = 0
+		candidate, err = d.generateLrCandidateList(r, uid, candidateKey)
+		if err != nil {
+			return
+		}
+	Loop:
+		for len(ret) < reqCount && currentOffset < len(candidate) {
+			var tmp []int64
+			if len(candidate)-currentOffset < int(reqCount) {
+				tmp = candidate[currentOffset:]
+			} else {
+				tmp = candidate[currentOffset : currentOffset+reqCount]
+			}
+			//去重
+			for _, id := range tmp {
+				_, ok := existMap[id]
+				currentOffset += 1
+				if !ok {
+					ret = append(ret, id)
+					if len(ret) >= int(reqCount) {
+						break Loop
+					}
+				}
+			}
+		}
+
+		d.addToRecommended(r, uid, ret)
+		d.saveOffset(r, uid, currentOffset)
+	} else {
+		candidateLen, err = redis.Int(r.Do("LLEN", candidateKey))
+		if err != nil {
+			return
+		}
+
+		var offset int
+		offset, _ = redis.Int(r.Do("GET", fmt.Sprintf(_recommendOffsetKey, uid)))
+		if offset > (candidateLen - 1) {
+			return
+		}
+		var currentOffset = offset
+	Loop2:
+		for len(ret) < reqCount && currentOffset < candidateLen {
+			var ids []int64
+			ids, err = redis.Int64s(r.Do("LRANGE", candidateKey, currentOffset, currentOffset+reqCount-1))
+			if err != nil {
+				err = errors.WithStack(err)
+				return
+			}
+			// 去重
+			for _, id := range ids {
+				currentOffset++
+				_, ok := existMap[id]
+				if !ok {
+					ret = append(ret, id)
+					if len(ret) >= int(reqCount) {
+						break Loop2
+					}
+				}
+			}
+			if len(ids) == 0 {
+				log.Error("Cannot get recommend candidate, key=%s, offset=%d, count=%d", candidateKey, offset, reqCount)
+				break
+			}
+		}
+
+		d.addToRecommended(r, uid, ret)
+		d.saveOffset(r, uid, currentOffset)
+	}
+	return
+}
+
+// GetLrRecRoomIds 在GetRandomRoomIds的基础上进行LR计算并返回倒排的房间号列表
+// 与GetRandomRoomIds有相同的输入输出结构
+func (d *Dao) GetLrRecRoomIds(r redis.Conn, uid int64, candidateIds []int64) (ret []int64, err error) {
+	var areas string
+	areaIds := map[int64]struct{}{}
+	areas, err = redis.String(r.Do("GET", fmt.Sprintf(recconst.UserAreaKey, uid)))
+	if err != nil && err != redis.ErrNil {
+		log.Error("redis GET error: %v", err)
+		return
+	}
+	err = nil
+	if areas != "" {
+		split := strings.Split(areas, ";")
+		for _, areaIdStr := range split {
+			areaId, _ := strconv.ParseInt(areaIdStr, 10, 64)
+			areaIds[areaId] = struct{}{}
+		}
+	}
+
+	weightVector := makeWeightVec(d.c)
+	roomFeatures, ok := roomFeatureValue.Load().(map[int64][]int64)
+	if !ok {
+		ret = candidateIds
+		return
+	}
+	roomScoreSlice := ScoreSlice{}
+	for _, roomId := range candidateIds {
+		if fv, ok := roomFeatures[roomId]; ok {
+			featureVector := make([]int64, len(fv))
+			copy(featureVector, fv)
+			areaId := featureVector[0]
+			if _, ok := areaIds[areaId]; ok {
+				featureVector[0] = 1
+			} else {
+				featureVector[0] = 0
+			}
+			counter := Counter{roomId: roomId, score: calcScore(weightVector, featureVector)}
+			roomScoreSlice = append(roomScoreSlice, counter)
+		}
+	}
+
+	sort.Sort(roomScoreSlice)
+	for _, counter := range roomScoreSlice {
+		ret = append(ret, counter.roomId)
+	}
+	return
+}
+
+// generateCandidateList 得到候选集
+func (d *Dao) generateCandidateList(r redis.Conn, uid int64, candidateKey string) (ret []int64, err error) {
+
+	// 第一步 itemcf，优先级最高。
+	itemCFKey := fmt.Sprintf(recconst.UserItemCFRecKey, uid)
+	var itemCFList []int64
+	itemCFList, err = redis.Int64s(r.Do("ZREVRANGE", itemCFKey, 0, -1))
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	itemCFOnlineIds := d.FilterOnlineRoomIds(itemCFList)
+	if len(itemCFOnlineIds) == 0 {
+		log.Info("No item-cf room online for user, uid=%d, before online filter room ids: %+v", uid, itemCFList)
+	}
+
+	// 第二步 取兴趣分区的房间 人气超过100的房间
+	var areas string
+	areas, err = redis.String(r.Do("GET", fmt.Sprintf(recconst.UserAreaKey, uid)))
+	if err != nil && err != redis.ErrNil {
+		err = errors.WithStack(err)
+		return
+	}
+	err = nil
+	var areaRoomIDs []int64
+	if areas != "" {
+		split := strings.Split(areas, ";")
+		for _, areaIdStr := range split {
+			areaId, _ := strconv.ParseInt(areaIdStr, 10, 64)
+			var ids = d.getAreaRoomIds(areaId)
+			areaRoomIDs = append(areaRoomIDs, ids...)
+		}
+	}
+
+	// 第三步 取兴趣分区大分区的100个 先不做
+	// 第四步 减去已经推荐过的
+	day := time.Now().Format("20060102")
+	var recommendedList []int64
+	edKey := fmt.Sprintf(_recommendedKey, uid, day)
+	recommendedList, err = redis.Int64s(r.Do("SMEMBERS", edKey))
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	recommended := map[int64]struct{}{}
+	for _, id := range recommendedList {
+		recommended[id] = struct{}{}
+	}
+	var itemCFFinalIDs []int64
+	for _, id := range itemCFOnlineIds {
+		_, exist := recommended[id]
+		if !exist {
+			itemCFFinalIDs = append(itemCFFinalIDs, id)
+		}
+	}
+
+	var areaRoomFinalIDs []int64
+	for _, id := range areaRoomIDs {
+		_, exist := recommended[id]
+		if !exist {
+			areaRoomFinalIDs = append(areaRoomFinalIDs, id)
+		}
+	}
+
+	ret = mergeArr(itemCFFinalIDs, areaRoomFinalIDs)
+
+	log.Info("UserRecommend : uid=%d total=%d, "+
+		"itemcf.original=%d, itemcf.online=%d, itemcf.noviewd=%d, "+
+		"areaRoom.original=%d, itemcf.noviewd=%d viewed=%d",
+		uid, len(ret), len(itemCFList), len(itemCFOnlineIds), len(itemCFFinalIDs),
+		len(areaRoomIDs), len(areaRoomFinalIDs), len(recommendedList))
+	return
+}
+
+// generateCandidateList 得到进过LR的候选集
+func (d *Dao) generateLrCandidateList(r redis.Conn, uid int64, candidateKey string) (ret []int64, err error) {
+	roomIDs, err := d.generateCandidateList(r, uid, candidateKey)
+	if err != nil {
+		log.Error("generateLrCandidateList failed 1, error:%v", err)
+		return
+	}
+
+	if len(ret) > 0 {
+		ret, err = d.GetLrRecRoomIds(r, uid, roomIDs)
+		if err != nil {
+			log.Error("generateLrCandidateList failed 2, error:%v", err)
+			return
+		}
+	}
+
+	// 召回源不足的情况下补足推荐房间数
+	if len(ret) < 150 {
+		ids, ok := recDefaultRoomIds.Load().([]int64)
+		if !ok {
+			return
+		}
+		ret1, err1 := d.GetLrRecRoomIds(r, uid, ids)
+		if err1 != nil {
+			log.Error("generateLrCandidateList failed 3, error:%v", err1)
+			return
+		}
+		ret = mergeArrWithOrder(ret, ret1, 150) // TODO:当前ret1的结果是没有过滤掉今天看过的房间的, 看后面是否需要优化
+	}
+	{
+		for _, roomID := range ret {
+			r.Send("RPUSH", candidateKey, roomID)
+		}
+		r.Send("EXPIRE", candidateKey, 60*2)
+		err = r.Flush()
+		if err != nil {
+			err = errors.WithStack(err)
+			return
+		}
+		for i := 0; i < len(ret)+1; i++ {
+			r.Receive()
+		}
+	}
+	return
+}
+
+// Ping dao ping
+func (d *Dao) Ping(ctx context.Context) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	_, err = conn.Do("ping")
+	if err != nil {
+		err = errors.Wrap(err, "dao Ping err")
+	}
+	return err
+}
+
+// Counter 房间-分数结构体, 用于构建一个可排序的slice
+type Counter struct {
+	roomId int64
+	score  float32
+}
+
+// ScoreSlice Counter对象的slice
+type ScoreSlice []Counter
+
+func (s ScoreSlice) Len() int {
+	return len(s)
+}
+
+func (s ScoreSlice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s ScoreSlice) Less(i, j int) bool {
+	return s[j].score < s[i].score
+}
+
+func calcScore(weightVector []float32, featureVector []int64) (score float32) {
+	if len(weightVector) != len(featureVector) {
+		panic(fmt.Sprintf("权重数量和特征数量不匹配, 请检查配置或逻辑, weight: %+v, feature: %+v", weightVector, featureVector))
+	}
+	for i := 0; i < min(len(weightVector), len(featureVector)); i++ {
+		score += weightVector[i] * float32(featureVector[i])
+	}
+	return
+}
+
+func min(x int, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+// 合并两个集合
+func mergeArr(x []int64, y []int64) (ret []int64) {
+	tmpMap := map[int64]struct{}{}
+	for _, id := range x {
+		tmpMap[id] = struct{}{}
+	}
+	for _, id := range y {
+		tmpMap[id] = struct{}{}
+	}
+	for id := range tmpMap {
+		ret = append(ret, id)
+	}
+	return
+}
+
+// 按x, y的顺序合并两个集合, 当x的长度不小于limit则直接返回
+func mergeArrWithOrder(x []int64, y []int64, limit int) (ret []int64) {
+	if len(x) >= limit {
+		ret = x
+		return
+	}
+	tmpMap := map[int64]struct{}{}
+	ret = append(ret, x...)
+	num := len(ret)
+	for _, id := range x {
+		tmpMap[id] = struct{}{}
+	}
+	for _, id := range y {
+		if _, ok := tmpMap[id]; ok {
+			continue
+		}
+		num += 1
+		tmpMap[id] = struct{}{}
+		ret = append(ret, id)
+		if num >= limit {
+			break
+		}
+	}
+	return
+}
+
+func makeWeightVec(c *conf.Config) (ret []float32) {
+	ret = append(ret, c.CommonFeature.UserAreaInterest.Weights...)
+	ret = append(ret, c.CommonFeature.FansNum.Weights...)
+	ret = append(ret, c.CommonFeature.CornerSign.Weights...)
+	ret = append(ret, c.CommonFeature.Online.Weights...)
+	return
+}

app/service/live/recommend/internal/dao/dao_test.go

@@ -0,0 +1,30 @@
+package dao
+
+import (
+	"reflect"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestMergeArrWithOrder(t *testing.T) {
+	Convey("mergeArrWithOrder", t, func() {
+		a := []int64{1, 2, 3, 4}
+		b := []int64{4, 5, 6, 7}
+		c := mergeArrWithOrder(a, b, len(a)+len(b))
+		So(reflect.DeepEqual(c, []int64{1, 2, 3, 4, 5, 6, 7}), ShouldBeTrue)
+		c = mergeArrWithOrder(a, b, 2)
+		So(reflect.DeepEqual(c, []int64{1, 2, 3, 4}), ShouldBeTrue)
+		c = mergeArrWithOrder(a, b, 6)
+		So(reflect.DeepEqual(c, []int64{1, 2, 3, 4, 5, 6}), ShouldBeTrue)
+	})
+}
+
+func TestMergeArr(t *testing.T) {
+	Convey("mergeArr", t, func() {
+		a := []int64{1, 2, 3, 4}
+		b := []int64{4, 5, 6, 7}
+		c := mergeArr(a, b)
+		So(len(c) == 7, ShouldBeTrue)
+	})
+}

app/service/live/recommend/internal/dao/online_filter.go

@@ -0,0 +1,72 @@
+package dao
+
+import (
+	"context"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/service/live/room/api/liverpc/v1"
+	"go-common/library/log"
+)
+
+var onlineRoomIdValue atomic.Value
+var areaRoomsValue atomic.Value
+
+// StartRefreshJob 更新在线房间信息
+func StartRefreshJob() {
+	t := time.Tick(time.Second * 60)
+	refreshOnlineRoomData(context.Background())
+	for range t {
+		refreshOnlineRoomData(context.Background())
+	}
+}
+
+// refreshOnlineRoomData 更新RoomId
+func refreshOnlineRoomData(ctx context.Context) (err error) {
+	resp, err := RoomAPI.V1Room.AllLiveForBigdata(ctx, &v1.RoomAllLiveForBigdataReq{})
+	if err != nil {
+		return
+	}
+	onlineRooms := map[int64]struct{}{}
+	areaRooms := map[int64][]int64{}
+	for _, info := range resp.Data {
+		roomID := info.Roomid
+
+		onlineRooms[roomID] = struct{}{}
+		if info.Online > 100 {
+			areaRooms[info.AreaV2Id] = append(areaRooms[info.AreaV2Id], roomID)
+		}
+	}
+	log.Info("refreshOnlineRoomData: count=%d", len(onlineRooms))
+	log.Info("refreshOnlineRoomData area Rooms: %+v", areaRooms)
+	onlineRoomIdValue.Store(onlineRooms)
+	areaRoomsValue.Store(areaRooms)
+	return
+}
+
+func (d *Dao) getAreaRoomIds(areaId int64) (ret []int64) {
+	ret = make([]int64, 0)
+	areaRooms, ok := areaRoomsValue.Load().(map[int64][]int64)
+	if !ok {
+		log.Warn("cannot load current area room ids")
+		return
+	}
+	ret = areaRooms[areaId]
+	return
+}
+
+// FilterOnlineRoomIds 给定一批room id 返回所有在线的
+func (d *Dao) FilterOnlineRoomIds(roomIds []int64) (ret []int64) {
+	ret = make([]int64, 0)
+	currentIds, ok := onlineRoomIdValue.Load().(map[int64]struct{})
+	if !ok {
+		log.Warn("cannot load current online room ids")
+		return
+	}
+	for _, roomId := range roomIds {
+		if _, ok := currentIds[roomId]; ok {
+			ret = append(ret, roomId)
+		}
+	}
+	return
+}

app/service/live/recommend/internal/dao/room_feature.go

@@ -0,0 +1,204 @@
+package dao
+
+import (
+	"context"
+	"errors"
+	"regexp"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/service/live/recommend/internal/conf"
+	relationV1 "go-common/app/service/live/relation/api/liverpc/v1"
+	roomV1 "go-common/app/service/live/room/api/liverpc/v1"
+	roomV2 "go-common/app/service/live/room/api/liverpc/v2"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+)
+
+var roomFeatureValue atomic.Value
+var recDefaultRoomIds atomic.Value
+
+// StartRoomFeatureJob 更新在线房间的特征信息
+func StartRoomFeatureJob(c *conf.Config) {
+	t := time.Tick(time.Second * 30)
+	refreshRoomFeature(context.Background(), c)
+	for range t {
+		refreshRoomFeature(context.Background(), c)
+	}
+}
+
+func refreshRoomFeature(ctx context.Context, c *conf.Config) (err error) {
+	n := 20
+	currentIds, ok := onlineRoomIdValue.Load().(map[int64]struct{})
+	if !ok {
+		log.Warn("cannot load current online room ids")
+		err = errors.New("cannot load current online room ids")
+		return
+	}
+	keys := make([]int64, 0, len(currentIds))
+	for k := range currentIds {
+		keys = append(keys, k)
+	}
+	chunkIdsArray := sliceArray(keys, n)
+	roomFeatures := map[int64][]int64{}
+	var lock sync.Mutex
+	var eg errgroup.Group
+	for _, tmp := range chunkIdsArray {
+		chunkIds := tmp
+		eg.Go(func() (err error) {
+			resp, err := RoomAPI.V2Room.GetByIds(ctx, &roomV2.RoomGetByIdsReq{Ids: chunkIds})
+			if err != nil || resp.GetCode() != 0 {
+				log.Error("dao.RoomAPI.V2Room.GetByIds (%v) error(%v) resp(%v)", chunkIds, err, resp)
+				return
+			}
+
+			resp1, err1 := RoomAPI.V1RoomPendant.GetPendantByIds(ctx, &roomV1.RoomPendantGetPendantByIdsReq{Ids: chunkIds, Type: "mobile_index_badge", Position: 2})
+			if err1 != nil || resp1.GetCode() != 0 {
+				log.Error("dao.RoomAPI.V1Room.GetPendantByIds (%v) error(%v) resp(%v)", chunkIds, err1, resp1)
+				return
+			}
+
+			uids := make([]int64, 0, n)
+			for _, r := range resp.Data {
+				uids = append(uids, r.Uid)
+			}
+			resp2, err2 := RelationAPI.V1Feed.GetUserFcBatch(ctx, &relationV1.FeedGetUserFcBatchReq{Uids: uids})
+			if err2 != nil || resp.GetCode() != 0 {
+				log.Error("dao.RelationAPI.V1Relation.GetUserFcBatch (%v) error(%v) resp(%v)", chunkIds, err2, resp2)
+				return
+			}
+
+			roomPendantInfo := resp1.Data.Result
+			fansCountInfo := resp2.Data
+
+			for roomId, r := range resp.Data {
+				cornerTag := ""
+				fansNum := int64(0)
+				if PendantInfo, ok := roomPendantInfo[roomId]; ok && PendantInfo != nil {
+					cornerTag = PendantInfo.Value
+				}
+				if fans, ok := fansCountInfo[r.Uid]; ok {
+					fansNum = fans.Fc
+				}
+				featureVector := createFeature(c, r.AreaV2Id, cornerTag, fansNum, r.Online)
+				lock.Lock()
+				roomFeatures[roomId] = featureVector
+				lock.Unlock()
+			}
+			return
+		})
+	}
+	eg.Wait()
+	roomFeatureValue.Store(roomFeatures)
+	//创建默认推荐房间列表
+	roomScoreSlice := ScoreSlice{}
+	for roomId, vec := range roomFeatures {
+		featureVector := make([]int64, len(vec))
+		copy(featureVector, vec)
+		featureVector[0] = 0
+		counter := Counter{roomId: roomId, score: calcScore(makeWeightVec(c), featureVector)}
+		roomScoreSlice = append(roomScoreSlice, counter)
+	}
+	sort.Sort(roomScoreSlice)
+	//默认的召回源
+	limit := 400
+	recDefault := make([]int64, 0, limit)
+	for _, counter := range roomScoreSlice {
+		limit = limit - 1
+		if limit < 0 {
+			break
+		}
+		recDefault = append(recDefault, counter.roomId)
+	}
+	recDefaultRoomIds.Store(recDefault)
+	log.Info("refreshRoomFeature success, total num:%d recDefault_num:%d, recDefault:%+v", len(roomFeatures), len(recDefault), recDefault)
+	return
+}
+
+//建立房间相关的特征向量
+func createFeature(c *conf.Config, areaV2Id int64, cornerTag string, fansNum int64, onlineValue int64) (featureVector []int64) {
+	fansMilestone := c.CommonFeature.FansNum.Values
+	onlineMilestone := c.CommonFeature.Online.Values
+	cornerSignList := c.CommonFeature.CornerSign.Values
+
+	featureVector = append(featureVector, areaV2Id) //分区id, 留待在线计算的时候替换成0,1
+	featureVector = append(featureVector, oneHotEncode(fansNum, fansMilestone)...)
+	featureVector = append(featureVector, oneHotTextEncode(cornerTag, cornerSignList)...)
+	featureVector = append(featureVector, oneHotEncode(onlineValue, onlineMilestone)...)
+	return
+}
+
+//把slice按大小切成多个等大的小slice(除了最后一块)
+func sliceArray(arr []int64, n int) (ret [][]int64) {
+	remainder := len(arr) % n
+	quotient := (len(arr) - remainder) / n
+	num := int(quotient)
+	if remainder > 0 {
+		num = num + 1
+	}
+	ret = make([][]int64, 0, num)
+	for i := 0; i < num; i++ {
+		if i < num-1 {
+			ret = append(ret, arr[n*i:n*(i+1)])
+		} else {
+			ret = append(ret, arr[n*i:])
+		}
+	}
+	return
+}
+
+//构建0,1组成的特征向量; 如果x<0, 返回全为0的向量
+func int2Slice(x int, n int) []int64 {
+	p := make([]int64, n)
+	if x < 0 {
+		return p
+	}
+	p[x] = 1
+	return p
+}
+
+func compAndSet(value int64, vList []int64) int {
+	place := 0
+	for _, v := range vList {
+		if value < v {
+			return place
+		}
+		place = place + 1
+	}
+	return place
+}
+
+func oneHotEncode(value int64, milestone []int64) []int64 {
+	place := compAndSet(value, milestone)
+	return int2Slice(place, len(milestone)+1)
+}
+
+// textList ["", A, B ]
+// 如果targetText空或者没匹配到 ret[0] = 1
+func oneHotTextEncode(targetText string, textList []string) (ret []int64) {
+	place := 0
+	ret = make([]int64, len(textList))
+	if targetText == "" {
+		ret[0] = 1
+		return
+	}
+	for i, text := range textList {
+		if text == "" {
+			continue
+		}
+		match, err := regexp.MatchString(text, targetText)
+		if err != nil {
+			log.Error("oneHotTextEncode regex error " + text)
+			place = 0
+			break
+		}
+		if match {
+			place = i
+			break
+		}
+	}
+	ret[place] = 1
+	return
+}

app/service/live/recommend/internal/dao/room_feature_test.go

@@ -0,0 +1,63 @@
+package dao
+
+import (
+	"reflect"
+	"testing"
+
+	"flag"
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/service/live/recommend/internal/conf"
+)
+
+func init() {
+	flag.Set("conf", "../../cmd/test.toml")
+	var err error
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+}
+
+func TestOneHotTextEncode(t *testing.T) {
+	Convey("oneHotTextEncode", t, func() {
+		arr := oneHotTextEncode("", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{1, 0, 0, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("23人存活", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 1, 0, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("决赛圈", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 1, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("正在抽奖", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 1, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("上小时电台No.1", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 0, 1, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("年度五强主播", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 0, 0, 1}), ShouldBeTrue)
+	})
+}
+
+func TestOneHotEncode(t *testing.T) {
+	Convey("oneHotEncode", t, func() {
+		arr := oneHotEncode(78, []int64{23, 54, 100, 120})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 1, 0, 0}), ShouldBeTrue)
+		arr = oneHotEncode(7, []int64{23, 54, 100, 120})
+		So(reflect.DeepEqual(arr, []int64{1, 0, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotEncode(200, []int64{23, 54, 100, 120})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 0, 1}), ShouldBeTrue)
+	})
+}
+
+func TestSliceArray(t *testing.T) {
+	Convey("sliceArray", t, func() {
+		arr := sliceArray([]int64{1, 2, 3, 4, 5, 6, 7, 8, 9}, 4)
+		So(reflect.DeepEqual(arr[0], []int64{1, 2, 3, 4}), ShouldBeTrue)
+		So(reflect.DeepEqual(arr[1], []int64{5, 6, 7, 8}), ShouldBeTrue)
+		So(reflect.DeepEqual(arr[2], []int64{9}), ShouldBeTrue)
+	})
+}
+
+func TestCreateRoomFeature(t *testing.T) {
+	Convey("createRoomFeature", t, func() {
+		c := conf.Conf
+		arr := createFeature(c, 21, "决赛圈", 2000, 1000)
+		So(reflect.DeepEqual(arr, []int64{21, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0}), ShouldBeTrue)
+	})
+}