Create & Init Project...

app/service/live/recommend/BUILD

@@ -0,0 +1,24 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/live/recommend/api/grpc/v1:all-srcs",
+        "//app/service/live/recommend/cmd:all-srcs",
+        "//app/service/live/recommend/internal/conf:all-srcs",
+        "//app/service/live/recommend/internal/dao:all-srcs",
+        "//app/service/live/recommend/internal/model:all-srcs",
+        "//app/service/live/recommend/internal/server/grpc:all-srcs",
+        "//app/service/live/recommend/internal/server/http:all-srcs",
+        "//app/service/live/recommend/internal/service:all-srcs",
+        "//app/service/live/recommend/recconst:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/CHANGELOG.md

@@ -0,0 +1,28 @@
+### v1.0.0
+1. 上线功能xxx
+
+### v1.0.1
+1. 上线LR层功能
+
+### v1.0.2
+1. 修复ErrNil问题
+
+### v1.0.3
+1. 为无协同过滤数据的用户创建默认的LR推荐结果
+
+### v1.0.4
+1. 修改上一次提交的一个bug, 原bug会导致计算出来的LR分数没有失去意义
+2. 修复一次atomicValue的load操作没有判断成功
+
+### v1.0.5
+1. 加上了人气特征
+
+### v1.0.6
+1. 角标增加特征增加一个空角标特征
+2. 补充单元测试
+
+### v1.0.7
+1. 用默认的召回源数据补全用户的推荐房间列表
+
+### v1.0.8
+1. 完善了特征配置, 方便扩展

app/service/live/recommend/CONTRIBUTORS.md

@@ -0,0 +1,7 @@
+# Owner
+liugang
+
+# Author
+
+# Reviewer
+liuzhen

app/service/live/recommend/OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- liugang
+labels:
+- live
+- service
+- service/live/recommend
+options:
+  no_parent_owners: true
+reviewers:
+- iver
+- liuzhen

app/service/live/recommend/README.md

@@ -0,0 +1,24 @@
+# recommend-service
+
+## 项目简介
+1. 推荐服务当前由协同过滤提供召回源, 由LR提供召回源排序输出
+2. 当前的特征向量有按特征位排序:
+   1. 房间所在分区是否是用户常访问的
+   2. 粉丝数∈(0, 1000)
+   3. 粉丝数∈[1000, 1w)
+   4. 粉丝数∈[1w, 10w)
+   5. 粉丝数∈[10w, 30w)
+   6. 粉丝数∈[30w, +∞)
+   7. 吃鸡角标
+   8. 决赛圈角标
+   9. 抽奖角标
+   10. 小时榜角标
+   11~20. 人气特征
+
+## 编译环境
+
+
+## 依赖包
+
+
+## 编译执行

app/service/live/recommend/api/grpc/v1/BUILD

@@ -0,0 +1,63 @@
+load(
+    "@io_bazel_rules_go//proto:def.bzl",
+    "go_proto_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+proto_library(
+    name = "v1_proto",
+    srcs = ["api.proto"],
+    tags = ["automanaged"],
+    deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
+)
+
+go_proto_library(
+    name = "v1_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:gogofast_grpc"],
+    importpath = "go-common/app/service/live/recommend/api/grpc/v1",
+    proto = ":v1_proto",
+    tags = ["automanaged"],
+    deps = ["@com_github_gogo_protobuf//gogoproto:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.bm.go",
+        "client.go",
+        "generate.go",
+    ],
+    embed = [":v1_go_proto"],
+    importpath = "go-common/app/service/live/recommend/api/grpc/v1",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/binding:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "@com_github_gogo_protobuf//gogoproto:go_default_library",
+        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_x_net//context:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/api/grpc/v1/api.bm.go

@@ -0,0 +1,71 @@
+// Code generated by protoc-gen-bm v0.1, DO NOT EDIT.
+// source: api/grpc/v1/api.proto
+
+/*
+Package v1 is a generated blademaster stub package.
+This code was generated with go-common/app/tool/bmgen/protoc-gen-bm v0.1.
+
+It is generated from these files:
+	api/grpc/v1/api.proto
+*/
+package v1
+
+import (
+	"context"
+
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/binding"
+)
+
+// to suppressed 'imported but not used warning'
+var _ *bm.Context
+var _ context.Context
+var _ binding.StructValidator
+
+// ===================
+// Recommend Interface
+// ===================
+
+type Recommend interface {
+	// 获取n个推荐, 得到的结果是在线的房间
+	// 去重，不会重复推荐
+	// 如果没有足够推荐的结果则返回空的结果，调用方需要补位
+	RandomRecsByUser(ctx context.Context, req *GetRandomRecReq) (resp *GetRandomRecResp, err error)
+
+	// 清空推荐缓存，清空推荐过的集合
+	ClearRecommendCache(ctx context.Context, req *ClearRecommendRequest) (resp *ClearRecommendResponse, err error)
+}
+
+var v1RecommendSvc Recommend
+
+// @params GetRandomRecReq
+// @router GET /xlive/recommend/v1/recommend/random_recs_by_user
+// @response GetRandomRecResp
+func recommendRandomRecsByUser(c *bm.Context) {
+	p := new(GetRandomRecReq)
+	if err := c.BindWith(p, binding.Default(c.Request.Method, c.Request.Header.Get("Content-Type"))); err != nil {
+		return
+	}
+	resp, err := v1RecommendSvc.RandomRecsByUser(c, p)
+	c.JSON(resp, err)
+}
+
+// @params ClearRecommendRequest
+// @router GET /xlive/recommend/v1/recommend/clear_recommend_cache
+// @response ClearRecommendResponse
+func recommendClearRecommendCache(c *bm.Context) {
+	p := new(ClearRecommendRequest)
+	if err := c.BindWith(p, binding.Default(c.Request.Method, c.Request.Header.Get("Content-Type"))); err != nil {
+		return
+	}
+	resp, err := v1RecommendSvc.ClearRecommendCache(c, p)
+	c.JSON(resp, err)
+}
+
+// RegisterV1RecommendService Register the blademaster route with middleware map
+// midMap is the middleware map, the key is defined in proto
+func RegisterV1RecommendService(e *bm.Engine, svc Recommend, midMap map[string]bm.HandlerFunc) {
+	v1RecommendSvc = svc
+	e.GET("/xlive/recommend/v1/recommend/random_recs_by_user", recommendRandomRecsByUser)
+	e.GET("/xlive/recommend/v1/recommend/clear_recommend_cache", recommendClearRecommendCache)
+}

app/service/live/recommend/api/grpc/v1/api.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package live.recommend.v1;
+
+option go_package = "v1";
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+service Recommend {
+    // 获取n个推荐, 得到的结果是在线的房间
+    // 去重，不会重复推荐
+    // 如果没有足够推荐的结果则返回空的结果，调用方需要补位
+    rpc random_recs_by_user (GetRandomRecReq) returns (GetRandomRecResp);
+    // 清空推荐缓存，清空推荐过的集合
+    rpc clear_recommend_cache (ClearRecommendRequest) returns (ClearRecommendResponse);
+}
+
+message ClearRecommendRequest {
+    // 用户uid
+    int64 uid = 1 [(gogoproto.moretags) = 'validate:"gt=0"'];
+}
+
+message ClearRecommendResponse {
+
+}
+
+message GetRandomRecReq {
+    // 用户uid
+    int64 uid = 1 [(gogoproto.moretags) = 'validate:"gt=0"'];
+    // 获取数量
+    uint32 count = 2 [(gogoproto.moretags) = 'validate:"gt=0"'];
+    // room_id去重
+    repeated int64 exist_ids = 3;
+}
+
+message GetRandomRecResp {
+    // 返回数量
+    uint32 count = 1;
+    // 房间id
+    repeated int64 room_ids = 2;
+}

app/service/live/recommend/api/grpc/v1/api.recommend.md

@@ -0,0 +1,48 @@
+## 获取n个推荐, 得到的结果是在线的房间
+ 去重，不会重复推荐
+ 如果没有足够推荐的结果则返回空的结果，调用方需要补位
+
+`GET http://api.live.bilibili.com/xlive/recommend/v1/recommend/random_recs_by_user`
+
+### 请求参数
+
+|参数名|必选|类型|描述|
+|:---|:---|:---|:---|
+|uid|否|integer| 用户uid|
+|count|否|integer| 获取数量|
+|exist_ids|否|多个integer| room_id去重|
+
+```json
+{
+    "code": 0,
+    "message": "ok",
+    "data": {
+        //  返回数量
+        "count": 0,
+        //  房间id
+        "room_ids": [
+            0
+        ]
+    }
+}
+```
+
+## 清空推荐缓存，清空推荐过的集合
+
+`GET http://api.live.bilibili.com/xlive/recommend/v1/recommend/clear_recommend_cache`
+
+### 请求参数
+
+|参数名|必选|类型|描述|
+|:---|:---|:---|:---|
+|uid|否|integer| 用户uid|
+
+```json
+{
+    "code": 0,
+    "message": "ok",
+    "data": {
+    }
+}
+```
+

app/service/live/recommend/api/grpc/v1/client.go

@@ -0,0 +1,29 @@
+package v1
+
+import (
+	"context"
+
+	"google.golang.org/grpc"
+
+	"go-common/library/net/rpc/warden"
+)
+
+// AppID discovery id
+const AppID = "live.recommend"
+
+// Client recommend client
+type Client struct {
+	RecommendClient
+}
+
+// NewClient new grpc client
+func NewClient(cfg *warden.ClientConfig, opts ...grpc.DialOption) (*Client, error) {
+	client := warden.NewClient(cfg, opts...)
+	conn, err := client.Dial(context.Background(), "discovery://default/"+AppID)
+	if err != nil {
+		return nil, err
+	}
+	cli := &Client{}
+	cli.RecommendClient = NewRecommendClient(conn)
+	return cli, nil
+}

app/service/live/recommend/api/grpc/v1/generate.go

@@ -0,0 +1,3 @@
+package v1
+
+//go:generate bmgen --nobm

app/service/live/recommend/api/http/api.md

@@ -0,0 +1 @@
+# HTTP API文档

app/service/live/recommend/cmd/BUILD

@@ -0,0 +1,49 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["test.toml"],
+    importpath = "go-common/app/service/live/recommend/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//app/service/live/recommend/internal/dao:go_default_library",
+        "//app/service/live/recommend/internal/server/grpc:go_default_library",
+        "//app/service/live/recommend/internal/server/http:go_default_library",
+        "//app/service/live/recommend/internal/service:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/live/recommend/cmd/client:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/cmd/client/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "client",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "go-common/app/service/live/recommend/cmd/client",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/api/grpc/v1:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/cmd/client/main.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"time"
+
+	"go-common/app/service/live/recommend/api/grpc/v1"
+	"go-common/library/net/rpc/warden"
+	xtime "go-common/library/time"
+)
+
+var name, addr string
+
+func init() {
+	flag.StringVar(&name, "name", "lily", "name")
+	flag.StringVar(&addr, "addr", "127.0.0.1:9000", "server addr")
+}
+func main() {
+	flag.Parse()
+	cfg := &warden.ClientConfig{
+		Dial:    xtime.Duration(time.Second * 3),
+		Timeout: xtime.Duration(time.Second * 3),
+	}
+	cc, err := warden.NewClient(cfg).Dial(context.Background(), addr)
+	if err != nil {
+		log.Fatalf("new client failed!err:=%v", err)
+		return
+	}
+	client := v1.NewRecommendClient(cc)
+	resp, err := client.RandomRecsByUser(context.Background(), &v1.GetRandomRecReq{
+		Uid: 4158272, Count: 5,
+	})
+	if err != nil {
+		log.Fatalf("say hello failed!err:=%v", err)
+		return
+	}
+	fmt.Printf("got HelloReply:%+v", resp)
+}

app/service/live/recommend/cmd/main.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/service/live/recommend/internal/conf"
+	"go-common/app/service/live/recommend/internal/dao"
+	"go-common/app/service/live/recommend/internal/server/grpc"
+	"go-common/app/service/live/recommend/internal/server/http"
+	"go-common/app/service/live/recommend/internal/service"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+var runJob = false
+var jobFilePath = ""
+var jobWorkerNum = 1000
+var jobOffset = -1
+
+func main() {
+	flag.BoolVar(&runJob, "runJob", false, "跑redis脚本")
+	flag.StringVar(&jobFilePath, "jobFile", "", "推荐文件地址")
+	flag.IntVar(&jobOffset, "jobOffset", -1, "操作偏移，即从n行开始跑, 默认会从上一次中断的地方开始")
+	flag.IntVar(&jobWorkerNum, "jobWorkerNum", 1000, "worker数量")
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("recommend-service start")
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	ecode.Init(conf.Conf.Ecode)
+
+	svc := service.New(conf.Conf)
+	http.Init(conf.Conf, svc)
+	grpc.Init(conf.Conf)
+	go dao.StartRefreshJob()
+	go dao.StartRoomFeatureJob(conf.Conf)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			svc.Close()
+			log.Info("recommend-service exit")
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/service/live/recommend/cmd/test.toml

@@ -0,0 +1,35 @@
+[log]
+    stdout = true
+
+[redis]
+	name = "recommend-service"
+	proto = "tcp"
+	addr = "127.0.0.1:6379"
+	idle = 20
+	active = 20
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[Feature]
+    WeightVector = [3.0519, -1.6184, -2.0771, -1.9381, -1.9873, -1.6225, -2.0000, -1.8740, -1.7568, -1.7802, -1.9143, -1.8643, -4.1616, -2.6550, -2.2589, -2.3561, -2.1690, -1.9138, -1.6899, -1.6309, -1.3955, -1.2116]
+
+[CommonFeature]
+    [CommonFeature.UserAreaInterest]
+        Type = "NumMatch"
+        Values = [1]
+        Weights = [3.0519]
+    [CommonFeature.FansNum]
+        Type = "RangeSplit"
+        Values = [100, 500, 2500, 10000]
+        Weights = [-1.6184, -2.0771, -1.9381, -1.9873, -1.6225]
+    [CommonFeature.CornerSign]
+        Type = "ReMatch"
+        Values = ["", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播", "周星主播", "BLS.*"]
+        Weights = [-2.0000, -1.8740, -1.7568, -1.7802, -1.9143, -1.8643, -1.8643, -1.8643]
+    [CommonFeature.Online]
+        Type = "RangeSplit"
+        Values = [10, 50, 100, 300, 1600, 3600, 6000, 10000, 100000]
+        Weights = [-4.1616, -2.6550, -2.2589, -2.3561, -2.1690, -1.9138, -1.6899, -1.6309, -1.3955, -1.2116]

app/service/live/recommend/internal/conf/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/service/live/recommend/internal/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/rpc/liverpc:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/internal/conf/conf.go

@@ -0,0 +1,126 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"github.com/BurntSushi/toml"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/rpc/liverpc"
+	"go-common/library/net/trace"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config .
+type Config struct {
+	Log           *log.Config
+	BM            *bm.ServerConfig
+	Verify        *verify.Config
+	Tracer        *trace.Config
+	Redis         *redis.Config
+	Memcache      *memcache.Config
+	MySQL         *sql.Config
+	Ecode         *ecode.Config
+	LiveRpc       map[string]*liverpc.ClientConfig
+	Feature       *FeatureConf
+	CommonFeature *CommonFeatureConf
+}
+
+// CommonFeatureConf 细分维度的特征权重配置
+type CommonFeatureConf struct {
+	UserAreaInterest NumMatch
+	FansNum          RangeSplit
+	CornerSign       ReMatch
+	Online           RangeSplit
+}
+
+// NumMatch 通过int匹配获得权重
+type NumMatch struct {
+	Type    string
+	Values  []int
+	Weights []float32
+}
+
+// ReMatch 通过文字匹配获得权重
+type ReMatch struct {
+	Type    string
+	Values  []string
+	Weights []float32
+}
+
+// RangeSplit 通过分割区间获得权重
+type RangeSplit struct {
+	Type    string
+	Values  []int64
+	Weights []float32
+}
+
+// FeatureConf 特征配置
+type FeatureConf struct {
+	WeightVector []float32
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/service/live/recommend/internal/dao/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "online_filter.go",
+        "room_feature.go",
+    ],
+    importpath = "go-common/app/service/live/recommend/internal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//app/service/live/recommend/recconst:go_default_library",
+        "//app/service/live/relation/api/liverpc:go_default_library",
+        "//app/service/live/relation/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc:go_default_library",
+        "//app/service/live/room/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc/v2:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/liverpc:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "room_feature_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/service/live/recommend/internal/dao/dao.go

@@ -0,0 +1,465 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"go-common/app/service/live/recommend/internal/conf"
+	"go-common/app/service/live/recommend/recconst"
+	relation_api "go-common/app/service/live/relation/api/liverpc"
+	room_api "go-common/app/service/live/room/api/liverpc"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"go-common/library/net/rpc/liverpc"
+)
+
+var _userRecCandidateKey = "rec_candidate_%d"
+var _recommendOffsetKey = "rec_offset_%d"
+
+// 已经推荐过的池子，用户+日期
+var _recommendedKey = "recommended_%d_%s"
+
+// RoomAPI room liverpc client
+var RoomAPI *room_api.Client
+
+// RelationAPI relation liverpc client
+var RelationAPI *relation_api.Client
+
+// Dao dao
+type Dao struct {
+	c     *conf.Config
+	redis *redis.Pool
+}
+
+func init() {
+	RoomAPI = room_api.New(getConf("room"))
+	RelationAPI = relation_api.New(getConf("relation"))
+}
+
+func getConf(appName string) *liverpc.ClientConfig {
+	c := conf.Conf.LiveRpc
+	if c != nil {
+		return c[appName]
+	}
+	return nil
+}
+
+// ClearRecommend 清空该用户相关的推荐缓存
+func (d *Dao) ClearRecommend(ctx context.Context, uid int64) error {
+	candidateKey := fmt.Sprintf(_userRecCandidateKey, uid)
+	recommendedKey := fmt.Sprintf(_recommendedKey, uid, time.Now().Format("20060102"))
+	offsetKey := fmt.Sprintf(_recommendOffsetKey, uid)
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	_, err := conn.Do("DEL", candidateKey, recommendedKey, offsetKey)
+	return errors.WithStack(err)
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:     c,
+		redis: redis.NewPool(c.Redis),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+}
+
+func (d *Dao) saveOffset(conn redis.Conn, uid int64, offset int) {
+	conn.Do("SETEX", fmt.Sprintf(_recommendOffsetKey, uid), 86400, offset)
+}
+
+func (d *Dao) addToRecommended(conn redis.Conn, uid int64, ids []int64) {
+	if len(ids) == 0 {
+		return
+	}
+	day := time.Now().Format("20060102")
+
+	key := fmt.Sprintf(_recommendedKey, uid, day)
+	var is []interface{}
+	is = append(is, key)
+	for _, id := range ids {
+		is = append(is, id)
+	}
+
+	conn.Send("EXPIRE", key, 86400)
+	conn.Send("SADD", is...)
+	conn.Flush()
+	conn.Receive()
+	_, err := conn.Receive()
+	if err != nil {
+		log.Info("addToRecommended error +%v", err)
+	}
+}
+
+// GetRandomRoomIds 随机获取count个推荐
+// 如果总数量total比count小，则返回total个
+func (d *Dao) GetRandomRoomIds(ctx context.Context, uid int64, reqCount int, existRoomIDs []int64) (ret []int64, err error) {
+	if reqCount == 0 {
+		return
+	}
+	var (
+		candidateLen int
+	)
+	r := d.redis.Get(ctx)
+	defer r.Close()
+
+	candidateKey := fmt.Sprintf(_userRecCandidateKey, uid)
+	exists, err := redis.Int(r.Do("exists", candidateKey))
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+
+	existMap := map[int64]struct{}{}
+	for _, id := range existRoomIDs {
+		existMap[id] = struct{}{}
+	}
+	if exists == 0 {
+		var candidate []int64
+		var currentOffset = 0
+		candidate, err = d.generateLrCandidateList(r, uid, candidateKey)
+		if err != nil {
+			return
+		}
+	Loop:
+		for len(ret) < reqCount && currentOffset < len(candidate) {
+			var tmp []int64
+			if len(candidate)-currentOffset < int(reqCount) {
+				tmp = candidate[currentOffset:]
+			} else {
+				tmp = candidate[currentOffset : currentOffset+reqCount]
+			}
+			//去重
+			for _, id := range tmp {
+				_, ok := existMap[id]
+				currentOffset += 1
+				if !ok {
+					ret = append(ret, id)
+					if len(ret) >= int(reqCount) {
+						break Loop
+					}
+				}
+			}
+		}
+
+		d.addToRecommended(r, uid, ret)
+		d.saveOffset(r, uid, currentOffset)
+	} else {
+		candidateLen, err = redis.Int(r.Do("LLEN", candidateKey))
+		if err != nil {
+			return
+		}
+
+		var offset int
+		offset, _ = redis.Int(r.Do("GET", fmt.Sprintf(_recommendOffsetKey, uid)))
+		if offset > (candidateLen - 1) {
+			return
+		}
+		var currentOffset = offset
+	Loop2:
+		for len(ret) < reqCount && currentOffset < candidateLen {
+			var ids []int64
+			ids, err = redis.Int64s(r.Do("LRANGE", candidateKey, currentOffset, currentOffset+reqCount-1))
+			if err != nil {
+				err = errors.WithStack(err)
+				return
+			}
+			// 去重
+			for _, id := range ids {
+				currentOffset++
+				_, ok := existMap[id]
+				if !ok {
+					ret = append(ret, id)
+					if len(ret) >= int(reqCount) {
+						break Loop2
+					}
+				}
+			}
+			if len(ids) == 0 {
+				log.Error("Cannot get recommend candidate, key=%s, offset=%d, count=%d", candidateKey, offset, reqCount)
+				break
+			}
+		}
+
+		d.addToRecommended(r, uid, ret)
+		d.saveOffset(r, uid, currentOffset)
+	}
+	return
+}
+
+// GetLrRecRoomIds 在GetRandomRoomIds的基础上进行LR计算并返回倒排的房间号列表
+// 与GetRandomRoomIds有相同的输入输出结构
+func (d *Dao) GetLrRecRoomIds(r redis.Conn, uid int64, candidateIds []int64) (ret []int64, err error) {
+	var areas string
+	areaIds := map[int64]struct{}{}
+	areas, err = redis.String(r.Do("GET", fmt.Sprintf(recconst.UserAreaKey, uid)))
+	if err != nil && err != redis.ErrNil {
+		log.Error("redis GET error: %v", err)
+		return
+	}
+	err = nil
+	if areas != "" {
+		split := strings.Split(areas, ";")
+		for _, areaIdStr := range split {
+			areaId, _ := strconv.ParseInt(areaIdStr, 10, 64)
+			areaIds[areaId] = struct{}{}
+		}
+	}
+
+	weightVector := makeWeightVec(d.c)
+	roomFeatures, ok := roomFeatureValue.Load().(map[int64][]int64)
+	if !ok {
+		ret = candidateIds
+		return
+	}
+	roomScoreSlice := ScoreSlice{}
+	for _, roomId := range candidateIds {
+		if fv, ok := roomFeatures[roomId]; ok {
+			featureVector := make([]int64, len(fv))
+			copy(featureVector, fv)
+			areaId := featureVector[0]
+			if _, ok := areaIds[areaId]; ok {
+				featureVector[0] = 1
+			} else {
+				featureVector[0] = 0
+			}
+			counter := Counter{roomId: roomId, score: calcScore(weightVector, featureVector)}
+			roomScoreSlice = append(roomScoreSlice, counter)
+		}
+	}
+
+	sort.Sort(roomScoreSlice)
+	for _, counter := range roomScoreSlice {
+		ret = append(ret, counter.roomId)
+	}
+	return
+}
+
+// generateCandidateList 得到候选集
+func (d *Dao) generateCandidateList(r redis.Conn, uid int64, candidateKey string) (ret []int64, err error) {
+
+	// 第一步 itemcf，优先级最高。
+	itemCFKey := fmt.Sprintf(recconst.UserItemCFRecKey, uid)
+	var itemCFList []int64
+	itemCFList, err = redis.Int64s(r.Do("ZREVRANGE", itemCFKey, 0, -1))
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	itemCFOnlineIds := d.FilterOnlineRoomIds(itemCFList)
+	if len(itemCFOnlineIds) == 0 {
+		log.Info("No item-cf room online for user, uid=%d, before online filter room ids: %+v", uid, itemCFList)
+	}
+
+	// 第二步 取兴趣分区的房间 人气超过100的房间
+	var areas string
+	areas, err = redis.String(r.Do("GET", fmt.Sprintf(recconst.UserAreaKey, uid)))
+	if err != nil && err != redis.ErrNil {
+		err = errors.WithStack(err)
+		return
+	}
+	err = nil
+	var areaRoomIDs []int64
+	if areas != "" {
+		split := strings.Split(areas, ";")
+		for _, areaIdStr := range split {
+			areaId, _ := strconv.ParseInt(areaIdStr, 10, 64)
+			var ids = d.getAreaRoomIds(areaId)
+			areaRoomIDs = append(areaRoomIDs, ids...)
+		}
+	}
+
+	// 第三步 取兴趣分区大分区的100个 先不做
+	// 第四步 减去已经推荐过的
+	day := time.Now().Format("20060102")
+	var recommendedList []int64
+	edKey := fmt.Sprintf(_recommendedKey, uid, day)
+	recommendedList, err = redis.Int64s(r.Do("SMEMBERS", edKey))
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	recommended := map[int64]struct{}{}
+	for _, id := range recommendedList {
+		recommended[id] = struct{}{}
+	}
+	var itemCFFinalIDs []int64
+	for _, id := range itemCFOnlineIds {
+		_, exist := recommended[id]
+		if !exist {
+			itemCFFinalIDs = append(itemCFFinalIDs, id)
+		}
+	}
+
+	var areaRoomFinalIDs []int64
+	for _, id := range areaRoomIDs {
+		_, exist := recommended[id]
+		if !exist {
+			areaRoomFinalIDs = append(areaRoomFinalIDs, id)
+		}
+	}
+
+	ret = mergeArr(itemCFFinalIDs, areaRoomFinalIDs)
+
+	log.Info("UserRecommend : uid=%d total=%d, "+
+		"itemcf.original=%d, itemcf.online=%d, itemcf.noviewd=%d, "+
+		"areaRoom.original=%d, itemcf.noviewd=%d viewed=%d",
+		uid, len(ret), len(itemCFList), len(itemCFOnlineIds), len(itemCFFinalIDs),
+		len(areaRoomIDs), len(areaRoomFinalIDs), len(recommendedList))
+	return
+}
+
+// generateCandidateList 得到进过LR的候选集
+func (d *Dao) generateLrCandidateList(r redis.Conn, uid int64, candidateKey string) (ret []int64, err error) {
+	roomIDs, err := d.generateCandidateList(r, uid, candidateKey)
+	if err != nil {
+		log.Error("generateLrCandidateList failed 1, error:%v", err)
+		return
+	}
+
+	if len(ret) > 0 {
+		ret, err = d.GetLrRecRoomIds(r, uid, roomIDs)
+		if err != nil {
+			log.Error("generateLrCandidateList failed 2, error:%v", err)
+			return
+		}
+	}
+
+	// 召回源不足的情况下补足推荐房间数
+	if len(ret) < 150 {
+		ids, ok := recDefaultRoomIds.Load().([]int64)
+		if !ok {
+			return
+		}
+		ret1, err1 := d.GetLrRecRoomIds(r, uid, ids)
+		if err1 != nil {
+			log.Error("generateLrCandidateList failed 3, error:%v", err1)
+			return
+		}
+		ret = mergeArrWithOrder(ret, ret1, 150) // TODO:当前ret1的结果是没有过滤掉今天看过的房间的, 看后面是否需要优化
+	}
+	{
+		for _, roomID := range ret {
+			r.Send("RPUSH", candidateKey, roomID)
+		}
+		r.Send("EXPIRE", candidateKey, 60*2)
+		err = r.Flush()
+		if err != nil {
+			err = errors.WithStack(err)
+			return
+		}
+		for i := 0; i < len(ret)+1; i++ {
+			r.Receive()
+		}
+	}
+	return
+}
+
+// Ping dao ping
+func (d *Dao) Ping(ctx context.Context) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	_, err = conn.Do("ping")
+	if err != nil {
+		err = errors.Wrap(err, "dao Ping err")
+	}
+	return err
+}
+
+// Counter 房间-分数结构体, 用于构建一个可排序的slice
+type Counter struct {
+	roomId int64
+	score  float32
+}
+
+// ScoreSlice Counter对象的slice
+type ScoreSlice []Counter
+
+func (s ScoreSlice) Len() int {
+	return len(s)
+}
+
+func (s ScoreSlice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s ScoreSlice) Less(i, j int) bool {
+	return s[j].score < s[i].score
+}
+
+func calcScore(weightVector []float32, featureVector []int64) (score float32) {
+	if len(weightVector) != len(featureVector) {
+		panic(fmt.Sprintf("权重数量和特征数量不匹配, 请检查配置或逻辑, weight: %+v, feature: %+v", weightVector, featureVector))
+	}
+	for i := 0; i < min(len(weightVector), len(featureVector)); i++ {
+		score += weightVector[i] * float32(featureVector[i])
+	}
+	return
+}
+
+func min(x int, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+// 合并两个集合
+func mergeArr(x []int64, y []int64) (ret []int64) {
+	tmpMap := map[int64]struct{}{}
+	for _, id := range x {
+		tmpMap[id] = struct{}{}
+	}
+	for _, id := range y {
+		tmpMap[id] = struct{}{}
+	}
+	for id := range tmpMap {
+		ret = append(ret, id)
+	}
+	return
+}
+
+// 按x, y的顺序合并两个集合, 当x的长度不小于limit则直接返回
+func mergeArrWithOrder(x []int64, y []int64, limit int) (ret []int64) {
+	if len(x) >= limit {
+		ret = x
+		return
+	}
+	tmpMap := map[int64]struct{}{}
+	ret = append(ret, x...)
+	num := len(ret)
+	for _, id := range x {
+		tmpMap[id] = struct{}{}
+	}
+	for _, id := range y {
+		if _, ok := tmpMap[id]; ok {
+			continue
+		}
+		num += 1
+		tmpMap[id] = struct{}{}
+		ret = append(ret, id)
+		if num >= limit {
+			break
+		}
+	}
+	return
+}
+
+func makeWeightVec(c *conf.Config) (ret []float32) {
+	ret = append(ret, c.CommonFeature.UserAreaInterest.Weights...)
+	ret = append(ret, c.CommonFeature.FansNum.Weights...)
+	ret = append(ret, c.CommonFeature.CornerSign.Weights...)
+	ret = append(ret, c.CommonFeature.Online.Weights...)
+	return
+}

app/service/live/recommend/internal/dao/dao_test.go

@@ -0,0 +1,30 @@
+package dao
+
+import (
+	"reflect"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestMergeArrWithOrder(t *testing.T) {
+	Convey("mergeArrWithOrder", t, func() {
+		a := []int64{1, 2, 3, 4}
+		b := []int64{4, 5, 6, 7}
+		c := mergeArrWithOrder(a, b, len(a)+len(b))
+		So(reflect.DeepEqual(c, []int64{1, 2, 3, 4, 5, 6, 7}), ShouldBeTrue)
+		c = mergeArrWithOrder(a, b, 2)
+		So(reflect.DeepEqual(c, []int64{1, 2, 3, 4}), ShouldBeTrue)
+		c = mergeArrWithOrder(a, b, 6)
+		So(reflect.DeepEqual(c, []int64{1, 2, 3, 4, 5, 6}), ShouldBeTrue)
+	})
+}
+
+func TestMergeArr(t *testing.T) {
+	Convey("mergeArr", t, func() {
+		a := []int64{1, 2, 3, 4}
+		b := []int64{4, 5, 6, 7}
+		c := mergeArr(a, b)
+		So(len(c) == 7, ShouldBeTrue)
+	})
+}

app/service/live/recommend/internal/dao/online_filter.go

@@ -0,0 +1,72 @@
+package dao
+
+import (
+	"context"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/service/live/room/api/liverpc/v1"
+	"go-common/library/log"
+)
+
+var onlineRoomIdValue atomic.Value
+var areaRoomsValue atomic.Value
+
+// StartRefreshJob 更新在线房间信息
+func StartRefreshJob() {
+	t := time.Tick(time.Second * 60)
+	refreshOnlineRoomData(context.Background())
+	for range t {
+		refreshOnlineRoomData(context.Background())
+	}
+}
+
+// refreshOnlineRoomData 更新RoomId
+func refreshOnlineRoomData(ctx context.Context) (err error) {
+	resp, err := RoomAPI.V1Room.AllLiveForBigdata(ctx, &v1.RoomAllLiveForBigdataReq{})
+	if err != nil {
+		return
+	}
+	onlineRooms := map[int64]struct{}{}
+	areaRooms := map[int64][]int64{}
+	for _, info := range resp.Data {
+		roomID := info.Roomid
+
+		onlineRooms[roomID] = struct{}{}
+		if info.Online > 100 {
+			areaRooms[info.AreaV2Id] = append(areaRooms[info.AreaV2Id], roomID)
+		}
+	}
+	log.Info("refreshOnlineRoomData: count=%d", len(onlineRooms))
+	log.Info("refreshOnlineRoomData area Rooms: %+v", areaRooms)
+	onlineRoomIdValue.Store(onlineRooms)
+	areaRoomsValue.Store(areaRooms)
+	return
+}
+
+func (d *Dao) getAreaRoomIds(areaId int64) (ret []int64) {
+	ret = make([]int64, 0)
+	areaRooms, ok := areaRoomsValue.Load().(map[int64][]int64)
+	if !ok {
+		log.Warn("cannot load current area room ids")
+		return
+	}
+	ret = areaRooms[areaId]
+	return
+}
+
+// FilterOnlineRoomIds 给定一批room id 返回所有在线的
+func (d *Dao) FilterOnlineRoomIds(roomIds []int64) (ret []int64) {
+	ret = make([]int64, 0)
+	currentIds, ok := onlineRoomIdValue.Load().(map[int64]struct{})
+	if !ok {
+		log.Warn("cannot load current online room ids")
+		return
+	}
+	for _, roomId := range roomIds {
+		if _, ok := currentIds[roomId]; ok {
+			ret = append(ret, roomId)
+		}
+	}
+	return
+}

app/service/live/recommend/internal/dao/room_feature.go

@@ -0,0 +1,204 @@
+package dao
+
+import (
+	"context"
+	"errors"
+	"regexp"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/service/live/recommend/internal/conf"
+	relationV1 "go-common/app/service/live/relation/api/liverpc/v1"
+	roomV1 "go-common/app/service/live/room/api/liverpc/v1"
+	roomV2 "go-common/app/service/live/room/api/liverpc/v2"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+)
+
+var roomFeatureValue atomic.Value
+var recDefaultRoomIds atomic.Value
+
+// StartRoomFeatureJob 更新在线房间的特征信息
+func StartRoomFeatureJob(c *conf.Config) {
+	t := time.Tick(time.Second * 30)
+	refreshRoomFeature(context.Background(), c)
+	for range t {
+		refreshRoomFeature(context.Background(), c)
+	}
+}
+
+func refreshRoomFeature(ctx context.Context, c *conf.Config) (err error) {
+	n := 20
+	currentIds, ok := onlineRoomIdValue.Load().(map[int64]struct{})
+	if !ok {
+		log.Warn("cannot load current online room ids")
+		err = errors.New("cannot load current online room ids")
+		return
+	}
+	keys := make([]int64, 0, len(currentIds))
+	for k := range currentIds {
+		keys = append(keys, k)
+	}
+	chunkIdsArray := sliceArray(keys, n)
+	roomFeatures := map[int64][]int64{}
+	var lock sync.Mutex
+	var eg errgroup.Group
+	for _, tmp := range chunkIdsArray {
+		chunkIds := tmp
+		eg.Go(func() (err error) {
+			resp, err := RoomAPI.V2Room.GetByIds(ctx, &roomV2.RoomGetByIdsReq{Ids: chunkIds})
+			if err != nil || resp.GetCode() != 0 {
+				log.Error("dao.RoomAPI.V2Room.GetByIds (%v) error(%v) resp(%v)", chunkIds, err, resp)
+				return
+			}
+
+			resp1, err1 := RoomAPI.V1RoomPendant.GetPendantByIds(ctx, &roomV1.RoomPendantGetPendantByIdsReq{Ids: chunkIds, Type: "mobile_index_badge", Position: 2})
+			if err1 != nil || resp1.GetCode() != 0 {
+				log.Error("dao.RoomAPI.V1Room.GetPendantByIds (%v) error(%v) resp(%v)", chunkIds, err1, resp1)
+				return
+			}
+
+			uids := make([]int64, 0, n)
+			for _, r := range resp.Data {
+				uids = append(uids, r.Uid)
+			}
+			resp2, err2 := RelationAPI.V1Feed.GetUserFcBatch(ctx, &relationV1.FeedGetUserFcBatchReq{Uids: uids})
+			if err2 != nil || resp.GetCode() != 0 {
+				log.Error("dao.RelationAPI.V1Relation.GetUserFcBatch (%v) error(%v) resp(%v)", chunkIds, err2, resp2)
+				return
+			}
+
+			roomPendantInfo := resp1.Data.Result
+			fansCountInfo := resp2.Data
+
+			for roomId, r := range resp.Data {
+				cornerTag := ""
+				fansNum := int64(0)
+				if PendantInfo, ok := roomPendantInfo[roomId]; ok && PendantInfo != nil {
+					cornerTag = PendantInfo.Value
+				}
+				if fans, ok := fansCountInfo[r.Uid]; ok {
+					fansNum = fans.Fc
+				}
+				featureVector := createFeature(c, r.AreaV2Id, cornerTag, fansNum, r.Online)
+				lock.Lock()
+				roomFeatures[roomId] = featureVector
+				lock.Unlock()
+			}
+			return
+		})
+	}
+	eg.Wait()
+	roomFeatureValue.Store(roomFeatures)
+	//创建默认推荐房间列表
+	roomScoreSlice := ScoreSlice{}
+	for roomId, vec := range roomFeatures {
+		featureVector := make([]int64, len(vec))
+		copy(featureVector, vec)
+		featureVector[0] = 0
+		counter := Counter{roomId: roomId, score: calcScore(makeWeightVec(c), featureVector)}
+		roomScoreSlice = append(roomScoreSlice, counter)
+	}
+	sort.Sort(roomScoreSlice)
+	//默认的召回源
+	limit := 400
+	recDefault := make([]int64, 0, limit)
+	for _, counter := range roomScoreSlice {
+		limit = limit - 1
+		if limit < 0 {
+			break
+		}
+		recDefault = append(recDefault, counter.roomId)
+	}
+	recDefaultRoomIds.Store(recDefault)
+	log.Info("refreshRoomFeature success, total num:%d recDefault_num:%d, recDefault:%+v", len(roomFeatures), len(recDefault), recDefault)
+	return
+}
+
+//建立房间相关的特征向量
+func createFeature(c *conf.Config, areaV2Id int64, cornerTag string, fansNum int64, onlineValue int64) (featureVector []int64) {
+	fansMilestone := c.CommonFeature.FansNum.Values
+	onlineMilestone := c.CommonFeature.Online.Values
+	cornerSignList := c.CommonFeature.CornerSign.Values
+
+	featureVector = append(featureVector, areaV2Id) //分区id, 留待在线计算的时候替换成0,1
+	featureVector = append(featureVector, oneHotEncode(fansNum, fansMilestone)...)
+	featureVector = append(featureVector, oneHotTextEncode(cornerTag, cornerSignList)...)
+	featureVector = append(featureVector, oneHotEncode(onlineValue, onlineMilestone)...)
+	return
+}
+
+//把slice按大小切成多个等大的小slice(除了最后一块)
+func sliceArray(arr []int64, n int) (ret [][]int64) {
+	remainder := len(arr) % n
+	quotient := (len(arr) - remainder) / n
+	num := int(quotient)
+	if remainder > 0 {
+		num = num + 1
+	}
+	ret = make([][]int64, 0, num)
+	for i := 0; i < num; i++ {
+		if i < num-1 {
+			ret = append(ret, arr[n*i:n*(i+1)])
+		} else {
+			ret = append(ret, arr[n*i:])
+		}
+	}
+	return
+}
+
+//构建0,1组成的特征向量; 如果x<0, 返回全为0的向量
+func int2Slice(x int, n int) []int64 {
+	p := make([]int64, n)
+	if x < 0 {
+		return p
+	}
+	p[x] = 1
+	return p
+}
+
+func compAndSet(value int64, vList []int64) int {
+	place := 0
+	for _, v := range vList {
+		if value < v {
+			return place
+		}
+		place = place + 1
+	}
+	return place
+}
+
+func oneHotEncode(value int64, milestone []int64) []int64 {
+	place := compAndSet(value, milestone)
+	return int2Slice(place, len(milestone)+1)
+}
+
+// textList ["", A, B ]
+// 如果targetText空或者没匹配到 ret[0] = 1
+func oneHotTextEncode(targetText string, textList []string) (ret []int64) {
+	place := 0
+	ret = make([]int64, len(textList))
+	if targetText == "" {
+		ret[0] = 1
+		return
+	}
+	for i, text := range textList {
+		if text == "" {
+			continue
+		}
+		match, err := regexp.MatchString(text, targetText)
+		if err != nil {
+			log.Error("oneHotTextEncode regex error " + text)
+			place = 0
+			break
+		}
+		if match {
+			place = i
+			break
+		}
+	}
+	ret[place] = 1
+	return
+}

app/service/live/recommend/internal/dao/room_feature_test.go

@@ -0,0 +1,63 @@
+package dao
+
+import (
+	"reflect"
+	"testing"
+
+	"flag"
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/service/live/recommend/internal/conf"
+)
+
+func init() {
+	flag.Set("conf", "../../cmd/test.toml")
+	var err error
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+}
+
+func TestOneHotTextEncode(t *testing.T) {
+	Convey("oneHotTextEncode", t, func() {
+		arr := oneHotTextEncode("", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{1, 0, 0, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("23人存活", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 1, 0, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("决赛圈", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 1, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("正在抽奖", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 1, 0, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("上小时电台No.1", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 0, 1, 0}), ShouldBeTrue)
+		arr = oneHotTextEncode("年度五强主播", []string{"", ".*人存活", "决赛圈", "正在抽奖", ".*No\\.\\d+", "年度.*主播"})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 0, 0, 1}), ShouldBeTrue)
+	})
+}
+
+func TestOneHotEncode(t *testing.T) {
+	Convey("oneHotEncode", t, func() {
+		arr := oneHotEncode(78, []int64{23, 54, 100, 120})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 1, 0, 0}), ShouldBeTrue)
+		arr = oneHotEncode(7, []int64{23, 54, 100, 120})
+		So(reflect.DeepEqual(arr, []int64{1, 0, 0, 0, 0}), ShouldBeTrue)
+		arr = oneHotEncode(200, []int64{23, 54, 100, 120})
+		So(reflect.DeepEqual(arr, []int64{0, 0, 0, 0, 1}), ShouldBeTrue)
+	})
+}
+
+func TestSliceArray(t *testing.T) {
+	Convey("sliceArray", t, func() {
+		arr := sliceArray([]int64{1, 2, 3, 4, 5, 6, 7, 8, 9}, 4)
+		So(reflect.DeepEqual(arr[0], []int64{1, 2, 3, 4}), ShouldBeTrue)
+		So(reflect.DeepEqual(arr[1], []int64{5, 6, 7, 8}), ShouldBeTrue)
+		So(reflect.DeepEqual(arr[2], []int64{9}), ShouldBeTrue)
+	})
+}
+
+func TestCreateRoomFeature(t *testing.T) {
+	Convey("createRoomFeature", t, func() {
+		c := conf.Conf
+		arr := createFeature(c, 21, "决赛圈", 2000, 1000)
+		So(reflect.DeepEqual(arr, []int64{21, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0}), ShouldBeTrue)
+	})
+}

app/service/live/recommend/internal/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/service/live/recommend/internal/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/internal/model/model.go

@@ -0,0 +1 @@
+package model

app/service/live/recommend/internal/server/grpc/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["server.go"],
+    importpath = "go-common/app/service/live/recommend/internal/server/grpc",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/api/grpc/v1:go_default_library",
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//app/service/live/recommend/internal/service/v1:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/internal/server/grpc/server.go

@@ -0,0 +1,20 @@
+package grpc
+
+import (
+	v1pb "go-common/app/service/live/recommend/api/grpc/v1"
+	"go-common/app/service/live/recommend/internal/conf"
+	svc "go-common/app/service/live/recommend/internal/service/v1"
+	"go-common/library/log"
+	"go-common/library/net/rpc/warden"
+)
+
+// Init grpc server
+func Init(c *conf.Config) {
+	s := warden.NewServer(nil)
+	v1pb.RegisterRecommendServer(s.Server(), svc.NewRecommendService(c))
+	_, err := s.Start()
+	if err != nil {
+		log.Error("grpc Start error(%v)", err)
+		panic(err)
+	}
+}

app/service/live/recommend/internal/server/http/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/service/live/recommend/internal/server/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/api/grpc/v1:go_default_library",
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//app/service/live/recommend/internal/service:go_default_library",
+        "//app/service/live/recommend/internal/service/v1:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/internal/server/http/http.go

@@ -0,0 +1,56 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/service/live/recommend/api/grpc/v1"
+	"go-common/app/service/live/recommend/internal/conf"
+	"go-common/app/service/live/recommend/internal/service"
+	v12 "go-common/app/service/live/recommend/internal/service/v1"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	vfy *verify.Verify
+	svc *service.Service
+)
+
+// Init init
+func Init(c *conf.Config, s *service.Service) {
+	svc = s
+	vfy = verify.New(c.Verify)
+	engine := bm.DefaultServer(c.BM)
+	route(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("bm Start error(%v)", err)
+		panic(err)
+	}
+}
+
+func route(e *bm.Engine) {
+	e.Ping(ping)
+	e.Register(register)
+	g := e.Group("/x/recommend")
+	{
+		g.GET("/start", vfy.Verify, howToStart)
+	}
+	v1.RegisterV1RecommendService(e, v12.NewRecommendService(conf.Conf), nil)
+}
+
+func ping(ctx *bm.Context) {
+	if err := svc.Ping(ctx); err != nil {
+		log.Error("ping error(%+v)", err)
+		ctx.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}
+
+// example for http request handler
+func howToStart(c *bm.Context) {
+	c.String(0, "Golang 大法好 !!!")
+}

app/service/live/recommend/internal/service/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["service.go"],
+    importpath = "go-common/app/service/live/recommend/internal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//app/service/live/recommend/internal/dao:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/live/recommend/internal/service/v1:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/internal/service/service.go

@@ -0,0 +1,33 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/service/live/recommend/internal/conf"
+	"go-common/app/service/live/recommend/internal/dao"
+)
+
+// Service struct
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:   c,
+		dao: dao.New(c),
+	}
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(ctx context.Context) (err error) {
+	return s.dao.Ping(ctx)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/service/live/recommend/internal/service/v1/BUILD

@@ -0,0 +1,47 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["recommend_test.go"],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/live/recommend/api/grpc/v1:go_default_library",
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["recommend.go"],
+    importpath = "go-common/app/service/live/recommend/internal/service/v1",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/recommend/api/grpc/v1:go_default_library",
+        "//app/service/live/recommend/internal/conf:go_default_library",
+        "//app/service/live/recommend/internal/dao:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/internal/service/v1/recommend.go

@@ -0,0 +1,50 @@
+package v1
+
+import (
+	"context"
+
+	v1pb "go-common/app/service/live/recommend/api/grpc/v1"
+	"go-common/app/service/live/recommend/internal/conf"
+	"go-common/app/service/live/recommend/internal/dao"
+	"go-common/library/log"
+)
+
+// RecommendService struct
+type RecommendService struct {
+	conf *conf.Config
+	// optionally add other properties here, such as dao
+	dao *dao.Dao
+}
+
+//NewRecommendService init
+func NewRecommendService(c *conf.Config) (s *RecommendService) {
+	s = &RecommendService{
+		conf: c,
+		dao:  dao.New(c),
+	}
+	return s
+}
+
+// RandomRecsByUser implementation
+// 随机获取n个推荐
+func (s *RecommendService) RandomRecsByUser(ctx context.Context, req *v1pb.GetRandomRecReq) (resp *v1pb.GetRandomRecResp, err error) {
+	resp = &v1pb.GetRandomRecResp{}
+
+	ids, err := s.dao.GetRandomRoomIds(ctx, req.Uid, int(req.Count), req.ExistIds)
+	if err != nil {
+		log.Error("RandomRecsByUser err: %+v req:%+v", err, req)
+		return
+	}
+	resp.RoomIds = ids
+	resp.Count = uint32(len(ids))
+	log.Info("RandomRecsByUser uid:%d, reqCount:%d count:%d roomIds: %v", req.Uid, req.Count, resp.Count, resp.RoomIds)
+	return
+}
+
+// ClearRecommendCache implementation
+// 清空推荐缓存，清空推荐过的集合
+func (s *RecommendService) ClearRecommendCache(ctx context.Context, req *v1pb.ClearRecommendRequest) (resp *v1pb.ClearRecommendResponse, err error) {
+	resp = &v1pb.ClearRecommendResponse{}
+	err = s.dao.ClearRecommend(ctx, req.Uid)
+	return
+}

app/service/live/recommend/internal/service/v1/recommend_test.go

@@ -0,0 +1,37 @@
+package v1
+
+import (
+	"context"
+	"flag"
+	"testing"
+
+	"go-common/app/service/live/recommend/api/grpc/v1"
+	"go-common/app/service/live/recommend/internal/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *RecommendService
+)
+
+func init() {
+	flag.Set("conf", "../../../cmd/test.toml")
+	var err error
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+	s = NewRecommendService(conf.Conf)
+}
+
+// go test  -test.v -test.run TestRecommend_RandomRecsByUser
+func TestRecommend_RandomRecsByUser(t *testing.T) {
+	Convey("TestRecommend_RandomRecsByUser", t, func() {
+		res, err := s.RandomRecsByUser(context.TODO(), &v1.GetRandomRecReq{Uid: 4158272, Count: 3})
+		t.Logf("%v msg", res)
+		if err != nil {
+			t.Logf("err=%+v", err)
+		}
+		So(err, ShouldBeNil)
+	})
+}

app/service/live/recommend/recconst/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["redis_keys.go"],
+    importpath = "go-common/app/service/live/recommend/recconst",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/recommend/recconst/redis_keys.go

@@ -0,0 +1,7 @@
+package recconst
+
+// UserItemCFRecKey 协同过滤推荐key zset
+var UserItemCFRecKey = "user_item_rec_%d"
+
+// UserAreaKey 保存用户的兴趣分区 多个分区;分隔
+var UserAreaKey = "user_area_%d"