Create & Init Project...

app/job/main/reply-feed/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/reply-feed/cmd:all-srcs",
+        "//app/job/main/reply-feed/conf:all-srcs",
+        "//app/job/main/reply-feed/dao:all-srcs",
+        "//app/job/main/reply-feed/model:all-srcs",
+        "//app/job/main/reply-feed/server/http:all-srcs",
+        "//app/job/main/reply-feed/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/reply-feed/CHANGELOG.md

@@ -0,0 +1,57 @@
+
+# v1.1.4
+1. 修复attr为uint32
+
+# v1.1.3
+1. optimize smembers and get multi
+
+# v1.1.2
+1. fix hour = 23
+
+# v1.1.1
+1. 去重逻辑
+
+# v1.1.0
+1. 增加uv
+
+# v1.0.13
+1. 优化algorithm判断
+
+# v1.0.12
+1. 减少limit
+
+# v1.0.11
+1. 新增两组算法, order by like和线上默认逻辑
+
+# v1.0.10
+1. 优化order by like查询
+
+# v1.0.9
+1. 去掉20和3的限制
+
+# v1.0.8
+1. 优化in查询
+
+# v1.0.7
+1. 优化sql
+
+# v1.0.6
+1. redis zadd batch
+
+# v1.0.5
+1. 缓存异步操作
+
+# v1.0.4
+1. 开启reply flow,新增top和untop
+
+# v1.0.3
+1. 增加热门评论的统计
+
+# v1.0.2
+1. --操作前进行判断
+
+# v1.0.1
+1. stat consume
+
+# v1.0.0
+1. init

app/job/main/reply-feed/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+caoguoliang
+yangjiankun
+
+# Author
+caoguoliang
+yangjiankun
+
+# Reviewer
+caoguoliang
+yangjiankun

app/job/main/reply-feed/OWNERS

@@ -0,0 +1,14 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- caoguoliang
+- yangjiankun
+labels:
+- job
+- job/main/reply-feed
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- caoguoliang
+- yangjiankun

app/job/main/reply-feed/README.md

@@ -0,0 +1,12 @@
+# reply-feed-job
+
+# 项目简介
+1.
+
+# 编译环境
+
+
+# 依赖包
+
+
+# 编译执行

app/job/main/reply-feed/api/http/api.md

@@ -0,0 +1 @@
+# HTTP API文档

app/job/main/reply-feed/cmd/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["test.toml"],
+    importpath = "go-common/app/job/main/reply-feed/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/reply-feed/conf:go_default_library",
+        "//app/job/main/reply-feed/server/http:go_default_library",
+        "//app/job/main/reply-feed/service:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/reply-feed/cmd/main.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/main/reply-feed/conf"
+	"go-common/app/job/main/reply-feed/server/http"
+	"go-common/app/job/main/reply-feed/service"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Warn("reply-feed job start")
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	ecode.Init(conf.Conf.Ecode)
+	svc := service.New(conf.Conf)
+	http.Init(conf.Conf, svc)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Warn("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			svc.Close()
+			log.Warn("reply-feed job exit")
+			time.Sleep(time.Second * 3)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/reply-feed/cmd/test.toml

@@ -0,0 +1,118 @@
+refreshTime = 30
+
+[log]
+stdout=true
+
+[httpClient]
+key = "c1a1cb2d89c33794"
+secret = "dda47eeca111e03e6845017505baea13"
+dial = "300ms"
+timeout = "800ms"
+keepAlive = "60s"
+timer = 512
+[httpClient.url]
+"https://api.bilibili.co/x/internal/v2/reply/ishot" = {timeout="200ms"}
+[httpClient.breaker]
+window  = "10s"
+sleep   = "50ms"
+bucket  = 10
+ratio   = 0.5
+request = 100
+
+[mysql]
+    [mysql.db]
+    addr = "172.22.34.101:3306"
+    dsn = "test_3306:UJPZaGKjpb2ylFx3HNhmLuwOYft4MCAi@tcp(172.22.34.101:3306)/bilibili_reply?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8"
+    active = 10
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "500ms"
+    execTimeout = "500ms"
+    tranTimeout = "500ms"
+    [mysql.db.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+    [mysql.dbSlave]
+    addr = "172.22.34.101:3306"
+    dsn = "test_3306:UJPZaGKjpb2ylFx3HNhmLuwOYft4MCAi@tcp(172.22.34.101:3306)/bilibili_reply?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8"
+    active = 10
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "500ms"
+    execTimeout = "500ms"
+    tranTimeout = "500ms"
+    [mysql.dbSlave.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+	name = "reply-feed-job"
+	proto = "tcp"
+	addr = "172.18.33.60:6889"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[redisExpire]
+    redisReplySetExpire = "12h"
+    redisReplyZSetExpire = "12h"
+    redisRefreshExpire = "12h"
+
+[memcache]
+	name = "reply-feed-job"
+	proto = "tcp"
+	addr = "172.18.33.61:11213"
+	active = 50
+	idle = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "24h"
+
+ [memcacheExpire]
+    mcStatExpire = "12h"
+
+[databus]
+    [databus.event]
+    key = "170e302355453683"
+    secret = "3d0e8db7bed0503949e545a469789279"
+    group = "ReplyFeed-MainCommunity-S"
+    topic = "ReplyFeed-T"
+    action ="sub"
+    name = "reply-feed/event"
+    proto = "tcp"
+    addr = "172.18.33.50:6205"
+    idle = 2
+    active = 5
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+    expire = "1h"
+    [databus.stats]
+    key = "170e302355453683"
+    secret = "3d0e8db7bed0503949e545a469789279"
+    group = "Reply-MainCommunity-S"
+    topic = "Reply-T"
+    action ="sub"
+    name = "reply/stats"
+    proto = "tcp"
+    addr = "172.18.33.50:6205"
+    idle = 2
+    active = 5
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+    expire = "1h"

app/job/main/reply-feed/conf/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/reply-feed/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/reply-feed/conf/conf.go

@@ -0,0 +1,125 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config .
+type Config struct {
+	Log            *log.Config
+	BM             *bm.ServerConfig
+	Verify         *verify.Config
+	Tracer         *trace.Config
+	Redis          *redis.Config
+	RedisExpire    *RedisExpire
+	Memcache       *memcache.Config
+	MemcacheExpire *MemcacheExpire
+	MySQL          *MySQL
+	Databus        *Databus
+	Ecode          *ecode.Config
+	RefreshTime    int64
+	HTTPClient     *bm.ClientConfig
+}
+
+// RedisExpire Redis
+type RedisExpire struct {
+	RedisReplySetExpire  xtime.Duration
+	RedisReplyZSetExpire xtime.Duration
+	RedisRefreshExpire   xtime.Duration
+}
+
+// MemcacheExpire Memcache
+type MemcacheExpire struct {
+	McStatExpire xtime.Duration
+}
+
+// Databus databus
+type Databus struct {
+	Stats *databus.Config
+	Event *databus.Config
+}
+
+// MySQL mysql config
+type MySQL struct {
+	DB      *sql.Config
+	DBSlave *sql.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() (err error) {
+	if confPath != "" {
+		return local()
+	}
+	if err = remote(); err != nil {
+		return
+	}
+	if Conf.RefreshTime <= 0 {
+		panic("refresh time illegal.")
+	}
+	return
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/reply-feed/dao/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "db.go",
+        "http_client.go",
+        "mc.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/reply-feed/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/reply-feed/conf:go_default_library",
+        "//app/job/main/reply-feed/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/time:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "db_test.go",
+        "http_client_test.go",
+        "mc_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/reply-feed/conf:go_default_library",
+        "//app/job/main/reply-feed/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/reply-feed/dao/dao.go

@@ -0,0 +1,64 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/reply-feed/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao dao
+type Dao struct {
+	c        *conf.Config
+	mc       *memcache.Pool
+	mcExpire int32
+
+	redis                *redis.Pool
+	redisReplySetExpire  int
+	redisReplyZSetExpire int
+	redisRefreshExpire   int
+	db                   *xsql.DB
+	dbSlave              *xsql.DB
+
+	httpCli *bm.Client
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:                    c,
+		mc:                   memcache.NewPool(c.Memcache),
+		mcExpire:             int32(time.Duration(c.MemcacheExpire.McStatExpire) / time.Second),
+		redis:                redis.NewPool(c.Redis),
+		redisReplySetExpire:  int(time.Duration(c.RedisExpire.RedisReplySetExpire) / time.Second),
+		redisReplyZSetExpire: int(time.Duration(c.RedisExpire.RedisReplyZSetExpire) / time.Second),
+		redisRefreshExpire:   int(time.Duration(c.RedisExpire.RedisRefreshExpire) / time.Second),
+		db:                   xsql.NewMySQL(c.MySQL.DB),
+		dbSlave:              xsql.NewMySQL(c.MySQL.DBSlave),
+		httpCli:              bm.NewClient(c.HTTPClient),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.mc.Close()
+	d.redis.Close()
+	d.dbSlave.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	if err := d.PingRedis(c); err != nil {
+		return err
+	}
+	if err := d.PingMc(c); err != nil {
+		return err
+	}
+	return d.db.Ping(c)
+}

app/job/main/reply-feed/dao/dao_test.go

@@ -0,0 +1,35 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/job/main/reply-feed/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.community.reply-feed-job")
+		flag.Set("conf_token", "bbc0aa3966a0d3ba92a533fd3dd8f704")
+		flag.Set("tree_id", "71051")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/reply-feed/dao/db.go

@@ -0,0 +1,307 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	xtime "go-common/library/time"
+	"go-common/library/xstr"
+)
+
+const (
+	_chunkedSize       = 200
+	_getReplyStatsByID = "SELECT id, `like`, hate, rcount, ctime FROM reply_%d WHERE id IN (%s)"
+	_getReplyStats     = "SELECT id, `like`, hate, rcount, ctime FROM reply_%d WHERE oid=? AND type=? AND root=0 AND state in (0,1,2,5,6) ORDER BY `like` DESC LIMIT 2000"
+	_getReplyReport    = "SELECT rpid, count FROM reply_report_%d WHERE rpid IN (%s)"
+	_getSubjectStat    = "SELECT ctime from reply_subject_%d where oid=? and type=?"
+	_getRpID           = "SELECT id FROM reply_%d WHERE oid=? AND type=? AND root=0 AND state in (0,1,2,5,6) ORDER BY `like` DESC LIMIT 2000"
+
+	_getSlotStats = "SELECT slot, name, algorithm, weight FROM reply_abtest_strategy"
+
+	_getSlotsMapping = "SELECT name, slot FROM reply_abtest_strategy"
+
+	_upsertStatisticsT = "INSERT INTO reply_abtest_statistics (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s"
+)
+
+var (
+	_upsertStatistics = genSQL()
+)
+
+func genSQL() string {
+	var (
+		slot1 []string
+		slot2 []string
+		slot3 []string
+	)
+	slot1 = append(slot1, model.StatisticsDatabaseI...)
+	slot1 = append(slot1, model.StatisticsDatabaseU...)
+	slot1 = append(slot1, model.StatisticsDatabaseS...)
+	for range model.StatisticsDatabaseI {
+		slot2 = append(slot2, "?")
+	}
+	for _, c := range model.StatisticsDatabaseU {
+		slot2 = append(slot2, "?")
+		slot3 = append(slot3, c+"="+c+"+?")
+	}
+	for _, c := range model.StatisticsDatabaseS {
+		slot2 = append(slot2, "?")
+		slot3 = append(slot3, c+"="+"?")
+	}
+	return fmt.Sprintf(_upsertStatisticsT, strings.Join(slot1, ","), strings.Join(slot2, ","), strings.Join(slot3, ","))
+}
+
+func reportHit(oid int64) int64 {
+	return oid % 200
+}
+
+func replyHit(oid int64) int64 {
+	return oid % 200
+}
+
+func subjectHit(oid int64) int64 {
+	return oid % 50
+}
+
+func splitReplyScore(buf []*model.ReplyScore, limit int) [][]*model.ReplyScore {
+	var chunk []*model.ReplyScore
+	chunks := make([][]*model.ReplyScore, 0, len(buf)/limit+1)
+	for len(buf) >= limit {
+		chunk, buf = buf[:limit], buf[limit:]
+		chunks = append(chunks, chunk)
+	}
+	if len(buf) > 0 {
+		chunks = append(chunks, buf)
+	}
+	return chunks
+}
+
+func splitString(buf []string, limit int) [][]string {
+	var chunk []string
+	chunks := make([][]string, 0, len(buf)/limit+1)
+	for len(buf) >= limit {
+		chunk, buf = buf[:limit], buf[limit:]
+		chunks = append(chunks, chunk)
+	}
+	if len(buf) > 0 {
+		chunks = append(chunks, buf)
+	}
+	return chunks
+}
+
+func split(buf []int64, limit int) [][]int64 {
+	var chunk []int64
+	chunks := make([][]int64, 0, len(buf)/limit+1)
+	for len(buf) >= limit {
+		chunk, buf = buf[:limit], buf[limit:]
+		chunks = append(chunks, chunk)
+	}
+	if len(buf) > 0 {
+		chunks = append(chunks, buf)
+	}
+	return chunks
+}
+
+// SlotStats get slot stat
+func (d *Dao) SlotStats(ctx context.Context) (ss []*model.SlotStat, err error) {
+	rows, err := d.db.Query(ctx, _getSlotStats)
+	if err != nil {
+		log.Error("db.Query(%s) error(%v)", _getSlotStats, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		s := new(model.SlotStat)
+		if err = rows.Scan(&s.Slot, &s.Name, &s.Algorithm, &s.Weight); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		ss = append(ss, s)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// RpIDs return rpIDs should in hot reply list.
+func (d *Dao) RpIDs(ctx context.Context, oid int64, tp int) (rpIDs []int64, err error) {
+	query := fmt.Sprintf(_getRpID, replyHit(oid))
+	rows, err := d.dbSlave.Query(ctx, query, oid, tp)
+	if err != nil {
+		log.Error("db.Query(%s) error(%v)", query, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var ID int64
+		if err = rows.Scan(&ID); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		rpIDs = append(rpIDs, ID)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// ReportStatsByID get report stats from database by ID
+func (d *Dao) ReportStatsByID(ctx context.Context, oid int64, rpIDs []int64) (reportMap map[int64]*model.ReplyStat, err error) {
+	reportMap = make(map[int64]*model.ReplyStat)
+	chunkedRpIDs := split(rpIDs, _chunkedSize)
+	for _, ids := range chunkedRpIDs {
+		var (
+			query = fmt.Sprintf(_getReplyReport, reportHit(oid), xstr.JoinInts(ids))
+			rows  *sql.Rows
+		)
+		rows, err = d.dbSlave.Query(ctx, query)
+		if err != nil {
+			log.Error("db.Query(%s) error(%v)", query, err)
+			return
+		}
+		for rows.Next() {
+			var stat = new(model.ReplyStat)
+			if err = rows.Scan(&stat.RpID, &stat.Report); err != nil {
+				log.Error("rows.Scan() error(%v)", err)
+				return
+			}
+			reportMap[stat.RpID] = stat
+		}
+		if err = rows.Err(); err != nil {
+			rows.Close()
+			log.Error("rows.Err() error(%v)", err)
+			return
+		}
+		rows.Close()
+	}
+	return
+}
+
+// ReplyLHRCStatsByID return a reply like hate reply ctime stat by rpid.
+func (d *Dao) ReplyLHRCStatsByID(ctx context.Context, oid int64, rpIDs []int64) (replyMap map[int64]*model.ReplyStat, err error) {
+	replyMap = make(map[int64]*model.ReplyStat)
+	chunkedRpIDs := split(rpIDs, _chunkedSize)
+	for _, ids := range chunkedRpIDs {
+		var (
+			query = fmt.Sprintf(_getReplyStatsByID, replyHit(oid), xstr.JoinInts(ids))
+			rows  *sql.Rows
+		)
+		rows, err = d.dbSlave.Query(ctx, query)
+		if err != nil {
+			log.Error("db.Query(%s) error(%v)", query, err)
+			return
+		}
+		for rows.Next() {
+			var (
+				ctime xtime.Time
+				stat  = new(model.ReplyStat)
+			)
+			if err = rows.Scan(&stat.RpID, &stat.Like, &stat.Hate, &stat.Reply, &ctime); err != nil {
+				log.Error("rows.Scan() error(%v)", err)
+				return
+			}
+			stat.ReplyTime = ctime
+			replyMap[stat.RpID] = stat
+		}
+		if err = rows.Err(); err != nil {
+			rows.Close()
+			log.Error("rows.Err() error(%v)", err)
+			return
+		}
+		rows.Close()
+	}
+	return
+}
+
+// SubjectStats get subject ctime from database
+func (d *Dao) SubjectStats(ctx context.Context, oid int64, tp int) (ctime xtime.Time, err error) {
+	query := fmt.Sprintf(_getSubjectStat, subjectHit(oid))
+	if err = d.dbSlave.QueryRow(ctx, query, oid, tp).Scan(&ctime); err != nil {
+		log.Error("db.QueryRow(%s) args(%d, %d) error(%v)", query, oid, tp, err)
+		return
+	}
+	return
+}
+
+// ReplyLHRCStats get reply like, hate, reply, ctime stat from database, only get root reply which like>3, call it when back to source.
+func (d *Dao) ReplyLHRCStats(ctx context.Context, oid int64, tp int) (replyMap map[int64]*model.ReplyStat, err error) {
+	replyMap = make(map[int64]*model.ReplyStat)
+	query := fmt.Sprintf(_getReplyStats, replyHit(oid))
+	rows, err := d.dbSlave.Query(ctx, query, oid, tp)
+	if err != nil {
+		log.Error("db.Query(%s) args(%d, %d) error(%v)", query, oid, tp, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			ctime xtime.Time
+			stat  = new(model.ReplyStat)
+		)
+		if err = rows.Scan(&stat.RpID, &stat.Like, &stat.Hate, &stat.Reply, &ctime); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		stat.ReplyTime = ctime
+		replyMap[stat.RpID] = stat
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// SlotsMapping get slots and name mapping.
+func (d *Dao) SlotsMapping(ctx context.Context) (slotsMap map[string]*model.SlotsMapping, err error) {
+	slotsMap = make(map[string]*model.SlotsMapping)
+	rows, err := d.db.Query(ctx, _getSlotsMapping)
+	if err != nil {
+		log.Error("db.Query(%s) args(%s) error(%v)", _getSlotsMapping, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			name string
+			slot int
+		)
+		if err = rows.Scan(&name, &slot); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		slotsMapping, ok := slotsMap[name]
+		if ok {
+			slotsMapping.Slots = append(slotsMapping.Slots, slot)
+		} else {
+			slotsMapping = &model.SlotsMapping{
+				Name:  name,
+				Slots: []int{slot},
+			}
+		}
+		slotsMap[name] = slotsMapping
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// UpsertStatistics insert or update statistics into database
+func (d *Dao) UpsertStatistics(ctx context.Context, name string, date, hour int, s *model.StatisticsStat) (err error) {
+	if _, err = d.db.Exec(ctx, _upsertStatistics,
+		name, date, hour,
+		s.HotLike, s.HotHate, s.HotReport, s.HotChildReply, s.TotalLike, s.TotalHate, s.TotalReport, s.TotalRootReply, s.TotalChildReply,
+		s.HotLikeUV, s.HotHateUV, s.HotReportUV, s.HotChildUV, s.TotalLikeUV, s.TotalHateUV, s.TotalReportUV, s.TotalChildUV, s.TotalRootUV,
+		s.HotLike, s.HotHate, s.HotReport, s.HotChildReply, s.TotalLike, s.TotalHate, s.TotalReport, s.TotalRootReply, s.TotalChildReply,
+		s.HotLikeUV, s.HotHateUV, s.HotReportUV, s.HotChildUV, s.TotalLikeUV, s.TotalHateUV, s.TotalReportUV, s.TotalChildUV, s.TotalRootUV,
+	); err != nil {
+		log.Error("upsert statistics failed. error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/reply-feed/dao/db_test.go

@@ -0,0 +1,208 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/reply-feed/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestGenSQL(t *testing.T) {
+	convey.Convey("genSQL", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := genSQL()
+			t.Log(p1)
+		})
+	})
+}
+
+func TestDaoreportHit(t *testing.T) {
+	convey.Convey("reportHit", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := reportHit(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoreplyHit(t *testing.T) {
+	convey.Convey("replyHit", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := replyHit(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaosubjectHit(t *testing.T) {
+	convey.Convey("subjectHit", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := subjectHit(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaosplit(t *testing.T) {
+	convey.Convey("split", t, func(ctx convey.C) {
+		var (
+			s = []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := split(s, 5)
+			p2 := split(s, 10)
+			p3 := split(s, 20)
+			p4 := split(s, 3)
+			p5 := split(s, 1)
+			p6 := split(s, 2)
+			p7 := split(s, 4)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(len(p1), convey.ShouldEqual, 2)
+				ctx.So(len(p2), convey.ShouldEqual, 1)
+				ctx.So(len(p3), convey.ShouldEqual, 1)
+				ctx.So(len(p4), convey.ShouldEqual, 4)
+				ctx.So(len(p5), convey.ShouldEqual, 10)
+				ctx.So(len(p6), convey.ShouldEqual, 5)
+				ctx.So(len(p7), convey.ShouldEqual, 3)
+			})
+		})
+	})
+}
+
+func TestDaoSlotStats(t *testing.T) {
+	convey.Convey("SlotStats", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ss, err := d.SlotStats(context.Background())
+			ctx.Convey("Then err should be nil.ss should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ss, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRpIDs(t *testing.T) {
+	convey.Convey("RpIDs", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			rpIDs, err := d.RpIDs(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.rpIDs should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rpIDs, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoReportStatsByID(t *testing.T) {
+	convey.Convey("ReportStatsByID", t, func(ctx convey.C) {
+		var (
+			oid   = int64(0)
+			rpIDs = []int64{-1}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			reportMap, err := d.ReportStatsByID(context.Background(), oid, rpIDs)
+			ctx.Convey("Then err should be nil.reportMap should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(reportMap, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoReplyLHRCStatsByID(t *testing.T) {
+	convey.Convey("ReplyLHRCStatsByID", t, func(ctx convey.C) {
+		var (
+			oid   = int64(0)
+			rpIDs = []int64{-1}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			replyMap, err := d.ReplyLHRCStatsByID(context.Background(), oid, rpIDs)
+			ctx.Convey("Then err should be nil.replyMap should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(replyMap, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSubjectStats(t *testing.T) {
+	convey.Convey("SubjectStats", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ctime, err := d.SubjectStats(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.ctime should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ctime, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoReplyLHRCStats(t *testing.T) {
+	convey.Convey("ReplyLHRCStats", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			replyMap, err := d.ReplyLHRCStats(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.replyMap should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(replyMap, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSlotsMapping(t *testing.T) {
+	convey.Convey("SlotsMapping", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			slotsMap, err := d.SlotsMapping(context.Background())
+			ctx.Convey("Then err should be nil.slotsMap should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(slotsMap, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoUpsertStatistics(t *testing.T) {
+	convey.Convey("UpsertStatistics", t, func(ctx convey.C) {
+		var (
+			name = ""
+			date = int(0)
+			hour = int(0)
+			s    = &model.StatisticsStat{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.UpsertStatistics(context.Background(), name, date, hour, s)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/reply-feed/dao/http_client.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_isHotURL = "http://api.bilibili.co/x/internal/v2/reply/ishot"
+)
+
+// IsOriginHot return is origin hot reply.
+func (d *Dao) IsOriginHot(ctx context.Context, oid, rpID int64, tp int) (isHot bool, err error) {
+	params := url.Values{}
+	params.Set("oid", strconv.FormatInt(oid, 10))
+	params.Set("rpid", strconv.FormatInt(rpID, 10))
+	params.Set("type", strconv.Itoa(tp))
+	var res struct {
+		Code int `json:"code"`
+		Data *struct {
+			IsHot bool `json:"isHot"`
+		} `json:"data"`
+	}
+	if err = d.httpCli.Get(ctx, _isHotURL, "", params, &res); err != nil {
+		log.Error("d.httpCli.Get(%s, %s) error(%v)", _isHotURL, params.Encode(), err)
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		err = ecode.Int(res.Code)
+		return
+	}
+	if res.Data != nil {
+		isHot = res.Data.IsHot
+	}
+	return
+}

app/job/main/reply-feed/dao/http_client_test.go

@@ -0,0 +1,25 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoIsOriginHot(t *testing.T) {
+	convey.Convey("IsOriginHot", t, func(ctx convey.C) {
+		var (
+			oid  = int64(1)
+			rpID = int64(1)
+			tp   = int(1)
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			isHot, err := d.IsOriginHot(context.Background(), oid, rpID, tp)
+			ctx.Convey("Then err should be nil.isHot should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(isHot, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/reply-feed/dao/mc.go

@@ -0,0 +1,93 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_replyStatFormat = "rs_%d"
+)
+
+// PingMc ping
+func (d *Dao) PingMc(ctx context.Context) (err error) {
+	conn := d.mc.Get(ctx)
+	defer conn.Close()
+	item := memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}
+	return conn.Set(&item)
+}
+
+func keyReplyStat(rpID int64) string {
+	return fmt.Sprintf(_replyStatFormat, rpID)
+}
+
+// RemReplyStatMc ...
+func (d *Dao) RemReplyStatMc(ctx context.Context, rpID int64) (err error) {
+	conn := d.mc.Get(ctx)
+	defer conn.Close()
+	return conn.Delete(keyReplyStat(rpID))
+}
+
+// SetReplyStatMc set reply stat into mc.
+func (d *Dao) SetReplyStatMc(ctx context.Context, rs *model.ReplyStat) (err error) {
+	conn := d.mc.Get(ctx)
+	defer conn.Close()
+	key := keyReplyStat(rs.RpID)
+	item := &memcache.Item{
+		Key:        key,
+		Object:     rs,
+		Expiration: d.mcExpire,
+		Flags:      memcache.FlagJSON,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("memcache Set(%s, %v), error(%v)", key, item, err)
+	}
+	return
+}
+
+// ReplyStatsMc get multi repies stat from memcache.
+func (d *Dao) ReplyStatsMc(ctx context.Context, rpIDs []int64) (rsMap map[int64]*model.ReplyStat, missIDs []int64, err error) {
+	rsMap = make(map[int64]*model.ReplyStat)
+	keys := make([]string, len(rpIDs))
+	mapping := make(map[string]int64)
+	for i, rpID := range rpIDs {
+		key := keyReplyStat(rpID)
+		keys[i] = key
+		mapping[key] = rpID
+	}
+	for _, chunkedKeys := range splitString(keys, 2000) {
+		var (
+			conn  = d.mc.Get(ctx)
+			items map[string]*memcache.Item
+		)
+		if items, err = conn.GetMulti(chunkedKeys); err != nil {
+			if err == memcache.ErrNotFound {
+				missIDs = rpIDs
+				err = nil
+				conn.Close()
+				return
+			}
+			conn.Close()
+			log.Error("memcache GetMulti error(%v)", err)
+			return
+		}
+		for _, item := range items {
+			stat := new(model.ReplyStat)
+			if err = conn.Scan(item, stat); err != nil {
+				log.Error("memcache Scan(%v) error(%v)", item.Value, err)
+				continue
+			}
+			rsMap[mapping[item.Key]] = stat
+			delete(mapping, item.Key)
+		}
+		conn.Close()
+	}
+	for _, rpID := range mapping {
+		missIDs = append(missIDs, rpID)
+	}
+	return
+}

app/job/main/reply-feed/dao/mc_test.go

@@ -0,0 +1,88 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/reply-feed/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoPingMc(t *testing.T) {
+	convey.Convey("PingMc", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.PingMc(context.Background())
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaokeyReplyStat(t *testing.T) {
+	convey.Convey("keyReplyStat", t, func(ctx convey.C) {
+		var (
+			rpID = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyReplyStat(rpID)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSetReplyStatMc(t *testing.T) {
+	convey.Convey("SetReplyStatMc", t, func(ctx convey.C) {
+		var (
+			rs = &model.ReplyStat{RpID: 0}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.SetReplyStatMc(context.Background(), rs)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoReplyStatsMc(t *testing.T) {
+	convey.Convey("ReplyStatsMc", t, func(ctx convey.C) {
+		var (
+			rpIDs       = []int64{}
+			hitRpIDs    = []int64{}
+			missedRpIDs = []int64{}
+			c           = context.Background()
+		)
+		for i := 0; i < 5000; i++ {
+			d.RemReplyStatMc(c, int64(i))
+		}
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			for i := 0; i < 5000; i++ {
+				if i%2 == 0 {
+					err := d.SetReplyStatMc(c, &model.ReplyStat{RpID: int64(i)})
+					if err != nil {
+						t.Fatal(err)
+					}
+					hitRpIDs = append(hitRpIDs, int64(i))
+				} else {
+					missedRpIDs = append(missedRpIDs, int64(i))
+				}
+				rpIDs = append(rpIDs, int64(i))
+			}
+			rsMap, missIDs, err := d.ReplyStatsMc(c, rpIDs)
+			ctx.Convey("Then err should be nil.rsMap,missIDs should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(len(rsMap), convey.ShouldEqual, len(hitRpIDs))
+				ctx.So(len(missIDs), convey.ShouldEqual, len(missedRpIDs))
+			})
+			for _, rpID := range hitRpIDs {
+				if err = d.RemReplyStatMc(c, rpID); err != nil {
+					t.Fatal(err)
+				}
+			}
+		})
+	})
+}

app/job/main/reply-feed/dao/redis.go

@@ -0,0 +1,427 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	// r_<实验组名>_<oid>_<type>
+	// 用redis ZSet存储热门评论列表，score为热评分数，member为rpID
+	_replyZSetFormat = "r_%s_%d_%d"
+
+	// c_<oid>_<type>
+	_refreshCheckerFormat = "c_%d_%d"
+
+	// h_<oid>_<type>
+	// 用一个set来存某一个评论区下的应该存在于热门评论列表的评论ID
+	// 上热评的最低门槛，点赞数大于3，且该评论区根评论数目大于20
+	_replyListFormat = "h_%d_%d"
+
+	// 行为, 小时，slot, 种类(全量或者针对热评)
+	_uvFormat = "uv_%s_%d_%d_%s"
+
+	_uvExp = 3600
+)
+
+func keyRefreshChecker(oid int64, tp int) string {
+	return fmt.Sprintf(_refreshCheckerFormat, oid, tp)
+}
+
+func keyReplyZSet(name string, oid int64, tp int) string {
+	return fmt.Sprintf(_replyZSetFormat, name, oid, tp)
+}
+
+func keyReplySet(oid int64, tp int) string {
+	return fmt.Sprintf(_replyListFormat, oid, tp)
+}
+
+// KeyUV ...
+func (d *Dao) KeyUV(action string, hour, slot int, kind string) string {
+	return keyUV(action, hour, slot, kind)
+}
+
+func keyUV(action string, hour, slot int, kind string) string {
+	return fmt.Sprintf(_uvFormat, action, hour, slot, kind)
+}
+
+// PingRedis redis health check.
+func (d *Dao) PingRedis(ctx context.Context) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	_, err = conn.Do("SET", "ping", "pong")
+	return
+}
+
+// AddUV ...
+func (d *Dao) AddUV(ctx context.Context, action string, hour, slot int, mid int64, kind string) (err error) {
+	key := keyUV(action, hour, slot, kind)
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	if err = conn.Send("SADD", key, mid); err != nil {
+		log.Error("redis SADD(%s, %d) error(%v)", key, mid, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, _uvExp); err != nil {
+		log.Error("redis EXPIRE(%s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("redis Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("redis Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// CountUV ...
+func (d *Dao) CountUV(ctx context.Context, keys []string) (counts []int64, err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	j := 0
+	for _, key := range keys {
+		if err = conn.Send("SCARD", key); err != nil {
+			log.Error("redis SCARD(%s) error(%v)", key, err)
+			return
+		}
+		j++
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("redis Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < j; i++ {
+		var count int64
+		if count, err = redis.Int64(conn.Receive()); err != nil && err != redis.ErrNil {
+			log.Error("redis Receive() error(%v)", err)
+			return
+		}
+		counts = append(counts, count)
+	}
+	return
+}
+
+// ExpireCheckerRds expire checker.
+func (d *Dao) ExpireCheckerRds(ctx context.Context, oid int64, tp int) (ok bool, err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyRefreshChecker(oid, tp)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.redisRefreshExpire)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis EXPIRE key(%s) error(%v)", key, err)
+	}
+	return
+}
+
+// ExpireReplyZSetRds expire reply list.
+func (d *Dao) ExpireReplyZSetRds(ctx context.Context, name string, oid int64, tp int) (ok bool, err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplyZSet(name, oid, tp)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.redisReplyZSetExpire)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis EXPIRE key(%s) error(%v)", key, err)
+	}
+	return
+}
+
+// ExpireReplySetRds expire reply set.
+func (d *Dao) ExpireReplySetRds(ctx context.Context, oid int64, tp int) (ok bool, err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplySet(oid, tp)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.redisReplySetExpire)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis EXPIRE key(%s) error(%v)", key, err)
+	}
+	return
+}
+
+// ReplySetRds get reply rpIDs from redis set.
+func (d *Dao) ReplySetRds(ctx context.Context, oid int64, tp int) (rpIDs []int64, err error) {
+	key := keyReplySet(oid, tp)
+	var startCursor, endCursor int64
+	endCursor = 1
+	for endCursor != 0 {
+		var (
+			conn         = d.redis.Get(ctx)
+			chunkedRpIDs []int64
+			values       []interface{}
+		)
+		if values, err = redis.Values(conn.Do("SSCAN", key, startCursor)); err != nil {
+			log.Error("redis SSCAN(%s) error(%v)", key, err)
+			conn.Close()
+			return
+		}
+		if _, err = redis.Scan(values, &endCursor, &chunkedRpIDs); err != nil {
+			log.Error("redis Scan(%v) error(%v)", values, err)
+			conn.Close()
+			return
+		}
+		startCursor = endCursor
+		rpIDs = append(rpIDs, chunkedRpIDs...)
+		conn.Close()
+	}
+	return
+}
+
+// SetReplySetRds set reply list batch, call it when back to source.
+func (d *Dao) SetReplySetRds(ctx context.Context, oid int64, tp int, rpIDs []int64) (err error) {
+	if len(rpIDs) < 1 {
+		return
+	}
+	for _, chunkedRpIDs := range split(rpIDs, 5000) {
+		var (
+			key  = keyReplySet(oid, tp)
+			args = make([]interface{}, 0, len(chunkedRpIDs)+1)
+			conn = d.redis.Get(ctx)
+		)
+		args = append(args, key)
+		for _, rpID := range chunkedRpIDs {
+			args = append(args, rpID)
+		}
+		if err = conn.Send("SADD", args...); err != nil {
+			log.Error("redis SADD(%v) error(%v)", args, err)
+			conn.Close()
+			return
+		}
+		if err = conn.Send("EXPIRE", key, d.redisReplySetExpire); err != nil {
+			log.Error("redis EXPIRE(%s) error(%v)", key, err)
+			conn.Close()
+			return
+		}
+		if err = conn.Flush(); err != nil {
+			log.Error("redis Flush() error(%v)", err)
+			conn.Close()
+			return
+		}
+		for i := 0; i < 2; i++ {
+			if _, err = conn.Receive(); err != nil {
+				log.Error("redis Receive() error(%v)", err)
+				conn.Close()
+				return
+			}
+		}
+		conn.Close()
+	}
+	return
+}
+
+// RemReplySetRds remove one rp from set.
+func (d *Dao) RemReplySetRds(ctx context.Context, oid, rpID int64, tp int) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplySet(oid, tp)
+	if _, err = redis.Int64(conn.Do("SREM", key, rpID)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("SREM conn.Do(%s,%d) err(%v)", key, rpID, err)
+	}
+	return
+}
+
+// DelReplySetRds delete a set key.
+func (d *Dao) DelReplySetRds(ctx context.Context, oid int64, tp int) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplySet(oid, tp)
+	if _, err = conn.Do("DEL", key); err != nil {
+		log.Error("conn.Do(DEL %s) error(%v)", key, err)
+	}
+	return
+}
+
+// AddReplySetRds add a reply into redis set, make sure expire the key first.
+func (d *Dao) AddReplySetRds(ctx context.Context, oid int64, tp int, rpID int64) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplySet(oid, tp)
+	if _, err = conn.Do("SADD", key, rpID); err != nil {
+		log.Error("redis SADD(%s, %d) error(%v)", key, rpID, err)
+	}
+	return
+}
+
+// ReplyZSetRds get reply list from redis sorted set.
+func (d *Dao) ReplyZSetRds(ctx context.Context, name string, oid int64, tp, start, end int) (rpIDs []int64, err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplyZSet(name, oid, tp)
+	values, err := redis.Values(conn.Do("ZREVRANGE", key, start, end))
+	if err != nil {
+		log.Error("redis ZREVRANGE(%s, %d, %d) error(%v)", key, start, end, err)
+		return
+	}
+	if err = redis.ScanSlice(values, &rpIDs); err != nil {
+		log.Error("redis ScanSlice(%v) error(%v)", values, err)
+	}
+	return
+}
+
+// SetReplyZSetRds set reply list batch, call it when back to source.
+func (d *Dao) SetReplyZSetRds(ctx context.Context, name string, oid int64, tp int, rs []*model.ReplyScore) (err error) {
+	if len(rs) < 1 {
+		return
+	}
+	for _, chunkedReplyStats := range splitReplyScore(rs, 5000) {
+		var (
+			count = 0
+			key   = keyReplyZSet(name, oid, tp)
+			conn  = d.redis.Get(ctx)
+			args  = make([]interface{}, 0, len(chunkedReplyStats)*2+1)
+		)
+		args = append(args, key)
+		for _, s := range chunkedReplyStats {
+			args = append(args, s.Score)
+			args = append(args, s.RpID)
+		}
+		if err = conn.Send("ZADD", args...); err != nil {
+			log.Error("redis ZADD(%s, %v) error(%v)", key, args, err)
+			conn.Close()
+			return
+		}
+		count++
+		if err = conn.Send("EXPIRE", key, d.redisReplyZSetExpire); err != nil {
+			log.Error("redis EXPIRE(%s) error(%v)", key, err)
+			conn.Close()
+			return
+		}
+		count++
+		if err = conn.Flush(); err != nil {
+			log.Error("redis Flush error(%v)", err)
+			conn.Close()
+			return
+		}
+		for i := 0; i < count; i++ {
+			if _, err = conn.Receive(); err != nil {
+				log.Error("redis Receive (key: %s, %f, %d) error(%v)", key, rs[i].Score, rs[i].RpID, err)
+				conn.Close()
+				return
+			}
+		}
+		conn.Close()
+	}
+	return
+}
+
+// RangeReplyZSetRds ...
+func (d *Dao) RangeReplyZSetRds(ctx context.Context, name string, oid int64, tp, start, end int) (rpIDs []int64, err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplyZSet(name, oid, tp)
+	values, err := redis.Values(conn.Do("ZREVRANGE", key, start, end))
+	if err != nil {
+		log.Error("conn.Do(ZREVRANGE, %s) error(%v)", key, err)
+		return
+	}
+	if len(values) == 0 {
+		return
+	}
+	if err = redis.ScanSlice(values, &rpIDs); err != nil {
+		log.Error("redis.ScanSlice(%v) error(%v)", values, err)
+		return
+	}
+	return
+}
+
+// AddReplyZSetRds add a reply into redis sorted set, make sure expire the key first.
+func (d *Dao) AddReplyZSetRds(ctx context.Context, name string, oid int64, tp int, rs *model.ReplyScore) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplyZSet(name, oid, tp)
+	if _, err = conn.Do("ZADD", key, rs.Score, rs.RpID); err != nil {
+		log.Error("redis ZADD(%s, %f, %d) error(%v)", key, rs.Score, rs.RpID, err)
+	}
+	return
+}
+
+// RemReplyZSetRds remove one rpID from reply ZSet.
+func (d *Dao) RemReplyZSetRds(ctx context.Context, name string, oid int64, tp int, rpID int64) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyReplyZSet(name, oid, tp)
+	if _, err = conn.Do("ZREM", key, rpID); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis ZREM(%s, %d) error(%v)", key, rpID, err)
+	}
+	return
+}
+
+// DelReplyZSetRds del a key from reply ZSet.
+func (d *Dao) DelReplyZSetRds(ctx context.Context, names []string, oid int64, tp int) (err error) {
+	if len(names) < 1 {
+		return
+	}
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	count := 0
+	for _, name := range names {
+		key := keyReplyZSet(name, oid, tp)
+		if err = conn.Send("DEL", key); err != nil {
+			log.Error("conn.Do(DEL %s) error(%v)", key, err)
+			return
+		}
+		count++
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("redis Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < count; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("redis Receive error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// CheckerTsRds get refresh checker timestamp from redis, if time.Now()-ts > strategy.time, then refresh reply list.
+func (d *Dao) CheckerTsRds(ctx context.Context, oid int64, tp int) (ts int64, err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyRefreshChecker(oid, tp)
+	if ts, err = redis.Int64(conn.Do("GET", key)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis GET(%s) error(%v)", key, err)
+	}
+	return
+}
+
+// SetCheckerTsRds set refresh checker's timestamp as time.Now(), call it when refresh reply list.
+func (d *Dao) SetCheckerTsRds(ctx context.Context, oid int64, tp int) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	key := keyRefreshChecker(oid, tp)
+	if err = conn.Send("SETEX", key, d.redisRefreshExpire, time.Now().Unix()); err != nil {
+		log.Error("redis SETEX(%s) error(%v)", key, err)
+	}
+	return
+}

app/job/main/reply-feed/dao/redis_test.go

@@ -0,0 +1,401 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/reply-feed/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaokeyUV(t *testing.T) {
+	convey.Convey("keyUV", t, func(ctx convey.C) {
+		var (
+			action = ""
+			hour   = 0
+			slot   = 0
+			kind   = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyUV(action, hour, slot, kind)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoUV(t *testing.T) {
+	convey.Convey("addUV+countUV", t, func(ctx convey.C) {
+		var (
+			action = "test"
+			hour   = 0
+			slot   = 0
+			kind   = "test"
+			mid    = int64(0)
+			err    error
+			counts []int64
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err = d.AddUV(context.Background(), action, hour, slot, mid, kind)
+			ctx.So(err, convey.ShouldBeNil)
+			keys := []string{keyUV(action, hour, slot, kind)}
+			counts, err = d.CountUV(context.Background(), keys)
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(len(counts), convey.ShouldEqual, 1)
+		})
+	})
+}
+
+func TestDaokeyRefreshChecker(t *testing.T) {
+	convey.Convey("keyRefreshChecker", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyRefreshChecker(oid, tp)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaokeyReplyZSet(t *testing.T) {
+	convey.Convey("keyReplyZSet", t, func(ctx convey.C) {
+		var (
+			name = ""
+			oid  = int64(0)
+			tp   = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyReplyZSet(name, oid, tp)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaokeyReplySet(t *testing.T) {
+	convey.Convey("keyReplySet", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyReplySet(oid, tp)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoPingRedis(t *testing.T) {
+	convey.Convey("PingRedis", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.PingRedis(context.Background())
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoExpireCheckerRds(t *testing.T) {
+	convey.Convey("ExpireCheckerRds", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ok, err := d.ExpireCheckerRds(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ok, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoExpireReplyZSetRds(t *testing.T) {
+	convey.Convey("ExpireReplyZSetRds", t, func(ctx convey.C) {
+		var (
+			name = ""
+			oid  = int64(0)
+			tp   = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ok, err := d.ExpireReplyZSetRds(context.Background(), name, oid, tp)
+			ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ok, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoExpireReplySetRds(t *testing.T) {
+	convey.Convey("ExpireReplySetRds", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ok, err := d.ExpireReplySetRds(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ok, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetReplySetRds(t *testing.T) {
+	convey.Convey("ReplySetRds", t, func(ctx convey.C) {
+		var (
+			oid   = int64(-1)
+			tp    = int(-1)
+			idMap = make(map[int64]struct{})
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			for i := 0; i < 10; i++ {
+				if err := d.AddReplySetRds(context.Background(), oid, tp, int64(i)); err != nil {
+					return
+				}
+				idMap[int64(i)] = struct{}{}
+			}
+			d.ExpireReplySetRds(context.Background(), oid, tp)
+			rpIDs, err := d.ReplySetRds(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.rpIDs should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(len(rpIDs), convey.ShouldEqual, 10)
+				for _, rpID := range rpIDs {
+					if _, ok := idMap[rpID]; !ok {
+						t.Fatal("id not match")
+					}
+				}
+			})
+		})
+	})
+}
+
+func TestDaoSetReplySetRds(t *testing.T) {
+	convey.Convey("SetReplySetRds", t, func(ctx convey.C) {
+		var (
+			oid   = int64(0)
+			tp    = int(0)
+			rpIDs = []int64{}
+			c     = context.Background()
+		)
+		for i := 0; i < 10000; i++ {
+			rpIDs = append(rpIDs, int64(i))
+		}
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.SetReplySetRds(c, oid, tp, rpIDs)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+			receivedRpIDs, err := d.ReplySetRds(c, oid, tp)
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx.So(len(receivedRpIDs), convey.ShouldEqual, len(rpIDs))
+		})
+		d.DelReplySetRds(c, oid, tp)
+	})
+}
+
+func TestDaoRemReplySetRds(t *testing.T) {
+	convey.Convey("RemReplySetRds", t, func(ctx convey.C) {
+		var (
+			oid  = int64(0)
+			rpID = int64(0)
+			tp   = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.RemReplySetRds(context.Background(), oid, rpID, tp)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelReplySetRds(t *testing.T) {
+	convey.Convey("DelReplySetRds", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.DelReplySetRds(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoAddReplySetRds(t *testing.T) {
+	convey.Convey("AddReplySetRds", t, func(ctx convey.C) {
+		var (
+			oid  = int64(0)
+			tp   = int(0)
+			rpID = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.AddReplySetRds(context.Background(), oid, tp, rpID)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetReplyZSetRds(t *testing.T) {
+	convey.Convey("ReplyZSetRds", t, func(ctx convey.C) {
+		var (
+			name  = ""
+			oid   = int64(0)
+			tp    = int(0)
+			start = int(0)
+			end   = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			rpIDs, err := d.ReplyZSetRds(context.Background(), name, oid, tp, start, end)
+			ctx.Convey("Then err should be nil.rpIDs should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rpIDs, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSetReplyZSetRds(t *testing.T) {
+	convey.Convey("SetReplyZSetRds", t, func(ctx convey.C) {
+		var (
+			name = "test"
+			oid  = int64(0)
+			tp   = int(0)
+			c    = context.Background()
+			rs   = []*model.ReplyScore{}
+		)
+		for i := 0; i < 10000; i++ {
+			rs = append(rs, &model.ReplyScore{RpID: int64(i), Score: float64(i)})
+		}
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.SetReplyZSetRds(c, name, oid, tp, rs)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+			rpIDs, err := d.ReplyZSetRds(c, name, oid, tp, 0, -1)
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx.So(len(rpIDs), convey.ShouldEqual, len(rs))
+		})
+		d.DelReplyZSetRds(c, []string{name}, oid, tp)
+	})
+}
+
+func TestDaoAddReplyZSetRds(t *testing.T) {
+	convey.Convey("AddReplyZSetRds", t, func(ctx convey.C) {
+		var (
+			name = ""
+			oid  = int64(0)
+			tp   = int(0)
+			rs   = &model.ReplyScore{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.AddReplyZSetRds(context.Background(), name, oid, tp, rs)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRemReplyZSetRds(t *testing.T) {
+	convey.Convey("RemReplyZSetRds", t, func(ctx convey.C) {
+		var (
+			name = ""
+			oid  = int64(0)
+			tp   = int(0)
+			rpID = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.RemReplyZSetRds(context.Background(), name, oid, tp, rpID)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelReplyZSetRds(t *testing.T) {
+	convey.Convey("DelReplyZSetRds", t, func(ctx convey.C) {
+		var (
+			names = []string{}
+			oid   = int64(0)
+			tp    = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.DelReplyZSetRds(context.Background(), names, oid, tp)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetCheckerTsRds(t *testing.T) {
+	convey.Convey("CheckerTsRds", t, func(ctx convey.C) {
+		var (
+			oid = int64(-1)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ts, err := d.CheckerTsRds(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.ts should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ts, convey.ShouldEqual, 0)
+			})
+		})
+	})
+}
+
+func TestDaoSetCheckerTsRds(t *testing.T) {
+	convey.Convey("SetCheckerTsRds", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.SetCheckerTsRds(context.Background(), oid, tp)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRangeReplyZSetRds(t *testing.T) {
+	convey.Convey("RangeReplyZSetRds", t, func(ctx convey.C) {
+		var (
+			oid  = int64(0)
+			tp   = int(0)
+			name = ""
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			rpIDs, err := d.RangeReplyZSetRds(context.Background(), name, oid, tp, 0, 0)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(len(rpIDs), convey.ShouldEqual, 0)
+			})
+		})
+	})
+}

app/job/main/reply-feed/model/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "algorithm.go",
+        "model.go",
+    ],
+    importpath = "go-common/app/job/main/reply-feed/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/reply-feed/model/algorithm.go

@@ -0,0 +1,197 @@
+package model
+
+import (
+	"math"
+	"time"
+)
+
+// Algorithm Algorithm
+type Algorithm interface {
+	Score(stat *ReplyStat) *ReplyScore
+	Slots() []int
+	Name() string
+}
+
+/*
+Origin Algorithm.
+*/
+
+// Origin origin algorithm
+type Origin struct {
+	name  string
+	slots []int
+}
+
+// NewOrigin NewLikeDesc Aogorithm
+func NewOrigin(name string, slots []int) *Origin {
+	return &Origin{
+		name:  name,
+		slots: slots,
+	}
+}
+
+// Name get name
+func (o *Origin) Name() string {
+	return o.name
+}
+
+// Slots get slots
+func (o *Origin) Slots() []int {
+	return o.slots
+}
+
+// Score calc score
+func (o *Origin) Score(stat *ReplyStat) (rs *ReplyScore) {
+	rs = new(ReplyScore)
+	rs.RpID = stat.RpID
+	if stat.Like < 0 || stat.Reply < 0 || stat.Report < 0 || stat.Hate < 0 {
+		return
+	}
+	score := int64(100 * ((stat.Like + 2) / (stat.Hate + 4 + stat.Report)))
+	score = score<<32 | (int64(stat.Reply) & 0xFFFFFFFF)
+	rs.Score = float64(score)
+	return
+}
+
+/*
+LikeDesc Algorithm order by like desc.
+*/
+
+// LikeDesc like desc
+type LikeDesc struct {
+	name  string
+	slots []int
+}
+
+// NewLikeDesc NewLikeDesc Aogorithm
+func NewLikeDesc(name string, slots []int) *LikeDesc {
+	return &LikeDesc{
+		name:  name,
+		slots: slots,
+	}
+}
+
+// Name get name
+func (l *LikeDesc) Name() string {
+	return l.name
+}
+
+// Slots get slots
+func (l *LikeDesc) Slots() []int {
+	return l.slots
+}
+
+// Score calc score
+func (l *LikeDesc) Score(stat *ReplyStat) (rs *ReplyScore) {
+	rs = new(ReplyScore)
+	rs.RpID = stat.RpID
+	rs.Score = float64(stat.Like)
+	return
+}
+
+/*
+WilsonLHRR wilson algorithm
+like,reply
+hate,report
+*/
+
+// WilsonLHRR WilsonLHRR
+type WilsonLHRR struct {
+	name   string
+	slots  []int
+	weight *WilsonLHRRWeight
+}
+
+// NewWilsonLHRR NewWilsonLHRR
+func NewWilsonLHRR(name string, slots []int, weight *WilsonLHRRWeight) *WilsonLHRR {
+	return &WilsonLHRR{
+		name:   name,
+		slots:  slots,
+		weight: weight,
+	}
+}
+
+// Name get name
+func (w *WilsonLHRR) Name() string {
+	return w.name
+}
+
+// Slots get slots
+func (w *WilsonLHRR) Slots() []int {
+	return w.slots
+}
+
+// Score calc score
+func (w *WilsonLHRR) Score(stat *ReplyStat) (rs *ReplyScore) {
+	rs = new(ReplyScore)
+	rs.RpID = stat.RpID
+	if stat.Like < 0 || stat.Reply < 0 || stat.Report < 0 || stat.Hate < 0 {
+		return
+	}
+	ups := float64(stat.Like)*w.weight.Like + float64(stat.Reply)*w.weight.Reply
+	downs := float64(stat.Hate)*w.weight.Hate + float64(stat.Report)*w.weight.Report
+	n := ups + downs
+	if n == 0 {
+		return
+	}
+	z := float64(2)
+	p := ups / n
+	rs.Score = (p + math.Pow(z, 2)/(2*n) - (z/(2*n))*math.Sqrt(4*n*(1-p)*p+math.Pow(z, 2))) / (1 + math.Pow(z, 2)/n)
+	return
+}
+
+/*
+WilsonLHRRFluid wilson algorightm dynamic score by time
+like,reply
+hate,report
+*/
+
+// WilsonLHRRFluid WilsonLHRRFluid
+type WilsonLHRRFluid struct {
+	name   string
+	slots  []int
+	weight *WilsonLHRRFluidWeight
+}
+
+// NewWilsonLHRRFluid NewWilsonLHRRFluid
+func NewWilsonLHRRFluid(name string, slots []int, weight *WilsonLHRRFluidWeight) *WilsonLHRRFluid {
+	return &WilsonLHRRFluid{
+		name:   name,
+		slots:  slots,
+		weight: weight,
+	}
+}
+
+// Name get name
+func (w *WilsonLHRRFluid) Name() string {
+	return w.name
+}
+
+// Slots get slots
+func (w *WilsonLHRRFluid) Slots() []int {
+	return w.slots
+}
+
+func coolDownFunc(weight *WilsonLHRRFluidWeight, duration float64) (coefficient float64) {
+	return 1.5 - (0.5 / (1 + math.Exp(-weight.Slope*(duration))))
+}
+
+// Score calc score
+func (w *WilsonLHRRFluid) Score(stat *ReplyStat) (rs *ReplyScore) {
+	rs = new(ReplyScore)
+	rs.RpID = stat.RpID
+	if stat.Like < 0 || stat.Reply < 0 || stat.Report < 0 || stat.Hate < 0 {
+		return
+	}
+	ups := float64(stat.Like)*w.weight.Like + float64(stat.Reply)*w.weight.Reply
+	downs := float64(stat.Hate)*w.weight.Hate + float64(stat.Report)*w.weight.Report
+	n := ups + downs
+	if n == 0 {
+		return
+	}
+	z := float64(2)
+	p := ups / n
+	coefficient := coolDownFunc(w.weight, float64(time.Now().Unix()-int64(stat.ReplyTime))/86400)
+	rs.Score = ((p + math.Pow(z, 2)/(2*n) - (z/(2*n))*math.Sqrt(4*n*(1-p)*p+math.Pow(z, 2))) / (1 + math.Pow(z, 2)/n)) * coefficient
+	return
+}

app/job/main/reply-feed/model/model.go

@@ -0,0 +1,302 @@
+package model
+
+import (
+	xtime "go-common/library/time"
+)
+
+// const varialble
+const (
+	DefaultAlgorithm         = "default"
+	WilsonLHRRAlgorithm      = "wilsonLHRR"
+	WilsonLHRRFluidAlgorithm = "wilsonLHRRFluid"
+	OriginAlgorithm          = "origin"
+	LikeDescAlgorithm        = "likeDesc"
+	StateInactive            = int(0)
+	StateActive              = int(1)
+	StateDelete              = int(2)
+
+	// 用于redis中统计uv
+	StatisticActionRootReply  = "rr"
+	StatisticActionChildReply = "cr"
+	StatisticActionLike       = "l"
+	StatisticActionHate       = "h"
+	StatisticActionReport     = "r"
+
+	StatisticKindTotal = "t"
+	StatisticKindHot   = "h"
+
+	DatabusActionReply      = "reply"
+	DatabusActionReport     = "report_add"
+	DatabusActionLike       = "like"
+	DatabusActionCancelLike = "like_cancel"
+	DatabusActionHate       = "hate"
+	DatabusActionCancelHate = "hate_cancel"
+	// user upper or admin delete
+	DatabusActionDel = "reply_del"
+	// admin delete by report
+	DatabusActionRptDel = "report_del"
+	// admin recover
+	DatabusActionRecover = "reply_recover"
+	// admin or upper top reply
+	DatabusActionTop = "top"
+	// admin or upper untop reply
+	DatabusActionUnTop = "untop"
+
+	DatabusActionReIdx = "re_idx"
+
+	// 只有大于等于3个赞且评论区根评论数目多余20才会被加入热门评论列表
+	MinLikeCount      = 3
+	MinRootReplyCount = 20
+	SlotsNum          = 100
+)
+
+// Statistics const
+var (
+	StatisticActions    = []string{StatisticActionRootReply, StatisticActionChildReply, StatisticActionLike, StatisticActionHate, StatisticActionReport}
+	StatisticKinds      = []string{StatisticKindTotal, StatisticKindHot}
+	StatisticsDatabaseI = []string{"`name`", "`date`", "`hour`"}
+	StatisticsDatabaseU = []string{"hot_like", "hot_hate", "hot_report", "hot_child", "total_like", "total_hate", "total_report", "total_root", "total_child"}
+	StatisticsDatabaseS = []string{"hot_like_uv", "hot_hate_uv", "hot_report_uv", "hot_child_uv", "total_like_uv", "total_hate_uv", "total_report_uv", "total_child_uv", "total_root_uv"}
+)
+
+// ReplyScore reply score
+type ReplyScore struct {
+	RpID  int64
+	Score float64
+}
+
+// ReplyStat 放在MC里的衡量一条根评论质量的各个参数
+type ReplyStat struct {
+	RpID        int64      `json:"rpid"`
+	Like        int        `json:"like"`
+	Hate        int        `json:"hate"`
+	Reply       int        `json:"reply"`
+	Report      int        `json:"report"`
+	SubjectTime xtime.Time `json:"subject_time"`
+	ReplyTime   xtime.Time `json:"reply_time"`
+}
+
+// ReplyResp 返回给reply-interface的评论ID数组，已按热度排好序
+type ReplyResp struct {
+	RpIDs []int64
+	// 属于哪一个实验组
+	TestSetName string
+}
+
+// ReplyList 存在redis sorted set中的数据结构
+type ReplyList struct {
+	RpID []int64
+}
+
+// SlotStat slot stat
+type SlotStat struct {
+	Name      string
+	Slot      int
+	Algorithm string
+	Weight    string
+}
+
+// SlotsStat SlotsStat
+type SlotsStat struct {
+	Name      string
+	Slots     []int
+	Algorithm string
+	Weight    string
+}
+
+// SlotsMapping E group slots
+type SlotsMapping struct {
+	Name  string
+	Slots []int
+}
+
+// StatisticsStat 实验组或者对照组的各项统计
+type StatisticsStat struct {
+	// 流量所属槽位 0~99
+	Slot int
+	// 所属实验组名
+	Name string
+	// 用户在评论首页看到的热门评论被点赞点踩评论以及举报的次数
+	HotLike       int64
+	HotHate       int64
+	HotChildReply int64
+	HotReport     int64
+	// 整个评论区
+	TotalLike       int64
+	TotalHate       int64
+	TotalReport     int64
+	TotalRootReply  int64
+	TotalChildReply int64
+
+	HotLikeUV   int64
+	HotHateUV   int64
+	HotReportUV int64
+	HotChildUV  int64
+
+	TotalLikeUV   int64
+	TotalHateUV   int64
+	TotalReportUV int64
+	TotalChildUV  int64
+	TotalRootUV   int64
+}
+
+// Merge merge two statistics
+func (stat1 *StatisticsStat) Merge(stat2 *StatisticsStat) (stat3 *StatisticsStat) {
+	stat3 = new(StatisticsStat)
+	stat3.TotalLike = stat1.TotalLike + stat2.TotalLike
+	stat3.TotalHate = stat1.TotalHate + stat2.TotalHate
+	stat3.TotalReport = stat1.TotalReport + stat2.TotalReport
+	stat3.TotalRootReply = stat1.TotalRootReply + stat2.TotalRootReply
+	stat3.TotalChildReply = stat1.TotalChildReply + stat2.TotalChildReply
+	stat3.HotLike = stat1.HotLike + stat2.HotLike
+	stat3.HotHate = stat1.HotHate + stat2.HotHate
+	stat3.HotReport = stat1.HotReport + stat2.HotReport
+	stat3.HotChildReply = stat1.HotChildReply + stat2.HotChildReply
+
+	stat3.HotLikeUV = stat1.HotLikeUV + stat2.HotLikeUV
+	stat3.HotHateUV = stat1.HotHateUV + stat2.HotHateUV
+	stat3.HotReportUV = stat1.HotReportUV + stat2.HotReportUV
+	stat3.HotChildUV = stat1.HotChildUV + stat2.HotChildUV
+
+	stat3.TotalLikeUV = stat1.TotalLikeUV + stat2.TotalLikeUV
+	stat3.TotalHateUV = stat1.TotalHateUV + stat2.TotalHateUV
+	stat3.TotalReportUV = stat1.TotalReportUV + stat2.TotalReportUV
+	stat3.TotalRootUV = stat1.TotalRootUV + stat2.TotalRootUV
+	stat3.TotalChildUV = stat1.TotalChildUV + stat2.TotalChildUV
+	return
+}
+
+// StrategyStat 实验组所使用算法，以及各个参数情况
+type StrategyStat struct {
+	Name      string             `json:"name"`
+	Percent   int                `json:"percent"`
+	Algorithm string             `json:"algorithm"`
+	Args      map[string]float64 `json:"args"`
+}
+
+// RefreshChecker 刷新热门评论的触发条件，用来对同一个评论区的所有请求进行聚合
+type RefreshChecker struct {
+	Oid           int64
+	Type          int
+	LastTimeStamp int64
+}
+
+// WilsonLHRRWeight wilson score interval weight
+type WilsonLHRRWeight struct {
+	Like   float64
+	Hate   float64
+	Reply  float64
+	Report float64
+}
+
+// WilsonLHRRFluidWeight wilson
+type WilsonLHRRFluidWeight struct {
+	Like   float64
+	Hate   float64
+	Reply  float64
+	Report float64
+	Slope  float64
+}
+
+// EventMsg event message
+type EventMsg struct {
+	Action string `json:"action"`
+	Oid    int64  `json:"oid"`
+	Tp     int    `json:"tp"`
+}
+
+// StatsMsg stats message
+type StatsMsg struct {
+	Action  string   `json:"action"`
+	Mid     int64    `json:"mid"`
+	Subject *Subject `json:"subject"`
+	Reply   *Reply   `json:"reply"`
+	Report  *Report  `json:"report,omitempty"`
+}
+
+// Sharding 返回该用户属于哪一个组
+// 将流量划分为100份
+func (r *StatsMsg) Sharding() int64 {
+	return r.Mid % SlotsNum
+}
+
+// HotCondition return if should check exists in hot reply
+func (r *StatsMsg) HotCondition() bool {
+	if r.Action == DatabusActionReply && !r.Reply.IsRoot() {
+		return true
+	}
+	if r.Reply.IsRoot() && r.Reply.Like >= MinLikeCount &&
+		(r.Action == DatabusActionLike || r.Action == DatabusActionHate ||
+			r.Action == DatabusActionCancelLike || r.Action == DatabusActionCancelHate || r.Action == DatabusActionReport) {
+		return true
+	}
+	return false
+}
+
+// Reply define reply object
+type Reply struct {
+	RpID   int64      `json:"rpid"`
+	Mid    int64      `json:"mid"`
+	Root   int64      `json:"root"`
+	Parent int64      `json:"parent"`
+	RCount int        `json:"rcount"`
+	Floor  int        `json:"floor"`
+	State  int8       `json:"state"`
+	Attr   uint32     `json:"attr"`
+	CTime  xtime.Time `json:"ctime"`
+	Like   int        `json:"like"`
+	Hate   int        `json:"hate"`
+}
+
+// Legal return a reply legal
+func (r *Reply) Legal() bool {
+	// 0,1,2,5,6 所有需要显示给用户的评论state
+	return r.State == 0 || r.State == 1 || r.State == 2 || r.State == 5 || r.State == 6
+}
+
+// ShowAfterAudit ShowAfterAudit
+func (r *Reply) ShowAfterAudit() bool {
+	return r.State == 11
+}
+
+// AuditButShow AuditButShow
+func (r *Reply) AuditButShow() bool {
+	return r.State == 5
+}
+
+// IsRoot IsRoot
+func (r *Reply) IsRoot() bool {
+	return r.Root == 0
+}
+
+// Qualified Qualified
+func (r *Reply) Qualified() bool {
+	return r.Like >= MinLikeCount
+}
+
+// Report define reply report
+type Report struct {
+	RpID  int64      `json:"rpid"`
+	Mid   int64      `json:"mid"`
+	Count int        `json:"count"`
+	Score int        `json:"score"`
+	State int8       `json:"state"`
+	CTime xtime.Time `json:"ctime"`
+	Attr  uint32     `json:"attr"`
+}
+
+// Subject is subject of reply
+type Subject struct {
+	Oid    int64      `json:"oid"`
+	Type   int        `json:"type"`
+	Mid    int64      `json:"mid"`
+	RCount int        `json:"rcount"`
+	State  int8       `json:"state"`
+	Attr   uint32     `json:"attr"`
+	CTime  xtime.Time `json:"ctime"`
+}
+
+// ShowHotReply if show
+func (s *Subject) ShowHotReply() bool {
+	return s.RCount >= MinRootReplyCount
+}

app/job/main/reply-feed/server/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/reply-feed/server/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/reply-feed/conf:go_default_library",
+        "//app/job/main/reply-feed/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/reply-feed/server/http/http.go

@@ -0,0 +1,41 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/main/reply-feed/conf"
+	"go-common/app/job/main/reply-feed/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	svc *service.Service
+)
+
+// Init init
+func Init(c *conf.Config, s *service.Service) {
+	svc = s
+	engine := bm.DefaultServer(c.BM)
+	router(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("engine.Start error(%v)", err)
+		panic(err)
+	}
+}
+
+func router(e *bm.Engine) {
+	e.Ping(ping)
+	e.Register(register)
+}
+
+func ping(c *bm.Context) {
+	if err := svc.Ping(c); err != nil {
+		log.Error("ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}

app/job/main/reply-feed/service/BUILD

@@ -0,0 +1,49 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "databus.go",
+        "loader.go",
+        "reply_set.go",
+        "reply_zset.go",
+        "service.go",
+        "stat.go",
+        "statistics.go",
+    ],
+    importpath = "go-common/app/job/main/reply-feed/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/reply-feed/conf:go_default_library",
+        "//app/job/main/reply-feed/dao:go_default_library",
+        "//app/job/main/reply-feed/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/netutil:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/sync/pipeline/fanout:go_default_library",
+        "//vendor/github.com/ivpusic/grpool:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/reply-feed/service/databus.go

@@ -0,0 +1,201 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/log"
+)
+
+// func (s *Service) eventproc() {
+// 	defer s.waiter.Done()
+// 	msgs := s.eventConsumer.Messages()
+// 	ctx := context.Background()
+// 	for {
+// 		msg, ok := <-msgs
+// 		if !ok {
+// 			log.Warn("databus consumer channel has been closed.")
+// 			return
+// 		}
+// 		if msg.Topic != s.c.Databus.Event.Topic {
+// 			log.Warn("wrong topic actual (%s) expect (%s)", msg.Topic, s.c.Databus.Stats.Topic)
+// 			continue
+// 		}
+// 		value := &model.EventMsg{}
+// 		if err := json.Unmarshal(msg.Value, value); err != nil {
+// 			log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
+// 			continue
+// 		}
+// 		switch value.Action {
+// 		case model.DatabusActionReIdx:
+// 			s.setReplySetBatch(ctx, value.Oid, value.Tp)
+// 			s.upsertZSet(ctx, value.Oid, value.Tp)
+// 		default:
+// 			continue
+// 		}
+// 		msg.Commit()
+// 		log.Info("consumer topic:%s, partitionId:%d, offset:%d, Key:%s, Value:%s", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
+// 	}
+// }
+
+func (s *Service) statsproc() {
+	defer s.waiter.Done()
+	msgs := s.statsConsumer.Messages()
+	for {
+		msg, ok := <-msgs
+		if !ok {
+			log.Warn("databus consumer channel has been closed.")
+			return
+		}
+		if msg.Topic != s.c.Databus.Stats.Topic {
+			log.Warn("wrong topic actual (%s) expect (%s)", msg.Topic, s.c.Databus.Stats.Topic)
+			continue
+		}
+		value := &model.StatsMsg{}
+		if err := json.Unmarshal(msg.Value, value); err != nil {
+			log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
+			continue
+		}
+		// 脏数据
+		if value.Reply == nil || value.Subject == nil || (value.Action == model.DatabusActionReport && value.Report == nil) {
+			log.Error("illegal message (%v)", value)
+			continue
+		}
+		ctx := context.Background()
+		// 针对评论列表的流程
+		s.replyListFlow(ctx, value)
+		// 针对统计数据的流程
+		s.statisticsFlow(ctx, value)
+		msg.Commit()
+		log.Info("consumer topic:%s, partitionId:%d, offset:%d, Key:%s, Value:%s", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
+	}
+}
+
+func (s *Service) statisticsFlow(ctx context.Context, value *model.StatsMsg) {
+	var (
+		reply      = value.Reply
+		oid        = value.Subject.Oid
+		tp         = value.Subject.Type
+		rpID       = reply.RpID
+		isHotReply bool
+		name       string
+		err        error
+	)
+	s.statisticsLock.RLock()
+	name = s.statisticsStats[value.Sharding()].Name
+	s.statisticsLock.RUnlock()
+	if value.HotCondition() {
+		if !reply.IsRoot() {
+			rpID = reply.Root
+		}
+		if name == model.DefaultAlgorithm {
+			if isHotReply, err = s.dao.IsOriginHot(ctx, oid, rpID, tp); err != nil {
+				return
+			}
+		} else {
+			if isHotReply, err = s.isHot(ctx, name, oid, rpID, tp); err != nil {
+				return
+			}
+		}
+	}
+	s.addUV(ctx, value, isHotReply)
+	switch value.Action {
+	case model.DatabusActionLike:
+		if isHotReply {
+			s.statisticsStats[value.Sharding()].HotLike++
+		}
+		s.statisticsStats[value.Sharding()].TotalLike++
+	case model.DatabusActionHate:
+		if isHotReply {
+			s.statisticsStats[value.Sharding()].HotHate++
+		}
+		s.statisticsStats[value.Sharding()].TotalHate++
+	case model.DatabusActionCancelLike:
+		if isHotReply && s.statisticsStats[value.Sharding()].HotLike > 0 {
+			s.statisticsStats[value.Sharding()].HotLike--
+		}
+		if s.statisticsStats[value.Sharding()].TotalLike > 0 {
+			s.statisticsStats[value.Sharding()].TotalLike--
+		}
+	case model.DatabusActionCancelHate:
+		if isHotReply && s.statisticsStats[value.Sharding()].HotHate > 0 {
+			s.statisticsStats[value.Sharding()].HotHate--
+		}
+		if s.statisticsStats[value.Sharding()].TotalHate > 0 {
+			s.statisticsStats[value.Sharding()].TotalHate--
+		}
+	case model.DatabusActionReport:
+		if isHotReply {
+			s.statisticsStats[value.Sharding()].HotReport++
+		}
+		s.statisticsStats[value.Sharding()].TotalReport++
+	case model.DatabusActionReply:
+		if reply.IsRoot() {
+			s.statisticsStats[value.Sharding()].TotalRootReply++
+		} else {
+			if isHotReply {
+				s.statisticsStats[value.Sharding()].HotChildReply++
+			}
+			s.statisticsStats[value.Sharding()].TotalChildReply++
+		}
+	}
+}
+
+func (s *Service) replyListFlow(ctx context.Context, value *model.StatsMsg) {
+	var (
+		subject     = value.Subject
+		reply       = value.Reply
+		oid         = subject.Oid
+		tp          = subject.Type
+		stat        *model.ReplyStat
+		reportCount int
+		err         error
+	)
+	if value.Report == nil {
+		reportCount = 0
+	} else {
+		reportCount = value.Report.Count
+	}
+	// if root reply get stat, else get root reply stat
+	if reply.IsRoot() {
+		stat = &model.ReplyStat{
+			RpID:        reply.RpID,
+			Reply:       reply.RCount,
+			Like:        reply.Like,
+			Hate:        reply.Hate,
+			Report:      reportCount,
+			SubjectTime: subject.CTime,
+			ReplyTime:   reply.CTime,
+		}
+	} else {
+		if stat, err = s.GetStatByID(ctx, oid, tp, reply.Root); err != nil || stat == nil {
+			return
+		}
+	}
+	if reply.IsRoot() {
+		switch value.Action {
+		case model.DatabusActionTop, model.DatabusActionDel, model.DatabusActionRptDel:
+			s.remReply(ctx, oid, tp, stat.RpID)
+		case model.DatabusActionUnTop, model.DatabusActionRecover, model.DatabusActionReply:
+			s.addReplySet(ctx, oid, tp, stat.RpID)
+		case model.DatabusActionLike, model.DatabusActionCancelLike, model.DatabusActionCancelHate, model.DatabusActionHate, model.DatabusActionReport:
+			s.updateStat(ctx, stat.RpID, stat)
+		default:
+			return
+		}
+	} else {
+		switch value.Action {
+		case model.DatabusActionReply, model.DatabusActionRecover:
+			stat.Reply++
+		case model.DatabusActionDel, model.DatabusActionRptDel:
+			if stat.Reply > 0 {
+				stat.Reply--
+			}
+		default:
+			return
+		}
+		s.updateStat(ctx, stat.RpID, stat)
+	}
+	s.upsertZSet(ctx, oid, tp)
+}

app/job/main/reply-feed/service/loader.go

@@ -0,0 +1,100 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/log"
+)
+
+func (s *Service) loadAlgorithm() (err error) {
+	ss, err := s.dao.SlotStats(context.Background())
+	if err != nil {
+		return
+	}
+	// 按name聚合slot
+	ssMap := make(map[string]*model.SlotsStat)
+	for _, s := range ss {
+		if v, exists := ssMap[s.Name]; exists {
+			v.Slots = append(v.Slots, s.Slot)
+		} else {
+			ssMap[s.Name] = &model.SlotsStat{
+				Name:      s.Name,
+				Slots:     []int{s.Slot},
+				Algorithm: s.Algorithm,
+				Weight:    s.Weight,
+			}
+		}
+	}
+	var algorithms []model.Algorithm
+	for name, ss := range ssMap {
+		if ss.Weight == "" {
+			continue
+		}
+		var (
+			algorithm model.Algorithm
+			w         interface{}
+		)
+		if ss.Algorithm == model.WilsonLHRRAlgorithm || ss.Algorithm == model.WilsonLHRRFluidAlgorithm {
+			if err = json.Unmarshal([]byte(ss.Weight), &w); err != nil {
+				log.Error("json.Unmarshal() error(%v), name (%s), algorightm (%s), weight (%s)", err, name, ss.Algorithm, ss.Weight)
+				return
+			}
+		}
+		switch ss.Algorithm {
+		case model.WilsonLHRRAlgorithm:
+			weight := w.(map[string]interface{})
+			algorithm = model.NewWilsonLHRR(name, ss.Slots, &model.WilsonLHRRWeight{
+				Like:   weight["like"].(float64),
+				Hate:   weight["hate"].(float64),
+				Reply:  weight["reply"].(float64),
+				Report: weight["report"].(float64),
+			})
+		case model.WilsonLHRRFluidAlgorithm:
+			weight := w.(map[string]interface{})
+			algorithm = model.NewWilsonLHRRFluid(name, ss.Slots, &model.WilsonLHRRFluidWeight{
+				Like:   weight["like"].(float64),
+				Hate:   weight["hate"].(float64),
+				Reply:  weight["reply"].(float64),
+				Report: weight["report"].(float64),
+				Slope:  weight["slope"].(float64),
+			})
+		case model.OriginAlgorithm:
+			algorithm = model.NewOrigin(name, ss.Slots)
+		case model.LikeDescAlgorithm:
+			algorithm = model.NewLikeDesc(name, ss.Slots)
+		case model.DefaultAlgorithm:
+			continue
+		default:
+			log.Warn("invalid algorithm")
+			continue
+		}
+		if algorithm != nil {
+			algorithms = append(algorithms, algorithm)
+		}
+	}
+	s.algorithmsLock.Lock()
+	s.algorithms = algorithms
+	s.algorithmsLock.Unlock()
+	return
+}
+
+func (s *Service) loadSlots() (err error) {
+	ctx := context.Background()
+	slotsMap, err := s.dao.SlotsMapping(ctx)
+	if err != nil {
+		return
+	}
+	s.statisticsLock.Lock()
+	for name, mapping := range slotsMap {
+		for _, slot := range mapping.Slots {
+			s.statisticsStats[slot].Name = name
+			s.statisticsStats[slot].Slot = slot
+		}
+		log.Warn("name stat (name: %s, slots: %v)", name, mapping.Slots)
+	}
+	log.Warn("statistics stat (%v)", s.statisticsStats)
+	s.statisticsLock.Unlock()
+	return
+}

app/job/main/reply-feed/service/reply_set.go

@@ -0,0 +1,91 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/log"
+)
+
+// setReplySetBatch set reply set batch.
+func (s *Service) setReplySetBatch(ctx context.Context, oid int64, tp int) (err error) {
+	var (
+		stats []*model.ReplyStat
+		rpIDs []int64
+	)
+	// 从DB查出满足热门评论条件的评论ID
+	if rpIDs, err = s.dao.RpIDs(ctx, oid, tp); err != nil || len(rpIDs) <= 0 {
+		return
+	}
+	// 从MC或者DB中取出reply stat
+	if stats, err = s.GetStatsByID(ctx, oid, tp, rpIDs); err != nil {
+		return
+	}
+	for _, stat := range stats {
+		stat := stat
+		s.statQ.Do(ctx, func(ctx context.Context) {
+			s.dao.SetReplyStatMc(ctx, stat)
+		})
+	}
+	return s.dao.SetReplySetRds(ctx, oid, tp, rpIDs)
+}
+
+// addReplySet add one rpID into redis reply set.
+func (s *Service) addReplySet(ctx context.Context, oid int64, tp int, rpID int64) (err error) {
+	ok, err := s.dao.ExpireReplySetRds(ctx, oid, tp)
+	if err != nil {
+		return
+	}
+	if ok {
+		if err = s.dao.AddReplySetRds(ctx, oid, tp, rpID); err != nil {
+			return
+		}
+	} else {
+		if err = s.setReplySetBatch(ctx, oid, tp); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (s *Service) remSet(ctx context.Context, oid, rpID int64, tp int) (err error) {
+	if err = s.dao.RemReplySetRds(ctx, oid, rpID, tp); err != nil {
+		log.Error("Remove rpID from set error (%v)", err)
+	}
+	return
+}
+
+// func (s *Service) delSet(ctx context.Context, oid int64, tp int) (err error) {
+// 	if err = s.dao.DelReplySetRds(ctx, oid, tp); err != nil {
+// 		log.Error("delete reply set(oid: %d, type: %d)", oid, tp)
+// 	}
+// 	return
+// }
+
+// func (s *Service) delReply(ctx context.Context, oid int64, tp int) {
+// 	var err error
+// 	if err = s.delSet(ctx, oid, tp); err != nil {
+// 		s.replyListQ.Do(ctx, func(ctx context.Context) {
+// 			s.delSet(ctx, oid, tp)
+// 		})
+// 	}
+// 	if err = s.delZSet(ctx, oid, tp); err != nil {
+// 		s.replyListQ.Do(ctx, func(ctx context.Context) {
+// 			s.delZSet(ctx, oid, tp)
+// 		})
+// 	}
+// }
+
+func (s *Service) remReply(ctx context.Context, oid int64, tp int, rpID int64) {
+	var err error
+	if err = s.remSet(ctx, oid, rpID, tp); err != nil {
+		s.replyListQ.Do(ctx, func(ctx context.Context) {
+			s.remSet(ctx, oid, rpID, tp)
+		})
+	}
+	if err = s.remZSet(ctx, oid, tp, rpID); err != nil {
+		s.replyListQ.Do(ctx, func(ctx context.Context) {
+			s.remZSet(ctx, oid, tp, rpID)
+		})
+	}
+}

app/job/main/reply-feed/service/reply_zset.go

@@ -0,0 +1,138 @@
+package service
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+// func (s *Service) delZSet(ctx context.Context, oid int64, tp int) (err error) {
+// 	var names []string
+// 	s.algorithmsLock.RLock()
+// 	for _, algorithm := range s.algorithms {
+// 		names = append(names, algorithm.Name())
+// 	}
+// 	s.algorithmsLock.RUnlock()
+// 	if err = s.dao.DelReplyZSetRds(ctx, names, oid, tp); err != nil {
+// 		log.Error("Del ZSet from redis oid(%d) type(%d) error(%v)", oid, tp, err)
+// 	}
+// 	return
+// }
+
+func (s *Service) remZSet(ctx context.Context, oid int64, tp int, rpID int64) (err error) {
+	var (
+		names []string
+	)
+	s.algorithmsLock.RLock()
+	for _, algorithm := range s.algorithms {
+		names = append(names, algorithm.Name())
+	}
+	s.algorithmsLock.RUnlock()
+	for _, name := range names {
+		if err = s.dao.RemReplyZSetRds(ctx, name, oid, tp, rpID); err != nil {
+			log.Error("Remove reply (name: %s, oid: %d, type: %d, rpID: %d) from ZSet failed.", name, oid, tp, rpID)
+			return
+		}
+	}
+	return
+}
+
+func (s *Service) upsertZSet(ctx context.Context, oid int64, tp int) {
+	var (
+		rpIDs []int64
+		rs    []*model.ReplyStat
+		err   error
+		ts    int64
+	)
+	// 获取计时器
+	if ts, err = s.dao.CheckerTsRds(ctx, oid, tp); err != nil && err != redis.ErrNil {
+		// 出错不刷新，如果缓存里还没有的话刷新
+		return
+	} else if time.Now().Unix()-ts < s.c.RefreshTime {
+		// 小于CD时间不刷新
+		return
+	}
+	// 从reply set中取rpIDs
+	ok, err := s.dao.ExpireReplySetRds(ctx, oid, tp)
+	if err != nil {
+		return
+	}
+	if ok {
+		// 缓存有则从redis中取
+		if rpIDs, err = s.dao.ReplySetRds(ctx, oid, tp); err != nil {
+			return
+		}
+	} else {
+		// 缓存中没有从DB中取
+		if rpIDs, err = s.dao.RpIDs(ctx, oid, tp); err != nil {
+			return
+		}
+		// 异步回源
+		s.taskQ.Do(ctx, func(ctx context.Context) {
+			s.setReplySetBatch(ctx, oid, tp)
+		})
+	}
+	// 从MC中获取reply stat
+	if rs, err = s.GetStatsByID(ctx, oid, tp, rpIDs); err != nil {
+		return
+	}
+	// 重新计算分值
+	rsMap, err := s.recalculateScore(ctx, rs)
+	if err != nil {
+		return
+	}
+	for name, rs := range rsMap {
+		name, rs := name, rs
+		s.replyListQ.Do(ctx, func(ctx context.Context) {
+			s.dao.SetReplyZSetRds(ctx, name, oid, tp, rs)
+		})
+	}
+	// 更新完后更新计时器
+	if err = s.dao.SetCheckerTsRds(ctx, oid, tp); err != nil {
+		log.Error("set refresh checker error (%v)", err)
+	}
+}
+
+// recalculateScore recalculate all e group reply list score.
+func (s *Service) recalculateScore(ctx context.Context, stats []*model.ReplyStat) (rsMap map[string][]*model.ReplyScore, err error) {
+	rsMap = make(map[string][]*model.ReplyScore)
+	s.algorithmsLock.RLock()
+	defer s.algorithmsLock.RUnlock()
+	for _, algorithm := range s.algorithms {
+		wg := sync.WaitGroup{}
+		rs := make([]*model.ReplyScore, len(stats))
+		for i := range stats {
+			j := i
+			wg.Add(1)
+			s.calculator.JobQueue <- func() {
+				rs[j] = algorithm.Score(stats[j])
+				wg.Done()
+			}
+		}
+		wg.Wait()
+		rsMap[algorithm.Name()] = rs
+	}
+	return
+}
+
+func (s *Service) isHot(ctx context.Context, name string, oid, rpID int64, tp int) (isHot bool, err error) {
+	rpIDs, err := s.dao.RangeReplyZSetRds(ctx, name, oid, tp, 0, 5)
+	if err != nil || len(rpIDs) <= 0 {
+		return
+	}
+	rs, err := s.GetStatsByID(ctx, oid, tp, rpIDs)
+	if err != nil {
+		return
+	}
+	for _, r := range rs {
+		if r.RpID == rpID && r.Like >= 3 {
+			isHot = true
+			return
+		}
+	}
+	return
+}

app/job/main/reply-feed/service/service.go

@@ -0,0 +1,111 @@
+package service
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/reply-feed/conf"
+	"go-common/app/job/main/reply-feed/dao"
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/log"
+	"go-common/library/net/netutil"
+	"go-common/library/queue/databus"
+	"go-common/library/sync/pipeline/fanout"
+
+	"github.com/ivpusic/grpool"
+	"github.com/robfig/cron"
+)
+
+// Service struct
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+	// 定时任务
+	cron *cron.Cron
+	// backoff
+	bc            netutil.BackoffConfig
+	statsConsumer *databus.Databus
+	// eventConsumer  *databus.Databus
+	taskQ      *fanout.Fanout
+	uvQ        *fanout.Fanout
+	statQ      *fanout.Fanout
+	replyListQ *fanout.Fanout
+	waiter     sync.WaitGroup
+
+	// 专门计算热评分数的goroutine pool
+	calculator *grpool.Pool
+
+	statisticsStats [model.SlotsNum]model.StatisticsStat
+
+	algorithmsLock sync.RWMutex
+	statisticsLock sync.RWMutex
+	algorithms     []model.Algorithm
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:    c,
+		dao:  dao.New(c),
+		cron: cron.New(),
+		bc: netutil.BackoffConfig{
+			MaxDelay:  1 * time.Second,
+			BaseDelay: 100 * time.Millisecond,
+			Factor:    1.6,
+			Jitter:    0.2,
+		},
+		statsConsumer: databus.New(c.Databus.Stats),
+		// eventConsumer: databus.New(c.Databus.Event),
+		// 处理异步写任务的goroutine
+		taskQ:      fanout.New("task"),
+		uvQ:        fanout.New("uv-task", fanout.Worker(4), fanout.Buffer(2048)),
+		statQ:      fanout.New("memcache", fanout.Worker(4), fanout.Buffer(2048)),
+		replyListQ: fanout.New("redis", fanout.Worker(4), fanout.Buffer(2048)),
+		calculator: grpool.NewPool(4, 2048),
+	}
+	var err error
+	if err = s.loadAlgorithm(); err != nil {
+		panic(err)
+	}
+	if err = s.loadSlots(); err != nil {
+		panic(err)
+	}
+	go s.loadproc()
+
+	// 消费databus
+	s.waiter.Add(1)
+	go s.statsproc()
+	// s.waiter.Add(1)
+	// go s.eventproc()
+
+	// 每整小时执行一次将统计数据写入DB
+	s.cron.AddFunc("@hourly", func() {
+		s.persistStatistics()
+	})
+	s.cron.Start()
+	return s
+}
+
+func (s *Service) loadproc() {
+	for {
+		time.Sleep(time.Minute)
+		s.loadAlgorithm()
+		s.loadSlots()
+	}
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.statsConsumer.Close()
+	// s.eventConsumer.Close()
+	log.Warn("consumer closed")
+	s.waiter.Wait()
+	s.persistStatistics()
+	s.dao.Close()
+}

app/job/main/reply-feed/service/stat.go

@@ -0,0 +1,117 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+func (s *Service) updateStat(ctx context.Context, rpID int64, stat *model.ReplyStat) {
+	s.statQ.Do(ctx, func(ctx context.Context) {
+		s.dao.SetReplyStatMc(ctx, stat)
+	})
+}
+
+// GetStatByID 从缓存获取单条评论stat获取不到则从DB获取
+func (s *Service) GetStatByID(ctx context.Context, oid int64, tp int, rpID int64) (stat *model.ReplyStat, err error) {
+	stats, err := s.GetStatsByID(ctx, oid, tp, []int64{rpID})
+	if err != nil {
+		return
+	}
+	if len(stats) > 0 {
+		stat = stats[0]
+	} else {
+		err = ecode.ReplyNotExist
+		log.Error("reply not exists rpID %d", rpID)
+	}
+	return
+}
+
+// GetStatsByID 从缓存获取多条评论stat获取不到则从DB获取
+func (s *Service) GetStatsByID(ctx context.Context, oid int64, tp int, rpIDs []int64) (rs []*model.ReplyStat, err error) {
+	var (
+		rsMap   map[int64]*model.ReplyStat
+		missIDs []int64
+		missed  map[int64]*model.ReplyStat
+	)
+	if rsMap, missIDs, err = s.dao.ReplyStatsMc(ctx, rpIDs); err != nil {
+		return
+	}
+	for _, stat := range rsMap {
+		rs = append(rs, stat)
+	}
+	if len(missIDs) > 0 {
+		// miss从DB查
+		if missed, err = s.getStatsByIDDB(ctx, oid, tp, missIDs); err != nil {
+			rs = nil
+			return
+		}
+		for _, stat := range missed {
+			stat := stat
+			rs = append(rs, stat)
+			s.statQ.Do(ctx, func(ctx context.Context) {
+				s.dao.SetReplyStatMc(ctx, stat)
+			})
+		}
+	}
+	return
+}
+
+// getStatsByIDDB 从数据库获取热门评论stats，这里不需要一致性，所以跨表查再聚合
+func (s *Service) getStatsByIDDB(ctx context.Context, oid int64, tp int, rpIDs []int64) (rs map[int64]*model.ReplyStat, err error) {
+	if len(rpIDs) == 0 {
+		return
+	}
+	replyMap, err := s.dao.ReplyLHRCStatsByID(ctx, oid, rpIDs)
+	if err != nil {
+		return
+	}
+	reportMap, err := s.dao.ReportStatsByID(ctx, oid, rpIDs)
+	if err != nil {
+		return
+	}
+	ctime, err := s.dao.SubjectStats(ctx, oid, tp)
+	if err != nil {
+		return
+	}
+	for rpID := range replyMap {
+		r, ok := reportMap[rpID]
+		if ok && r != nil {
+			replyMap[rpID].Report = r.Report
+		}
+		replyMap[rpID].SubjectTime = ctime
+	}
+	rs = replyMap
+	return
+}
+
+// GetStatsDB 从数据库获取热门评论stats，这里不需要一致性，所以跨表查再聚合
+func (s *Service) GetStatsDB(ctx context.Context, oid int64, tp int) (rs []*model.ReplyStat, err error) {
+	replyMap, err := s.dao.ReplyLHRCStats(ctx, oid, tp)
+	if err != nil {
+		return
+	}
+	var RpIDs []int64
+	for rpID := range replyMap {
+		RpIDs = append(RpIDs, rpID)
+	}
+	reportMap, err := s.dao.ReportStatsByID(ctx, oid, RpIDs)
+	if err != nil {
+		return
+	}
+	ctime, err := s.dao.SubjectStats(ctx, oid, tp)
+	if err != nil {
+		return
+	}
+	for rpID := range replyMap {
+		r, ok := reportMap[rpID]
+		if ok && r != nil {
+			replyMap[rpID].Report = r.Report
+		}
+		replyMap[rpID].SubjectTime = ctime
+		rs = append(rs, replyMap[rpID])
+	}
+	return
+}

app/job/main/reply-feed/service/statistics.go

@@ -0,0 +1,162 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/reply-feed/model"
+	"go-common/library/log"
+)
+
+func hourNow() int {
+	return time.Now().Hour()
+}
+
+func lastHour() int {
+	hour := hourNow()
+	if hour == 0 {
+		return 23
+	}
+	return hour - 1
+}
+
+// AddUV ...
+func (s *Service) addUV(ctx context.Context, value *model.StatsMsg, isHot bool) {
+	var action string
+	switch value.Action {
+	case model.DatabusActionLike:
+		action = model.StatisticActionLike
+	case model.DatabusActionHate:
+		action = model.StatisticActionHate
+	case model.DatabusActionReport:
+		action = model.StatisticActionReport
+	case model.DatabusActionReply:
+		if value.Reply.IsRoot() {
+			action = model.StatisticActionRootReply
+		} else {
+			action = model.StatisticActionChildReply
+		}
+	}
+	if action == "" || value.Mid == 0 {
+		return
+	}
+	s.uvQ.Do(ctx, func(ctx context.Context) {
+		if isHot {
+			s.dao.AddUV(ctx, action, hourNow(), int(value.Sharding()), value.Mid, model.StatisticKindHot)
+		}
+		s.dao.AddUV(ctx, action, hourNow(), int(value.Sharding()), value.Mid, model.StatisticKindTotal)
+	})
+}
+
+// uvStatistics ...
+func (s *Service) uvStatistics(ctx context.Context, slots []int, stat *model.StatisticsStat) {
+	var (
+		keys     []string
+		lastHour = lastHour()
+		x, y, z  = len(model.StatisticKinds), len(model.StatisticActions), len(slots)
+		idxMap   = make([][][]int, x)
+		idx      int
+	)
+	for i, kind := range model.StatisticKinds {
+		idxMap[i] = make([][]int, y)
+		for j, action := range model.StatisticActions {
+			idxMap[i][j] = make([]int, z)
+			for k, slot := range slots {
+				keys = append(keys, s.dao.KeyUV(action, lastHour, slot, kind))
+				idxMap[i][j][k] = idx
+				idx++
+			}
+		}
+	}
+	counts, err := s.dao.CountUV(ctx, keys)
+	if err != nil || len(counts) != len(keys) {
+		return
+	}
+	for i, kind := range model.StatisticKinds {
+		for j, action := range model.StatisticActions {
+			for k := range slots {
+				count := counts[idxMap[i][j][k]]
+				switch {
+				case kind == model.StatisticKindTotal:
+					switch action {
+					case model.StatisticActionRootReply:
+						stat.TotalRootUV += count
+					case model.StatisticActionChildReply:
+						stat.TotalChildUV += count
+					case model.StatisticActionLike:
+						stat.TotalLikeUV += count
+					case model.StatisticActionHate:
+						stat.TotalHateUV += count
+					case model.StatisticActionReport:
+						stat.TotalReportUV += count
+					}
+				case kind == model.StatisticKindHot:
+					switch action {
+					case model.StatisticActionChildReply:
+						stat.HotChildUV += count
+					case model.StatisticActionLike:
+						stat.HotLikeUV += count
+					case model.StatisticActionHate:
+						stat.HotHateUV += count
+					case model.StatisticActionReport:
+						stat.HotReportUV += count
+					}
+				}
+			}
+		}
+	}
+}
+
+// persistStatistics persist statistics
+func (s *Service) persistStatistics() {
+	ctx := context.Background()
+	statisticsMap := make(map[string]*model.StatisticsStat)
+	nameMapping := make(map[string][]int)
+	s.statisticsLock.RLock()
+	for slot, stat := range s.statisticsStats {
+		nameMapping[stat.Name] = append(nameMapping[stat.Name], slot)
+		s, ok := statisticsMap[stat.Name]
+		if ok {
+			statisticsMap[stat.Name] = s.Merge(&stat)
+		} else {
+			statisticsMap[stat.Name] = &stat
+		}
+	}
+	s.statisticsLock.RUnlock()
+	now := time.Now()
+	year, month, day := now.Date()
+	date := year*10000 + int(month)*100 + day
+	hour := now.Hour()
+	for name, stat := range statisticsMap {
+		s.uvStatistics(ctx, nameMapping[name], stat)
+		err := s.dao.UpsertStatistics(ctx, name, date, hour, stat)
+		var (
+			retryTimes    = 0
+			maxRetryTimes = 5
+		)
+		for err != nil && retryTimes < maxRetryTimes {
+			time.Sleep(s.bc.Backoff(retryTimes))
+			err = s.dao.UpsertStatistics(ctx, name, date, hour, stat)
+			retryTimes++
+		}
+		if retryTimes >= maxRetryTimes {
+			log.Error("upsert statistics error retry reached max retry times.")
+		}
+	}
+	for i := range s.statisticsStats {
+		reset(&s.statisticsStats[i])
+	}
+	log.Warn("reset statistics at %v", now)
+}
+
+func reset(stat *model.StatisticsStat) {
+	stat.HotChildReply = 0
+	stat.HotHate = 0
+	stat.HotLike = 0
+	stat.HotReport = 0
+	stat.TotalChildReply = 0
+	stat.TotalRootReply = 0
+	stat.TotalReport = 0
+	stat.TotalLike = 0
+	stat.TotalHate = 0
+}