Create & Init Project...

app/job/live/push-search/CHANGELOG.md

@@ -0,0 +1,9 @@
+
+# v1.0.2
+1. 用户信息 multiV3 切换为 account
+
+# v1.0.1
+1. waitgroup
+
+# v1.0.0
+1. 上线功能xxx

app/job/live/push-search/CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+# Owner
+fuyu01
+
+# Author
+
+# Reviewer

app/job/live/push-search/OWNERS

@@ -0,0 +1,14 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- fuyu01
+labels:
+- job
+- job/live/push-search
+- live
+options:
+  no_parent_owners: true
+reviewers:
+- liuzhen
+- xiehaishen
+- yangbaibing

app/job/live/push-search/README.md

@@ -0,0 +1,12 @@
+# push-search-service
+
+# 项目简介
+1.
+
+# 编译环境
+
+
+# 依赖包
+
+
+# 编译执行

app/job/live/push-search/cmd/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["push-search-test.toml"],
+    importpath = "go-common/app/job/live/push-search/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//app/job/live/push-search/http:go_default_library",
+        "//app/job/live/push-search/service/migrate:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/cmd/main.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"flag"
+	"go-common/library/net/trace"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/live/push-search/conf"
+	"go-common/app/job/live/push-search/http"
+	"go-common/app/job/live/push-search/service/migrate"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("push-search-service start")
+	if len(os.Args[1:]) > 0 && os.Args[1:][0] == "migrate" {
+		roomId := os.Args[1:][1]
+		isTest := os.Args[1:][2]
+		ms := migrate.NewMigrateS(conf.Conf)
+		go ms.Migrate(roomId, isTest)
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+		for {
+			m := <-c
+			switch m {
+			case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+				ms.Close()
+				log.Info("push-search-service-migrate exit")
+				time.Sleep(time.Second)
+				return
+			case syscall.SIGHUP:
+			default:
+				return
+			}
+		}
+	}
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	ecode.Init(conf.Conf.Ecode)
+	http.Init(conf.Conf)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			http.Srv.Close()
+			log.Info("push-search-service exit")
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/live/push-search/cmd/push-search-test.toml

@@ -0,0 +1,97 @@
+[log]
+    stdout=true
+[group]
+    [group.RoomInfo]
+        num = 32
+        chan = 1024
+    [group.Attention]
+        num = 32
+        chan = 1024
+    [group.UserInfo]
+        num = 32
+        chan = 1024
+[databus]
+    [databus.PushSearch]
+        key = "ec4c0820d525d67b"
+        secret= "e20f8f664bf10722efeb6aac0cc16011"
+        group= "RoomUpdateToSearch-LiveLive-P"
+        topic= "RoomUpdateToSearch-T"
+        action="pub"
+        name = "live-search"
+        proto = "tcp"
+        addr = "172.18.33.50:6205"
+        idle = 100
+        active = 100
+        dialTimeout = "10s"
+        readTimeout = "40s"
+        writeTimeout = "10s"
+        idleTimeout = "60s"
+    [databus.RoomInfo]
+        key = "ec4c0820d525d67b"
+        secret= "e20f8f664bf10722efeb6aac0cc16011"
+        group= "ApRoomBinlog-LiveLive-S"
+        topic= "ApRoomBinlog-T"
+        action="sub"
+        name = "live-roomInfo-change"
+        proto = "tcp"
+        addr = "172.18.33.50:6205"
+        idle = 100
+        active = 100
+        dialTimeout = "10s"
+        readTimeout = "40s"
+        writeTimeout = "10s"
+        idleTimeout = "60s"
+    [databus.Attention]
+        key = "ec4c0820d525d67b"
+        secret= "e20f8f664bf10722efeb6aac0cc16011"
+        group= "LiveRelationChanged-LiveLive-S"
+        topic= "LiveRelationChanged-T"
+        action="sub"
+        name = "live-attention-change"
+        proto = "tcp"
+        addr = "172.18.33.50:6205"
+        idle = 100
+        active = 100
+        dialTimeout = "10s"
+        readTimeout = "40s"
+        writeTimeout = "10s"
+        idleTimeout = "60s"
+    [databus.UserName]
+        key = "ec4c0820d525d67b"
+        secret= "e20f8f664bf10722efeb6aac0cc16011"
+        group= "BannedUserSyn-LiveLive-Search-S"
+        topic= "BannedUserSyn-T"
+        action="sub"
+        name = "live-userName-change"
+        proto = "tcp"
+        addr = "172.18.33.50:6205"
+        idle = 100
+        active = 100
+        dialTimeout = "10s"
+        readTimeout = "40s"
+        writeTimeout = "10s"
+        idleTimeout = "60s"
+
+[SearchHBase]
+    master = ""
+    meta = ""
+    dialTimeout = "1s"
+    readTimeout = "2s"
+    readsTimeout = "5s"
+    writeTimeout = "2s"
+    writesTimeout = "5s"
+    [SearchHBase.zookeeper]
+        root = ""
+        addrs = ["172.18.33.131:2181","172.18.33.168:2181","172.18.33.169:2181"]
+        timeout = "30s"
+
+[liverpc]
+    [liverpc.room]
+        AppID = "live.room"
+        ConnTimeout = "50ms"
+    [liverpc.relation]
+        AppID = "live.relation"
+        ConnTimeout = "50ms"
+    [liverpc.user]
+        AppID = "live.user"
+        ConnTimeout = "50ms"

app/job/live/push-search/conf/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/live/push-search/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/rpc/liverpc:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/conf/conf.go

@@ -0,0 +1,126 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/trace"
+
+	"github.com/BurntSushi/toml"
+	"go-common/library/queue/databus"
+	"go-common/library/database/hbase.v2"
+	"go-common/library/net/rpc/liverpc"
+	xtime "go-common/library/time"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config .
+type Config struct {
+	Log      *log.Config
+	BM       *bm.ServerConfig
+	Verify   *verify.Config
+	Tracer   *trace.Config
+	Redis    *redis.Config
+	Memcache *memcache.Config
+	MySQL    *sql.Config
+	Ecode    *ecode.Config
+	LiveRpc  map[string]*liverpc.ClientConfig
+	//DataBus
+	DataBus     *DataBus
+	Group       *Group
+	SearchHBase *hbaseConf
+	MigrateNum	int
+}
+
+type DataBus struct {
+	RoomInfo *databus.Config
+	Attention *databus.Config
+	UserName *databus.Config
+	PushSearch *databus.Config
+}
+
+// Group group.
+type Group struct {
+	RoomInfo *GroupConf
+	Attention *GroupConf
+	UserInfo *GroupConf
+}
+
+// GroupConf group conf.
+type GroupConf struct {
+	Num    int
+	Chan   int
+}
+
+type hbaseConf struct {
+	hbase.Config
+	ReadTimeout   xtime.Duration
+	ReadsTimeout  xtime.Duration
+	WriteTimeout  xtime.Duration
+	WritesTimeout xtime.Duration
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/live/push-search/dao/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "dao.go",
+        "pub.go",
+    ],
+    importpath = "go-common/app/job/live/push-search/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//app/service/live/relation/api/liverpc:go_default_library",
+        "//app/service/live/room/api/liverpc:go_default_library",
+        "//app/service/live/user/api/liverpc:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/liverpc:go_default_library",
+        "//library/queue/databus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/live/push-search/dao/migrate:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/dao/api.go

@@ -0,0 +1,28 @@
+package dao
+
+import (
+	"go-common/app/job/live/push-search/conf"
+	userApi "go-common/app/service/live/user/api/liverpc"
+	relationApi "go-common/app/service/live/relation/api/liverpc"
+	roomApi "go-common/app/service/live/room/api/liverpc"
+	"go-common/library/net/rpc/liverpc"
+)
+
+var UserApi *userApi.Client
+var RelationApi *relationApi.Client
+var RoomApi *roomApi.Client
+
+// InitAPI init all service APIs
+func InitAPI() {
+	UserApi = userApi.New(getConf("user"))
+	RelationApi = relationApi.New(getConf("relation"))
+	RoomApi = roomApi.New(getConf("room"))
+}
+
+func getConf(appName string) *liverpc.ClientConfig {
+	c := conf.Conf.LiveRpc
+	if c != nil {
+		return c[appName]
+	}
+	return nil
+}

app/job/live/push-search/dao/dao.go

@@ -0,0 +1,46 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/live/push-search/conf"
+	"go-common/library/queue/databus"
+	"go-common/library/database/hbase.v2"
+)
+
+// Dao dao
+type Dao struct {
+	c                 *conf.Config
+	RoomInfoDataBus   *databus.Databus
+	AttentionDataBus  *databus.Databus
+	UserNameDataBus   *databus.Databus
+	PushSearchDataBus *databus.Databus
+	SearchHBase       *hbase.Client
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:                 c,
+		RoomInfoDataBus:   databus.New(c.DataBus.RoomInfo),
+		AttentionDataBus:  databus.New(c.DataBus.Attention),
+		UserNameDataBus:   databus.New(c.DataBus.UserName),
+		PushSearchDataBus: databus.New(c.DataBus.PushSearch),
+		SearchHBase:       hbase.NewClient(&c.SearchHBase.Config),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.RoomInfoDataBus.Close()
+	d.AttentionDataBus.Close()
+	d.UserNameDataBus.Close()
+	return
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return nil
+}

app/job/live/push-search/dao/migrate/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dao.go"],
+    importpath = "go-common/app/job/live/push-search/dao/migrate",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/sql:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/dao/migrate/dao.go

@@ -0,0 +1,37 @@
+package migrate
+
+import (
+	"context"
+	"go-common/app/job/live/push-search/conf"
+	"go-common/library/database/hbase.v2"
+	"go-common/library/database/sql"
+)
+
+// Dao dao
+type Dao struct {
+	c                 *conf.Config
+	SearchHBase       *hbase.Client
+	RoomDb			  *sql.DB
+}
+
+// New init mysql db
+func NewMigrate(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:           c,
+		SearchHBase: hbase.NewClient(&c.SearchHBase.Config),
+		RoomDb:      sql.NewMySQL(c.MySQL),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.RoomDb.Close()
+	return
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return nil
+}

app/job/live/push-search/dao/pub.go

@@ -0,0 +1,16 @@
+package dao
+
+import (
+	"context"
+	"go-common/library/log"
+	"strconv"
+)
+
+func (d *Dao) Pub(c context.Context, uid int64, msg interface{}) error {
+	key := strconv.FormatInt(uid, 10)
+	err := d.PushSearchDataBus.Send(c, key, msg)
+	if err != nil {
+		log.Error("pub wallet change failed uid:%d, msg:%+v", uid, msg)
+	}
+	return err
+}

app/job/live/push-search/http/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/live/push-search/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//app/job/live/push-search/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/http/http.go

@@ -0,0 +1,53 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/live/push-search/conf"
+	"go-common/app/job/live/push-search/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	Srv *service.Service
+	vfy *verify.Verify
+)
+
+// Init init
+func Init(c *conf.Config) {
+	Srv = service.New(c)
+	vfy = verify.New(c.Verify)
+	engine := bm.DefaultServer(c.BM)
+	route(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("bm Start error(%v)", err)
+		panic(err)
+	}
+}
+
+func route(e *bm.Engine) {
+	e.Ping(ping)
+	e.Register(register)
+	g := e.Group("/x/push-search")
+	{
+		g.GET("/start", vfy.Verify, howToStart)
+	}
+}
+
+func ping(c *bm.Context) {
+	if err := Srv.Ping(c); err != nil {
+		log.Error("ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}
+
+// example for http request handler
+func howToStart(c *bm.Context) {
+	c.String(0, "Golang 大法好 !!!")
+}

app/job/live/push-search/model/BUILD

@@ -0,0 +1,31 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "model.go",
+        "room_info.go",
+    ],
+    importpath = "go-common/app/job/live/push-search/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/model/model.go

@@ -0,0 +1 @@
+package model

app/job/live/push-search/model/room_info.go

@@ -0,0 +1,75 @@
+package model
+
+import (
+	"encoding/json"
+)
+
+// NotifyInfo notify info.
+type ApRoomNotifyInfo struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+type LiveDatabusAttention struct {
+	Topic	string `json:"topic"`
+	MsgId	string `json:"msg_id"`
+	MsgContent *AttentionNotifyInfo `json:"msg_content"`
+}
+
+// NotifyInfo notify info.
+type AttentionNotifyInfo struct {
+	Uid      int64  `json:"uid"`
+	UpUid    int64  `json:"up_uid"`
+	ExtInfo  *ExInfo `json:"ext_info"`
+}
+
+type ExInfo struct {
+	UpUidFans int `json:"up_uid_fans"`
+}
+
+type LiveDatabus struct {
+	Topic	string `json:"topic"`
+	MsgId	string `json:"msg_id"`
+	MsgContent string `json:"msg_content"`
+}
+
+type UnameNotifyInfo struct{
+	Uid int64 `json:"uid"`
+	Uname string `json:"uname"`
+	Identification int `json:"identification"`
+}
+
+type TableField struct {
+	RoomId         int    `json:"roomid"`
+	ShortId        int    `json:"short_id"`
+	Uid            int64  `json:"uid"`
+	UName          string `json:"uname"`
+	Area           int    `json:"area"`
+	Title          string `json:"title"`
+	Tag            string `json:"tags"`
+	MTime          string `json:"mtime"`
+	CTime          string `json:"ctime"`
+	TryTime        string `json:"try_time"`
+	Cover          string `json:"cover"`
+	UserCover      string `json:"user_cover"`
+	LockStatus     string    `json:"lock_status"`
+	HiddenStatus   string    `json:"hidden_status"`
+	Attentions     int    `json:"attentions"`
+	Online         int    `json:"online"`
+	LiveTime       string `json:"live_time"`
+	AreaV2Id       int    `json:"area_v2_id"`
+	AreaV2Name	   string `json:"area_v2_name"`
+	AreaV2ParentId int    `json:"area_v2_parent_id"`
+	Virtual        int    `json:"virtual"`
+	RoundStatus    int    `json:"round_status"`
+	OnFlag         int    `json:"on_flag"`
+}
+
+type DataMap struct {
+	Action string
+	Table  string
+	New    *TableField
+	Old    *TableField
+}

app/job/live/push-search/service/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "attention_notify.go",
+        "common.go",
+        "roominfo_notify.go",
+        "service.go",
+        "uname_notify.go",
+    ],
+    importpath = "go-common/app/job/live/push-search/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//app/job/live/push-search/dao:go_default_library",
+        "//app/job/live/push-search/model:go_default_library",
+        "//app/service/live/relation/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc/v2:go_default_library",
+        "//app/service/live/user/api/liverpc/v3:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/liverpc/context:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/live/push-search/service/migrate:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/service/attention_notify.go

@@ -0,0 +1,154 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"strconv"
+)
+
+const (
+	_retry = 3
+)
+
+func (s *Service) attentionNotifyConsumeProc() {
+	defer s.waiter.Done()
+	for {
+		msg, ok := <-s.dao.AttentionDataBus.Messages()
+		if !ok {
+			log.Error("attentionNotifyConsumeProc closed")
+			if err := s.dao.AttentionDataBus.Close(); err != nil {
+				log.Error("s.dao.AttentionDataBus.Close() error(%v)", err)
+			}
+			return
+		}
+
+		m := &message{data: msg}
+
+		p := new(model.LiveDatabusAttention)
+
+		if err := json.Unmarshal(msg.Value, p); err != nil {
+			msg.Commit()
+			log.Error("[AttentionDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+
+		if p.MsgContent == nil {
+			log.Error("[AttentionDataBus]attentionNotifyConsumeProc msg object msgContent is nil, msg:%+v", string(msg.Value))
+			return
+		}
+
+		m.object = p
+		s.attentionMergeChan[p.MsgContent.UpUid%int64(s.c.Group.Attention.Num)] <- m
+	}
+}
+
+func (s *Service) attentionNotifyHandleProc(c chan *message) {
+	defer s.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			log.Error("[AttentionDataBus]attentionNotifyHandleProc closed")
+			return
+		}
+		//先提交防止阻塞,关闭时等待任务执行完
+		msgData.data.Commit()
+
+		p, assertOk := msgData.object.(*model.LiveDatabusAttention)
+
+		if !assertOk {
+			log.Error("[AttentionDataBus]attentionNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
+			return
+		}
+
+		uid := p.MsgContent.UpUid
+		uName := ""
+		newMap := &model.TableField{}
+
+		wg := errgroup.Group{}
+		wg.Go(func() (err error) {
+			userInfo, err := s.getMultiUserInfo(uid)
+			if err == nil && userInfo != nil && userInfo.Uname != "" {
+				uName = userInfo.Uname
+			}
+			return
+		})
+
+		wg.Go(func() (err error) {
+			roomInfo, err := s.getBaseRoomInfo(uid)
+			if err == nil && roomInfo != nil {
+				newMap.RoomId = int(roomInfo.Roomid)
+				newMap.ShortId = int(roomInfo.ShortId)
+				newMap.Uid = roomInfo.Uid
+				newMap.UName = roomInfo.Uname
+				newMap.Area = int(roomInfo.Area)
+				newMap.Title = roomInfo.Title
+				newMap.Tag = roomInfo.Tags
+				newMap.TryTime = roomInfo.TryTime
+				newMap.Cover = roomInfo.Cover
+				newMap.UserCover = roomInfo.UserCover
+				newMap.LockStatus = roomInfo.LockStatus
+				newMap.HiddenStatus = roomInfo.HiddenStatus
+				newMap.Attentions = int(roomInfo.Attentions)
+				newMap.Online = int(roomInfo.Online)
+				newMap.LiveTime = roomInfo.LiveTime
+				newMap.AreaV2Id = int(roomInfo.AreaV2Id)
+				newMap.AreaV2ParentId = int(roomInfo.AreaV2ParentId)
+				newMap.Virtual = int(roomInfo.Virtual)
+				newMap.AreaV2Name = roomInfo.AreaV2Name
+				newMap.CTime = roomInfo.Ctime
+				newMap.MTime = roomInfo.Mtime
+				newMap.RoundStatus = int(roomInfo.RoundStatus)
+				newMap.OnFlag = int(roomInfo.OnFlag)
+			}
+			return
+		})
+
+		err := wg.Wait()
+		if err == nil && newMap.RoomId != 0 {
+			ret, retByte := s.generateSearchInfo("update", _tableArchive, newMap, nil)
+			if uName != "" {
+				ret["new"].(map[string]interface{})["uname"] = uName
+				retByte["uname"] = []byte(uName)
+			}
+			if p.MsgContent.ExtInfo != nil {
+				ret["new"].(map[string]interface{})["attentions"] = p.MsgContent.ExtInfo.UpUidFans
+				ret["new"].(map[string]interface{})["attention"] = p.MsgContent.ExtInfo.UpUidFans
+				retByte["attentions"] = []byte(strconv.Itoa(p.MsgContent.ExtInfo.UpUidFans))
+				retByte["attention"] = []byte(strconv.Itoa(p.MsgContent.ExtInfo.UpUidFans))
+			}
+			//构造假old
+			ret["old"].(map[string]interface{})["attention"] = 0
+			ret["old"].(map[string]interface{})["attentions"] = 0
+
+			wg := errgroup.Group{}
+			wg.Go(func() (err error) {
+				for i := 0; i < _retry; i++ {
+					hbaseErr := s.saveHBase(context.TODO(), s.rowKey(newMap.RoomId), retByte)
+					err = hbaseErr
+					if hbaseErr != nil {
+						continue
+					}
+					break
+				}
+				if err != nil {
+					log.Error("[AttentionDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Go(func() (err error) {
+				err = s.dao.Pub(context.TODO(), int64(newMap.RoomId), ret)
+				if err != nil {
+					log.Error("[AttentionDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Wait()
+			log.Info("[AttentionDataBus]success to handle, error(%v), msg:(%v)", err, ret)
+			continue
+		}
+		log.Error("[AttentionDataBus]fail to getData, error(%v),msg:(%v)", err, p)
+	}
+}

app/job/live/push-search/service/common.go

@@ -0,0 +1,344 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"go-common/app/job/live/push-search/dao"
+	"go-common/app/job/live/push-search/model"
+	relationV1 "go-common/app/service/live/relation/api/liverpc/v1"
+	roomV1 "go-common/app/service/live/room/api/liverpc/v1"
+	roomV2 "go-common/app/service/live/room/api/liverpc/v2"
+	userV3 "go-common/app/service/live/user/api/liverpc/v3"
+	accountApi "go-common/app/service/main/account/api"
+
+	"go-common/library/log"
+	rpccontext "go-common/library/net/rpc/liverpc/context"
+	"strconv"
+	"time"
+)
+
+var (
+	hbaseTable  = "live:PushSearch"
+	hbaseFamily = "search"
+	fields      = []string{
+		"roomid",
+		"short_id",
+		"uid",
+		"uname",
+		"area",
+		"title",
+		"tags",
+		"try_time",
+		"cover",
+		"user_cover",
+		"lock_status",
+		"hidden_status",
+		"attentions",
+		"online",
+		"live_time",
+		"area_v2_id",
+		"area_v2_parent_id",
+		"virtual",
+		"round_status",
+		"on_flag",
+		"area_v2_name",
+		"ctime",
+		"mtime",
+	}
+)
+
+func (s *Service) getBaseRoomInfo(uid int64) (roomInfo *roomV2.RoomGetByIdsResp_RoomInfo, err error) {
+
+	roomIdResp, err := dao.RoomApi.V2Room.RoomIdByUid(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV2.RoomRoomIdByUidReq{
+		Uid: uid,
+	})
+	if err != nil {
+		log.Error("[getBaseRoomInfo]RoomIdByUid rpc error, error:%+v", err)
+		return
+	}
+	if roomIdResp.Code != 0 {
+		log.Error("[getBaseRoomInfo]RoomIdByUid return error, code:%d, msg:%s", roomIdResp.Code, roomIdResp.Msg)
+		err = errors.New("getRoomId return error")
+		return
+	}
+	if roomIdResp.Data == nil {
+		log.Error("[getBaseRoomInfo]GetMultiple empty data")
+		err = errors.New("getRoomId empty error")
+		return
+	}
+
+	if roomIdResp.Data.RoomId == 0 {
+		log.Error("[getBaseRoomInfo]GetMultiple empty data")
+		err = errors.New("roomId not found error")
+		return
+	}
+
+	roomInfoResp := &roomV2.RoomGetByIdsResp{}
+	roomInfoResp, err = dao.RoomApi.V2Room.GetByIds(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV2.RoomGetByIdsReq{
+		Ids:    []int64{roomIdResp.Data.RoomId},
+		From:   "push-search",
+		Fields: fields,
+	})
+	if err != nil {
+		log.Error("[getBaseRoomInfo]GetByIds rpc error, error:%+v", err)
+		return
+	}
+	if roomInfoResp.Code != 0 {
+		log.Error("[getBaseRoomInfo]GetByIds return error, code:%d, msg:%s", roomInfoResp.Code, roomInfoResp.Msg)
+		err = errors.New("GetByIds return error")
+		return
+	}
+	if roomInfoResp.Data == nil {
+		log.Error("[getBaseRoomInfo]GetByIds empty data")
+		err = errors.New("GetByIds empty error")
+		return
+	}
+
+	info, ok := roomInfoResp.Data[roomIdResp.Data.RoomId]
+	if !ok {
+		log.Error("[getBaseRoomInfo]GetByIds not found")
+		err = errors.New("roomId not found error")
+		return
+	}
+
+	roomInfo = info
+	return
+}
+
+func (s *Service) getMultiUserInfo(uid int64) (userInfo *userV3.UserGetMultipleResp_Info, err error) {
+	userInfo = &userV3.UserGetMultipleResp_Info{}
+	pr, err := s.AccountClient.Profile3(context.TODO(), &accountApi.MidReq{Mid: uid})
+
+	if err != nil {
+		log.Error("[getMultiUserInfo]Profile3 rpc error, error:%+v", err)
+		return
+	}
+
+	if pr == nil {
+		log.Error("[getMultiUserInfo]Profile3 empty data")
+		err = errors.New("user empty error")
+		return
+	}
+
+	userInfo.Uid = uid
+	userInfo.Uname = pr.GetProfile().GetName()
+	userInfo.Face = pr.GetProfile().GetFace()
+
+	if userInfo.Uname != "" {
+		return
+	}
+
+	err = errors.New("user not found")
+	log.Error("[getMultiUserInfo]GetMultiple no user, data:%+v", userInfo)
+
+	return
+}
+
+func (s *Service) getFc(uid int64) (fc int, err error) {
+	fcResp := &relationV1.FeedGetUserFcResp{}
+	fcResp, err = dao.RelationApi.V1Feed.GetUserFc(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &relationV1.FeedGetUserFcReq{
+		Follow: uid,
+	})
+	if err != nil {
+		log.Error("[getFc]GetFc rpc error, error:%+v", err)
+		return
+	}
+	if fcResp.Code != 0 {
+		log.Error("[getFc]GetFc return error, code:%d, msg:%s", fcResp.Code, fcResp.Msg)
+		err = errors.New("fc return error")
+		return
+	}
+
+	if fcResp.Data == nil {
+		log.Error("[getFc]GetFc empty data")
+		err = errors.New("fc empty error")
+		return
+	}
+
+	fc = int(fcResp.Data.Fc)
+	return
+}
+
+func (s *Service) saveHBase(c context.Context, key string, columnInfo map[string][]byte) (err error) {
+	var ctx, cancel = context.WithTimeout(c, time.Duration(s.c.SearchHBase.WriteTimeout)*time.Millisecond)
+
+	defer cancel()
+
+	values := map[string]map[string][]byte{hbaseFamily: columnInfo}
+	if _, err = s.dao.SearchHBase.PutStr(ctx, hbaseTable, key, values); err != nil {
+		log.Error("SearchHBase.PutStr error(%v), table(%s), values(%+v)", err, hbaseTable, values)
+	}
+	return
+}
+
+func (s *Service) getLockStatus(lockStatus string) int {
+	status := 0
+	if lockStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+
+func (s *Service) getHiddenStatus(HiddenStatus string) int {
+	status := 0
+	if HiddenStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+
+//hbase key roomID md5
+func (s *Service) rowKey(roomId int) string {
+	key := fmt.Sprintf("%d_%d", roomId%10, roomId)
+	return key
+}
+
+func (s *Service) generateSearchInfo(action string, table string, new *model.TableField, old *model.TableField) (ret map[string]interface{}, retByte map[string][]byte) {
+	ret = make(map[string]interface{})
+	ret["action"] = action
+	ret["table"] = table
+	//搜索字段转换
+	newMap := make(map[string]interface{})
+	newMap["id"] = new.RoomId
+	newMap["short_id"] = new.ShortId
+	newMap["uid"] = new.Uid
+	newMap["uname"] = new.UName
+	newMap["category"] = new.Area
+	newMap["title"] = new.Title
+	newMap["tag"] = new.Tag
+	newMap["try_time"] = new.TryTime
+	newMap["cover"] = new.Cover
+	newMap["user_cover"] = new.UserCover
+	newMap["lock_status"] = s.getLockStatus(new.LockStatus)
+	newMap["hidden_status"] = s.getHiddenStatus(new.HiddenStatus)
+	newMap["attentions"] = new.Attentions
+	newMap["attention"] = new.Attentions
+	newMap["online"] = new.Online
+	newMap["live_time"] = new.LiveTime
+	newMap["area_v2_id"] = new.AreaV2Id
+	newMap["ord"] = new.AreaV2ParentId
+	newMap["arcrank"] = new.Virtual
+	newMap["lastupdate"] = s.getLastUpdate(new)
+	newMap["is_live"] = s.getLiveStatus(new)
+	newMap["s_category"] = new.AreaV2Name
+	ret["new"] = newMap
+	oldMap := make(map[string]interface{})
+	if old != nil {
+		oldMap["id"] = old.RoomId
+		oldMap["short_id"] = old.ShortId
+		oldMap["uid"] = old.Uid
+		oldMap["uname"] = old.UName
+		oldMap["category"] = old.Area
+		oldMap["title"] = old.Title
+		oldMap["tag"] = old.Tag
+		oldMap["try_time"] = old.TryTime
+		oldMap["cover"] = old.Cover
+		oldMap["user_cover"] = old.UserCover
+		oldMap["lock_status"] = s.getLockStatus(old.LockStatus)
+		oldMap["hidden_status"] = s.getHiddenStatus(old.HiddenStatus)
+		oldMap["attentions"] = old.Attentions
+		oldMap["attention"] = old.Attentions
+		oldMap["online"] = old.Online
+		oldMap["live_time"] = old.LiveTime
+		oldMap["area_v2_id"] = old.AreaV2Id
+		oldMap["area_v2_name"] = old.AreaV2Name
+		oldMap["ord"] = old.AreaV2ParentId
+		oldMap["arcrank"] = old.Virtual
+		oldMap["lastupdate"] = s.getLastUpdate(old)
+		oldMap["is_live"] = s.getLiveStatus(old)
+	}
+	if action != "insert" && old == nil {
+		oldMap["id"] = new.RoomId
+		oldMap["short_id"] = new.ShortId
+		oldMap["uid"] = new.Uid
+		oldMap["uname"] = new.UName
+		oldMap["category"] = new.Area
+		oldMap["title"] = new.Title
+		oldMap["tag"] = new.Tag
+		oldMap["try_time"] = new.TryTime
+		oldMap["cover"] = new.Cover
+		oldMap["user_cover"] = new.UserCover
+		oldMap["lock_status"] = s.getLockStatus(new.LockStatus)
+		oldMap["hidden_status"] = s.getHiddenStatus(new.HiddenStatus)
+		oldMap["attentions"] = new.Attentions
+		oldMap["attention"] = new.Attentions
+		oldMap["online"] = new.Online
+		oldMap["live_time"] = new.LiveTime
+		oldMap["area_v2_id"] = new.AreaV2Id
+		oldMap["area_v2_name"] = new.AreaV2Name
+		oldMap["ord"] = new.AreaV2ParentId
+		oldMap["arcrank"] = new.Virtual
+		oldMap["lastupdate"] = s.getLastUpdate(new)
+		oldMap["is_live"] = s.getLiveStatus(new)
+	}
+	ret["old"] = oldMap
+
+	newByteMap := make(map[string][]byte)
+	newByteMap["id"] = []byte(strconv.Itoa(new.RoomId))
+	newByteMap["short_id"] = []byte(strconv.Itoa(new.ShortId))
+	newByteMap["uid"] = []byte(strconv.FormatInt(new.Uid, 10))
+	newByteMap["uname"] = []byte(new.UName)
+	newByteMap["category"] = []byte(strconv.Itoa(new.Area))
+	newByteMap["title"] = []byte(new.Title)
+	newByteMap["tag"] = []byte(new.Tag)
+	newByteMap["try_time"] = []byte(new.TryTime)
+	newByteMap["cover"] = []byte(new.Cover)
+	newByteMap["user_cover"] = []byte(new.UserCover)
+	newByteMap["lock_status"] = []byte(strconv.Itoa(s.getLockStatus(new.LockStatus)))
+	newByteMap["hidden_status"] = []byte(strconv.Itoa(s.getHiddenStatus(new.HiddenStatus)))
+	newByteMap["attentions"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["attention"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["online"] = []byte(strconv.Itoa(new.Online))
+	newByteMap["live_time"] = []byte(new.LiveTime)
+	newByteMap["area_v2_id"] = []byte(strconv.Itoa(new.AreaV2Id))
+	newByteMap["ord"] = []byte(strconv.Itoa(new.AreaV2ParentId))
+	newByteMap["arcrank"] = []byte(strconv.Itoa(new.Virtual))
+	newByteMap["lastupdate"] = []byte(s.getLastUpdate(new))
+	newByteMap["is_live"] = []byte(strconv.Itoa(s.getLiveStatus(new)))
+	newByteMap["s_category"] = []byte(new.AreaV2Name)
+	return ret, newByteMap
+}
+
+//获取直播状态
+func (s *Service) getLiveStatus(roomInfo *model.TableField) int {
+	if roomInfo.LiveTime != "0000-00-00 00:00:00" {
+		return 1
+	}
+
+	if roomInfo.RoundStatus == 1 && roomInfo.OnFlag == 1 {
+		return 2
+	}
+
+	return 0
+}
+
+//获取房间最后更新时间
+func (s *Service) getLastUpdate(roomInfo *model.TableField) string {
+	if roomInfo.MTime != "0000-00-00 00:00:00" {
+		return roomInfo.MTime
+	}
+	return roomInfo.CTime
+}
+
+func (s *Service) getAreaV2Detail(areaV2Id int) (areaInfo *roomV1.AreaGetDetailResp_AreaInfo, err error) {
+	areaResp, err := dao.RoomApi.V1Area.GetDetail(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV1.AreaGetDetailReq{
+		Id: int64(areaV2Id),
+	})
+	if err != nil {
+		log.Error("[getAreaV2Detail]GetMultiple rpc error, error:%+v", err)
+		return
+	}
+	if areaResp.Code != 0 {
+		log.Error("[getAreaV2Detail]GetMultiple return error, code:%d, msg:%s", areaResp.Code, areaResp.Msg)
+		err = errors.New("user return error")
+		return
+	}
+	if areaResp.Data == nil {
+		log.Error("[getAreaV2Detail]GetMultiple empty data")
+		err = errors.New("area detail empty error")
+		return
+	}
+
+	return areaResp.Data, err
+}

app/job/live/push-search/service/migrate/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["service.go"],
+    importpath = "go-common/app/job/live/push-search/service/migrate",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//app/job/live/push-search/dao/migrate:go_default_library",
+        "//app/job/live/push-search/model:go_default_library",
+        "//library/database/sql:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/service/migrate/service.go

@@ -0,0 +1,243 @@
+package migrate
+
+import (
+	"context"
+	defaultsql "database/sql"
+	"fmt"
+	"go-common/app/job/live/push-search/conf"
+	"go-common/app/job/live/push-search/dao/migrate"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/database/sql"
+	"strconv"
+	"sync"
+	"time"
+)
+
+const _sql = "select roomid, short_id,uid,uname,area,title,tags, mtime,a.ctime,try_time,user_cover,a.lock_status,hidden_status,attentions,live_time,area_v2_id,area_v2_parent_id,b.name as area_v2_name, virtual,round_status,on_flag,online, cover from ap_room a left join ap_room_area_v2 b on a.area_v2_id=b.id where roomid > %d order by roomid asc limit 100 "
+
+const hbaseTable   = "live:PushSearch"
+const hbaseFamily  = "search"
+
+type message struct {
+	rowKey string
+	values map[string]map[string][]byte
+}
+
+type MService struct {
+	c                  *conf.Config
+	dao                *migrate.Dao
+	hChan			   []chan *message
+	waiterChan			*sync.WaitGroup
+	mainWaiter			*sync.WaitGroup
+}
+
+
+func NewMigrateS(c *conf.Config) (s *MService) {
+	s = &MService{
+		c:   c,
+		dao: migrate.NewMigrate(c),
+		hChan: make([]chan *message, c.MigrateNum),
+		waiterChan:  new(sync.WaitGroup),
+		mainWaiter:  new(sync.WaitGroup),
+	}
+
+	//ap room 表 binlog qps 高, hash roomId 并行
+	for i := 0; i < c.MigrateNum; i++ {
+		ch := make(chan *message, 1024)
+		s.hChan[i] = ch
+		go s.handle(ch)
+	}
+
+	return s
+}
+
+func (ms *MService) Migrate (roomid string, isTest string) {
+
+	id, err := strconv.Atoi(roomid)
+	if err != nil {
+		fmt.Println("roomid error")
+	}
+	ms.mainWaiter.Add(1)
+	defer ms.mainWaiter.Done()
+	var rows *sql.Rows
+	online := &defaultsql.NullInt64{}
+	cover := &defaultsql.NullString{}
+	areaV2Name := &defaultsql.NullString{}
+	for {
+		rows, err := ms.dao.RoomDb.Query(context.TODO(), fmt.Sprintf(_sql, id))
+		if err != nil {
+			fmt.Println("query error:%+v", err)
+			return
+		}
+		for rows.Next() {
+			r := new(model.TableField)
+			if err = rows.Scan(&r.RoomId, &r.ShortId, &r.Uid, &r.UName, &r.Area, &r.Title, &r.Tag, &r.MTime, &r.CTime, &r.TryTime, &r.UserCover, &r.LockStatus, &r.HiddenStatus, &r.Attentions, &r.LiveTime, &r.AreaV2Id, &r.AreaV2ParentId, areaV2Name, &r.Virtual, &r.RoundStatus, &r.OnFlag, online, cover); err != nil {
+				if !online.Valid {
+					r.Online = 0
+				}
+				if !cover.Valid {
+					r.Cover = ""
+				}
+			}
+			r.AreaV2Name = areaV2Name.String
+			r.Online = int(online.Int64)
+			r.Cover = cover.String
+
+			zijie := ms.generateSearchInfo(r)
+			if r.LiveTime != "0000-00-00 00:00:00" {
+				fmt.Println(r.RoomId, "jump live room")
+				continue
+			}
+			values := map[string]map[string][]byte{hbaseFamily: zijie}
+			rowKey := ms.rowKey(r.RoomId)
+			m := &message{
+				rowKey: rowKey,
+				values: values,
+			}
+			ms.hChan[r.RoomId % ms.c.MigrateNum] <- m
+			fmt.Println(r.RoomId)
+			if isTest == "1" {
+				return
+			}
+			id = r.RoomId
+		}
+	}
+	rows.Close()
+}
+
+func (ms *MService) handle(c chan *message) {
+	ms.waiterChan.Add(1)
+	defer ms.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			fmt.Println("close chan")
+			return
+		}
+		ms.dao.SearchHBase.PutStr(context.TODO(), hbaseTable, msgData.rowKey, msgData.values)
+	}
+}
+
+func (ms *MService) Close() {
+	ms.dao.Close()
+	ms.mainWaiter.Wait()
+	for _, ch := range ms.hChan {
+		close(ch)
+	}
+	ms.waiterChan.Wait()
+	ms.dao.SearchHBase.Close()
+}
+func (ms *MService) rowKey(roomId int) string{
+	key := fmt.Sprintf("%d_%d", roomId % 10, roomId)
+	return key
+}
+func (ms *MService) generateSearchInfo(new *model.TableField) (retByte map[string][]byte){
+	newByteMap := make(map[string][]byte)
+	newByteMap["id"] = []byte(strconv.Itoa(new.RoomId))
+	newByteMap["short_id"] = []byte(strconv.Itoa(new.ShortId))
+	newByteMap["uid"] = []byte(strconv.FormatInt(new.Uid, 10))
+	newByteMap["uname"] = []byte(new.UName)
+	newByteMap["category"] = []byte(strconv.Itoa(new.Area))
+	newByteMap["title"] = []byte(new.Title)
+	newByteMap["tag"] = []byte(new.Tag)
+
+	tryTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.TryTime, time.Local)
+	tryTimeStr := tryTime.Format("2006-01-02 15:04:05")
+	if tryTimeStr == "0001-01-01 00:00:00" {
+		tryTimeStr = "0000-00-00 00:00:00"
+		new.TryTime = tryTimeStr
+	}
+	newByteMap["try_time"] = []byte(tryTimeStr)
+
+	newByteMap["cover"] = []byte(new.Cover)
+	newByteMap["user_cover"] = []byte(new.UserCover)
+
+	lockStatus, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.LockStatus, time.Local)
+	lockStatusStr := lockStatus.Format("2006-01-02 15:04:05")
+	if lockStatusStr == "0001-01-01 00:00:00" {
+		lockStatusStr = "0000-00-00 00:00:00"
+		new.LockStatus = lockStatusStr
+	}
+	newByteMap["lock_status"] = []byte(strconv.Itoa(ms.getLockStatus(lockStatusStr)))
+
+	hiddenStatus, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.HiddenStatus, time.Local)
+	hiddenStatusStr := hiddenStatus.Format("2006-01-02 15:04:05")
+	if hiddenStatusStr == "0001-01-01 00:00:00" {
+		hiddenStatusStr = "0000-00-00 00:00:00"
+		new.HiddenStatus = hiddenStatusStr
+	}
+	newByteMap["hidden_status"] = []byte(strconv.Itoa(ms.getHiddenStatus(hiddenStatusStr)))
+
+	newByteMap["attentions"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["attention"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["online"] = []byte(strconv.Itoa(new.Online))
+
+	liveTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.LiveTime, time.Local)
+	liveTimeStr := liveTime.Format("2006-01-02 15:04:05")
+	if liveTimeStr == "0001-01-01 00:00:00" {
+		liveTimeStr = "0000-00-00 00:00:00"
+		new.LiveTime = liveTimeStr
+	}
+	newByteMap["live_time"] = []byte(liveTimeStr)
+
+	newByteMap["area_v2_id"] = []byte(strconv.Itoa(new.AreaV2Id))
+	newByteMap["ord"] = []byte(strconv.Itoa(new.AreaV2ParentId))
+	newByteMap["arcrank"] = []byte(strconv.Itoa(new.Virtual))
+
+	cTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.CTime, time.Local)
+	cTimeStr := cTime.Format("2006-01-02 15:04:05")
+	if cTimeStr == "0001-01-01 00:00:00" {
+		new.CTime = "0000-00-00 00:00:00"
+	}else{
+		new.CTime = cTimeStr
+	}
+	mTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.MTime, time.Local)
+	mTimeStr := mTime.Format("2006-01-02 15:04:05")
+	if mTimeStr == "0001-01-01 00:00:00" {
+		new.MTime = "0000-00-00 00:00:00"
+	}else{
+		new.MTime = mTimeStr
+	}
+	newByteMap["lastupdate"] = []byte(ms.getLastUpdate(new))
+	newByteMap["is_live"] = []byte(strconv.Itoa(ms.getLiveStatus(new)))
+	newByteMap["s_category"] = []byte(new.AreaV2Name)
+	return newByteMap
+}
+
+//获取直播状态
+func (ms *MService) getLiveStatus(roomInfo *model.TableField) int{
+	if roomInfo.LiveTime != "0000-00-00 00:00:00" {
+		return 1
+	}
+
+	if roomInfo.RoundStatus == 1 && roomInfo.OnFlag == 1{
+		return 2
+	}
+
+	return 0
+}
+
+//获取房间最后更新时间
+func (ms *MService) getLastUpdate(roomInfo *model.TableField) string{
+	if roomInfo.MTime != "0000-00-00 00:00:00" {
+		return roomInfo.MTime
+	}
+	return roomInfo.CTime
+}
+
+func (ms *MService) getLockStatus(lockStatus string) int{
+	status := 0
+	if lockStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+
+func (ms *MService) getHiddenStatus(HiddenStatus string) int{
+	status := 0
+	if HiddenStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+

app/job/live/push-search/service/roominfo_notify.go

@@ -0,0 +1,194 @@
+package service
+
+import (
+	"encoding/json"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/log"
+
+	"context"
+	roomV1 "go-common/app/service/live/room/api/liverpc/v1"
+	"go-common/library/sync/errgroup"
+	"strconv"
+)
+
+const (
+	_updateAct = "update"
+	_insertAct = "insert"
+)
+
+func (s *Service) roomInfoNotifyConsumeProc() {
+	defer s.waiter.Done()
+	for {
+		msg, ok := <-s.dao.RoomInfoDataBus.Messages()
+		// databus关闭chan导致,服务自杀或异常退出
+		if !ok {
+			log.Error("roomInfoNotifyConsumeProc closed")
+			if err := s.dao.RoomInfoDataBus.Close(); err != nil {
+				log.Error("s.dao.RoomInfoDataBus.Close() error(%v)", err)
+			}
+			return
+		}
+
+		m := &message{data: msg}
+		p := new(model.ApRoomNotifyInfo)
+
+		if err := json.Unmarshal(msg.Value, p); err != nil {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+
+		if p.Action != _insertAct && p.Action != _updateAct {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]Action Invalid error(%v)", p.Action)
+			continue
+		}
+
+		//判断是否是关注or昵称变更,如果是则跳过,顺便解出新旧map
+		isAttentionUpdate, oldMap, newMap, err := isAttentionChange(p.Action, p.Old, p.New)
+		if err != nil {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]isAttentionChange,json.Unmarshal(old:%s, new:%s) error(%v)", string(p.Old), string(p.New), err)
+			continue
+		}
+		if isAttentionUpdate {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]attention change pass")
+			continue
+		}
+
+		//hash chan
+
+		if newMap == nil || newMap.RoomId <= 0 {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]roomId type conversion error, roomId:%+v", newMap)
+			continue
+		}
+
+		dataMap := new(model.DataMap)
+		dataMap.Action = p.Action
+		dataMap.Table = p.Table
+		dataMap.New = newMap
+		dataMap.Old = oldMap
+		m.object = dataMap
+
+		log.Info("[RoomInfoDataBus]roomInfoNotifyConsumeProc key:%s partition:%d offset:%d", msg.Key, msg.Partition, msg.Offset)
+
+		s.binLogMergeChan[newMap.RoomId%s.c.Group.RoomInfo.Num] <- m
+	}
+}
+
+func isAttentionChange(action string, old []byte, new []byte) (bool, *model.TableField, *model.TableField, error) {
+	newMap := &model.TableField{}
+	oldMap := &model.TableField{}
+	err := json.Unmarshal(new, newMap)
+	if err != nil {
+		return false, oldMap, newMap, err
+	}
+
+	if action == _updateAct {
+		err := json.Unmarshal(old, oldMap)
+		if err != nil {
+			return false, oldMap, newMap, err
+		}
+		if oldMap != nil && oldMap.Attentions != newMap.Attentions {
+			return true, oldMap, newMap, err
+		}
+	}
+
+	if action == _insertAct {
+		oldMap = nil
+	}
+
+	return false, oldMap, newMap, err
+}
+
+func (s *Service) roomInfoNotifyHandleProc(c chan *message) {
+	defer s.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			log.Error("[RoomInfoDataBus]roomInfoNotifyHandleProc closed")
+			return
+		}
+
+		msgData.data.Commit()
+
+		p, assertOk := msgData.object.(*model.DataMap)
+
+		if !assertOk {
+			log.Error("[RoomInfoDataBus]roomInfoNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
+			return
+		}
+
+		uid := p.New.Uid
+
+		wg := errgroup.Group{}
+		uName := ""
+		fc := 0
+		areaInfo := &roomV1.AreaGetDetailResp_AreaInfo{}
+
+		wg.Go(func() (err error) {
+			userInfo, err := s.getMultiUserInfo(uid)
+			if err == nil && userInfo != nil && userInfo.Uname != "" {
+				uName = userInfo.Uname
+			}
+			return
+		})
+
+		//fc任何错误都要返回,不然fc为0无法判断是接口返回0还是初始化的0!!!!
+		wg.Go(func() (err error) {
+			fc, err = s.getFc(uid)
+			return
+		})
+
+		wg.Go(func() (err error) {
+			areaInfo, err = s.getAreaV2Detail(p.New.AreaV2Id)
+			return
+		})
+
+		err := wg.Wait()
+
+		//成功返回则替换,否则输出原数据
+		ret, retByte := s.generateSearchInfo(p.Action, p.Table, p.New, p.Old)
+		if err == nil {
+			if uName != "" {
+				ret["new"].(map[string]interface{})["uname"] = uName
+				retByte["uname"] = []byte(uName)
+			}
+			if areaInfo != nil && areaInfo.Name != "" {
+				ret["new"].(map[string]interface{})["s_category"] = areaInfo.Name
+				retByte["s_category"] = []byte(areaInfo.Name)
+			}
+			ret["new"].(map[string]interface{})["attentions"] = fc
+			ret["new"].(map[string]interface{})["attention"] = fc
+			retByte["attentions"] = []byte(strconv.Itoa(fc))
+			retByte["attention"] = []byte(strconv.Itoa(fc))
+		}
+		writeWg := errgroup.Group{}
+		writeWg.Go(func() (err error) {
+			for i := 0; i < _retry; i++ {
+				hbaseErr := s.saveHBase(context.TODO(), s.rowKey(p.New.RoomId), retByte)
+				err = hbaseErr
+				if hbaseErr != nil {
+					continue
+				}
+				break
+			}
+			if err != nil {
+				log.Error("[RoomInfoDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
+			}
+			return
+		})
+		writeWg.Go(func() (err error) {
+			err = s.dao.Pub(context.TODO(), int64(p.New.RoomId), ret)
+			if err != nil {
+				log.Error("[RoomInfoDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
+			}
+			return
+		})
+		wg.Wait()
+		log.Info("[RoomInfoDataBus]success handle, error(%v),msg:(%v)", err, ret)
+
+	}
+}

app/job/live/push-search/service/service.go

@@ -0,0 +1,109 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/live/push-search/conf"
+	"go-common/app/job/live/push-search/dao"
+	accountApi "go-common/app/service/main/account/api"
+
+	"go-common/library/queue/databus"
+	"sync"
+)
+
+const (
+	_tableArchive = "ap_room"
+)
+
+// Service struct
+type Service struct {
+	c                  *conf.Config
+	dao                *dao.Dao
+	binLogMergeChan    []chan *message
+	attentionMergeChan []chan *message
+	unameMergeChan     []chan *message
+	waiter             *sync.WaitGroup
+	waiterChan         *sync.WaitGroup
+	AccountClient      accountApi.AccountClient
+}
+
+type message struct {
+	next   *message
+	data   *databus.Message
+	object interface{}
+	done   bool
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	dao.InitAPI()
+	s = &Service{
+		c:                  c,
+		dao:                dao.New(c),
+		binLogMergeChan:    make([]chan *message, c.Group.RoomInfo.Num),
+		attentionMergeChan: make([]chan *message, c.Group.Attention.Num),
+		unameMergeChan:     make([]chan *message, c.Group.UserInfo.Num),
+		waiter:             new(sync.WaitGroup),
+		waiterChan:         new(sync.WaitGroup),
+	}
+	accountClient, err := accountApi.NewClient(nil)
+	if err != nil {
+		panic(err)
+	}
+	s.AccountClient = accountClient
+
+	//ap room 表 binlog qps 高, hash roomId 并行
+	for i := 0; i < c.Group.RoomInfo.Num; i++ {
+		ch := make(chan *message, c.Group.RoomInfo.Chan)
+		s.binLogMergeChan[i] = ch
+		s.waiterChan.Add(1)
+		go s.roomInfoNotifyHandleProc(ch)
+	}
+
+	for i := 0; i < c.Group.Attention.Num; i++ {
+		ch := make(chan *message, c.Group.Attention.Chan)
+		s.attentionMergeChan[i] = ch
+		s.waiterChan.Add(1)
+		go s.attentionNotifyHandleProc(ch)
+	}
+
+	for i := 0; i < c.Group.UserInfo.Num; i++ {
+		ch := make(chan *message, c.Group.UserInfo.Chan)
+		s.unameMergeChan[i] = ch
+		s.waiterChan.Add(1)
+		go s.unameNotifyHandleProc(ch)
+	}
+	s.waiter.Add(1)
+	go s.roomInfoNotifyConsumeProc()
+	s.waiter.Add(1)
+	go s.attentionNotifyConsumeProc()
+	s.waiter.Add(1)
+	go s.unameNotifyConsumeProc()
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	//databus chan close
+	s.dao.Close()
+	s.waiter.Wait()
+	//task goroutine close
+	for _, ch := range s.binLogMergeChan {
+		close(ch)
+	}
+
+	for _, ch := range s.attentionMergeChan {
+		close(ch)
+	}
+
+	for _, ch := range s.unameMergeChan {
+		close(ch)
+	}
+	s.waiterChan.Wait()
+	s.dao.PushSearchDataBus.Close()
+}

app/job/live/push-search/service/uname_notify.go

@@ -0,0 +1,155 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"strconv"
+)
+
+func (s *Service) unameNotifyConsumeProc() {
+	defer s.waiter.Done()
+	for {
+		msg, ok := <-s.dao.UserNameDataBus.Messages()
+		if !ok {
+			log.Error("unameNotifyConsumeProc closed")
+			if err := s.dao.UserNameDataBus.Close(); err != nil {
+				log.Error("s.dao.UserNameDataBus.Close() error(%v)", err)
+			}
+			return
+		}
+		//先提交防止阻塞,关闭时等待任务执行完
+		m := &message{data: msg}
+		raw := new(model.LiveDatabus)
+
+		if err := json.Unmarshal(msg.Value, raw); err != nil {
+			msg.Commit()
+			log.Error("[UnameDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+
+		p := new(model.UnameNotifyInfo)
+		if err := json.Unmarshal([]byte(raw.MsgContent), p); err != nil {
+			msg.Commit()
+			log.Error("[UnameDataBus]json.Unmarshal(%s) error(%v)", raw.MsgContent, err)
+			continue
+		}
+
+		m.object = p
+		s.unameMergeChan[p.Uid%int64(s.c.Group.UserInfo.Num)] <- m
+	}
+}
+
+func (s *Service) unameNotifyHandleProc(c chan *message) {
+	defer s.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			log.Error("[UnameDataBus]unameNotifyHandleProc closed")
+			return
+		}
+		//先提交防止阻塞,关闭时等待任务执行完
+		msgData.data.Commit()
+
+		p, assertOk := msgData.object.(*model.UnameNotifyInfo)
+
+		if !assertOk {
+			log.Error("[UnameDataBus]unameNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
+			return
+		}
+
+		uid := p.Uid
+
+		if uid == 0 {
+			log.Error("[UnameDataBus]empty uid, uid:%d", uid)
+			continue
+		}
+		fc := 0
+		newMap := &model.TableField{}
+
+		wg := errgroup.Group{}
+		wg.Go(func() (err error) {
+			fc, err = s.getFc(uid)
+			return
+		})
+
+		wg.Go(func() (err error) {
+			roomInfo, err := s.getBaseRoomInfo(uid)
+			if err == nil && roomInfo != nil {
+				newMap.RoomId = int(roomInfo.Roomid)
+				newMap.ShortId = int(roomInfo.ShortId)
+				newMap.Uid = roomInfo.Uid
+				newMap.UName = roomInfo.Uname
+				newMap.Area = int(roomInfo.Area)
+				newMap.Title = roomInfo.Title
+				newMap.Tag = roomInfo.Tags
+				newMap.TryTime = roomInfo.TryTime
+				newMap.Cover = roomInfo.Cover
+				newMap.UserCover = roomInfo.UserCover
+				newMap.LockStatus = roomInfo.LockStatus
+				newMap.HiddenStatus = roomInfo.HiddenStatus
+				newMap.Attentions = int(roomInfo.Attentions)
+				newMap.Online = int(roomInfo.Online)
+				newMap.LiveTime = roomInfo.LiveTime
+				newMap.AreaV2Id = int(roomInfo.AreaV2Id)
+				newMap.AreaV2ParentId = int(roomInfo.AreaV2ParentId)
+				newMap.Virtual = int(roomInfo.Virtual)
+				newMap.AreaV2Name = roomInfo.AreaV2Name
+				newMap.CTime = roomInfo.Ctime
+				newMap.MTime = roomInfo.Mtime
+				newMap.RoundStatus = int(roomInfo.RoundStatus)
+				newMap.OnFlag = int(roomInfo.OnFlag)
+			}
+			return
+		})
+
+		err := wg.Wait()
+
+		if err == nil && newMap.RoomId != 0 {
+			//非uname更新
+			if p.Uname == newMap.UName {
+				log.Info("[UnameDataBus]uname no change, msg:(%v)", p)
+				continue
+			}
+			ret, retByte := s.generateSearchInfo("update", _tableArchive, newMap, nil)
+			if p.Uname != "" {
+				ret["new"].(map[string]interface{})["uname"] = p.Uname
+				retByte["uname"] = []byte(p.Uname)
+			}
+			ret["new"].(map[string]interface{})["attentions"] = fc
+			ret["new"].(map[string]interface{})["attention"] = fc
+			retByte["attentions"] = []byte(strconv.Itoa(fc))
+			retByte["attention"] = []byte(strconv.Itoa(fc))
+			ret["old"].(map[string]interface{})["uname"] = ""
+
+			wg := errgroup.Group{}
+			wg.Go(func() (err error) {
+				for i := 0; i < _retry; i++ {
+					hbaseErr := s.saveHBase(context.TODO(), s.rowKey(newMap.RoomId), retByte)
+					err = hbaseErr
+					if hbaseErr != nil {
+						continue
+					}
+					break
+				}
+				if err != nil {
+					log.Error("[UnameDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Go(func() (err error) {
+				err = s.dao.Pub(context.TODO(), int64(newMap.RoomId), ret)
+				if err != nil {
+					log.Error("[UnameDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Wait()
+			log.Info("[UnameDataBus]success to handle, error(%v), msg:(%v)", err, ret)
+			continue
+		}
+		log.Error("[UnameDataBus]fail to getData, error(%v),msg:(%v)", err, p)
+	}
+}