Create & Init Project...

app/job/live/push-search/service/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "attention_notify.go",
+        "common.go",
+        "roominfo_notify.go",
+        "service.go",
+        "uname_notify.go",
+    ],
+    importpath = "go-common/app/job/live/push-search/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//app/job/live/push-search/dao:go_default_library",
+        "//app/job/live/push-search/model:go_default_library",
+        "//app/service/live/relation/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc/v1:go_default_library",
+        "//app/service/live/room/api/liverpc/v2:go_default_library",
+        "//app/service/live/user/api/liverpc/v3:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/liverpc/context:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/live/push-search/service/migrate:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/service/attention_notify.go

@@ -0,0 +1,154 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"strconv"
+)
+
+const (
+	_retry = 3
+)
+
+func (s *Service) attentionNotifyConsumeProc() {
+	defer s.waiter.Done()
+	for {
+		msg, ok := <-s.dao.AttentionDataBus.Messages()
+		if !ok {
+			log.Error("attentionNotifyConsumeProc closed")
+			if err := s.dao.AttentionDataBus.Close(); err != nil {
+				log.Error("s.dao.AttentionDataBus.Close() error(%v)", err)
+			}
+			return
+		}
+
+		m := &message{data: msg}
+
+		p := new(model.LiveDatabusAttention)
+
+		if err := json.Unmarshal(msg.Value, p); err != nil {
+			msg.Commit()
+			log.Error("[AttentionDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+
+		if p.MsgContent == nil {
+			log.Error("[AttentionDataBus]attentionNotifyConsumeProc msg object msgContent is nil, msg:%+v", string(msg.Value))
+			return
+		}
+
+		m.object = p
+		s.attentionMergeChan[p.MsgContent.UpUid%int64(s.c.Group.Attention.Num)] <- m
+	}
+}
+
+func (s *Service) attentionNotifyHandleProc(c chan *message) {
+	defer s.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			log.Error("[AttentionDataBus]attentionNotifyHandleProc closed")
+			return
+		}
+		//先提交防止阻塞,关闭时等待任务执行完
+		msgData.data.Commit()
+
+		p, assertOk := msgData.object.(*model.LiveDatabusAttention)
+
+		if !assertOk {
+			log.Error("[AttentionDataBus]attentionNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
+			return
+		}
+
+		uid := p.MsgContent.UpUid
+		uName := ""
+		newMap := &model.TableField{}
+
+		wg := errgroup.Group{}
+		wg.Go(func() (err error) {
+			userInfo, err := s.getMultiUserInfo(uid)
+			if err == nil && userInfo != nil && userInfo.Uname != "" {
+				uName = userInfo.Uname
+			}
+			return
+		})
+
+		wg.Go(func() (err error) {
+			roomInfo, err := s.getBaseRoomInfo(uid)
+			if err == nil && roomInfo != nil {
+				newMap.RoomId = int(roomInfo.Roomid)
+				newMap.ShortId = int(roomInfo.ShortId)
+				newMap.Uid = roomInfo.Uid
+				newMap.UName = roomInfo.Uname
+				newMap.Area = int(roomInfo.Area)
+				newMap.Title = roomInfo.Title
+				newMap.Tag = roomInfo.Tags
+				newMap.TryTime = roomInfo.TryTime
+				newMap.Cover = roomInfo.Cover
+				newMap.UserCover = roomInfo.UserCover
+				newMap.LockStatus = roomInfo.LockStatus
+				newMap.HiddenStatus = roomInfo.HiddenStatus
+				newMap.Attentions = int(roomInfo.Attentions)
+				newMap.Online = int(roomInfo.Online)
+				newMap.LiveTime = roomInfo.LiveTime
+				newMap.AreaV2Id = int(roomInfo.AreaV2Id)
+				newMap.AreaV2ParentId = int(roomInfo.AreaV2ParentId)
+				newMap.Virtual = int(roomInfo.Virtual)
+				newMap.AreaV2Name = roomInfo.AreaV2Name
+				newMap.CTime = roomInfo.Ctime
+				newMap.MTime = roomInfo.Mtime
+				newMap.RoundStatus = int(roomInfo.RoundStatus)
+				newMap.OnFlag = int(roomInfo.OnFlag)
+			}
+			return
+		})
+
+		err := wg.Wait()
+		if err == nil && newMap.RoomId != 0 {
+			ret, retByte := s.generateSearchInfo("update", _tableArchive, newMap, nil)
+			if uName != "" {
+				ret["new"].(map[string]interface{})["uname"] = uName
+				retByte["uname"] = []byte(uName)
+			}
+			if p.MsgContent.ExtInfo != nil {
+				ret["new"].(map[string]interface{})["attentions"] = p.MsgContent.ExtInfo.UpUidFans
+				ret["new"].(map[string]interface{})["attention"] = p.MsgContent.ExtInfo.UpUidFans
+				retByte["attentions"] = []byte(strconv.Itoa(p.MsgContent.ExtInfo.UpUidFans))
+				retByte["attention"] = []byte(strconv.Itoa(p.MsgContent.ExtInfo.UpUidFans))
+			}
+			//构造假old
+			ret["old"].(map[string]interface{})["attention"] = 0
+			ret["old"].(map[string]interface{})["attentions"] = 0
+
+			wg := errgroup.Group{}
+			wg.Go(func() (err error) {
+				for i := 0; i < _retry; i++ {
+					hbaseErr := s.saveHBase(context.TODO(), s.rowKey(newMap.RoomId), retByte)
+					err = hbaseErr
+					if hbaseErr != nil {
+						continue
+					}
+					break
+				}
+				if err != nil {
+					log.Error("[AttentionDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Go(func() (err error) {
+				err = s.dao.Pub(context.TODO(), int64(newMap.RoomId), ret)
+				if err != nil {
+					log.Error("[AttentionDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Wait()
+			log.Info("[AttentionDataBus]success to handle, error(%v), msg:(%v)", err, ret)
+			continue
+		}
+		log.Error("[AttentionDataBus]fail to getData, error(%v),msg:(%v)", err, p)
+	}
+}

app/job/live/push-search/service/common.go

@@ -0,0 +1,344 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"go-common/app/job/live/push-search/dao"
+	"go-common/app/job/live/push-search/model"
+	relationV1 "go-common/app/service/live/relation/api/liverpc/v1"
+	roomV1 "go-common/app/service/live/room/api/liverpc/v1"
+	roomV2 "go-common/app/service/live/room/api/liverpc/v2"
+	userV3 "go-common/app/service/live/user/api/liverpc/v3"
+	accountApi "go-common/app/service/main/account/api"
+
+	"go-common/library/log"
+	rpccontext "go-common/library/net/rpc/liverpc/context"
+	"strconv"
+	"time"
+)
+
+var (
+	hbaseTable  = "live:PushSearch"
+	hbaseFamily = "search"
+	fields      = []string{
+		"roomid",
+		"short_id",
+		"uid",
+		"uname",
+		"area",
+		"title",
+		"tags",
+		"try_time",
+		"cover",
+		"user_cover",
+		"lock_status",
+		"hidden_status",
+		"attentions",
+		"online",
+		"live_time",
+		"area_v2_id",
+		"area_v2_parent_id",
+		"virtual",
+		"round_status",
+		"on_flag",
+		"area_v2_name",
+		"ctime",
+		"mtime",
+	}
+)
+
+func (s *Service) getBaseRoomInfo(uid int64) (roomInfo *roomV2.RoomGetByIdsResp_RoomInfo, err error) {
+
+	roomIdResp, err := dao.RoomApi.V2Room.RoomIdByUid(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV2.RoomRoomIdByUidReq{
+		Uid: uid,
+	})
+	if err != nil {
+		log.Error("[getBaseRoomInfo]RoomIdByUid rpc error, error:%+v", err)
+		return
+	}
+	if roomIdResp.Code != 0 {
+		log.Error("[getBaseRoomInfo]RoomIdByUid return error, code:%d, msg:%s", roomIdResp.Code, roomIdResp.Msg)
+		err = errors.New("getRoomId return error")
+		return
+	}
+	if roomIdResp.Data == nil {
+		log.Error("[getBaseRoomInfo]GetMultiple empty data")
+		err = errors.New("getRoomId empty error")
+		return
+	}
+
+	if roomIdResp.Data.RoomId == 0 {
+		log.Error("[getBaseRoomInfo]GetMultiple empty data")
+		err = errors.New("roomId not found error")
+		return
+	}
+
+	roomInfoResp := &roomV2.RoomGetByIdsResp{}
+	roomInfoResp, err = dao.RoomApi.V2Room.GetByIds(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV2.RoomGetByIdsReq{
+		Ids:    []int64{roomIdResp.Data.RoomId},
+		From:   "push-search",
+		Fields: fields,
+	})
+	if err != nil {
+		log.Error("[getBaseRoomInfo]GetByIds rpc error, error:%+v", err)
+		return
+	}
+	if roomInfoResp.Code != 0 {
+		log.Error("[getBaseRoomInfo]GetByIds return error, code:%d, msg:%s", roomInfoResp.Code, roomInfoResp.Msg)
+		err = errors.New("GetByIds return error")
+		return
+	}
+	if roomInfoResp.Data == nil {
+		log.Error("[getBaseRoomInfo]GetByIds empty data")
+		err = errors.New("GetByIds empty error")
+		return
+	}
+
+	info, ok := roomInfoResp.Data[roomIdResp.Data.RoomId]
+	if !ok {
+		log.Error("[getBaseRoomInfo]GetByIds not found")
+		err = errors.New("roomId not found error")
+		return
+	}
+
+	roomInfo = info
+	return
+}
+
+func (s *Service) getMultiUserInfo(uid int64) (userInfo *userV3.UserGetMultipleResp_Info, err error) {
+	userInfo = &userV3.UserGetMultipleResp_Info{}
+	pr, err := s.AccountClient.Profile3(context.TODO(), &accountApi.MidReq{Mid: uid})
+
+	if err != nil {
+		log.Error("[getMultiUserInfo]Profile3 rpc error, error:%+v", err)
+		return
+	}
+
+	if pr == nil {
+		log.Error("[getMultiUserInfo]Profile3 empty data")
+		err = errors.New("user empty error")
+		return
+	}
+
+	userInfo.Uid = uid
+	userInfo.Uname = pr.GetProfile().GetName()
+	userInfo.Face = pr.GetProfile().GetFace()
+
+	if userInfo.Uname != "" {
+		return
+	}
+
+	err = errors.New("user not found")
+	log.Error("[getMultiUserInfo]GetMultiple no user, data:%+v", userInfo)
+
+	return
+}
+
+func (s *Service) getFc(uid int64) (fc int, err error) {
+	fcResp := &relationV1.FeedGetUserFcResp{}
+	fcResp, err = dao.RelationApi.V1Feed.GetUserFc(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &relationV1.FeedGetUserFcReq{
+		Follow: uid,
+	})
+	if err != nil {
+		log.Error("[getFc]GetFc rpc error, error:%+v", err)
+		return
+	}
+	if fcResp.Code != 0 {
+		log.Error("[getFc]GetFc return error, code:%d, msg:%s", fcResp.Code, fcResp.Msg)
+		err = errors.New("fc return error")
+		return
+	}
+
+	if fcResp.Data == nil {
+		log.Error("[getFc]GetFc empty data")
+		err = errors.New("fc empty error")
+		return
+	}
+
+	fc = int(fcResp.Data.Fc)
+	return
+}
+
+func (s *Service) saveHBase(c context.Context, key string, columnInfo map[string][]byte) (err error) {
+	var ctx, cancel = context.WithTimeout(c, time.Duration(s.c.SearchHBase.WriteTimeout)*time.Millisecond)
+
+	defer cancel()
+
+	values := map[string]map[string][]byte{hbaseFamily: columnInfo}
+	if _, err = s.dao.SearchHBase.PutStr(ctx, hbaseTable, key, values); err != nil {
+		log.Error("SearchHBase.PutStr error(%v), table(%s), values(%+v)", err, hbaseTable, values)
+	}
+	return
+}
+
+func (s *Service) getLockStatus(lockStatus string) int {
+	status := 0
+	if lockStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+
+func (s *Service) getHiddenStatus(HiddenStatus string) int {
+	status := 0
+	if HiddenStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+
+//hbase key roomID md5
+func (s *Service) rowKey(roomId int) string {
+	key := fmt.Sprintf("%d_%d", roomId%10, roomId)
+	return key
+}
+
+func (s *Service) generateSearchInfo(action string, table string, new *model.TableField, old *model.TableField) (ret map[string]interface{}, retByte map[string][]byte) {
+	ret = make(map[string]interface{})
+	ret["action"] = action
+	ret["table"] = table
+	//搜索字段转换
+	newMap := make(map[string]interface{})
+	newMap["id"] = new.RoomId
+	newMap["short_id"] = new.ShortId
+	newMap["uid"] = new.Uid
+	newMap["uname"] = new.UName
+	newMap["category"] = new.Area
+	newMap["title"] = new.Title
+	newMap["tag"] = new.Tag
+	newMap["try_time"] = new.TryTime
+	newMap["cover"] = new.Cover
+	newMap["user_cover"] = new.UserCover
+	newMap["lock_status"] = s.getLockStatus(new.LockStatus)
+	newMap["hidden_status"] = s.getHiddenStatus(new.HiddenStatus)
+	newMap["attentions"] = new.Attentions
+	newMap["attention"] = new.Attentions
+	newMap["online"] = new.Online
+	newMap["live_time"] = new.LiveTime
+	newMap["area_v2_id"] = new.AreaV2Id
+	newMap["ord"] = new.AreaV2ParentId
+	newMap["arcrank"] = new.Virtual
+	newMap["lastupdate"] = s.getLastUpdate(new)
+	newMap["is_live"] = s.getLiveStatus(new)
+	newMap["s_category"] = new.AreaV2Name
+	ret["new"] = newMap
+	oldMap := make(map[string]interface{})
+	if old != nil {
+		oldMap["id"] = old.RoomId
+		oldMap["short_id"] = old.ShortId
+		oldMap["uid"] = old.Uid
+		oldMap["uname"] = old.UName
+		oldMap["category"] = old.Area
+		oldMap["title"] = old.Title
+		oldMap["tag"] = old.Tag
+		oldMap["try_time"] = old.TryTime
+		oldMap["cover"] = old.Cover
+		oldMap["user_cover"] = old.UserCover
+		oldMap["lock_status"] = s.getLockStatus(old.LockStatus)
+		oldMap["hidden_status"] = s.getHiddenStatus(old.HiddenStatus)
+		oldMap["attentions"] = old.Attentions
+		oldMap["attention"] = old.Attentions
+		oldMap["online"] = old.Online
+		oldMap["live_time"] = old.LiveTime
+		oldMap["area_v2_id"] = old.AreaV2Id
+		oldMap["area_v2_name"] = old.AreaV2Name
+		oldMap["ord"] = old.AreaV2ParentId
+		oldMap["arcrank"] = old.Virtual
+		oldMap["lastupdate"] = s.getLastUpdate(old)
+		oldMap["is_live"] = s.getLiveStatus(old)
+	}
+	if action != "insert" && old == nil {
+		oldMap["id"] = new.RoomId
+		oldMap["short_id"] = new.ShortId
+		oldMap["uid"] = new.Uid
+		oldMap["uname"] = new.UName
+		oldMap["category"] = new.Area
+		oldMap["title"] = new.Title
+		oldMap["tag"] = new.Tag
+		oldMap["try_time"] = new.TryTime
+		oldMap["cover"] = new.Cover
+		oldMap["user_cover"] = new.UserCover
+		oldMap["lock_status"] = s.getLockStatus(new.LockStatus)
+		oldMap["hidden_status"] = s.getHiddenStatus(new.HiddenStatus)
+		oldMap["attentions"] = new.Attentions
+		oldMap["attention"] = new.Attentions
+		oldMap["online"] = new.Online
+		oldMap["live_time"] = new.LiveTime
+		oldMap["area_v2_id"] = new.AreaV2Id
+		oldMap["area_v2_name"] = new.AreaV2Name
+		oldMap["ord"] = new.AreaV2ParentId
+		oldMap["arcrank"] = new.Virtual
+		oldMap["lastupdate"] = s.getLastUpdate(new)
+		oldMap["is_live"] = s.getLiveStatus(new)
+	}
+	ret["old"] = oldMap
+
+	newByteMap := make(map[string][]byte)
+	newByteMap["id"] = []byte(strconv.Itoa(new.RoomId))
+	newByteMap["short_id"] = []byte(strconv.Itoa(new.ShortId))
+	newByteMap["uid"] = []byte(strconv.FormatInt(new.Uid, 10))
+	newByteMap["uname"] = []byte(new.UName)
+	newByteMap["category"] = []byte(strconv.Itoa(new.Area))
+	newByteMap["title"] = []byte(new.Title)
+	newByteMap["tag"] = []byte(new.Tag)
+	newByteMap["try_time"] = []byte(new.TryTime)
+	newByteMap["cover"] = []byte(new.Cover)
+	newByteMap["user_cover"] = []byte(new.UserCover)
+	newByteMap["lock_status"] = []byte(strconv.Itoa(s.getLockStatus(new.LockStatus)))
+	newByteMap["hidden_status"] = []byte(strconv.Itoa(s.getHiddenStatus(new.HiddenStatus)))
+	newByteMap["attentions"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["attention"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["online"] = []byte(strconv.Itoa(new.Online))
+	newByteMap["live_time"] = []byte(new.LiveTime)
+	newByteMap["area_v2_id"] = []byte(strconv.Itoa(new.AreaV2Id))
+	newByteMap["ord"] = []byte(strconv.Itoa(new.AreaV2ParentId))
+	newByteMap["arcrank"] = []byte(strconv.Itoa(new.Virtual))
+	newByteMap["lastupdate"] = []byte(s.getLastUpdate(new))
+	newByteMap["is_live"] = []byte(strconv.Itoa(s.getLiveStatus(new)))
+	newByteMap["s_category"] = []byte(new.AreaV2Name)
+	return ret, newByteMap
+}
+
+//获取直播状态
+func (s *Service) getLiveStatus(roomInfo *model.TableField) int {
+	if roomInfo.LiveTime != "0000-00-00 00:00:00" {
+		return 1
+	}
+
+	if roomInfo.RoundStatus == 1 && roomInfo.OnFlag == 1 {
+		return 2
+	}
+
+	return 0
+}
+
+//获取房间最后更新时间
+func (s *Service) getLastUpdate(roomInfo *model.TableField) string {
+	if roomInfo.MTime != "0000-00-00 00:00:00" {
+		return roomInfo.MTime
+	}
+	return roomInfo.CTime
+}
+
+func (s *Service) getAreaV2Detail(areaV2Id int) (areaInfo *roomV1.AreaGetDetailResp_AreaInfo, err error) {
+	areaResp, err := dao.RoomApi.V1Area.GetDetail(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV1.AreaGetDetailReq{
+		Id: int64(areaV2Id),
+	})
+	if err != nil {
+		log.Error("[getAreaV2Detail]GetMultiple rpc error, error:%+v", err)
+		return
+	}
+	if areaResp.Code != 0 {
+		log.Error("[getAreaV2Detail]GetMultiple return error, code:%d, msg:%s", areaResp.Code, areaResp.Msg)
+		err = errors.New("user return error")
+		return
+	}
+	if areaResp.Data == nil {
+		log.Error("[getAreaV2Detail]GetMultiple empty data")
+		err = errors.New("area detail empty error")
+		return
+	}
+
+	return areaResp.Data, err
+}

app/job/live/push-search/service/migrate/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["service.go"],
+    importpath = "go-common/app/job/live/push-search/service/migrate",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/live/push-search/conf:go_default_library",
+        "//app/job/live/push-search/dao/migrate:go_default_library",
+        "//app/job/live/push-search/model:go_default_library",
+        "//library/database/sql:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/live/push-search/service/migrate/service.go

@@ -0,0 +1,243 @@
+package migrate
+
+import (
+	"context"
+	defaultsql "database/sql"
+	"fmt"
+	"go-common/app/job/live/push-search/conf"
+	"go-common/app/job/live/push-search/dao/migrate"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/database/sql"
+	"strconv"
+	"sync"
+	"time"
+)
+
+const _sql = "select roomid, short_id,uid,uname,area,title,tags, mtime,a.ctime,try_time,user_cover,a.lock_status,hidden_status,attentions,live_time,area_v2_id,area_v2_parent_id,b.name as area_v2_name, virtual,round_status,on_flag,online, cover from ap_room a left join ap_room_area_v2 b on a.area_v2_id=b.id where roomid > %d order by roomid asc limit 100 "
+
+const hbaseTable   = "live:PushSearch"
+const hbaseFamily  = "search"
+
+type message struct {
+	rowKey string
+	values map[string]map[string][]byte
+}
+
+type MService struct {
+	c                  *conf.Config
+	dao                *migrate.Dao
+	hChan			   []chan *message
+	waiterChan			*sync.WaitGroup
+	mainWaiter			*sync.WaitGroup
+}
+
+
+func NewMigrateS(c *conf.Config) (s *MService) {
+	s = &MService{
+		c:   c,
+		dao: migrate.NewMigrate(c),
+		hChan: make([]chan *message, c.MigrateNum),
+		waiterChan:  new(sync.WaitGroup),
+		mainWaiter:  new(sync.WaitGroup),
+	}
+
+	//ap room 表 binlog qps 高, hash roomId 并行
+	for i := 0; i < c.MigrateNum; i++ {
+		ch := make(chan *message, 1024)
+		s.hChan[i] = ch
+		go s.handle(ch)
+	}
+
+	return s
+}
+
+func (ms *MService) Migrate (roomid string, isTest string) {
+
+	id, err := strconv.Atoi(roomid)
+	if err != nil {
+		fmt.Println("roomid error")
+	}
+	ms.mainWaiter.Add(1)
+	defer ms.mainWaiter.Done()
+	var rows *sql.Rows
+	online := &defaultsql.NullInt64{}
+	cover := &defaultsql.NullString{}
+	areaV2Name := &defaultsql.NullString{}
+	for {
+		rows, err := ms.dao.RoomDb.Query(context.TODO(), fmt.Sprintf(_sql, id))
+		if err != nil {
+			fmt.Println("query error:%+v", err)
+			return
+		}
+		for rows.Next() {
+			r := new(model.TableField)
+			if err = rows.Scan(&r.RoomId, &r.ShortId, &r.Uid, &r.UName, &r.Area, &r.Title, &r.Tag, &r.MTime, &r.CTime, &r.TryTime, &r.UserCover, &r.LockStatus, &r.HiddenStatus, &r.Attentions, &r.LiveTime, &r.AreaV2Id, &r.AreaV2ParentId, areaV2Name, &r.Virtual, &r.RoundStatus, &r.OnFlag, online, cover); err != nil {
+				if !online.Valid {
+					r.Online = 0
+				}
+				if !cover.Valid {
+					r.Cover = ""
+				}
+			}
+			r.AreaV2Name = areaV2Name.String
+			r.Online = int(online.Int64)
+			r.Cover = cover.String
+
+			zijie := ms.generateSearchInfo(r)
+			if r.LiveTime != "0000-00-00 00:00:00" {
+				fmt.Println(r.RoomId, "jump live room")
+				continue
+			}
+			values := map[string]map[string][]byte{hbaseFamily: zijie}
+			rowKey := ms.rowKey(r.RoomId)
+			m := &message{
+				rowKey: rowKey,
+				values: values,
+			}
+			ms.hChan[r.RoomId % ms.c.MigrateNum] <- m
+			fmt.Println(r.RoomId)
+			if isTest == "1" {
+				return
+			}
+			id = r.RoomId
+		}
+	}
+	rows.Close()
+}
+
+func (ms *MService) handle(c chan *message) {
+	ms.waiterChan.Add(1)
+	defer ms.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			fmt.Println("close chan")
+			return
+		}
+		ms.dao.SearchHBase.PutStr(context.TODO(), hbaseTable, msgData.rowKey, msgData.values)
+	}
+}
+
+func (ms *MService) Close() {
+	ms.dao.Close()
+	ms.mainWaiter.Wait()
+	for _, ch := range ms.hChan {
+		close(ch)
+	}
+	ms.waiterChan.Wait()
+	ms.dao.SearchHBase.Close()
+}
+func (ms *MService) rowKey(roomId int) string{
+	key := fmt.Sprintf("%d_%d", roomId % 10, roomId)
+	return key
+}
+func (ms *MService) generateSearchInfo(new *model.TableField) (retByte map[string][]byte){
+	newByteMap := make(map[string][]byte)
+	newByteMap["id"] = []byte(strconv.Itoa(new.RoomId))
+	newByteMap["short_id"] = []byte(strconv.Itoa(new.ShortId))
+	newByteMap["uid"] = []byte(strconv.FormatInt(new.Uid, 10))
+	newByteMap["uname"] = []byte(new.UName)
+	newByteMap["category"] = []byte(strconv.Itoa(new.Area))
+	newByteMap["title"] = []byte(new.Title)
+	newByteMap["tag"] = []byte(new.Tag)
+
+	tryTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.TryTime, time.Local)
+	tryTimeStr := tryTime.Format("2006-01-02 15:04:05")
+	if tryTimeStr == "0001-01-01 00:00:00" {
+		tryTimeStr = "0000-00-00 00:00:00"
+		new.TryTime = tryTimeStr
+	}
+	newByteMap["try_time"] = []byte(tryTimeStr)
+
+	newByteMap["cover"] = []byte(new.Cover)
+	newByteMap["user_cover"] = []byte(new.UserCover)
+
+	lockStatus, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.LockStatus, time.Local)
+	lockStatusStr := lockStatus.Format("2006-01-02 15:04:05")
+	if lockStatusStr == "0001-01-01 00:00:00" {
+		lockStatusStr = "0000-00-00 00:00:00"
+		new.LockStatus = lockStatusStr
+	}
+	newByteMap["lock_status"] = []byte(strconv.Itoa(ms.getLockStatus(lockStatusStr)))
+
+	hiddenStatus, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.HiddenStatus, time.Local)
+	hiddenStatusStr := hiddenStatus.Format("2006-01-02 15:04:05")
+	if hiddenStatusStr == "0001-01-01 00:00:00" {
+		hiddenStatusStr = "0000-00-00 00:00:00"
+		new.HiddenStatus = hiddenStatusStr
+	}
+	newByteMap["hidden_status"] = []byte(strconv.Itoa(ms.getHiddenStatus(hiddenStatusStr)))
+
+	newByteMap["attentions"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["attention"] = []byte(strconv.Itoa(new.Attentions))
+	newByteMap["online"] = []byte(strconv.Itoa(new.Online))
+
+	liveTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.LiveTime, time.Local)
+	liveTimeStr := liveTime.Format("2006-01-02 15:04:05")
+	if liveTimeStr == "0001-01-01 00:00:00" {
+		liveTimeStr = "0000-00-00 00:00:00"
+		new.LiveTime = liveTimeStr
+	}
+	newByteMap["live_time"] = []byte(liveTimeStr)
+
+	newByteMap["area_v2_id"] = []byte(strconv.Itoa(new.AreaV2Id))
+	newByteMap["ord"] = []byte(strconv.Itoa(new.AreaV2ParentId))
+	newByteMap["arcrank"] = []byte(strconv.Itoa(new.Virtual))
+
+	cTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.CTime, time.Local)
+	cTimeStr := cTime.Format("2006-01-02 15:04:05")
+	if cTimeStr == "0001-01-01 00:00:00" {
+		new.CTime = "0000-00-00 00:00:00"
+	}else{
+		new.CTime = cTimeStr
+	}
+	mTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.MTime, time.Local)
+	mTimeStr := mTime.Format("2006-01-02 15:04:05")
+	if mTimeStr == "0001-01-01 00:00:00" {
+		new.MTime = "0000-00-00 00:00:00"
+	}else{
+		new.MTime = mTimeStr
+	}
+	newByteMap["lastupdate"] = []byte(ms.getLastUpdate(new))
+	newByteMap["is_live"] = []byte(strconv.Itoa(ms.getLiveStatus(new)))
+	newByteMap["s_category"] = []byte(new.AreaV2Name)
+	return newByteMap
+}
+
+//获取直播状态
+func (ms *MService) getLiveStatus(roomInfo *model.TableField) int{
+	if roomInfo.LiveTime != "0000-00-00 00:00:00" {
+		return 1
+	}
+
+	if roomInfo.RoundStatus == 1 && roomInfo.OnFlag == 1{
+		return 2
+	}
+
+	return 0
+}
+
+//获取房间最后更新时间
+func (ms *MService) getLastUpdate(roomInfo *model.TableField) string{
+	if roomInfo.MTime != "0000-00-00 00:00:00" {
+		return roomInfo.MTime
+	}
+	return roomInfo.CTime
+}
+
+func (ms *MService) getLockStatus(lockStatus string) int{
+	status := 0
+	if lockStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+
+func (ms *MService) getHiddenStatus(HiddenStatus string) int{
+	status := 0
+	if HiddenStatus != "0000-00-00 00:00:00" {
+		status = 1
+	}
+	return status
+}
+

app/job/live/push-search/service/roominfo_notify.go

@@ -0,0 +1,194 @@
+package service
+
+import (
+	"encoding/json"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/log"
+
+	"context"
+	roomV1 "go-common/app/service/live/room/api/liverpc/v1"
+	"go-common/library/sync/errgroup"
+	"strconv"
+)
+
+const (
+	_updateAct = "update"
+	_insertAct = "insert"
+)
+
+func (s *Service) roomInfoNotifyConsumeProc() {
+	defer s.waiter.Done()
+	for {
+		msg, ok := <-s.dao.RoomInfoDataBus.Messages()
+		// databus关闭chan导致,服务自杀或异常退出
+		if !ok {
+			log.Error("roomInfoNotifyConsumeProc closed")
+			if err := s.dao.RoomInfoDataBus.Close(); err != nil {
+				log.Error("s.dao.RoomInfoDataBus.Close() error(%v)", err)
+			}
+			return
+		}
+
+		m := &message{data: msg}
+		p := new(model.ApRoomNotifyInfo)
+
+		if err := json.Unmarshal(msg.Value, p); err != nil {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+
+		if p.Action != _insertAct && p.Action != _updateAct {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]Action Invalid error(%v)", p.Action)
+			continue
+		}
+
+		//判断是否是关注or昵称变更,如果是则跳过,顺便解出新旧map
+		isAttentionUpdate, oldMap, newMap, err := isAttentionChange(p.Action, p.Old, p.New)
+		if err != nil {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]isAttentionChange,json.Unmarshal(old:%s, new:%s) error(%v)", string(p.Old), string(p.New), err)
+			continue
+		}
+		if isAttentionUpdate {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]attention change pass")
+			continue
+		}
+
+		//hash chan
+
+		if newMap == nil || newMap.RoomId <= 0 {
+			msg.Commit()
+			log.Error("[RoomInfoDataBus]roomId type conversion error, roomId:%+v", newMap)
+			continue
+		}
+
+		dataMap := new(model.DataMap)
+		dataMap.Action = p.Action
+		dataMap.Table = p.Table
+		dataMap.New = newMap
+		dataMap.Old = oldMap
+		m.object = dataMap
+
+		log.Info("[RoomInfoDataBus]roomInfoNotifyConsumeProc key:%s partition:%d offset:%d", msg.Key, msg.Partition, msg.Offset)
+
+		s.binLogMergeChan[newMap.RoomId%s.c.Group.RoomInfo.Num] <- m
+	}
+}
+
+func isAttentionChange(action string, old []byte, new []byte) (bool, *model.TableField, *model.TableField, error) {
+	newMap := &model.TableField{}
+	oldMap := &model.TableField{}
+	err := json.Unmarshal(new, newMap)
+	if err != nil {
+		return false, oldMap, newMap, err
+	}
+
+	if action == _updateAct {
+		err := json.Unmarshal(old, oldMap)
+		if err != nil {
+			return false, oldMap, newMap, err
+		}
+		if oldMap != nil && oldMap.Attentions != newMap.Attentions {
+			return true, oldMap, newMap, err
+		}
+	}
+
+	if action == _insertAct {
+		oldMap = nil
+	}
+
+	return false, oldMap, newMap, err
+}
+
+func (s *Service) roomInfoNotifyHandleProc(c chan *message) {
+	defer s.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			log.Error("[RoomInfoDataBus]roomInfoNotifyHandleProc closed")
+			return
+		}
+
+		msgData.data.Commit()
+
+		p, assertOk := msgData.object.(*model.DataMap)
+
+		if !assertOk {
+			log.Error("[RoomInfoDataBus]roomInfoNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
+			return
+		}
+
+		uid := p.New.Uid
+
+		wg := errgroup.Group{}
+		uName := ""
+		fc := 0
+		areaInfo := &roomV1.AreaGetDetailResp_AreaInfo{}
+
+		wg.Go(func() (err error) {
+			userInfo, err := s.getMultiUserInfo(uid)
+			if err == nil && userInfo != nil && userInfo.Uname != "" {
+				uName = userInfo.Uname
+			}
+			return
+		})
+
+		//fc任何错误都要返回,不然fc为0无法判断是接口返回0还是初始化的0!!!!
+		wg.Go(func() (err error) {
+			fc, err = s.getFc(uid)
+			return
+		})
+
+		wg.Go(func() (err error) {
+			areaInfo, err = s.getAreaV2Detail(p.New.AreaV2Id)
+			return
+		})
+
+		err := wg.Wait()
+
+		//成功返回则替换,否则输出原数据
+		ret, retByte := s.generateSearchInfo(p.Action, p.Table, p.New, p.Old)
+		if err == nil {
+			if uName != "" {
+				ret["new"].(map[string]interface{})["uname"] = uName
+				retByte["uname"] = []byte(uName)
+			}
+			if areaInfo != nil && areaInfo.Name != "" {
+				ret["new"].(map[string]interface{})["s_category"] = areaInfo.Name
+				retByte["s_category"] = []byte(areaInfo.Name)
+			}
+			ret["new"].(map[string]interface{})["attentions"] = fc
+			ret["new"].(map[string]interface{})["attention"] = fc
+			retByte["attentions"] = []byte(strconv.Itoa(fc))
+			retByte["attention"] = []byte(strconv.Itoa(fc))
+		}
+		writeWg := errgroup.Group{}
+		writeWg.Go(func() (err error) {
+			for i := 0; i < _retry; i++ {
+				hbaseErr := s.saveHBase(context.TODO(), s.rowKey(p.New.RoomId), retByte)
+				err = hbaseErr
+				if hbaseErr != nil {
+					continue
+				}
+				break
+			}
+			if err != nil {
+				log.Error("[RoomInfoDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
+			}
+			return
+		})
+		writeWg.Go(func() (err error) {
+			err = s.dao.Pub(context.TODO(), int64(p.New.RoomId), ret)
+			if err != nil {
+				log.Error("[RoomInfoDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
+			}
+			return
+		})
+		wg.Wait()
+		log.Info("[RoomInfoDataBus]success handle, error(%v),msg:(%v)", err, ret)
+
+	}
+}

app/job/live/push-search/service/service.go

@@ -0,0 +1,109 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/live/push-search/conf"
+	"go-common/app/job/live/push-search/dao"
+	accountApi "go-common/app/service/main/account/api"
+
+	"go-common/library/queue/databus"
+	"sync"
+)
+
+const (
+	_tableArchive = "ap_room"
+)
+
+// Service struct
+type Service struct {
+	c                  *conf.Config
+	dao                *dao.Dao
+	binLogMergeChan    []chan *message
+	attentionMergeChan []chan *message
+	unameMergeChan     []chan *message
+	waiter             *sync.WaitGroup
+	waiterChan         *sync.WaitGroup
+	AccountClient      accountApi.AccountClient
+}
+
+type message struct {
+	next   *message
+	data   *databus.Message
+	object interface{}
+	done   bool
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	dao.InitAPI()
+	s = &Service{
+		c:                  c,
+		dao:                dao.New(c),
+		binLogMergeChan:    make([]chan *message, c.Group.RoomInfo.Num),
+		attentionMergeChan: make([]chan *message, c.Group.Attention.Num),
+		unameMergeChan:     make([]chan *message, c.Group.UserInfo.Num),
+		waiter:             new(sync.WaitGroup),
+		waiterChan:         new(sync.WaitGroup),
+	}
+	accountClient, err := accountApi.NewClient(nil)
+	if err != nil {
+		panic(err)
+	}
+	s.AccountClient = accountClient
+
+	//ap room 表 binlog qps 高, hash roomId 并行
+	for i := 0; i < c.Group.RoomInfo.Num; i++ {
+		ch := make(chan *message, c.Group.RoomInfo.Chan)
+		s.binLogMergeChan[i] = ch
+		s.waiterChan.Add(1)
+		go s.roomInfoNotifyHandleProc(ch)
+	}
+
+	for i := 0; i < c.Group.Attention.Num; i++ {
+		ch := make(chan *message, c.Group.Attention.Chan)
+		s.attentionMergeChan[i] = ch
+		s.waiterChan.Add(1)
+		go s.attentionNotifyHandleProc(ch)
+	}
+
+	for i := 0; i < c.Group.UserInfo.Num; i++ {
+		ch := make(chan *message, c.Group.UserInfo.Chan)
+		s.unameMergeChan[i] = ch
+		s.waiterChan.Add(1)
+		go s.unameNotifyHandleProc(ch)
+	}
+	s.waiter.Add(1)
+	go s.roomInfoNotifyConsumeProc()
+	s.waiter.Add(1)
+	go s.attentionNotifyConsumeProc()
+	s.waiter.Add(1)
+	go s.unameNotifyConsumeProc()
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	//databus chan close
+	s.dao.Close()
+	s.waiter.Wait()
+	//task goroutine close
+	for _, ch := range s.binLogMergeChan {
+		close(ch)
+	}
+
+	for _, ch := range s.attentionMergeChan {
+		close(ch)
+	}
+
+	for _, ch := range s.unameMergeChan {
+		close(ch)
+	}
+	s.waiterChan.Wait()
+	s.dao.PushSearchDataBus.Close()
+}

app/job/live/push-search/service/uname_notify.go

@@ -0,0 +1,155 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"go-common/app/job/live/push-search/model"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"strconv"
+)
+
+func (s *Service) unameNotifyConsumeProc() {
+	defer s.waiter.Done()
+	for {
+		msg, ok := <-s.dao.UserNameDataBus.Messages()
+		if !ok {
+			log.Error("unameNotifyConsumeProc closed")
+			if err := s.dao.UserNameDataBus.Close(); err != nil {
+				log.Error("s.dao.UserNameDataBus.Close() error(%v)", err)
+			}
+			return
+		}
+		//先提交防止阻塞,关闭时等待任务执行完
+		m := &message{data: msg}
+		raw := new(model.LiveDatabus)
+
+		if err := json.Unmarshal(msg.Value, raw); err != nil {
+			msg.Commit()
+			log.Error("[UnameDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+
+		p := new(model.UnameNotifyInfo)
+		if err := json.Unmarshal([]byte(raw.MsgContent), p); err != nil {
+			msg.Commit()
+			log.Error("[UnameDataBus]json.Unmarshal(%s) error(%v)", raw.MsgContent, err)
+			continue
+		}
+
+		m.object = p
+		s.unameMergeChan[p.Uid%int64(s.c.Group.UserInfo.Num)] <- m
+	}
+}
+
+func (s *Service) unameNotifyHandleProc(c chan *message) {
+	defer s.waiterChan.Done()
+	for {
+		msgData, ok := <-c
+		if !ok {
+			log.Error("[UnameDataBus]unameNotifyHandleProc closed")
+			return
+		}
+		//先提交防止阻塞,关闭时等待任务执行完
+		msgData.data.Commit()
+
+		p, assertOk := msgData.object.(*model.UnameNotifyInfo)
+
+		if !assertOk {
+			log.Error("[UnameDataBus]unameNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
+			return
+		}
+
+		uid := p.Uid
+
+		if uid == 0 {
+			log.Error("[UnameDataBus]empty uid, uid:%d", uid)
+			continue
+		}
+		fc := 0
+		newMap := &model.TableField{}
+
+		wg := errgroup.Group{}
+		wg.Go(func() (err error) {
+			fc, err = s.getFc(uid)
+			return
+		})
+
+		wg.Go(func() (err error) {
+			roomInfo, err := s.getBaseRoomInfo(uid)
+			if err == nil && roomInfo != nil {
+				newMap.RoomId = int(roomInfo.Roomid)
+				newMap.ShortId = int(roomInfo.ShortId)
+				newMap.Uid = roomInfo.Uid
+				newMap.UName = roomInfo.Uname
+				newMap.Area = int(roomInfo.Area)
+				newMap.Title = roomInfo.Title
+				newMap.Tag = roomInfo.Tags
+				newMap.TryTime = roomInfo.TryTime
+				newMap.Cover = roomInfo.Cover
+				newMap.UserCover = roomInfo.UserCover
+				newMap.LockStatus = roomInfo.LockStatus
+				newMap.HiddenStatus = roomInfo.HiddenStatus
+				newMap.Attentions = int(roomInfo.Attentions)
+				newMap.Online = int(roomInfo.Online)
+				newMap.LiveTime = roomInfo.LiveTime
+				newMap.AreaV2Id = int(roomInfo.AreaV2Id)
+				newMap.AreaV2ParentId = int(roomInfo.AreaV2ParentId)
+				newMap.Virtual = int(roomInfo.Virtual)
+				newMap.AreaV2Name = roomInfo.AreaV2Name
+				newMap.CTime = roomInfo.Ctime
+				newMap.MTime = roomInfo.Mtime
+				newMap.RoundStatus = int(roomInfo.RoundStatus)
+				newMap.OnFlag = int(roomInfo.OnFlag)
+			}
+			return
+		})
+
+		err := wg.Wait()
+
+		if err == nil && newMap.RoomId != 0 {
+			//非uname更新
+			if p.Uname == newMap.UName {
+				log.Info("[UnameDataBus]uname no change, msg:(%v)", p)
+				continue
+			}
+			ret, retByte := s.generateSearchInfo("update", _tableArchive, newMap, nil)
+			if p.Uname != "" {
+				ret["new"].(map[string]interface{})["uname"] = p.Uname
+				retByte["uname"] = []byte(p.Uname)
+			}
+			ret["new"].(map[string]interface{})["attentions"] = fc
+			ret["new"].(map[string]interface{})["attention"] = fc
+			retByte["attentions"] = []byte(strconv.Itoa(fc))
+			retByte["attention"] = []byte(strconv.Itoa(fc))
+			ret["old"].(map[string]interface{})["uname"] = ""
+
+			wg := errgroup.Group{}
+			wg.Go(func() (err error) {
+				for i := 0; i < _retry; i++ {
+					hbaseErr := s.saveHBase(context.TODO(), s.rowKey(newMap.RoomId), retByte)
+					err = hbaseErr
+					if hbaseErr != nil {
+						continue
+					}
+					break
+				}
+				if err != nil {
+					log.Error("[UnameDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Go(func() (err error) {
+				err = s.dao.Pub(context.TODO(), int64(newMap.RoomId), ret)
+				if err != nil {
+					log.Error("[UnameDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
+				}
+				return
+			})
+			wg.Wait()
+			log.Info("[UnameDataBus]success to handle, error(%v), msg:(%v)", err, ret)
+			continue
+		}
+		log.Error("[UnameDataBus]fail to getData, error(%v),msg:(%v)", err, p)
+	}
+}