445 lines
11 KiB
Go
445 lines
11 KiB
Go
|
package upcrmservice
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"errors"
|
|||
|
|
"sync"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"go-common/app/admin/main/up/dao/global"
|
|||
|
|
"go-common/app/admin/main/up/model/datamodel"
|
|||
|
|
"go-common/app/admin/main/up/model/upcrmmodel"
|
|||
|
|
"go-common/app/admin/main/up/util"
|
|||
|
|
accgrpc "go-common/app/service/main/account/api"
|
|||
|
|
"go-common/library/database/elastic"
|
|||
|
|
"go-common/library/log"
|
|||
|
|
"go-common/library/net/metadata"
|
|||
|
|
"go-common/library/sync/errgroup"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
maxSearchItemCount = 10000
|
|||
|
|
maxBatchCount = 100
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
var (
|
|||
|
|
//ErrTooManySearchItem too many search items, 搜索只支持返回前10000条数据
|
|||
|
|
ErrTooManySearchItem = errors.New("筛选仅支持展示前1万条")
|
|||
|
|
//ErrNoMid no mid
|
|||
|
|
ErrNoMid = errors.New("Mid为空")
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
//UpBaseInfoQuery query
|
|||
|
|
func (s *Service) UpBaseInfoQuery(context context.Context, arg *upcrmmodel.InfoQueryArgs) (result *upcrmmodel.InfoQueryResult, err error) {
|
|||
|
|
|
|||
|
|
var data, e = s.upBaseInfoQueryBatch(s.crmdb.QueryUpBaseInfoBatchByMid, arg.Mid)
|
|||
|
|
err = e
|
|||
|
|
// 没找到按照错误处理
|
|||
|
|
if err != nil || len(data) == 0 {
|
|||
|
|
log.Error("get from db fail, req=%+v, err=%+v", arg, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
result = data[0]
|
|||
|
|
|
|||
|
|
log.Info("query base info ok, req=%+v, err=%+v", arg, result)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//QueryDbFunc query func type
|
|||
|
|
type QueryDbFunc func(fields string, mid ...int64) (result []upcrmmodel.UpBaseInfo, err error)
|
|||
|
|
|
|||
|
|
func (s *Service) upBaseInfoQueryBatch(queryfunc QueryDbFunc, ids ...int64) (result []*upcrmmodel.InfoQueryResult, err error) {
|
|||
|
|
var data, e = queryfunc("*", ids...)
|
|||
|
|
err = e
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("get from db fail, err=%+v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, v := range data {
|
|||
|
|
var info = upcrmmodel.InfoQueryResult{}
|
|||
|
|
info.CopyFromBaseInfo(v)
|
|||
|
|
info.CalculateAttr()
|
|||
|
|
result = append(result, &info)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// UpAccountInfo get account info
|
|||
|
|
func (s *Service) UpAccountInfo(c context.Context, arg *upcrmmodel.InfoAccountInfoArgs) (res []*accgrpc.Info, err error) {
|
|||
|
|
var (
|
|||
|
|
infosReply *accgrpc.InfosReply
|
|||
|
|
mids = util.ExplodeInt64(arg.Mids, ",")
|
|||
|
|
)
|
|||
|
|
if infosReply, err = global.GetAccClient().Infos3(c, &accgrpc.MidsReq{Mids: mids, RealIp: metadata.String(c, metadata.RemoteIP)}); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if infosReply == nil || infosReply.Infos == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
for _, v := range infosReply.Infos {
|
|||
|
|
res = append(res, v)
|
|||
|
|
}
|
|||
|
|
log.Info("query acount info ok, req=%+v, result=%+v", arg, res)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//SearchResult struct
|
|||
|
|
type SearchResult struct {
|
|||
|
|
AccountState int `json:"account_state"`
|
|||
|
|
Activity int `json:"activity"`
|
|||
|
|
Attr int `json:"attr"`
|
|||
|
|
ArticleCountAccumulate int `json:"article_count_accumulate"`
|
|||
|
|
ID uint32
|
|||
|
|
Mid int64
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func getAttrFormat(attrs upcrmmodel.UpAttr) (result []int) {
|
|||
|
|
// 什么要shift，因为es的位是从1开始的，而存储的位是从0开始的
|
|||
|
|
const shift = 1
|
|||
|
|
if attrs.AttrVideo != 0 {
|
|||
|
|
result = append(result, upcrmmodel.AttrBitVideo+shift)
|
|||
|
|
}
|
|||
|
|
if attrs.AttrAudio != 0 {
|
|||
|
|
result = append(result, upcrmmodel.AttrBitAudio+shift)
|
|||
|
|
}
|
|||
|
|
if attrs.AttrArticle != 0 {
|
|||
|
|
result = append(result, upcrmmodel.AttrBitArticle+shift)
|
|||
|
|
}
|
|||
|
|
if attrs.AttrPhoto != 0 {
|
|||
|
|
result = append(result, upcrmmodel.AttrBitPhoto+shift)
|
|||
|
|
}
|
|||
|
|
if attrs.AttrSign != 0 {
|
|||
|
|
result = append(result, upcrmmodel.AttrBitSign+shift)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if attrs.AttrGrowup != 0 {
|
|||
|
|
result = append(result, upcrmmodel.AttrBitGrowup+shift)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if attrs.AttrVerify != 0 {
|
|||
|
|
result = append(result, upcrmmodel.AttrBitVerify+shift)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func getEsCombo(attrs upcrmmodel.UpAttr) (combos []*elastic.Combo) {
|
|||
|
|
const shift = 1
|
|||
|
|
|
|||
|
|
var attrs1, attrs2 []interface{}
|
|||
|
|
var attrFlagList = getAttrFormat(attrs)
|
|||
|
|
for _, v := range attrFlagList {
|
|||
|
|
if _, ok := upcrmmodel.AttrGroup1[v-shift]; ok {
|
|||
|
|
attrs1 = append(attrs1, v)
|
|||
|
|
} else if _, ok := upcrmmodel.AttrGroup2[v-shift]; ok {
|
|||
|
|
attrs2 = append(attrs2, v)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if attrs1 != nil {
|
|||
|
|
var attrGroup = make(map[string][]interface{})
|
|||
|
|
attrGroup["attr_format"] = attrs1
|
|||
|
|
cmb := &elastic.Combo{}
|
|||
|
|
cmb.ComboIn([]map[string][]interface{}{
|
|||
|
|
attrGroup},
|
|||
|
|
).MinIn(1).MinAll(1)
|
|||
|
|
combos = append(combos, cmb)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if attrs2 != nil {
|
|||
|
|
var attrGroup = make(map[string][]interface{})
|
|||
|
|
attrGroup["attr_format"] = attrs2
|
|||
|
|
cmb := &elastic.Combo{}
|
|||
|
|
cmb.ComboIn([]map[string][]interface{}{
|
|||
|
|
attrGroup},
|
|||
|
|
).MinIn(1).MinAll(1)
|
|||
|
|
combos = append(combos, cmb)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//UpInfoSearch info search
|
|||
|
|
func (s *Service) UpInfoSearch(c context.Context, arg *upcrmmodel.InfoSearchArgs) (result upcrmmodel.InfoSearchResult, err error) {
|
|||
|
|
//调用搜索的接口
|
|||
|
|
var searchData esResult
|
|||
|
|
searchData, err = s.searchFromEs(c, arg)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("search arg=%+v, err=%+v", arg, err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if len(searchData.Result) == 0 {
|
|||
|
|
log.Info("no data return from search, just return")
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
var ids []int64
|
|||
|
|
for _, v := range searchData.Result {
|
|||
|
|
ids = append(ids, int64(v.ID))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
result.Result, err = s.queryUpBaseInfo(c, ids...)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("query up base info fail, err=%+v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
result.PageInfo = searchData.Page.ToPageInfo()
|
|||
|
|
|
|||
|
|
log.Info("res=%+v, page=%+v", searchData, result.PageInfo)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type esPage struct {
|
|||
|
|
Num int `json:"num"`
|
|||
|
|
Size int `json:"size"`
|
|||
|
|
Total int `json:"total"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//ToPageInfo cast to page info
|
|||
|
|
func (e *esPage) ToPageInfo() (pageInfo upcrmmodel.PageInfo) {
|
|||
|
|
if e == nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
pageInfo.TotalCount = e.Total
|
|||
|
|
pageInfo.Size = e.Size
|
|||
|
|
pageInfo.Page = e.Num
|
|||
|
|
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type esResult struct {
|
|||
|
|
Page *esPage `json:"page"`
|
|||
|
|
Result []*SearchResult `json:"result"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) searchFromEs(c context.Context, arg *upcrmmodel.InfoSearchArgs) (searchData esResult, err error) {
|
|||
|
|
if arg.Page*arg.Size > maxSearchItemCount {
|
|||
|
|
err = ErrTooManySearchItem
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var searchSdk = elastic.NewElastic(nil)
|
|||
|
|
var r = searchSdk.NewRequest("up_crm_info")
|
|||
|
|
r.Pn(arg.Page).Ps(arg.Size).Index("up_base_info").
|
|||
|
|
Fields("id")
|
|||
|
|
if arg.Mid != 0 {
|
|||
|
|
r.WhereEq("mid", arg.Mid)
|
|||
|
|
} else {
|
|||
|
|
if arg.AccountState != 0 {
|
|||
|
|
// 字段有0值，所以接口改为1，2表示字段的0，1，接口的0表示没有此条件
|
|||
|
|
var realArg = arg.AccountState - 1
|
|||
|
|
r.WhereEq("account_state", realArg)
|
|||
|
|
}
|
|||
|
|
if arg.Activity != 0 {
|
|||
|
|
r.WhereEq("activity", arg.Activity)
|
|||
|
|
}
|
|||
|
|
var startdate, _ = time.Parse(upcrmmodel.TimeFmtDate, arg.FirstDateBegin)
|
|||
|
|
var enddate, _ = time.Parse(upcrmmodel.TimeFmtDate, arg.FirstDateEnd)
|
|||
|
|
enddate = enddate.AddDate(0, 0, 1)
|
|||
|
|
var startStr = startdate.Format(upcrmmodel.TimeFmtMysql)
|
|||
|
|
var endStr = enddate.Format(upcrmmodel.TimeFmtMysql)
|
|||
|
|
if arg.FirstDateBegin != "" && arg.FirstDateEnd != "" {
|
|||
|
|
r.WhereRange("first_up_time", startStr, endStr, elastic.RangeScopeLcRc)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if arg.Order.Order == "" {
|
|||
|
|
arg.Order.Order = "desc"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if arg.Order.Field == "" {
|
|||
|
|
arg.Order.Field = "first_up_time"
|
|||
|
|
arg.Order.Order = "desc"
|
|||
|
|
}
|
|||
|
|
var combos = getEsCombo(arg.Attrs)
|
|||
|
|
r.WhereCombo(combos...)
|
|||
|
|
r.Order(arg.Order.Field, arg.Order.Order)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
err = r.Scan(c, &searchData)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) queryUpBaseInfo(c context.Context, ids ...int64) (result []*upcrmmodel.InfoQueryResult, err error) {
|
|||
|
|
|
|||
|
|
var group, ctx = errgroup.WithContext(c)
|
|||
|
|
var infoData []*upcrmmodel.InfoQueryResult
|
|||
|
|
group.Go(func() error {
|
|||
|
|
var e error
|
|||
|
|
infoData, e = s.upBaseInfoQueryBatch(s.crmdb.QueryUpBaseInfoBatchByID, ids...)
|
|||
|
|
if e != nil {
|
|||
|
|
err = e
|
|||
|
|
log.Error("get base info fail, err=%+v", err)
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
var tidMap = make(map[int64]*datamodel.UpArchiveTypeData)
|
|||
|
|
var mapLock sync.Mutex
|
|||
|
|
for _, mid := range ids {
|
|||
|
|
group.Go(func() error {
|
|||
|
|
var arg = datamodel.GetUpArchiveTypeInfoArg{Mid: mid}
|
|||
|
|
var tidData, e = s.dataService.GetUpArchiveTypeInfo(ctx, &arg)
|
|||
|
|
if e != nil || tidData == nil {
|
|||
|
|
log.Error("get up type info err, err=%v", e)
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
mapLock.Lock()
|
|||
|
|
tidMap[arg.Mid] = tidData
|
|||
|
|
mapLock.Unlock()
|
|||
|
|
return nil
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
if err = group.Wait(); err != nil {
|
|||
|
|
log.Error("get data error, err=%v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var infoIDMap = make(map[uint32]*upcrmmodel.InfoQueryResult)
|
|||
|
|
for _, v := range infoData {
|
|||
|
|
infoIDMap[v.ID] = v
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, v := range ids {
|
|||
|
|
var info, ok = infoIDMap[uint32(v)]
|
|||
|
|
if !ok {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if typeInfo, ok := tidMap[v]; ok {
|
|||
|
|
info.ActiveTid = typeInfo.Tid
|
|||
|
|
info.ActiveSubtid = typeInfo.SubTid
|
|||
|
|
}
|
|||
|
|
result = append(result, info)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//QueryUpInfoWithViewerData query with view data
|
|||
|
|
func (s *Service) QueryUpInfoWithViewerData(c context.Context, arg *upcrmmodel.UpInfoWithViewerArg) (result upcrmmodel.UpInfoWithViewerResult, err error) {
|
|||
|
|
if arg.Page*arg.Size > maxSearchItemCount {
|
|||
|
|
err = ErrTooManySearchItem
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
// 如果是0，则默认设置所有的tag
|
|||
|
|
if arg.Flag == 0 {
|
|||
|
|
arg.Flag = -1
|
|||
|
|
}
|
|||
|
|
var mids []int64
|
|||
|
|
if arg.Mids != "" {
|
|||
|
|
mids = util.ExplodeInt64(arg.Mids, ",")
|
|||
|
|
var midlen = len(mids)
|
|||
|
|
if midlen == 0 {
|
|||
|
|
err = ErrNoMid
|
|||
|
|
log.Error("no mid get from mids, arg=%+v", arg)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if midlen > maxBatchCount {
|
|||
|
|
mids = mids[:maxBatchCount]
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
var searchSdk = elastic.NewElastic(nil)
|
|||
|
|
var r = searchSdk.NewRequest("up_crm_info")
|
|||
|
|
if arg.Size > maxBatchCount {
|
|||
|
|
arg.Size = maxBatchCount
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
r.Pn(arg.Page).Ps(arg.Size).Index("up_base_info").
|
|||
|
|
Fields("mid").
|
|||
|
|
Order(arg.Sort, arg.Order)
|
|||
|
|
|
|||
|
|
var searchData esResult
|
|||
|
|
err = r.Scan(c, &searchData)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("fail to get from search, arg=%+v", arg)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, v := range searchData.Result {
|
|||
|
|
mids = append(mids, int64(v.Mid))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
result.PageInfo = searchData.Page.ToPageInfo()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var group, ctx = errgroup.WithContext(c)
|
|||
|
|
var infoData []*upcrmmodel.InfoQueryResult
|
|||
|
|
var playData []*upcrmmodel.UpPlayInfo
|
|||
|
|
group.Go(func() error {
|
|||
|
|
if arg.Flag&upcrmmodel.FlagUpBaseData != 0 {
|
|||
|
|
infoData, err = s.upBaseInfoQueryBatch(s.crmdb.QueryUpBaseInfoBatchByMid, mids...)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("query up base error, err=%v", err)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if arg.Flag&upcrmmodel.FlagUpPlayData != 0 {
|
|||
|
|
playData, err = s.crmdb.QueryPlayInfoBatch(mids, upcrmmodel.BusinessTypeVideo)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("query play info err, err=%v", err)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
var dataMap = make(map[int64]*upcrmmodel.UpInfoWithViewerData)
|
|||
|
|
if arg.Flag&upcrmmodel.FlagViewData != 0 {
|
|||
|
|
for _, v := range mids {
|
|||
|
|
var mid = v // copy this v
|
|||
|
|
group.Go(func() error {
|
|||
|
|
var info, e = s.dataService.GetViewData(ctx, mid)
|
|||
|
|
if e != nil {
|
|||
|
|
err = e
|
|||
|
|
log.Error("query up view info from hbase error, err=%v", err)
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
var data = getOrCreateUpInfo(dataMap, mid)
|
|||
|
|
data.ViewerBase = info.Base
|
|||
|
|
data.ViewerTrend = info.Trend
|
|||
|
|
data.ViewerArea = info.Area
|
|||
|
|
return err
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if err = group.Wait(); err != nil {
|
|||
|
|
log.Error("get data fail, err=%v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, baseInfo := range infoData {
|
|||
|
|
var data = getOrCreateUpInfo(dataMap, baseInfo.Mid)
|
|||
|
|
data.UpBaseInfo = baseInfo
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, playInfo := range playData {
|
|||
|
|
var data = getOrCreateUpInfo(dataMap, playInfo.Mid)
|
|||
|
|
data.UpPlayInfo = playInfo
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, mid := range mids {
|
|||
|
|
var data, ok = dataMap[mid]
|
|||
|
|
if !ok {
|
|||
|
|
log.Warn("up info not found, mid=%d", mid)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
data.Mid = mid
|
|||
|
|
result.Result = append(result.Result, data)
|
|||
|
|
}
|
|||
|
|
log.Info("query up with view ok, arg=%+v, result count=%d", arg, len(mids))
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var dataMapMutex sync.Mutex
|
|||
|
|
|
|||
|
|
func getOrCreateUpInfo(dataMap map[int64]*upcrmmodel.UpInfoWithViewerData, mid int64) (result *upcrmmodel.UpInfoWithViewerData) {
|
|||
|
|
dataMapMutex.Lock()
|
|||
|
|
defer dataMapMutex.Unlock()
|
|||
|
|
|
|||
|
|
var ok bool
|
|||
|
|
if result, ok = dataMap[mid]; !ok {
|
|||
|
|
result = &upcrmmodel.UpInfoWithViewerData{}
|
|||
|
|
dataMap[mid] = result
|
|||
|
|
}
|
|||
|
|
return result
|
|||
|
|
}
|