go-common/app/job/main/reply/service/service.go
2019-04-22 18:49:16 +08:00

367 lines
9.0 KiB
Go

package service
import (
"context"
"encoding/json"
"fmt"
"strconv"
"sync"
"time"
artrpc "go-common/app/interface/openplatform/article/rpc/client"
"go-common/app/job/main/reply/conf"
"go-common/app/job/main/reply/dao/message"
"go-common/app/job/main/reply/dao/notice"
"go-common/app/job/main/reply/dao/reply"
"go-common/app/job/main/reply/dao/search"
"go-common/app/job/main/reply/dao/spam"
"go-common/app/job/main/reply/dao/stat"
model "go-common/app/job/main/reply/model/reply"
accrpc "go-common/app/service/main/account/api"
arcrpc "go-common/app/service/main/archive/api/gorpc"
assrpc "go-common/app/service/main/assist/rpc/client"
eprpc "go-common/app/service/openplatform/pgc-season/api/grpc/episode/v1"
es "go-common/library/database/elastic"
"go-common/library/log"
xhttp "go-common/library/net/http/blademaster"
"go-common/library/net/rpc/warden"
"go-common/library/queue/databus"
"go-common/library/sync/pipeline/fanout"
)
const (
_chLen = 2048
)
var (
_rpChs []chan *databus.Message
_likeChs []chan *databus.Message
)
// action the message struct of kafka
type consumerMsg struct {
Action string `json:"action"`
Data json.RawMessage `json:"data"`
}
type searchFlush struct {
OldState int8
Reply *model.Reply
Report *model.Report
}
func (s *searchFlush) Key() (key string) {
if s.Report != nil {
return fmt.Sprintf("%d%d", s.Report.RpID, s.Report.ID)
}
return fmt.Sprintf("%d", s.Reply.RpID)
}
// Service is reply-job service
type Service struct {
c *conf.Config
waiter *sync.WaitGroup
dataConsumer *databus.Databus
likeConsumer *databus.Databus
searchChan chan *searchFlush
// rpc client
accSrv accrpc.AccountClient
arcSrv *arcrpc.Service2
articleSrv *artrpc.Service
assistSrv *assrpc.Service
bangumiSrv eprpc.EpisodeClient
// depend
messageDao *message.Dao
// notice
noticeDao *notice.Dao
// stat
statDao *stat.Dao
// reply
dao *reply.Dao
// spam
spam *spam.Cache
// search
searchDao *search.Dao
batchNumber int
es *es.Elastic
notify *fanout.Fanout
typeMapping map[int32]string
aliasMapping map[string]int32
marker *fanout.Fanout
}
// New return new service
func New(c *conf.Config) (s *Service) {
if c.Job.BatchNumber <= 0 {
c.Job.BatchNumber = 2000
}
searchHTTPClient = xhttp.NewClient(c.HTTPClient)
wardenClient := warden.DefaultClient()
cc, err := wardenClient.Dial(context.Background(), "discovery://default/season.service")
if err != nil {
panic(err)
}
bangumiClient := eprpc.NewEpisodeClient(cc)
s = &Service{
c: c,
bangumiSrv: bangumiClient,
waiter: new(sync.WaitGroup),
searchChan: make(chan *searchFlush, 1024),
dataConsumer: databus.New(c.Databus.Consumer),
likeConsumer: databus.New(c.Databus.Like),
//rpc
arcSrv: arcrpc.New2(c.RPCClient2.Archive),
articleSrv: artrpc.New(c.RPCClient2.Article),
assistSrv: assrpc.New(c.RPCClient2.Assist),
messageDao: message.NewMessageDao(c),
searchDao: search.New(c),
noticeDao: notice.New(c),
// stat
statDao: stat.New(c),
// init reply dao
dao: reply.New(c),
// init spam cache
batchNumber: c.Job.BatchNumber,
spam: spam.NewCache(c.Redis.Config),
notify: fanout.New("cache", fanout.Worker(1), fanout.Buffer(2048)),
typeMapping: make(map[int32]string),
aliasMapping: make(map[string]int32),
es: es.NewElastic(c.Es),
marker: fanout.New("marker", fanout.Worker(1), fanout.Buffer(1024)),
}
accSvc, err := accrpc.NewClient(c.AccountClient)
if err != nil {
panic(err)
}
s.accSrv = accSvc
time.Sleep(time.Second)
_rpChs = make([]chan *databus.Message, c.Job.Proc)
_likeChs = make([]chan *databus.Message, c.Job.Proc)
for i := 0; i < c.Job.Proc; i++ {
_rpChs[i] = make(chan *databus.Message, _chLen)
_likeChs[i] = make(chan *databus.Message, _chLen)
s.waiter.Add(1)
go s.consumeproc(i)
s.waiter.Add(1)
go s.consumelikeproc(i)
}
s.waiter.Add(1)
go s.likeConsume()
s.waiter.Add(1)
go s.dataConsume()
go s.searchproc()
go s.mappingproc()
return
}
func (s *Service) addSearchUp(c context.Context, oldState int8, rp *model.Reply, rpt *model.Report) {
select {
case s.searchChan <- &searchFlush{OldState: oldState, Reply: rp, Report: rpt}:
default:
log.Error("addSearchUp chan full, type:%d oid:%d rpID:%d", rp.Type, rp.Oid, rp.RpID)
}
}
func (s *Service) searchproc() {
var (
m *searchFlush
merge = make(map[string]*searchFlush)
num = s.c.Job.SearchNum
ticker = time.NewTicker(time.Duration(s.c.Job.SearchFlush))
)
for {
select {
case m = <-s.searchChan:
merge[m.Key()] = m
if len(merge) < num {
continue
}
case <-ticker.C:
}
if len(merge) > 0 {
s.callSearchUp(context.Background(), merge)
merge = make(map[string]*searchFlush)
}
}
}
func (s *Service) likeConsume() {
defer func() {
s.waiter.Done()
for i := 0; i < s.c.Job.Proc; i++ {
close(_rpChs[i])
}
}()
msgs := s.likeConsumer.Messages()
for {
msg, ok := <-msgs
if !ok {
log.Warn("[service.dataConsume|reply] dataConsumer has been closed.")
return
}
if msg.Topic != s.c.Databus.Like.Topic {
continue
}
rpid, err := strconv.ParseInt(string(msg.Key), 10, 64)
if err != nil {
continue
}
_likeChs[rpid%int64(s.c.Job.Proc)] <- msg
}
}
func (s *Service) dataConsume() {
defer func() {
s.waiter.Done()
for i := 0; i < s.c.Job.Proc; i++ {
close(_rpChs[i])
}
}()
msgs := s.dataConsumer.Messages()
for {
msg, ok := <-msgs
if !ok {
log.Warn("[service.dataConsume|reply] dataConsumer has been closed.")
return
}
if msg.Topic != s.c.Databus.Consumer.Topic {
continue
}
oid, err := strconv.ParseInt(string(msg.Key), 10, 64)
if err != nil {
continue
}
_rpChs[oid%int64(s.c.Job.Proc)] <- msg
}
}
// StatMsg stat msg.
type StatMsg struct {
Type string `json:"type,omitempty"`
ID int64 `json:"id,omitempty"`
Count int `json:"count,omitempty"`
Oid int64 `json:"origin_id,omitempty"`
DislikeCount int `json:"dislike_count,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
Mid int64 `json:"mid,omitempty"`
}
func (s *Service) consumelikeproc(i int) {
defer s.waiter.Done()
for {
msg, ok := <-_likeChs[i]
if !ok {
log.Info("consumeproc exit")
return
}
cmsg := &StatMsg{}
if err := json.Unmarshal(msg.Value, cmsg); err != nil {
log.Error("json.Unmarshal() error(%v)", err)
continue
}
if cmsg.Type != "reply" {
continue
}
s.setLike(context.Background(), cmsg)
msg.Commit()
log.Info("consumer topic:%s, partitionId:%d, offset:%d, Key:%s, Value:%s", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
}
}
func (s *Service) consumeproc(i int) {
defer s.waiter.Done()
for {
msg, ok := <-_rpChs[i]
if !ok {
log.Info("consumeproc exit")
return
}
cmsg := &consumerMsg{}
if err := json.Unmarshal(msg.Value, cmsg); err != nil {
log.Error("json.Unmarshal() error(%v)", err)
continue
}
switch cmsg.Action {
case "add":
s.actionAdd(context.Background(), cmsg)
case "add_top":
s.addTopCache(context.Background(), cmsg)
case "rpt":
s.actionRpt(context.Background(), cmsg)
case "act":
//s.actionAct(context.Background(), cmsg)
s.recAct(context.Background(), cmsg)
case "re_idx":
s.actionRecoverIndex(context.Background(), cmsg)
case "idx_floor":
s.acionRecoverFloorIdx(context.Background(), cmsg)
case "re_rt_idx":
s.actionRecoverRootIndex(context.Background(), cmsg)
case "idx_dialog":
s.actionRecoverDialog(context.Background(), cmsg)
case "fix_dialog":
s.actionRecoverFixDialog(context.Background(), cmsg)
case "re_act":
// s.actionRecoverAction(context.Background(),cmsg)
case "up":
s.actionUp(context.Background(), cmsg)
case "admin":
s.actionAdmin(context.Background(), cmsg)
case "spam":
s.addRecReply(context.Background(), cmsg)
s.addDailyReply(context.Background(), cmsg)
case "folder":
s.folderHanlder(context.Background(), cmsg)
default:
log.Error("invalid action %s, cmsg is %v", cmsg.Action, cmsg)
}
msg.Commit()
log.Info("consumer topic:%s, partitionId:%d, offset:%d, Key:%s, Value:%s", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
}
}
// TypeToAlias map type to alias
func (s *Service) TypeToAlias(t int32) (alias string, exists bool) {
alias, exists = s.typeMapping[t]
return
}
// AliasToType map alias to type
func (s *Service) AliasToType(alias string) (t int32, exists bool) {
t, exists = s.aliasMapping[alias]
return
}
func (s *Service) mappingproc() {
for {
if business, err := s.ListBusiness(context.Background()); err != nil {
log.Error("s.ListBusiness error(%v)", err)
} else {
for _, b := range business {
s.typeMapping[b.Type] = b.Alias
s.aliasMapping[b.Alias] = b.Type
}
}
time.Sleep(time.Duration(time.Minute * 5))
}
}
// Close close service
func (s *Service) Close() error {
return s.dataConsumer.Close()
}
// Wait wait all chan close
func (s *Service) Wait() {
s.waiter.Wait()
}
// Ping check service health
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}