package service import ( "context" "encoding/json" "fmt" "strconv" "sync" "time" artrpc "go-common/app/interface/openplatform/article/rpc/client" "go-common/app/job/main/reply/conf" "go-common/app/job/main/reply/dao/message" "go-common/app/job/main/reply/dao/notice" "go-common/app/job/main/reply/dao/reply" "go-common/app/job/main/reply/dao/search" "go-common/app/job/main/reply/dao/spam" "go-common/app/job/main/reply/dao/stat" model "go-common/app/job/main/reply/model/reply" accrpc "go-common/app/service/main/account/api" arcrpc "go-common/app/service/main/archive/api/gorpc" assrpc "go-common/app/service/main/assist/rpc/client" eprpc "go-common/app/service/openplatform/pgc-season/api/grpc/episode/v1" es "go-common/library/database/elastic" "go-common/library/log" xhttp "go-common/library/net/http/blademaster" "go-common/library/net/rpc/warden" "go-common/library/queue/databus" "go-common/library/sync/pipeline/fanout" ) const ( _chLen = 2048 ) var ( _rpChs []chan *databus.Message _likeChs []chan *databus.Message ) // action the message struct of kafka type consumerMsg struct { Action string `json:"action"` Data json.RawMessage `json:"data"` } type searchFlush struct { OldState int8 Reply *model.Reply Report *model.Report } func (s *searchFlush) Key() (key string) { if s.Report != nil { return fmt.Sprintf("%d%d", s.Report.RpID, s.Report.ID) } return fmt.Sprintf("%d", s.Reply.RpID) } // Service is reply-job service type Service struct { c *conf.Config waiter *sync.WaitGroup dataConsumer *databus.Databus likeConsumer *databus.Databus searchChan chan *searchFlush // rpc client accSrv accrpc.AccountClient arcSrv *arcrpc.Service2 articleSrv *artrpc.Service assistSrv *assrpc.Service bangumiSrv eprpc.EpisodeClient // depend messageDao *message.Dao // notice noticeDao *notice.Dao // stat statDao *stat.Dao // reply dao *reply.Dao // spam spam *spam.Cache // search searchDao *search.Dao batchNumber int es *es.Elastic notify *fanout.Fanout typeMapping map[int32]string aliasMapping map[string]int32 marker *fanout.Fanout } // New return new service func New(c *conf.Config) (s *Service) { if c.Job.BatchNumber <= 0 { c.Job.BatchNumber = 2000 } searchHTTPClient = xhttp.NewClient(c.HTTPClient) wardenClient := warden.DefaultClient() cc, err := wardenClient.Dial(context.Background(), "discovery://default/season.service") if err != nil { panic(err) } bangumiClient := eprpc.NewEpisodeClient(cc) s = &Service{ c: c, bangumiSrv: bangumiClient, waiter: new(sync.WaitGroup), searchChan: make(chan *searchFlush, 1024), dataConsumer: databus.New(c.Databus.Consumer), likeConsumer: databus.New(c.Databus.Like), //rpc arcSrv: arcrpc.New2(c.RPCClient2.Archive), articleSrv: artrpc.New(c.RPCClient2.Article), assistSrv: assrpc.New(c.RPCClient2.Assist), messageDao: message.NewMessageDao(c), searchDao: search.New(c), noticeDao: notice.New(c), // stat statDao: stat.New(c), // init reply dao dao: reply.New(c), // init spam cache batchNumber: c.Job.BatchNumber, spam: spam.NewCache(c.Redis.Config), notify: fanout.New("cache", fanout.Worker(1), fanout.Buffer(2048)), typeMapping: make(map[int32]string), aliasMapping: make(map[string]int32), es: es.NewElastic(c.Es), marker: fanout.New("marker", fanout.Worker(1), fanout.Buffer(1024)), } accSvc, err := accrpc.NewClient(c.AccountClient) if err != nil { panic(err) } s.accSrv = accSvc time.Sleep(time.Second) _rpChs = make([]chan *databus.Message, c.Job.Proc) _likeChs = make([]chan *databus.Message, c.Job.Proc) for i := 0; i < c.Job.Proc; i++ { _rpChs[i] = make(chan *databus.Message, _chLen) _likeChs[i] = make(chan *databus.Message, _chLen) s.waiter.Add(1) go s.consumeproc(i) s.waiter.Add(1) go s.consumelikeproc(i) } s.waiter.Add(1) go s.likeConsume() s.waiter.Add(1) go s.dataConsume() go s.searchproc() go s.mappingproc() return } func (s *Service) addSearchUp(c context.Context, oldState int8, rp *model.Reply, rpt *model.Report) { select { case s.searchChan <- &searchFlush{OldState: oldState, Reply: rp, Report: rpt}: default: log.Error("addSearchUp chan full, type:%d oid:%d rpID:%d", rp.Type, rp.Oid, rp.RpID) } } func (s *Service) searchproc() { var ( m *searchFlush merge = make(map[string]*searchFlush) num = s.c.Job.SearchNum ticker = time.NewTicker(time.Duration(s.c.Job.SearchFlush)) ) for { select { case m = <-s.searchChan: merge[m.Key()] = m if len(merge) < num { continue } case <-ticker.C: } if len(merge) > 0 { s.callSearchUp(context.Background(), merge) merge = make(map[string]*searchFlush) } } } func (s *Service) likeConsume() { defer func() { s.waiter.Done() for i := 0; i < s.c.Job.Proc; i++ { close(_rpChs[i]) } }() msgs := s.likeConsumer.Messages() for { msg, ok := <-msgs if !ok { log.Warn("[service.dataConsume|reply] dataConsumer has been closed.") return } if msg.Topic != s.c.Databus.Like.Topic { continue } rpid, err := strconv.ParseInt(string(msg.Key), 10, 64) if err != nil { continue } _likeChs[rpid%int64(s.c.Job.Proc)] <- msg } } func (s *Service) dataConsume() { defer func() { s.waiter.Done() for i := 0; i < s.c.Job.Proc; i++ { close(_rpChs[i]) } }() msgs := s.dataConsumer.Messages() for { msg, ok := <-msgs if !ok { log.Warn("[service.dataConsume|reply] dataConsumer has been closed.") return } if msg.Topic != s.c.Databus.Consumer.Topic { continue } oid, err := strconv.ParseInt(string(msg.Key), 10, 64) if err != nil { continue } _rpChs[oid%int64(s.c.Job.Proc)] <- msg } } // StatMsg stat msg. type StatMsg struct { Type string `json:"type,omitempty"` ID int64 `json:"id,omitempty"` Count int `json:"count,omitempty"` Oid int64 `json:"origin_id,omitempty"` DislikeCount int `json:"dislike_count,omitempty"` Timestamp int64 `json:"timestamp,omitempty"` Mid int64 `json:"mid,omitempty"` } func (s *Service) consumelikeproc(i int) { defer s.waiter.Done() for { msg, ok := <-_likeChs[i] if !ok { log.Info("consumeproc exit") return } cmsg := &StatMsg{} if err := json.Unmarshal(msg.Value, cmsg); err != nil { log.Error("json.Unmarshal() error(%v)", err) continue } if cmsg.Type != "reply" { continue } s.setLike(context.Background(), cmsg) msg.Commit() log.Info("consumer topic:%s, partitionId:%d, offset:%d, Key:%s, Value:%s", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value) } } func (s *Service) consumeproc(i int) { defer s.waiter.Done() for { msg, ok := <-_rpChs[i] if !ok { log.Info("consumeproc exit") return } cmsg := &consumerMsg{} if err := json.Unmarshal(msg.Value, cmsg); err != nil { log.Error("json.Unmarshal() error(%v)", err) continue } switch cmsg.Action { case "add": s.actionAdd(context.Background(), cmsg) case "add_top": s.addTopCache(context.Background(), cmsg) case "rpt": s.actionRpt(context.Background(), cmsg) case "act": //s.actionAct(context.Background(), cmsg) s.recAct(context.Background(), cmsg) case "re_idx": s.actionRecoverIndex(context.Background(), cmsg) case "idx_floor": s.acionRecoverFloorIdx(context.Background(), cmsg) case "re_rt_idx": s.actionRecoverRootIndex(context.Background(), cmsg) case "idx_dialog": s.actionRecoverDialog(context.Background(), cmsg) case "fix_dialog": s.actionRecoverFixDialog(context.Background(), cmsg) case "re_act": // s.actionRecoverAction(context.Background(),cmsg) case "up": s.actionUp(context.Background(), cmsg) case "admin": s.actionAdmin(context.Background(), cmsg) case "spam": s.addRecReply(context.Background(), cmsg) s.addDailyReply(context.Background(), cmsg) case "folder": s.folderHanlder(context.Background(), cmsg) default: log.Error("invalid action %s, cmsg is %v", cmsg.Action, cmsg) } msg.Commit() log.Info("consumer topic:%s, partitionId:%d, offset:%d, Key:%s, Value:%s", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value) } } // TypeToAlias map type to alias func (s *Service) TypeToAlias(t int32) (alias string, exists bool) { alias, exists = s.typeMapping[t] return } // AliasToType map alias to type func (s *Service) AliasToType(alias string) (t int32, exists bool) { t, exists = s.aliasMapping[alias] return } func (s *Service) mappingproc() { for { if business, err := s.ListBusiness(context.Background()); err != nil { log.Error("s.ListBusiness error(%v)", err) } else { for _, b := range business { s.typeMapping[b.Type] = b.Alias s.aliasMapping[b.Alias] = b.Type } } time.Sleep(time.Duration(time.Minute * 5)) } } // Close close service func (s *Service) Close() error { return s.dataConsumer.Close() } // Wait wait all chan close func (s *Service) Wait() { s.waiter.Wait() } // Ping check service health func (s *Service) Ping(c context.Context) (err error) { return s.dao.Ping(c) }