package service import ( "context" "encoding/json" "go-common/app/job/main/videoup-report/model/archive" "go-common/app/job/main/videoup-report/model/manager" "go-common/library/log" ) var ( _archive = "archive" _video = "archive_video" _insertAct = "insert" _updateAct = "update" //_delete = "delete" ) // consumer binlog func (s *Service) arcCanalConsume() { defer s.waiter.Done() var ( msgs = s.archiveSub.Messages() err error ) for { msg, ok := <-msgs if !ok { log.Error("s.archiveSub.Message closed", err) return } msg.Commit() m := &archive.Message{} if err = json.Unmarshal(msg.Value, m); err != nil { log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err) continue } log.Info("arcCanalConsume msg(%s)", msg.Value) log.Info("arcCanalConsume topic(%s) partition(%d) offset(%d) commit start", msg.Topic, msg.Partition, msg.Offset) if msg.Offset >= s.c.BeginOffset { log.Info("arcCanalConsume offset(%d) is hit BeginOffset(%d) and start track data", msg.Offset, s.c.BeginOffset) if m.Table == _archiveTable { s.putArcChan(m.Action, m.New, m.Old) } if m.Table == _videoTable { s.putVideoChan(m.Action, m.New, m.Old) } } else { log.Info("arcCanalConsume offset(%d) not hit BeginOffset(%d) and pass", msg.Offset, s.c.BeginOffset) } //todo 异步消费 if m.Table == _video && m.Action == _updateAct { s.hdlVideoUpdateBinLog(m.New, m.Old) } if m.Table == _archive { s.hdlArchiveMessage(m.Action, m.New, m.Old) } } } func (s *Service) videoupConsumer() { defer s.waiter.Done() var ( msgs = s.videoupSub.Messages() err error c = context.TODO() ) for { msg, ok := <-msgs if !ok { log.Error("s.videoupSub.Messages closed") return } msg.Commit() m := &archive.VideoupMsg{} if err = json.Unmarshal(msg.Value, m); err != nil { log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err) return } log.Info("videoupMessage key(%s) value(%s) partition(%d) offset(%d) route(%s) commit start", msg.Key, msg.Value, msg.Partition, msg.Offset, m.Route) switch m.Route { case archive.RoutePostFirstRound: err = s.postFirstRound(c, m) case archive.RouteSecondRound: err = s.secondRound(c, m) case archive.RouteAddArchive: err = s.addArchive(c, m) case archive.RouteModifyArchive: err = s.modifyArchive(c, m) case archive.RouteAutoOpen: err = s.autoOpen(c, m) case archive.RouteDelayOpen: err = s.delayOpen(c, m) default: log.Warn("videoupConsumer unknown message route(%s)", m.Route) } if err == nil { log.Info("videoupMessage key(%s) value(%s) partition(%d) offset(%d) end", msg.Key, msg.Value, msg.Partition, msg.Offset) } else { log.Error("videoupMessage key(%s) value(%s) partition(%d) offset(%d) error(%v)", msg.Key, msg.Value, msg.Partition, msg.Offset, err) } } } // managerDBConsume 消费manager binlog func (s *Service) managerDBConsume() { defer s.waiter.Done() var ( err error msgs = s.ManagerDBSub.Messages() ) for { msg, open := <-msgs if !open { log.Info("managerDBConsume s.arcResultSub.Messages is closed") return } if msg == nil { continue } msg.Commit() log.Info("managerDBConsume consume key(%s) offset(%d) value(%s)", msg.Key, msg.Offset, string(msg.Value)) m := &manager.BinMsg{} if err = json.Unmarshal(msg.Value, m); err != nil { log.Error("managerDBConsume json.Unmarshal error(%v)", err) continue } switch m.Table { case _upsTable: s.hdlManagerUpsBinlog(m) } } }