122 lines
2.8 KiB
Go
122 lines
2.8 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"time"
|
|
|
|
"go-common/app/job/main/passport/model"
|
|
"go-common/library/log"
|
|
"go-common/library/queue/databus"
|
|
)
|
|
|
|
const (
|
|
_addHBaseRetryCount = 3
|
|
_addHBaseRetryDuration = time.Second
|
|
)
|
|
|
|
func (s *Service) logconsumeproc() {
|
|
mergeRoutineNum := int64(s.c.Group.Log.Num)
|
|
for {
|
|
msg, ok := <-s.dsLog.Messages()
|
|
if !ok {
|
|
log.Error("s.logconsumeproc closed")
|
|
return
|
|
}
|
|
// marked head to first commit
|
|
m := &message{data: msg}
|
|
p := &model.LoginLog{}
|
|
if err := json.Unmarshal(msg.Value, p); err != nil {
|
|
log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
|
|
continue
|
|
}
|
|
s.logMu.Lock()
|
|
if s.logHead == nil {
|
|
s.logHead = m
|
|
s.logLast = m
|
|
} else {
|
|
s.logLast.next = m
|
|
s.logLast = m
|
|
}
|
|
s.logMu.Unlock()
|
|
m.object = p
|
|
// use specify goroutine to merge messages
|
|
s.logMergeChans[p.Mid%mergeRoutineNum] <- m
|
|
log.Info("logconsumeproc key:%s partition:%d offset:%d", msg.Key, msg.Partition, msg.Offset)
|
|
}
|
|
}
|
|
|
|
func (s *Service) logcommitproc() {
|
|
commits := make(map[int32]*databus.Message, s.c.Group.Log.Size)
|
|
for {
|
|
done := <-s.logDoneChan
|
|
// merge partitions to commit offset
|
|
for _, d := range done {
|
|
d.done = true
|
|
}
|
|
s.logMu.Lock()
|
|
for ; s.logHead != nil && s.logHead.done; s.logHead = s.logHead.next {
|
|
commits[s.logHead.data.Partition] = s.logHead.data
|
|
}
|
|
s.logMu.Unlock()
|
|
for k, m := range commits {
|
|
log.Info("logcommitproc committed, key:%s partition:%d offset:%d", m.Key, m.Partition, m.Offset)
|
|
m.Commit()
|
|
delete(commits, k)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) logmergeproc(c chan *message) {
|
|
var (
|
|
max = s.c.Group.Log.Size
|
|
merges = make([]*model.LoginLog, 0, max)
|
|
marked = make([]*message, 0, max)
|
|
ticker = time.NewTicker(time.Duration(s.c.Group.Log.Ticker))
|
|
)
|
|
for {
|
|
select {
|
|
case msg, ok := <-c:
|
|
if !ok {
|
|
log.Error("s.logmergeproc closed")
|
|
return
|
|
}
|
|
p, assertOk := msg.object.(*model.LoginLog)
|
|
if assertOk {
|
|
merges = append(merges, p)
|
|
}
|
|
marked = append(marked, msg)
|
|
if len(marked) < max && len(merges) < max {
|
|
continue
|
|
}
|
|
case <-ticker.C:
|
|
}
|
|
if len(merges) > 0 {
|
|
s.processMerges(merges)
|
|
merges = make([]*model.LoginLog, 0, max)
|
|
}
|
|
if len(marked) > 0 {
|
|
s.logDoneChan <- marked
|
|
marked = make([]*message, 0, max)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) processMerges(merges []*model.LoginLog) {
|
|
s.d.AddLoginLog(merges)
|
|
for _, v := range merges {
|
|
s.addLoginLog(context.TODO(), v)
|
|
}
|
|
}
|
|
|
|
func (s *Service) addLoginLog(c context.Context, v *model.LoginLog) (err error) {
|
|
for i := 0; i < _addHBaseRetryCount; i++ {
|
|
if err = s.d.AddLoginLogHBase(c, v); err == nil {
|
|
return
|
|
}
|
|
log.Error("failed to add login log to hbase, service.dao.AddLoginLogHBase(%+v) error(%v)", v, err)
|
|
time.Sleep(_addHBaseRetryDuration)
|
|
}
|
|
return
|
|
}
|