Create & Init Project...

app/job/main/passport/service/login_log.go

@@ -0,0 +1,121 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/passport/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_addHBaseRetryCount    = 3
+	_addHBaseRetryDuration = time.Second
+)
+
+func (s *Service) logconsumeproc() {
+	mergeRoutineNum := int64(s.c.Group.Log.Num)
+	for {
+		msg, ok := <-s.dsLog.Messages()
+		if !ok {
+			log.Error("s.logconsumeproc closed")
+			return
+		}
+		// marked head to first commit
+		m := &message{data: msg}
+		p := &model.LoginLog{}
+		if err := json.Unmarshal(msg.Value, p); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+		s.logMu.Lock()
+		if s.logHead == nil {
+			s.logHead = m
+			s.logLast = m
+		} else {
+			s.logLast.next = m
+			s.logLast = m
+		}
+		s.logMu.Unlock()
+		m.object = p
+		// use specify goroutine to merge messages
+		s.logMergeChans[p.Mid%mergeRoutineNum] <- m
+		log.Info("logconsumeproc key:%s partition:%d offset:%d", msg.Key, msg.Partition, msg.Offset)
+	}
+}
+
+func (s *Service) logcommitproc() {
+	commits := make(map[int32]*databus.Message, s.c.Group.Log.Size)
+	for {
+		done := <-s.logDoneChan
+		// merge partitions to commit offset
+		for _, d := range done {
+			d.done = true
+		}
+		s.logMu.Lock()
+		for ; s.logHead != nil && s.logHead.done; s.logHead = s.logHead.next {
+			commits[s.logHead.data.Partition] = s.logHead.data
+		}
+		s.logMu.Unlock()
+		for k, m := range commits {
+			log.Info("logcommitproc committed, key:%s partition:%d offset:%d", m.Key, m.Partition, m.Offset)
+			m.Commit()
+			delete(commits, k)
+		}
+	}
+}
+
+func (s *Service) logmergeproc(c chan *message) {
+	var (
+		max    = s.c.Group.Log.Size
+		merges = make([]*model.LoginLog, 0, max)
+		marked = make([]*message, 0, max)
+		ticker = time.NewTicker(time.Duration(s.c.Group.Log.Ticker))
+	)
+	for {
+		select {
+		case msg, ok := <-c:
+			if !ok {
+				log.Error("s.logmergeproc closed")
+				return
+			}
+			p, assertOk := msg.object.(*model.LoginLog)
+			if assertOk {
+				merges = append(merges, p)
+			}
+			marked = append(marked, msg)
+			if len(marked) < max && len(merges) < max {
+				continue
+			}
+		case <-ticker.C:
+		}
+		if len(merges) > 0 {
+			s.processMerges(merges)
+			merges = make([]*model.LoginLog, 0, max)
+		}
+		if len(marked) > 0 {
+			s.logDoneChan <- marked
+			marked = make([]*message, 0, max)
+		}
+	}
+}
+
+func (s *Service) processMerges(merges []*model.LoginLog) {
+	s.d.AddLoginLog(merges)
+	for _, v := range merges {
+		s.addLoginLog(context.TODO(), v)
+	}
+}
+
+func (s *Service) addLoginLog(c context.Context, v *model.LoginLog) (err error) {
+	for i := 0; i < _addHBaseRetryCount; i++ {
+		if err = s.d.AddLoginLogHBase(c, v); err == nil {
+			return
+		}
+		log.Error("failed to add login log to hbase, service.dao.AddLoginLogHBase(%+v) error(%v)", v, err)
+		time.Sleep(_addHBaseRetryDuration)
+	}
+	return
+}