Create & Init Project...

app/job/main/passport-game-cloud/service/bin_log.go

@@ -0,0 +1,137 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/passport-game-cloud/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+func (s *Service) binlogconsumeproc() {
+	mergeRoutineNum := int64(s.c.Group.BinLog.Num)
+	for {
+		msg, ok := <-s.binLogDataBus.Messages()
+		if !ok {
+			log.Error("binlogconsumeproc closed")
+			return
+		}
+		// marked head to first commit
+		m := &message{data: msg}
+		s.mu.Lock()
+		if s.head == nil {
+			s.head = m
+			s.last = m
+		} else {
+			s.last.next = m
+			s.last = m
+		}
+		s.mu.Unlock()
+		bmsg := new(model.BMsg)
+		if err := json.Unmarshal(msg.Value, bmsg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+		mid := int64(0)
+		if bmsg.Table == _asoAccountTable {
+			t := new(model.OriginAsoAccount)
+			if err := json.Unmarshal(bmsg.New, t); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+				continue
+			}
+			mid = t.Mid
+			bmsg.MTS = s.asoAccountInterval.MTS(context.TODO(), t.Mtime)
+		} else if strings.HasPrefix(bmsg.Table, _tokenTablePrefix) {
+			t := new(model.OriginPerm)
+			if err := json.Unmarshal(bmsg.New, t); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+				continue
+			}
+			mid = t.Mid
+			bmsg.MTS = s.tokenInterval.MTS(context.TODO(), t.Mtime)
+		} else if strings.HasPrefix(bmsg.Table, _memberTablePrefix) {
+			t := new(model.OriginMember)
+			if err := json.Unmarshal(bmsg.New, t); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+				continue
+			}
+			mid = t.Mid
+			bmsg.MTS = s.memberInterval.MTS(context.TODO(), t.Mtime)
+		}
+		m.object = bmsg
+		// use specify goroutine to merge messages
+		s.binLogMergeChans[mid%mergeRoutineNum] <- m
+		log.Info("binlogconsumeproc table:%s key:%s partition:%d offset:%d", bmsg.Table, msg.Key, msg.Partition, msg.Offset)
+	}
+}
+
+func (s *Service) binlogcommitproc() {
+	commits := make(map[int32]*databus.Message, s.c.Group.BinLog.Size)
+	for {
+		done := <-s.binLogDoneChan
+		// merge partitions to commit offset
+		for _, d := range done {
+			d.done = true
+		}
+		s.mu.Lock()
+		for ; s.head != nil && s.head.done; s.head = s.head.next {
+			commits[s.head.data.Partition] = s.head.data
+		}
+		s.mu.Unlock()
+		for k, m := range commits {
+			log.Info("binlogcommitproc committed, key:%s partition:%d offset:%d", m.Key, m.Partition, m.Offset)
+			m.Commit()
+			delete(commits, k)
+		}
+	}
+}
+
+func (s *Service) binlogmergeproc(c chan *message) {
+	var (
+		max    = s.c.Group.BinLog.Size
+		merges = make([]*model.BMsg, 0, max)
+		marked = make([]*message, 0, max)
+		ticker = time.NewTicker(time.Duration(s.c.Group.BinLog.Ticker))
+	)
+	for {
+		select {
+		case msg, ok := <-c:
+			if !ok {
+				log.Error("binlogmergeproc closed")
+				return
+			}
+			p, assertOk := msg.object.(*model.BMsg)
+			if assertOk {
+				merges = append(merges, p)
+			}
+			marked = append(marked, msg)
+			if len(marked) < max && len(merges) < max {
+				continue
+			}
+		case <-ticker.C:
+		}
+		if len(merges) > 0 {
+			s.process(merges)
+			merges = make([]*model.BMsg, 0, max)
+		}
+		if len(marked) > 0 {
+			s.binLogDoneChan <- marked
+			marked = make([]*message, 0, max)
+		}
+	}
+}
+
+func (s *Service) process(bmsgs []*model.BMsg) {
+	for _, bmsg := range bmsgs {
+		if bmsg.Table == _asoAccountTable {
+			s.processUserInfo(bmsg)
+		} else if strings.HasPrefix(bmsg.Table, _tokenTablePrefix) {
+			s.processToken(bmsg)
+		} else if strings.HasPrefix(bmsg.Table, _memberTablePrefix) {
+			s.processMemberInfo(bmsg)
+		}
+	}
+}