Create & Init Project...

app/job/main/account-summary/service/BUILD

@@ -0,0 +1,48 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "block.go",
+        "initial.go",
+        "member.go",
+        "passport.go",
+        "relation.go",
+        "service.go",
+        "summary.go",
+        "sync.go",
+    ],
+    importpath = "go-common/app/job/main/account-summary/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/account-summary/conf:go_default_library",
+        "//app/job/main/account-summary/dao:go_default_library",
+        "//app/job/main/account-summary/model:go_default_library",
+        "//app/service/main/member/model:go_default_library",
+        "//app/service/main/member/model/block:go_default_library",
+        "//app/service/main/relation/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/account-summary/service/block.go

@@ -0,0 +1,47 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/job/main/account-summary/model"
+	"go-common/library/log"
+)
+
+func (s *Service) blockBinLogproc(ctx context.Context) {
+	for msg := range s.BlockBinLog.Messages() {
+		blog := &model.CanalBinLog{}
+		if err := json.Unmarshal(msg.Value, blog); err != nil {
+			log.Error("Failed to unmarshal canal bin log: %+v, value: %s: %+v", msg, string(msg.Value), err)
+			msg.Commit()
+			continue
+		}
+
+		log.Info("Handling message key: %s, value: %s", msg.Key, string(msg.Value))
+		s.blockBinLogHandle(ctx, blog)
+		msg.Commit()
+	}
+}
+
+func (s *Service) blockBinLogHandle(ctx context.Context, blog *model.CanalBinLog) {
+	if len(blog.New) == 0 {
+		log.Error("Failed to sync to hbase with empty new field: %+v", blog)
+		return
+	}
+
+	switch blog.Table {
+	case "block_user":
+		midl := &model.MidBinLog{}
+		if err := json.Unmarshal(blog.New, midl); err != nil {
+			log.Error("Failed to unmarsha new data: %s: %+v", string(blog.New), err)
+			return
+		}
+		// FIXME: 一段时间后改用 syncBlock
+		if err := s.SyncOne(ctx, midl.Mid); err != nil {
+			log.Error("Failed to sync block with mid: %d: %+v", midl.Mid, err)
+			return
+		}
+	default:
+		log.Warn("Unable to hanlde binlog: %+v, old: %s, new: %s", blog, string(blog.Old), string(blog.New))
+	}
+}

app/job/main/account-summary/service/initial.go

@@ -0,0 +1,122 @@
+package service
+
+// import (
+// 	"context"
+// 	"sync"
+// 	"time"
+
+// 	"go-common/library/log"
+// )
+
+// func (s *Service) initialproc(ctx context.Context) {
+// 	dataWg := sync.WaitGroup{}
+// 	workerWg := sync.WaitGroup{}
+
+// 	jobQueue := make(chan func(), 4096)
+// 	initialWorker := func() {
+// 		worker := uint64(50)
+// 		if s.c.AccountSummary.InitialWriteWorker > 0 {
+// 			worker = s.c.AccountSummary.InitialWriteWorker
+// 		}
+// 		log.Info("Start %d initial write worker", worker)
+// 		for i := uint64(0); i < worker; i++ {
+// 			workerWg.Add(1)
+// 			go func() {
+// 				defer workerWg.Done()
+// 				for job := range jobQueue {
+// 					job()
+// 				}
+// 			}()
+// 		}
+// 	}
+// 	initialWorker()
+
+// 	initBase := func() {
+// 		log.Info("Start to initial member base")
+// 		defer dataWg.Done()
+// 		baseCh := s.dao.AllMemberBase(ctx)
+// 		for chunk := range baseCh {
+// 			for _, b := range chunk {
+// 				b := b
+// 				jobQueue <- func() {
+// 					if err := s.SyncToHBase(ctx, b); err != nil {
+// 						log.Error("Failed to sync member base in initial process: base: %+v: %+v", b, err)
+// 					}
+// 				}
+// 			}
+// 		}
+// 	}
+// 	initExp := func() {
+// 		log.Info("Start to initial member exp")
+// 		defer dataWg.Done()
+// 		expCh := s.dao.AllMemberExp(ctx)
+// 		for chunk := range expCh {
+// 			for _, e := range chunk {
+// 				e := e
+// 				jobQueue <- func() {
+// 					if err := s.SyncToHBase(ctx, e); err != nil {
+// 						log.Error("Failed to sync member exp in initial process: exp: %+v: %+v", e, err)
+// 					}
+// 				}
+// 			}
+// 			time.Sleep(time.Second)
+// 		}
+// 	}
+// 	initOfficial := func() {
+// 		log.Info("Start to initial member official")
+// 		defer dataWg.Done()
+// 		official, err := s.dao.AllOfficial(ctx)
+// 		if err != nil {
+// 			log.Error("Failed to get all member official: %+v", err)
+// 			return
+// 		}
+// 		for _, o := range official {
+// 			o := o
+// 			jobQueue <- func() {
+// 				if err := s.SyncToHBase(ctx, o); err != nil {
+// 					log.Error("Failed to sync member official in initial process: official: %+v: %+v", o, err)
+// 				}
+// 			}
+// 		}
+// 	}
+// 	initStat := func() {
+// 		log.Info("Start to initial relation stat")
+// 		defer dataWg.Done()
+// 		statsCh := s.dao.AllRelationStat(ctx)
+// 		for chunk := range statsCh {
+// 			for _, stat := range chunk {
+// 				stat := stat
+// 				jobQueue <- func() {
+// 					if err := s.SyncToHBase(ctx, stat); err != nil {
+// 						log.Error("Failed to sync relation stat in initial process: stat: %+v: %+v", stat, err)
+// 					}
+// 				}
+// 			}
+// 			time.Sleep(time.Second)
+// 		}
+// 	}
+
+// 	if s.c.FeatureGate.InitialMemberBase {
+// 		dataWg.Add(1)
+// 		go initBase()
+// 	}
+
+// 	if s.c.FeatureGate.InitialMemberExp {
+// 		dataWg.Add(1)
+// 		go initExp()
+// 	}
+
+// 	if s.c.FeatureGate.InitialMemberOfficial {
+// 		dataWg.Add(1)
+// 		go initOfficial()
+// 	}
+
+// 	if s.c.FeatureGate.InitialRelationStat {
+// 		dataWg.Add(1)
+// 		go initStat()
+// 	}
+
+// 	dataWg.Wait()
+// 	close(jobQueue) // all job is enqueued
+// 	workerWg.Wait()
+// }

app/job/main/account-summary/service/member.go

@@ -0,0 +1,48 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+
+	"go-common/app/job/main/account-summary/model"
+	"go-common/library/log"
+)
+
+func (s *Service) memberBinLogproc(ctx context.Context) {
+	for msg := range s.MemberBinLog.Messages() {
+		blog := &model.CanalBinLog{}
+		if err := json.Unmarshal(msg.Value, blog); err != nil {
+			log.Error("Failed to unmarshal canal bin log: %+v, value: %s: %+v", msg, string(msg.Value), err)
+			msg.Commit()
+			continue
+		}
+
+		log.Info("Handling message key: %s, value: %s", msg.Key, string(msg.Value))
+		s.memberBinLogHandle(ctx, blog)
+		msg.Commit()
+	}
+}
+
+func (s *Service) memberBinLogHandle(ctx context.Context, blog *model.CanalBinLog) {
+	if len(blog.New) == 0 {
+		log.Error("Failed to sync to hbase with empty new field: %+v", blog)
+		return
+	}
+
+	switch {
+	case strings.HasPrefix(blog.Table, "user_base_") || strings.HasPrefix(blog.Table, "user_official") || strings.HasPrefix(blog.Table, "user_exp_"):
+		midl := &model.MidBinLog{}
+		if err := json.Unmarshal(blog.New, midl); err != nil {
+			log.Error("Failed to unmarsha new data: %s: %+v", string(blog.New), err)
+			return
+		}
+		// FIXME: 一段时间后该用 syncMember
+		if err := s.SyncOne(ctx, midl.Mid); err != nil {
+			log.Error("Failed to sync member with mid: %d: %+v", midl.Mid, err)
+			return
+		}
+	default:
+		log.Warn("Unable to hanlde binlog: %+v, old: %s, new: %s", blog, string(blog.Old), string(blog.New))
+	}
+}

app/job/main/account-summary/service/passport.go

@@ -0,0 +1,47 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+
+	"go-common/app/job/main/account-summary/model"
+	"go-common/library/log"
+)
+
+func (s *Service) passportBinLogproc(ctx context.Context) {
+	for msg := range s.PassportBinLog.Messages() {
+		blog := &model.CanalBinLog{}
+		if err := json.Unmarshal(msg.Value, blog); err != nil {
+			log.Error("Failed to unmarshal canal bin log: %+v, value: %s: %+v", msg, string(msg.Value), err)
+			msg.Commit()
+			continue
+		}
+
+		log.Info("Handling message key: %s, value: %s", msg.Key, string(msg.Value))
+		s.passportBinLogHandle(ctx, blog)
+		msg.Commit()
+	}
+}
+
+func (s *Service) passportBinLogHandle(ctx context.Context, blog *model.CanalBinLog) {
+	if len(blog.New) == 0 {
+		log.Error("Failed to sync to hbase with empty new field: %+v", blog)
+		return
+	}
+
+	switch {
+	case strings.HasPrefix(blog.Table, "aso_account"):
+		midl := &model.MidBinLog{}
+		if err := json.Unmarshal(blog.New, midl); err != nil {
+			log.Error("Failed to unmarsha new data: %s: %+v", string(blog.New), err)
+			return
+		}
+		if err := s.syncPassportSummary(ctx, midl.Mid); err != nil {
+			log.Error("Failed to sync passport summary with mid: %d: %+v", midl.Mid, err)
+			return
+		}
+	default:
+		log.Warn("Unable to hanlde binlog: %+v, old: %s, new: %s", blog, string(blog.Old), string(blog.New))
+	}
+}

app/job/main/account-summary/service/relation.go

@@ -0,0 +1,47 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+
+	"go-common/app/job/main/account-summary/model"
+	"go-common/library/log"
+)
+
+func (s *Service) relationBinLogproc(ctx context.Context) {
+	for msg := range s.RelationBinLog.Messages() {
+		blog := &model.CanalBinLog{}
+		if err := json.Unmarshal(msg.Value, blog); err != nil {
+			log.Error("Failed to unmarshal canal bin log: %+v, value: %s: %+v", msg, string(msg.Value), err)
+			msg.Commit()
+			continue
+		}
+
+		log.Info("Handling message key: %s, value: %s", msg.Key, string(msg.Value))
+		s.relationBinLogHandle(ctx, blog)
+		msg.Commit()
+	}
+}
+
+func (s *Service) relationBinLogHandle(ctx context.Context, blog *model.CanalBinLog) {
+	if len(blog.New) == 0 {
+		log.Error("Failed to sync to hbase with empty new field: %+v", blog)
+		return
+	}
+
+	switch {
+	case strings.HasPrefix(blog.Table, "user_relation_stat_"):
+		midl := &model.MidBinLog{}
+		if err := json.Unmarshal(blog.New, midl); err != nil {
+			log.Error("Failed to unmarsha new data: %s: %+v", string(blog.New), err)
+			return
+		}
+		if err := s.syncRelationStat(ctx, midl.Mid); err != nil {
+			log.Error("Failed to sync relation stat with mid: %d: %+v", midl.Mid, err)
+			return
+		}
+	default:
+		log.Warn("Unable to hanlde binlog: %+v, old: %s, new: %s", blog, string(blog.Old), string(blog.New))
+	}
+}

app/job/main/account-summary/service/service.go

@@ -0,0 +1,96 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/account-summary/conf"
+	"go-common/app/job/main/account-summary/dao"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+// Service struct
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+
+	MemberBinLog           *databus.Databus
+	BlockBinLog            *databus.Databus
+	PassportBinLog         *databus.Databus
+	RelationBinLog         *databus.Databus
+	AccountSummaryProducer *databus.Databus
+}
+
+// New init
+func New(c *conf.Config) *Service {
+	s := &Service{
+		c:                      c,
+		dao:                    dao.New(c),
+		RelationBinLog:         databus.New(c.RelationBinLog),
+		MemberBinLog:           databus.New(c.MemberBinLog),
+		BlockBinLog:            databus.New(c.BlockBinLog),
+		PassportBinLog:         databus.New(c.PassportBinLog),
+		AccountSummaryProducer: databus.New(c.AccountSummaryProducer),
+	}
+	s.Main()
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) error {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}
+
+// Main is
+func (s *Service) Main() {
+	subproc := func() {
+		worker := s.c.AccountSummary.SubProcessWorker
+		if worker <= 0 {
+			worker = 1
+		}
+		log.Info("Starting sub process with %d workers", worker)
+		for i := uint64(0); i < worker; i++ {
+			go s.memberBinLogproc(context.Background())
+			go s.blockBinLogproc(context.Background())
+			go s.passportBinLogproc(context.Background())
+			go s.relationBinLogproc(context.Background())
+		}
+	}
+
+	syncrange := func() {
+		start := s.c.AccountSummary.SyncRangeStart
+		if start <= 0 {
+			start = 1
+		}
+		end := s.c.AccountSummary.SyncRangeEnd
+		if end <= 0 {
+			end = 1
+		}
+		worker := s.c.AccountSummary.SyncRangeWorker
+		if worker <= 0 {
+			worker = 1
+		}
+		go s.syncRangeproc(context.Background(), start, end, worker)
+	}
+
+	// initial := func() {
+	// go s.initialproc(context.Background())
+	// }
+
+	if !s.c.FeatureGate.DisableSubProcess {
+		subproc()
+	}
+
+	if s.c.FeatureGate.SyncRange {
+		syncrange()
+	}
+
+	// if s.c.FeatureGate.Initial {
+	// 	initial()
+	// }
+}

app/job/main/account-summary/service/summary.go

@@ -0,0 +1,65 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/account-summary/model"
+	"go-common/library/log"
+)
+
+// Syncable is
+type Syncable interface {
+	Key() (string, error)
+	Marshal() (map[string][]byte, error)
+}
+
+// SyncToHBase is
+func (s *Service) SyncToHBase(ctx context.Context, body Syncable) error {
+	key, err := body.Key()
+	if err != nil {
+		log.Error("Failed to struct key with body: %+v: %+v", body, err)
+		return err
+	}
+	data, err := body.Marshal()
+	if err != nil {
+		log.Error("Failed to sync to hbase with key: %s, body: %+v: %+v", key, body, err)
+		return err
+	}
+	defer s.logging(ctx, key)
+	return s.dao.Save(ctx, key, data)
+}
+
+func (s *Service) logging(ctx context.Context, key string) {
+	sum, _ := s.dao.GetByKey(ctx, key)
+	log.Info("Sync to hbase result: key: %s, summary: %+v", key, sum)
+}
+
+// SyncOne is
+func (s *Service) SyncOne(ctx context.Context, mid int64) error {
+	// member
+	if err := s.syncMember(ctx, mid); err != nil {
+		log.Error("Failed to sync member with mid: %d: %+v", mid, err)
+	}
+
+	// relation
+	if err := s.syncRelationStat(ctx, mid); err != nil {
+		log.Error("Failed to sync relation stat with mid: %d: %+v", mid, err)
+	}
+
+	// block
+	if err := s.syncBlock(ctx, mid); err != nil {
+		log.Error("Failed to sync block with mid: %d: %+v", mid, err)
+	}
+
+	// passport
+	if err := s.syncPassportSummary(ctx, mid); err != nil {
+		log.Error("Failed to sync passport summary with mid: %d: %+v", mid, err)
+	}
+
+	return nil
+}
+
+// GetOne is
+func (s *Service) GetOne(ctx context.Context, mid int64) (*model.AccountSummary, error) {
+	return s.dao.GetByKey(ctx, model.MidKey(mid))
+}

app/job/main/account-summary/service/sync.go

@@ -0,0 +1,208 @@
+package service
+
+import (
+	"context"
+	"sync"
+
+	"go-common/app/job/main/account-summary/model"
+	member "go-common/app/service/main/member/model"
+	"go-common/app/service/main/member/model/block"
+	relation "go-common/app/service/main/relation/model"
+	"go-common/library/log"
+	xtime "go-common/library/time"
+)
+
+func (s *Service) block(ctx context.Context, mid int64) (*model.BlockSummary, error) {
+	bl, err := s.dao.MemberService.BlockInfo(ctx, &block.RPCArgInfo{MID: mid})
+	if err != nil {
+		return nil, err
+	}
+
+	blSummary := &model.BlockSummary{
+		EmbedMid:    model.EmbedMid{Mid: bl.MID},
+		BlockStatus: int64(bl.BlockStatus),
+		StartTime:   model.Datetime(xtime.Time(bl.StartTime)),
+		EndTime:     model.Datetime(xtime.Time(bl.EndTime)),
+	}
+	return blSummary, nil
+}
+
+func (s *Service) relationStat(ctx context.Context, mid int64) (*model.RelationStat, error) {
+	stat, err := s.dao.RelationService.Stat(ctx, &relation.ArgMid{Mid: mid})
+	if err != nil {
+		return nil, err
+	}
+
+	reStat := &model.RelationStat{
+		EmbedMid:  model.EmbedMid{Mid: stat.Mid},
+		Follower:  stat.Follower,
+		Following: stat.Following,
+		Whisper:   stat.Whisper,
+		Black:     stat.Black,
+	}
+	return reStat, nil
+}
+
+func (s *Service) passportSummary(ctx context.Context, mid int64) (*model.PassportSummary, error) {
+	ps := &model.PassportSummary{
+		EmbedMid: model.EmbedMid{Mid: mid},
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	go func() error {
+		defer wg.Done()
+		pp, err := s.dao.PassportProfile(ctx, mid)
+		if err != nil {
+			log.Error("Failed to fetch passport profile: %+v", err)
+			return err
+		}
+		ps.TelStatus = pp.TelStatus()
+		ps.CountryID = pp.CountryCode
+		ps.JoinIP = pp.JoinIP
+		ps.JoinTime = model.Datetime(pp.JoinTime)
+		ps.EmailSuffix = pp.EmailSuffix()
+		return nil
+	}()
+	wg.Add(1)
+	go func() error {
+		defer wg.Done()
+		origin, err := s.dao.AsoAccountRegOrigin(ctx, mid)
+		if err != nil {
+			log.Error("Failed to fetch passport aso account reg origin: %+v", err)
+			return err
+		}
+		ps.RegType = origin.RegType
+		ps.OriginType = origin.OriginType
+		return nil
+	}()
+	wg.Wait()
+
+	return ps, nil
+}
+
+func (s *Service) member(ctx context.Context, mid int64) (*model.MemberBase, *model.MemberExp, *model.MemberOfficial, error) {
+	mb, err := s.dao.MemberService.Member(ctx, &member.ArgMemberMid{Mid: mid})
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	var base *model.MemberBase
+	if mb.BaseInfo != nil {
+		base = &model.MemberBase{
+			EmbedMid: model.EmbedMid{Mid: mb.Mid},
+			Name:     mb.Name,
+			Face:     mb.Face,
+			Rank:     int64(mb.Rank),
+			Sex:      mb.Sex,
+			Sign:     mb.Sign,
+			Birthday: model.Date(mb.Birthday),
+		}
+	}
+
+	var exp *model.MemberExp
+	if mb.LevelInfo != nil {
+		exp = &model.MemberExp{
+			EmbedMid: model.EmbedMid{Mid: mb.Mid},
+			Exp:      int64(mb.NowExp),
+		}
+	}
+
+	var of *model.MemberOfficial
+	if mb.OfficialInfo != nil {
+		of = &model.MemberOfficial{
+			EmbedMid:    model.EmbedMid{Mid: mb.Mid},
+			Role:        int64(mb.Role),
+			Title:       mb.Title,
+			Description: mb.Desc,
+		}
+	}
+
+	return base, exp, of, nil
+}
+
+func (s *Service) syncMember(ctx context.Context, mid int64) error {
+	base, exp, of, err := s.member(ctx, mid)
+	if err != nil {
+		return err
+	}
+
+	syncable := make([]Syncable, 0, 3)
+	if base != nil {
+		syncable = append(syncable, base)
+	}
+	if exp != nil {
+		syncable = append(syncable, exp)
+	}
+	if of != nil {
+		syncable = append(syncable, of)
+	}
+
+	for _, data := range syncable {
+		if err := s.SyncToHBase(ctx, data); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) syncRelationStat(ctx context.Context, mid int64) error {
+	reStat, err := s.relationStat(ctx, mid)
+	if err != nil {
+		return err
+	}
+	if err := s.SyncToHBase(ctx, reStat); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *Service) syncBlock(ctx context.Context, mid int64) error {
+	blSummary, err := s.block(ctx, mid)
+	if err != nil {
+		return err
+	}
+	if err := s.SyncToHBase(ctx, blSummary); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *Service) syncPassportSummary(ctx context.Context, mid int64) error {
+	ps, err := s.passportSummary(ctx, mid)
+	if err != nil {
+		return err
+	}
+	if err := s.SyncToHBase(ctx, ps); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *Service) syncRangeproc(ctx context.Context, start, end int64, worker uint64) {
+	log.Info("Sync with range: start: %d, end %d, worker: %d", start, end, worker)
+
+	syncChan := make(chan int64, worker*128)
+	defer close(syncChan)
+
+	// initial
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	for i := uint64(0); i < worker; i++ {
+		go func() {
+			defer wg.Done()
+			for mid := range syncChan {
+				if err := s.SyncOne(context.Background(), mid); err != nil {
+					log.Error("Failed to sync user with mid: %d: %+v", mid, err)
+				}
+			}
+		}()
+	}
+
+	for j := start; j <= end; j++ {
+		syncChan <- j
+	}
+
+	wg.Wait()
+}