Create & Init Project...

app/job/main/passport-game-data/service/BUILD

@@ -0,0 +1,64 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "batch_query_test.go",
+        "cloud2local_test.go",
+        "parse_diff_log_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/passport-game-data/conf:go_default_library",
+        "//app/job/main/passport-game-data/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "batch_query.go",
+        "cloud2local.go",
+        "compare_only.go",
+        "diff.go",
+        "init_cloud.go",
+        "local2cloud.go",
+        "log_and_fix.go",
+        "parse_diff_log.go",
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/passport-game-data/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-game-data/conf:go_default_library",
+        "//app/job/main/passport-game-data/dao:go_default_library",
+        "//app/job/main/passport-game-data/model:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-game-data/service/batch_query.go

@@ -0,0 +1,106 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-game-data/model"
+	"go-common/library/log"
+)
+
+func (s *Service) batchQueryLocalNonMiss(c context.Context, mids []int64, batchSize, batchMissRetryCount int) (res []*model.OriginAsoAccount) {
+	as, miss := s.batchQueryLocalByMid(c, mids, batchSize)
+	if len(miss) == 0 {
+		return as
+	}
+
+	res = as
+	for i := 0; i < batchMissRetryCount; i++ {
+		log.Info("try for the %dth retry, miss mids: %v", miss)
+		as, miss = s.batchQueryLocalByMid(c, miss, batchSize)
+
+		res = append(res, as...)
+
+		if len(miss) == 0 {
+			return
+		}
+
+		if i == batchMissRetryCount-1 {
+			log.Error("still miss those mids: %v after %d tries", miss, batchMissRetryCount)
+		}
+	}
+	return
+}
+
+func (s *Service) batchQueryLocalByMid(c context.Context, mids []int64, batchSize int) (res []*model.OriginAsoAccount, miss []int64) {
+	if len(mids) == 0 {
+		return
+	}
+	res = make([]*model.OriginAsoAccount, 0)
+	miss = make([]int64, 0)
+	bc := len(mids)/batchSize + 1
+	for i := 0; i < bc; i++ {
+		start := i * batchSize
+		end := (i + 1) * batchSize
+		if end > len(mids) {
+			end = len(mids)
+		}
+		partMids := mids[start:end]
+		as, err := s.d.AsoAccountsLocal(c, partMids)
+		if err != nil {
+			miss = append(miss, partMids...)
+			continue
+		}
+		res = append(res, as...)
+	}
+	return
+}
+
+func (s *Service) batchQueryCloudNonMiss(c context.Context, mids []int64, batchSize, batchMissRetryCount int) (res []*model.AsoAccount) {
+	if len(mids) == 0 {
+		return
+	}
+	as, miss := s.batchQueryCloudByMid(c, mids, batchSize)
+	if len(miss) == 0 {
+		return as
+	}
+
+	res = as
+	for i := 0; i < batchMissRetryCount; i++ {
+		log.Info("try for the %dth times, miss mids: %v", i, miss)
+		as, miss = s.batchQueryCloudByMid(c, miss, batchSize)
+
+		res = append(res, as...)
+		if len(miss) == 0 {
+			return
+		}
+
+		if i == batchMissRetryCount-1 {
+			log.Error("still miss those mids: %v after %d tries", miss, batchMissRetryCount)
+		}
+	}
+	return
+}
+
+func (s *Service) batchQueryCloudByMid(c context.Context, mids []int64, batchSize int) (res []*model.AsoAccount, miss []int64) {
+	if len(mids) == 0 {
+		return
+	}
+	res = make([]*model.AsoAccount, 0)
+	miss = make([]int64, 0)
+	bc := len(mids)/batchSize + 1
+	for i := 0; i < bc; i++ {
+		start := i * batchSize
+		end := (i + 1) * batchSize
+		if end > len(mids) {
+			end = len(mids)
+		}
+		partMids := mids[start:end]
+		as, err := s.d.AsoAccountsCloud(c, partMids)
+		if err != nil {
+			miss = append(miss, partMids...)
+			continue
+		}
+		res = append(res, as...)
+	}
+	return
+}

app/job/main/passport-game-data/service/batch_query_test.go

@@ -0,0 +1,36 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_batchQueryCloudByMid(t *testing.T) {
+	once.Do(startService)
+
+	Convey("batch query cloud by mid", t, func() {
+		mids := []int64{88888970, 88894849}
+		res, miss := s.batchQueryCloudByMid(context.TODO(), mids, 1)
+		str, _ := json.Marshal(res)
+		t.Logf("res: %s", str)
+
+		So(len(miss), ShouldEqual, 0)
+		So(len(res), ShouldEqual, 2)
+	})
+}
+
+func TestService_batchQueryCloudByMidNonMiss(t *testing.T) {
+	once.Do(startService)
+
+	Convey("batch query cloud by mid", t, func() {
+		mids := []int64{88888970, 88894849}
+		res := s.batchQueryCloudNonMiss(context.TODO(), mids, 1001, 1)
+		str, _ := json.Marshal(res)
+		t.Logf("res: %s", str)
+
+		So(len(res), ShouldEqual, 2)
+	})
+}

app/job/main/passport-game-data/service/cloud2local.go

@@ -0,0 +1,156 @@
+package service
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"go-common/app/job/main/passport-game-data/model"
+	"go-common/library/log"
+)
+
+// cloud2localcompareproc compare aso accounts between cloud and local.
+// select last modified from cloud
+// load batchSize from origin
+// compare:
+// if cloud_mtime >= local_mtime, directly compare
+// if cloud_mtime < local_mtime, sleep and reload from cloud, then do compare again
+func (s *Service) cloud2localcompareproc() {
+	var (
+		err      error
+		cloudRes []*model.AsoAccount
+
+		ack = false
+	)
+
+	cc := s.c2lC
+
+	delay := cc.DelayDuration
+
+	cc.st = cc.StartTime
+	cc.ed = cc.st.Add(cc.StepDuration)
+
+	offsetFile, err := os.Create(cc.OffsetFilePath)
+	if err != nil {
+		log.Error("failed to create offset file, os.Create(%s) error(%v)", cc.OffsetFilePath, err)
+		return
+	}
+	defer offsetFile.Close()
+	log.Info("created offset file %s", cc.OffsetFilePath)
+
+	for {
+		time.Sleep(cc.LoopDuration)
+
+		cc.sleeping = false
+
+		if ack {
+			cc.st = cc.st.Add(cc.StepDuration)
+			cc.ed = cc.ed.Add(cc.StepDuration)
+		}
+
+		st, ed := cc.st, cc.ed
+
+		if err = ioutil.WriteFile(cc.OffsetFilePath, []byte(st.Format(_timeFormat)), os.ModeAppend); err != nil {
+			log.Error("failed to write offset, ioutil.WriteFile(%s, %s, os.ModeAppend), error(%v)", cc.OffsetFilePath, st.Format(_timeFormat), err)
+			continue
+		}
+
+		if cc.Debug {
+			log.Info("st: %s, ed: %s", st.Format(_timeFormat), ed.Format(_timeFormat))
+		}
+
+		if cc.End && st.After(cc.EndTime) {
+			log.Info("st:%s is after endTime:%s, all data compares ok, cloud2localcompareproc exit", st.Format(_timeFormat), cc.EndTime.Format(_timeFormat))
+			return
+		}
+
+		now := time.Now()
+
+		if now.Sub(st) <= delay {
+			delta := int64(delay/time.Second) - (now.Unix() - st.Unix())
+			log.Info("now time is just after st by %d seconds, not greater than delay duration: %v, will sleep %d seconds", int64(delay/time.Second)-delta, delay, delta)
+
+			cc.sleeping = true
+			cc.sleepingSeconds = delta
+			cc.sleepFromTs = now.Unix()
+
+			time.Sleep(time.Duration(int64(time.Second) * delta))
+			continue
+		}
+
+		if cloudRes, err = s.d.AsoAccountRangeCloud(context.TODO(), st, ed); err != nil {
+			continue
+		}
+		cc.rangeCount = len(cloudRes)
+		cc.totalCount += len(cloudRes)
+
+		if err = s.cloud2LocalCompare(context.TODO(), cloudRes); err != nil {
+			continue
+		}
+
+		ack = true
+	}
+}
+
+func (s *Service) cloud2LocalCompare(c context.Context, cloudRes []*model.AsoAccount) (err error) {
+	mids := make([]int64, 0)
+	for _, item := range cloudRes {
+		mids = append(mids, item.Mid)
+	}
+
+	cc := s.c2lC
+
+	localRes := s.batchQueryLocalNonMiss(context.TODO(), mids, cc.BatchSize, cc.BatchMissRetryCount)
+
+	m := make(map[int64]*model.OriginAsoAccount)
+	for _, item := range localRes {
+		m[item.Mid] = item
+	}
+
+	// compare
+	pendingMids := make([]int64, 0)
+	for _, item := range cloudRes {
+		cloud := item
+		local := m[item.Mid]
+		status := doCompare(cloud, local, true)
+		switch status {
+		case _statusOK:
+			// do nothing
+		case _statusNo:
+			cc.diffCount++
+			s.doLog(cloud, local, false)
+			if cc.Fix {
+				s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
+			}
+		case _statusPending:
+			pendingMids = append(pendingMids, item.Mid)
+		}
+	}
+
+	if len(pendingMids) == 0 {
+		return
+	}
+
+	// reload pending mids from cloud
+	var pendingRes []*model.AsoAccount
+	if pendingRes, err = s.d.AsoAccountsCloud(context.TODO(), pendingMids); err != nil {
+		return
+	}
+	// compare
+	for _, item := range pendingRes {
+		cloud := item
+		local := m[item.Mid]
+		status := doCompare(item, m[item.Mid], false)
+		switch status {
+		case _statusOK:
+		case _statusNo:
+			cc.diffCount++
+			s.doLog(cloud, local, true)
+			if cc.Fix {
+				s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
+			}
+		}
+	}
+	return
+}

app/job/main/passport-game-data/service/cloud2local_test.go

@@ -0,0 +1,14 @@
+package service
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_DoCompare(t *testing.T) {
+	once.Do(startService)
+
+	Convey("compare", t, func() {
+	})
+}

app/job/main/passport-game-data/service/compare_only.go

@@ -0,0 +1,72 @@
+package service
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"os"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/passport-game-data/conf"
+	"go-common/app/job/main/passport-game-data/dao"
+	"go-common/library/log"
+)
+
+// NewCompareOnly new a service for compare only.
+func NewCompareOnly(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:    c,
+		d:    dao.New(c),
+		l2cC: newCompareConfigFrom(c.Compare.Local2Cloud),
+	}
+	return
+}
+
+// CompareFromMidListFile load mid list from file and compare.
+func (s *Service) CompareFromMidListFile(c context.Context, fn string) (err error) {
+	var f *os.File
+	if f, err = os.Open(fn); err != nil {
+		log.Error("failed to open file %s, error(%v)", fn, err)
+		return
+	}
+	defer f.Close()
+
+	cc := s.l2cC
+
+	rd := bufio.NewReader(f)
+	skippedCount := 0
+	var (
+		mid      int64
+		mids     = make([]int64, 0)
+		line     []byte
+		isPrefix bool
+	)
+	for {
+		line, isPrefix, err = rd.ReadLine()
+
+		if isPrefix || err != nil || err == io.EOF {
+			break
+		}
+		mid, err = strconv.ParseInt(string(line), 10, 64)
+		if err != nil {
+			log.Error("failed to parse mid, strconv.ParseInt(%s, 10 ,64) error(%v), skip", line, err)
+			skippedCount++
+			continue
+		}
+		mids = append(mids, mid)
+	}
+	log.Info("mid list len: %d, total skipped count: %d", len(mids), skippedCount)
+	if len(mids) == 0 {
+		return
+	}
+
+	for {
+		time.Sleep(cc.LoopDuration)
+		if err = s.local2CloudCompare(context.TODO(), s.batchQueryLocalNonMiss(context.TODO(), mids, cc.BatchSize, cc.BatchMissRetryCount)); err == nil {
+			break
+		}
+		log.Error("failed to compare mids %v, retrying", mids)
+	}
+	return
+}

app/job/main/passport-game-data/service/diff.go

@@ -0,0 +1,29 @@
+package service
+
+import (
+	"go-common/app/job/main/passport-game-data/model"
+	"go-common/library/log"
+)
+
+const (
+	_statusOK      = 0
+	_statusPending = 1
+	_statusNo      = 2
+)
+
+func doCompare(cloud *model.AsoAccount, local *model.OriginAsoAccount, pending bool) int {
+	if cloud == nil || local == nil {
+		log.Info("either cloud or local aso account is nil, cloud %+v, local: %+v", cloud, local)
+		return _statusNo
+	}
+	if cloud.Mtime.After(local.Mtime) {
+		if model.Default(local).Equals(cloud) {
+			return _statusOK
+		}
+		return _statusNo
+	}
+	if pending {
+		return _statusPending
+	}
+	return _statusNo
+}

app/job/main/passport-game-data/service/init_cloud.go

@@ -0,0 +1,146 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/passport-game-data/conf"
+	"go-common/app/job/main/passport-game-data/dao"
+	"go-common/app/job/main/passport-game-data/model"
+	"go-common/library/log"
+)
+
+const (
+	_defaultInitCloudOffsetFilePath = "/data/passport-game-data-job.initcloud.offset"
+
+	_defaultInitCloudSleep = time.Second
+)
+
+type initCloudConfig struct {
+	OffsetFilePath string
+	UseOldOffset   bool
+
+	Start, End int64
+
+	Batch int
+
+	Sleep time.Duration
+}
+
+func newInitCloudConfigFrom(c *conf.Config) (ic *initCloudConfig) {
+	ic = &initCloudConfig{
+		OffsetFilePath: c.InitCloud.OffsetFilePath,
+		UseOldOffset:   c.InitCloud.UseOldOffset,
+
+		Start: c.InitCloud.Start,
+		End:   c.InitCloud.End,
+
+		Batch: c.InitCloud.Batch,
+
+		Sleep: time.Duration(c.InitCloud.Sleep),
+	}
+
+	ic.fix()
+
+	if ic.UseOldOffset {
+		data, err := ioutil.ReadFile(ic.OffsetFilePath)
+		if err != nil {
+			log.Error("failed to read old offset, skip")
+			return
+		}
+
+		oldOffset, err := strconv.ParseInt(string(data), 10, 64)
+		if err != nil {
+			log.Error("failed to parse offset, strconv.ParseInt(%s, 10, 64)", string(data), err)
+			return
+		}
+
+		if oldOffset > 0 {
+			ic.Start = oldOffset
+		}
+	}
+	return
+}
+
+func (ic *initCloudConfig) fix() {
+	if len(ic.OffsetFilePath) == 0 {
+		ic.OffsetFilePath = _defaultInitCloudOffsetFilePath
+	}
+	if ic.Start < 0 {
+		ic.Start = 0
+	}
+	if ic.End < 0 {
+		ic.End = 0
+	}
+
+	if ic.Batch <= 0 {
+		ic.Batch = _defaultBatchSize
+	}
+
+	if int64(ic.Sleep) < 0 {
+		ic.Sleep = _defaultInitCloudSleep
+	}
+}
+
+// NewInitCloud new a service for initiating cloud.
+func NewInitCloud(c *conf.Config) (s *Service) {
+	s = &Service{
+		c: c,
+		d: dao.New(c),
+
+		ic: newInitCloudConfigFrom(c),
+	}
+	return
+}
+
+// InitCloud init cloud.
+func (s *Service) InitCloud(c context.Context) {
+	var err error
+	ic := s.ic
+
+	dstFile, err := os.Create(ic.OffsetFilePath)
+	if err != nil {
+		log.Error("failed to open file %s, error(%v)", ic.OffsetFilePath, err)
+		return
+	}
+	defer dstFile.Close()
+
+	for i := ic.Start; i <= ic.End; {
+		time.Sleep(ic.Sleep)
+		if err = ioutil.WriteFile(ic.OffsetFilePath, []byte(strconv.FormatInt(i, 10)), os.ModeAppend); err != nil {
+			log.Error("failed to record offset, offsetFilePath: %s, offset: %d, error(%v)", ic.OffsetFilePath, i, err)
+			continue
+		}
+		st := i
+		ed := i + int64(ic.Batch)
+		if ed > ic.End {
+			ed = ic.End
+		}
+
+		mids := make([]int64, 0)
+		for j := st; j <= ed; j++ {
+			mids = append(mids, j)
+		}
+
+		var as []*model.OriginAsoAccount
+		if as, err = s.d.AsoAccountsLocal(c, mids); err != nil {
+			log.Error("failed to get local aso accounts by mids, service.dao.AsoAccountsLocal(%v) error(%v)", mids, err)
+			continue
+		}
+
+		cloudAs := make([]*model.AsoAccount, 0)
+		for _, a := range as {
+			cloudAs = append(cloudAs, model.Default(a))
+		}
+		if err = s.d.AddAsoAccountsCloud(c, cloudAs); err != nil {
+			str, _ := json.Marshal(cloudAs)
+			log.Error("failed to add aso accounts to cloud, service.dao.AddAsoAccountsCloud(%v) error(%v)", str, err)
+			continue
+		}
+		i += int64(ic.Batch)
+	}
+}

app/job/main/passport-game-data/service/local2cloud.go

@@ -0,0 +1,162 @@
+package service
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"go-common/app/job/main/passport-game-data/model"
+	"go-common/library/log"
+)
+
+// local2cloudcompareproc compare aso accounts between local and cloud, delay for the given duration.
+// select last modified from cloud
+// load batchSize from origin
+// compare:
+// if cloud_mtime >= local_mtime, directly compare
+// if cloud_mtime < local_mtime, sleep and reload from cloud, then do compare again
+func (s *Service) local2cloudcompareproc() {
+	var (
+		err      error
+		localRes []*model.OriginAsoAccount
+
+		ack = false
+	)
+
+	cc := s.l2cC
+
+	delay := cc.DelayDuration
+
+	cc.st = cc.StartTime
+	cc.ed = cc.st.Add(cc.StepDuration)
+
+	offsetFile, err := os.Create(cc.OffsetFilePath)
+	if err != nil {
+		log.Error("failed to create offset file, os.Create(%s) error(%v)", cc.OffsetFilePath, err)
+		return
+	}
+	defer offsetFile.Close()
+	log.Info("created offset file %s", cc.OffsetFilePath)
+
+	for {
+		time.Sleep(cc.LoopDuration)
+
+		cc.sleeping = false
+
+		if ack {
+			cc.st = cc.st.Add(cc.StepDuration)
+			cc.ed = cc.ed.Add(cc.StepDuration)
+		}
+
+		st, ed := cc.st, cc.ed
+
+		if err = ioutil.WriteFile(cc.OffsetFilePath, []byte(st.Format(_timeFormat)), os.ModeAppend); err != nil {
+			log.Error("failed to write offset, ioutil.WriteFile(%s, %s, os.ModeAppend), error(%v)", cc.OffsetFilePath, st.Format(_timeFormat), err)
+			continue
+		}
+
+		if cc.Debug {
+			log.Info("st: %s, ed: %s", st.Format(_timeFormat), ed.Format(_timeFormat))
+		}
+
+		if cc.End && st.After(cc.EndTime) {
+			log.Info("st:%s is after endTime:%s, all data compares ok, local2cloudcompareproc exit", st.Format(_timeFormat), cc.EndTime.Format(_timeFormat))
+			return
+		}
+
+		now := time.Now()
+
+		if now.Sub(st) <= delay {
+			delta := int64(delay/time.Second) - (now.Unix() - st.Unix())
+			log.Info("now time is just after st by %d seconds, not greater than delay duration: %v, will sleep %d seconds", int64(delay/time.Second)-delta, delay, delta)
+
+			cc.sleeping = true
+			cc.sleepingSeconds = delta
+			cc.sleepFromTs = now.Unix()
+
+			time.Sleep(time.Duration(int64(time.Second) * delta))
+			continue
+		}
+
+		if localRes, err = s.d.AsoAccountRangeLocal(context.TODO(), st, ed); err != nil {
+			continue
+		}
+		cc.rangeCount = len(localRes)
+		cc.totalCount += len(localRes)
+
+		if err = s.local2CloudCompare(context.TODO(), localRes); err != nil {
+			continue
+		}
+
+		ack = true
+	}
+}
+
+func (s *Service) local2CloudCompare(c context.Context, lRes []*model.OriginAsoAccount) (err error) {
+	mids := make([]int64, 0)
+	for _, item := range lRes {
+		mids = append(mids, item.Mid)
+	}
+
+	cc := s.l2cC
+	// query from cloud
+	cRes := s.batchQueryCloudNonMiss(context.TODO(), mids, cc.BatchSize, cc.BatchMissRetryCount)
+
+	cM := make(map[int64]*model.AsoAccount)
+	for _, item := range cRes {
+		cM[item.Mid] = item
+	}
+
+	// compare cloud with local
+	pendingMids := make([]int64, 0)
+	for _, item := range lRes {
+		local := item
+		cloud := cM[item.Mid]
+		status := doCompare(cloud, local, true)
+		switch status {
+		case _statusOK:
+			// do nothing
+		case _statusNo:
+			cc.diffCount++
+			s.doLog(cloud, local, false)
+			if cc.Fix {
+				s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
+			}
+		case _statusPending:
+			pendingMids = append(pendingMids, item.Mid)
+		}
+	}
+
+	if len(pendingMids) == 0 {
+		return
+	}
+
+	// reload pending mids from cloud
+	var pendingRes []*model.AsoAccount
+	if pendingRes, err = s.d.AsoAccountsCloud(context.TODO(), pendingMids); err != nil {
+		return
+	}
+
+	lM := make(map[int64]*model.OriginAsoAccount)
+	for _, item := range lRes {
+		lM[item.Mid] = item
+	}
+
+	// compare
+	for _, item := range pendingRes {
+		cloud := item
+		local := lM[item.Mid]
+		status := doCompare(item, local, false)
+		switch status {
+		case _statusOK:
+		case _statusNo:
+			cc.diffCount++
+			s.doLog(cloud, local, true)
+			if cc.Fix {
+				s.fixCloudRecord(context.TODO(), model.Default(local), cloud)
+			}
+		}
+	}
+	return
+}

app/job/main/passport-game-data/service/log_and_fix.go

@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/job/main/passport-game-data/model"
+	"go-common/library/log"
+)
+
+func (s *Service) fixCloudRecord(c context.Context, newRecord *model.AsoAccount, old *model.AsoAccount) {
+	if old == nil {
+		affected, err := s.d.AddIgnoreAsoAccount(c, newRecord)
+		if err != nil {
+			oldStr, _ := json.Marshal(old)
+			newStr, _ := json.Marshal(newRecord)
+			log.Error("failed to fix cloud record by adding,  old(%s) new(%s) error(%v)", oldStr, newStr, err)
+			return
+		}
+		if affected == 0 {
+			oldStr, _ := json.Marshal(old)
+			newStr, _ := json.Marshal(newRecord)
+			log.Error("failed to fix cloud record by adding because of concurrent update, old(%s) new(%s)", oldStr, newStr)
+			return
+		}
+
+		oldStr, _ := json.Marshal(old)
+		newStr, _ := json.Marshal(newRecord)
+		log.Info("fix cloud record by adding ok, old(%s) new(%s)", oldStr, newStr)
+		return
+	}
+	affected, err := s.d.UpdateAsoAccountCloud(c, newRecord, old.Mtime)
+	if err != nil {
+		oldStr, _ := json.Marshal(old)
+		newStr, _ := json.Marshal(newRecord)
+		log.Error("failed to fix cloud record by updating, old(%s) new(%s) error(%v)", oldStr, newStr, err)
+		return
+	}
+	if affected == 0 {
+		oldStr, _ := json.Marshal(old)
+		newStr, _ := json.Marshal(newRecord)
+		log.Error("failed to fix cloud record by updating because of concurrent update, old(%s) new(%s)", oldStr, newStr)
+		return
+	}
+
+	oldStr, _ := json.Marshal(old)
+	newStr, _ := json.Marshal(newRecord)
+	log.Info("fix cloud record by updating ok, old(%s) new(%s)", oldStr, newStr)
+}
+
+func (s *Service) doLog(cloud *model.AsoAccount, local *model.OriginAsoAccount, afterPending bool) {
+	localStr := []byte("nil")
+	localEncStr := []byte("nil")
+	if local != nil {
+		localStr, _ = json.Marshal(local)
+		localEncStr, _ = json.Marshal(model.Default(local))
+	}
+	cloudStr := []byte("nil")
+	if cloud != nil {
+		cloudStr, _ = json.Marshal(cloud)
+	}
+	if afterPending {
+		log.Info("failed to compare, because cloud record is not updated in time, local(%s) local_encrypted(%s) cloud(%s)", localStr, localEncStr, cloudStr)
+		return
+	}
+	log.Info("compare diff, local(%s) local_encrypted(%s) cloud(%s)", localStr, localEncStr, cloudStr)
+}

app/job/main/passport-game-data/service/parse_diff_log.go

@@ -0,0 +1,255 @@
+package service
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+
+	"go-common/app/job/main/passport-game-data/model"
+	"go-common/library/log"
+)
+
+const (
+	_cloudJobGoroutineNum = 32
+)
+
+// ParseDiffLog parse diff log printed by compare proc.
+func ParseDiffLog(src, dst string) (err error) {
+	f, err := os.Open(src)
+	if err != nil {
+		log.Error("failed to open file %s, error(%v)", src, err)
+		return
+	}
+	defer f.Close()
+
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		log.Error("failed to open file %s, error(%v)", dst, err)
+		return
+	}
+	defer dstFile.Close()
+
+	var (
+		line         string
+		skippedCount = 0
+		res          = make([]*model.CompareRes, 0)
+		rd           = bufio.NewReader(f)
+	)
+	for {
+		line, err = rd.ReadString('\n')
+
+		if err != nil || io.EOF == err {
+			break
+		}
+
+		idx := strings.LastIndex(line, "]")
+		if idx == -1 {
+			log.Error("failed to parse log, expected have ] in string but not")
+			skippedCount++
+			continue
+		}
+
+		logJSON := line[idx+1:]
+		l := new(model.Log)
+		if err = json.Unmarshal([]byte(logJSON), &l); err != nil {
+			log.Error("failed to parse log, json.Unmarshal(%s) error(%v), skip", logJSON, err)
+			skippedCount++
+			continue
+		}
+
+		var cRes *model.CompareRes
+		if cRes, err = diffLog2CompareRes(l.Log); err != nil {
+			log.Error("diffLog2CompareRes(%s) error(%v), skip", l.Log, err)
+			skippedCount++
+			continue
+		}
+
+		// compare local encrypted and cloud, parse diff flags
+		flags := diff(cRes.Cloud, cRes.LocalEncrypted)
+
+		if flags == _diffTypeNon {
+			continue
+		}
+
+		cRes.Flags = flags
+		cRes.FlagsDesc = formatFlags(flags)
+		cRes.Seq = cRes.Local.Mid % _cloudJobGoroutineNum
+		res = append(res, cRes)
+	}
+
+	percentMap := make(map[uint8]*model.CountAndPercent)
+	seqMap := make(map[int64]*model.SeqCountAndPercent)
+
+	for _, v := range res {
+		percent, ok := percentMap[v.Flags]
+		if !ok {
+			percent = &model.CountAndPercent{
+				DiffType: v.FlagsDesc,
+			}
+			percentMap[v.Flags] = percent
+		}
+		percent.Count++
+
+		seq, ok := seqMap[v.Seq]
+		if !ok {
+			seq = &model.SeqCountAndPercent{
+				Seq: v.Seq,
+			}
+			seqMap[v.Seq] = seq
+		}
+		seq.Count++
+	}
+
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].Cloud.Mtime.After(res[j].Cloud.Mtime)
+	})
+
+	percentList := make([]*model.CountAndPercent, 0)
+	for _, v := range percentMap {
+		v.Percent = fmt.Sprintf("%0.2f", 100*float64(v.Count)/float64(len(res))) + "%"
+		percentList = append(percentList, v)
+	}
+	sort.Slice(percentList, func(i, j int) bool {
+		return percentList[i].Count > percentList[j].Count
+	})
+
+	seqList := make([]*model.SeqCountAndPercent, 0)
+	for _, v := range seqMap {
+		v.Percent = fmt.Sprintf("%0.2f", 100*float64(v.Count)/float64(_cloudJobGoroutineNum)) + "%"
+		seqList = append(seqList, v)
+	}
+	sort.Slice(seqList, func(i, j int) bool {
+		return seqList[i].Count > seqList[j].Count
+	})
+
+	stat := &model.DiffParseResp{
+		Total:            len(res),
+		SeqAndPercents:   seqList,
+		CompareResList:   res,
+		CountAndPercents: percentList,
+	}
+
+	str, _ := json.Marshal(stat)
+	_, err = dstFile.WriteString(string(str))
+	if err != nil {
+		log.Info("failed to write parse diff log result to file %s, error(%v)", dst, err)
+	}
+
+	log.Info("len res: %d, write ok", len(res))
+	return
+}
+
+func diffLog2CompareRes(str string) (*model.CompareRes, error) {
+	idx := strings.Index(str, "local")
+	if idx == -1 {
+		return nil, fmt.Errorf("failed to parse diff log, expected have local in string but not")
+	}
+	res := replace(str[idx:])
+	cRes := new(model.CompareRes)
+	err := json.Unmarshal([]byte(res), &cRes)
+	return cRes, err
+}
+
+// parse string like "local({\"mid\":1}) local_encrypted({\"mid\":1}) cloud({\"mid\":1})" to json string {"local":{},"local_encrypted":{},"cloud":{}}
+func replace(str string) string {
+	res := strings.Replace(str, "local(", `{"local":`, -1)
+
+	res = strings.Replace(res, "local_encrypted(", `"local_encrypted":`, -1)
+
+	res = strings.Replace(res, "cloud(", `"cloud":`, -1)
+
+	res = strings.Replace(res, ")", ",", -1)
+
+	res = strings.Replace(res, "\\", "", -1)
+
+	if strings.HasSuffix(res, ",") {
+		res = res[:len(res)-1]
+	}
+
+	res = res + "}"
+	return res
+}
+
+const (
+	_diffTypeNon            = uint8(0)  // 0x00000000
+	_diffTypePwd            = uint8(1)  // 0x00000001
+	_diffTypeEmail          = uint8(2)  // 0x00000010
+	_diffTypeTel            = uint8(4)  // 0x00000100
+	_diffTypeCountryID      = uint8(16) // 0x00001000
+	_diffTypeMobileVerified = uint8(32) // 0x00010000
+	_diffTypeIsLeak         = uint8(64) // 0x00100000
+)
+
+func formatFlags(flags uint8) string {
+	fs := make([]string, 0)
+
+	if flags&_diffTypePwd > 0 {
+		fs = append(fs, "pwd")
+	}
+
+	if flags&_diffTypeEmail > 0 {
+		fs = append(fs, "email")
+	}
+
+	if flags&_diffTypeTel > 0 {
+		fs = append(fs, "tel")
+	}
+
+	if flags&_diffTypeCountryID > 0 {
+		fs = append(fs, "country_id")
+	}
+
+	if flags&_diffTypeMobileVerified > 0 {
+		fs = append(fs, "mobile_verified")
+	}
+
+	if flags&_diffTypeIsLeak > 0 {
+		fs = append(fs, "is_leak")
+	}
+
+	if len(fs) == 0 {
+		return "non"
+	}
+	return strings.Join(fs, ",")
+}
+
+func diff(cloud, localEncrypted *model.AsoAccount) uint8 {
+	if localEncrypted == cloud {
+		return _diffTypeNon
+	}
+	if localEncrypted == nil || cloud == nil {
+		return _diffTypePwd | _diffTypeEmail | _diffTypeTel
+	}
+
+	res := _diffTypeNon
+
+	if cloud.Salt != localEncrypted.Salt || cloud.Pwd != localEncrypted.Pwd {
+		res = res | _diffTypePwd
+	}
+
+	if cloud.Email != localEncrypted.Email {
+		res = res | _diffTypeEmail
+	}
+
+	if cloud.Tel != localEncrypted.Tel {
+		res = res | _diffTypeTel
+	}
+
+	if cloud.CountryID != localEncrypted.CountryID {
+		res = res | _diffTypeCountryID
+	}
+
+	if cloud.MobileVerified != localEncrypted.MobileVerified {
+		res = res | _diffTypeMobileVerified
+	}
+
+	if cloud.Isleak != localEncrypted.Isleak {
+		res = res | _diffTypeIsLeak
+	}
+
+	return res
+}

app/job/main/passport-game-data/service/parse_diff_log_test.go

@@ -0,0 +1,28 @@
+package service
+
+import (
+	"testing"
+
+	"encoding/json"
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/job/main/passport-game-data/model"
+)
+
+func TestParseDiffLog(t *testing.T) {
+	Convey("parse log text", t, func() {
+		str := `local({\"mid\":80793085,\"userid\":\"adeqdiffer\",\"uname\":\"adeqdiffer\",\"pwd\":\"33a5fd6290550b88cc229275e9f790f7\",\"salt\":\"SFhrkmK3\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"modify_time\":\"2018-01-21T21:36:50+08:00\"}) local_encrypted({\"mid\":80793085,\"userid\":\"adeqdiffer\",\"uname\":\"adeqdiffer\",\"pwd\":\"7f0aa1b3dadda0c483aa78c3f3b048cf\",\"salt\":\"SFhrkmK3\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"0001-01-01T00:00:00Z\",\"mtime\":\"2018-01-21T21:36:50+08:00\"}) cloud({\"mid\":80793085,\"userid\":\"adeqdiffer\",\"uname\":\"adeqdiffer\",\"pwd\":\"7f0aa1b3dadda0c483aa78c3f3b048cf\",\"salt\":\"SFhrkmK3\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"2017-11-16T19:47:16+08:00\",\"mtime\":\"2018-01-21T21:36:50+08:00\"}`
+
+		str = `local({\"mid\":83768597,\"userid\":\"difficenemy\",\"uname\":\"difficenemy\",\"pwd\":\"7e9f9a98269eb6fcc717f2d6e3a25fc2\",\"salt\":\"8pscksH6\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"modify_time\":\"2018-01-10T18:18:05+08:00\"}) local_encrypted({\"mid\":83768597,\"userid\":\"difficenemy\",\"uname\":\"difficenemy\",\"pwd\":\"08fa599f4497e876f7b4c7861f748361\",\"salt\":\"8pscksH6\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"0001-01-01T00:00:00Z\",\"mtime\":\"2018-01-10T18:18:05+08:00\"}) cloud({\"mid\":83768597,\"userid\":\"difficenemy\",\"uname\":\"difficenemy\",\"pwd\":\"08fa599f4497e876f7b4c7861f748361\",\"salt\":\"8pscksH6\",\"email\":\"\",\"tel\":\"\",\"country_id\":1,\"mobile_verified\":0,\"isleak\":0,\"ctime\":\"2017-11-16T23:59:21+08:00\",\"mtime\":\"2018-01-10T18:18:05+08:00\"})`
+
+		res := replace(str)
+
+		t.Logf("res: %s", res)
+		cRes := new(model.CompareRes)
+
+		err := json.Unmarshal([]byte(res), &cRes)
+		So(err, ShouldBeNil)
+
+		rStr, _ := json.Marshal(cRes)
+		t.Logf("res: %s, cRes: %s", res, rStr)
+	})
+}

app/job/main/passport-game-data/service/service.go

@@ -0,0 +1,175 @@
+package service
+
+import (
+	"context"
+	"io/ioutil"
+	"time"
+
+	"go-common/app/job/main/passport-game-data/conf"
+	"go-common/app/job/main/passport-game-data/dao"
+	"go-common/library/log"
+)
+
+const (
+	_defaultDelayDuration = time.Minute * 0
+	_defaultStepDuration  = time.Minute * 15
+	_defaultLoopDuration  = time.Second * 3
+
+	_defaultBatchSize           = 1000
+	_defaultBatchMissRetryCount = 3
+
+	_timeFormat = "2006-01-02 15:04:05"
+)
+
+var (
+	_loc = time.Now().Location()
+)
+
+// Service service.
+type Service struct {
+	c *conf.Config
+	d *dao.Dao
+
+	// init cloud
+	ic *initCloudConfig
+
+	// c2l
+	c2lC *compareConfig
+
+	// l2c
+	l2cC *compareConfig
+}
+
+type compareConfig struct {
+	On             bool
+	OffsetFilePath string
+	UseOldOffset   bool
+
+	End bool
+
+	StartTime     time.Time
+	EndTime       time.Time
+	DelayDuration time.Duration
+	StepDuration  time.Duration
+	LoopDuration  time.Duration
+
+	BatchSize           int
+	BatchMissRetryCount int
+
+	Debug bool
+	Fix   bool
+
+	// runtime
+	st, ed     time.Time
+	rangeCount int
+	totalCount int
+	diffCount  int
+
+	sleeping        bool
+	sleepingSeconds int64
+	sleepFromTs     int64
+}
+
+func newCompareConfigFrom(c *conf.CompareConfig) (cc *compareConfig) {
+	st, err := time.ParseInLocation(_timeFormat, c.StartTime, _loc)
+	if err != nil {
+		log.Error("failed to parse end time, time.ParseInLocation(%s, %s, %v), error(%v)", _timeFormat, c.StartTime, _loc, err)
+		return
+	}
+	ed, err := time.ParseInLocation(_timeFormat, c.EndTime, _loc)
+	if err != nil {
+		log.Error("failed to parse end time, time.ParseInLocation(%s, %s, %v), error(%v)", _timeFormat, c.EndTime, _loc, err)
+		return
+	}
+
+	cc = &compareConfig{
+		On:    c.On,
+		Debug: c.Debug,
+
+		OffsetFilePath: c.OffsetFilePath,
+		UseOldOffset:   c.UseOldOffset,
+		End:            c.End,
+
+		StartTime: st,
+		EndTime:   ed,
+
+		DelayDuration: time.Duration(c.DelayDuration),
+		StepDuration:  time.Duration(c.StepDuration),
+		LoopDuration:  time.Duration(c.LoopDuration),
+
+		BatchSize:           c.BatchSize,
+		BatchMissRetryCount: c.BatchMissRetryCount,
+
+		Fix: c.Fix,
+	}
+
+	if cc.UseOldOffset {
+		if oldOffset, err := parseOldOffset(cc.OffsetFilePath); err == nil {
+			cc.StartTime = oldOffset
+		}
+	}
+
+	cc.fix()
+	return
+}
+
+func parseOldOffset(path string) (oldOffset time.Time, err error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		log.Error("failed to read old offset, ioutil.ReadFile(%s) error(%v) skip", path, err)
+		return
+	}
+
+	if oldOffset, err = time.ParseInLocation(_timeFormat, string(data), _loc); err != nil {
+		log.Error("failed to parse offset, time.ParseInLocation(%s, %s, %v) error(%v)", _timeFormat, string(data), _loc, err)
+	}
+	return
+}
+
+func (cc *compareConfig) fix() {
+	if int64(cc.DelayDuration) < 0 {
+		cc.DelayDuration = _defaultDelayDuration
+	}
+	if int64(cc.StepDuration) < 0 {
+		cc.StepDuration = _defaultStepDuration
+	}
+	if int64(cc.LoopDuration) < 0 {
+		cc.LoopDuration = _defaultLoopDuration
+	}
+
+	if cc.BatchSize <= 0 {
+		cc.BatchSize = _defaultBatchSize
+	}
+	if cc.BatchMissRetryCount < 0 {
+		cc.BatchMissRetryCount = _defaultBatchMissRetryCount
+	}
+}
+
+// New new a service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c: c,
+		d: dao.New(c),
+	}
+	if c.Compare.Cloud2Local.On {
+		s.c2lC = newCompareConfigFrom(c.Compare.Cloud2Local)
+		go s.cloud2localcompareproc()
+	}
+	if c.Compare.Local2Cloud.On {
+		s.l2cC = newCompareConfigFrom(c.Compare.Local2Cloud)
+		go s.local2cloudcompareproc()
+	}
+	return
+}
+
+// Ping check server ok.
+func (s *Service) Ping(c context.Context) (err error) {
+	err = s.d.Ping(c)
+	return
+}
+
+// Close close service.
+func (s *Service) Close() (err error) {
+	s.d.Close()
+	return
+}

app/job/main/passport-game-data/service/service_test.go

@@ -0,0 +1,22 @@
+package service
+
+import (
+	"flag"
+
+	"go-common/app/job/main/passport-game-data/conf"
+	"sync"
+)
+
+var (
+	once sync.Once
+	s    *Service
+)
+
+func startService() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	// service init
+	s = New(conf.Conf)
+}

app/job/main/passport-game-data/service/stat.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/passport-game-data/model"
+)
+
+// CompareProcStat get actual compare proc stat.
+func (s *Service) CompareProcStat(c context.Context) *model.ProcStat {
+	var c2lStat *model.CompareProcStat
+	if s.c.Compare.Cloud2Local.On {
+		cc := s.c2lC
+		c2lStat = &model.CompareProcStat{
+			StartTime:     s.c.Compare.Cloud2Local.StartTime,
+			EndTime:       s.c.Compare.Cloud2Local.EndTime,
+			StepDuration:  model.JsonDuration(cc.StepDuration),
+			LoopDuration:  model.JsonDuration(cc.LoopDuration),
+			DelayDuration: model.JsonDuration(cc.DelayDuration),
+
+			BatchSize:           cc.BatchSize,
+			BatchMissRetryCount: cc.BatchMissRetryCount,
+
+			Debug: cc.Debug,
+			Fix:   cc.Fix,
+
+			CurrentRangeStart:        model.JSONTime(cc.st),
+			CurrentRangeEnd:          model.JSONTime(cc.ed),
+			CurrentRangeRecordsCount: cc.rangeCount,
+			TotalRangeRecordsCount:   cc.totalCount,
+			DiffCount:                cc.diffCount,
+
+			Sleeping: cc.sleeping,
+		}
+		if cc.sleeping {
+			c2lStat.SleepSeconds = cc.sleepingSeconds
+			c2lStat.SleepFrom = time.Unix(cc.sleepFromTs, 0).Format(_timeFormat)
+			c2lStat.SleepRemainSeconds = cc.sleepingSeconds - (time.Now().Unix() - cc.sleepFromTs)
+		}
+	}
+
+	var l2cStat *model.CompareProcStat
+	if s.c.Compare.Local2Cloud.On {
+		cc := s.l2cC
+		l2cStat = &model.CompareProcStat{
+			StartTime:     s.c.Compare.Local2Cloud.StartTime,
+			EndTime:       s.c.Compare.Local2Cloud.EndTime,
+			StepDuration:  model.JsonDuration(cc.StepDuration),
+			LoopDuration:  model.JsonDuration(cc.LoopDuration),
+			DelayDuration: model.JsonDuration(cc.DelayDuration),
+
+			BatchSize:           cc.BatchSize,
+			BatchMissRetryCount: cc.BatchMissRetryCount,
+
+			Debug: cc.Debug,
+			Fix:   cc.Fix,
+
+			CurrentRangeStart:        model.JSONTime(cc.st),
+			CurrentRangeEnd:          model.JSONTime(cc.ed),
+			CurrentRangeRecordsCount: cc.rangeCount,
+			TotalRangeRecordsCount:   cc.totalCount,
+			DiffCount:                cc.diffCount,
+
+			Sleeping: cc.sleeping,
+		}
+		if cc.sleeping {
+			l2cStat.SleepSeconds = cc.sleepingSeconds
+			l2cStat.SleepFrom = time.Unix(cc.sleepFromTs, 0).Format(_timeFormat)
+			l2cStat.SleepRemainSeconds = cc.sleepingSeconds - (time.Now().Unix() - cc.sleepFromTs)
+		}
+	}
+
+	return &model.ProcStat{
+		Cloud2Local: c2lStat,
+		Local2Cloud: l2cStat,
+	}
+}