Create & Init Project...

app/job/main/passport-encrypt/service/BUILD

@@ -0,0 +1,60 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aes.go",
+        "aso.go",
+        "aso_full_migration.go",
+        "aso_incr_migration.go",
+        "passport_util.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/passport-encrypt/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/dao:go_default_library",
+        "//app/job/main/passport-encrypt/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "aes_test.go",
+        "aso_full_migration_test.go",
+        "aso_incr_migration_test.go",
+        "passport_util_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/passport-encrypt/service/aes.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+)
+
+var key = []byte("bili_account_enc")
+
+// Encrypt aes encrypt
+func Encrypt(origData []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	blockSize := block.BlockSize()
+	origData = PKCS5Padding(origData, blockSize)
+	blockMode := cipher.NewCBCEncrypter(block, key[:blockSize])
+	crypted := make([]byte, len(origData))
+	blockMode.CryptBlocks(crypted, origData)
+	return crypted, nil
+}
+
+// Decrypt aes encrypt
+func Decrypt(crypted []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	blockSize := block.BlockSize()
+	blockMode := cipher.NewCBCDecrypter(block, key[:blockSize])
+	origData := make([]byte, len(crypted))
+	blockMode.CryptBlocks(origData, crypted)
+	origData = PKCS5UnPadding(origData)
+	return origData, nil
+}
+
+// PKCS5Padding padding
+func PKCS5Padding(ciphertext []byte, blockSize int) []byte {
+	padding := blockSize - len(ciphertext)%blockSize
+	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
+	return append(ciphertext, padtext...)
+}
+
+// PKCS5UnPadding unpadding
+func PKCS5UnPadding(origData []byte) []byte {
+	length := len(origData)
+	unpadding := int(origData[length-1])
+	return origData[:(length - unpadding)]
+}

app/job/main/passport-encrypt/service/aes_test.go

@@ -0,0 +1,30 @@
+package service
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_Encrypt(t *testing.T) {
+	Convey("Encrypt  param ", t, func() {
+		input := []byte("13122119298")
+		res, err := Encrypt(input)
+		ShouldBeNil(err)
+		ShouldNotBeNil(res)
+		fmt.Printf("(%+v) \n", res)
+	})
+
+}
+
+func TestService_Decrypt(t *testing.T) {
+	Convey("Decrypt param ", t, func() {
+		input := []byte{70, 251, 31, 100, 140, 98, 156, 101, 27, 21, 201, 100, 159, 107, 90, 147}
+		fmt.Printf("input is  (%+v) \n", input)
+		res, err := Decrypt(input)
+		ShouldBeNil(err)
+		ShouldNotBeNil(res)
+		fmt.Printf("(%+v) \n", string(res))
+	})
+}

app/job/main/passport-encrypt/service/aso.go

@@ -0,0 +1,31 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-encrypt/model"
+)
+
+func (s *Service) saveEncryptAccount(c context.Context, account *model.EncryptAccount) (err error) {
+	var affect int64
+	if affect, err = s.d.AddAsoAccount(c, account); err != nil || affect == 0 {
+		return
+	}
+	return
+}
+
+func (s *Service) updateEncryptAccount(c context.Context, account *model.EncryptAccount) (err error) {
+	var affect int64
+	if affect, err = s.d.UpdateAsoAccount(c, account); err != nil || affect == 0 {
+		return
+	}
+	return
+}
+
+func (s *Service) delEncryptAccount(c context.Context, mid int64) (err error) {
+	var affect int64
+	if affect, err = s.d.DelAsoAccount(c, mid); err != nil || affect == 0 {
+		return
+	}
+	return
+}

app/job/main/passport-encrypt/service/aso_full_migration.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-encrypt/model"
+	"go-common/library/log"
+)
+
+// fullMigration full migration
+func (s *Service) fullMigration(start, end, inc, limit int64, group string) {
+	var count int64
+
+	for {
+		log.Info(" group is %s,start %d, end %d,inc %d,limit %d", group, start, end, inc, limit)
+		var (
+			res      []*model.OriginAccount
+			err      error
+			effected int64
+		)
+		if res, err = s.d.AsoAccounts(context.TODO(), start, end); err != nil {
+			log.Error("error is (%+v)\n", err)
+			break
+		}
+
+		for _, acc := range res {
+			enAcc := EncryptAccount(acc)
+			if effected, err = s.d.AddAsoAccount(context.TODO(), enAcc); err != nil || effected == 0 {
+				log.Error("insert error (%+v)", err)
+			}
+		}
+		start = end
+		count = end
+		end = end + inc
+
+		if count >= limit {
+			break
+		}
+	}
+}

app/job/main/passport-encrypt/service/aso_full_migration_test.go

@@ -0,0 +1,14 @@
+package service
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_FullMigration(t *testing.T) {
+	once.Do(startService)
+	Convey("test full migration ", t, func() {
+		s.fullMigration(0, 10, 5, 50, "")
+	})
+}

app/job/main/passport-encrypt/service/aso_incr_migration.go

@@ -0,0 +1,130 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/passport-encrypt/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+func (s *Service) asobinlogconsumeproc() {
+	mergeNum := int64(s.c.Group.AsoBinLog.Num)
+	for {
+		msg, ok := <-s.dsAsoBinLogSub.Messages()
+		if !ok {
+			log.Error("asobinlogconsumeproc closed")
+			return
+		}
+		// marked head to first commit
+		m := &message{data: msg}
+		s.mu.Lock()
+		if s.head == nil {
+			s.head = m
+			s.last = m
+		} else {
+			s.last.next = m
+			s.last = m
+		}
+		s.mu.Unlock()
+		bmsg := new(model.BMsg)
+		if err := json.Unmarshal(msg.Value, bmsg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+		mid := int64(0)
+		if bmsg.Table == _asoAccountTable {
+			t := new(model.OriginAccount)
+			if err := json.Unmarshal(bmsg.New, t); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+			}
+			mid = t.Mid
+			m.object = bmsg
+			log.Info("asobinlogconsumeproc table:%s key:%s partition:%d offset:%d", bmsg.Table, msg.Key, msg.Partition, msg.Offset)
+		} else {
+			continue
+		}
+		s.merges[mid%mergeNum] <- m
+	}
+}
+
+func (s *Service) asobinlogcommitproc() {
+	for {
+		done := <-s.done
+		commits := make(map[int32]*databus.Message)
+		for _, d := range done {
+			d.done = true
+		}
+		s.mu.Lock()
+		for ; s.head != nil && s.head.done; s.head = s.head.next {
+			commits[s.head.data.Partition] = s.head.data
+		}
+		s.mu.Unlock()
+		for _, m := range commits {
+			m.Commit()
+		}
+	}
+}
+
+func (s *Service) asobinlogmergeproc(c chan *message) {
+	var (
+		max    = s.c.Group.AsoBinLog.Size
+		merges = make([]*model.BMsg, 0, max)
+		marked = make([]*message, 0, max)
+		ticker = time.NewTicker(time.Duration(s.c.Group.AsoBinLog.Ticker))
+	)
+	for {
+		select {
+		case msg, ok := <-c:
+			if !ok {
+				log.Error("asobinlogmergeproc closed")
+				return
+			}
+			p, assertOk := msg.object.(*model.BMsg)
+			if assertOk && p.Action != "" && (p.Table == _asoAccountTable) {
+				merges = append(merges, p)
+			}
+			marked = append(marked, msg)
+			if len(marked) < max && len(merges) < max {
+				continue
+			}
+		case <-ticker.C:
+		}
+		if len(merges) > 0 {
+			s.processAsoAccLogInfo(merges)
+			merges = make([]*model.BMsg, 0, max)
+		}
+		if len(marked) > 0 {
+			s.done <- marked
+			marked = make([]*message, 0, max)
+		}
+	}
+}
+
+func (s *Service) processAsoAccLogInfo(bmsgs []*model.BMsg) {
+	for _, msg := range bmsgs {
+		s.processAsoAccLog(msg)
+	}
+}
+
+func (s *Service) processAsoAccLog(msg *model.BMsg) {
+	aso := new(model.OriginAccount)
+	if err := json.Unmarshal(msg.New, aso); err != nil {
+		log.Error("failed to parse binlog new, json.Unmarshal(%s) error(%v)", string(msg.New), err)
+		return
+	}
+	if _updateAction == msg.Action {
+		enAso := EncryptAccount(aso)
+		s.updateEncryptAccount(context.TODO(), enAso)
+	}
+	if _insertAction == msg.Action {
+		enAso := EncryptAccount(aso)
+		s.saveEncryptAccount(context.TODO(), enAso)
+	}
+	if _deleteAction == msg.Action {
+		s.delEncryptAccount(context.TODO(), aso.Mid)
+	}
+
+}

app/job/main/passport-encrypt/service/aso_incr_migration_test.go

@@ -0,0 +1,23 @@
+package service
+
+import (
+	"testing"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_IncMigration(t *testing.T) {
+	once.Do(startService)
+	c := conf.Conf
+	Convey("test full migration ", t, func() {
+		s.asobinlogconsumeproc()
+		for i := 0; i < c.Group.AsoBinLog.Num; i++ {
+			ch := make(chan *message, c.Group.AsoBinLog.Chan)
+			s.merges[i] = ch
+			s.asobinlogmergeproc(ch)
+		}
+		s.asobinlogcommitproc()
+	})
+}

app/job/main/passport-encrypt/service/passport_util.go

@@ -0,0 +1,67 @@
+package service
+
+import "go-common/app/job/main/passport-encrypt/model"
+
+// EncryptAccount Encrypt Account
+func EncryptAccount(a *model.OriginAccount) (res *model.EncryptAccount) {
+	return &model.EncryptAccount{
+		Mid:            a.Mid,
+		UserID:         a.UserID,
+		Uname:          a.Uname,
+		Pwd:            a.Pwd,
+		Salt:           a.Salt,
+		Email:          a.Email,
+		Tel:            doEncrypt(a.Tel),
+		CountryID:      a.CountryID,
+		MobileVerified: a.MobileVerified,
+		Isleak:         a.Isleak,
+		Mtime:          a.Mtime,
+	}
+}
+
+// DecryptAccount Decrypt Account
+func DecryptAccount(a *model.EncryptAccount) (res *model.OriginAccount) {
+	return &model.OriginAccount{
+		Mid:            a.Mid,
+		UserID:         a.UserID,
+		Uname:          a.Uname,
+		Pwd:            a.Pwd,
+		Salt:           a.Salt,
+		Email:          a.Email,
+		Tel:            doDecrypt(a.Tel),
+		CountryID:      a.CountryID,
+		MobileVerified: a.MobileVerified,
+		Isleak:         a.Isleak,
+		Mtime:          a.Mtime,
+	}
+}
+
+func doEncrypt(param string) []byte {
+	var (
+		err error
+		res = make([]byte, 0)
+	)
+	if param == "" || len(param) == 0 {
+		return nil
+	}
+	input := []byte(param)
+	if res, err = Encrypt(input); err != nil {
+		return input
+	}
+	return res
+}
+
+func doDecrypt(param []byte) string {
+	var (
+		err error
+		res = make([]byte, 0)
+	)
+	if param == nil {
+		return ""
+	}
+	input := []byte(param)
+	if res, err = Decrypt(input); err != nil {
+		return string(param)
+	}
+	return string(res)
+}

app/job/main/passport-encrypt/service/passport_util_test.go

@@ -0,0 +1,82 @@
+package service
+
+import (
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/passport-encrypt/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_EncryptAccount(t *testing.T) {
+	Convey("Encrypt a right account", t, func() {
+		account := &model.OriginAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            "13122110209",
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := EncryptAccount(account)
+		fmt.Printf("right (+%v)", enAcc)
+	})
+
+	Convey("Encrypt a empty tel account", t, func() {
+		account := &model.OriginAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            "",
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := EncryptAccount(account)
+		fmt.Printf("empty tel (+%v)", enAcc)
+	})
+}
+
+func TestService_DecryptAccount(t *testing.T) {
+	Convey("Decrypt right account", t, func() {
+		account := &model.EncryptAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            []byte{216, 28, 241, 48, 17, 243, 198, 234, 205, 33, 216, 25, 75, 146, 97, 203},
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := DecryptAccount(account)
+		fmt.Printf("(+%v)", enAcc)
+	})
+
+	Convey("Decrypt empty tel  account", t, func() {
+		account := &model.EncryptAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            []byte{73, 224, 88, 238, 85, 242, 37, 46, 200, 70, 5, 49, 73, 107, 179, 51},
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := DecryptAccount(account)
+		fmt.Printf("(+%v)", enAcc)
+	})
+}

app/job/main/passport-encrypt/service/service.go

@@ -0,0 +1,80 @@
+package service
+
+import (
+	"context"
+	"sync"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+	"go-common/app/job/main/passport-encrypt/dao"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+const (
+	// table and duration
+	_asoAccountTable = "aso_account"
+	_insertAction    = "insert"
+	_updateAction    = "update"
+	_deleteAction    = "delete"
+)
+
+// Service service.
+type Service struct {
+	c *conf.Config
+	d *dao.Dao
+	// aso binlog databus
+	dsAsoBinLogSub *databus.Databus
+	merges         []chan *message
+	done           chan []*message
+	// proc
+	head, last *message
+	mu         sync.Mutex
+}
+
+type message struct {
+	next   *message
+	data   *databus.Message
+	object interface{}
+	done   bool
+}
+
+// New new a service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:              c,
+		d:              dao.New(c),
+		dsAsoBinLogSub: databus.New(c.DataBus.AsoBinLogSub),
+		merges:         make([]chan *message, c.Group.AsoBinLog.Num),
+		done:           make(chan []*message, c.Group.AsoBinLog.Chan),
+	}
+	if c.DataSwitch.Full {
+		go s.fullMigration(c.StepGroup.Group1.Start, c.StepGroup.Group1.End, c.StepGroup.Group1.Inc, c.StepGroup.Group1.Limit, "group1")
+		go s.fullMigration(c.StepGroup.Group2.Start, c.StepGroup.Group2.End, c.StepGroup.Group2.Inc, c.StepGroup.Group2.Limit, "group2")
+		go s.fullMigration(c.StepGroup.Group3.Start, c.StepGroup.Group3.End, c.StepGroup.Group3.Inc, c.StepGroup.Group3.Limit, "group3")
+		go s.fullMigration(c.StepGroup.Group4.Start, c.StepGroup.Group4.End, c.StepGroup.Group4.Inc, c.StepGroup.Group4.Limit, "group4")
+	}
+	if c.DataSwitch.Inc {
+		go s.asobinlogcommitproc()
+		for i := 0; i < c.Group.AsoBinLog.Num; i++ {
+			ch := make(chan *message, c.Group.AsoBinLog.Chan)
+			s.merges[i] = ch
+			go s.asobinlogmergeproc(ch)
+		}
+		go s.asobinlogconsumeproc()
+	}
+
+	return
+}
+
+// Ping check server ok.
+func (s *Service) Ping(c context.Context) (err error) {
+	return
+}
+
+// Close close service, including databus and outer service.
+func (s *Service) Close() (err error) {
+	if err = s.dsAsoBinLogSub.Close(); err != nil {
+		log.Error("srv.asoBinLog.Close() error(%v)", err)
+	}
+	return
+}

app/job/main/passport-encrypt/service/service_test.go

@@ -0,0 +1,21 @@
+package service
+
+import (
+	"flag"
+	"sync"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+)
+
+var (
+	once sync.Once
+	s    *Service
+)
+
+func startService() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	s = New(conf.Conf)
+}