Create & Init Project...

app/job/main/passport-encrypt/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/passport-encrypt/cmd:all-srcs",
+        "//app/job/main/passport-encrypt/conf:all-srcs",
+        "//app/job/main/passport-encrypt/dao:all-srcs",
+        "//app/job/main/passport-encrypt/http:all-srcs",
+        "//app/job/main/passport-encrypt/model:all-srcs",
+        "//app/job/main/passport-encrypt/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-encrypt/CHANGELOG.md

@@ -0,0 +1,9 @@
+## passport-encrypt
+
+#### Version 1.0.0
+> 1. 版本初始化
+> 2. 解决冲突
+> 3. opts code
+> 4. full migration group
+> 5. full migration group improve
+> 6. format

app/job/main/passport-encrypt/CONTRIBUTORS.md

@@ -0,0 +1,9 @@
+# Owner
+linmiao
+
+# Author 
+wutao
+
+# Reviewer
+wutao
+linmiao

app/job/main/passport-encrypt/OWNERS

@@ -0,0 +1,14 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- linmiao
+- wutao
+labels:
+- job
+- job/main/passport-encrypt
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- linmiao
+- wutao

app/job/main/passport-encrypt/README.md

@@ -0,0 +1,13 @@
+## passport-encrypt
+
+#### 项目简介
+> 1.游戏云账号市北job，主要用于订阅源站变更的数据
+
+#### 编译环境
+> 请只用golang v1.8.x以上版本编译执行。
+
+#### 依赖包
+> 1.公共包go-common
+
+#### 特别说明
+> passport-encrypt 用于账号核心数据加密.

app/job/main/passport-encrypt/cmd/BUILD

@@ -0,0 +1,41 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_binary",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["passport-encrypt-job.toml"],
+    importpath = "go-common/app/job/main/passport-encrypt/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/http:go_default_library",
+        "//app/job/main/passport-encrypt/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-encrypt/cmd/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+	"go-common/app/job/main/passport-encrypt/http"
+	"go-common/app/job/main/passport-encrypt/service"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	// service init
+	srv := service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	// signal handler
+	log.Info("passport-encrypt-job start")
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("passport-encrypt-job get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			srv.Close()
+			time.Sleep(time.Second * 2)
+			log.Info("passport-encrypt-job exit")
+			return
+		case syscall.SIGHUP:
+			// TODO reload
+		default:
+			return
+		}
+	}
+}

app/job/main/passport-encrypt/cmd/passport-encrypt-job.toml

@@ -0,0 +1,107 @@
+# This is a TOML document. Boom.
+
+version = "1.0.0"
+user = "noboby"
+dir = "./"
+family = "passport-encrypt"
+env = "qa"
+
+[xlog]
+dir = "/data/log/passport-encrypt"
+
+[tracer]
+family = "passport-encrypt"
+proto = "unixgram"
+addr = "/var/run/dapper-collect/dapper-collect.sock"
+
+[stepgroup]
+    [stepgroup.group1]
+        start = 0
+        end = 400000
+        inc = 200000
+        limit = 300000000
+    [stepgroup.group2]
+        start = 0
+        end = 400000
+        inc = 200000
+        limit = 300000000
+    [stepgroup.group3]
+        start = 0
+        end = 400000
+        inc = 200000
+        limit = 300000000
+    [stepgroup.group4]
+        start = 0
+        end = 400000
+        inc = 200000
+        limit = 300000000
+
+[dataSwitch]
+    full = true
+    inc = false
+
+[bm]
+    [bm.inner]
+    addr = "0.0.0.0:6981"
+    timeout = "1s"
+    [bm.local]
+    addr = "0.0.0.0:6982"
+    timeout = "1s"
+
+[db]
+    [db.originDB]
+        addr = "172.16.33.205:3306"
+        dsn = "aso:hA0DAnENNFz78kYB@tcp(172.16.33.205:3306)/aso?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+        active = 5
+        idle = 2
+        queryTimeout = "1s"
+        execTimeout = "2s"
+        tranTimeout = "2s"
+        [db.originDB.breaker]
+        window = "3s"
+        sleep = "100ms"
+        bucket = 10
+        ratio = 0.5
+        request = 100
+
+    [db.encryptDB]
+        addr = "127.0.0.1:3306"
+        dsn = "root:123456789@tcp(127.0.0.1:3306)/test?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+        active = 5
+        idle = 2
+        queryTimeout = "1s"
+        execTimeout = "2s"
+        tranTimeout = "2s"
+        [db.encryptDB.breaker]
+        window = "3s"
+        sleep = "100ms"
+        bucket = 10
+        ratio = 0.5
+        request = 100
+
+
+[group]
+    [group.asoBinLog]
+    size = 200
+    num = 32
+    ticker = "1s"
+    chan = 1024
+
+[databus]
+    [databus.asoBinLogSub]
+    key = "4ba46ba31f9a44ef"
+    secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
+    group = "PassportAccount-MainAccount-S"
+    topic = "PassportAccount-T"
+    action = "sub"
+    offset = "old"
+    buffer = 2048
+    name = "passport-encrypt/databus"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 1
+    active = 1
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"

app/job/main/passport-encrypt/conf/BUILD

@@ -0,0 +1,36 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/passport-encrypt/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-encrypt/conf/conf.go

@@ -0,0 +1,147 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	"go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	// Conf conf.
+	Conf   = &Config{}
+	client *conf.Client
+)
+
+// Config config.
+type Config struct {
+	// common conf
+	// log
+	Xlog *log.Config
+	// Tracer tracer
+	Tracer *trace.Config
+	// http
+	BM *BM
+	// Database
+	DB *DB
+	// Step
+	StepGroup *StepGroup
+	// DataSwitch
+	DataSwitch *DataSwitch
+	// Group group
+	Group *Group
+	//Databus databus
+	DataBus *DataBus
+}
+
+// BM http server config
+type BM struct {
+	Inner *bm.ServerConfig
+	Local *bm.ServerConfig
+}
+
+// DB db config
+type DB struct {
+	OriginDB  *sql.Config
+	EncryptDB *sql.Config
+}
+
+// Group multi group config collection.
+type Group struct {
+	AsoBinLog *GroupConfig
+}
+
+// GroupConfig group config.
+type GroupConfig struct {
+	// Size merge size
+	Size int
+	// Num merge goroutine num
+	Num int
+	// Ticker duration of submit merges when no new message
+	Ticker time.Duration
+	// Chan size of merge chan and done chan
+	Chan int
+}
+
+// DataBus databus.
+type DataBus struct {
+	AsoBinLogSub *databus.Config
+}
+
+// StepGroup group
+type StepGroup struct {
+	Group1 *Step
+	Group2 *Step
+	Group3 *Step
+	Group4 *Step
+}
+
+// Step data step
+type Step struct {
+	Start, End, Inc, Limit int64
+}
+
+// DataSwitch data trans swtich
+type DataSwitch struct {
+	Full, Inc bool
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init config.
+func Init() (err error) {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf = &Config{}
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/passport-encrypt/dao/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "mysql_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+    ],
+    importpath = "go-common/app/job/main/passport-encrypt/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-encrypt/dao/dao.go

@@ -0,0 +1,48 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+// Dao dao
+type Dao struct {
+	c         *conf.Config
+	originDB  *sql.DB
+	encryptDB *sql.DB
+}
+
+// New new dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:         c,
+		originDB:  sql.NewMySQL(c.DB.OriginDB),
+		encryptDB: sql.NewMySQL(c.DB.EncryptDB),
+	}
+	return
+}
+
+// Ping check dao ok.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.originDB.Ping(c); err != nil {
+		log.Info("dao.originDB.Ping() error(%v)", err)
+	}
+	if err = d.encryptDB.Ping(c); err != nil {
+		log.Info("dao.encryptDB.Ping() error(%v)", err)
+	}
+	return
+}
+
+// Close close connections of mc, cloudDB.
+func (d *Dao) Close() (err error) {
+	if d.originDB != nil {
+		d.originDB.Close()
+	}
+	if d.encryptDB != nil {
+		d.encryptDB.Close()
+	}
+	return
+}

app/job/main/passport-encrypt/dao/dao_test.go

@@ -0,0 +1,20 @@
+package dao
+
+import (
+	"fmt"
+	"sync"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+)
+
+var (
+	once sync.Once
+	d    *Dao
+)
+
+func startDao() {
+	if err := conf.Init(); err != nil {
+		panic(fmt.Sprintf("conf.Init() error(%v)", err))
+	}
+	d = New(conf.Conf)
+}

app/job/main/passport-encrypt/dao/mysql.go

@@ -0,0 +1,83 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+
+	"go-common/app/job/main/passport-encrypt/model"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_getOriginAccountSQL = "SELECT mid,userid,uname,pwd,salt,email,tel,mobile_verified,isleak,country_id,modify_time FROM aso_account where mid >= ? and mid < ?"
+	_addAsoAccountSQL    = "INSERT INTO aso_account (mid,userid,uname,pwd,salt,email,tel,country_id,mobile_verified,isleak) VALUES(?,?,?,?,?,?,?,?,?,?)"
+	_updateAsoAccountSQL = "UPDATE aso_account SET userid=?,uname=?,pwd=?,salt=?,email=?,tel=?,country_id=?,mobile_verified=?,isleak=? WHERE mid=?"
+	_deleteAsoAccountSQL = "DELETE FROM aso_account WHERE mid=?"
+)
+
+// AsoAccounts get tokens by mid.
+func (d *Dao) AsoAccounts(c context.Context, start, end int64) (res []*model.OriginAccount, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.originDB.Query(c, _getOriginAccountSQL, start, end); err != nil {
+		log.Error("failed to get AsoAccounts, dao.originDB.AsoAccounts(%s) error(%v)", _getOriginAccountSQL, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var telPtr, emailPtr *string
+		r := new(model.OriginAccount)
+		if err = rows.Scan(&r.Mid, &r.UserID, &r.Uname, &r.Pwd, &r.Salt, &emailPtr, &telPtr, &r.MobileVerified, &r.Isleak, &r.CountryID, &r.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			res = nil
+			return
+		}
+		if telPtr != nil {
+			r.Tel = *telPtr
+		}
+		if emailPtr != nil {
+			r.Email = *emailPtr
+		}
+		res = append(res, r)
+	}
+	return
+}
+
+// AddAsoAccount add aso account.
+func (d *Dao) AddAsoAccount(c context.Context, a *model.EncryptAccount) (affected int64, err error) {
+	var res sql.Result
+	var emailPtr *string
+
+	if a.Email != "" {
+		emailPtr = &a.Email
+	}
+	if res, err = d.encryptDB.Exec(c, _addAsoAccountSQL, a.Mid, a.UserID, a.Uname, a.Pwd, a.Salt, emailPtr, a.Tel, a.CountryID, a.MobileVerified, a.Isleak); err != nil {
+		log.Error("failed to add aso account, dao.encryptDB.Exec() error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// UpdateAsoAccount update aso account.
+func (d *Dao) UpdateAsoAccount(c context.Context, a *model.EncryptAccount) (affected int64, err error) {
+	var res sql.Result
+	var emailPtr *string
+	if a.Email != "" {
+		emailPtr = &a.Email
+	}
+	if res, err = d.encryptDB.Exec(c, _updateAsoAccountSQL, a.UserID, a.Uname, a.Pwd, a.Salt, emailPtr, a.Tel, a.CountryID, a.MobileVerified, a.Isleak, a.Mid); err != nil {
+		log.Error("failed to update aso account, dao.encryptDB.Exec() error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// DelAsoAccount delete aso account.
+func (d *Dao) DelAsoAccount(c context.Context, mid int64) (affected int64, err error) {
+	var res sql.Result
+	if res, err = d.encryptDB.Exec(c, _deleteAsoAccountSQL, mid); err != nil {
+		log.Error("failed to delete aso account, dao.encryptDB.Exec(%s) error(%v)", mid, err)
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/passport-encrypt/dao/mysql_test.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/passport-encrypt/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_PingMySQL(t *testing.T) {
+	once.Do(startDao)
+	if err := d.encryptDB.Ping(context.TODO()); err != nil {
+		t.Errorf("dao.cloudDB.Ping() error(%v)", err)
+		t.FailNow()
+	}
+}
+
+func TestDao_UpdateAsoAccount(t *testing.T) {
+	once.Do(startDao)
+	Convey("update a aso account", t, func() {
+		account := &model.EncryptAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "62fe0d616162f56ecab3e12a2de83ea6",
+			Tel:            []byte("bdb27b0300e3984e48e7aea5c672a243"),
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		affected, err := d.UpdateAsoAccount(context.TODO(), account)
+		So(err, ShouldBeNil)
+
+		So(affected, ShouldEqual, 1)
+	})
+}

app/job/main/passport-encrypt/http/BUILD

@@ -0,0 +1,32 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/passport-encrypt/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-encrypt/http/http.go

@@ -0,0 +1,37 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+	"go-common/app/job/main/passport-encrypt/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	srv *service.Service
+)
+
+// Init init http sever instance.
+func Init(c *conf.Config, s *service.Service) {
+	engine := bm.DefaultServer(c.BM.Inner)
+	innerRouter(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+// ping service
+func ping(c *bm.Context) {
+	if err := srv.Ping(c); err != nil {
+		log.Error("ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}

app/job/main/passport-encrypt/model/BUILD

@@ -0,0 +1,29 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "message.go",
+        "passport.go",
+    ],
+    importpath = "go-common/app/job/main/passport-encrypt/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-encrypt/model/message.go

@@ -0,0 +1,19 @@
+package model
+
+import "encoding/json"
+
+// BMsg databus binlog message.
+type BMsg struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+// PMsg Push msg
+type PMsg struct {
+	Action string         `json:"action"`
+	Table  string         `json:"table"`
+	Data   *OriginAccount `json:"data"`
+	MTS    int64
+}

app/job/main/passport-encrypt/model/passport.go

@@ -0,0 +1,31 @@
+package model
+
+// EncryptAccount aso account.
+type EncryptAccount struct {
+	Mid            int64  `json:"mid"`
+	UserID         string `json:"userid"`
+	Uname          string `json:"uname"`
+	Pwd            string `json:"pwd"`
+	Salt           string `json:"salt"`
+	Email          string `json:"email"`
+	Tel            []byte `json:"tel"`
+	CountryID      int64  `json:"country_id"`
+	MobileVerified int8   `json:"mobile_verified"`
+	Isleak         int8   `json:"isleak"`
+	Mtime          string `json:"mtime"`
+}
+
+// OriginAccount origin aso account.
+type OriginAccount struct {
+	Mid            int64  `json:"mid"`
+	UserID         string `json:"userid"`
+	Uname          string `json:"uname"`
+	Pwd            string `json:"pwd"`
+	Salt           string `json:"salt"`
+	Email          string `json:"email"`
+	Tel            string `json:"tel"`
+	CountryID      int64  `json:"country_id"`
+	MobileVerified int8   `json:"mobile_verified"`
+	Isleak         int8   `json:"isleak"`
+	Mtime          string `json:"modify_time"`
+}

app/job/main/passport-encrypt/service/BUILD

@@ -0,0 +1,60 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aes.go",
+        "aso.go",
+        "aso_full_migration.go",
+        "aso_incr_migration.go",
+        "passport_util.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/passport-encrypt/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/dao:go_default_library",
+        "//app/job/main/passport-encrypt/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "aes_test.go",
+        "aso_full_migration_test.go",
+        "aso_incr_migration_test.go",
+        "passport_util_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/passport-encrypt/conf:go_default_library",
+        "//app/job/main/passport-encrypt/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/passport-encrypt/service/aes.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+)
+
+var key = []byte("bili_account_enc")
+
+// Encrypt aes encrypt
+func Encrypt(origData []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	blockSize := block.BlockSize()
+	origData = PKCS5Padding(origData, blockSize)
+	blockMode := cipher.NewCBCEncrypter(block, key[:blockSize])
+	crypted := make([]byte, len(origData))
+	blockMode.CryptBlocks(crypted, origData)
+	return crypted, nil
+}
+
+// Decrypt aes encrypt
+func Decrypt(crypted []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	blockSize := block.BlockSize()
+	blockMode := cipher.NewCBCDecrypter(block, key[:blockSize])
+	origData := make([]byte, len(crypted))
+	blockMode.CryptBlocks(origData, crypted)
+	origData = PKCS5UnPadding(origData)
+	return origData, nil
+}
+
+// PKCS5Padding padding
+func PKCS5Padding(ciphertext []byte, blockSize int) []byte {
+	padding := blockSize - len(ciphertext)%blockSize
+	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
+	return append(ciphertext, padtext...)
+}
+
+// PKCS5UnPadding unpadding
+func PKCS5UnPadding(origData []byte) []byte {
+	length := len(origData)
+	unpadding := int(origData[length-1])
+	return origData[:(length - unpadding)]
+}

app/job/main/passport-encrypt/service/aes_test.go

@@ -0,0 +1,30 @@
+package service
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_Encrypt(t *testing.T) {
+	Convey("Encrypt  param ", t, func() {
+		input := []byte("13122119298")
+		res, err := Encrypt(input)
+		ShouldBeNil(err)
+		ShouldNotBeNil(res)
+		fmt.Printf("(%+v) \n", res)
+	})
+
+}
+
+func TestService_Decrypt(t *testing.T) {
+	Convey("Decrypt param ", t, func() {
+		input := []byte{70, 251, 31, 100, 140, 98, 156, 101, 27, 21, 201, 100, 159, 107, 90, 147}
+		fmt.Printf("input is  (%+v) \n", input)
+		res, err := Decrypt(input)
+		ShouldBeNil(err)
+		ShouldNotBeNil(res)
+		fmt.Printf("(%+v) \n", string(res))
+	})
+}

app/job/main/passport-encrypt/service/aso.go

@@ -0,0 +1,31 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-encrypt/model"
+)
+
+func (s *Service) saveEncryptAccount(c context.Context, account *model.EncryptAccount) (err error) {
+	var affect int64
+	if affect, err = s.d.AddAsoAccount(c, account); err != nil || affect == 0 {
+		return
+	}
+	return
+}
+
+func (s *Service) updateEncryptAccount(c context.Context, account *model.EncryptAccount) (err error) {
+	var affect int64
+	if affect, err = s.d.UpdateAsoAccount(c, account); err != nil || affect == 0 {
+		return
+	}
+	return
+}
+
+func (s *Service) delEncryptAccount(c context.Context, mid int64) (err error) {
+	var affect int64
+	if affect, err = s.d.DelAsoAccount(c, mid); err != nil || affect == 0 {
+		return
+	}
+	return
+}

app/job/main/passport-encrypt/service/aso_full_migration.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-encrypt/model"
+	"go-common/library/log"
+)
+
+// fullMigration full migration
+func (s *Service) fullMigration(start, end, inc, limit int64, group string) {
+	var count int64
+
+	for {
+		log.Info(" group is %s,start %d, end %d,inc %d,limit %d", group, start, end, inc, limit)
+		var (
+			res      []*model.OriginAccount
+			err      error
+			effected int64
+		)
+		if res, err = s.d.AsoAccounts(context.TODO(), start, end); err != nil {
+			log.Error("error is (%+v)\n", err)
+			break
+		}
+
+		for _, acc := range res {
+			enAcc := EncryptAccount(acc)
+			if effected, err = s.d.AddAsoAccount(context.TODO(), enAcc); err != nil || effected == 0 {
+				log.Error("insert error (%+v)", err)
+			}
+		}
+		start = end
+		count = end
+		end = end + inc
+
+		if count >= limit {
+			break
+		}
+	}
+}

app/job/main/passport-encrypt/service/aso_full_migration_test.go

@@ -0,0 +1,14 @@
+package service
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_FullMigration(t *testing.T) {
+	once.Do(startService)
+	Convey("test full migration ", t, func() {
+		s.fullMigration(0, 10, 5, 50, "")
+	})
+}

app/job/main/passport-encrypt/service/aso_incr_migration.go

@@ -0,0 +1,130 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/passport-encrypt/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+func (s *Service) asobinlogconsumeproc() {
+	mergeNum := int64(s.c.Group.AsoBinLog.Num)
+	for {
+		msg, ok := <-s.dsAsoBinLogSub.Messages()
+		if !ok {
+			log.Error("asobinlogconsumeproc closed")
+			return
+		}
+		// marked head to first commit
+		m := &message{data: msg}
+		s.mu.Lock()
+		if s.head == nil {
+			s.head = m
+			s.last = m
+		} else {
+			s.last.next = m
+			s.last = m
+		}
+		s.mu.Unlock()
+		bmsg := new(model.BMsg)
+		if err := json.Unmarshal(msg.Value, bmsg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+		mid := int64(0)
+		if bmsg.Table == _asoAccountTable {
+			t := new(model.OriginAccount)
+			if err := json.Unmarshal(bmsg.New, t); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+			}
+			mid = t.Mid
+			m.object = bmsg
+			log.Info("asobinlogconsumeproc table:%s key:%s partition:%d offset:%d", bmsg.Table, msg.Key, msg.Partition, msg.Offset)
+		} else {
+			continue
+		}
+		s.merges[mid%mergeNum] <- m
+	}
+}
+
+func (s *Service) asobinlogcommitproc() {
+	for {
+		done := <-s.done
+		commits := make(map[int32]*databus.Message)
+		for _, d := range done {
+			d.done = true
+		}
+		s.mu.Lock()
+		for ; s.head != nil && s.head.done; s.head = s.head.next {
+			commits[s.head.data.Partition] = s.head.data
+		}
+		s.mu.Unlock()
+		for _, m := range commits {
+			m.Commit()
+		}
+	}
+}
+
+func (s *Service) asobinlogmergeproc(c chan *message) {
+	var (
+		max    = s.c.Group.AsoBinLog.Size
+		merges = make([]*model.BMsg, 0, max)
+		marked = make([]*message, 0, max)
+		ticker = time.NewTicker(time.Duration(s.c.Group.AsoBinLog.Ticker))
+	)
+	for {
+		select {
+		case msg, ok := <-c:
+			if !ok {
+				log.Error("asobinlogmergeproc closed")
+				return
+			}
+			p, assertOk := msg.object.(*model.BMsg)
+			if assertOk && p.Action != "" && (p.Table == _asoAccountTable) {
+				merges = append(merges, p)
+			}
+			marked = append(marked, msg)
+			if len(marked) < max && len(merges) < max {
+				continue
+			}
+		case <-ticker.C:
+		}
+		if len(merges) > 0 {
+			s.processAsoAccLogInfo(merges)
+			merges = make([]*model.BMsg, 0, max)
+		}
+		if len(marked) > 0 {
+			s.done <- marked
+			marked = make([]*message, 0, max)
+		}
+	}
+}
+
+func (s *Service) processAsoAccLogInfo(bmsgs []*model.BMsg) {
+	for _, msg := range bmsgs {
+		s.processAsoAccLog(msg)
+	}
+}
+
+func (s *Service) processAsoAccLog(msg *model.BMsg) {
+	aso := new(model.OriginAccount)
+	if err := json.Unmarshal(msg.New, aso); err != nil {
+		log.Error("failed to parse binlog new, json.Unmarshal(%s) error(%v)", string(msg.New), err)
+		return
+	}
+	if _updateAction == msg.Action {
+		enAso := EncryptAccount(aso)
+		s.updateEncryptAccount(context.TODO(), enAso)
+	}
+	if _insertAction == msg.Action {
+		enAso := EncryptAccount(aso)
+		s.saveEncryptAccount(context.TODO(), enAso)
+	}
+	if _deleteAction == msg.Action {
+		s.delEncryptAccount(context.TODO(), aso.Mid)
+	}
+
+}

app/job/main/passport-encrypt/service/aso_incr_migration_test.go

@@ -0,0 +1,23 @@
+package service
+
+import (
+	"testing"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_IncMigration(t *testing.T) {
+	once.Do(startService)
+	c := conf.Conf
+	Convey("test full migration ", t, func() {
+		s.asobinlogconsumeproc()
+		for i := 0; i < c.Group.AsoBinLog.Num; i++ {
+			ch := make(chan *message, c.Group.AsoBinLog.Chan)
+			s.merges[i] = ch
+			s.asobinlogmergeproc(ch)
+		}
+		s.asobinlogcommitproc()
+	})
+}

app/job/main/passport-encrypt/service/passport_util.go

@@ -0,0 +1,67 @@
+package service
+
+import "go-common/app/job/main/passport-encrypt/model"
+
+// EncryptAccount Encrypt Account
+func EncryptAccount(a *model.OriginAccount) (res *model.EncryptAccount) {
+	return &model.EncryptAccount{
+		Mid:            a.Mid,
+		UserID:         a.UserID,
+		Uname:          a.Uname,
+		Pwd:            a.Pwd,
+		Salt:           a.Salt,
+		Email:          a.Email,
+		Tel:            doEncrypt(a.Tel),
+		CountryID:      a.CountryID,
+		MobileVerified: a.MobileVerified,
+		Isleak:         a.Isleak,
+		Mtime:          a.Mtime,
+	}
+}
+
+// DecryptAccount Decrypt Account
+func DecryptAccount(a *model.EncryptAccount) (res *model.OriginAccount) {
+	return &model.OriginAccount{
+		Mid:            a.Mid,
+		UserID:         a.UserID,
+		Uname:          a.Uname,
+		Pwd:            a.Pwd,
+		Salt:           a.Salt,
+		Email:          a.Email,
+		Tel:            doDecrypt(a.Tel),
+		CountryID:      a.CountryID,
+		MobileVerified: a.MobileVerified,
+		Isleak:         a.Isleak,
+		Mtime:          a.Mtime,
+	}
+}
+
+func doEncrypt(param string) []byte {
+	var (
+		err error
+		res = make([]byte, 0)
+	)
+	if param == "" || len(param) == 0 {
+		return nil
+	}
+	input := []byte(param)
+	if res, err = Encrypt(input); err != nil {
+		return input
+	}
+	return res
+}
+
+func doDecrypt(param []byte) string {
+	var (
+		err error
+		res = make([]byte, 0)
+	)
+	if param == nil {
+		return ""
+	}
+	input := []byte(param)
+	if res, err = Decrypt(input); err != nil {
+		return string(param)
+	}
+	return string(res)
+}

app/job/main/passport-encrypt/service/passport_util_test.go

@@ -0,0 +1,82 @@
+package service
+
+import (
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/passport-encrypt/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_EncryptAccount(t *testing.T) {
+	Convey("Encrypt a right account", t, func() {
+		account := &model.OriginAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            "13122110209",
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := EncryptAccount(account)
+		fmt.Printf("right (+%v)", enAcc)
+	})
+
+	Convey("Encrypt a empty tel account", t, func() {
+		account := &model.OriginAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            "",
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := EncryptAccount(account)
+		fmt.Printf("empty tel (+%v)", enAcc)
+	})
+}
+
+func TestService_DecryptAccount(t *testing.T) {
+	Convey("Decrypt right account", t, func() {
+		account := &model.EncryptAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            []byte{216, 28, 241, 48, 17, 243, 198, 234, 205, 33, 216, 25, 75, 146, 97, 203},
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := DecryptAccount(account)
+		fmt.Printf("(+%v)", enAcc)
+	})
+
+	Convey("Decrypt empty tel  account", t, func() {
+		account := &model.EncryptAccount{
+			Mid:            12047569,
+			UserID:         "bili_1710676855",
+			Uname:          "Bili_12047569",
+			Pwd:            "3686c9d96ae6896fe117319ba6c07087",
+			Salt:           "pdMXF856",
+			Email:          "",
+			Tel:            []byte{73, 224, 88, 238, 85, 242, 37, 46, 200, 70, 5, 49, 73, 107, 179, 51},
+			CountryID:      1,
+			MobileVerified: 1,
+			Isleak:         0,
+		}
+		enAcc := DecryptAccount(account)
+		fmt.Printf("(+%v)", enAcc)
+	})
+}

app/job/main/passport-encrypt/service/service.go

@@ -0,0 +1,80 @@
+package service
+
+import (
+	"context"
+	"sync"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+	"go-common/app/job/main/passport-encrypt/dao"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+const (
+	// table and duration
+	_asoAccountTable = "aso_account"
+	_insertAction    = "insert"
+	_updateAction    = "update"
+	_deleteAction    = "delete"
+)
+
+// Service service.
+type Service struct {
+	c *conf.Config
+	d *dao.Dao
+	// aso binlog databus
+	dsAsoBinLogSub *databus.Databus
+	merges         []chan *message
+	done           chan []*message
+	// proc
+	head, last *message
+	mu         sync.Mutex
+}
+
+type message struct {
+	next   *message
+	data   *databus.Message
+	object interface{}
+	done   bool
+}
+
+// New new a service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:              c,
+		d:              dao.New(c),
+		dsAsoBinLogSub: databus.New(c.DataBus.AsoBinLogSub),
+		merges:         make([]chan *message, c.Group.AsoBinLog.Num),
+		done:           make(chan []*message, c.Group.AsoBinLog.Chan),
+	}
+	if c.DataSwitch.Full {
+		go s.fullMigration(c.StepGroup.Group1.Start, c.StepGroup.Group1.End, c.StepGroup.Group1.Inc, c.StepGroup.Group1.Limit, "group1")
+		go s.fullMigration(c.StepGroup.Group2.Start, c.StepGroup.Group2.End, c.StepGroup.Group2.Inc, c.StepGroup.Group2.Limit, "group2")
+		go s.fullMigration(c.StepGroup.Group3.Start, c.StepGroup.Group3.End, c.StepGroup.Group3.Inc, c.StepGroup.Group3.Limit, "group3")
+		go s.fullMigration(c.StepGroup.Group4.Start, c.StepGroup.Group4.End, c.StepGroup.Group4.Inc, c.StepGroup.Group4.Limit, "group4")
+	}
+	if c.DataSwitch.Inc {
+		go s.asobinlogcommitproc()
+		for i := 0; i < c.Group.AsoBinLog.Num; i++ {
+			ch := make(chan *message, c.Group.AsoBinLog.Chan)
+			s.merges[i] = ch
+			go s.asobinlogmergeproc(ch)
+		}
+		go s.asobinlogconsumeproc()
+	}
+
+	return
+}
+
+// Ping check server ok.
+func (s *Service) Ping(c context.Context) (err error) {
+	return
+}
+
+// Close close service, including databus and outer service.
+func (s *Service) Close() (err error) {
+	if err = s.dsAsoBinLogSub.Close(); err != nil {
+		log.Error("srv.asoBinLog.Close() error(%v)", err)
+	}
+	return
+}

app/job/main/passport-encrypt/service/service_test.go

@@ -0,0 +1,21 @@
+package service
+
+import (
+	"flag"
+	"sync"
+
+	"go-common/app/job/main/passport-encrypt/conf"
+)
+
+var (
+	once sync.Once
+	s    *Service
+)
+
+func startService() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	s = New(conf.Conf)
+}