Create & Init Project...

app/job/main/passport-auth/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/passport-auth/cmd:all-srcs",
+        "//app/job/main/passport-auth/conf:all-srcs",
+        "//app/job/main/passport-auth/dao:all-srcs",
+        "//app/job/main/passport-auth/http:all-srcs",
+        "//app/job/main/passport-auth/model:all-srcs",
+        "//app/job/main/passport-auth/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-auth/CHANGELOG.md

@@ -0,0 +1,26 @@
+### v1.3.0
+1. save deleted records
+
+### v1.2.2
+1. skip -400 error when clean cache
+
+### v1.2.1
+1. add log
+
+### v1.2.0
+1. 清缓存逻辑优化
+
+### v1.1.1
+1. 添加通知日志
+
+### v1.1.0
+1. 修复清理cookie缓存错误
+
+### v1.1.0
+1. 添加清理aso服务缓存逻辑
+
+### v1.0.1
+1. 修复passport auth服务发现
+
+### v1.0.0
+1. 初始化passport auth

app/job/main/passport-auth/CONTRIBUTORS.md

@@ -0,0 +1,7 @@
+# Owner
+wanghuan01
+wutao
+
+# Author
+
+# Reviewer

app/job/main/passport-auth/OWNERS

@@ -0,0 +1,11 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- wanghuan01
+- wutao
+labels:
+- job
+- job/main/passport-auth
+- main
+options:
+  no_parent_owners: true

app/job/main/passport-auth/README.md

@@ -0,0 +1,14 @@
+# passport-auth
+
+# 项目简介
+1.
+
+# 编译环境
+
+
+# 依赖包
+
+
+# 编译执行
+
+	

app/job/main/passport-auth/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["passport-auth-job.toml"],
+    importpath = "go-common/app/job/main/passport-auth/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/http:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-auth/cmd/main.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/passport-auth/conf"
+	"go-common/app/job/main/passport-auth/http"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("passport-auth-job start")
+	// init trace
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	// service init
+	http.Init(conf.Conf)
+	// init pprof conf.Conf.Perf
+	// init signal
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("passport-auth get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("passport-auth-job exit")
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/passport-auth/cmd/passport-auth-job.toml

@@ -0,0 +1,110 @@
+synclines = 1000
+
+[AuthJobConfig]
+asoCleanUrl = "http://passport.bilibili.co/intranet/auth/delCache"
+
+[mysql]
+	addr = "172.22.34.101:3308"
+	dsn = "auth:ABQ3ewVrxyZsLpWb0EkJuCvN9M8aidf7@tcp(172.22.34.101:3308)/passport_auth?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["auth_reader:3odEZJ7YlOLpNzt4XVDxHTw9bi08C1M2@tcp(172.22.34.101:3308)/passport_auth?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 10
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "100ms"
+	execTimeout = "100ms"
+	tranTimeout = "200ms"
+	[mysql.breaker]
+	window = "3s"
+	sleep = "100ms"
+	bucket = 10
+	ratio = 0.5
+	request = 100
+
+[OldMySQL]
+	addr = "172.22.34.101:3309"
+	dsn = "aso:WsjbodupJcZlBeWPEiYyK8HOTIcp3n3Z@tcp(172.22.34.101:3309)/aso?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["aso:WsjbodupJcZlBeWPEiYyK8HOTIcp3n3Z@tcp(172.22.34.101:3309)/aso?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 10
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "100ms"
+	execTimeout = "100ms"
+	tranTimeout = "200ms"
+	[OldMySQL.breaker]
+	window = "3s"
+	sleep = "100ms"
+	bucket = 10
+	ratio = 0.5
+	request = 100
+
+[databus]
+	key = "4ba46ba31f9a44ef"
+	secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
+	group = "Passport-MainAccount-AuthSync-S"
+	topic = "Passport-T"
+	action = "sub"
+	offset = "old"
+	buffer = 2048
+	name = "passport-auth-job/databus"
+	proto = "tcp"
+	addr = "172.18.33.50:6205"
+	idle = 1
+	active = 1
+	dialTimeout = "1s"
+	readTimeout = "60s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+
+[AuthDataBus]
+	key = "4ba46ba31f9a44ef"
+	secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
+	group = "PassportAuthBinlog-MainAccount-S"
+	topic = "PassportAuthBinlog-T"
+	action = "sub"
+	offset = "old"
+	buffer = 2048
+	name = "passport-auth-job/databus"
+	proto = "tcp"
+	addr = "172.18.33.50:6205"
+	idle = 1
+	active = 1
+	dialTimeout = "1s"
+	readTimeout = "60s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+#	[AuthDataBus.discovery]
+#   		domain = "api.bilibili.co"
+#   		key = "7634436ea852e3f4"
+#   		secret = "test"
+
+[databusUtil]
+	size = 1
+	num = 2
+	ticker = "500ms"
+	chan = 100
+
+[memcache]
+	name = "passport-auth-job"
+	proto = "tcp"
+	addr = ""
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "24h"
+
+[HttpClientConfig]
+    key = "7634436ea852e3f4"
+    secret = "test"
+    dial = "100ms"
+    timeout = "350ms"
+    keepAlive = "60s"
+    timer = 1024
+    [HttpClientConfig.breaker]
+        window  = "3s"
+        sleep   = "100ms"
+        bucket  = 10
+        ratio   = 1.0
+        request = 100

app/job/main/passport-auth/conf/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/passport-auth/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/databusutil:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-auth/conf/conf.go

@@ -0,0 +1,114 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	"go-common/library/queue/databus/databusutil"
+
+	"github.com/BurntSushi/toml"
+)
+
+// global var
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config config set
+type Config struct {
+	// base
+	// elk
+	Log *log.Config
+	// http
+	BM *bm.ServerConfig
+	// identify
+	//Identify *identify.Config
+	// tracer
+	Tracer *trace.Config
+	// MySQL
+	MySQL *sql.Config
+	// MySQL
+	OldMySQL *sql.Config
+	// Databus
+	Databus *databus.Config
+	// AuthDataBus authService binlog
+	AuthDataBus *databus.Config
+	// DataUtil config
+	DatabusUtil *databusutil.Config
+	// auth rpc
+	AuthRPC *rpc.ClientConfig
+	// HTTPClientConfig
+	HTTPClientConfig *bm.ClientConfig
+	// AuthJobConfig job config
+	AuthJobConfig *AuthJobConfig
+	// user defined
+	SyncLines int64
+	IDXFrom   int64
+	IDXTo     int64
+}
+
+// AuthJobConfig auth job config
+type AuthJobConfig struct {
+	AsoCleanURL string
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Value("passport-auth-job.toml"); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/passport-auth/dao/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "clean_cache.go",
+        "cookie.go",
+        "dao.go",
+        "refresh.go",
+        "token.go",
+    ],
+    importpath = "go-common/app/job/main/passport-auth/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "clean_cache_test.go",
+        "cookie_test.go",
+        "dao_test.go",
+        "refresh_test.go",
+        "token_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/passport-auth/dao/clean_cache.go

@@ -0,0 +1,36 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// AsoCleanCache aso clean cache
+func (d *Dao) AsoCleanCache(c context.Context, token, session string, mid int64) (err error) {
+	log.Info("aso clean cache,mid = %d,token = %s, session = %s", mid, token, session)
+	params := url.Values{}
+	params.Set("token", token)
+	params.Set("session", session)
+	params.Set("mid", strconv.Itoa(int(mid)))
+	res := &struct {
+		Code int `json:"code"`
+	}{}
+	if err = d.httpClient.Get(c, d.c.AuthJobConfig.AsoCleanURL, "127.0.0.2", params, res); err != nil {
+		log.Error("AsoCleanCache HTTP request err %v,token = %s,session = %s,mid = %d", err, token, session, mid)
+		return
+	}
+	if res.Code != 0 {
+		log.Error("AsoCleanCache server err_code %d,token = %s,session = %s,mid=%d", res.Code, token, session, mid)
+		if res.Code == ecode.RequestErr.Code() {
+			err = nil
+			return
+		}
+		err = ecode.Int(res.Code)
+		return
+	}
+	return
+}

app/job/main/passport-auth/dao/clean_cache_test.go

@@ -0,0 +1,25 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoAsoCleanCache(t *testing.T) {
+	convey.Convey("AsoCleanCache", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			token   = ""
+			session = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+			mid     = int64(1)
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			err := d.AsoCleanCache(c, token, session, mid)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/passport-auth/dao/cookie.go

@@ -0,0 +1,48 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+	"go-common/library/log"
+)
+
+const (
+	_addCookieSQL          = "INSERT IGNORE INTO user_cookie_%s (mid,session,csrf,type,expires) VALUES (?,?,?,?,?)"
+	_delCookieBySessionSQL = "DELETE FROM user_cookie_%s where session = ?"
+
+	_addCookieDeletedSQL = "INSERT IGNORE INTO user_cookie_deleted_%s (mid,session,csrf,type,expires,ctime) VALUES (?,?,?,?,?,?)"
+)
+
+// AddCookie save cookie
+func (d *Dao) AddCookie(c context.Context, cookie *model.Cookie, session, csrf []byte, ct time.Time) (affected int64, err error) {
+	row, err := d.db.Exec(c, fmt.Sprintf(_addCookieSQL, formatSuffix(ct)), cookie.Mid, session, csrf, cookie.Type, cookie.Expires)
+	if err != nil {
+		log.Error("dao.db.Exec(%v) err(%v)", cookie, err)
+		return
+	}
+	return row.RowsAffected()
+}
+
+// DelCookie del cookie by session
+func (d *Dao) DelCookie(c context.Context, session []byte, ct time.Time) (affected int64, err error) {
+	var res sql.Result
+	if res, err = d.db.Exec(c, fmt.Sprintf(_delCookieBySessionSQL, formatSuffix(ct)), session); err != nil {
+		log.Error("del cookie by session , dao.db.Exec(%s) error(%v)", session, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// AddCookieDeleted save cookie deleted
+func (d *Dao) AddCookieDeleted(c context.Context, cookie *model.Cookie, session, csrf []byte, ct time.Time) (affected int64, err error) {
+	row, err := d.db.Exec(c, fmt.Sprintf(_addCookieDeletedSQL, formatSuffix(ct)), cookie.Mid, session, csrf, cookie.Type, cookie.Expires, cookie.Ctime)
+	if err != nil {
+		log.Error("fail to add cookie deleted, cookie(%+v), tx.Exec() error(%+v)", cookie, err)
+		return
+	}
+	return row.RowsAffected()
+}

app/job/main/passport-auth/dao/cookie_test.go

@@ -0,0 +1,66 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoAddCookie(t *testing.T) {
+	convey.Convey("AddCookie", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			cookie  = &model.Cookie{}
+			session = []byte("712b7a22,1535703191,c07e44d8")
+			csrf    = []byte("0273f9216fa8d6d77c3dd5499a1d0d4a")
+			ct      = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.AddCookie(c, cookie, session, csrf, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelCookie(t *testing.T) {
+	convey.Convey("DelCookie", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			session = []byte("712b7a22,1535703191,c07e44d8")
+			ct      = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.DelCookie(c, session, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoAddCookieDeleted(t *testing.T) {
+	convey.Convey("AddCookieDeleted", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			cookie  = &model.Cookie{}
+			session = []byte("712b7a22,1535703191,c07e44d8")
+			csrf    = []byte("0273f9216fa8d6d77c3dd5499a1d0d4a")
+			ct      = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.AddCookieDeleted(c, cookie, session, csrf, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/passport-auth/dao/dao.go

@@ -0,0 +1,46 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-auth/conf"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao dao
+type Dao struct {
+	c     *conf.Config
+	db    *xsql.DB
+	olddb *xsql.DB
+	// httpClient
+	httpClient *bm.Client
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:     c,
+		db:    xsql.NewMySQL(c.MySQL),
+		olddb: xsql.NewMySQL(c.OldMySQL),
+		// httpClient
+		httpClient: bm.NewClient(c.HTTPClientConfig),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.db.Close()
+	d.olddb.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	return d.pingMC(c)
+}
+
+// pingMc ping
+func (d *Dao) pingMC(c context.Context) (err error) {
+	return
+}

app/job/main/passport-auth/dao/dao_test.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"flag"
+	"go-common/app/job/main/passport-auth/conf"
+	"os"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.passport.passport-auth-job")
+		flag.Set("conf_token", "ebcb53159bf293283b853091f57aa254")
+		flag.Set("tree_id", "35674")
+		flag.Set("conf_version", "uat-docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/passport-auth-job.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/passport-auth/dao/refresh.go

@@ -0,0 +1,48 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+	"go-common/library/log"
+)
+
+const (
+	_addRefreshSQL = "INSERT IGNORE INTO user_refresh_%s (mid,appid,refresh,token,expires) VALUES (?,?,?,?,?)"
+	_delRefreshSQL = "DELETE FROM user_refresh_%s WHERE refresh = ?"
+)
+
+// AddRefresh save token
+func (d *Dao) AddRefresh(c context.Context, t *model.Refresh, refresh, token []byte, ct time.Time) (affected int64, err error) {
+	var row sql.Result
+	if row, err = d.db.Exec(c, fmt.Sprintf(_addRefreshSQL, formatRefreshSuffix(ct)), t.Mid, t.AppID, refresh, token, t.Expires); err != nil {
+		log.Error("d.AddToken(%v) err(%v)", t, err)
+		return
+	}
+	return row.RowsAffected()
+}
+
+// DelRefresh del token
+func (d *Dao) DelRefresh(c context.Context, refresh []byte, ct time.Time) (affected int64, err error) {
+	var res sql.Result
+	if res, err = d.db.Exec(c, fmt.Sprintf(_delRefreshSQL, formatRefreshSuffix(ct)), refresh); err != nil {
+		log.Error("del token failed, dao.db.Exec(%s) error(%v)", hex.EncodeToString(refresh), err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+func formatRefreshSuffix(t time.Time) string {
+	return formatByDate(t.Year(), int(t.Month()))
+}
+
+func formatByDate(year, month int) string {
+	if month%2 == 1 {
+		return fmt.Sprintf("%4d%02d", year, month)
+	}
+	return fmt.Sprintf("%4d%02d", year, month-1)
+}

app/job/main/passport-auth/dao/refresh_test.go

@@ -0,0 +1,75 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/job/main/passport-auth/model"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoAddRefresh(t *testing.T) {
+	convey.Convey("AddRefresh", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			no      = &model.Refresh{}
+			refresh = []byte("9df38fe4b94a47baad001ad823b84110")
+			token   = []byte("61c13e530b1418653e2fdc265b3f0fe6")
+			ct      = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.AddRefresh(c, no, refresh, token, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelRefresh(t *testing.T) {
+	convey.Convey("DelRefresh", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			refresh = []byte("9df38fe4b94a47baad001ad823b84110")
+			ct      = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.DelRefresh(c, refresh, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoformatRefreshSuffix(t *testing.T) {
+	convey.Convey("formatRefreshSuffix", t, func(ctx convey.C) {
+		var (
+			no = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			p1 := formatRefreshSuffix(no)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoformatByDate(t *testing.T) {
+	convey.Convey("formatByDate", t, func(ctx convey.C) {
+		var (
+			year  = int(0)
+			month = int(0)
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			p1 := formatByDate(year, month)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/passport-auth/dao/token.go

@@ -0,0 +1,53 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+	"go-common/library/log"
+)
+
+const (
+	_addTokenSQL = "INSERT IGNORE INTO user_token_%s (mid,appid,token,expires,type) VALUES (?,?,?,?,?)"
+	_delTokenSQL = "DELETE FROM user_token_%s where token = ?"
+
+	_addTokenDeletedSQL = "INSERT IGNORE INTO user_token_deleted_%s (mid,appid,token,expires,type,ctime) VALUES (?,?,?,?,?,?)"
+)
+
+// AddToken save token
+func (d *Dao) AddToken(c context.Context, t *model.Token, token []byte, ct time.Time) (affected int64, err error) {
+	var row sql.Result
+	if row, err = d.db.Exec(c, fmt.Sprintf(_addTokenSQL, formatSuffix(ct)), t.Mid, t.AppID, token, t.Expires, t.Type); err != nil {
+		log.Error("d.AddToken(%v) err(%v)", t, err)
+		return
+	}
+	return row.RowsAffected()
+}
+
+// DelToken del token
+func (d *Dao) DelToken(c context.Context, token []byte, ct time.Time) (affected int64, err error) {
+	var res sql.Result
+	if res, err = d.db.Exec(c, fmt.Sprintf(_delTokenSQL, formatSuffix(ct)), token); err != nil {
+		log.Error("del token failed, dao.db.Exec(%s) error(%v)", hex.EncodeToString(token), err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// AddTokenDeleted save token deleted
+func (d *Dao) AddTokenDeleted(c context.Context, t *model.Token, token []byte, ct time.Time) (affected int64, err error) {
+	row, err := d.db.Exec(c, fmt.Sprintf(_addTokenDeletedSQL, formatSuffix(ct)), t.Mid, t.AppID, token, t.Expires, t.Type, t.Ctime)
+	if err != nil {
+		log.Error("fail to add token deleted, token(%+v), tx.Exec() error(%+v)", t, err)
+		return
+	}
+	return row.RowsAffected()
+}
+
+func formatSuffix(t time.Time) string {
+	return t.Format("200601")
+}

app/job/main/passport-auth/dao/token_test.go

@@ -0,0 +1,78 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoAddToken(t *testing.T) {
+	convey.Convey("AddToken", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			no    = &model.Token{}
+			token = []byte("9df38fe4b94a47baad001ad823b84110")
+			ct    = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.AddToken(c, no, token, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelToken(t *testing.T) {
+	convey.Convey("DelToken", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			token = []byte("9df38fe4b94a47baad001ad823b84110")
+			ct    = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.DelToken(c, token, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoAddTokenDeleted(t *testing.T) {
+	convey.Convey("AddTokenDeleted", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			no    = &model.Token{}
+			token = []byte("9df38fe4b94a47baad001ad823b84110")
+			ct    = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			affected, err := d.AddTokenDeleted(c, no, token, ct)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoformatSuffix(t *testing.T) {
+	convey.Convey("formatSuffix", t, func(ctx convey.C) {
+		var (
+			no = time.Now()
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			p1 := formatSuffix(no)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/passport-auth/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/passport-auth/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-auth/http/http.go

@@ -0,0 +1,45 @@
+package http
+
+import (
+	"go-common/app/job/main/passport-auth/conf"
+	"go-common/app/job/main/passport-auth/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	svc *service.Service
+)
+
+// Init init
+func Init(c *conf.Config) {
+	initService(c)
+	// init router
+	engineInner := bm.DefaultServer(c.BM)
+	outerRouter(engineInner)
+	if err := engineInner.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// initService init services.
+func initService(c *conf.Config) {
+	svc = service.New(c)
+}
+
+// outerRouter init outer router api path.
+func outerRouter(e *bm.Engine) {
+	//init api
+	e.Ping(ping)
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+}
+
+// this can delete
+func howToStart(c *bm.Context) {
+	out := "[\n {\n\ttitle: 如有问题请联系(企业微信) |&&|,\n\tname: 刘玄(小鱼生)\n },\n {\n\ttitle: 一键初始化项目文档,\n\turl: http://info.bilibili.co/pages/viewpage.action?pageId=7548250\n }\n]"
+	c.String(0, out)
+}

app/job/main/passport-auth/model/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["token.go"],
+    importpath = "go-common/app/job/main/passport-auth/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/passport-auth/model/token.go

@@ -0,0 +1,94 @@
+package model
+
+import (
+	"encoding/json"
+
+	go_common_time "go-common/library/time"
+)
+
+// BMsg databus binlog message.
+type BMsg struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+// OldToken old token
+type OldToken struct {
+	ID           int64  `json:"id"`
+	Mid          int64  `json:"mid"`
+	AppID        int64  `json:"appid"`
+	AccessToken  string `json:"access_token"`
+	RefreshToken string `json:"refresh_token"`
+	AppSubID     int64  `json:"app_subid"`
+	CreateAt     int64  `json:"create_at"`
+	Expires      int64  `json:"expires"`
+	Type         int64  `json:"type"`
+	CTime        string `json:"ctime"`
+}
+
+//OldCookie old cookie
+type OldCookie struct {
+	ID         int64  `json:"id"`
+	Mid        int64  `json:"mid"`
+	Session    string `json:"session_data"`
+	CSRFToken  string `json:"csrf_token"`
+	Type       int64  `json:"type"`
+	Expires    int64  `json:"expire_time"`
+	ModifyTime string `json:"modify_time"`
+}
+
+// Cookie for auth
+type Cookie struct {
+	ID      int64
+	Mid     int64
+	Session string
+	CSRF    string
+	Type    int64
+	Expires int64
+	Ctime   go_common_time.Time
+	Mtime   go_common_time.Time
+}
+
+// AuthCookie for auth
+type AuthCookie struct {
+	ID      int64  `json:"id"`
+	Mid     int64  `json:"mid"`
+	Session string `json:"session"`
+	CSRF    string `json:"csrf"`
+	Type    int64  `json:"type"`
+	Expires int64  `json:"expires"`
+}
+
+// AuthToken for auth
+type AuthToken struct {
+	ID      int64  `json:"id"`
+	Mid     int64  `json:"mid"`
+	AppID   int64  `json:"appid"`
+	Token   string `json:"token"`
+	Expires int64  `json:"expires"`
+	Type    int64  `json:"type"`
+}
+
+// Token for auth
+type Token struct {
+	ID      int64
+	Mid     int64
+	AppID   int64
+	Token   string
+	Expires int64
+	Type    int64
+	Ctime   go_common_time.Time
+}
+
+// Refresh for auth
+type Refresh struct {
+	ID      int64
+	Mid     int64
+	AppID   int64
+	Refresh string
+	Token   string
+	Expires int64
+	Ctime   go_common_time.Time
+}

app/job/main/passport-auth/service/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "clean_cache.go",
+        "service.go",
+        "syn_auth.go",
+        "sync_all.go",
+    ],
+    importpath = "go-common/app/job/main/passport-auth/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/dao:go_default_library",
+        "//app/job/main/passport-auth/model:go_default_library",
+        "//app/service/main/passport-auth/rpc/client:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/databusutil:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "clean_cache_test.go",
+        "service_test.go",
+        "syn_auth_test.go",
+        "sync_all_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/passport-auth/service/clean_cache.go

@@ -0,0 +1,138 @@
+package service
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+type authTokenBMsg struct {
+	Action string
+	Table  string
+	New    *model.AuthToken
+}
+
+type authCookieBMsg struct {
+	Action string
+	Table  string
+	New    *model.AuthCookie
+}
+
+func (s *Service) authConsumeProc() {
+	// fill callbacks
+	s.authGroup.New = func(msg *databus.Message) (res interface{}, err error) {
+		bmsg := new(model.BMsg)
+		if err = json.Unmarshal(msg.Value, &bmsg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			return
+		}
+		log.Info("receive auth msg action(%s) table(%s) key(%s) partition(%d) offset(%d) timestamp(%d) New(%s) Old(%s)",
+			bmsg.Action, bmsg.Table, msg.Key, msg.Partition, msg.Offset, msg.Timestamp, string(bmsg.New), string(bmsg.Old))
+		if strings.HasPrefix(bmsg.Table, "user_token_") {
+			tokenBMsg := &authTokenBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newToken := new(model.AuthToken)
+			if err = json.Unmarshal(bmsg.New, &newToken); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			tokenBMsg.New = newToken
+			return tokenBMsg, nil
+		} else if strings.HasPrefix(bmsg.Table, "user_cookie_") {
+			cookieBMsg := &authCookieBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newCookie := new(model.AuthCookie)
+			if err = json.Unmarshal(bmsg.New, newCookie); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			cookieBMsg.New = newCookie
+			return cookieBMsg, nil
+		}
+		return
+	}
+	s.authGroup.Split = func(msg *databus.Message, data interface{}) int {
+		if t, ok := data.(*authTokenBMsg); ok {
+			return int(t.New.Mid)
+		} else if t, ok := data.(*authCookieBMsg); ok {
+			return int(t.New.Mid)
+		}
+		return 0
+	}
+	s.authGroup.Do = func(msgs []interface{}) {
+		for _, m := range msgs {
+			if msg, ok := m.(*authTokenBMsg); ok {
+				if msg.Action != "delete" {
+					return
+				}
+				for {
+					if err := s.cleanTokenCache(msg.New.Token, msg.New.Mid); err != nil {
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			} else if msg, ok := m.(*authCookieBMsg); ok {
+				if msg.Action != "delete" {
+					return
+				}
+				for {
+					if err := s.cleanCookieCache(msg.New.Session, msg.New.Mid); err != nil {
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			}
+		}
+	}
+	// start the group
+	s.authGroup.Start()
+}
+
+func (s *Service) cleanTokenCache(tokenBase64 string, mid int64) (err error) {
+	var bytes []byte
+	if bytes, err = base64.StdEncoding.DecodeString(tokenBase64); err != nil {
+		log.Error("cleanTokenCache base64 decode err %v", err)
+		err = nil
+		return
+	}
+	token := hex.EncodeToString(bytes)
+	if err = s.authRPC.DelTokenCache(context.Background(), token); err != nil {
+		log.Error("cleanTokenCache err, %v", err)
+		return
+	}
+	if err = s.dao.AsoCleanCache(context.Background(), token, "", mid); err != nil {
+		return
+	}
+	return
+}
+
+func (s *Service) cleanCookieCache(cookieBase64 string, mid int64) (err error) {
+	var bytes []byte
+	if bytes, err = base64.StdEncoding.DecodeString(cookieBase64); err != nil {
+		log.Error("cleanCookieCache base64 decode err %v", err)
+		err = nil
+		return
+	}
+	session := string(bytes)
+	if err = s.authRPC.DelCookieCookie(context.Background(), session); err != nil {
+		log.Error("cleanCookieCache err, %v", err)
+		return
+	}
+	if err = s.dao.AsoCleanCache(context.Background(), "", session, mid); err != nil {
+		return
+	}
+	return
+}

app/job/main/passport-auth/service/clean_cache_test.go

@@ -0,0 +1,26 @@
+package service
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestServiceBase64Decode(t *testing.T) {
+	convey.Convey("", t, func() {
+		bytes, err := base64.StdEncoding.DecodeString("igIDgs/yFxaFI+oiu2HoDw==")
+		convey.So(bytes, convey.ShouldNotBeEmpty)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(hex.EncodeToString(bytes), convey.ShouldEqual, "8a020382cff217168523ea22bb61e80f")
+	})
+}
+
+func TestService_cleanTokenCache(t *testing.T) {
+	once.Do(startService)
+	convey.Convey("cleanTokenCache", t, func() {
+		err := s.cleanTokenCache("igIDgs/yFxaFI+oiu2HoDw==", 0)
+		convey.So(err, convey.ShouldNotBeNil)
+	})
+}

app/job/main/passport-auth/service/service.go

@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-auth/conf"
+	"go-common/app/job/main/passport-auth/dao"
+	auth "go-common/app/service/main/passport-auth/rpc/client"
+	"go-common/library/queue/databus"
+	"go-common/library/queue/databus/databusutil"
+)
+
+// Service struct
+type Service struct {
+	c               *conf.Config
+	dao             *dao.Dao
+	g               *databusutil.Group
+	oldAuthConsumer *databus.Databus
+	authRPC         *auth.Service
+	authConsumer    *databus.Databus
+	authGroup       *databusutil.Group
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:               c,
+		dao:             dao.New(c),
+		oldAuthConsumer: databus.New(c.Databus),
+		authRPC:         auth.New(c.AuthRPC),
+		authConsumer:    databus.New(c.AuthDataBus),
+	}
+	// new a group
+	s.g = databusutil.NewGroup(
+		c.DatabusUtil,
+		s.oldAuthConsumer.Messages(),
+	)
+	s.authGroup = databusutil.NewGroup(
+		c.DatabusUtil,
+		s.authConsumer.Messages(),
+	)
+	s.consumeproc()
+	s.authConsumeProc()
+	// go s.syncCookie()
+	// for i := c.IDXFrom; i < c.IDXTo; i ++ {
+	// 	go s.syncCookie(int64(i))
+	// }
+
+	// go s.syncToken("201804", 0, 50000000)
+	// go s.syncToken("201804", 50000001, 100000000)
+	// go s.syncToken("201804", 100000001, 150000000)
+	// go s.syncToken("201804", 150000001, 0)
+
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.g.Close()
+	s.authGroup.Close()
+	s.dao.Close()
+}

app/job/main/passport-auth/service/service_test.go

@@ -0,0 +1,20 @@
+package service
+
+import (
+	"sync"
+
+	"go-common/app/job/main/passport-auth/conf"
+)
+
+var (
+	once sync.Once
+	s    *Service
+)
+
+func startService() {
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	// service init
+	s = New(conf.Conf)
+}

app/job/main/passport-auth/service/syn_auth.go

@@ -0,0 +1,236 @@
+package service
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+)
+
+var (
+	local, _ = time.LoadLocation("Local")
+)
+
+type tokenBMsg struct {
+	Action string
+	Table  string
+	New    *model.OldToken
+}
+
+type cookieBMsg struct {
+	Action string
+	Table  string
+	New    *model.OldCookie
+}
+
+func (s *Service) consumeproc() {
+	// fill callbacks
+	s.g.New = func(msg *databus.Message) (res interface{}, err error) {
+		bmsg := new(model.BMsg)
+		if err = json.Unmarshal(msg.Value, &bmsg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			return
+		}
+		log.Info("receive aso msg action(%s) table(%s) key(%s) partition(%d) offset(%d) timestamp(%d) New(%s) Old(%s)",
+			bmsg.Action, bmsg.Table, msg.Key, msg.Partition, msg.Offset, msg.Timestamp, string(bmsg.New), string(bmsg.Old))
+		if strings.HasPrefix(bmsg.Table, "aso_app_perm") {
+			tokenBMsg := &tokenBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newToken := new(model.OldToken)
+			if err = json.Unmarshal(bmsg.New, &newToken); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			tokenBMsg.New = newToken
+			return tokenBMsg, nil
+		} else if strings.HasPrefix(bmsg.Table, "aso_cookie_token") {
+			cookieBMsg := &cookieBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newCookie := new(model.OldCookie)
+			if err = json.Unmarshal(bmsg.New, newCookie); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			cookieBMsg.New = newCookie
+			return cookieBMsg, nil
+		}
+		return
+	}
+	s.g.Split = func(msg *databus.Message, data interface{}) int {
+		if t, ok := data.(*tokenBMsg); ok {
+			return int(t.New.Mid)
+		} else if t, ok := data.(*cookieBMsg); ok {
+			return int(t.New.Mid)
+		}
+		return 0
+	}
+	s.g.Do = func(msgs []interface{}) {
+		for _, m := range msgs {
+			if msg, ok := m.(*tokenBMsg); ok {
+				for {
+					if err := s.handleToken(msg); err != nil {
+						log.Error("do handleToken error", err)
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			} else if msg, ok := m.(*cookieBMsg); ok {
+				for {
+					if err := s.handleCookie(msg); err != nil {
+						log.Error("do handleCookie error", err)
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			}
+		}
+	}
+	// start the group
+	s.g.Start()
+}
+
+func (s *Service) handleCookie(cookie *cookieBMsg) (err error) {
+	newCookie := &model.Cookie{
+		Mid:     cookie.New.Mid,
+		Session: cookie.New.Session,
+		CSRF:    cookie.New.CSRFToken,
+		Expires: cookie.New.Expires,
+	}
+	csrfByte, _ := hex.DecodeString(cookie.New.CSRFToken)
+	if strings.ToLower(cookie.Action) == "insert" {
+		if _, err = s.dao.AddCookie(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, time.Now()); err != nil {
+			return
+		}
+		return
+	} else if strings.ToLower(cookie.Action) == "delete" {
+		now := time.Now()
+		if _, err = s.dao.DelCookie(context.Background(), []byte(cookie.New.Session), now); err != nil {
+			log.Error("del db cookie(%s) error(%+v)", cookie.New.Session, err)
+			return
+		}
+		if _, err = s.dao.DelCookie(context.Background(), []byte(cookie.New.Session), previousMonth(now, -1)); err != nil {
+			log.Error("del db cookie(%s) error(%+v)", cookie.New.Session, err)
+			return
+		}
+		if newCookie.Expires > now.Unix() {
+			var t time.Time
+			if cookie.New.ModifyTime == "0000-00-00 00:00:00" {
+				t = time.Now()
+			} else {
+				t, err = time.ParseInLocation("2006-01-02 15:04:05", cookie.New.ModifyTime, local)
+				if err != nil {
+					log.Error("handleCookie error: ctime parse error(%+v) cookie(%+v)", err, cookie.New)
+					return
+				}
+			}
+			timestamp := xtime.Time(t.Unix())
+			newCookie.Ctime = timestamp
+			if _, err = s.dao.AddCookieDeleted(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, t); err != nil {
+				return
+			}
+		}
+		if err = s.authRPC.DelCookieCookie(context.Background(), cookie.New.Session); err != nil {
+			log.Error("del cache cookie(%s) error(%+v)", cookie.New.Session, err)
+		}
+		if err = s.dao.AsoCleanCache(context.Background(), "", cookie.New.Session, cookie.New.Mid); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (s *Service) handleToken(token *tokenBMsg) (err error) {
+	var t time.Time
+	t, err = time.ParseInLocation("2006-01-02 15:04:05", token.New.CTime, local)
+	if err != nil {
+		log.Error("handleToken error: ctime parse err. (%+v)", token.New)
+		return
+	}
+	timestamp := xtime.Time(t.Unix())
+	newToken := &model.Token{
+		Mid:     token.New.Mid,
+		Token:   token.New.AccessToken,
+		AppID:   token.New.AppID,
+		Expires: token.New.Expires,
+		Type:    token.New.Type,
+		Ctime:   timestamp,
+	}
+	tokenByte, _ := hex.DecodeString(newToken.Token)
+	if strings.ToLower(token.Action) == "insert" {
+		if _, err = s.dao.AddToken(context.Background(), newToken, tokenByte, t); err != nil {
+			return
+		}
+		if token.New.RefreshToken == "" {
+			return
+		}
+		newRefresh := &model.Refresh{
+			Mid:     token.New.Mid,
+			Refresh: token.New.RefreshToken,
+			Token:   token.New.AccessToken,
+			AppID:   token.New.AppID,
+			Expires: token.New.Expires, // 需要考虑+30天
+		}
+		tokenByteRefresh, _ := hex.DecodeString(newRefresh.Token)
+		refreshByte, _ := hex.DecodeString(newRefresh.Refresh)
+		if _, err = s.dao.AddRefresh(context.Background(), newRefresh, refreshByte, tokenByteRefresh, t); err != nil {
+			return
+		}
+	} else if strings.ToLower(token.Action) == "delete" {
+		now := time.Now()
+		if _, err = s.dao.DelToken(context.Background(), tokenByte, now); err != nil {
+			log.Error("del db token(%s) error(%+v)", token.New.AccessToken, err)
+			return
+		}
+		if _, err = s.dao.DelToken(context.Background(), tokenByte, previousMonth(now, -1)); err != nil {
+			log.Error("del db token(%s) error(%+v)", token.New.AccessToken, err)
+			return
+		}
+		if _, err = s.dao.DelToken(context.Background(), tokenByte, previousMonth(now, -2)); err != nil {
+			log.Error("del db token(%s) error(%+v)", token.New.AccessToken, err)
+			return
+		}
+		if _, err = s.dao.AddTokenDeleted(context.Background(), newToken, tokenByte, t); err != nil {
+			return
+		}
+		if err = s.authRPC.DelTokenCache(context.Background(), token.New.AccessToken); err != nil {
+			log.Error("del cache token(%s) error(%+v)", token.New.AccessToken, err)
+		}
+		if err = s.dao.AsoCleanCache(context.Background(), token.New.AccessToken, "", token.New.Mid); err != nil {
+			return
+		}
+		if token.New.RefreshToken == "" {
+			return
+		}
+		refreshByte, _ := hex.DecodeString(token.New.RefreshToken)
+		if _, err = s.dao.DelRefresh(context.Background(), refreshByte, t); err != nil {
+			log.Error("del db refresh(%s) error(%+v)", token.New.RefreshToken, err)
+			return
+		}
+		if _, err = s.dao.DelRefresh(context.Background(), refreshByte, previousMonth(t, -2)); err != nil {
+			log.Error("del db refresh(%s) error(%+v)", token.New.RefreshToken, err)
+			return
+		}
+	}
+	return
+}
+
+func previousMonth(t time.Time, delta int) time.Time {
+	if delta == 0 {
+		return t
+	}
+	year, month, _ := t.Date()
+	thisMonthFirstDay := time.Date(year, month, 1, 1, 1, 1, 1, t.Location())
+	return thisMonthFirstDay.AddDate(0, delta, 0)
+}

app/job/main/passport-auth/service/syn_auth_test.go

@@ -0,0 +1,47 @@
+package service
+
+import (
+	"testing"
+
+	"go-common/app/job/main/passport-auth/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_HandleToken(t *testing.T) {
+	once.Do(startService)
+	Convey("Test del cookie by cookie", t, func() {
+		token := &tokenBMsg{
+			Action: "insert",
+			Table:  "asp_app_perm_201701",
+			New: &model.OldToken{
+				Mid:          4186264,
+				AppID:        1,
+				AccessToken:  "test token",
+				RefreshToken: "test refresh",
+				AppSubID:     0,
+				CreateAt:     123456789000,
+			},
+		}
+		err := s.handleToken(token)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_HandleCookie(t *testing.T) {
+	once.Do(startService)
+	Convey("Test del cookie by cookie", t, func() {
+		cookie := &cookieBMsg{
+			Action: "insert",
+			Table:  "asp_app_perm_201701",
+			New: &model.OldCookie{
+				Mid:       4186264,
+				Session:   "test session",
+				CSRFToken: "test csrf",
+				Expires:   1234567890000,
+			},
+		}
+		err := s.handleCookie(cookie)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/passport-auth/service/sync_all.go

@@ -0,0 +1,308 @@
+package service
+
+//var (
+//may = time.Now().AddDate(0, -1, 0)
+//apr = time.Now().AddDate(0, -2, 0)
+//)
+
+/*
+func (s *Service) syncToken(tabIdx string, min, max int64) {
+	var minID int64
+	var t time.Time
+	if tabIdx == "201805" {
+		t = may
+		minID = 167003927
+	} else if tabIdx == "201804" {
+		t = apr
+		minID = min
+	}
+	for {
+		tokens, err := s.dao.TokenMultiQuery(context.Background(), tabIdx, minID, s.c.SyncLines)
+		if err != nil {
+			log.Error("s.dao.TokenMultiQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(tokens) == 0 {
+			break
+		}
+		minID = setToken(tokens, s, minID, t)
+		if max > 0 && max <= minID {
+			break
+		}
+	}
+}
+
+func setToken(tokens []*model.OldToken, s *Service, minID int64, t time.Time) int64 {
+	newTokens := make([]*model.Token, 0, 51)
+	newRefreshs := make([]*model.Refresh, 0, 51)
+	for _, token := range tokens {
+		newToken := &model.Token{
+			Mid:     token.Mid,
+			AppID:   token.AppID,
+			Token:   token.AccessToken,
+			Expires: token.Expires,
+			Type:    token.Type,
+		}
+		newTokens = append(newTokens, newToken)
+		if len(newTokens) == 50 {
+			s.dao.BatchAddToken(context.Background(), newTokens, t)
+			newTokens = make([]*model.Token, 0, 51)
+		}
+		if minID < token.ID {
+			minID = token.ID
+		}
+		if token.RefreshToken == "" || token.Expires < time.Now().Unix() {
+			continue
+		}
+		newRefresh := &model.Refresh{
+			Mid:     token.Mid,
+			AppID:   token.AppID,
+			Refresh: token.RefreshToken,
+			Token:   token.AccessToken,
+			Expires: token.Expires,
+		}
+		newRefreshs = append(newRefreshs, newRefresh)
+		if len(newRefreshs) == 50 {
+			s.dao.BatchAddRefresh(context.Background(), newRefreshs, t)
+			newRefreshs = make([]*model.Refresh, 0, 51)
+		}
+	}
+	if len(newTokens) != 0 {
+		for _, token := range newTokens {
+			tokenByte, err := hex.DecodeString(token.Token)
+			if err != nil {
+				log.Error("setNewToken error: tokenByte decode hex err: %v", token)
+				continue
+			}
+			s.dao.AddToken(context.Background(), token, tokenByte, t)
+		}
+	}
+	if len(newRefreshs) != 0 {
+		for _, refresh := range newRefreshs {
+			refreshByte, err := hex.DecodeString(refresh.Refresh)
+			tokenByteR, err1 := hex.DecodeString(refresh.Refresh)
+			if err != nil || err1 != nil {
+				log.Error("setNewToken error: refreshByte decode hex err: %v", refresh)
+				continue
+			}
+			s.dao.AddRefresh(context.Background(), refresh, refreshByte, tokenByteR, t)
+		}
+	}
+	return minID
+}
+
+func (s *Service) setNewToken(token *model.OldToken) (err error) {
+	newToken := &model.Token{
+		Mid:     token.Mid,
+		AppID:   token.AppID,
+		Token:   token.AccessToken,
+		Expires: token.Expires,
+		Type:    token.Type,
+	}
+	var t time.Time
+	t, err = time.ParseInLocation("2006-01-02T15:04:05Z07:00", token.CTime, time.Local)
+	if err != nil {
+		log.Error("setNewToken error: ctime parse err. %v", token)
+		return
+	}
+	tokenByte, err := hex.DecodeString(token.AccessToken)
+	if err != nil {
+		log.Error("setNewToken error: tokenByte decode hex err: %v", token)
+		return
+	}
+	s.dao.AddToken(context.Background(), newToken, tokenByte, t)
+	if token.RefreshToken == "" || t.Unix() < 1525104000 {
+		return
+	}
+	newRefresh := &model.Refresh{
+		Mid:     token.Mid,
+		AppID:   token.AppID,
+		Refresh: token.RefreshToken,
+		Token:   token.AccessToken,
+		Expires: token.Expires,
+	}
+	refreshByte, err := hex.DecodeString(token.RefreshToken)
+	if err != nil {
+		log.Error("setNewToken error: refreshByte decode hex err: %v", token)
+		return
+	}
+	tokenByteR, err := hex.DecodeString(token.AccessToken)
+	if err != nil {
+		log.Error("setNewToken error: tokenByte decode hex err: %v", token)
+		return
+	}
+	s.dao.AddRefresh(context.Background(), newRefresh, refreshByte, tokenByteR, t)
+	return
+}
+
+func (s *Service) syncCookie(tabIdx int64) {
+	var minID int64
+	for {
+		now := time.Now().Unix()
+		cookies, err := s.dao.CookieMultiQuery(context.Background(), tabIdx, minID, now, s.c.SyncLines)
+		if err != nil {
+			log.Error("s.dao.CookieMultiQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(cookies) == 0 {
+			log.Info("syncCookie finished tabIdx : %d", tabIdx)
+			break
+		}
+		minID = s.setCookie(cookies, now, minID)
+	}
+}
+
+func (s *Service) setCookie(cookies []*model.OldCookie, now int64, minID int64) int64 {
+	today := time.Now()
+	newCookies := make([]*model.Cookie, 0, 51)
+	var count int64
+	for _, cookie := range cookies {
+		if count%1000 == 0 {
+			// for down cpu
+			time.Sleep(10 * time.Millisecond)
+		}
+		if minID < cookie.ID {
+			minID = cookie.ID
+		}
+		if len(newCookies) == 50 {
+			s.dao.AddMultiCookie(context.Background(), newCookies, today)
+			newCookies = make([]*model.Cookie, 0, 51)
+		}
+		var t time.Time
+		t, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", cookie.ModifyTime, time.Local)
+		if err != nil {
+			log.Error("setNewToken error: ctime parse err. %v", cookie)
+			continue
+		}
+		if cookie.Expires > now && t.Unix() > 1527782400 {
+			newCookies = append(newCookies, &model.Cookie{
+				Mid:     cookie.Mid,
+				Session: cookie.Session,
+				CSRF:    cookie.CSRFToken,
+				Type:    cookie.Type,
+				Expires: cookie.Expires,
+			})
+		}
+		count = count + 1
+	}
+	if len(newCookies) != 0 {
+		for _, newCookie := range newCookies {
+			csrfByte, err := hex.DecodeString(newCookie.CSRF)
+			if err != nil {
+				log.Error("hex.DecodeString csrf err: %v, %v", newCookie, err)
+				continue
+			}
+			s.dao.AddCookie(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, today)
+		}
+	}
+	return minID
+}
+*/
+// func (s *Service) setNewCookie(cookie *model.OldCookie) (err error) {
+// 	newCookie := &model.Cookie{
+// 		Mid:     cookie.Mid,
+// 		Session: cookie.Session,
+// 		CSRF:    cookie.CSRFToken,
+// 		Type:    cookie.Type,
+// 		Expires: cookie.Expires,
+// 	}
+// 	csrfByte, err := hex.DecodeString(newCookie.CSRF)
+// 	if err != nil {
+// 		log.Error("hex.DecodeString csrf err: %v, %v", cookie, err)
+// 		return
+// 	}
+// 	if _, err = s.dao.AddCookie(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, time.Now()); err != nil {
+// 		log.Error("s.dao.AddCookie %v, %v", cookie, err)
+// 		return
+// 	}
+// 	return nil
+// }
+/*
+func (s *Service) compareToken(tabIdx string, min int64, max int64) {
+	var t time.Time
+	if tabIdx == "201805" {
+		t = may
+	} else if tabIdx == "201804" {
+		t = apr
+	} else {
+		t = time.Now()
+	}
+	minID := min
+	for {
+		tokens, err := s.dao.BatchGetToken(context.Background(), minID, s.c.SyncLines, t)
+		if err != nil {
+			log.Error("compareToken error: s.dao.BatchGetToken err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(tokens) == 0 {
+			break
+		}
+		tokenMap, err := s.dao.TokenBatchQuery(context.Background(), tabIdx, tokens)
+		if err != nil {
+			log.Error("compareToken error: s.dao.TokenBatchQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		for _, newToken := range tokens {
+			if tokenMap[newToken.Token] == "" {
+				tokenByte, err := hex.DecodeString(newToken.Token)
+				if err != nil {
+					log.Error("compareToken error: tokenByte decode hex err: %v", err)
+					continue
+				}
+				s.dao.DelToken(context.Background(), tokenByte, t)
+			}
+		}
+		minID = tokens[len(tokens)-1].ID
+		if max > 0 && max <= minID {
+			break
+		}
+	}
+	log.Info("token compare finished %s, %d - %d", tabIdx, minID, max)
+}
+
+func (s *Service) compareCookie(tabIdx string, min int64) {
+	var t time.Time
+	if tabIdx == "201805" {
+		t = may
+	} else if tabIdx == "201804" {
+		t = apr
+	} else {
+		t = time.Now()
+	}
+	minID := min
+	for {
+		cookies, err := s.dao.BatchGetCookie(context.Background(), minID, s.c.SyncLines, t)
+		if err != nil {
+			log.Error("compareCookie error: s.dao.BatchGetCookie err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(cookies) == 0 {
+			break
+		}
+		tabIdxMap := make(map[int64][]string)
+		for _, newCookie := range cookies {
+			oldTabIdx := newCookie.Mid % 30
+			tabIdxMap[oldTabIdx] = append(tabIdxMap[oldTabIdx], newCookie.Session)
+		}
+
+		sessionMap, err := s.dao.CookieBatchQuery(context.Background(), tabIdxMap)
+		if err != nil {
+			log.Error("compareCookie error: s.dao.CookieBatchQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		for _, newCookie := range cookies {
+			if sessionMap[newCookie.Session] == "" {
+				s.dao.DelCookie(context.Background(), []byte(newCookie.Session), t)
+			}
+		}
+		minID = cookies[len(cookies)-1].ID
+	}
+}*/

app/job/main/passport-auth/service/sync_all_test.go

@@ -0,0 +1,21 @@
+package service
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_compareToken(t *testing.T) {
+	once.Do(startService)
+	Convey("Test compareToken", t, func() {
+		//s.compareToken("201806", 0, 10)
+	})
+}
+
+func Test_compareCookie(t *testing.T) {
+	once.Do(startService)
+	Convey("Test compareCookie", t, func() {
+		//s.compareCookie("201806", 0)
+	})
+}