Create & Init Project...

app/job/main/passport-auth/service/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "clean_cache.go",
+        "service.go",
+        "syn_auth.go",
+        "sync_all.go",
+    ],
+    importpath = "go-common/app/job/main/passport-auth/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/dao:go_default_library",
+        "//app/job/main/passport-auth/model:go_default_library",
+        "//app/service/main/passport-auth/rpc/client:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/databusutil:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "clean_cache_test.go",
+        "service_test.go",
+        "syn_auth_test.go",
+        "sync_all_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/passport-auth/conf:go_default_library",
+        "//app/job/main/passport-auth/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/passport-auth/service/clean_cache.go

@@ -0,0 +1,138 @@
+package service
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+type authTokenBMsg struct {
+	Action string
+	Table  string
+	New    *model.AuthToken
+}
+
+type authCookieBMsg struct {
+	Action string
+	Table  string
+	New    *model.AuthCookie
+}
+
+func (s *Service) authConsumeProc() {
+	// fill callbacks
+	s.authGroup.New = func(msg *databus.Message) (res interface{}, err error) {
+		bmsg := new(model.BMsg)
+		if err = json.Unmarshal(msg.Value, &bmsg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			return
+		}
+		log.Info("receive auth msg action(%s) table(%s) key(%s) partition(%d) offset(%d) timestamp(%d) New(%s) Old(%s)",
+			bmsg.Action, bmsg.Table, msg.Key, msg.Partition, msg.Offset, msg.Timestamp, string(bmsg.New), string(bmsg.Old))
+		if strings.HasPrefix(bmsg.Table, "user_token_") {
+			tokenBMsg := &authTokenBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newToken := new(model.AuthToken)
+			if err = json.Unmarshal(bmsg.New, &newToken); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			tokenBMsg.New = newToken
+			return tokenBMsg, nil
+		} else if strings.HasPrefix(bmsg.Table, "user_cookie_") {
+			cookieBMsg := &authCookieBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newCookie := new(model.AuthCookie)
+			if err = json.Unmarshal(bmsg.New, newCookie); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			cookieBMsg.New = newCookie
+			return cookieBMsg, nil
+		}
+		return
+	}
+	s.authGroup.Split = func(msg *databus.Message, data interface{}) int {
+		if t, ok := data.(*authTokenBMsg); ok {
+			return int(t.New.Mid)
+		} else if t, ok := data.(*authCookieBMsg); ok {
+			return int(t.New.Mid)
+		}
+		return 0
+	}
+	s.authGroup.Do = func(msgs []interface{}) {
+		for _, m := range msgs {
+			if msg, ok := m.(*authTokenBMsg); ok {
+				if msg.Action != "delete" {
+					return
+				}
+				for {
+					if err := s.cleanTokenCache(msg.New.Token, msg.New.Mid); err != nil {
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			} else if msg, ok := m.(*authCookieBMsg); ok {
+				if msg.Action != "delete" {
+					return
+				}
+				for {
+					if err := s.cleanCookieCache(msg.New.Session, msg.New.Mid); err != nil {
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			}
+		}
+	}
+	// start the group
+	s.authGroup.Start()
+}
+
+func (s *Service) cleanTokenCache(tokenBase64 string, mid int64) (err error) {
+	var bytes []byte
+	if bytes, err = base64.StdEncoding.DecodeString(tokenBase64); err != nil {
+		log.Error("cleanTokenCache base64 decode err %v", err)
+		err = nil
+		return
+	}
+	token := hex.EncodeToString(bytes)
+	if err = s.authRPC.DelTokenCache(context.Background(), token); err != nil {
+		log.Error("cleanTokenCache err, %v", err)
+		return
+	}
+	if err = s.dao.AsoCleanCache(context.Background(), token, "", mid); err != nil {
+		return
+	}
+	return
+}
+
+func (s *Service) cleanCookieCache(cookieBase64 string, mid int64) (err error) {
+	var bytes []byte
+	if bytes, err = base64.StdEncoding.DecodeString(cookieBase64); err != nil {
+		log.Error("cleanCookieCache base64 decode err %v", err)
+		err = nil
+		return
+	}
+	session := string(bytes)
+	if err = s.authRPC.DelCookieCookie(context.Background(), session); err != nil {
+		log.Error("cleanCookieCache err, %v", err)
+		return
+	}
+	if err = s.dao.AsoCleanCache(context.Background(), "", session, mid); err != nil {
+		return
+	}
+	return
+}

app/job/main/passport-auth/service/clean_cache_test.go

@@ -0,0 +1,26 @@
+package service
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestServiceBase64Decode(t *testing.T) {
+	convey.Convey("", t, func() {
+		bytes, err := base64.StdEncoding.DecodeString("igIDgs/yFxaFI+oiu2HoDw==")
+		convey.So(bytes, convey.ShouldNotBeEmpty)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(hex.EncodeToString(bytes), convey.ShouldEqual, "8a020382cff217168523ea22bb61e80f")
+	})
+}
+
+func TestService_cleanTokenCache(t *testing.T) {
+	once.Do(startService)
+	convey.Convey("cleanTokenCache", t, func() {
+		err := s.cleanTokenCache("igIDgs/yFxaFI+oiu2HoDw==", 0)
+		convey.So(err, convey.ShouldNotBeNil)
+	})
+}

app/job/main/passport-auth/service/service.go

@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/passport-auth/conf"
+	"go-common/app/job/main/passport-auth/dao"
+	auth "go-common/app/service/main/passport-auth/rpc/client"
+	"go-common/library/queue/databus"
+	"go-common/library/queue/databus/databusutil"
+)
+
+// Service struct
+type Service struct {
+	c               *conf.Config
+	dao             *dao.Dao
+	g               *databusutil.Group
+	oldAuthConsumer *databus.Databus
+	authRPC         *auth.Service
+	authConsumer    *databus.Databus
+	authGroup       *databusutil.Group
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:               c,
+		dao:             dao.New(c),
+		oldAuthConsumer: databus.New(c.Databus),
+		authRPC:         auth.New(c.AuthRPC),
+		authConsumer:    databus.New(c.AuthDataBus),
+	}
+	// new a group
+	s.g = databusutil.NewGroup(
+		c.DatabusUtil,
+		s.oldAuthConsumer.Messages(),
+	)
+	s.authGroup = databusutil.NewGroup(
+		c.DatabusUtil,
+		s.authConsumer.Messages(),
+	)
+	s.consumeproc()
+	s.authConsumeProc()
+	// go s.syncCookie()
+	// for i := c.IDXFrom; i < c.IDXTo; i ++ {
+	// 	go s.syncCookie(int64(i))
+	// }
+
+	// go s.syncToken("201804", 0, 50000000)
+	// go s.syncToken("201804", 50000001, 100000000)
+	// go s.syncToken("201804", 100000001, 150000000)
+	// go s.syncToken("201804", 150000001, 0)
+
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.g.Close()
+	s.authGroup.Close()
+	s.dao.Close()
+}

app/job/main/passport-auth/service/service_test.go

@@ -0,0 +1,20 @@
+package service
+
+import (
+	"sync"
+
+	"go-common/app/job/main/passport-auth/conf"
+)
+
+var (
+	once sync.Once
+	s    *Service
+)
+
+func startService() {
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	// service init
+	s = New(conf.Conf)
+}

app/job/main/passport-auth/service/syn_auth.go

@@ -0,0 +1,236 @@
+package service
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/passport-auth/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+)
+
+var (
+	local, _ = time.LoadLocation("Local")
+)
+
+type tokenBMsg struct {
+	Action string
+	Table  string
+	New    *model.OldToken
+}
+
+type cookieBMsg struct {
+	Action string
+	Table  string
+	New    *model.OldCookie
+}
+
+func (s *Service) consumeproc() {
+	// fill callbacks
+	s.g.New = func(msg *databus.Message) (res interface{}, err error) {
+		bmsg := new(model.BMsg)
+		if err = json.Unmarshal(msg.Value, &bmsg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			return
+		}
+		log.Info("receive aso msg action(%s) table(%s) key(%s) partition(%d) offset(%d) timestamp(%d) New(%s) Old(%s)",
+			bmsg.Action, bmsg.Table, msg.Key, msg.Partition, msg.Offset, msg.Timestamp, string(bmsg.New), string(bmsg.Old))
+		if strings.HasPrefix(bmsg.Table, "aso_app_perm") {
+			tokenBMsg := &tokenBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newToken := new(model.OldToken)
+			if err = json.Unmarshal(bmsg.New, &newToken); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			tokenBMsg.New = newToken
+			return tokenBMsg, nil
+		} else if strings.HasPrefix(bmsg.Table, "aso_cookie_token") {
+			cookieBMsg := &cookieBMsg{
+				Action: bmsg.Action,
+				Table:  bmsg.Table,
+			}
+			newCookie := new(model.OldCookie)
+			if err = json.Unmarshal(bmsg.New, newCookie); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", bmsg.New, err)
+				return
+			}
+			cookieBMsg.New = newCookie
+			return cookieBMsg, nil
+		}
+		return
+	}
+	s.g.Split = func(msg *databus.Message, data interface{}) int {
+		if t, ok := data.(*tokenBMsg); ok {
+			return int(t.New.Mid)
+		} else if t, ok := data.(*cookieBMsg); ok {
+			return int(t.New.Mid)
+		}
+		return 0
+	}
+	s.g.Do = func(msgs []interface{}) {
+		for _, m := range msgs {
+			if msg, ok := m.(*tokenBMsg); ok {
+				for {
+					if err := s.handleToken(msg); err != nil {
+						log.Error("do handleToken error", err)
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			} else if msg, ok := m.(*cookieBMsg); ok {
+				for {
+					if err := s.handleCookie(msg); err != nil {
+						log.Error("do handleCookie error", err)
+						time.Sleep(100 * time.Millisecond)
+						continue
+					}
+					break
+				}
+			}
+		}
+	}
+	// start the group
+	s.g.Start()
+}
+
+func (s *Service) handleCookie(cookie *cookieBMsg) (err error) {
+	newCookie := &model.Cookie{
+		Mid:     cookie.New.Mid,
+		Session: cookie.New.Session,
+		CSRF:    cookie.New.CSRFToken,
+		Expires: cookie.New.Expires,
+	}
+	csrfByte, _ := hex.DecodeString(cookie.New.CSRFToken)
+	if strings.ToLower(cookie.Action) == "insert" {
+		if _, err = s.dao.AddCookie(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, time.Now()); err != nil {
+			return
+		}
+		return
+	} else if strings.ToLower(cookie.Action) == "delete" {
+		now := time.Now()
+		if _, err = s.dao.DelCookie(context.Background(), []byte(cookie.New.Session), now); err != nil {
+			log.Error("del db cookie(%s) error(%+v)", cookie.New.Session, err)
+			return
+		}
+		if _, err = s.dao.DelCookie(context.Background(), []byte(cookie.New.Session), previousMonth(now, -1)); err != nil {
+			log.Error("del db cookie(%s) error(%+v)", cookie.New.Session, err)
+			return
+		}
+		if newCookie.Expires > now.Unix() {
+			var t time.Time
+			if cookie.New.ModifyTime == "0000-00-00 00:00:00" {
+				t = time.Now()
+			} else {
+				t, err = time.ParseInLocation("2006-01-02 15:04:05", cookie.New.ModifyTime, local)
+				if err != nil {
+					log.Error("handleCookie error: ctime parse error(%+v) cookie(%+v)", err, cookie.New)
+					return
+				}
+			}
+			timestamp := xtime.Time(t.Unix())
+			newCookie.Ctime = timestamp
+			if _, err = s.dao.AddCookieDeleted(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, t); err != nil {
+				return
+			}
+		}
+		if err = s.authRPC.DelCookieCookie(context.Background(), cookie.New.Session); err != nil {
+			log.Error("del cache cookie(%s) error(%+v)", cookie.New.Session, err)
+		}
+		if err = s.dao.AsoCleanCache(context.Background(), "", cookie.New.Session, cookie.New.Mid); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (s *Service) handleToken(token *tokenBMsg) (err error) {
+	var t time.Time
+	t, err = time.ParseInLocation("2006-01-02 15:04:05", token.New.CTime, local)
+	if err != nil {
+		log.Error("handleToken error: ctime parse err. (%+v)", token.New)
+		return
+	}
+	timestamp := xtime.Time(t.Unix())
+	newToken := &model.Token{
+		Mid:     token.New.Mid,
+		Token:   token.New.AccessToken,
+		AppID:   token.New.AppID,
+		Expires: token.New.Expires,
+		Type:    token.New.Type,
+		Ctime:   timestamp,
+	}
+	tokenByte, _ := hex.DecodeString(newToken.Token)
+	if strings.ToLower(token.Action) == "insert" {
+		if _, err = s.dao.AddToken(context.Background(), newToken, tokenByte, t); err != nil {
+			return
+		}
+		if token.New.RefreshToken == "" {
+			return
+		}
+		newRefresh := &model.Refresh{
+			Mid:     token.New.Mid,
+			Refresh: token.New.RefreshToken,
+			Token:   token.New.AccessToken,
+			AppID:   token.New.AppID,
+			Expires: token.New.Expires, // 需要考虑+30天
+		}
+		tokenByteRefresh, _ := hex.DecodeString(newRefresh.Token)
+		refreshByte, _ := hex.DecodeString(newRefresh.Refresh)
+		if _, err = s.dao.AddRefresh(context.Background(), newRefresh, refreshByte, tokenByteRefresh, t); err != nil {
+			return
+		}
+	} else if strings.ToLower(token.Action) == "delete" {
+		now := time.Now()
+		if _, err = s.dao.DelToken(context.Background(), tokenByte, now); err != nil {
+			log.Error("del db token(%s) error(%+v)", token.New.AccessToken, err)
+			return
+		}
+		if _, err = s.dao.DelToken(context.Background(), tokenByte, previousMonth(now, -1)); err != nil {
+			log.Error("del db token(%s) error(%+v)", token.New.AccessToken, err)
+			return
+		}
+		if _, err = s.dao.DelToken(context.Background(), tokenByte, previousMonth(now, -2)); err != nil {
+			log.Error("del db token(%s) error(%+v)", token.New.AccessToken, err)
+			return
+		}
+		if _, err = s.dao.AddTokenDeleted(context.Background(), newToken, tokenByte, t); err != nil {
+			return
+		}
+		if err = s.authRPC.DelTokenCache(context.Background(), token.New.AccessToken); err != nil {
+			log.Error("del cache token(%s) error(%+v)", token.New.AccessToken, err)
+		}
+		if err = s.dao.AsoCleanCache(context.Background(), token.New.AccessToken, "", token.New.Mid); err != nil {
+			return
+		}
+		if token.New.RefreshToken == "" {
+			return
+		}
+		refreshByte, _ := hex.DecodeString(token.New.RefreshToken)
+		if _, err = s.dao.DelRefresh(context.Background(), refreshByte, t); err != nil {
+			log.Error("del db refresh(%s) error(%+v)", token.New.RefreshToken, err)
+			return
+		}
+		if _, err = s.dao.DelRefresh(context.Background(), refreshByte, previousMonth(t, -2)); err != nil {
+			log.Error("del db refresh(%s) error(%+v)", token.New.RefreshToken, err)
+			return
+		}
+	}
+	return
+}
+
+func previousMonth(t time.Time, delta int) time.Time {
+	if delta == 0 {
+		return t
+	}
+	year, month, _ := t.Date()
+	thisMonthFirstDay := time.Date(year, month, 1, 1, 1, 1, 1, t.Location())
+	return thisMonthFirstDay.AddDate(0, delta, 0)
+}

app/job/main/passport-auth/service/syn_auth_test.go

@@ -0,0 +1,47 @@
+package service
+
+import (
+	"testing"
+
+	"go-common/app/job/main/passport-auth/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_HandleToken(t *testing.T) {
+	once.Do(startService)
+	Convey("Test del cookie by cookie", t, func() {
+		token := &tokenBMsg{
+			Action: "insert",
+			Table:  "asp_app_perm_201701",
+			New: &model.OldToken{
+				Mid:          4186264,
+				AppID:        1,
+				AccessToken:  "test token",
+				RefreshToken: "test refresh",
+				AppSubID:     0,
+				CreateAt:     123456789000,
+			},
+		}
+		err := s.handleToken(token)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_HandleCookie(t *testing.T) {
+	once.Do(startService)
+	Convey("Test del cookie by cookie", t, func() {
+		cookie := &cookieBMsg{
+			Action: "insert",
+			Table:  "asp_app_perm_201701",
+			New: &model.OldCookie{
+				Mid:       4186264,
+				Session:   "test session",
+				CSRFToken: "test csrf",
+				Expires:   1234567890000,
+			},
+		}
+		err := s.handleCookie(cookie)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/passport-auth/service/sync_all.go

@@ -0,0 +1,308 @@
+package service
+
+//var (
+//may = time.Now().AddDate(0, -1, 0)
+//apr = time.Now().AddDate(0, -2, 0)
+//)
+
+/*
+func (s *Service) syncToken(tabIdx string, min, max int64) {
+	var minID int64
+	var t time.Time
+	if tabIdx == "201805" {
+		t = may
+		minID = 167003927
+	} else if tabIdx == "201804" {
+		t = apr
+		minID = min
+	}
+	for {
+		tokens, err := s.dao.TokenMultiQuery(context.Background(), tabIdx, minID, s.c.SyncLines)
+		if err != nil {
+			log.Error("s.dao.TokenMultiQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(tokens) == 0 {
+			break
+		}
+		minID = setToken(tokens, s, minID, t)
+		if max > 0 && max <= minID {
+			break
+		}
+	}
+}
+
+func setToken(tokens []*model.OldToken, s *Service, minID int64, t time.Time) int64 {
+	newTokens := make([]*model.Token, 0, 51)
+	newRefreshs := make([]*model.Refresh, 0, 51)
+	for _, token := range tokens {
+		newToken := &model.Token{
+			Mid:     token.Mid,
+			AppID:   token.AppID,
+			Token:   token.AccessToken,
+			Expires: token.Expires,
+			Type:    token.Type,
+		}
+		newTokens = append(newTokens, newToken)
+		if len(newTokens) == 50 {
+			s.dao.BatchAddToken(context.Background(), newTokens, t)
+			newTokens = make([]*model.Token, 0, 51)
+		}
+		if minID < token.ID {
+			minID = token.ID
+		}
+		if token.RefreshToken == "" || token.Expires < time.Now().Unix() {
+			continue
+		}
+		newRefresh := &model.Refresh{
+			Mid:     token.Mid,
+			AppID:   token.AppID,
+			Refresh: token.RefreshToken,
+			Token:   token.AccessToken,
+			Expires: token.Expires,
+		}
+		newRefreshs = append(newRefreshs, newRefresh)
+		if len(newRefreshs) == 50 {
+			s.dao.BatchAddRefresh(context.Background(), newRefreshs, t)
+			newRefreshs = make([]*model.Refresh, 0, 51)
+		}
+	}
+	if len(newTokens) != 0 {
+		for _, token := range newTokens {
+			tokenByte, err := hex.DecodeString(token.Token)
+			if err != nil {
+				log.Error("setNewToken error: tokenByte decode hex err: %v", token)
+				continue
+			}
+			s.dao.AddToken(context.Background(), token, tokenByte, t)
+		}
+	}
+	if len(newRefreshs) != 0 {
+		for _, refresh := range newRefreshs {
+			refreshByte, err := hex.DecodeString(refresh.Refresh)
+			tokenByteR, err1 := hex.DecodeString(refresh.Refresh)
+			if err != nil || err1 != nil {
+				log.Error("setNewToken error: refreshByte decode hex err: %v", refresh)
+				continue
+			}
+			s.dao.AddRefresh(context.Background(), refresh, refreshByte, tokenByteR, t)
+		}
+	}
+	return minID
+}
+
+func (s *Service) setNewToken(token *model.OldToken) (err error) {
+	newToken := &model.Token{
+		Mid:     token.Mid,
+		AppID:   token.AppID,
+		Token:   token.AccessToken,
+		Expires: token.Expires,
+		Type:    token.Type,
+	}
+	var t time.Time
+	t, err = time.ParseInLocation("2006-01-02T15:04:05Z07:00", token.CTime, time.Local)
+	if err != nil {
+		log.Error("setNewToken error: ctime parse err. %v", token)
+		return
+	}
+	tokenByte, err := hex.DecodeString(token.AccessToken)
+	if err != nil {
+		log.Error("setNewToken error: tokenByte decode hex err: %v", token)
+		return
+	}
+	s.dao.AddToken(context.Background(), newToken, tokenByte, t)
+	if token.RefreshToken == "" || t.Unix() < 1525104000 {
+		return
+	}
+	newRefresh := &model.Refresh{
+		Mid:     token.Mid,
+		AppID:   token.AppID,
+		Refresh: token.RefreshToken,
+		Token:   token.AccessToken,
+		Expires: token.Expires,
+	}
+	refreshByte, err := hex.DecodeString(token.RefreshToken)
+	if err != nil {
+		log.Error("setNewToken error: refreshByte decode hex err: %v", token)
+		return
+	}
+	tokenByteR, err := hex.DecodeString(token.AccessToken)
+	if err != nil {
+		log.Error("setNewToken error: tokenByte decode hex err: %v", token)
+		return
+	}
+	s.dao.AddRefresh(context.Background(), newRefresh, refreshByte, tokenByteR, t)
+	return
+}
+
+func (s *Service) syncCookie(tabIdx int64) {
+	var minID int64
+	for {
+		now := time.Now().Unix()
+		cookies, err := s.dao.CookieMultiQuery(context.Background(), tabIdx, minID, now, s.c.SyncLines)
+		if err != nil {
+			log.Error("s.dao.CookieMultiQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(cookies) == 0 {
+			log.Info("syncCookie finished tabIdx : %d", tabIdx)
+			break
+		}
+		minID = s.setCookie(cookies, now, minID)
+	}
+}
+
+func (s *Service) setCookie(cookies []*model.OldCookie, now int64, minID int64) int64 {
+	today := time.Now()
+	newCookies := make([]*model.Cookie, 0, 51)
+	var count int64
+	for _, cookie := range cookies {
+		if count%1000 == 0 {
+			// for down cpu
+			time.Sleep(10 * time.Millisecond)
+		}
+		if minID < cookie.ID {
+			minID = cookie.ID
+		}
+		if len(newCookies) == 50 {
+			s.dao.AddMultiCookie(context.Background(), newCookies, today)
+			newCookies = make([]*model.Cookie, 0, 51)
+		}
+		var t time.Time
+		t, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", cookie.ModifyTime, time.Local)
+		if err != nil {
+			log.Error("setNewToken error: ctime parse err. %v", cookie)
+			continue
+		}
+		if cookie.Expires > now && t.Unix() > 1527782400 {
+			newCookies = append(newCookies, &model.Cookie{
+				Mid:     cookie.Mid,
+				Session: cookie.Session,
+				CSRF:    cookie.CSRFToken,
+				Type:    cookie.Type,
+				Expires: cookie.Expires,
+			})
+		}
+		count = count + 1
+	}
+	if len(newCookies) != 0 {
+		for _, newCookie := range newCookies {
+			csrfByte, err := hex.DecodeString(newCookie.CSRF)
+			if err != nil {
+				log.Error("hex.DecodeString csrf err: %v, %v", newCookie, err)
+				continue
+			}
+			s.dao.AddCookie(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, today)
+		}
+	}
+	return minID
+}
+*/
+// func (s *Service) setNewCookie(cookie *model.OldCookie) (err error) {
+// 	newCookie := &model.Cookie{
+// 		Mid:     cookie.Mid,
+// 		Session: cookie.Session,
+// 		CSRF:    cookie.CSRFToken,
+// 		Type:    cookie.Type,
+// 		Expires: cookie.Expires,
+// 	}
+// 	csrfByte, err := hex.DecodeString(newCookie.CSRF)
+// 	if err != nil {
+// 		log.Error("hex.DecodeString csrf err: %v, %v", cookie, err)
+// 		return
+// 	}
+// 	if _, err = s.dao.AddCookie(context.Background(), newCookie, []byte(newCookie.Session), csrfByte, time.Now()); err != nil {
+// 		log.Error("s.dao.AddCookie %v, %v", cookie, err)
+// 		return
+// 	}
+// 	return nil
+// }
+/*
+func (s *Service) compareToken(tabIdx string, min int64, max int64) {
+	var t time.Time
+	if tabIdx == "201805" {
+		t = may
+	} else if tabIdx == "201804" {
+		t = apr
+	} else {
+		t = time.Now()
+	}
+	minID := min
+	for {
+		tokens, err := s.dao.BatchGetToken(context.Background(), minID, s.c.SyncLines, t)
+		if err != nil {
+			log.Error("compareToken error: s.dao.BatchGetToken err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(tokens) == 0 {
+			break
+		}
+		tokenMap, err := s.dao.TokenBatchQuery(context.Background(), tabIdx, tokens)
+		if err != nil {
+			log.Error("compareToken error: s.dao.TokenBatchQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		for _, newToken := range tokens {
+			if tokenMap[newToken.Token] == "" {
+				tokenByte, err := hex.DecodeString(newToken.Token)
+				if err != nil {
+					log.Error("compareToken error: tokenByte decode hex err: %v", err)
+					continue
+				}
+				s.dao.DelToken(context.Background(), tokenByte, t)
+			}
+		}
+		minID = tokens[len(tokens)-1].ID
+		if max > 0 && max <= minID {
+			break
+		}
+	}
+	log.Info("token compare finished %s, %d - %d", tabIdx, minID, max)
+}
+
+func (s *Service) compareCookie(tabIdx string, min int64) {
+	var t time.Time
+	if tabIdx == "201805" {
+		t = may
+	} else if tabIdx == "201804" {
+		t = apr
+	} else {
+		t = time.Now()
+	}
+	minID := min
+	for {
+		cookies, err := s.dao.BatchGetCookie(context.Background(), minID, s.c.SyncLines, t)
+		if err != nil {
+			log.Error("compareCookie error: s.dao.BatchGetCookie err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if len(cookies) == 0 {
+			break
+		}
+		tabIdxMap := make(map[int64][]string)
+		for _, newCookie := range cookies {
+			oldTabIdx := newCookie.Mid % 30
+			tabIdxMap[oldTabIdx] = append(tabIdxMap[oldTabIdx], newCookie.Session)
+		}
+
+		sessionMap, err := s.dao.CookieBatchQuery(context.Background(), tabIdxMap)
+		if err != nil {
+			log.Error("compareCookie error: s.dao.CookieBatchQuery err, sleep 1s %v", err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		for _, newCookie := range cookies {
+			if sessionMap[newCookie.Session] == "" {
+				s.dao.DelCookie(context.Background(), []byte(newCookie.Session), t)
+			}
+		}
+		minID = cookies[len(cookies)-1].ID
+	}
+}*/

app/job/main/passport-auth/service/sync_all_test.go

@@ -0,0 +1,21 @@
+package service
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_compareToken(t *testing.T) {
+	once.Do(startService)
+	Convey("Test compareToken", t, func() {
+		//s.compareToken("201806", 0, 10)
+	})
+}
+
+func Test_compareCookie(t *testing.T) {
+	once.Do(startService)
+	Convey("Test compareCookie", t, func() {
+		//s.compareCookie("201806", 0)
+	})
+}