Create & Init Project...

app/job/main/identify/service/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "identify_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/identify/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "auth.go",
+        "check_task.go",
+        "identify.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/identify/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/identify/conf:go_default_library",
+        "//app/job/main/identify/dao:go_default_library",
+        "//app/job/main/identify/model:go_default_library",
+        "//app/service/main/identify/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/databusutil:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/identify/service/auth.go

@@ -0,0 +1,116 @@
+package service
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"strings"
+
+	"go-common/app/job/main/identify/model"
+	mdl "go-common/app/service/main/identify/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+func (s *Service) new(msg *databus.Message) (interface{}, error) {
+	bmsg := new(model.BMsg)
+	if err := json.Unmarshal(msg.Value, bmsg); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+		return nil, err
+	}
+	if bmsg.Action != "delete" {
+		return bmsg, nil
+	}
+	if strings.HasPrefix(bmsg.Table, "user_token_") {
+		t := new(model.AuthToken)
+		if err := json.Unmarshal(bmsg.New, t); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+			return nil, err
+		}
+		log.Info("identifyconsumeproc table:%s key:%s partition:%d offset:%d", bmsg.Table, msg.Key, msg.Partition, msg.Offset)
+		return t, nil
+	} else if strings.HasPrefix(bmsg.Table, "user_cookie_") {
+		t := new(model.AuthCookie)
+		if err := json.Unmarshal(bmsg.New, t); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+			return nil, err
+		}
+		log.Info("identifyconsumeproc table:%s key:%s partition:%d offset:%d", bmsg.Table, msg.Key, msg.Partition, msg.Offset)
+		return t, nil
+	}
+	return bmsg, nil
+}
+
+func (s *Service) spilt(msg *databus.Message, data interface{}) int {
+	switch t := data.(type) {
+	case *model.AuthToken:
+		return int(t.Mid)
+	case *model.AuthCookie:
+		return int(t.Mid)
+	default:
+		return 0
+	}
+}
+
+func (s *Service) processAuthBinlog2(bmsgs []interface{}) {
+	for _, msg := range bmsgs {
+		switch t := msg.(type) {
+		case *model.AuthToken:
+			var (
+				bytes []byte
+				err   error
+			)
+			if bytes, err = base64.StdEncoding.DecodeString(t.Token); err != nil {
+				log.Error("cleanCookieCache base64 decode err %v", err)
+				err = nil
+				return
+			}
+			info := &mdl.IdentifyInfo{
+				Mid:     t.Mid,
+				Expires: int32(t.Expires),
+			}
+
+			if ok := isGameAppID(t.AppID); ok {
+				continue
+			}
+			for {
+				log.Info("auth service process token databus, key(%s)", hex.EncodeToString(bytes))
+				if err := s.processIdentify("delete", hex.EncodeToString(bytes), info); err != nil {
+					continue
+				}
+				break
+			}
+		case *model.AuthCookie:
+			var (
+				bytes     []byte
+				bytesCSRF []byte
+				err       error
+			)
+			if bytes, err = base64.StdEncoding.DecodeString(t.Session); err != nil {
+				log.Error("cleanCookieCache base64 decode err %v", err)
+				err = nil
+				return
+			}
+			if bytesCSRF, err = base64.StdEncoding.DecodeString(t.CSRF); err != nil {
+				log.Error("cleanCookieCache base64 decode err %v", err)
+				err = nil
+				return
+			}
+			info := &mdl.IdentifyInfo{
+				Mid:     t.Mid,
+				Csrf:    hex.EncodeToString(bytesCSRF),
+				Expires: int32(t.Expires),
+			}
+
+			for {
+				log.Info("auth service process cookie databus, key(%s)", string(bytes))
+				if err := s.processIdentify("delete", string(bytes), info); err != nil {
+					continue
+				}
+				break
+			}
+		default:
+			return
+		}
+	}
+}

app/job/main/identify/service/check_task.go

@@ -0,0 +1,223 @@
+package service
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/identify/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+	"go-common/library/stat/prom"
+)
+
+func (s *Service) queryCookieDeleted() {
+	var wg sync.WaitGroup
+	ticker := time.NewTicker(time.Duration(s.c.CheckConf.Ticker))
+	for {
+		now := time.Now()
+		for i := 0; i < 2; i++ {
+			wg.Add(1)
+			go func(i int, now time.Time) {
+				defer wg.Done()
+				s.readCookieData(dateFormat(addMonth(now, -i)))
+			}(i, now)
+		}
+		wg.Wait()
+		<-ticker.C
+	}
+}
+
+func (s *Service) readCookieData(moth string) {
+	var start int64
+	for {
+		cookies, err := s.d.CookieDeleted(context.Background(), start, s.c.CheckConf.Count, moth)
+		if err != nil {
+			log.Error("fail to get CookieDeleted error(%+v)", err)
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		if len(cookies) == 0 {
+			log.Info("check cookie(%s) finished!", moth)
+			return
+		}
+		maxID := int64(0)
+		for _, a := range cookies {
+			s.cookieCh[a.ID%int64(s.c.CheckConf.ChanNum)] <- a
+			if maxID < a.ID {
+				maxID = a.ID
+			}
+		}
+		start = maxID
+	}
+}
+
+func (s *Service) queryTokenDeleted() {
+	var wg sync.WaitGroup
+	ticker := time.NewTicker(time.Duration(s.c.CheckConf.Ticker))
+	for {
+		now := time.Now()
+		for i := 0; i < 3; i++ {
+			wg.Add(1)
+			go func(i int, now time.Time) {
+				defer wg.Done()
+				s.readTokenData(dateFormat(addMonth(now, -i)))
+			}(i, now)
+		}
+		wg.Wait()
+		<-ticker.C
+	}
+}
+
+func (s *Service) readTokenData(moth string) {
+	var start int64
+	for {
+		tokens, err := s.d.TokenDeleted(context.Background(), start, s.c.CheckConf.Count, moth)
+		if err != nil {
+			log.Error("fail to get TokenDeleted error(%+v)", err)
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		if len(tokens) == 0 {
+			log.Info("check token(%s) finished!", moth)
+			return
+		}
+		maxID := int64(0)
+		for _, a := range tokens {
+			s.tokenCh[a.ID%int64(s.c.CheckConf.ChanNum)] <- a
+			if maxID < a.ID {
+				maxID = a.ID
+			}
+		}
+		start = maxID
+	}
+}
+
+func (s *Service) checkCookie(c chan *model.AuthCookie) {
+	count := 0
+	for {
+		cookie, ok := <-c
+		if !ok {
+			log.Error("cookieChan closed")
+			return
+		}
+		count = count + 1
+		if count%10000 == 0 {
+			count = 0
+			time.Sleep(100 * time.Millisecond)
+		}
+		// auth
+		for {
+			res, err := s.d.CookieCache(context.Background(), cookie.Session)
+			if err != nil {
+				log.Error("fail to get cookie(%+v) cache from auth , error(%+v)", cookie, err)
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+			if res != nil {
+				prom.BusinessErrCount.Incr("auth:cacheNotDeleted")
+				log.Error("auth cache not deleted, session(%s) mid(%d)", cookie.Session, cookie.Mid)
+				for {
+					err = s.d.DelCookieCache(context.Background(), cookie.Session)
+					if err == nil {
+						break
+					}
+					time.Sleep(100 * time.Millisecond)
+				}
+			}
+			break
+		}
+		// identify
+		s.checkIdentifyCache(cookie.Session, cookie.Mid)
+	}
+}
+
+func (s *Service) checkToken(c chan *model.AuthToken) {
+	count := 0
+	for {
+		token, ok := <-c
+		if !ok {
+			log.Error("tokenChan closed")
+			return
+		}
+		count = count + 1
+		if count%10000 == 0 {
+			count = 0
+			time.Sleep(100 * time.Millisecond)
+		}
+		// auth
+		for {
+			res, err := s.d.TokenCache(context.Background(), token.Token)
+			if err != nil {
+				log.Error("fail to get token(%+v) cache from auth , error(%+v)", token, err)
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+			if res != nil {
+				prom.BusinessErrCount.Incr("auth:cacheNotDeleted")
+				log.Error("auth cache not deleted, token(%s) mid(%d)", token.Token, token.Mid)
+				for {
+					err = s.d.DelTokenCache(context.Background(), token.Token)
+					if err == nil {
+						break
+					}
+					time.Sleep(100 * time.Millisecond)
+				}
+			}
+			break
+		}
+		// identify
+		s.checkIdentifyCache(token.Token, token.Mid)
+	}
+}
+
+func (s *Service) checkIdentifyCache(k string, mid int64) {
+	for name, p := range s.poolm {
+		mcc, ok := s.c.Memcaches[name]
+		if !ok || mcc == nil {
+			return
+		}
+		key := mcc.Prefix + k
+		for {
+			conn := p.Get(context.Background())
+			res, err := conn.Get(key)
+			conn.Close()
+			if err != nil {
+				if err == memcache.ErrNotFound {
+					break
+				}
+				log.Error("fail to get cache(%s) from identify , error(%+v)", key, err)
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+			if res != nil {
+				prom.BusinessErrCount.Incr("identify:cacheNotDeleted")
+				log.Error("identify cache not deleted, key(%s) mid(%d) cache(%s)", key, mid, name)
+				for {
+					conn := p.Get(context.Background())
+					err := conn.Delete(key)
+					conn.Close()
+					if err == nil || err == memcache.ErrNotFound {
+						break
+					}
+					log.Error("dao.DelCache(%s) error(%+v)", key, err)
+					time.Sleep(100 * time.Millisecond)
+				}
+			}
+			break
+		}
+	}
+}
+
+func dateFormat(t time.Time) string {
+	return t.Format("200601")
+}
+
+func addMonth(t time.Time, delta int) time.Time {
+	if delta == 0 {
+		return t
+	}
+	year, month, _ := t.Date()
+	thisMonthFirstDay := time.Date(year, month, 1, 1, 1, 1, 1, t.Location())
+	return thisMonthFirstDay.AddDate(0, delta, 0)
+}

app/job/main/identify/service/identify.go

@@ -0,0 +1,157 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/identify/model"
+	mdl "go-common/app/service/main/identify/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+type actionToken struct {
+	model.Token
+	Action string
+}
+
+type actionCookie struct {
+	model.Cookie
+	Action string
+}
+
+func (s *Service) identifyNew(msg *databus.Message) (interface{}, error) {
+	bmsg := new(model.BMsg)
+	if err := json.Unmarshal(msg.Value, bmsg); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+		return nil, err
+	}
+	log.Info("identify service process databus message, table(%s)", bmsg.Table)
+	if strings.HasPrefix(bmsg.Table, _tokenTable) {
+		t := new(actionToken)
+		if err := json.Unmarshal(bmsg.New, t); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+			return nil, err
+		}
+		log.Info("process databus message, table(%s), action(%s)", bmsg.Table, bmsg.Action)
+		t.Action = bmsg.Action
+		return t, nil
+	}
+	if strings.HasPrefix(bmsg.Table, _cookieTable) {
+		t := new(actionCookie)
+		if err := json.Unmarshal(bmsg.New, t); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
+			return nil, err
+		}
+		log.Info("process databus message, table(%s), action(%s)", bmsg.Table, bmsg.Action)
+		t.Action = bmsg.Action
+		return t, nil
+	}
+	return bmsg, nil
+}
+
+func (s *Service) identifySplit(msg *databus.Message, data interface{}) int {
+	mergeNum := int64(s.c.Databusutil.Num)
+	mid := int64(0)
+	switch t := data.(type) {
+	case *model.Cookie:
+		mid = t.Mid
+	case *model.Token:
+		mid = t.Mid
+	}
+	return int(mid % mergeNum)
+}
+
+func (s *Service) processIdentifyInfo(bmsgs []interface{}) {
+	for _, msg := range bmsgs {
+		switch t := msg.(type) {
+		case *actionToken:
+			info := &mdl.IdentifyInfo{
+				Mid:     t.Mid,
+				Expires: t.Expires,
+			}
+			if ok := isGameAppID(t.APPID); ok {
+				continue
+			}
+			s.processIdentify(t.Action, t.AccessToken, info)
+		case *actionCookie:
+			info := &mdl.IdentifyInfo{
+				Mid:     t.Mid,
+				Csrf:    t.CSRFToken,
+				Expires: t.ExpireTime,
+			}
+			s.processIdentify(t.Action, t.SessionData, info)
+		}
+	}
+}
+
+func (s *Service) processIdentify(action, key string, info *mdl.IdentifyInfo) (err error) {
+	if err = s.processMc(action, key, info); err != nil {
+		return
+	}
+	log.Info("identify process action(%s) mid(%d) csrf(%s) key(%s) success", action, info.Mid, info.Csrf, key[:8])
+	return
+}
+
+// processMc .
+func (s *Service) processMc(action, k string, info *mdl.IdentifyInfo) (err error) {
+	if len(s.poolm) == 0 {
+		return
+	}
+	if _insertAction == action {
+		expire := info.Expires
+		if expire < int32(time.Now().Unix()) {
+			log.Error("identify expire error(%d,%d)", info.Expires, time.Now().Unix())
+			return
+		}
+		for name, p := range s.poolm {
+			mcc, ok := s.c.Memcaches[name]
+			if !ok || mcc == nil {
+				return
+			}
+			key := mcc.Prefix + k
+			conn := p.Get(context.Background())
+			err = conn.Set(&memcache.Item{Key: key, Object: info, Flags: memcache.FlagProtobuf, Expiration: expire})
+			conn.Close()
+			if err != nil {
+				log.Error("old identify set error(%s,%d,%v)", key, info.Expires, err)
+				err = nil
+			}
+		}
+		return
+	}
+	if _delteAction == action {
+		for name, p := range s.poolm {
+			mcc, ok := s.c.Memcaches[name]
+			if !ok || mcc == nil {
+				return
+			}
+			key := mcc.Prefix + k
+			// if delete failed, retry until success
+			for {
+				conn := p.Get(context.Background())
+				err := conn.Delete(key)
+				if err == nil || err == memcache.ErrNotFound {
+					conn.Close()
+					break
+				}
+				log.Error("dao.DelCache(%s) error(%v)", key, err)
+				conn.Close()
+				time.Sleep(time.Second)
+			}
+		}
+	}
+	return
+}
+
+func isGameAppID(appid int64) (res bool) {
+	for i := 0; i < len(_gameAppID); i++ {
+		if _gameAppID[i] == appid {
+			return true
+		}
+	}
+	return false
+}

app/job/main/identify/service/identify_test.go

@@ -0,0 +1,56 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestServie_Ping(t *testing.T) {
+	Convey("test identify consumer proc", t, func() {
+		once.Do(startService)
+
+		s.Ping(context.Background())
+	})
+}
+
+func TestService_identify(t *testing.T) {
+	Convey("test identify consumer proc", t, func() {
+		once.Do(startService)
+		var (
+			bmsg interface{}
+			err  error
+			res  int
+		)
+		msg := <-s.identifySub.Messages()
+		bmsg, err = s.identifyNew(msg)
+		So(err, ShouldBeNil)
+		So(bmsg, ShouldNotBeNil)
+		res = s.identifySplit(msg, bmsg)
+		So(res, ShouldNotBeNil)
+
+		bmsgs := []interface{}{bmsg}
+		s.processIdentifyInfo(bmsgs)
+	})
+}
+
+func TestService_auth(t *testing.T) {
+	Convey("test identify consumer proc", t, func() {
+		once.Do(startService)
+		var (
+			m   interface{}
+			err error
+			res int
+		)
+		msg := <-s.authDataBus.Messages()
+		m, err = s.new(msg)
+		So(err, ShouldBeNil)
+		So(m, ShouldNotBeNil)
+		res = s.spilt(msg, m)
+		So(res, ShouldNotBeNil)
+
+		ms := []interface{}{m}
+		s.processIdentifyInfo(ms)
+	})
+}

app/job/main/identify/service/service.go

@@ -0,0 +1,100 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/identify/conf"
+	"go-common/app/job/main/identify/dao"
+	"go-common/app/job/main/identify/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/queue/databus"
+	"go-common/library/queue/databus/databusutil"
+)
+
+const (
+	_tokenTable   = "aso_app_perm"
+	_cookieTable  = "aso_cookie_token"
+	_insertAction = "insert"
+	_delteAction  = "delete"
+)
+
+var (
+	_gameAppID = [3]int64{432, 876, 849}
+)
+
+// Service is a identify service.
+type Service struct {
+	c           *conf.Config
+	d           *dao.Dao
+	identifySub *databus.Databus
+	authDataBus *databus.Databus
+	// mc
+	poolm map[string]*memcache.Pool
+	// databus group
+	authGroup     *databusutil.Group
+	identifyGroup *databusutil.Group
+
+	cookieCh []chan *model.AuthCookie
+	tokenCh  []chan *model.AuthToken
+}
+
+// New new a identify service.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:           c,
+		d:           dao.New(c),
+		identifySub: databus.New(c.DataBus.IdentifySub),
+		authDataBus: databus.New(c.DataBus.AuthDataBus),
+		cookieCh:    make([]chan *model.AuthCookie, c.CheckConf.ChanNum),
+		tokenCh:     make([]chan *model.AuthToken, c.CheckConf.ChanNum),
+	}
+	if len(s.c.Memcaches) > 0 {
+		pm := make(map[string]*memcache.Pool, len(s.c.Memcaches))
+		for name, mcc := range s.c.Memcaches {
+			p := memcache.NewPool(mcc.Config)
+			pm[name] = p
+		}
+		s.poolm = pm
+	}
+
+	s.authGroup = databusutil.NewGroup(c.Databusutil, s.authDataBus.Messages())
+	s.authGroup.New = s.new
+	s.authGroup.Split = s.spilt
+	s.authGroup.Do = s.processAuthBinlog2
+	s.authGroup.Start()
+
+	s.identifyGroup = databusutil.NewGroup(c.Databusutil, s.identifySub.Messages())
+	s.identifyGroup.New = s.identifyNew
+	s.identifyGroup.Split = s.identifySplit
+	s.identifyGroup.Do = s.processIdentifyInfo
+	s.identifyGroup.Start()
+
+	if c.CheckConf.Switch {
+		for i := 0; i < c.CheckConf.ChanNum; i++ {
+			cookie := make(chan *model.AuthCookie, c.CheckConf.ChanSize)
+			token := make(chan *model.AuthToken, c.CheckConf.ChanSize)
+			s.cookieCh[i] = cookie
+			s.tokenCh[i] = token
+			go s.checkCookie(cookie)
+			go s.checkToken(token)
+		}
+		go s.queryCookieDeleted()
+		go s.queryTokenDeleted()
+	}
+
+	return
+}
+
+// Ping .
+func (s *Service) Ping(c context.Context) (err error) {
+	return nil
+}
+
+// Close close.
+func (s *Service) Close() (err error) {
+	s.identifySub.Close()
+	s.authDataBus.Close()
+	s.authGroup.Close()
+	s.identifyGroup.Close()
+	return nil
+}

app/job/main/identify/service/service_test.go

@@ -0,0 +1,18 @@
+package service
+
+import (
+	"go-common/app/job/main/identify/conf"
+	"sync"
+)
+
+var (
+	once sync.Once
+	s    *Service
+)
+
+func startService() {
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	s = New(conf.Conf)
+}