Create & Init Project...

app/job/main/stat/service/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/stat/conf:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/stat/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/stat/conf:go_default_library",
+        "//app/job/main/stat/dao:go_default_library",
+        "//app/job/main/stat/model:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/stat/service/service.go

@@ -0,0 +1,218 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/stat/conf"
+	"go-common/app/job/main/stat/dao"
+	"go-common/app/job/main/stat/model"
+	arcmdl "go-common/app/service/main/archive/api"
+	archive "go-common/app/service/main/archive/api/gorpc"
+	"go-common/library/cache/memcache"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_sharding = 100
+)
+
+type lastTmStat struct {
+	last int64
+	stat *arcmdl.Stat
+}
+
+// Service is stat job service.
+type Service struct {
+	c *conf.Config
+	// dao
+	dao *dao.Dao
+	// wait
+	waiter sync.WaitGroup
+	closed bool
+	// databus
+	subMap     map[string]*databus.Databus
+	subMonitor map[string]*model.Monitor
+	subStatCh  []chan *model.StatMsg
+	mu         sync.Mutex
+	// stat map
+	statSM []map[int64]*lastTmStat
+	// rpc
+	arcRPC  *archive.Service2
+	arcRPC2 *archive.Service2
+	// max aid
+	maxAid    int64
+	memcaches []*memcache.Pool
+}
+
+// New is stat-job service implementation.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c: c,
+		// dao
+		dao: dao.New(c),
+		// rpc
+		arcRPC:     archive.New2(c.ArchiveRPC),
+		arcRPC2:    archive.New2(c.ArchiveRPC2),
+		subMap:     make(map[string]*databus.Databus),
+		subMonitor: make(map[string]*model.Monitor),
+	}
+	for _, mc := range s.c.Memcaches {
+		s.memcaches = append(s.memcaches, memcache.NewPool(mc))
+	}
+	// view
+	s.subMap[model.TypeForView] = databus.New(c.ViewSub)
+	s.subMonitor[model.TypeForView] = &model.Monitor{Topic: c.ViewSub.Topic, Count: 0}
+	// dm
+	s.subMap[model.TypeForDm] = databus.New(c.DmSub)
+	s.subMonitor[model.TypeForDm] = &model.Monitor{Topic: c.DmSub.Topic, Count: 0}
+	// reply
+	s.subMap[model.TypeForReply] = databus.New(c.ReplySub)
+	s.subMonitor[model.TypeForReply] = &model.Monitor{Topic: c.ReplySub.Topic, Count: 0}
+	// fav
+	s.subMap[model.TypeForFav] = databus.New(c.FavSub)
+	s.subMonitor[model.TypeForFav] = &model.Monitor{Topic: c.FavSub.Topic, Count: 0}
+	// coin
+	s.subMap[model.TypeForCoin] = databus.New(c.CoinSub)
+	s.subMonitor[model.TypeForCoin] = &model.Monitor{Topic: c.CoinSub.Topic, Count: 0}
+	// share
+	s.subMap[model.TypeForShare] = databus.New(c.ShareSub)
+	s.subMonitor[model.TypeForShare] = &model.Monitor{Topic: c.ShareSub.Topic, Count: 0}
+	// rank
+	s.subMap[model.TypeForRank] = databus.New(c.RankSub)
+	// like
+	s.subMap[model.TypeForLike] = databus.New(c.LikeSub)
+	s.subMonitor[model.TypeForLike] = &model.Monitor{Topic: c.LikeSub.Topic, Count: 0}
+	for i := int64(0); i < _sharding; i++ {
+		s.subStatCh = append(s.subStatCh, make(chan *model.StatMsg, 10240))
+		s.statSM = append(s.statSM, map[int64]*lastTmStat{})
+		s.waiter.Add(1)
+		go s.statDealproc(i)
+	}
+	go s.loadproc()
+	if env.DeployEnv == env.DeployEnvProd {
+		go s.monitorproc()
+	}
+	for k, d := range s.subMap {
+		s.waiter.Add(1)
+		go s.consumerproc(k, d)
+	}
+	return
+}
+
+func (s *Service) loadproc() {
+	for {
+		time.Sleep(1 * time.Minute)
+		id, err := s.dao.MaxAID(context.TODO())
+		if err != nil {
+			s.maxAid = 0
+			log.Error("s.dao.MaxAid error(%+v)", err)
+			continue
+		}
+		s.maxAid = id
+	}
+}
+
+func (s *Service) monitorproc() {
+	for {
+		time.Sleep(90 * time.Second)
+		s.mu.Lock()
+		for _, mo := range s.subMonitor {
+			if mo.Count == 0 {
+				s.dao.SendQiyeWX(fmt.Sprintf("日志报警:stat-job topic(%s) 没消费！！！！", mo.Topic))
+			}
+			mo.Count = 0
+		}
+		s.mu.Unlock()
+	}
+}
+
+// consumerproc consumer all topic
+func (s *Service) consumerproc(k string, d *databus.Databus) {
+	defer s.waiter.Done()
+	var msgs = d.Messages()
+	for {
+		var (
+			err error
+			ok  bool
+			msg *databus.Message
+			now = time.Now().Unix()
+		)
+		msg, ok = <-msgs
+		if !ok || s.closed {
+			log.Info("databus(%s) consumer exit", k)
+			return
+		}
+		msg.Commit()
+		var ms = &model.StatCount{}
+		if err = json.Unmarshal(msg.Value, ms); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+		if ms.Aid <= 0 || (ms.Type != "archive" && ms.Type != "archive_his") {
+			log.Warn("message(%s) error", msg.Value)
+			continue
+		}
+		if now-ms.TimeStamp > 8*60*60 {
+			log.Warn("topic(%s) message(%s) too early", msg.Topic, msg.Value)
+			continue
+		}
+		stat := &model.StatMsg{Aid: ms.Aid, Type: k, Ts: ms.TimeStamp}
+		switch k {
+		case model.TypeForView:
+			stat.Click = ms.Count
+		case model.TypeForDm:
+			stat.DM = ms.Count
+		case model.TypeForReply:
+			stat.Reply = ms.Count
+		case model.TypeForFav:
+			stat.Fav = ms.Count
+		case model.TypeForCoin:
+			stat.Coin = ms.Count
+		case model.TypeForShare:
+			stat.Share = ms.Count
+		case model.TypeForRank:
+			stat.HisRank = ms.Count
+		case model.TypeForLike:
+			stat.Like = ms.Count
+			stat.DisLike = ms.DisLike
+		default:
+			log.Error("unknow type(%s) message(%s)", k, msg.Value)
+			continue
+		}
+		s.mu.Lock()
+		if _, ok := s.subMonitor[k]; ok {
+			s.subMonitor[k].Count++
+		}
+		s.mu.Unlock()
+		s.subStatCh[stat.Aid%_sharding] <- stat
+		log.Info("got message(%+v)", stat)
+	}
+}
+
+// Close Databus consumer close.
+func (s *Service) Close() (err error) {
+	s.closed = true
+	time.Sleep(2 * time.Second)
+	log.Info("start close job")
+	for k, d := range s.subMap {
+		d.Close()
+		log.Info("databus(%s) cloesed", k)
+	}
+	for i := int64(0); i < _sharding; i++ {
+		close(s.subStatCh[i])
+	}
+	log.Info("end close job")
+	s.waiter.Wait()
+	return
+}
+
+// Ping check server ok
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}

app/job/main/stat/service/service_test.go

@@ -0,0 +1,41 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/stat/conf"
+	"go-common/app/service/main/archive/api"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/stat-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+func Test_Ping(t *testing.T) {
+	Convey("Ping", t, func() {
+		s.Ping(context.TODO())
+	})
+}
+
+func Test_UpdateCache(t *testing.T) {
+	Convey("updateCache", t, func() {
+		err := s.updateCache(&api.Stat{
+			Aid:  1,
+			Coin: 22,
+			View: 200,
+		})
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/stat/service/stat.go

@@ -0,0 +1,169 @@
+package service
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/stat/model"
+	"go-common/app/service/main/archive/api"
+	"go-common/app/service/main/archive/model/archive"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_prefixStatPB  = "stp_"
+	_prefixClickPB = "clkp_"
+)
+
+func statPBKey(aid int64) string {
+	return _prefixStatPB + strconv.FormatInt(aid, 10)
+}
+
+func clickPBKey(aid int64) string {
+	return _prefixClickPB + strconv.FormatInt(aid, 10)
+}
+
+func (s *Service) statDealproc(i int64) {
+	defer s.waiter.Done()
+	var (
+		ch  = s.subStatCh[i]
+		sm  = s.statSM[i]
+		c   = context.TODO()
+		ls  *lastTmStat
+		err error
+	)
+	for {
+		now := time.Now().Unix()
+		ms, ok := <-ch
+		if !ok {
+			s.multiUpdateDB(i, sm)
+			log.Warn("statDealproc(%d) quit", i)
+			return
+		}
+		if s.maxAid > 0 && s.maxAid+300 < ms.Aid {
+			log.Warn("aid(%d) too big maxAid(%d)", ms.Aid, s.maxAid)
+			continue
+		}
+		// get stat
+		if ls, ok = sm[ms.Aid]; !ok {
+			var stat *api.Stat
+			if stat, err = s.dao.Stat(c, ms.Aid); err != nil {
+				log.Error("s.dao.Stat(%d) error(%v)", ms.Aid, err)
+				continue
+			}
+			ls = &lastTmStat{}
+			if stat == nil {
+				ls.stat = &api.Stat{Aid: ms.Aid}
+				ls.last = 0 // NOTE: make sure update db in first.
+			} else {
+				ls.stat = stat
+				ls.last = time.Now().Unix()
+			}
+			sm[ms.Aid] = ls
+		}
+		model.Merge(ms, ls.stat)
+		if now-ms.Ts < 60 {
+			// update cache
+			s.updateCache(ls.stat)
+		}
+		// update db when after 60s
+		if time.Now().Unix()-ls.last > 120 {
+			s.updateDB(ls.stat)
+			delete(sm, ms.Aid) // NOTE: delete make sure the normal scope of memory and can be save all in 120s when close chan.
+		}
+	}
+}
+
+// updateDB update stat in db.
+func (s *Service) updateDB(stat *api.Stat) (err error) {
+	if _, err := s.dao.Update(context.TODO(), stat); err != nil {
+		log.Error("s.dao.Update(%v) error(%v)", stat, err)
+	}
+	log.Info("update db aid(%d) stat(%+v) success", stat.Aid, stat)
+	return
+}
+
+// multiUpdateDB update some stat in db.
+func (s *Service) multiUpdateDB(yu int64, sm map[int64]*lastTmStat) (err error) {
+	log.Info("start close(%d) multi update stat start", yu)
+	var (
+		c     = context.TODO()
+		alloc = [1000]*api.Stat{}
+		stats = alloc[:0]
+		i     int
+	)
+	for aid, ls := range sm {
+		stats = append(stats, ls.stat)
+		if i > 0 && i%1000 == 0 {
+			s.dao.MultiUpdate(c, yu, stats...)
+		} else if i+1 == len(sm) {
+			s.dao.MultiUpdate(c, yu, stats...)
+		} else {
+			log.Info("start close(%d) aid(%d) append", i, aid)
+			continue
+		}
+		log.Info("start close(%d) multi update stat %d", i, aid)
+		stats = alloc[:0]
+	}
+	log.Info("start close(%d) multi update stat endm", yu)
+	return
+}
+
+// updateCache purge stat info in cache
+func (s *Service) updateCache(st *api.Stat) (err error) {
+	var (
+		stat3 = &api.Stat{
+			Aid:     st.Aid,
+			View:    int32(st.View),
+			Danmaku: int32(st.Danmaku),
+			Reply:   int32(st.Reply),
+			Fav:     int32(st.Fav),
+			Coin:    int32(st.Coin),
+			Share:   int32(st.Share),
+			NowRank: int32(st.NowRank),
+			HisRank: int32(st.HisRank),
+			Like:    int32(st.Like),
+			DisLike: 0,
+		}
+		click   *archive.Click3
+		upclick = true
+	)
+	if click, err = s.dao.Click(context.TODO(), st.Aid); err != nil {
+		upclick = false
+	}
+	if click == nil {
+		click = &archive.Click3{}
+	}
+	for _, mc := range s.memcaches {
+		var c = context.TODO()
+		conn := mc.Get(c)
+		if err = conn.Set(&memcache.Item{Key: statPBKey(stat3.Aid), Object: stat3, Flags: memcache.FlagProtobuf, Expiration: 0}); err != nil {
+			log.Error("conn1.Set(%s, %+v) error(%v)", statPBKey(stat3.Aid), stat3, err)
+		}
+		if upclick {
+			if err = conn.Set(&memcache.Item{Key: clickPBKey(stat3.Aid), Object: click, Flags: memcache.FlagProtobuf, Expiration: 0}); err != nil {
+				log.Error("conn1.Set(%s, %+v) error(%v)", clickPBKey(stat3.Aid), click, err)
+			}
+		}
+		if err == nil {
+			log.Info("update cache aid(%d) stat(%+v) success", st.Aid, stat3)
+			log.Info("update cache aid(%d) click(%+v) success", st.Aid, click)
+		}
+		conn.Close()
+	}
+	return
+}
+
+// Purge purge arc's stat cache
+func (s *Service) Purge(c context.Context, aids []int64) (err error) {
+	for _, aid := range aids {
+		var stat *api.Stat
+		if stat, err = s.dao.Stat(c, aid); err != nil {
+			return
+		}
+		s.updateCache(stat)
+	}
+	return
+}