Create & Init Project...

app/job/main/playlist/service/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/playlist/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/playlist/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/playlist/model:go_default_library",
+        "//app/interface/main/playlist/rpc/client:go_default_library",
+        "//app/job/main/playlist/conf:go_default_library",
+        "//app/job/main/playlist/dao:go_default_library",
+        "//app/job/main/playlist/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/service/service.go

@@ -0,0 +1,189 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	plmdl "go-common/app/interface/main/playlist/model"
+	plarpc "go-common/app/interface/main/playlist/rpc/client"
+	"go-common/app/job/main/playlist/conf"
+	"go-common/app/job/main/playlist/dao"
+	"go-common/app/job/main/playlist/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/stat/prom"
+)
+
+const (
+	_sharding = 10 // goroutines for dealing the stat
+	_chanSize = 10240
+)
+
+// Service .
+type Service struct {
+	c                *conf.Config
+	dao              *dao.Dao
+	waiter           *sync.WaitGroup
+	closed           bool
+	playlistViewSub  *databus.Databus
+	playlistFavSub   *databus.Databus
+	playlistReplySub *databus.Databus
+	playlistShareSub *databus.Databus
+	playlistRPC      *plarpc.Service
+	updateDbInterval int64
+	statCh           [_sharding]chan *model.StatM
+}
+
+// New creates a Service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:                c,
+		dao:              dao.New(c),
+		waiter:           new(sync.WaitGroup),
+		playlistViewSub:  databus.New(c.PlaylistViewSub),
+		playlistFavSub:   databus.New(c.PlaylistFavSub),
+		playlistReplySub: databus.New(c.PlaylistReplySub),
+		playlistShareSub: databus.New(c.PlaylistShareSub),
+		playlistRPC:      plarpc.New(c.PlaylistRPC),
+		updateDbInterval: int64(time.Duration(c.Job.UpdateDbInterval) / time.Second),
+	}
+	for i := int64(0); i < _sharding; i++ {
+		// for stat
+		s.statCh[i] = make(chan *model.StatM, _chanSize)
+		s.waiter.Add(1)
+		go s.viewproc(i)
+	}
+	s.waiter.Add(1)
+	go s.consumeView()
+	s.waiter.Add(1)
+	go s.consumeFav()
+	s.waiter.Add(1)
+	go s.consumeReply()
+	s.waiter.Add(1)
+	go s.consumeShare()
+	return
+}
+
+// consumeView consumes playlist's view.
+func (s *Service) consumeView() {
+	defer s.waiter.Done()
+	for {
+		if s.closed {
+			for i := 0; i < _sharding; i++ {
+				close(s.statCh[i])
+			}
+			return
+		}
+		msg, ok := <-s.playlistViewSub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job view consumer exit!")
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+		msg.Commit()
+		viewSM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, viewSM); err != nil {
+			dao.PromError("service:解析计数databus消息", "json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if viewSM.Type != plmdl.PlDBusType || viewSM.ID <= 0 {
+			continue
+		}
+		key := viewSM.ID % _sharding
+		s.statCh[key] <- viewSM
+		prom.BusinessInfoCount.State(fmt.Sprintf("statChan-%v", key), int64(len(s.statCh[key])))
+		log.Info("consumeView key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, viewSM.String(model.ViewCountType))
+	}
+}
+
+// consumeFav  consumes playlist's favorite.
+func (s *Service) consumeFav() {
+	defer s.waiter.Done()
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.playlistFavSub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job favorite consumer exit!")
+			return
+		}
+		msg.Commit()
+		favSM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, favSM); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if favSM.Type != plmdl.PlDBusType || favSM.ID <= 0 {
+			continue
+		}
+		s.upStat(c, favSM, model.FavCountType)
+		log.Info("consumeFav key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, favSM.String(model.FavCountType))
+	}
+}
+
+// consumeReply  consumes playlist's reply.
+func (s *Service) consumeReply() {
+	defer s.waiter.Done()
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.playlistReplySub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job reply consumer exit!")
+			return
+		}
+		msg.Commit()
+		replySM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, replySM); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if replySM.Type != plmdl.PlDBusType || replySM.ID <= 0 {
+			continue
+		}
+		s.upStat(c, replySM, model.ReplyCountType)
+		log.Info("consumeReply key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, replySM.String(model.ReplyCountType))
+	}
+}
+
+// consumeShare  consumes playlist's share.
+func (s *Service) consumeShare() {
+	defer s.waiter.Done()
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.playlistShareSub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job share consumer exit!")
+			return
+		}
+		msg.Commit()
+		shareSM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, shareSM); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if shareSM.Type != plmdl.PlDBusType || shareSM.ID <= 0 {
+			continue
+		}
+		s.upStat(c, shareSM, model.ShareCountType)
+		log.Info("consumeShare key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, shareSM.String(model.ShareCountType))
+	}
+}
+
+// Ping reports the heath of services.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close releases resources which owned by the Service instance.
+func (s *Service) Close() (err error) {
+	defer s.waiter.Wait()
+	s.closed = true
+	s.playlistViewSub.Close()
+	s.playlistFavSub.Close()
+	s.playlistReplySub.Close()
+	s.playlistShareSub.Close()
+	log.Info("playlist-job has been closed.")
+	return
+}

app/job/main/playlist/service/service_test.go

@@ -0,0 +1,39 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/playlist/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	svr *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/playlist-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	svr = New(conf.Conf)
+	time.Sleep(time.Second)
+
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		f(svr)
+	}
+}
+
+func TestService_Ping(t *testing.T) {
+	Convey("test ping", t, WithService(func(s *Service) {
+		err := s.Ping(context.TODO())
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/playlist/service/stat.go

@@ -0,0 +1,126 @@
+package service
+
+import (
+	"context"
+
+	plamdl "go-common/app/interface/main/playlist/model"
+	"go-common/app/job/main/playlist/conf"
+	"go-common/app/job/main/playlist/dao"
+	"go-common/app/job/main/playlist/model"
+	"go-common/library/log"
+)
+
+func (s *Service) viewproc(i int64) {
+	defer s.waiter.Done()
+	var (
+		c  = context.TODO()
+		ch = s.statCh[i]
+	)
+	for {
+		sm, ok := <-ch
+		if !ok {
+			log.Warn("statproc(%d) quit", i)
+			return
+		}
+		// filter view count
+		if conf.Conf.Job.InterceptOn && sm.Count != nil && *sm.Count > 0 {
+			if s.intercept(sm) {
+				log.Info("intercept view count (pid:%d, aid:%d, ip:%s)", sm.ID, sm.Aid, sm.IP)
+				dao.PromInfo("stat:访问计数拦截")
+				continue
+			}
+		}
+		s.upStat(c, sm, model.ViewCountType)
+	}
+}
+
+func (s *Service) upStat(c context.Context, sm *model.StatM, tp string) {
+	// update cache
+	s.updateCache(c, sm, tp)
+	// update db
+	s.updateDB(c, sm, tp)
+}
+
+// updateDB update stat in db.
+func (s *Service) updateDB(c context.Context, stat *model.StatM, tp string) (err error) {
+	if _, err = s.dao.Update(context.TODO(), stat, tp); err != nil {
+		return
+	}
+	log.Info("update db success "+tp+" pid(%d) count(%d) ", stat.ID, *stat.Count)
+	dao.PromInfo("stat:更新计数DB")
+	return
+}
+
+// updateCache update stat in cache
+func (s *Service) updateCache(c context.Context, sm *model.StatM, tp string) (err error) {
+	var (
+		mid  int64
+		fid  int64
+		st   *plamdl.PlStat
+		stat *plamdl.ArgStats
+	)
+	if st, err = s.dao.Stat(c, sm.ID); err != nil {
+		log.Error("s.dao.Stat(%d) error(%v)", sm.ID, err)
+		return
+	}
+	switch tp {
+	case model.ViewCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  *sm.Count,
+			Reply: st.Reply,
+			Fav:   st.Fav,
+			Share: st.Share,
+			MTime: st.MTime,
+		}}
+	case model.FavCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  st.View,
+			Reply: st.Reply,
+			Fav:   *sm.Count,
+			Share: st.Share,
+			MTime: st.MTime,
+		}}
+	case model.ReplyCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  st.View,
+			Reply: *sm.Count,
+			Fav:   st.Fav,
+			Share: st.Share,
+			MTime: st.MTime,
+		}}
+	case model.ShareCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  st.View,
+			Reply: st.Reply,
+			Fav:   st.Fav,
+			Share: *sm.Count,
+			MTime: st.MTime,
+		}}
+	}
+	if err = s.playlistRPC.SetStat(c, stat); err != nil {
+		dao.PromError("stat:更新计数缓存",
+			"s.playlistRPC.SetStat "+tp+" pid(%d) fid(%d) mid(%d) view(%d) favorite(%d) reply(%d) share(%d) error(%v)",
+			sm.ID, fid, mid, *sm.Count, st.Fav, st.Reply, st.Share, err)
+	}
+	log.Info("update cache success "+tp+"  pid(%d)  aid(%d)  fid(%d) mid(%d) view(%d) favorite(%d) reply(%d) share(%d)",
+		sm.ID, sm.Aid, fid, mid, *sm.Count, st.Fav, st.Reply, st.Share)
+	dao.PromInfo("stat:更新计数缓存")
+	return
+}
+
+// intercept intercepts illegal views.
+func (s *Service) intercept(stat *model.StatM) bool {
+	return s.dao.Intercept(context.TODO(), stat.ID, stat.Aid, stat.IP)
+}