Create & Init Project...

app/job/main/dm/service/BUILD

@@ -0,0 +1,59 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dm_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dm.go",
+        "service.go",
+        "track.go",
+    ],
+    importpath = "go-common/app/job/main/dm/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/dao:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/sync/pipeline/fanout:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/dm/service/dm.go

@@ -0,0 +1,124 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// flushTrimQueue 将数据库数据填充到redis中
+func (s *Service) flushTrimQueue(c context.Context, tp int32, oid int64) (err error) {
+	var (
+		dms   []*model.DM
+		trims []*model.Trim
+	)
+	if dms, err = s.dao.DMInfos(c, tp, oid); err != nil {
+		return
+	}
+	for _, dm := range dms {
+		// NOTE 只有普通弹幕会被顶掉
+		if dm.Pool == model.PoolNormal && (dm.State == model.StateNormal || dm.State == model.StateMonitorAfter) {
+			trim := &model.Trim{ID: dm.ID, Attr: dm.AttrVal(model.AttrProtect)}
+			trims = append(trims, trim)
+		}
+	}
+	return s.dao.FlushTrimCache(c, tp, oid, trims)
+}
+
+// addTrimQueue add dm index redis trim queue and return segment need flush.
+func (s *Service) addTrimQueue(c context.Context, tp int32, oid, maxlimit int64, dms ...*model.DM) (err error) {
+	var (
+		ok             bool
+		trimCnt, count int64
+		trims          []*model.Trim
+		dmids          []int64
+	)
+	for _, dm := range dms {
+		// NOTE 只有普通弹幕并且弹幕状态处于正常或者先发后审状态的弹幕会被放入顶队列
+		if dm.Pool == model.PoolNormal && dm.NeedDisplay() {
+			trim := &model.Trim{ID: dm.ID, Attr: dm.AttrVal(model.AttrProtect)}
+			trims = append(trims, trim)
+		}
+	}
+	if len(trims) == 0 {
+		return
+	}
+	if ok, err = s.dao.ExpireTrimQueue(c, tp, oid); err != nil {
+		return
+	}
+	if !ok {
+		if err = s.flushTrimQueue(c, tp, oid); err != nil {
+			return
+		}
+	}
+	if count, err = s.dao.AddTrimQueueCache(c, tp, oid, trims); err != nil {
+		return
+	}
+	// NOTE 对于满弹幕的视频，始终保持两倍的候选弹幕集
+	if trimCnt = count - 2*maxlimit; trimCnt > 0 {
+		if dmids, err = s.dao.TrimCache(c, tp, oid, trimCnt); err != nil || len(trims) == 0 {
+			return
+		}
+		if len(dmids) == 0 {
+			return
+		}
+		if _, err = s.dao.UpdateDMStates(c, oid, dmids, model.StateHide); err != nil {
+			return
+		}
+		if err = s.dao.DelIdxContentCaches(c, tp, oid, dmids...); err != nil {
+			return
+		}
+		log.Info("oid:%d,trimCnt:%d,trims:%v", oid, len(dmids), dmids)
+	}
+	return
+}
+
+// recoverDM delete a dm and recover a hide state dm from db.
+func (s *Service) recoverDM(c context.Context, typ int32, oid, rcvCnt int64) (dms []*model.DM, err error) {
+	if dms, err = s.dao.DMHides(c, typ, oid, rcvCnt); err != nil {
+		return
+	}
+	if len(dms) > 0 {
+		var dmids []int64
+		for _, dm := range dms {
+			dmids = append(dmids, dm.ID)
+			dm.State = model.StateNormal
+		}
+		if _, err = s.dao.UpdateDMStates(c, oid, dmids, model.StateNormal); err != nil {
+			return
+		}
+		log.Info("recoverDM oid:%d dmids:%v", oid, dmids)
+	}
+	return
+}
+
+func (s *Service) subject(c context.Context, tp int32, oid int64) (sub *model.Subject, err error) {
+	var cache = true
+	if sub, err = s.dao.SubjectCache(c, tp, oid); err != nil {
+		err = nil
+		cache = false
+	}
+	if sub == nil {
+		if sub, err = s.dao.Subject(c, tp, oid); err != nil {
+			return
+		}
+		if sub == nil {
+			sub = &model.Subject{
+				Type: tp,
+				Oid:  oid,
+			}
+		}
+		if cache {
+			s.cache.Do(c, func(ctx context.Context) {
+				s.dao.SetSubjectCache(ctx, sub)
+			})
+		}
+	}
+	if sub.ID == 0 {
+		err = ecode.NothingFound
+		return
+	}
+	return
+}

app/job/main/dm/service/dm_test.go

@@ -0,0 +1,60 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestFlushTrimQueue(t *testing.T) {
+	var (
+		tp  int32 = 1
+		oid int64 = 1
+	)
+	Convey("", t, func() {
+		err := testSvc.flushTrimQueue(context.TODO(), tp, oid)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestAddTrimQueue(t *testing.T) {
+	var (
+		tp       int32 = 1
+		oid      int64 = 1
+		maxlimit int64 = 1
+		idx            = &model.DM{ID: 1, Type: tp, Oid: oid, Mid: 1, Progress: 1, State: 0, Pool: 2, Attr: 1}
+	)
+	Convey("", t, func() {
+		err := testSvc.addTrimQueue(context.TODO(), tp, oid, maxlimit, idx)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestRecoverDM(t *testing.T) {
+	var (
+		tp       int32 = 1
+		oid      int64 = 1
+		duration int64 = 10
+		maxlimit int64 = 1
+		sub            = &model.Subject{ID: 1, Type: tp, Oid: oid, ACount: 2, Count: 2, Maxlimit: maxlimit}
+	)
+	Convey("", t, func() {
+		_, err := testSvc.recoverDM(context.TODO(), sub.Type, sub.Oid, duration)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestSubject(t *testing.T) {
+	var (
+		tp  int32 = 1
+		oid int64 = 1
+	)
+	Convey("", t, func() {
+		sub, err := testSvc.subject(context.TODO(), tp, oid)
+		So(err, ShouldBeNil)
+		So(sub, ShouldNotBeNil)
+	})
+}

app/job/main/dm/service/service.go

@@ -0,0 +1,76 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"regexp"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/app/job/main/dm/dao"
+	"go-common/app/job/main/dm/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/sync/pipeline/fanout"
+)
+
+var (
+	errSubNotExist = errors.New("subject not exist")
+)
+
+// Service rpc service.
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+	// databus sub
+	dmMetaCsmr *databus.Databus
+	// cache
+	cache *fanout.Fanout
+}
+
+// New new rpc service.
+func New(c *conf.Config) *Service {
+	s := &Service{
+		c:          c,
+		dao:        dao.New(c),
+		dmMetaCsmr: databus.New(c.Databus.DMMetaCsmr),
+		cache:      fanout.New("cache", fanout.Worker(1), fanout.Buffer(1024)),
+	}
+	// 消费DMMeta-T消息
+	go s.dmMetaCsmproc()
+	return s
+}
+
+// Ping check if service is ok.
+func (s *Service) Ping(c context.Context) error {
+	return s.dao.Ping(c)
+}
+
+func (s *Service) dmMetaCsmproc() {
+	var (
+		err        error
+		c          = context.TODO()
+		regexIndex = regexp.MustCompile("dm_index_[0-9]+")
+	)
+	for {
+		msg, ok := <-s.dmMetaCsmr.Messages()
+		if !ok {
+			log.Error("dmmeta binlog consumer exit")
+			return
+		}
+		m := &model.BinlogMsg{}
+		if err = json.Unmarshal(msg.Value, &m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if regexIndex.MatchString(m.Table) {
+			if err = s.trackDMMeta(c, m); err != nil {
+				log.Error("s.trackDMMeta(%s) error(%v)", m, err)
+				continue
+			}
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("commit offset(%v) error(%v)", msg, err)
+		}
+	}
+}

app/job/main/dm/service/service_test.go

@@ -0,0 +1,22 @@
+package service
+
+import (
+	"os"
+	"testing"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/library/log"
+)
+
+var testSvc *Service
+
+func TestMain(m *testing.M) {
+	conf.ConfPath = "../cmd/dm-job-test.toml"
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	testSvc = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/dm/service/track.go

@@ -0,0 +1,80 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/log"
+)
+
+// trackDMMeta 顶弹幕逻辑  保持pool0的弹幕池只有maxlimt*2的数量
+func (s *Service) trackDMMeta(c context.Context, m *model.BinlogMsg) (err error) {
+	var (
+		sub *model.Subject
+		nw  = &model.DM{}
+	)
+	if err = json.Unmarshal(m.New, &nw); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", m.New, err)
+		return
+	}
+	if sub, err = s.subject(c, model.SubTypeVideo, nw.Oid); err != nil {
+		log.Error("s.subject(%d) error(%v)", nw.Oid, err)
+		return
+	}
+	if sub == nil {
+		err = errSubNotExist
+		return
+	}
+	switch m.Action {
+	case model.SyncInsert:
+		if sub.Count >= sub.Maxlimit {
+			if err = s.addTrimQueue(c, nw.Type, nw.Oid, sub.Maxlimit, nw); err != nil {
+				log.Error("s.addTrimQueue(%v) error(%v)", nw, err)
+				return err
+			}
+		}
+	case model.SyncUpdate:
+		old := &model.DM{}
+		if err = json.Unmarshal(m.Old, &old); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", m.Old, err)
+			return
+		}
+		if nw.NeedStateNormal(old) {
+			nw.State = model.StateNormal
+			if _, err = s.dao.UpdateDM(c, nw); err != nil {
+				log.Error("dao.UpdateDM(%v) error(%v)", nw, err)
+				return err
+			}
+		}
+		if sub.Count >= sub.Maxlimit {
+			dms := make([]*model.DM, 0)
+			if isDelOperation(nw, old) {
+				if err = s.dao.ZRemTrimCache(c, nw.Type, nw.Oid, nw.ID); err != nil {
+					return
+				}
+				if dms, err = s.recoverDM(c, nw.Type, nw.Oid, 1); err != nil {
+					log.Error("s.recoverIdx(%d) error(%v)", nw.Oid, err)
+					return
+				}
+			}
+			dms = append(dms, nw)
+			if err = s.addTrimQueue(c, nw.Type, nw.Oid, sub.Maxlimit, dms...); err != nil {
+				log.Error("s.addTrimQueue(%v) error(%v)", dms, err)
+				return
+			}
+		}
+	case model.SyncDelete:
+	}
+	return
+}
+
+func isDelOperation(nw, old *model.DM) bool {
+	if nw.State != model.StateHide && old.NeedDisplay() && !nw.NeedDisplay() { // 弹幕从展示变为非展示状态
+		return true
+	}
+	if nw.Pool != old.Pool && (nw.Pool == model.PoolSpecial || nw.Pool == model.PoolSubtitle) {
+		return true
+	}
+	return false
+}