Create & Init Project...

2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
--- a/app/job/main/dm/BUILD
+++ b/app/job/main/dm/BUILD
@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/dm/cmd:all-srcs",
+        "//app/job/main/dm/conf:all-srcs",
+        "//app/job/main/dm/dao:all-srcs",
+        "//app/job/main/dm/http:all-srcs",
+        "//app/job/main/dm/model:all-srcs",
+        "//app/job/main/dm/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/app/job/main/dm/CHANGELOG.md
+++ b/app/job/main/dm/CHANGELOG.md
@@ -0,0 +1,242 @@
+#### dm-job
+
+#### V3.8.5
+> 1. remove cache.Cache
+
+#### V3.8.4
+> 1. 移除删除缓存逻辑，只保留顶弹幕逻辑
+
+#### V3.8.3
+> 1. 分段弹幕空值缓存bug
+
+#### V3.8.2
+> 1. 修复顶弹幕逻辑时的刷新逻辑
+
+#### V3.8.1
+> 1. 修复redis缓存保护弹幕不显示的bug
+
+#### V3.8.0
+> 1. 彻底移除dm_index表的同步
+
+#### V3.7.10
+> 1. archive  duration缓存穿透
+
+#### V3.7.9
+> 1. 刷新缓存同步刷新字幕弹幕
+
+#### V3.7.8
+> 1. 移除dm_index insert事件的同步
+
+#### V3.7.7
+> 1. sync maxlimit
+
+#### V3.7.6
+> 1. 移除dm_index update事件的同步
+
+#### V3.7.5
+> 1. 移除chilpopl,maxlimit,state,aid,mid 同步
+
+#### V3.7.4
+> 1. subject 缓存穿透
+
+#### V3.7.3
+> 1. 重新构建master
+
+#### V3.7.2
+> 1. 移除多余Httpclient
+
+#### V3.7.1
+> 1. 删除comment
+
+#### V3.7.0
+> 1. 彻底移除老弹幕内容库到新库的同步逻辑
+
+#### V3.6.19
+> 1. update sortedset score in trim queue
+
+#### V3.6.18
+> 1. update sortedset score in trim queue
+
+#### V3.6.17
+> 1. update video duration cache
+
+#### V3.6.16
+> 1. 修复trimqueue中sortset key存在的bug
+
+#### V3.6.15
+> 1. 移除dm_index_filter的同步逻辑
+> 2. 迁移blademaster
+> 3. 弹幕实名增加白名单以及总开关
+> 4. 移除bilibili库
+
+#### V3.6.14
+> 1. 添加高级弹幕存储bfs逻辑
+
+#### V3.6.13
+> 1. dm_subject中新增mid同步
+
+#### V3.6.12
+> 1. 修复当分段弹幕被打满时，最后的弹幕无法展示的bug
+
+#### V3.6.11
+> 1. up主屏蔽词同步时清除mc缓存
+
+#### V3.6.10
+> 1. 迁移 main目录
+
+#### V3.6.9
+> 1. 同步老库dm_index_filter-->dm_filter_up_[00-09]
+
+#### V3.6.8
+> 1. 移除老库游客弹幕同步
+> 2. 移除老库活动弹幕同步
+
+#### V3.6.7
+> 1. 移除剧透弹幕相关逻辑
+
+#### V3.6.6
+> 1. 分段弹幕开启实名制
+
+#### V3.6.5
+> 1. 替换xml非法字符
+
+#### V3.6.4
+> 1. 迁移弹幕转移到dm2-job
+
+#### V3.6.3
+> 1. 修复subject mtime字段更新
+
+#### V3.6.2
+> 1. 增加禁止发送弹幕功能
+> 2. xml header中增加state字段
+
+#### V3.6.1
+>1. 限制分段弹幕数量为2*maxlimit条
+>2. 移除分段弹幕memcache 1M的限制
+
+#### V3.6.0
+>1. 迁移弹幕广播到dm interface
+>2. 停止同步弹幕老库的insert
+
+#### V3.5.16
+>1. 优化代码
+
+#### V3.5.15
+>1. 优化代码
+
+#### V3.5.14
+>1. 弹幕转移插入新库
+
+#### V3.5.13
+>1. 修改弹幕转移查库索引
+
+#### V3.5.12
+>1. 修改弹幕转移排序规则
+
+#### V3.5.11
+>1. 不再通过binlog同步新老库的insert
+
+#### V3.5.10
+>1. 删除弹幕后只恢复一条被顶掉的弹幕
+
+#### V3.5.9
+>1. 弹幕转移read使用新库
+
+#### V3.5.8
+>1. 修复顶弹幕逻辑
+
+#### V3.5.7
+>1. 弹幕转移重试
+
+#### V3.5.6
+>1. 减小弹幕转移并发
+>2. 增加弹幕转移并发锁
+
+#### V3.5.5
+>1. 修改弹幕转移任务限制
+
+#### V3.5.4
+>1. 插入subject 增加重试三次逻辑
+
+#### V3.5.3
+>1. 修复update dm mtime
+>2. add debug log
+
+#### V3.5.2
+> 1. 修复fontsize bug
+
+#### V3.5.1
+> 1. 修改弹幕转移的bug
+
+#### V3.5.0
+> 1. 订阅新弹幕库binlog进行弹幕顶逻辑处理
+> 2. 移除dm rpc client
+
+#### V3.4.17
+> 1. 移除childpool 3相应逻辑
+
+#### V3.4.16
+> 1. 删除被trim掉的缓存
+
+#### V3.4.15
+> 1. Video2 to Video3
+
+#### V3.4.14
+> 1. 修复新增pool=2弹幕被截断的问题
+
+#### V3.4.13
+> 1. 修复字号大于255导致的越界-1字号
+
+#### V3.4.12
+> 1.  更改dm_index和dm_content事务提交顺序
+
+#### V3.4.3
+> 1. 依据dmid查弹幕内容改为并发查询
+
+#### V3.4.2
+> 1. 优化慢查询sql语句
+
+#### V3.4.1
+> 1. 视频弹幕上限调整为2*maxlimit
+
+#### V3.4.0
+> 1.新增弹幕转移特性
+
+#### V3.3.12
+> 1.新增或变更弹幕时刷新当前段缓存
+
+#### V3.3.11
+> 1.修改为全局顶弹幕逻辑
+
+#### V3.3.10
+> 1.更改顶弹幕时的回源逻辑
+
+#### V3.3.9
+> 1.修复极端情况下mc的连接泄露问题
+
+#### V3.3.8
+> 1.弹幕族群job上线
+
+#### V3.3.7
+> 1.变更回源到redis
+
+#### V3.3.6
+> 1.弹幕迁移双写
+
+#### V3.3.5
+> 1.合并master分支
+
+#### V3.3.4
+> 1.修复弹幕广播bug
+
+#### V3.3.3
+> 1.弹幕举报状态
+
+#### V3.3.2
+> 1.修复弹幕rpc导致的panic
+
+#### V3.3.1
+> 1.修复弹幕rpc导致的panic
+
+#### V3.3.0
+> 1.弹幕举报定时删除
--- a/app/job/main/dm/CONTRIBUTORS.md
+++ b/app/job/main/dm/CONTRIBUTORS.md
@@ -0,0 +1,11 @@
+# Owner
+liangkai
+renwei
+
+# Author 
+liangkai
+renwei
+
+# Reviewer
+liangkai
+renwei
--- a/app/job/main/dm/OWNERS
+++ b/app/job/main/dm/OWNERS
@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- liangkai
+- renwei
+labels:
+- job
+- job/main/dm
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- guhao
+- liangkai
+- renwei
+- renyashun
--- a/app/job/main/dm/README.md
+++ b/app/job/main/dm/README.md
--- a/app/job/main/dm/cmd/BUILD
+++ b/app/job/main/dm/cmd/BUILD
@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["dm-job-test.toml"],
+    importpath = "go-common/app/job/main/dm/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/http:go_default_library",
+        "//app/job/main/dm/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/app/job/main/dm/cmd/dm-job-test.toml
+++ b/app/job/main/dm/cmd/dm-job-test.toml
@@ -0,0 +1,128 @@
+# This is a TOML document. Boom.
+
+version = "3.0.5"
+user = "nobody"
+pid = "/tmp/dm-job.pid"
+dir = "./"
+perf = "0.0.0.0:6215"
+checkFile = "/data/www/dm-job.html"
+
+[xlog]
+    dir = "/data/log/dm-job/"
+
+[tracer]
+    proto = "udp"
+    addr = "172.16.33.46:5140"
+    family = "dm-job"
+
+[httpServer]
+    addr = "0.0.0.0:6216"
+    maxListen = 100
+    timeout = "1s"
+
+[db]
+    [db.dmReader]
+    addr = "172.16.33.205:3310"
+    dsn = "test_3308:test_3308@tcp(172.16.33.205:3310)/bilibili_dm_meta?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8mb4"
+    active = 10
+    idle = 5
+    idleTimeout ="4h"
+    queryTimeout = "3s"
+    execTimeout = "3s"
+    tranTimeout = "3s"
+    [db.dmReader.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+    [db.dmWriter]
+    addr = "172.16.33.205:3310"
+    dsn = "test_3308:test_3308@tcp(172.16.33.205:3310)/bilibili_dm_meta?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8mb4"
+    active = 10
+    idle = 5
+    idleTimeout ="4h"
+    queryTimeout = "3s"
+    execTimeout = "3s"
+    tranTimeout = "3s"
+    [db.dmWriter.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+    name = "dm-job"
+    proto = "tcp"
+    addr = "172.16.33.54:6379"
+    active = 10
+    idle = 5
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+    expire = "24h"
+
+[memcache]
+    name = "dm-job"
+    proto = "tcp"
+    addr = "172.16.33.54:11211"
+    idle = 5
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "80s"
+    expire = "24h"
+
+[databus]
+    [databus.jobSub]
+    key = "0Pub71KvEMKXu63qtztq"
+    secret = "0Pub71KvEMKXu63qtztr"
+    group = "DMJob-UGC-S"
+    topic = "DMJob-T"
+    action = "sub"
+    name = "dm-job"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 1
+    active = 1
+    dialTimeout = "1s"
+    readTimeout = "40s"
+    writeTimeout = "1s"
+    idleTimeout = "60s"
+    [databus.dmMetaCsmr]
+    key = "0Pub71KvEMKXu63qtztq"
+    secret = "0Pub71KvEMKXu63qtztr"
+    group = "DMMeta-DMQueue-S"
+    topic = "DMMeta-T"
+    action = "sub"
+    name = "dm-job"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 1
+    active = 1
+    dialTimeout = "1s"
+    readTimeout = "40s"
+    writeTimeout = "1s"
+    idleTimeout = "60s"
+
+[archiveRPC]
+    timeout = "150ms"
+
+[realname]
+    switchOn = false
+    whitelist = [10109417,1221]
+    [realname.threshold]
+    "1,2" = 1010
+
+[job]
+    flushExpire = "10s"
+
+[bfs]
+    host = "http://uat-bfs.bilibili.co"
+    timeout = "5s"
+    bucket = "test"
+    key = "221bce6492eba70f"
+    secret = "6eb80603e85842542f9736eb13b7e3"
--- a/app/job/main/dm/cmd/main.go
+++ b/app/job/main/dm/cmd/main.go
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/app/job/main/dm/http"
+	"go-common/app/job/main/dm/service"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	// init log
+	log.Init(conf.Conf.Xlog)
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	defer log.Close()
+	// service init
+	svc := service.New(conf.Conf)
+	http.Init(conf.Conf, svc)
+	// init signal
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("dm-job get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			log.Info("dm-job exit")
+			return
+		case syscall.SIGHUP:
+		// TODO reload
+		default:
+			return
+		}
+	}
+}
--- a/app/job/main/dm/conf/BUILD
+++ b/app/job/main/dm/conf/BUILD
@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/dm/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/app/job/main/dm/conf/conf.go
+++ b/app/job/main/dm/conf/conf.go
@@ -0,0 +1,119 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	"go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// ConfPath dm-job config file path
+	ConfPath string
+	client   *conf.Client
+	// Conf config export var
+	Conf = &Config{}
+)
+
+// Config dm-job config struct
+type Config struct {
+	// base
+	// log
+	Xlog *log.Config
+	// tracer
+	Tracer *trace.Config
+	// http
+	HTTPServer *bm.ServerConfig
+	// databus
+	Databus *Databus
+	// database
+	DB *DB
+	// redis
+	Redis *Redis
+	// memcache
+	Memcache *Memcache
+}
+
+// Databus databus.
+type Databus struct {
+	DMMetaCsmr *databus.Config
+}
+
+// DB bilibili_dm
+type DB struct {
+	DMReader *sql.Config
+	DMWriter *sql.Config
+}
+
+// Redis dm redis
+type Redis struct {
+	*redis.Config
+	Expire time.Duration
+}
+
+// Memcache dm memcache
+type Memcache struct {
+	*memcache.Config
+	Expire time.Duration
+}
+
+func init() {
+	flag.StringVar(&ConfPath, "conf", "", "config path")
+}
+
+//Init int config
+func Init() error {
+	if ConfPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(ConfPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}
--- a/app/job/main/dm/dao/BUILD
+++ b/app/job/main/dm/dao/BUILD
@@ -0,0 +1,64 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "index_test.go",
+        "memcache_test.go",
+        "redis_test.go",
+        "subject_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "index.go",
+        "memcache.go",
+        "redis.go",
+        "subject.go",
+    ],
+    importpath = "go-common/app/job/main/dm/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/app/job/main/dm/dao/dao.go
+++ b/app/job/main/dm/dao/dao.go
@@ -0,0 +1,81 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_subjectSharding = 100
+	_indexSharding   = 1000
+)
+
+// Dao dao struct.
+type Dao struct {
+	// redis
+	redis       *redis.Pool
+	redisExpire int32
+	// memcache
+	mc       *memcache.Pool
+	mcExpire int32
+	// mysql
+	dmReader *sql.DB
+	dmWriter *sql.DB
+}
+
+// New return dm dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		// redis
+		redis:       redis.NewPool(c.Redis.Config),
+		redisExpire: int32(time.Duration(c.Redis.Expire) / time.Second),
+		// memcache
+		mc:       memcache.NewPool(c.Memcache.Config),
+		mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
+		// mysql
+		dmReader: sql.NewMySQL(c.DB.DMReader),
+		dmWriter: sql.NewMySQL(c.DB.DMWriter),
+	}
+	return
+}
+
+func (d *Dao) hitSubject(oid int64) int64 {
+	return oid % _subjectSharding
+}
+
+func (d *Dao) hitIndex(oid int64) int64 {
+	return oid % _indexSharding
+}
+
+// Ping dm dao ping.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.dmWriter.Ping(c); err != nil {
+		log.Error("dmWriter.Ping() error(%v)", err)
+		return
+	}
+	if err = d.dmReader.Ping(c); err != nil {
+		log.Error("dmReader.Ping() error(%v)", err)
+		return
+	}
+	// mc
+	mconn := d.mc.Get(c)
+	defer mconn.Close()
+	if err = mconn.Set(&memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}); err != nil {
+		log.Error("mc.Set error(%v)", err)
+		return
+	}
+	// dm redis
+	rconn := d.redis.Get(c)
+	defer rconn.Close()
+	if _, err = rconn.Do("SET", "ping", "pong"); err != nil {
+		rconn.Close()
+		log.Error("redis.Set error(%v)", err)
+	}
+	return
+}
--- a/app/job/main/dm/dao/dao_test.go
+++ b/app/job/main/dm/dao/dao_test.go
@@ -0,0 +1,24 @@
+package dao
+
+import (
+	"os"
+	"testing"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/library/log"
+)
+
+var (
+	testDao *Dao
+)
+
+func TestMain(m *testing.M) {
+	conf.ConfPath = "../cmd/dm-job-test.toml"
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	testDao = New(conf.Conf)
+	os.Exit(m.Run())
+}
--- a/app/job/main/dm/dao/index.go
+++ b/app/job/main/dm/dao/index.go
@@ -0,0 +1,76 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_selAllIdxSQL   = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d WHERE type=? AND oid=? AND state IN(0,6)"
+	_selIdxHidesSQL = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d FORCE INDEX(ix_oid_state) WHERE type=? AND oid=? AND state=2 ORDER BY id DESC limit ?" // NOTE slow query
+	_upIdxSQL       = "UPDATE dm_index_%03d SET mid=?,progress=?,state=?,pool=?,attr=? WHERE id=?"
+	_upIdxStatesSQL = "UPDATE dm_index_%03d SET state=? WHERE id IN(%s)"
+)
+
+// DMInfos get indexs of oid.
+func (d *Dao) DMInfos(c context.Context, tp int32, oid int64) (dms []*model.DM, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_selAllIdxSQL, d.hitIndex(oid)), tp, oid)
+	if err != nil {
+		log.Error("db.Query(%d %d) error(%v)", tp, oid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		dm := &model.DM{}
+		if err = rows.Scan(&dm.ID, &dm.Type, &dm.Oid, &dm.Mid, &dm.Progress, &dm.State, &dm.Pool, &dm.Attr, &dm.Ctime, &dm.Mtime); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		dms = append(dms, dm)
+	}
+	return
+}
+
+// DMHides get hide index info from db by oid and state.
+func (d *Dao) DMHides(c context.Context, typ int32, oid, limit int64) (res []*model.DM, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_selIdxHidesSQL, d.hitIndex(oid)), typ, oid, limit)
+	if err != nil {
+		log.Error("db.Query(%d %d) error(%v)", typ, oid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		dm := &model.DM{}
+		if err = rows.Scan(&dm.ID, &dm.Type, &dm.Oid, &dm.Mid, &dm.Progress, &dm.State, &dm.Pool, &dm.Attr, &dm.Ctime, &dm.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, dm)
+	}
+	return
+}
+
+// UpdateDM update index of dm.
+func (d *Dao) UpdateDM(c context.Context, m *model.DM) (affect int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_upIdxSQL, d.hitIndex(m.Oid)), m.Mid, m.Progress, m.State, m.Pool, m.Attr, m.ID)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// UpdateDMStates multi update index state of dm.
+func (d *Dao) UpdateDMStates(c context.Context, oid int64, dmids []int64, state int32) (affect int64, err error) {
+	upSQL := fmt.Sprintf(_upIdxStatesSQL, d.hitIndex(oid), xstr.JoinInts(dmids))
+	res, err := d.dmWriter.Exec(c, upSQL, state)
+	if err != nil {
+		log.Error("db.Exec(%s) error(%v)", upSQL, err)
+		return
+	}
+	return res.RowsAffected()
+}
--- a/app/job/main/dm/dao/index_test.go
+++ b/app/job/main/dm/dao/index_test.go
@@ -0,0 +1,53 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/dm/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDMInfos(t *testing.T) {
+	Convey("", t, func() {
+		dms, err := testDao.DMInfos(context.TODO(), 1, 1221)
+		if err != nil {
+			fmt.Println(err)
+		}
+		So(err, ShouldBeNil)
+		So(dms, ShouldNotBeEmpty)
+	})
+}
+
+func TestDMHides(t *testing.T) {
+	Convey("", t, func() {
+		dms, err := testDao.DMHides(context.TODO(), 1, 1221, 1)
+		if err != nil {
+			fmt.Println(err)
+		}
+		So(err, ShouldBeNil)
+		So(dms, ShouldNotBeEmpty)
+	})
+}
+
+func TestUpdateDM(t *testing.T) {
+	Convey("", t, func() {
+		dm := &model.DM{
+			ID:    1234555,
+			Oid:   1221,
+			State: 3,
+		}
+		_, err := testDao.UpdateDM(context.TODO(), dm)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestUpdateDMStates(t *testing.T) {
+	Convey("", t, func() {
+		dmids := []int64{709150141, 709150142}
+		_, err := testDao.UpdateDMStates(context.TODO(), 1221, dmids, 1)
+		So(err, ShouldBeNil)
+	})
+}
--- a/app/job/main/dm/dao/memcache.go
+++ b/app/job/main/dm/dao/memcache.go
@@ -0,0 +1,107 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_prefixSub   = "s_"
+	_keyDuration = "d_" // video duration
+)
+
+func keySubject(tp int32, oid int64) string {
+	return _prefixSub + fmt.Sprintf("%d_%d", tp, oid)
+}
+
+// keyDuration return video duration key.
+func keyDuration(oid int64) string {
+	return _keyDuration + strconv.FormatInt(oid, 10)
+}
+
+// SubjectCache get subject from memcache.
+func (d *Dao) SubjectCache(c context.Context, tp int32, oid int64) (sub *model.Subject, err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = keySubject(tp, oid)
+		rp   *memcache.Item
+	)
+	defer conn.Close()
+	if rp, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("mc.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	sub = &model.Subject{}
+	if err = conn.Scan(rp, &sub); err != nil {
+		log.Error("conn.Scan() error(%v)", err)
+	}
+	return
+}
+
+// SetSubjectCache add subject cache.
+func (d *Dao) SetSubjectCache(c context.Context, sub *model.Subject) (err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = keySubject(sub.Type, sub.Oid)
+	)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Object:     sub,
+		Flags:      memcache.FlagJSON,
+		Expiration: d.mcExpire,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%v) error(%v)", item, err)
+	}
+	return
+}
+
+// DurationCache return duration of video.
+func (d *Dao) DurationCache(c context.Context, oid int64) (duration int64, err error) {
+	var (
+		key  = keyDuration(oid)
+		conn = d.mc.Get(c)
+		item *memcache.Item
+	)
+	defer conn.Close()
+	if item, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			duration = model.NotFound
+			err = nil
+		} else {
+			log.Error("conn.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	if duration, err = strconv.ParseInt(string(item.Value), 10, 64); err != nil {
+		log.Error("strconv.ParseInt(%s) error(%v)", item.Value, err)
+	}
+	return
+}
+
+// SetDurationCache set video duration to redis.
+func (d *Dao) SetDurationCache(c context.Context, oid, duration int64) (err error) {
+	key := keyDuration(oid)
+	conn := d.mc.Get(c)
+	item := memcache.Item{
+		Key:        key,
+		Value:      []byte(fmt.Sprint(duration)),
+		Expiration: d.mcExpire,
+		Flags:      memcache.FlagRAW,
+	}
+	if err = conn.Set(&item); err != nil {
+		log.Error("mc.Set(%v) error(%v)", item, err)
+	}
+	conn.Close()
+	return
+}
--- a/app/job/main/dm/dao/memcache_test.go
+++ b/app/job/main/dm/dao/memcache_test.go
@@ -0,0 +1,43 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestSubjectCache(t *testing.T) {
+	Convey("subject cache", t, func() {
+		sub, err := testDao.SubjectCache(context.TODO(), 1, 1221)
+		So(err, ShouldBeNil)
+		So(sub, ShouldNotBeNil)
+	})
+}
+
+func TestSetSubjectCache(t *testing.T) {
+	sub := &model.Subject{
+		Type: 1,
+		Oid:  1221,
+	}
+	Convey("add subject cache", t, func() {
+		err := testDao.SetSubjectCache(context.TODO(), sub)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestDurationCache(t *testing.T) {
+	Convey("", t, func() {
+		_, err := testDao.DurationCache(context.TODO(), 1221)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestSetDurationCachee(t *testing.T) {
+	Convey("", t, func() {
+		err := testDao.SetDurationCache(context.TODO(), 1221, 10000)
+		So(err, ShouldBeNil)
+	})
+}
--- a/app/job/main/dm/dao/redis.go
+++ b/app/job/main/dm/dao/redis.go
@@ -0,0 +1,171 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_keyIdxContent = "c_%d_%d"    // dm content hash(c_type_oid, dmid, xml)
+	_keyTrimQueue  = "tq_n_%d_%d" // trim queue if dm_count > dm_maxlimit
+
+	divide = int64(34359738368) // 2^35
+)
+
+// keyIdxContent return key of different dm.
+func keyIdxContent(typ int32, oid int64) string {
+	return fmt.Sprintf(_keyIdxContent, typ, oid)
+}
+
+// keyIdxQueue return trim queue key.
+func keyTrimQueue(tp int32, oid int64) string {
+	return fmt.Sprintf(_keyTrimQueue, tp, oid)
+}
+
+// ExpireTrimQueue set expire time of index.
+func (d *Dao) ExpireTrimQueue(c context.Context, tp int32, oid int64) (ok bool, err error) {
+	key := keyTrimQueue(tp, oid)
+	conn := d.redis.Get(c)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.redisExpire)); err != nil {
+		log.Error("conn.Do(EXPIRE %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+func score(attr int32, id int64) (score float64) {
+	v := id / divide           // 2^63 / 2^35 = 2^28-1 整数部分最大值：268435455
+	k := id % divide           // 精度8位，最后5位可忽略
+	r := int64(attr)&1<<28 | v // NOTE v should less than 2^28
+	score, _ = strconv.ParseFloat(fmt.Sprintf("%d.%d", r, k), 64)
+	return
+}
+
+// AddTrimQueueCache add dm index into trim queue.
+func (d *Dao) AddTrimQueueCache(c context.Context, tp int32, oid int64, trims []*model.Trim) (count int64, err error) {
+	var (
+		key  = keyTrimQueue(tp, oid)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, trim := range trims {
+		if err = conn.Send("ZADD", key, score(trim.Attr, trim.ID), trim.ID); err != nil {
+			log.Error("conn.Send(ZADD %s %v) error(%v)", key, trim, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Send("ZCARD", key); err != nil {
+		log.Error("conn.Send(ZCARD %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < len(trims)+1; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	if count, err = redis.Int64(conn.Receive()); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+	}
+	return
+}
+
+// FlushTrimCache flush trim queue cache.
+func (d *Dao) FlushTrimCache(c context.Context, tp int32, oid int64, trims []*model.Trim) (err error) {
+	var (
+		key  = keyTrimQueue(tp, oid)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, trim := range trims {
+		if err = conn.Send("ZADD", key, score(trim.Attr, trim.ID), trim.ID); err != nil {
+			log.Error("conn.Send(ZADD %s %v) error(%v)", key, trim, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush(%s) error(%v)", key, err)
+		return
+	}
+	for i := 0; i < len(trims)+1; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// ZRemTrimCache ZREM trim from trim queue.
+func (d *Dao) ZRemTrimCache(c context.Context, tp int32, oid int64, dmID int64) (err error) {
+	var (
+		key  = keyTrimQueue(tp, oid)
+		conn = d.redis.Get(c)
+	)
+	if _, err = conn.Do("ZREM", key, dmID); err != nil {
+		log.Error("conn.Do(ZREM %s %v) error(%v)", key, dmID, err)
+	}
+	conn.Close()
+	return
+}
+
+// TrimCache trim trim queue and return trimed dmid from trim queue.
+func (d *Dao) TrimCache(c context.Context, tp int32, oid, count int64) (dmids []int64, err error) {
+	var (
+		key    = keyTrimQueue(tp, oid)
+		conn   = d.redis.Get(c)
+		replys [][]byte
+		dmID   int64
+	)
+	defer conn.Close()
+	if replys, err = redis.ByteSlices(conn.Do("ZRANGE", key, 0, count-1)); err != nil {
+		log.Error("conn.Do(ZRANGE %s) error(%v)", key, err)
+		return
+	}
+	for _, reply := range replys {
+		if err = json.Unmarshal(reply, &dmID); err != nil {
+			log.Error("json.Unmarshal(%s) error(v)", string(reply), err)
+			return
+		}
+		dmids = append(dmids, dmID)
+	}
+	if len(dmids) > 0 {
+		if _, err = conn.Do("ZREMRANGEBYRANK", key, 0, len(dmids)-1); err != nil {
+			log.Error("conn.Do(ZREMRANGEBYRANK %s) error(%v)", key, err)
+		}
+	}
+	return
+}
+
+// DelIdxContentCaches del index content cache.
+func (d *Dao) DelIdxContentCaches(c context.Context, typ int32, oid int64, dmids ...int64) (err error) {
+	key := keyIdxContent(typ, oid)
+	conn := d.redis.Get(c)
+	args := []interface{}{key}
+	for _, dmid := range dmids {
+		args = append(args, dmid)
+	}
+	if _, err = conn.Do("HDEL", args...); err != nil {
+		log.Error("conn.Do(HDEL %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
--- a/app/job/main/dm/dao/redis_test.go
+++ b/app/job/main/dm/dao/redis_test.go
@@ -0,0 +1,28 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm/conf"
+)
+
+func TestDuration(t *testing.T) {
+	var (
+		err      error
+		res      int64
+		cid      = int64(1)
+		duration = int64(100)
+		c        = context.TODO()
+	)
+	d := New(conf.Conf)
+	if err = d.SetDurationCache(c, cid, duration); err != nil {
+		t.Errorf("d.SetDurationCache(%d,%d) error(%v)", cid, duration, err)
+	}
+	if res, err = d.DurationCache(c, cid); err != nil {
+		t.Errorf("d.DurationCache(%d) error(%v)", cid, err)
+	}
+	if res != duration {
+		t.Errorf("not expect value %d,%d", duration, res)
+	}
+}
--- a/app/job/main/dm/dao/subject.go
+++ b/app/job/main/dm/dao/subject.go
@@ -0,0 +1,29 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_selSubSQL = "SELECT id,oid,type,pid,mid,state,attr,acount,count,mcount,move_count,maxlimit,childpool,ctime,mtime FROM dm_subject_%02d WHERE oid=? AND type=?"
+)
+
+// Subject get subject info from db.
+func (d *Dao) Subject(c context.Context, tp int32, oid int64) (s *model.Subject, err error) {
+	s = &model.Subject{}
+	row := d.dmReader.QueryRow(c, fmt.Sprintf(_selSubSQL, d.hitSubject(oid)), oid, tp)
+	if err = row.Scan(&s.ID, &s.Oid, &s.Type, &s.Pid, &s.Mid, &s.State, &s.Attr, &s.ACount, &s.Count, &s.MCount, &s.MoveCnt, &s.Maxlimit, &s.Childpool, &s.CTime, &s.MTime); err != nil {
+		if err == sql.ErrNoRows {
+			s = nil
+			err = nil
+		} else {
+			log.Error("row.Scan() error(%v)", err)
+		}
+	}
+	return
+}
--- a/app/job/main/dm/dao/subject_test.go
+++ b/app/job/main/dm/dao/subject_test.go
@@ -0,0 +1,17 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestSubject d.Subject() unit test.
+func TestSubject(t *testing.T) {
+	Convey("test dao.Subject", t, func() {
+		s, err := testDao.Subject(context.TODO(), 1, 1221)
+		So(err, ShouldBeNil)
+		So(s, ShouldNotBeNil)
+	})
+}
--- a/app/job/main/dm/http/BUILD
+++ b/app/job/main/dm/http/BUILD
@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/dm/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/app/job/main/dm/http/http.go
+++ b/app/job/main/dm/http/http.go
@@ -0,0 +1,39 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/app/job/main/dm/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	dmSvc *service.Service
+)
+
+// Init new dm job service
+func Init(c *conf.Config, s *service.Service) {
+	dmSvc = s
+	engine := bm.DefaultServer(c.HTTPServer)
+	initRouter(engine)
+	// run http server
+	if err := engine.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// initRouter init router.
+func initRouter(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	if err := dmSvc.Ping(c); err != nil {
+		log.Error("dm-job service ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
--- a/app/job/main/dm/model/BUILD
+++ b/app/job/main/dm/model/BUILD
@@ -0,0 +1,62 @@
+load(
+    "@io_bazel_rules_go//proto:def.bzl",
+    "go_proto_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+proto_library(
+    name = "model_proto",
+    srcs = ["dm.proto"],
+    tags = ["automanaged"],
+    deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
+)
+
+go_proto_library(
+    name = "model_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:gogofast_proto"],
+    importpath = "go-common/app/job/main/dm/model",
+    proto = ":model_proto",
+    tags = ["manual"],
+    deps = [
+        "//library/time:go_default_library",
+        "@com_github_gogo_protobuf//gogoproto:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dm.go",
+        "job.go",
+        "subject.go",
+    ],
+    embed = [":model_go_proto"],
+    importpath = "go-common/app/job/main/dm/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/time:go_default_library",
+        "@com_github_gogo_protobuf//gogoproto:go_default_library",
+        "@com_github_golang_protobuf//proto:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/app/job/main/dm/model/dm.go
+++ b/app/job/main/dm/model/dm.go
@@ -0,0 +1,116 @@
+package model
+
+import (
+	"database/sql/driver"
+	"strconv"
+	"time"
+)
+
+// all const variable used in job
+const (
+	AttrNo  = int32(0)
+	AttrYes = int32(1)
+
+	// platform
+	PlatUnknow  = int32(0)
+	PlatWeb     = int32(1)
+	PlatAndroid = int32(2)
+	PlatIPhone  = int32(3)
+	PlatWpM     = int32(4) // wp mobile
+	PlatIPad    = int32(5)
+	PlatPadHd   = int32(6) // ipad hd
+	PlatWpPc    = int32(7) // win10
+
+	// dm state
+	StateNormal        = int32(0) // 普通状态
+	StateDelete        = int32(1) // 删除状态
+	StateHide          = int32(2) // 隐藏状态
+	StateBlock         = int32(3) // 屏蔽状态
+	StateFilter        = int32(4) // 过滤状态
+	StateMonitorBefore = int32(5) // 先审后发
+	StateMonitorAfter  = int32(6) // 先发后审
+
+	// dm attribute
+	AttrProtect = uint(0) // 保护弹幕
+
+	// dm pool
+	PoolNormal   = int32(0) // 普通弹幕池
+	PoolSubtitle = int32(1) // 字幕弹幕池
+	PoolSpecial  = int32(2) // 特殊弹幕池
+
+	// dm mode
+	ModeNormal  = int32(1) // 正常滚动弹幕
+	ModeBottom  = int32(4) // 底部弹幕
+	ModeTop     = int32(5) // 顶部弹幕
+	ModeReverse = int32(6) // 逆向滚动弹幕
+	ModeAdvance = int32(7) // 高级弹幕
+	ModeCode    = int32(8) // 代码弹幕
+
+	NotFound = int64(-1)
+)
+
+// AttrVal return val of index'attr
+func (d *DM) AttrVal(bit uint) int32 {
+	return (d.Attr >> bit) & int32(1)
+}
+
+// AttrSet set val of index'attr
+func (d *DM) AttrSet(v int32, bit uint) {
+	d.Attr = d.Attr&(^(1 << bit)) | (v << bit)
+}
+
+// NeedDisplay 判断该条弹幕是否需要展示
+func (d *DM) NeedDisplay() bool {
+	return d.State == StateNormal || d.State == StateMonitorAfter
+}
+
+// NeedStateNormal 判断是否更新状态
+// pool0 变为 pool1 状态正常
+// 变为保护弹幕 状态正常
+func (d *DM) NeedStateNormal(old *DM) bool {
+	if (d.Pool != old.Pool) && d.Pool == PoolSubtitle {
+		return !(d.State == StateNormal)
+	}
+	if d.AttrVal(AttrProtect) == AttrYes && old.AttrVal(AttrProtect) == AttrNo {
+		return !(d.State == StateNormal)
+	}
+	return false
+}
+
+// Trim dmid and it's progress time will be trimed.
+type Trim struct {
+	ID   int64 `json:"id"`
+	Attr int32 `json:"-"`
+}
+
+type stime int64
+
+// Scan scan time.
+func (st *stime) Scan(src interface{}) (err error) {
+	switch sc := src.(type) {
+	case time.Time:
+		*st = stime(sc.Unix())
+	case string:
+		var i int64
+		i, err = strconv.ParseInt(sc, 10, 64)
+		*st = stime(i)
+	}
+	return
+}
+
+// Value get time value.
+func (st stime) Value() (driver.Value, error) {
+	return time.Unix(int64(st), 0), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (st *stime) UnmarshalJSON(data []byte) error {
+	timestamp, err := strconv.ParseInt(string(data), 10, 64)
+	if err == nil {
+		*st = stime(timestamp)
+		return nil
+	}
+	t, err := time.ParseInLocation(`"2006-01-02 15:04:05"`, string(data), time.Local)
+	*st = stime(t.Unix())
+	return err
+}
--- a/app/job/main/dm/model/dm.pb.go
+++ b/app/job/main/dm/model/dm.pb.go
--- a/app/job/main/dm/model/dm.proto
+++ b/app/job/main/dm/model/dm.proto
@@ -0,0 +1,37 @@
+syntax = "proto3";
+package model;
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+message DM {
+	int64 ID = 1 [(gogoproto.jsontag) = "id"];
+	int32 Type = 2 [(gogoproto.jsontag) = "type"];
+	int64 Oid = 3 [(gogoproto.jsontag) = "oid"];
+    int64 Mid = 4 [(gogoproto.jsontag) = "mid"];
+	int32 Progress = 5 [(gogoproto.jsontag) = "progress"];
+	int32 Pool = 6 [(gogoproto.jsontag) = "pool"];
+    int32 Attr = 7 [(gogoproto.jsontag) = "attr"];
+	int32 State = 8 [(gogoproto.jsontag) = "state"];
+	int64 Ctime = 9 [(gogoproto.jsontag) = "ctime", (gogoproto.casttype) = "stime"];
+	int64 Mtime = 10 [(gogoproto.jsontag) = "mtime", (gogoproto.casttype) = "stime"];
+    Content Content = 11 [(gogoproto.jsontag) = "content,omitempty"];
+    ContentSpecial ContentSpe = 12 [(gogoproto.jsontag) = "content_special,omitempty"];
+}
+
+message Content {
+	int64 ID = 1 [(gogoproto.jsontag) = "id"];
+	int32 FontSize = 2 [(gogoproto.jsontag) = "fontsize"];
+	int64 Color = 3 [(gogoproto.jsontag) = "color"];
+	int32 Mode = 4 [(gogoproto.jsontag) = "mode"];
+	int64 IP = 5 [(gogoproto.jsontag) = "ip"];
+	int32 Plat = 6 [(gogoproto.jsontag) = "plat"];
+	string Msg = 7 [(gogoproto.jsontag) = "msg"];
+	int64 Ctime = 8 [(gogoproto.jsontag) = "ctime", (gogoproto.casttype) = "go-common/library/time.Time"];
+	int64 Mtime = 9 [(gogoproto.jsontag) = "mtime", (gogoproto.casttype) = "go-common/library/time.Time"];
+}
+
+message ContentSpecial {
+	int64 ID = 1 [(gogoproto.jsontag) = "id"];
+	string Msg = 2 [(gogoproto.jsontag) = "msg"];
+	int64 Ctime = 3 [(gogoproto.jsontag) = "ctime", (gogoproto.casttype) = "go-common/library/time.Time"];
+	int64 Mtime = 4 [(gogoproto.jsontag) = "mtime", (gogoproto.casttype) = "go-common/library/time.Time"];
+}
--- a/app/job/main/dm/model/job.go
+++ b/app/job/main/dm/model/job.go
@@ -0,0 +1,21 @@
+package model
+
+import (
+	"encoding/json"
+)
+
+// All const variable used in job
+const (
+	// binlog type
+	SyncInsert = "insert"
+	SyncUpdate = "update"
+	SyncDelete = "delete"
+)
+
+// BinlogMsg dm binlog msg
+type BinlogMsg struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
--- a/app/job/main/dm/model/subject.go
+++ b/app/job/main/dm/model/subject.go
@@ -0,0 +1,128 @@
+package model
+
+import (
+	"go-common/library/time"
+)
+
+var (
+	_defaultSeg = &Segment{Start: 0, End: DefaultVideoEnd, Cnt: 1, Num: 1, Duration: 0}
+)
+
+const (
+	// segmentLength 分段长度，根据视频时长做分段，单位：毫秒
+	segmentLength = int64(6 * 60 * 1000)
+	// DefaultVideoEnd 当视频时长不存在或者为0时的默认视频结尾时间点
+	DefaultVideoEnd = int64(10 * 60 * 60 * 1000)
+
+	// SubTypeVideo 主题类型
+	SubTypeVideo = int32(1)
+	// SubStateOpen 主题打开
+	SubStateOpen = int32(0)
+	// SubStateClosed 主题关闭
+	SubStateClosed = int32(1)
+
+	// AttrSubGuest 允许游客弹幕
+	AttrSubGuest = uint(0)
+	// AttrSubSpolier 允许剧透弹幕
+	AttrSubSpolier = uint(1)
+	// AttrSubMission 允许活动弹幕
+	AttrSubMission = uint(2)
+	// AttrSubAdvance 允许高级弹幕
+	AttrSubAdvance = uint(3)
+	// AttrSubMonitorBefore 弹幕先审后发
+	AttrSubMonitorBefore = uint(4)
+	// AttrSubMonitorAfter 弹幕先发后审
+	AttrSubMonitorAfter = uint(5)
+)
+
+// Subject dm_subject.
+type Subject struct {
+	ID        int64     `json:"id"`
+	Type      int32     `json:"type"`
+	Oid       int64     `json:"oid"`
+	Pid       int64     `json:"pid"`
+	Mid       int64     `json:"mid"`
+	State     int32     `json:"state"`
+	Attr      int32     `json:"attr"`
+	ACount    int64     `json:"acount"`
+	Count     int64     `json:"count"`
+	MCount    int64     `json:"mcount"`
+	MoveCnt   int64     `json:"move_count"`
+	Maxlimit  int64     `json:"maxlimit"`
+	Childpool int32     `json:"childpool"`
+	CTime     time.Time `json:"ctime"`
+	MTime     time.Time `json:"mtime"`
+}
+
+// AttrVal return val of subject'attr.
+func (s *Subject) AttrVal(bit uint) int32 {
+	return (s.Attr >> bit) & int32(1)
+}
+
+// AttrSet set val of subject'attr.
+func (s *Subject) AttrSet(v int32, bit uint) {
+	s.Attr = s.Attr&(^(1 << bit)) | (v << bit)
+}
+
+// Segment dm segment struct
+type Segment struct {
+	Start    int64 `json:"ps"`       // 分段起始时间
+	End      int64 `json:"pe"`       // 分段结束时间
+	Cnt      int64 `json:"cnt"`      // 总分段数
+	Num      int64 `json:"num"`      // 当前第几段
+	Duration int64 `json:"duration"` // 视频总时长
+}
+
+// SegmentInfo get segment info by start time and video duration.
+func SegmentInfo(ps, duration int64) (s *Segment) {
+	var cnt, num, pe int64
+	if duration == 0 {
+		s = _defaultSeg
+		return
+	}
+	cnt = duration / segmentLength
+	if duration%segmentLength > 0 {
+		cnt++
+	}
+	for i := int64(0); i < cnt; i++ {
+		if ps >= i*segmentLength && ps < (i+1)*segmentLength {
+			ps = i * segmentLength
+			pe = (i + 1) * segmentLength
+			num = i + 1
+		}
+	}
+	if pe > duration {
+		pe = duration
+	}
+	if ps > duration {
+		ps = duration
+		pe = duration
+		num = cnt
+	}
+	s = &Segment{
+		Start:    ps,
+		End:      pe,
+		Cnt:      cnt,
+		Num:      num,
+		Duration: duration,
+	}
+	return
+}
+
+// SegmentPoint 根据当前段数和视频总时长计算分段的起始时间点
+func SegmentPoint(num, duration int64) (ps, pe int64) {
+	if duration == 0 {
+		ps = 0
+		pe = DefaultVideoEnd
+		return
+	}
+	pe = num * segmentLength
+	ps = pe - segmentLength
+	if pe > duration {
+		pe = duration
+	}
+	if ps < 0 {
+		ps = 0
+	}
+	return
+}
--- a/app/job/main/dm/service/BUILD
+++ b/app/job/main/dm/service/BUILD
@@ -0,0 +1,59 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dm_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dm.go",
+        "service.go",
+        "track.go",
+    ],
+    importpath = "go-common/app/job/main/dm/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/dao:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/sync/pipeline/fanout:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/app/job/main/dm/service/dm.go
+++ b/app/job/main/dm/service/dm.go
@@ -0,0 +1,124 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// flushTrimQueue 将数据库数据填充到redis中
+func (s *Service) flushTrimQueue(c context.Context, tp int32, oid int64) (err error) {
+	var (
+		dms   []*model.DM
+		trims []*model.Trim
+	)
+	if dms, err = s.dao.DMInfos(c, tp, oid); err != nil {
+		return
+	}
+	for _, dm := range dms {
+		// NOTE 只有普通弹幕会被顶掉
+		if dm.Pool == model.PoolNormal && (dm.State == model.StateNormal || dm.State == model.StateMonitorAfter) {
+			trim := &model.Trim{ID: dm.ID, Attr: dm.AttrVal(model.AttrProtect)}
+			trims = append(trims, trim)
+		}
+	}
+	return s.dao.FlushTrimCache(c, tp, oid, trims)
+}
+
+// addTrimQueue add dm index redis trim queue and return segment need flush.
+func (s *Service) addTrimQueue(c context.Context, tp int32, oid, maxlimit int64, dms ...*model.DM) (err error) {
+	var (
+		ok             bool
+		trimCnt, count int64
+		trims          []*model.Trim
+		dmids          []int64
+	)
+	for _, dm := range dms {
+		// NOTE 只有普通弹幕并且弹幕状态处于正常或者先发后审状态的弹幕会被放入顶队列
+		if dm.Pool == model.PoolNormal && dm.NeedDisplay() {
+			trim := &model.Trim{ID: dm.ID, Attr: dm.AttrVal(model.AttrProtect)}
+			trims = append(trims, trim)
+		}
+	}
+	if len(trims) == 0 {
+		return
+	}
+	if ok, err = s.dao.ExpireTrimQueue(c, tp, oid); err != nil {
+		return
+	}
+	if !ok {
+		if err = s.flushTrimQueue(c, tp, oid); err != nil {
+			return
+		}
+	}
+	if count, err = s.dao.AddTrimQueueCache(c, tp, oid, trims); err != nil {
+		return
+	}
+	// NOTE 对于满弹幕的视频，始终保持两倍的候选弹幕集
+	if trimCnt = count - 2*maxlimit; trimCnt > 0 {
+		if dmids, err = s.dao.TrimCache(c, tp, oid, trimCnt); err != nil || len(trims) == 0 {
+			return
+		}
+		if len(dmids) == 0 {
+			return
+		}
+		if _, err = s.dao.UpdateDMStates(c, oid, dmids, model.StateHide); err != nil {
+			return
+		}
+		if err = s.dao.DelIdxContentCaches(c, tp, oid, dmids...); err != nil {
+			return
+		}
+		log.Info("oid:%d,trimCnt:%d,trims:%v", oid, len(dmids), dmids)
+	}
+	return
+}
+
+// recoverDM delete a dm and recover a hide state dm from db.
+func (s *Service) recoverDM(c context.Context, typ int32, oid, rcvCnt int64) (dms []*model.DM, err error) {
+	if dms, err = s.dao.DMHides(c, typ, oid, rcvCnt); err != nil {
+		return
+	}
+	if len(dms) > 0 {
+		var dmids []int64
+		for _, dm := range dms {
+			dmids = append(dmids, dm.ID)
+			dm.State = model.StateNormal
+		}
+		if _, err = s.dao.UpdateDMStates(c, oid, dmids, model.StateNormal); err != nil {
+			return
+		}
+		log.Info("recoverDM oid:%d dmids:%v", oid, dmids)
+	}
+	return
+}
+
+func (s *Service) subject(c context.Context, tp int32, oid int64) (sub *model.Subject, err error) {
+	var cache = true
+	if sub, err = s.dao.SubjectCache(c, tp, oid); err != nil {
+		err = nil
+		cache = false
+	}
+	if sub == nil {
+		if sub, err = s.dao.Subject(c, tp, oid); err != nil {
+			return
+		}
+		if sub == nil {
+			sub = &model.Subject{
+				Type: tp,
+				Oid:  oid,
+			}
+		}
+		if cache {
+			s.cache.Do(c, func(ctx context.Context) {
+				s.dao.SetSubjectCache(ctx, sub)
+			})
+		}
+	}
+	if sub.ID == 0 {
+		err = ecode.NothingFound
+		return
+	}
+	return
+}
--- a/app/job/main/dm/service/dm_test.go
+++ b/app/job/main/dm/service/dm_test.go
@@ -0,0 +1,60 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestFlushTrimQueue(t *testing.T) {
+	var (
+		tp  int32 = 1
+		oid int64 = 1
+	)
+	Convey("", t, func() {
+		err := testSvc.flushTrimQueue(context.TODO(), tp, oid)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestAddTrimQueue(t *testing.T) {
+	var (
+		tp       int32 = 1
+		oid      int64 = 1
+		maxlimit int64 = 1
+		idx            = &model.DM{ID: 1, Type: tp, Oid: oid, Mid: 1, Progress: 1, State: 0, Pool: 2, Attr: 1}
+	)
+	Convey("", t, func() {
+		err := testSvc.addTrimQueue(context.TODO(), tp, oid, maxlimit, idx)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestRecoverDM(t *testing.T) {
+	var (
+		tp       int32 = 1
+		oid      int64 = 1
+		duration int64 = 10
+		maxlimit int64 = 1
+		sub            = &model.Subject{ID: 1, Type: tp, Oid: oid, ACount: 2, Count: 2, Maxlimit: maxlimit}
+	)
+	Convey("", t, func() {
+		_, err := testSvc.recoverDM(context.TODO(), sub.Type, sub.Oid, duration)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestSubject(t *testing.T) {
+	var (
+		tp  int32 = 1
+		oid int64 = 1
+	)
+	Convey("", t, func() {
+		sub, err := testSvc.subject(context.TODO(), tp, oid)
+		So(err, ShouldBeNil)
+		So(sub, ShouldNotBeNil)
+	})
+}
--- a/app/job/main/dm/service/service.go
+++ b/app/job/main/dm/service/service.go
@@ -0,0 +1,76 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"regexp"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/app/job/main/dm/dao"
+	"go-common/app/job/main/dm/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/sync/pipeline/fanout"
+)
+
+var (
+	errSubNotExist = errors.New("subject not exist")
+)
+
+// Service rpc service.
+type Service struct {
+	c   *conf.Config
+	dao *dao.Dao
+	// databus sub
+	dmMetaCsmr *databus.Databus
+	// cache
+	cache *fanout.Fanout
+}
+
+// New new rpc service.
+func New(c *conf.Config) *Service {
+	s := &Service{
+		c:          c,
+		dao:        dao.New(c),
+		dmMetaCsmr: databus.New(c.Databus.DMMetaCsmr),
+		cache:      fanout.New("cache", fanout.Worker(1), fanout.Buffer(1024)),
+	}
+	// 消费DMMeta-T消息
+	go s.dmMetaCsmproc()
+	return s
+}
+
+// Ping check if service is ok.
+func (s *Service) Ping(c context.Context) error {
+	return s.dao.Ping(c)
+}
+
+func (s *Service) dmMetaCsmproc() {
+	var (
+		err        error
+		c          = context.TODO()
+		regexIndex = regexp.MustCompile("dm_index_[0-9]+")
+	)
+	for {
+		msg, ok := <-s.dmMetaCsmr.Messages()
+		if !ok {
+			log.Error("dmmeta binlog consumer exit")
+			return
+		}
+		m := &model.BinlogMsg{}
+		if err = json.Unmarshal(msg.Value, &m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if regexIndex.MatchString(m.Table) {
+			if err = s.trackDMMeta(c, m); err != nil {
+				log.Error("s.trackDMMeta(%s) error(%v)", m, err)
+				continue
+			}
+		}
+		if err = msg.Commit(); err != nil {
+			log.Error("commit offset(%v) error(%v)", msg, err)
+		}
+	}
+}
--- a/app/job/main/dm/service/service_test.go
+++ b/app/job/main/dm/service/service_test.go
@@ -0,0 +1,22 @@
+package service
+
+import (
+	"os"
+	"testing"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/library/log"
+)
+
+var testSvc *Service
+
+func TestMain(m *testing.M) {
+	conf.ConfPath = "../cmd/dm-job-test.toml"
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	testSvc = New(conf.Conf)
+	os.Exit(m.Run())
+}
--- a/app/job/main/dm/service/track.go
+++ b/app/job/main/dm/service/track.go
@@ -0,0 +1,80 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/log"
+)
+
+// trackDMMeta 顶弹幕逻辑  保持pool0的弹幕池只有maxlimt*2的数量
+func (s *Service) trackDMMeta(c context.Context, m *model.BinlogMsg) (err error) {
+	var (
+		sub *model.Subject
+		nw  = &model.DM{}
+	)
+	if err = json.Unmarshal(m.New, &nw); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", m.New, err)
+		return
+	}
+	if sub, err = s.subject(c, model.SubTypeVideo, nw.Oid); err != nil {
+		log.Error("s.subject(%d) error(%v)", nw.Oid, err)
+		return
+	}
+	if sub == nil {
+		err = errSubNotExist
+		return
+	}
+	switch m.Action {
+	case model.SyncInsert:
+		if sub.Count >= sub.Maxlimit {
+			if err = s.addTrimQueue(c, nw.Type, nw.Oid, sub.Maxlimit, nw); err != nil {
+				log.Error("s.addTrimQueue(%v) error(%v)", nw, err)
+				return err
+			}
+		}
+	case model.SyncUpdate:
+		old := &model.DM{}
+		if err = json.Unmarshal(m.Old, &old); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", m.Old, err)
+			return
+		}
+		if nw.NeedStateNormal(old) {
+			nw.State = model.StateNormal
+			if _, err = s.dao.UpdateDM(c, nw); err != nil {
+				log.Error("dao.UpdateDM(%v) error(%v)", nw, err)
+				return err
+			}
+		}
+		if sub.Count >= sub.Maxlimit {
+			dms := make([]*model.DM, 0)
+			if isDelOperation(nw, old) {
+				if err = s.dao.ZRemTrimCache(c, nw.Type, nw.Oid, nw.ID); err != nil {
+					return
+				}
+				if dms, err = s.recoverDM(c, nw.Type, nw.Oid, 1); err != nil {
+					log.Error("s.recoverIdx(%d) error(%v)", nw.Oid, err)
+					return
+				}
+			}
+			dms = append(dms, nw)
+			if err = s.addTrimQueue(c, nw.Type, nw.Oid, sub.Maxlimit, dms...); err != nil {
+				log.Error("s.addTrimQueue(%v) error(%v)", dms, err)
+				return
+			}
+		}
+	case model.SyncDelete:
+	}
+	return
+}
+
+func isDelOperation(nw, old *model.DM) bool {
+	if nw.State != model.StateHide && old.NeedDisplay() && !nw.NeedDisplay() { // 弹幕从展示变为非展示状态
+		return true
+	}
+	if nw.Pool != old.Pool && (nw.Pool == model.PoolSpecial || nw.Pool == model.PoolSubtitle) {
+		return true
+	}
+	return false
+}