Create & Init Project...

app/job/main/dm2/dao/BUILD

@@ -0,0 +1,95 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bfs_test.go",
+        "dao_test.go",
+        "mask_test.go",
+        "mc_special_test.go",
+        "mc_subtitle_test.go",
+        "memcache_seg_test.go",
+        "memcache_test.go",
+        "mysql_dm_special_test.go",
+        "mysql_test.go",
+        "rank_list_test.go",
+        "redis_mask_test.go",
+        "redis_rct_test.go",
+        "redis_seg_test.go",
+        "redis_task_test.go",
+        "redis_test.go",
+        "task_test.go",
+        "transfer_test.go",
+        "videoup_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/dm2/conf:go_default_library",
+        "//app/job/main/dm2/model:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bfs.go",
+        "bnj.go",
+        "dao.go",
+        "mask.go",
+        "mc_special.go",
+        "mc_subtitle.go",
+        "memcache.go",
+        "memcache_seg.go",
+        "mysql.go",
+        "mysql_dm_special.go",
+        "rank_list.go",
+        "redis.go",
+        "redis_mask.go",
+        "redis_rct.go",
+        "redis_seg.go",
+        "redis_task.go",
+        "task.go",
+        "transfer.go",
+        "videoup.go",
+    ],
+    importpath = "go-common/app/job/main/dm2/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/dm2/conf:go_default_library",
+        "//app/job/main/dm2/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/bfs:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/dm2/dao/bfs.go

@@ -0,0 +1,39 @@
+package dao
+
+import (
+	"context"
+	"net/http"
+
+	"go-common/library/database/bfs"
+	"go-common/library/log"
+)
+
+// BfsData get bfs data
+func (d *Dao) BfsData(c context.Context, bfsURL string) (bs []byte, err error) {
+	var (
+		req *http.Request
+	)
+	if req, err = http.NewRequest(http.MethodGet, bfsURL, nil); err != nil {
+		log.Error("NewRequest(bfsURL:%v),error(%v)", bfsURL, err)
+		return
+	}
+	if bs, err = d.httpCli.Raw(c, req); err != nil {
+		log.Error("Raw(bfsURL:%v),error(%v)", bfsURL, err)
+		return
+	}
+	return
+}
+
+// BfsDmUpload .
+func (d *Dao) BfsDmUpload(c context.Context, fileName string, bs []byte) (location string, err error) {
+	if location, err = d.bfsCli.Upload(c, &bfs.Request{
+		Bucket:      d.conf.Bfs.Dm,
+		Filename:    fileName,
+		ContentType: "application/json",
+		File:        bs,
+	}); err != nil {
+		log.Error("bfs.BfsDmUpload error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/dm2/dao/bfs_test.go

@@ -0,0 +1,33 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoBfsData(t *testing.T) {
+	convey.Convey("BfsData", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			bfsURL = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.BfsData(c, bfsURL)
+		})
+	})
+}
+
+func TestDaoBfsDmUpload(t *testing.T) {
+	convey.Convey("BfsDmUpload", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			fileName = ""
+			bs       = []byte("231231231231")
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.BfsDmUpload(c, fileName, bs)
+		})
+	})
+}

app/job/main/dm2/dao/bnj.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+)
+
+const (
+	_bnjLiveConfig = "/activity/v0/bainian/config"
+)
+
+func (d *Dao) bnjConfigURI() string {
+	return d.conf.Host.APILive + _bnjLiveConfig
+}
+
+// BnjConfig .
+func (d *Dao) BnjConfig(c context.Context) (bnjConfig *model.BnjLiveConfig, err error) {
+	var (
+		res struct {
+			Code    int64                `json:"code"`
+			Message string               `json:"message"`
+			Data    *model.BnjLiveConfig `json:"data"`
+		}
+	)
+	if err = d.httpCli.Get(c, d.bnjConfigURI(), metadata.String(c, metadata.RemoteIP), nil, &res); err != nil {
+		log.Error("bnjLiveConfig BnjConfig(url:%v) error(%v)", d.bnjConfigURI(), err)
+		return
+	}
+	bnjConfig = res.Data
+	return
+}

app/job/main/dm2/dao/dao.go

@@ -0,0 +1,138 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/dm2/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/bfs"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+const (
+	_pageSize = 1000
+)
+
+// Dao dao struct
+type Dao struct {
+	conf *conf.Config
+	// batch query size
+	pageSize int
+	// database
+	dmWriter     *sql.DB
+	dmReader     *sql.DB
+	biliDMWriter *sql.DB
+	// redis
+	dmRds       *redis.Pool
+	dmRdsExpire int32
+	// recent dm redis
+	dmRctRds    *redis.Pool
+	dmRctExpire int32
+	// segment dm redis
+	dmSegRds    *redis.Pool
+	dmSegExpire int32
+	// memcache
+	mc               *memcache.Pool
+	mcExpire         int32
+	subtitleMc       *memcache.Pool
+	subtitleMcExpire int32
+	// memcache new
+	dmSegMC       *memcache.Pool
+	dmSegMCExpire int32
+	// recent dm redis
+	rctRds       *redis.Pool
+	rctRdsExpire int32
+
+	// http client
+	httpCli *bm.Client
+	// upload dm
+	bfsCli *bfs.BFS
+}
+
+// New new a dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		conf:             c,
+		dmWriter:         sql.NewMySQL(c.DB.DMWriter),
+		dmReader:         sql.NewMySQL(c.DB.DMReader),
+		biliDMWriter:     sql.NewMySQL(c.DB.BiliDMWriter),
+		dmRds:            redis.NewPool(c.Redis.DM.Config),
+		dmRdsExpire:      int32(time.Duration(c.Redis.DM.Expire) / time.Second),
+		dmRctRds:         redis.NewPool(c.Redis.DMRct.Config),
+		dmRctExpire:      int32(time.Duration(c.Redis.DMRct.Expire) / time.Second),
+		dmSegRds:         redis.NewPool(c.Redis.DMSeg.Config),
+		dmSegExpire:      int32(time.Duration(c.Redis.DMSeg.Expire) / time.Second),
+		mc:               memcache.NewPool(c.Memcache.Config),
+		mcExpire:         int32(time.Duration(c.Memcache.Expire) / time.Second),
+		subtitleMc:       memcache.NewPool(c.SubtitleMemcache.Config),
+		subtitleMcExpire: int32(time.Duration(c.SubtitleMemcache.Expire) / time.Second),
+		dmSegMC:          memcache.NewPool(c.DMMemcache.Config),
+		dmSegMCExpire:    int32(time.Duration(c.DMMemcache.Expire) / time.Second),
+		rctRds:           redis.NewPool(c.Redis.DMRct.Config),
+		rctRdsExpire:     int32(time.Duration(c.Redis.DMRct.Expire) / time.Second),
+		httpCli:          bm.NewClient(c.HTTPClient),
+		bfsCli:           bfs.New(c.Bfs.Client),
+		pageSize:         int(c.DB.QueryPageSize),
+	}
+	if d.pageSize <= 0 {
+		d.pageSize = _pageSize
+	}
+	return
+}
+
+// BeginTran begin mysql transaction
+func (d *Dao) BeginTran(c context.Context) (*sql.Tx, error) {
+	return d.dmWriter.Begin(c)
+}
+
+// BeginBiliDMTran .
+func (d *Dao) BeginBiliDMTran(c context.Context) (*sql.Tx, error) {
+	return d.biliDMWriter.Begin(c)
+}
+
+// Ping dm dao ping.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.dmWriter.Ping(c); err != nil {
+		log.Error("dmWriter.Ping() error(%v)", err)
+		return
+	}
+	if err = d.dmReader.Ping(c); err != nil {
+		log.Error("dmReader.Ping() error(%v)", err)
+		return
+	}
+	if err = d.biliDMWriter.Ping(c); err != nil {
+		log.Error("biliDMWriter.Ping() error(%v)", err)
+		return
+	}
+	// mc
+	mconn := d.mc.Get(c)
+	defer mconn.Close()
+	if err = mconn.Set(&memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}); err != nil {
+		log.Error("mc.Set error(%v)", err)
+		return
+	}
+	// dm redis
+	dmRdsConn := d.dmRds.Get(c)
+	defer dmRdsConn.Close()
+	if _, err = dmRdsConn.Do("SET", "ping", "pong"); err != nil {
+		log.Error("dmRds.Set error(%v)", err)
+		return
+	}
+	rctRdsConn := d.dmRctRds.Get(c)
+	defer rctRdsConn.Close()
+	if _, err = rctRdsConn.Do("SET", "ping", "pong"); err != nil {
+		log.Error("rctRds.Set error(%v)", err)
+		return
+	}
+	dmSegConn := d.dmSegRds.Get(c)
+	defer dmSegConn.Close()
+	if _, err = dmSegConn.Do("SET", "ping", "pong"); err != nil {
+		log.Error("dmSegConn.Set error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/dm2/dao/dao_test.go

@@ -0,0 +1,32 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/job/main/dm2/conf"
+	"go-common/library/log"
+)
+
+var testDao *Dao
+
+func TestMain(m *testing.M) {
+	flag.Set("app_id", "main.community.dm2-job")
+	flag.Set("conf_token", "m7xxj8RU7YxRK0fmRocmD9SoGUGzsZSA")
+	flag.Set("tree_id", "5391")
+	flag.Set("conf_version", "docker-1")
+	flag.Set("deploy_env", "uat")
+	flag.Set("conf_host", "config.bilibili.co")
+	flag.Set("conf_path", "/tmp")
+	flag.Set("region", "sh")
+	flag.Set("zone", "sh001")
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	testDao = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/dm2/dao/mask.go

@@ -0,0 +1,61 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"go-common/library/log"
+)
+
+const _mask = "/dl/api/masks/v1"
+
+func (d *Dao) maskURI() string {
+	return d.conf.Host.Mask + _mask
+}
+
+// GenerateMask ask AI to generate dm mask
+func (d *Dao) GenerateMask(c context.Context, cid, mid int64, plat int8, priority int32, aid int64, duration int64, typeID int32) (err error) {
+	var (
+		res struct {
+			Code    int64  `json:"errcode"`
+			Message string `json:"errmsg"`
+		}
+		params = make(map[string]interface{})
+	)
+	params["cid"] = cid
+	params["mask_platform"] = plat
+	params["priority"] = priority
+	params["mid"] = mid
+	params["aid"] = aid
+	params["duration"] = duration
+	params["region_2"] = typeID
+	data, err := json.Marshal(params)
+	if err != nil {
+		log.Error("json.Marshal(%v) error(%v)", params, err)
+		return
+	}
+	reader := bytes.NewReader(data)
+	req, err := http.NewRequest("POST", d.maskURI(), reader)
+	if err != nil {
+		log.Error("http.NewRequest error(%v)", err)
+		return
+	}
+	req.Header.Set("Content-Type", "application/json")
+	for i := 0; i < 3; i++ {
+		if err = d.httpCli.Do(c, req, &res); err != nil {
+			log.Error("d.httpCli.DO(%v) error(%v)", req, err)
+			continue
+		}
+		if res.Code != 200 {
+			err = fmt.Errorf("uri:%s,code:%d", d.maskURI(), res.Code)
+			log.Error("http code error(%v)", err)
+			continue
+		}
+		log.Info("send genarate mask request succeed(cid:%d)", cid)
+		break
+	}
+	return
+}

app/job/main/dm2/dao/mask_test.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaomaskURI(t *testing.T) {
+	convey.Convey("maskURI", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.maskURI()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGenerateMask(t *testing.T) {
+	convey.Convey("GenerateMask", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			cid      = int64(0)
+			mid      = int64(0)
+			plat     = int8(0)
+			priority = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.GenerateMask(c, cid, mid, plat, priority, 0, 0, 0)
+		})
+	})
+}

app/job/main/dm2/dao/mc_special.go

@@ -0,0 +1,52 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_fmtSpecialDm = "s_special_%d_%d"
+)
+
+func (d *Dao) specialDmKey(oid int64, tp int32) string {
+	return fmt.Sprintf(_fmtSpecialDm, oid, tp)
+}
+
+// DelSpecialDmCache .
+func (d *Dao) DelSpecialDmCache(c context.Context, oid int64, tp int32) (err error) {
+	var (
+		key  = d.specialDmKey(oid, tp)
+		conn = d.dmSegMC.Get(c)
+	)
+	defer conn.Close()
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("memcache.Delete(%s) error(%v)", key, err)
+		}
+	}
+	return
+}
+
+// AddSpecialDmCache add special content to memcache.
+func (d *Dao) AddSpecialDmCache(c context.Context, ds *model.DmSpecial) (err error) {
+	conn := d.dmSegMC.Get(c)
+	key := d.specialDmKey(ds.Oid, ds.Type)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Object:     ds,
+		Flags:      memcache.FlagJSON,
+		Expiration: d.dmSegMCExpire,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%s) error(%v)", key, err)
+	}
+	return
+}

app/job/main/dm2/dao/mc_special_test.go

@@ -0,0 +1,52 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/job/main/dm2/model"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaospecialDmKey(t *testing.T) {
+	convey.Convey("specialDmKey", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.specialDmKey(oid, tp)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelSpecialDmCache(t *testing.T) {
+	convey.Convey("DelSpecialDmCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			oid = int64(0)
+			tp  = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelSpecialDmCache(c, oid, tp)
+		})
+	})
+}
+
+func TestDaoAddSpecialDmCache(t *testing.T) {
+	convey.Convey("AddSpecialDmCache", t, func(ctx convey.C) {
+		var (
+			c  = context.Background()
+			ds = &model.DmSpecial{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := testDao.AddSpecialDmCache(c, ds)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/dm2/dao/mc_subtitle.go

@@ -0,0 +1,78 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_fmtSubtitle      = "s_subtitle_%d_%d"
+	_fmtVideoSubtitle = "s_video_%d_%d"
+	_fmtSubtitleDraft = "s_draft_%v_%v_%v_%v"
+)
+
+func (d *Dao) subtitleKey(oid int64, subtitleID int64) string {
+	return fmt.Sprintf(_fmtSubtitle, oid, subtitleID)
+}
+
+func (d *Dao) subtitleVideoKey(oid int64, tp int32) string {
+	return fmt.Sprintf(_fmtVideoSubtitle, oid, tp)
+}
+
+func (d *Dao) subtitleDraftKey(oid int64, tp int32, mid int64, lan uint8) string {
+	return fmt.Sprintf(_fmtSubtitleDraft, oid, tp, mid, lan)
+}
+
+// DelVideoSubtitleCache .
+func (d *Dao) DelVideoSubtitleCache(c context.Context, oid int64, tp int32) (err error) {
+	var (
+		key  = d.subtitleVideoKey(oid, tp)
+		conn = d.subtitleMc.Get(c)
+	)
+	defer conn.Close()
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("memcache.Delete(%s) error(%v)", key, err)
+		}
+	}
+	return
+}
+
+// DelSubtitleDraftCache .
+func (d *Dao) DelSubtitleDraftCache(c context.Context, oid int64, tp int32, mid int64, lan uint8) (err error) {
+	var (
+		key  = d.subtitleDraftKey(oid, tp, mid, lan)
+		conn = d.subtitleMc.Get(c)
+	)
+	defer conn.Close()
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("memcache.Delete(%s) error(%v)", key, err)
+		}
+	}
+	return
+}
+
+// DelSubtitleCache .
+func (d *Dao) DelSubtitleCache(c context.Context, oid int64, subtitleID int64) (err error) {
+	var (
+		key  = d.subtitleKey(oid, subtitleID)
+		conn = d.subtitleMc.Get(c)
+	)
+	defer conn.Close()
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("memcache.Delete(%s) error(%v)", key, err)
+		}
+	}
+	return
+}

app/job/main/dm2/dao/mc_subtitle_test.go

@@ -0,0 +1,96 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaosubtitleKey(t *testing.T) {
+	convey.Convey("subtitleKey", t, func(ctx convey.C) {
+		var (
+			oid        = int64(0)
+			subtitleID = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.subtitleKey(oid, subtitleID)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaosubtitleVideoKey(t *testing.T) {
+	convey.Convey("subtitleVideoKey", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.subtitleVideoKey(oid, tp)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaosubtitleDraftKey(t *testing.T) {
+	convey.Convey("subtitleDraftKey", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+			tp  = int32(0)
+			mid = int64(0)
+			lan = uint8(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.subtitleDraftKey(oid, tp, mid, lan)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelVideoSubtitleCache(t *testing.T) {
+	convey.Convey("DelVideoSubtitleCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			oid = int64(0)
+			tp  = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelVideoSubtitleCache(c, oid, tp)
+		})
+	})
+}
+
+func TestDaoDelSubtitleDraftCache(t *testing.T) {
+	convey.Convey("DelSubtitleDraftCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			oid = int64(0)
+			tp  = int32(0)
+			mid = int64(0)
+			lan = uint8(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelSubtitleDraftCache(c, oid, tp, mid, lan)
+		})
+	})
+}
+
+func TestDaoDelSubtitleCache(t *testing.T) {
+	convey.Convey("DelSubtitleCache", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			oid        = int64(0)
+			subtitleID = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelSubtitleCache(c, oid, subtitleID)
+		})
+	})
+}

app/job/main/dm2/dao/memcache.go

@@ -0,0 +1,280 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_prefixXML   = "dm_xml_"
+	_prefixSub   = "s_"
+	_prefixAjax  = "dm_ajax_"
+	_keyDuration = "d_" // video duration
+
+)
+
+func keyXML(oid int64) string {
+	return _prefixXML + strconv.FormatInt(oid, 10)
+}
+
+func keySubject(tp int32, oid int64) string {
+	return _prefixSub + fmt.Sprintf("%d_%d", tp, oid)
+}
+
+func keyAjax(oid int64) string {
+	return _prefixAjax + strconv.FormatInt(oid, 10)
+}
+
+// keyDuration return video duration key.
+func keyDuration(oid int64) string {
+	return _keyDuration + strconv.FormatInt(oid, 10)
+}
+
+func keyTransferLock() string {
+	return "dm_transfer_lock"
+}
+
+// DelXMLCache delete xml content.
+func (d *Dao) DelXMLCache(c context.Context, oid int64) (err error) {
+	conn := d.mc.Get(c)
+	key := keyXML(oid)
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("conn.Delete(%s) error(%v)", key, err)
+		}
+	}
+	conn.Close()
+	return
+}
+
+// AddXMLCache add xml content to memcache.
+func (d *Dao) AddXMLCache(c context.Context, oid int64, value []byte) (err error) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        keyXML(oid),
+		Value:      value,
+		Expiration: d.mcExpire,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%s) error(%v)", keyXML(oid), err)
+	}
+	return
+}
+
+// XMLCache get xml content.
+func (d *Dao) XMLCache(c context.Context, oid int64) (data []byte, err error) {
+	key := keyXML(oid)
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	item, err := conn.Get(key)
+	if err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("mc.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	data = item.Value
+	return
+}
+
+// SubjectCache get subject from memcache.
+func (d *Dao) SubjectCache(c context.Context, tp int32, oid int64) (sub *model.Subject, err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = keySubject(tp, oid)
+		rp   *memcache.Item
+	)
+	defer conn.Close()
+	if rp, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			sub = nil
+			err = nil
+		} else {
+			log.Error("mc.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	sub = &model.Subject{}
+	if err = conn.Scan(rp, &sub); err != nil {
+		log.Error("mc.Scan(%d) error(%v)", oid, err)
+	}
+	return
+}
+
+// SubjectsCache multi get subject from memcache.
+func (d *Dao) SubjectsCache(c context.Context, tp int32, oids []int64) (cached map[int64]*model.Subject, missed []int64, err error) {
+	var (
+		conn   = d.mc.Get(c)
+		keys   []string
+		oidMap = make(map[string]int64, len(oids))
+	)
+	cached = make(map[int64]*model.Subject, len(oids))
+	defer conn.Close()
+	for _, oid := range oids {
+		k := keySubject(tp, oid)
+		if _, ok := oidMap[k]; !ok {
+			keys = append(keys, k)
+			oidMap[k] = oid
+		}
+	}
+	rs, err := conn.GetMulti(keys)
+	if err != nil {
+		log.Error("conn.GetMulti(%v) error(%v)", keys, err)
+		return
+	}
+	for k, r := range rs {
+		sub := &model.Subject{}
+		if err = conn.Scan(r, sub); err != nil {
+			log.Error("conn.Scan(%s) error(%v)", r.Value, err)
+			err = nil
+			continue
+		}
+		cached[oidMap[k]] = sub
+		// delete hit key
+		delete(oidMap, k)
+	}
+	// missed key
+	missed = make([]int64, 0, len(oidMap))
+	for _, oid := range oidMap {
+		missed = append(missed, oid)
+	}
+	return
+}
+
+// AddSubjectCache add subject cache.
+func (d *Dao) AddSubjectCache(c context.Context, sub *model.Subject) (err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = keySubject(sub.Type, sub.Oid)
+	)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Object:     sub,
+		Flags:      memcache.FlagJSON,
+		Expiration: d.mcExpire,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%v) error(%v)", item, err)
+	}
+	return
+}
+
+// DelSubjectCache delete subject memcache cache.
+func (d *Dao) DelSubjectCache(c context.Context, tp int32, oid int64) (err error) {
+	conn := d.mc.Get(c)
+	key := keySubject(tp, oid)
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("conn.Delete(%s) error(%v)", key, err)
+		}
+	}
+	conn.Close()
+	return
+}
+
+// AddTransferLock 添加弹幕转移并发锁
+func (d *Dao) AddTransferLock(c context.Context) (succeed bool) {
+	var (
+		key  = keyTransferLock()
+		conn = d.mc.Get(c)
+	)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Value:      []byte("0"),
+		Expiration: 60,
+	}
+	if err := conn.Add(item); err != nil {
+		if err != memcache.ErrNotStored {
+			log.Error("conn.Add(%s) error(%v)", key, err)
+		}
+	} else {
+		succeed = true
+	}
+	return
+}
+
+// DelTransferLock 删除弹幕转移并发锁
+func (d *Dao) DelTransferLock(c context.Context) (err error) {
+	var (
+		key  = keyTransferLock()
+		conn = d.mc.Get(c)
+	)
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("conn.Delete(%s) error(%v)", key, err)
+		}
+	}
+	conn.Close()
+	return
+}
+
+// DelAjaxDMCache delete ajax dm from memcache.
+func (d *Dao) DelAjaxDMCache(c context.Context, oid int64) (err error) {
+	conn := d.mc.Get(c)
+	defer conn.Close()
+	key := keyAjax(oid)
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("DelAjaxDMCache.conn.Delete(%s) error(%v)", key, err)
+		}
+	}
+	return
+}
+
+// DurationCache return duration of video.
+func (d *Dao) DurationCache(c context.Context, oid int64) (duration int64, err error) {
+	var (
+		key  = keyDuration(oid)
+		conn = d.mc.Get(c)
+		item *memcache.Item
+	)
+	defer conn.Close()
+	if item, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			duration = model.NotFound
+			err = nil
+		} else {
+			log.Error("conn.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	if duration, err = strconv.ParseInt(string(item.Value), 10, 64); err != nil {
+		log.Error("strconv.ParseInt(%s) error(%v)", item.Value, err)
+	}
+	return
+}
+
+// SetDurationCache set video duration to redis.
+func (d *Dao) SetDurationCache(c context.Context, oid, duration int64) (err error) {
+	key := keyDuration(oid)
+	conn := d.mc.Get(c)
+	item := memcache.Item{
+		Key:        key,
+		Value:      []byte(fmt.Sprint(duration)),
+		Expiration: d.mcExpire,
+		Flags:      memcache.FlagRAW,
+	}
+	if err = conn.Set(&item); err != nil {
+		log.Error("mc.Set(%v) error(%v)", item, err)
+	}
+	conn.Close()
+	return
+}

app/job/main/dm2/dao/memcache_seg.go

@@ -0,0 +1,95 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_keySegMC = "sg_%d_%d_%d_%d"
+)
+
+func keySegMC(tp int32, oid, total, num int64) string {
+	return fmt.Sprintf(_keySegMC, tp, oid, total, num)
+}
+
+func keyXMLSeg(tp int32, oid, cnt, num int64) string {
+	return fmt.Sprintf("%d_%d_%d_%d", tp, oid, cnt, num)
+}
+
+// DelXMLSegCache delete segment xml content.
+func (d *Dao) DelXMLSegCache(c context.Context, tp int32, oid, cnt, num int64) (err error) {
+	conn := d.mc.Get(c)
+	key := keyXMLSeg(tp, oid, cnt, num)
+	if err = conn.Delete(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("conn.Delete(%s) error(%v)", key, err)
+		}
+	}
+	conn.Close()
+	return
+}
+
+// SetXMLSegCache set dm xml content into memcache.
+func (d *Dao) SetXMLSegCache(c context.Context, tp int32, oid, cnt, num int64, value []byte) (err error) {
+	key := keyXMLSeg(tp, oid, cnt, num)
+	conn := d.mc.Get(c)
+	item := memcache.Item{
+		Key:        key,
+		Value:      value,
+		Expiration: d.mcExpire,
+		Flags:      memcache.FlagRAW,
+	}
+	if err = conn.Set(&item); err != nil {
+		log.Error("mc.Set(%v) error(%v)", item, err)
+	}
+	conn.Close()
+	return
+}
+
+// SetDMSegCache set segment dm to cache.
+func (d *Dao) SetDMSegCache(c context.Context, tp int32, oid, total, num int64, dmSeg *model.DMSeg) (err error) {
+	key := keySegMC(tp, oid, total, num)
+	conn := d.dmSegMC.Get(c)
+	item := memcache.Item{
+		Key:        key,
+		Object:     dmSeg,
+		Expiration: d.mcExpire,
+		Flags:      memcache.FlagProtobuf | memcache.FlagGzip,
+	}
+	if err = conn.Set(&item); err != nil {
+		log.Error("conn.Set(%v) error(%v)", item, err)
+	}
+	conn.Close()
+	return
+}
+
+// DMSegCache dm segment pb cache.
+func (d *Dao) DMSegCache(c context.Context, tp int32, oid, total, num int64) (dmSeg *model.DMSeg, err error) {
+	var (
+		key  = keySegMC(tp, oid, total, num)
+		conn = d.dmSegMC.Get(c)
+		item *memcache.Item
+	)
+	dmSeg = new(model.DMSeg)
+	defer conn.Close()
+	if item, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+			dmSeg = nil
+		} else {
+			log.Error("mc.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	if err = conn.Scan(item, dmSeg); err != nil {
+		log.Error("conn.Scan() error(%v)", err)
+	}
+	return
+}

app/job/main/dm2/dao/memcache_seg_test.go

@@ -0,0 +1,35 @@
+package dao
+
+import (
+	"testing"
+
+	"go-common/app/job/main/dm2/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDelXMLSegCache(t *testing.T) {
+	Convey("check delete segment xml cache,error should be nil", t, func() {
+		err := testDao.DelXMLSegCache(c, 1, 1221, 1, 1)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestSetDMSegCache(t *testing.T) {
+	Convey("set dm segment cache, error should be nil", t, func() {
+		dmseg := new(model.DMSeg)
+		dmseg.Elems = append(dmseg.Elems, &model.Elem{Content: "dm msg"})
+		err := testDao.SetDMSegCache(c, 1, 1221, 1, 1, dmseg)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestDMSegCache(t *testing.T) {
+	Convey("get dm segment cache", t, func() {
+		dmseg, err := testDao.DMSegCache(c, 1, 1221, 1, 1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Logf("%+v", dmseg)
+	})
+}

app/job/main/dm2/dao/memcache_test.go

@@ -0,0 +1,194 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/job/main/dm2/model"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaokeyXML(t *testing.T) {
+	convey.Convey("keyXML", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyXML(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaokeySubject(t *testing.T) {
+	convey.Convey("keySubject", t, func(ctx convey.C) {
+		var (
+			tp  = int32(0)
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keySubject(tp, oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaokeyAjax(t *testing.T) {
+	convey.Convey("keyAjax", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyAjax(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaokeyTransferLock(t *testing.T) {
+	convey.Convey("keyTransferLock", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := keyTransferLock()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelXMLCache(t *testing.T) {
+	convey.Convey("DelXMLCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelXMLCache(c, oid)
+		})
+	})
+}
+
+func TestDaoAddXMLCache(t *testing.T) {
+	convey.Convey("AddXMLCache", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			oid   = int64(0)
+			value = []byte("")
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := testDao.AddXMLCache(c, oid, value)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoXMLCache(t *testing.T) {
+	convey.Convey("XMLCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.XMLCache(c, oid)
+		})
+	})
+}
+
+func TestDaoSubjectCache(t *testing.T) {
+	convey.Convey("SubjectCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			tp  = int32(0)
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.SubjectCache(c, tp, oid)
+		})
+	})
+}
+
+func TestDaoSubjectsCache(t *testing.T) {
+	convey.Convey("SubjectsCache", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			tp   = int32(0)
+			oids = []int64{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.SubjectsCache(c, tp, oids)
+		})
+	})
+}
+
+func TestDaoAddSubjectCache(t *testing.T) {
+	convey.Convey("AddSubjectCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			sub = &model.Subject{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := testDao.AddSubjectCache(c, sub)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelSubjectCache(t *testing.T) {
+	convey.Convey("DelSubjectCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			tp  = int32(0)
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelSubjectCache(c, tp, oid)
+		})
+	})
+}
+
+func TestDaoAddTransferLock(t *testing.T) {
+	convey.Convey("AddTransferLock", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			succeed := testDao.AddTransferLock(c)
+			ctx.Convey("Then succeed should not be nil.", func(ctx convey.C) {
+				ctx.So(succeed, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelTransferLock(t *testing.T) {
+	convey.Convey("DelTransferLock", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelTransferLock(c)
+		})
+	})
+}
+
+func TestDaoDelAjaxDMCache(t *testing.T) {
+	convey.Convey("DelAjaxDMCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DelAjaxDMCache(c, oid)
+		})
+	})
+}

app/job/main/dm2/dao/mysql.go

@@ -0,0 +1,502 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"go-common/library/xstr"
+)
+
+const (
+	_subtitleSharding = 100
+	_subjectSharding  = 100
+	_indexSharding    = 1000
+	_contentSharding  = 1000
+
+	_addSubjectSQL      = "INSERT INTO dm_subject_%02d(type,oid,pid,mid,maxlimit,attr) VALUES(?,?,?,?,?,?)"
+	_updateChildpoolSQL = "UPDATE dm_subject_%02d SET childpool=? WHERE type=? AND oid=?"
+	_updateSubMidSQL    = "UPDATE dm_subject_%02d SET mid=? WHERE type=? AND oid=?"
+	_updateSubAttrSQL   = "UPDATE dm_subject_%02d SET attr=? WHERE type=? AND oid=?"
+	_incrSubMCountSQL   = "UPDATE dm_subject_%02d SET mcount=mcount+1 WHERE type=? AND oid=?"
+	_incrSubCountSQL    = "UPDATE dm_subject_%02d SET acount=acount+?,count=count+?,childpool=? WHERE type=? AND oid=?"
+	_getSubjectSQL      = "SELECT id,type,oid,pid,mid,state,attr,acount,count,mcount,move_count,maxlimit,childpool,ctime,mtime FROM dm_subject_%02d WHERE type=? AND oid=?"
+	_addIndexSQL        = "INSERT INTO dm_index_%03d(id,type,oid,mid,progress,state,pool,attr,ctime) VALUES(?,?,?,?,?,?,?,?,?)"
+	_getIndexSQL        = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d WHERE type=? AND oid=? AND state IN(0,6)"
+	_idxSegIDSQL        = "SELECT id FROM dm_index_%03d WHERE type=? AND oid=? AND progress>=? AND progress<? AND state IN(0,6) AND pool = ? limit ?"
+	_idxSegSQL          = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d WHERE type=? AND oid=? AND state IN(0,6) AND progress>=? AND progress<? AND pool=? limit ?"
+	_idxIDSQL           = "SELECT id FROM dm_index_%03d WHERE type=? AND oid=? AND state IN(0,6) AND pool=?"
+	_idxsByidSQL        = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d WHERE id IN(%s)"
+	_idxsByPoolSQL      = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d WHERE type=? AND oid=? AND state IN(0,6) AND pool=?"
+	_addContentSQL      = "REPLACE INTO dm_content_%03d(dmid,fontsize,color,mode,ip,plat,msg,ctime) VALUES(?,?,?,?,?,?,?,?)"
+	_getContentsSQL     = "SELECT dmid,fontsize,color,mode,ip,plat,msg,ctime,mtime FROM dm_content_%03d WHERE dmid IN(%s)"
+	_getContentSQL      = "SELECT dmid,fontsize,color,mode,ip,plat,msg,ctime,mtime FROM dm_content_%03d WHERE dmid=?"
+	_addContentSpeSQL   = "REPLACE INTO dm_special_content(dmid,msg,ctime) VALUES(?,?,?)"
+	_getContentSpeSQL   = "SELECT dmid,msg,ctime,mtime FROM dm_special_content WHERE dmid=?"
+	_getContentsSpeSQL  = "SELECT dmid,msg,ctime,mtime FROM dm_special_content WHERE dmid IN(%s)"
+	//delete dm hide state
+	_delDMHideState = "UPDATE dm_index_%03d SET state=? WHERE oid=? AND id=? AND state=?"
+	// update subtitle upmid
+	_getSubtitle    = "SELECT id,oid,type,lan,status,mid,up_mid,subtitle_url,pub_time,reject_comment from subtitle_%02d WHERE id=?"
+	_getSubtitles   = "SELECT id,oid,type,lan,status,mid,up_mid,subtitle_url,pub_time,reject_comment from subtitle_%02d WHERE oid=? AND type=?"
+	_updateSubtitle = "UPDATE subtitle_%02d SET up_mid=?,status=?,pub_time=?,reject_comment=? WHERE id=?"
+	_addSubtitlePub = "INSERT INTO subtitle_pub(oid,type,lan,subtitle_id,is_delete) VALUES(?,?,?,?,?) ON DUPLICATE KEY UPDATE subtitle_id=?,is_delete=?"
+	// get mask mid
+	_getMaskMids = "SELECT mid from dm_mask_up where state=1"
+)
+
+func (d *Dao) hitSubject(oid int64) int64 {
+	return oid % _subjectSharding
+}
+
+func (d *Dao) hitIndex(oid int64) int64 {
+	return oid % _indexSharding
+}
+
+func (d *Dao) hitContent(oid int64) int64 {
+	return oid % _contentSharding
+}
+
+func (d *Dao) hitSubtile(oid int64) int64 {
+	return oid % _subtitleSharding
+}
+
+// AddSubject insert subject.
+func (d *Dao) AddSubject(c context.Context, tp int32, oid, pid, mid, maxlimit int64, attr int32) (lastID int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_addSubjectSQL, d.hitSubject(oid)), tp, oid, pid, mid, maxlimit, attr)
+	if err != nil {
+		log.Error("dmWriter.Exec(%d,%d,%d,%d,%d,%d) error(%v)", tp, oid, pid, mid, maxlimit, attr, err)
+		return
+	}
+	lastID, err = res.LastInsertId()
+	return
+}
+
+// UpdateSubAttr .
+func (d *Dao) UpdateSubAttr(c context.Context, tp int32, oid int64, attr int32) (affect int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_updateSubAttrSQL, d.hitSubject(oid)), attr, tp, oid)
+	if err != nil {
+		log.Error("dmWriter.Exec(%s,%d,%d) error(%v)", _updateSubMidSQL, oid, attr, err)
+		return
+	}
+	affect, err = res.RowsAffected()
+	return
+}
+
+// UpdateSubMid update mid in dm_subject.
+func (d *Dao) UpdateSubMid(c context.Context, tp int32, oid, mid int64) (affect int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_updateSubMidSQL, d.hitSubject(oid)), mid, tp, oid)
+	if err != nil {
+		log.Error("dmWriter.Exec(%s,%d,%d) error(%v)", _updateSubMidSQL, oid, mid, err)
+		return
+	}
+	affect, err = res.RowsAffected()
+	return
+}
+
+// Subject get subject info from db.
+func (d *Dao) Subject(c context.Context, tp int32, oid int64) (s *model.Subject, err error) {
+	s = &model.Subject{}
+	row := d.dmReader.QueryRow(c, fmt.Sprintf(_getSubjectSQL, d.hitSubject(oid)), tp, oid)
+	if err = row.Scan(&s.ID, &s.Type, &s.Oid, &s.Pid, &s.Mid, &s.State, &s.Attr, &s.ACount, &s.Count, &s.MCount, &s.MoveCnt, &s.Maxlimit, &s.Childpool, &s.Ctime, &s.Mtime); err != nil {
+		if err == sql.ErrNoRows {
+			s = nil
+			err = nil
+		} else {
+			log.Error("row.Scan() error(%v)", err)
+		}
+	}
+	return
+}
+
+// UpdateChildpool update childpool.
+func (d *Dao) UpdateChildpool(c context.Context, tp int32, oid int64, childpool int32) (affect int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_updateChildpoolSQL, d.hitSubject(oid)), childpool, tp, oid)
+	if err != nil {
+		log.Error("dmWriter.Exec(%s %d) error(%v)", _updateChildpoolSQL, oid, err)
+		return
+	}
+	affect, err = res.RowsAffected()
+	return
+}
+
+// TxIncrSubjectCount update acount,count,childpool of dm by transcation.
+func (d *Dao) TxIncrSubjectCount(tx *sql.Tx, tp int32, oid, acount, count int64, childpool int32) (affect int64, err error) {
+	res, err := tx.Exec(fmt.Sprintf(_incrSubCountSQL, d.hitSubject(oid)), acount, count, childpool, tp, oid)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// TxAddIndex add index of dm by transcation.
+func (d *Dao) TxAddIndex(tx *sql.Tx, m *model.DM) (id int64, err error) {
+	res, err := tx.Exec(fmt.Sprintf(_addIndexSQL, d.hitIndex(m.Oid)), m.ID, m.Type, m.Oid, m.Mid, m.Progress, m.State, m.Pool, m.Attr, m.Ctime)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.LastInsertId()
+}
+
+// Indexs get dm index by type and oid.
+func (d *Dao) Indexs(c context.Context, tp int32, oid int64) (idxMap map[int64]*model.DM, dmids, special []int64, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_getIndexSQL, d.hitIndex(oid)), tp, oid)
+	if err != nil {
+		log.Error("dmReader.Query(%d,%d) error(%v)", tp, oid, err)
+		return
+	}
+	defer rows.Close()
+	idxMap = make(map[int64]*model.DM)
+	for rows.Next() {
+		idx := &model.DM{}
+		if err = rows.Scan(&idx.ID, &idx.Type, &idx.Oid, &idx.Mid, &idx.Progress, &idx.State, &idx.Pool, &idx.Attr, &idx.Ctime, &idx.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			return
+		}
+		idxMap[idx.ID] = idx
+		dmids = append(dmids, idx.ID)
+		if idx.Pool == model.PoolSpecial {
+			special = append(special, idx.ID)
+		}
+	}
+	err = rows.Err()
+	return
+}
+
+// IndexsSeg get segment index info from db by ps and pe.
+func (d *Dao) IndexsSeg(c context.Context, tp int32, oid, ps, pe, limit int64, pool int32) (res []*model.DM, dmids []int64, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_idxSegSQL, d.hitIndex(oid)), tp, oid, ps, pe, pool, limit)
+	if err != nil {
+		log.Error("db.Query(%d %d %d %d) error(%v)", tp, oid, ps, pe, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		dm := &model.DM{}
+		if err = rows.Scan(&dm.ID, &dm.Type, &dm.Oid, &dm.Mid, &dm.Progress, &dm.State, &dm.Pool, &dm.Attr, &dm.Ctime, &dm.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, dm)
+		dmids = append(dmids, dm.ID)
+	}
+	return
+}
+
+// IndexsSegID get segment dmids.
+func (d *Dao) IndexsSegID(c context.Context, tp int32, oid, ps, pe, limit int64, pool int32) (dmids []int64, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_idxSegIDSQL, d.hitIndex(oid)), tp, oid, ps, pe, pool, limit)
+	if err != nil {
+		log.Error("db.Query() error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var dmid int64
+	for rows.Next() {
+		if err = rows.Scan(&dmid); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		dmids = append(dmids, dmid)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// IndexsID get dmids.
+func (d *Dao) IndexsID(c context.Context, tp int32, oid int64, pool int32) (dmids []int64, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_idxIDSQL, d.hitIndex(oid)), tp, oid, pool)
+	if err != nil {
+		log.Error("db.Query() error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	var dmid int64
+	for rows.Next() {
+		if err = rows.Scan(&dmid); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		dmids = append(dmids, dmid)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// IndexsByid get dm index by dmids.
+func (d *Dao) IndexsByid(c context.Context, tp int32, oid int64, dmids []int64) (idxMap map[int64]*model.DM, special []int64, err error) {
+	query := fmt.Sprintf(_idxsByidSQL, d.hitIndex(oid), xstr.JoinInts(dmids))
+	rows, err := d.dmReader.Query(c, query)
+	if err != nil {
+		log.Error("db.Query(%s) error(%v)", query, err)
+		return
+	}
+	defer rows.Close()
+	idxMap = make(map[int64]*model.DM)
+	for rows.Next() {
+		idx := new(model.DM)
+		if err = rows.Scan(&idx.ID, &idx.Type, &idx.Oid, &idx.Mid, &idx.Progress, &idx.State, &idx.Pool, &idx.Attr, &idx.Ctime, &idx.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			return
+		}
+		idxMap[idx.ID] = idx
+		if idx.Pool == model.PoolSpecial {
+			special = append(special, idx.ID)
+		}
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// IndexsByPool get dm index by type,oid and pool.
+func (d *Dao) IndexsByPool(c context.Context, tp int32, oid int64, pool int32) (dms []*model.DM, dmids []int64, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_idxsByPoolSQL, d.hitIndex(oid)), tp, oid, pool)
+	if err != nil {
+		log.Error("dmReader.Query(tp:%v,oid:%v) error(%v)", tp, oid, err)
+		return
+	}
+	defer rows.Close()
+	dms = make([]*model.DM, 0, 100)
+	for rows.Next() {
+		dm := &model.DM{}
+		if err = rows.Scan(&dm.ID, &dm.Type, &dm.Oid, &dm.Mid, &dm.Progress, &dm.State, &dm.Pool, &dm.Attr, &dm.Ctime, &dm.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			return
+		}
+		dms = append(dms, dm)
+		dmids = append(dmids, dm.ID)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// TxAddContent add content of dm by transcation.
+func (d *Dao) TxAddContent(tx *sql.Tx, oid int64, m *model.Content) (id int64, err error) {
+	res, err := tx.Exec(fmt.Sprintf(_addContentSQL, d.hitContent(oid)), m.ID, m.FontSize, m.Color, m.Mode, m.IP, m.Plat, m.Msg, m.Ctime)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.LastInsertId()
+}
+
+// TxAddContentSpecial add special dm by transcation.
+func (d *Dao) TxAddContentSpecial(tx *sql.Tx, m *model.ContentSpecial) (id int64, err error) {
+	res, err := tx.Exec(_addContentSpeSQL, m.ID, m.Msg, m.Ctime)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.LastInsertId()
+}
+
+// Content dm content by dmid
+func (d *Dao) Content(c context.Context, oid, dmid int64) (ct *model.Content, err error) {
+	ct = &model.Content{}
+	row := d.dmReader.QueryRow(c, fmt.Sprintf(_getContentSQL, d.hitContent(oid)), dmid)
+	if err = row.Scan(&ct.ID, &ct.FontSize, &ct.Color, &ct.Mode, &ct.IP, &ct.Plat, &ct.Msg, &ct.Ctime, &ct.Mtime); err != nil {
+		ct = nil
+		log.Error("row.Scan() error(%v)", err)
+	}
+	return
+}
+
+// Contents multi get dm content by dmids.
+func (d *Dao) Contents(c context.Context, oid int64, dmids []int64) (ctsMap map[int64]*model.Content, err error) {
+	var (
+		wg   errgroup.Group
+		lock sync.Mutex
+	)
+	ctsMap = make(map[int64]*model.Content)
+	pageNum := len(dmids) / d.pageSize
+	if len(dmids)%d.pageSize > 0 {
+		pageNum = pageNum + 1
+	}
+	for i := 0; i < pageNum; i++ {
+		start := i * d.pageSize
+		end := (i + 1) * d.pageSize
+		if end > len(dmids) {
+			end = len(dmids)
+		}
+		wg.Go(func() (err error) {
+			rows, err := d.dmReader.Query(c, fmt.Sprintf(_getContentsSQL, d.hitContent(oid), xstr.JoinInts(dmids[start:end])))
+			if err != nil {
+				log.Error("db.Query(%s) error(%v)", fmt.Sprintf(_getContentsSQL, d.hitContent(oid), xstr.JoinInts(dmids)), err)
+				return
+			}
+			defer rows.Close()
+			for rows.Next() {
+				ct := &model.Content{}
+				if err = rows.Scan(&ct.ID, &ct.FontSize, &ct.Color, &ct.Mode, &ct.IP, &ct.Plat, &ct.Msg, &ct.Ctime, &ct.Mtime); err != nil {
+					log.Error("rows.Scan() error(%v)", err)
+					return
+				}
+				lock.Lock()
+				ctsMap[ct.ID] = ct
+				lock.Unlock()
+			}
+			err = rows.Err()
+			return
+		})
+	}
+	if err = wg.Wait(); err != nil {
+		log.Error("wg.Wait() error(%v)", err)
+	}
+	return
+}
+
+// ContentsSpecial multi get special dm content by dmids.
+func (d *Dao) ContentsSpecial(c context.Context, dmids []int64) (res map[int64]*model.ContentSpecial, err error) {
+	res = make(map[int64]*model.ContentSpecial, len(dmids))
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_getContentsSpeSQL, xstr.JoinInts(dmids)))
+	if err != nil {
+		log.Error("db.Query() error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		content := &model.ContentSpecial{}
+		if err = rows.Scan(&content.ID, &content.Msg, &content.Ctime, &content.Mtime); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		res[content.ID] = content
+	}
+	return
+}
+
+// ContentSpecial get special dm content by dmids.
+func (d *Dao) ContentSpecial(c context.Context, dmid int64) (contentSpe *model.ContentSpecial, err error) {
+	contentSpe = &model.ContentSpecial{}
+	row := d.dmReader.QueryRow(c, _getContentSpeSQL, dmid)
+	if err = row.Scan(&contentSpe.ID, &contentSpe.Msg, &contentSpe.Ctime, &contentSpe.Mtime); err != nil {
+		log.Error("rows.Scan() error(%v)", err)
+	}
+	return
+}
+
+// DelDMHideState del dm hide state
+func (d *Dao) DelDMHideState(c context.Context, tp int32, oid int64, dmid int64) (affect int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_delDMHideState, d.hitIndex(oid)), model.StateNormal, oid, dmid, model.StateHide)
+	if err != nil {
+		log.Error("dmWriter.Exec(%s %d dmid=%d) error(%v)", _delDMHideState, oid, dmid, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// TxIncrSubMCount update monitor dm count.
+func (d *Dao) TxIncrSubMCount(tx *sql.Tx, tp int32, oid int64) (affect int64, err error) {
+	res, err := tx.Exec(fmt.Sprintf(_incrSubMCountSQL, d.hitSubject(oid)), tp, oid)
+	if err != nil {
+		log.Error("tx.Exec(%s,%d,%d) error(%v)", _incrSubMCountSQL, tp, oid, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// UpdateSubtitle update subtitle mid
+func (d *Dao) UpdateSubtitle(c context.Context, subtitle *model.Subtitle) (err error) {
+	if _, err = d.biliDMWriter.Exec(c, fmt.Sprintf(_updateSubtitle, d.hitSubtile(subtitle.Oid)), subtitle.UpMid, subtitle.Status,
+		subtitle.PubTime, subtitle.RejectComment, subtitle.ID); err != nil {
+		log.Error("biliDMWriter.Exec(query:%v,subtitle:%+v) error(%v)", _updateSubtitle, subtitle, err)
+		return
+	}
+	return
+}
+
+// GetSubtitles .
+func (d *Dao) GetSubtitles(c context.Context, tp int32, oid int64) (subtitles []*model.Subtitle, err error) {
+	rows, err := d.biliDMWriter.Query(c, fmt.Sprintf(_getSubtitles, d.hitSubtile(oid)), oid, tp)
+	if err != nil {
+		log.Error("biliDMWriter.Query(%s,%d,%d) error(%v)", _getSubtitles, oid, tp, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var subtitle = &model.Subtitle{}
+		if err = rows.Scan(&subtitle.ID, &subtitle.Oid, &subtitle.Type, &subtitle.Lan, &subtitle.Status, &subtitle.Mid, &subtitle.UpMid,
+			&subtitle.SubtitleURL, &subtitle.PubTime, &subtitle.RejectComment); err != nil {
+			log.Error("biliDMWriter.Scan(%s,%d,%d) error(%v)", _getSubtitles, oid, tp, err)
+			return
+		}
+		subtitles = append(subtitles, subtitle)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("biliDMWriter.rows.Err()(%s,%d,%d) error(%v)", _getSubtitles, oid, tp, err)
+		return
+	}
+	return
+}
+
+// GetSubtitle .
+func (d *Dao) GetSubtitle(c context.Context, oid int64, subtitleID int64) (subtitle *model.Subtitle, err error) {
+	subtitle = &model.Subtitle{}
+	row := d.biliDMWriter.QueryRow(c, fmt.Sprintf(_getSubtitle, d.hitSubtile(oid)), subtitleID)
+	if err = row.Scan(&subtitle.ID, &subtitle.Oid, &subtitle.Type, &subtitle.Lan, &subtitle.Status, &subtitle.Mid, &subtitle.UpMid,
+		&subtitle.SubtitleURL, &subtitle.PubTime, &subtitle.RejectComment); err != nil {
+		if err == sql.ErrNoRows {
+			subtitle = nil
+			err = nil
+		} else {
+			log.Error("row.Scan() error(%v)", err)
+		}
+	}
+	return
+}
+
+// TxUpdateSubtitle .
+func (d *Dao) TxUpdateSubtitle(tx *sql.Tx, subtitle *model.Subtitle) (err error) {
+	if _, err = tx.Exec(fmt.Sprintf(_updateSubtitle, d.hitSubtile(subtitle.Oid)), subtitle.UpMid, subtitle.Status,
+		subtitle.PubTime, subtitle.RejectComment, subtitle.ID); err != nil {
+		log.Error("params(%+v),error(%v)", subtitle, err)
+		return
+	}
+	return
+}
+
+// TxAddSubtitlePub .
+func (d *Dao) TxAddSubtitlePub(tx *sql.Tx, subtitlePub *model.SubtitlePub) (err error) {
+	if _, err = tx.Exec(_addSubtitlePub, subtitlePub.Oid, subtitlePub.Type, subtitlePub.Lan, subtitlePub.SubtitleID, subtitlePub.IsDelete, subtitlePub.SubtitleID, subtitlePub.IsDelete); err != nil {
+		log.Error("params(%+v),error(%v)", subtitlePub, err)
+		return
+	}
+	return
+}
+
+// MaskMids get mask mids from db.
+func (d *Dao) MaskMids(c context.Context) (mids []int64, err error) {
+	mids = make([]int64, 0, 100)
+	rows, err := d.biliDMWriter.Query(c, _getMaskMids)
+	if err != nil {
+		log.Error("biliDMWriter.Query(%s) error(%v)", _getMaskMids, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var mid int64
+		if err = rows.Scan(&mid); err != nil {
+			log.Error("biliDMWriter.Scan(%s) error(%v)", _getMaskMids, err)
+			return
+		}
+		mids = append(mids, mid)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("biliDMWriter.rows.Err() error(%v)", err)
+	}
+	return
+}

app/job/main/dm2/dao/mysql_dm_special.go

@@ -0,0 +1,39 @@
+package dao
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_addDmSpecialLocationSQL = "INSERT INTO dm_special_content_location (type,oid,locations,ctime) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE locations=?"
+	_getDmSpecialLocationSQL = "SELECT locations FROM dm_special_content_location WHERE oid=? AND type=?"
+)
+
+// UpsertDmSpecialLocation .
+func (d *Dao) UpsertDmSpecialLocation(c context.Context, tp int32, oid int64, locations string) (err error) {
+	if _, err = d.dmWriter.Exec(c, _addDmSpecialLocationSQL, tp, oid, locations, time.Now(), locations); err != nil {
+		log.Error("AddDmSpecialLocation.Exec error(%v)", err)
+	}
+	return
+}
+
+// DMSpecialLocations .
+func (d *Dao) DMSpecialLocations(c context.Context, tp int32, oid int64) (locations []string, err error) {
+	row := d.dmReader.QueryRow(c, _getDmSpecialLocationSQL, oid, tp)
+	var s string
+	if err = row.Scan(&s); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("DMSpecialLocations.Query error(%v)", err)
+		}
+		return
+	}
+	locations = strings.Split(s, ",")
+	return
+}

app/job/main/dm2/dao/mysql_dm_special_test.go

@@ -0,0 +1,38 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoUpsertDmSpecialLocation(t *testing.T) {
+	convey.Convey("UpsertDmSpecialLocation", t, func(ctx convey.C) {
+		var (
+			c         = context.Background()
+			tp        = int32(0)
+			oid       = int64(0)
+			locations = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := testDao.UpsertDmSpecialLocation(c, tp, oid, locations)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDMSpecialLocations(t *testing.T) {
+	convey.Convey("DMSpecialLocations", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			tp  = int32(1)
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.DMSpecialLocations(c, tp, oid)
+		})
+	})
+}

app/job/main/dm2/dao/mysql_test.go

@@ -0,0 +1,522 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/job/main/dm2/model"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaohitSubject(t *testing.T) {
+	convey.Convey("hitSubject", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.hitSubject(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaohitIndex(t *testing.T) {
+	convey.Convey("hitIndex", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.hitIndex(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaohitContent(t *testing.T) {
+	convey.Convey("hitContent", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.hitContent(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaohitSubtile(t *testing.T) {
+	convey.Convey("hitSubtile", t, func(ctx convey.C) {
+		var (
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.hitSubtile(oid)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoAddSubject(t *testing.T) {
+	convey.Convey("AddSubject", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			tp       = int32(0)
+			oid      = int64(0)
+			pid      = int64(0)
+			mid      = int64(0)
+			maxlimit = int64(0)
+			attr     = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.AddSubject(c, tp, oid, pid, mid, maxlimit, attr)
+		})
+	})
+}
+
+func TestDaoUpdateSubAttr(t *testing.T) {
+	convey.Convey("UpdateSubAttr", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			tp   = int32(0)
+			oid  = int64(0)
+			attr = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affect, err := testDao.UpdateSubAttr(c, tp, oid, attr)
+			ctx.Convey("Then err should be nil.affect should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affect, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoUpdateSubMid(t *testing.T) {
+	convey.Convey("UpdateSubMid", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			tp  = int32(0)
+			oid = int64(0)
+			mid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affect, err := testDao.UpdateSubMid(c, tp, oid, mid)
+			ctx.Convey("Then err should be nil.affect should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affect, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSubject(t *testing.T) {
+	convey.Convey("Subject", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			tp  = int32(0)
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			s, err := testDao.Subject(c, tp, oid)
+			ctx.Convey("Then err should be nil.s should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(s, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoUpdateChildpool(t *testing.T) {
+	convey.Convey("UpdateChildpool", t, func(ctx convey.C) {
+		var (
+			c         = context.Background()
+			tp        = int32(0)
+			oid       = int64(0)
+			childpool = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affect, err := testDao.UpdateChildpool(c, tp, oid, childpool)
+			ctx.Convey("Then err should be nil.affect should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affect, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoTxIncrSubjectCount(t *testing.T) {
+	convey.Convey("TxIncrSubjectCount", t, func(ctx convey.C) {
+		var (
+			tx, _     = testDao.BeginTran(c)
+			tp        = int32(0)
+			oid       = int64(0)
+			acount    = int64(0)
+			count     = int64(0)
+			childpool = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affect, err := testDao.TxIncrSubjectCount(tx, tp, oid, acount, count, childpool)
+			ctx.Convey("Then err should be nil.affect should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affect, convey.ShouldNotBeNil)
+			})
+		})
+		ctx.Reset(func() {
+			tx.Commit()
+		})
+	})
+}
+
+func TestDaoTxAddIndex(t *testing.T) {
+	convey.Convey("TxAddIndex", t, func(ctx convey.C) {
+		var (
+			tx, _ = testDao.BeginTran(c)
+			m     = &model.DM{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			id, err := testDao.TxAddIndex(tx, m)
+			ctx.Convey("Then err should be nil.id should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(id, convey.ShouldNotBeNil)
+			})
+		})
+		ctx.Reset(func() {
+			tx.Commit()
+		})
+	})
+}
+
+func TestDaoIndexs(t *testing.T) {
+	convey.Convey("Indexs", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			tp  = int32(0)
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.Indexs(c, tp, oid)
+		})
+	})
+}
+
+func TestDaoIndexsSeg(t *testing.T) {
+	convey.Convey("IndexsSeg", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			tp    = int32(0)
+			oid   = int64(0)
+			ps    = int64(0)
+			pe    = int64(0)
+			limit = int64(0)
+			pool  = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.IndexsSeg(c, tp, oid, ps, pe, limit, pool)
+		})
+	})
+}
+
+func TestDaoIndexsSegID(t *testing.T) {
+	convey.Convey("IndexsSegID", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			tp    = int32(0)
+			oid   = int64(0)
+			ps    = int64(0)
+			pe    = int64(0)
+			limit = int64(0)
+			pool  = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.IndexsSegID(c, tp, oid, ps, pe, limit, pool)
+		})
+	})
+}
+
+func TestDaoIndexsID(t *testing.T) {
+	convey.Convey("IndexsID", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			tp   = int32(0)
+			oid  = int64(0)
+			pool = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			dmids, err := testDao.IndexsID(c, tp, oid, pool)
+			ctx.Convey("Then err should be nil.dmids should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(dmids, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoIndexsByid(t *testing.T) {
+	convey.Convey("IndexsByid", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			tp    = int32(0)
+			oid   = int64(0)
+			dmids = []int64{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.IndexsByid(c, tp, oid, dmids)
+		})
+	})
+}
+
+func TestDaoIndexsByPool(t *testing.T) {
+	convey.Convey("IndexsByPool", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			tp   = int32(0)
+			oid  = int64(0)
+			pool = int32(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			dms, dmids, err := testDao.IndexsByPool(c, tp, oid, pool)
+			ctx.Convey("Then err should be nil.dms,dmids should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(dmids, convey.ShouldNotBeNil)
+				ctx.So(dms, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoTxAddContent(t *testing.T) {
+	convey.Convey("TxAddContent", t, func(ctx convey.C) {
+		var (
+			tx, _ = testDao.BeginTran(c)
+			oid   = int64(0)
+			m     = &model.Content{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			id, err := testDao.TxAddContent(tx, oid, m)
+			ctx.Convey("Then err should be nil.id should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(id, convey.ShouldNotBeNil)
+			})
+		})
+		ctx.Reset(func() {
+			tx.Commit()
+		})
+	})
+}
+
+func TestDaoTxAddContentSpecial(t *testing.T) {
+	convey.Convey("TxAddContentSpecial", t, func(ctx convey.C) {
+		var (
+			tx, _ = testDao.BeginTran(c)
+			m     = &model.ContentSpecial{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			id, err := testDao.TxAddContentSpecial(tx, m)
+			ctx.Convey("Then err should be nil.id should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(id, convey.ShouldNotBeNil)
+			})
+		})
+		ctx.Reset(func() {
+			tx.Commit()
+		})
+	})
+}
+
+func TestDaoContent(t *testing.T) {
+	convey.Convey("Content", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			oid  = int64(0)
+			dmid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ct, err := testDao.Content(c, oid, dmid)
+			ctx.Convey("Then err should be nil.ct should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ct, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoContents(t *testing.T) {
+	convey.Convey("Contents", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			oid   = int64(0)
+			dmids = []int64{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ctsMap, err := testDao.Contents(c, oid, dmids)
+			ctx.Convey("Then err should be nil.ctsMap should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ctsMap, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoContentsSpecial(t *testing.T) {
+	convey.Convey("ContentsSpecial", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			dmids = []int64{123}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.ContentsSpecial(c, dmids)
+		})
+	})
+}
+
+func TestDaoContentSpecial(t *testing.T) {
+	convey.Convey("ContentSpecial", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			dmid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			contentSpe, err := testDao.ContentSpecial(c, dmid)
+			ctx.Convey("Then err should be nil.contentSpe should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(contentSpe, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelDMHideState(t *testing.T) {
+	convey.Convey("DelDMHideState", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			tp   = int32(0)
+			oid  = int64(0)
+			dmid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affect, err := testDao.DelDMHideState(c, tp, oid, dmid)
+			ctx.Convey("Then err should be nil.affect should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affect, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoTxIncrSubMCount(t *testing.T) {
+	convey.Convey("TxIncrSubMCount", t, func(ctx convey.C) {
+		var (
+			tx, _ = testDao.BeginTran(c)
+			tp    = int32(0)
+			oid   = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affect, err := testDao.TxIncrSubMCount(tx, tp, oid)
+			ctx.Convey("Then err should be nil.affect should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affect, convey.ShouldNotBeNil)
+			})
+		})
+		ctx.Reset(func() {
+			tx.Commit()
+		})
+	})
+}
+
+func TestDaoUpdateSubtitle(t *testing.T) {
+	convey.Convey("UpdateSubtitle", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			subtitle = &model.Subtitle{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := testDao.UpdateSubtitle(c, subtitle)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetSubtitles(t *testing.T) {
+	convey.Convey("GetSubtitles", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			tp  = int32(0)
+			oid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			subtitles, err := testDao.GetSubtitles(c, tp, oid)
+			ctx.Convey("Then err should be nil.subtitles should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(subtitles, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetSubtitle(t *testing.T) {
+	convey.Convey("GetSubtitle", t, func(ctx convey.C) {
+		var (
+			c          = context.Background()
+			oid        = int64(0)
+			subtitleID = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.GetSubtitle(c, oid, subtitleID)
+		})
+	})
+}
+
+func TestDaoTxUpdateSubtitle(t *testing.T) {
+	convey.Convey("TxUpdateSubtitle", t, func(ctx convey.C) {
+		var (
+			tx, _    = testDao.BeginTran(c)
+			subtitle = &model.Subtitle{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.TxUpdateSubtitle(tx, subtitle)
+		})
+		ctx.Reset(func() {
+			tx.Commit()
+		})
+	})
+}
+
+func TestDaoTxAddSubtitlePub(t *testing.T) {
+	convey.Convey("TxAddSubtitlePub", t, func(ctx convey.C) {
+		var (
+			tx, _       = testDao.BeginTran(c)
+			subtitlePub = &model.SubtitlePub{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.TxAddSubtitlePub(tx, subtitlePub)
+		})
+		ctx.Reset(func() {
+			tx.Commit()
+		})
+	})
+}
+
+func TestDaoMaskMids(t *testing.T) {
+	convey.Convey("MaskMids", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			mids, err := testDao.MaskMids(c)
+			ctx.Convey("Then err should be nil.mids should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(mids, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/dm2/dao/rank_list.go

@@ -0,0 +1,21 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/log"
+)
+
+const (
+	_dataRankURI = "/data/rank/recent_region-%d-%d.json"
+)
+
+// RankList get data rank by tid
+func (d *Dao) RankList(c context.Context, tid int64, day int32) (resp *model.RankRecentResp, err error) {
+	if err = d.httpCli.RESTfulGet(c, d.conf.Host.DataRank+_dataRankURI, "", nil, &resp, tid, day); err != nil {
+		log.Error("RankList(tid:%v,day:%v),error(%v)", tid, day, err)
+		return
+	}
+	return
+}

app/job/main/dm2/dao/rank_list_test.go

@@ -0,0 +1,20 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestRankList(t *testing.T) {
+	Convey("", t, func() {
+		res, err := testDao.RankList(context.Background(), 185, 3)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+		t.Log(res)
+		for _, v := range res.List {
+			t.Log(v)
+		}
+	})
+}

app/job/main/dm2/dao/redis.go

@@ -0,0 +1,146 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	// dm xml list v1
+	_prefixDM = "dm_v1_%d_%d" // dm_v1_tpe_oid
+	divide    = 34359738368   // 2^35
+)
+
+func keyDM(tp int32, oid int64) (key string) {
+	return fmt.Sprintf(_prefixDM, tp, oid)
+}
+
+// 弹幕在redis sortset 中的score
+// 通过score保证弹幕在缓存中的排序为:普通弹幕、普通弹幕中的保护弹幕、字幕弹幕、脚本弹幕
+func score(dm *model.DM) (score float64) {
+	// NOTE redis score最多17位表示，这里采用整数十位+小数部分十位
+	v := dm.ID / divide                                // 2^63 / 2^35 = 2^28-1 整数部分最大值：268435455
+	k := dm.ID % divide                                // 精度8位，最后5位可忽略
+	r := int64(dm.Pool)<<29 | int64(dm.Attr)&1<<28 | v // NOTE v should less than 2^28
+	score, _ = strconv.ParseFloat(fmt.Sprintf("%d.%d", r, k), 64)
+	return
+}
+
+// AddDMCache add dm to redis.
+func (d *Dao) AddDMCache(c context.Context, dm *model.DM) (err error) {
+	var (
+		conn  = d.dmRds.Get(c)
+		value []byte
+		key   = keyDM(dm.Type, dm.Oid)
+	)
+	defer conn.Close()
+	if value, err = dm.Marshal(); err != nil {
+		log.Error("dm.Marshal(%v) error(%v)", dm, err)
+		return
+	}
+	if err = conn.Send("ZADD", key, score(dm), value); err != nil {
+		log.Error("conn.Send(ZADD %v) error(%v)", dm, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.dmRdsExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// SetDMCache flush dm list to redis.
+func (d *Dao) SetDMCache(c context.Context, tp int32, oid int64, dms []*model.DM) (err error) {
+	var (
+		value []byte
+		conn  = d.dmRds.Get(c)
+		key   = keyDM(tp, oid)
+	)
+	defer conn.Close()
+	for _, dm := range dms {
+		if value, err = dm.Marshal(); err != nil {
+			log.Error("dm.Marshal(%v) error(%v)", dm, err)
+			return
+		}
+		if err = conn.Send("ZADD", key, score(dm), value); err != nil {
+			log.Error("conn.Send(ZADD %v) error(%v)", dm, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.dmRdsExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < len(dms)+1; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// DelDMCache delete redis cache of oid.
+func (d *Dao) DelDMCache(c context.Context, tp int32, oid int64) (err error) {
+	var (
+		key  = keyDM(tp, oid)
+		conn = d.dmRds.Get(c)
+	)
+	if _, err = conn.Do("DEL", key); err != nil {
+		log.Error("conn.Do(DEL %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// ExpireDMCache expire dm.
+func (d *Dao) ExpireDMCache(c context.Context, tp int32, oid int64) (ok bool, err error) {
+	key := keyDM(tp, oid)
+	conn := d.dmRds.Get(c)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.dmRdsExpire)); err != nil {
+		log.Error("conn.Do(EXPIRE %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// DMCache 获取redis列表中的弹幕.
+func (d *Dao) DMCache(c context.Context, tp int32, oid int64) (res [][]byte, err error) {
+	conn := d.dmRds.Get(c)
+	key := keyDM(tp, oid)
+	if res, err = redis.ByteSlices(conn.Do("ZRANGE", key, 0, -1)); err != nil {
+		log.Error("conn.Do(ZRANGE %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// TrimDMCache 从redis列表中pop掉count条弹幕.
+func (d *Dao) TrimDMCache(c context.Context, tp int32, oid, count int64) (err error) {
+	conn := d.dmRds.Get(c)
+	key := keyDM(tp, oid)
+	if _, err = conn.Do("ZREMRANGEBYRANK", key, 0, count-1); err != nil {
+		log.Error("conn.Do(ZREMRANGEBYRANK %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}

app/job/main/dm2/dao/redis_mask.go

@@ -0,0 +1,64 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_maskJobKey = "mask_job"
+)
+
+// SetnxMaskJob setnx mask_job value
+func (d *Dao) SetnxMaskJob(c context.Context, value string) (ok bool, err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if ok, err = redis.Bool(conn.Do("SETNX", _maskJobKey, value)); err != nil {
+		log.Error("d.SetnxMask(value:%s),error(%v)", value, err)
+		return
+	}
+	return
+}
+
+// GetMaskJob .
+func (d *Dao) GetMaskJob(c context.Context) (value string, err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if value, err = redis.String(conn.Do("GET", _maskJobKey)); err != nil {
+		log.Error("d.GetMaskJob,error(%v)", err)
+		return
+	}
+	return
+}
+
+// GetSetMaskJob .
+func (d *Dao) GetSetMaskJob(c context.Context, value string) (old string, err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if old, err = redis.String(conn.Do("GETSET", _maskJobKey, value)); err != nil {
+		log.Error("d.GetSetMaskJob(value:%s),error(%v)", value, err)
+		return
+	}
+	return
+}
+
+// DelMaskJob .
+func (d *Dao) DelMaskJob(c context.Context) (err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("DEL", _maskJobKey); err != nil {
+		log.Error("d.DelMaskJob,error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/dm2/dao/redis_mask_test.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestSetnxMaskJob(t *testing.T) {
+	Convey("", t, func() {
+		ok, err := testDao.SetnxMaskJob(context.Background(), time.Now().String())
+		So(err, ShouldBeNil)
+		t.Logf("ok:%v", ok)
+	})
+}
+
+func TestGetMaskJob(t *testing.T) {
+	Convey("", t, func() {
+		value, err := testDao.GetMaskJob(context.Background())
+		So(err, ShouldBeNil)
+		t.Logf("ok:%v", value)
+	})
+}
+
+func TestGetSetMaskJob(t *testing.T) {
+	Convey("", t, func() {
+		value, err := testDao.GetSetMaskJob(context.Background(), time.Now().String())
+		So(err, ShouldBeNil)
+		t.Logf("ok:%v", value)
+	})
+}
+
+func TestDelMaskJob(t *testing.T) {
+	Convey("", t, func() {
+		err := testDao.DelMaskJob(context.Background())
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/dm2/dao/redis_rct.go

@@ -0,0 +1,89 @@
+package dao
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_prefixRecent = "dm_rct_"
+)
+
+func keyRecent(mid int64) string {
+	return _prefixRecent + strconv.FormatInt(mid, 10)
+}
+
+// AddRecentDM add recent dm of up to redis.
+func (d *Dao) AddRecentDM(c context.Context, mid int64, dm *model.DM) (count int64, err error) {
+	var (
+		conn  = d.dmRctRds.Get(c)
+		key   = keyRecent(mid)
+		value []byte
+	)
+	defer conn.Close()
+	if value, err = dm.Marshal(); err != nil {
+		log.Error("dm.Marshal(%v) error(%v)", dm, err)
+		return
+	}
+	if err = conn.Send("ZREMRANGEBYSCORE", key, dm.ID, dm.ID); err != nil {
+		log.Error("conn.Do(ZREMRANGEBYSCORE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Send("ZADD", key, dm.ID, value); err != nil {
+		log.Error("conn.Send(ZADD %v) error(%v)", dm, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.dmRctExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Send("ZCARD", key); err != nil {
+		log.Error("conn.Send(ZCARD %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < 3; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	if count, err = redis.Int64(conn.Receive()); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+		return
+	}
+	return
+}
+
+// ZRemRecentDM remove recent dm of up.
+func (d *Dao) ZRemRecentDM(c context.Context, mid, dmid int64) (err error) {
+	var (
+		conn = d.dmRctRds.Get(c)
+		key  = keyRecent(mid)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("ZREMRANGEBYSCORE", key, dmid, dmid); err != nil {
+		log.Error("conn.Do(ZREMRANGEBYSCORE %s) error(%v)", key, dmid)
+	}
+	return
+}
+
+// TrimRecentDM zrange remove recent dm of up.
+func (d *Dao) TrimRecentDM(c context.Context, mid, count int64) (err error) {
+	var (
+		conn = d.dmRctRds.Get(c)
+		key  = keyRecent(mid)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("ZREMRANGEBYRANK", key, 0, count-1); err != nil {
+		log.Error("conn.Do(ZREMRANGEBYRANK %s) error(%v)", key, err)
+	}
+	return
+}

app/job/main/dm2/dao/redis_rct_test.go

@@ -0,0 +1,26 @@
+package dao
+
+import (
+	"context"
+	"testing"
+)
+
+func TestAddRecentDM(t *testing.T) {
+	count, err := testDao.AddRecentDM(context.TODO(), 123, dm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("count:%d", count)
+}
+
+func TestZRemRecentDM(t *testing.T) {
+	if err := testDao.ZRemRecentDM(context.TODO(), 123, 719150142); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestTrimRecentDM(t *testing.T) {
+	if err := testDao.TrimRecentDM(context.TODO(), 123, 1); err != nil {
+		t.Fatal(err)
+	}
+}

app/job/main/dm2/dao/redis_seg.go

@@ -0,0 +1,282 @@
+package dao
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_keyIdx        = "i_%d_%d_%d_%d" // normal dm segment sortedset(s_type_oid_cnt_n, ctime, dmid)
+	_keyIdxSub     = "s_%d_%d"       // subtitle dm sortedset(s_type_oid, progress, dmid)
+	_keyIdxContent = "c_%d_%d"       // dm content hash(d_type_oid, dmid, xml)
+)
+
+func keyIdx(tp int32, oid, cnt, n int64) string {
+	return fmt.Sprintf(_keyIdx, tp, oid, cnt, n)
+}
+
+// keyIdxSub return dm idx key.
+func keyIdxSub(tp int32, oid int64) string {
+	return fmt.Sprintf(_keyIdxSub, tp, oid)
+}
+
+// keyIdxContent return key of different dm.
+func keyIdxContent(tp int32, oid int64) string {
+	return fmt.Sprintf(_keyIdxContent, tp, oid)
+}
+
+// ExpireDMID set expire time of index.
+func (d *Dao) ExpireDMID(c context.Context, tp int32, oid, cnt, n int64) (ok bool, err error) {
+	key := keyIdx(tp, oid, cnt, n)
+	conn := d.dmSegRds.Get(c)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.dmSegExpire)); err != nil {
+		log.Error("conn.Do(EXPIRE %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// DMIDCache return dm ids.
+func (d *Dao) DMIDCache(c context.Context, tp int32, oid int64, cnt, n, limit int64) (dmids []int64, err error) {
+	var (
+		conn = d.dmSegRds.Get(c)
+		key  = keyIdx(tp, oid, cnt, n)
+	)
+	defer conn.Close()
+	if dmids, err = redis.Int64s(conn.Do("ZRANGE", key, 0, -1)); err != nil {
+		log.Error("DMIDSPCache.conn.DO(ZRANGEBYSCORE %s) error(%v)", key, err)
+	}
+	return
+}
+
+// AddDMIDCache add dmid(normal and special) to segment redis.
+func (d *Dao) AddDMIDCache(c context.Context, tp int32, oid, cnt, n int64, dmids ...int64) (err error) {
+	key := keyIdx(tp, oid, cnt, n)
+	conn := d.dmSegRds.Get(c)
+	defer conn.Close()
+	for _, dmid := range dmids {
+		if err = conn.Send("ZADD", key, dmid, dmid); err != nil {
+			log.Error("conn.Send(ZADD %s) error(%v)", key, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.dmSegExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < len(dmids)+1; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// DelDMIDCache delete dm segment cache.
+func (d *Dao) DelDMIDCache(c context.Context, tp int32, oid, cnt, n int64) (err error) {
+	key := keyIdx(tp, oid, cnt, n)
+	conn := d.dmSegRds.Get(c)
+	if _, err = conn.Do("DEL", key); err != nil {
+		log.Error("conn.Do(DEL %s) err(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// ExpireDMIDSubtitle set expire time of subtitle dmid.
+func (d *Dao) ExpireDMIDSubtitle(c context.Context, tp int32, oid int64) (ok bool, err error) {
+	key := keyIdxSub(tp, oid)
+	conn := d.dmSegRds.Get(c)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.dmSegExpire)); err != nil {
+		log.Error("conn.Do(EXPIRE %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// DMIDSubtitleCache get subtitle dmid.
+func (d *Dao) DMIDSubtitleCache(c context.Context, tp int32, oid int64, ps, pe, limit int64) (dmids []int64, err error) {
+	var (
+		conn = d.dmSegRds.Get(c)
+		key  = keyIdxSub(tp, oid)
+	)
+	defer conn.Close()
+	if dmids, err = redis.Int64s(conn.Do("ZRANGEBYSCORE", key, ps, pe, "LIMIT", 0, limit)); err != nil {
+		log.Error("conn.DO(ZRANGEBYSCORE %s) error(%v)", key, err)
+	}
+	return
+}
+
+// AddDMIDSubtitleCache add subtitle dmid to redis.
+func (d *Dao) AddDMIDSubtitleCache(c context.Context, tp int32, oid int64, dms ...*model.DM) (err error) {
+	key := keyIdxSub(tp, oid)
+	conn := d.dmSegRds.Get(c)
+	defer conn.Close()
+	for _, dm := range dms {
+		if err = conn.Send("ZADD", key, dm.Progress, dm.ID); err != nil {
+			log.Error("conn.Send(ZADD %s) error(%v)", key, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.dmSegExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < len(dms)+1; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// DelDMIDSubtitleCache delete subtitle dmid cache.
+func (d *Dao) DelDMIDSubtitleCache(c context.Context, tp int32, oid int64) (err error) {
+	key := keyIdxSub(tp, oid)
+	conn := d.dmSegRds.Get(c)
+	if _, err = conn.Do("DEL", key); err != nil {
+		log.Error("conn.Do(DEL %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// AddIdxContentCaches add index content cache to redis.
+func (d *Dao) AddIdxContentCaches(c context.Context, tp int32, oid int64, dms ...*model.DM) (err error) {
+	var (
+		conn = d.dmSegRds.Get(c)
+		key  = keyIdxContent(tp, oid)
+	)
+	defer conn.Close()
+	for _, dm := range dms {
+		if err = conn.Send("HSET", key, dm.ID, dm.ToXMLSeg()); err != nil {
+			log.Error("conn.Send(HSET %s,%v) error(%v)", key, dm, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.dmSegExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i <= len(dms); i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// DelIdxContentCaches del index content cache.
+func (d *Dao) DelIdxContentCaches(c context.Context, tp int32, oid int64, dmids ...int64) (err error) {
+	key := keyIdxContent(tp, oid)
+	conn := d.dmSegRds.Get(c)
+	args := []interface{}{key}
+	for _, dmid := range dmids {
+		args = append(args, dmid)
+	}
+	if _, err = conn.Do("HDEL", args...); err != nil {
+		log.Error("conn.Do(HDEL %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+// IdxContentCache get xml info by dmid.
+func (d *Dao) IdxContentCache(c context.Context, tp int32, oid int64, dmids []int64) (res []byte, missed []int64, err error) {
+	var (
+		k      int
+		dmid   int64
+		values [][]byte
+		key    = keyIdxContent(tp, oid)
+		args   = []interface{}{key}
+	)
+	for _, dmid = range dmids {
+		args = append(args, dmid)
+	}
+	conn := d.dmSegRds.Get(c)
+	defer conn.Close()
+	if values, err = redis.ByteSlices(conn.Do("HMGET", args...)); err != nil {
+		log.Error("conn.Do(HMGET %v) error(%v)", args, err)
+		if err == redis.ErrNil {
+			return nil, nil, nil
+		}
+		return
+	}
+	for k, dmid = range dmids {
+		if len(values[k]) == 0 {
+			missed = append(missed, dmid)
+			continue
+		}
+		res = append(res, values[k]...)
+	}
+	return
+}
+
+// IdxContentCacheV2 get elems info by dmid.
+func (d *Dao) IdxContentCacheV2(c context.Context, tp int32, oid int64, dmids []int64) (elems []*model.Elem, missed []int64, err error) {
+	var (
+		values [][]byte
+		key    = keyIdxContent(tp, oid)
+		args   = []interface{}{key}
+	)
+	for _, dmid := range dmids {
+		args = append(args, dmid)
+	}
+	conn := d.dmSegRds.Get(c)
+	defer conn.Close()
+	if values, err = redis.ByteSlices(conn.Do("HMGET", args...)); err != nil {
+		log.Error("conn.Do(HMGET %v) error(%v)", args, err)
+		if err == redis.ErrNil {
+			return nil, nil, nil
+		}
+		return
+	}
+	for k, dmid := range dmids {
+		if len(values[k]) == 0 {
+			missed = append(missed, dmid)
+			continue
+		}
+		elem, err := d.xmlToElem(values[k])
+		if err != nil {
+			missed = append(missed, dmid)
+			continue
+		}
+		elems = append(elems, elem)
+	}
+	return
+}
+
+// 在缓存过渡期将<d p="弹幕ID,弹幕属性,播放时间,弹幕模式,字体大小,颜色,发送时间,弹幕池,用户hash id">弹幕内容</d>
+// 装换为 model.Elem结构
+func (d *Dao) xmlToElem(data []byte) (e *model.Elem, err error) {
+	var v struct {
+		XMLName   xml.Name `xml:"d"`
+		Attribute string   `xml:"p,attr"`
+		Content   string   `xml:",chardata"`
+	}
+	if err = xml.Unmarshal(data, &v); err != nil {
+		return
+	}
+	e = &model.Elem{Content: v.Content, Attribute: v.Attribute}
+	return
+}

app/job/main/dm2/dao/redis_seg_test.go

@@ -0,0 +1,53 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDMIDCache(t *testing.T) {
+	var (
+		tp  int32 = 1
+		oid int64 = 1508
+		cnt int64 = 26
+		num int64 = 1
+		c         = context.TODO()
+	)
+	Convey("", t, func() {
+		_, err := testDao.DMIDCache(c, tp, oid, cnt, num, 100)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestAddDMIDCache(t *testing.T) {
+	Convey("", t, func() {
+		err := testDao.AddDMIDCache(c, 1, 1508, 26, 1, 1233333333333)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestIdxContentCacheV2(t *testing.T) {
+	var (
+		tp    int32 = 1
+		oid   int64 = 1508
+		c           = context.TODO()
+		dmids       = []int64{2355015081, 2356915089}
+	)
+	Convey("", t, func() {
+		elems, missed, err := testDao.IdxContentCacheV2(c, tp, oid, dmids)
+		So(err, ShouldBeNil)
+		t.Logf("missed dmid:%v", missed)
+		t.Logf("elems:%+v", elems)
+	})
+}
+
+func TestXMLToElem(t *testing.T) {
+	Convey("convert xml tag to elem struct", t, func() {
+		s := []byte(`<d p="1,1,1,1,11,111,11,1,23123123">弹幕内容</d>`)
+		elem, err := testDao.xmlToElem(s)
+		So(err, ShouldBeNil)
+		t.Logf("%+v", elem)
+	})
+}

app/job/main/dm2/dao/redis_task.go

@@ -0,0 +1,64 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_taskJobKey = "task_job"
+)
+
+// SetnxTaskJob setnx task_job value
+func (d *Dao) SetnxTaskJob(c context.Context, value string) (ok bool, err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if ok, err = redis.Bool(conn.Do("SETNX", _taskJobKey, value)); err != nil {
+		log.Error("d.SetnxMask(value:%s),error(%v)", value, err)
+		return
+	}
+	return
+}
+
+// GetTaskJob .
+func (d *Dao) GetTaskJob(c context.Context) (value string, err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if value, err = redis.String(conn.Do("GET", _taskJobKey)); err != nil {
+		log.Error("d.GetMaskJob,error(%v)", err)
+		return
+	}
+	return
+}
+
+// GetSetTaskJob .
+func (d *Dao) GetSetTaskJob(c context.Context, value string) (old string, err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if old, err = redis.String(conn.Do("GETSET", _taskJobKey, value)); err != nil {
+		log.Error("d.GetSetTaskJob(value:%s),error(%v)", value, err)
+		return
+	}
+	return
+}
+
+// DelTaskJob .
+func (d *Dao) DelTaskJob(c context.Context) (err error) {
+	var (
+		conn = d.dmRds.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("DEL", _taskJobKey); err != nil {
+		log.Error("d.DelTaskJob,error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/dm2/dao/redis_task_test.go

@@ -0,0 +1,65 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoSetnxTaskJob(t *testing.T) {
+	convey.Convey("SetnxTaskJob", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			value = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ok, err := testDao.SetnxTaskJob(c, value)
+			ctx.Convey("Then err should be nil.ok should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ok, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetTaskJob(t *testing.T) {
+	convey.Convey("GetTaskJob", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.GetTaskJob(c)
+		})
+	})
+}
+
+func TestDaoGetSetTaskJob(t *testing.T) {
+	convey.Convey("GetSetTaskJob", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			value = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			old, err := testDao.GetSetTaskJob(c, value)
+			ctx.Convey("Then err should be nil.old should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(old, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelTaskJob(t *testing.T) {
+	convey.Convey("DelTaskJob", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := testDao.DelTaskJob(c)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/dm2/dao/redis_test.go

@@ -0,0 +1,71 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm2/model"
+)
+
+var (
+	c  = context.TODO()
+	dm = &model.DM{
+		ID:       719150142,
+		Oid:      1221,
+		Type:     1,
+		Mid:      478046,
+		Progress: 0,
+		State:    0,
+		Content: &model.Content{
+			ID:       719150142,
+			FontSize: 24,
+			Mode:     1,
+			Msg:      "aaa",
+		}}
+)
+
+func TestAddDMCache(t *testing.T) {
+	if err := testDao.AddDMCache(context.TODO(), dm); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestSetDMCache(t *testing.T) {
+	if err := testDao.SetDMCache(c, dm.Type, dm.Oid, []*model.DM{dm, dm}); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestDelDMCache(t *testing.T) {
+	if err := testDao.DelDMCache(context.TODO(), 1, 1221); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestExpireDMCache(t *testing.T) {
+	ok, err := testDao.ExpireDMCache(context.TODO(), 1, 1221)
+	if err != nil {
+		t.Error(err)
+	}
+	t.Log(ok)
+}
+
+func TestDMCache(t *testing.T) {
+	values, err := testDao.DMCache(context.TODO(), dm.Type, dm.Oid)
+	if err != nil {
+		t.Error(err)
+	}
+	for _, value := range values {
+		dmCache := &model.DM{}
+		if err = dmCache.Unmarshal(value); err != nil {
+			t.Errorf("Unmarshal(%s) error(%v)", value, err)
+		}
+		t.Log(dmCache)
+	}
+}
+
+func TestTrimDMCache(t *testing.T) {
+	if err := testDao.TrimDMCache(context.TODO(), 1, dm.Oid, 1); err != nil {
+		t.Error(err)
+	}
+}

app/job/main/dm2/dao/task.go

@@ -0,0 +1,230 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_taskInfo      = "SELECT id,topic,state,qcount,result,sub,last_index,priority,creator,reviewer,title FROM dm_task WHERE state=? order by priority desc,ctime asc"
+	_oneTask       = "SELECT id,topic,state,qcount,result,sub,last_index,priority,creator,reviewer,title FROM dm_task WHERE state in (3,9) order by priority desc,ctime asc limit 1"
+	_taskByID      = "SELECT id,topic,state,qcount,result,sub,last_index,priority,creator,reviewer,title FROM dm_task WHERE id=?"
+	_uptTask       = "UPDATE dm_task SET state=?,last_index=?,qcount=?,result=? WHERE id=?"
+	_uptSubTask    = "UPDATE dm_sub_task SET tcount=tcount+?,end=? WHERE task_id=?"
+	_selectSubTask = "SELECT id,operation,rate,tcount,start,end FROM dm_sub_task WHERE task_id=?"
+
+	_delDMIndex     = "UPDATE dm_index_%03d SET state=? WHERE oid=? AND id IN (%s)"
+	_uptSubCountSQL = "UPDATE dm_subject_%02d SET count=count-? WHERE type=? AND oid=?"
+
+	_merakMsgURI = "/"
+)
+
+// TaskInfos task infos.
+func (d *Dao) TaskInfos(c context.Context, state int32) (tasks []*model.TaskInfo, err error) {
+	rows, err := d.biliDMWriter.Query(c, _taskInfo, state)
+	if err != nil {
+		log.Error("d.biliDMWriter.Query(query:%s,state:%d) error(%v)", _taskInfo, state, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		task := &model.TaskInfo{}
+		if err = rows.Scan(&task.ID, &task.Topic, &task.State, &task.Count, &task.Result, &task.Sub, &task.LastIndex, &task.Priority, &task.Creator, &task.Reviewer, &task.Title); err != nil {
+			log.Error("d.biliDMWriter.Scan(query:%s,state:%d) error(%v)", _taskInfo, state, err)
+			return
+		}
+		tasks = append(tasks, task)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.biliDMWriter.rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// OneTask .
+func (d *Dao) OneTask(c context.Context) (task *model.TaskInfo, err error) {
+	task = &model.TaskInfo{}
+	row := d.biliDMWriter.QueryRow(c, _oneTask)
+	if err = row.Scan(&task.ID, &task.Topic, &task.State, &task.Count, &task.Result, &task.Sub, &task.LastIndex, &task.Priority, &task.Creator, &task.Reviewer, &task.Title); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			task = nil
+		} else {
+			log.Error("d.biliDMWriter.Scan(query:%s) error(%v)", _oneTask, err)
+		}
+	}
+	return
+}
+
+// TaskInfoByID .
+func (d *Dao) TaskInfoByID(c context.Context, id int64) (task *model.TaskInfo, err error) {
+	task = &model.TaskInfo{}
+	row := d.biliDMWriter.QueryRow(c, _taskByID, id)
+	if err = row.Scan(&task.ID, &task.Topic, &task.State, &task.Count, &task.Result, &task.Sub, &task.LastIndex, &task.Priority, &task.Creator, &task.Reviewer, &task.Title); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			task = nil
+		} else {
+			log.Error("d.biliDMWriter.Scan(query:%s,id:%d) error(%v)", _taskByID, id, err)
+		}
+	}
+	return
+}
+
+// UpdateTask update dm task.
+func (d *Dao) UpdateTask(c context.Context, task *model.TaskInfo) (affected int64, err error) {
+	row, err := d.biliDMWriter.Exec(c, _uptTask, task.State, task.LastIndex, task.Count, task.Result, task.ID)
+	if err != nil {
+		log.Error("d.biliDMWriter.Exec(query:%s,task:%+v) error(%v)", _uptTask, task, err)
+		return
+	}
+	return row.RowsAffected()
+}
+
+// UptSubTask uopdate dm sub task.
+func (d *Dao) UptSubTask(c context.Context, taskID, delCount int64, end time.Time) (affected int64, err error) {
+	row, err := d.biliDMWriter.Exec(c, _uptSubTask, delCount, end, taskID)
+	if err != nil {
+		log.Error("d.biliDMWriter.Exec(query:%s) error(%v)", _uptSubTask, err)
+		return
+	}
+	return row.RowsAffected()
+}
+
+// SubTask .
+func (d *Dao) SubTask(c context.Context, id int64) (subTask *model.SubTask, err error) {
+	// TODO: operation time
+	subTask = new(model.SubTask)
+	row := d.biliDMWriter.QueryRow(c, _selectSubTask, id)
+	if err = row.Scan(&subTask.ID, &subTask.Operation, &subTask.Rate, &subTask.Tcount, &subTask.Start, &subTask.End); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			subTask = nil
+		}
+		log.Error("biliDM.Scan(%s, taskID:%d) error*(%v)", _selectSubTask, id, err)
+		return
+	}
+	return
+}
+
+// DelDMs dm task del dms.
+func (d *Dao) DelDMs(c context.Context, oid int64, dmids []int64, state int32) (affected int64, err error) {
+	rows, err := d.dmWriter.Exec(c, fmt.Sprintf(_delDMIndex, d.hitIndex(oid), xstr.JoinInts(dmids)), state, oid)
+	if err != nil {
+		log.Error("d.dmWriter.Exec(query:%s,oid:%d,dmids:%v) error(%v)", _delDMIndex, oid, dmids, err)
+		return
+	}
+	return rows.RowsAffected()
+}
+
+// UptSubjectCount update count.
+func (d *Dao) UptSubjectCount(c context.Context, tp int32, oid, count int64) (affected int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_uptSubCountSQL, d.hitSubject(oid)), count, tp, oid)
+	if err != nil {
+		log.Error("dmWriter.Exec(query:%s,oid:%d) error(%v)", _uptSubCountSQL, oid, err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// TaskSearchRes get res from BI url
+func (d *Dao) TaskSearchRes(c context.Context, task *model.TaskInfo) (count int64, result string, state int32, err error) {
+
+	var (
+		resp *http.Response
+		res  struct {
+			Code     int64    `json:"code"`
+			StatusID int32    `json:"statusId"`
+			Path     []string `json:"hdfsPath"`
+			Count    int64    `json:"count"`
+		}
+		bs []byte
+	)
+	// may costing long time use default transport
+	if resp, err = http.Get(task.Topic); err != nil {
+		log.Error("http.Get(%s) error(%v)", task.Topic, err)
+		return
+	}
+	defer resp.Body.Close()
+	if bs, err = ioutil.ReadAll(resp.Body); err != nil {
+		log.Error("ioutil.ReadAll url:%v error(%v)", task.Topic, err)
+		return
+	}
+	if err = json.Unmarshal(bs, &res); err != nil {
+		return
+	}
+	if res.Code != 200 {
+		err = fmt.Errorf("%v", res)
+		log.Error("d.httpClient.Get(%s) code(%d)", task.Topic, res.Code)
+		return
+	}
+	if res.StatusID == model.TaskSearchSuc && len(res.Path) > 0 {
+		result = res.Path[0]
+		count = res.Count
+	}
+	return count, result, res.StatusID, err
+}
+
+// SendWechatWorkMsg send wechat work msg.
+func (d *Dao) SendWechatWorkMsg(c context.Context, content, title string, users []string) (err error) {
+	userMap := make(map[string]struct{}, len(users))
+	unames := make([]string, 0, len(users))
+	for _, user := range users {
+		if user == "" {
+			continue
+		}
+		if _, ok := userMap[user]; ok {
+			continue
+		}
+		userMap[user] = struct{}{}
+		unames = append(unames, user)
+	}
+	params := url.Values{}
+	params.Set("Action", "CreateWechatMessage")
+	params.Set("PublicKey", d.conf.TaskConf.MsgPublicKey)
+	params.Set("UserName", strings.Join(unames, ","))
+	params.Set("Title", title)
+	params.Set("Content", content)
+	params.Set("Signature", "")
+	params.Set("TreeId", "")
+	paramStr := params.Encode()
+	if strings.IndexByte(paramStr, '+') > -1 {
+		paramStr = strings.Replace(paramStr, "+", "%20", -1)
+	}
+	var (
+		buffer bytes.Buffer
+		querry string
+	)
+	buffer.WriteString(paramStr)
+	querry = buffer.String()
+	res := &struct {
+		Code int `json:"RetCode"`
+	}{}
+	url := d.conf.Host.MerakHost + _merakMsgURI
+	req, err := http.NewRequest("POST", url, strings.NewReader(querry))
+	if err != nil {
+		return
+	}
+	req.Header.Add("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
+	if err = d.httpCli.Do(c, req, res); err != nil {
+		log.Error("d.SendWechatWorkMsg.client.Do(%v,%v) error(%v)", req, querry, err)
+		return
+	}
+	if res.Code != 0 {
+		log.Error("d.SendWechatWorkMsg(%s,%s,%v) res.Code != 0, res(%v)", content, title, users, res)
+		err = fmt.Errorf("uri:%s,code:%d", url+querry, res.Code)
+	}
+	return
+}

app/job/main/dm2/dao/task_test.go

@@ -0,0 +1,152 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/dm2/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoTaskInfos(t *testing.T) {
+	convey.Convey("TaskInfos", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			state = int32(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			tasks, err := testDao.TaskInfos(c, state)
+			ctx.Convey("Then err should be nil.tasks should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(tasks, convey.ShouldNotBeNil)
+				for _, task := range tasks {
+					t.Logf("%+v", task)
+				}
+			})
+		})
+	})
+}
+
+func TestDaoUpdateTask(t *testing.T) {
+	convey.Convey("UpdateTask", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			task = &model.TaskInfo{}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affected, err := testDao.UpdateTask(c, task)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelDMs(t *testing.T) {
+	convey.Convey("DelDMs", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			oid   = int64(221)
+			dmids = []int64{719182141}
+			state = int32(12)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affected, err := testDao.DelDMs(c, oid, dmids, state)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoUptSubTask(t *testing.T) {
+	convey.Convey("UptSubTask", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			taskID   = int64(0)
+			delCount = int64(0)
+			end      = time.Now()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affected, err := testDao.UptSubTask(c, taskID, delCount, end)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoTaskSearchRes(t *testing.T) {
+	convey.Convey("TaskSearchRes", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			task = &model.TaskInfo{
+				Topic: "http://berserker.bilibili.co/m-avenger/api/hive/status/query/148/672bc22888af701529e8b3052fd2c4a7/1541066053/1389520",
+			}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, result, state, err := testDao.TaskSearchRes(c, task)
+			ctx.Convey("Then err should be nil.result,state should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(state, convey.ShouldNotBeNil)
+				ctx.So(result, convey.ShouldNotBeNil)
+				t.Log(result, state)
+			})
+		})
+	})
+}
+
+func TestDaoUptSubjectCount(t *testing.T) {
+	convey.Convey("UptSubjectCount", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			tp    = int32(1)
+			oid   = int64(0)
+			count = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			affected, err := testDao.UptSubjectCount(c, tp, oid, count)
+			ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(affected, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSendWechatWorkMsg(t *testing.T) {
+	convey.Convey("SendWechatWorkMsg", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			content = "test"
+			title   = "test"
+			users   = []string{"fengduzhen"}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := testDao.SendWechatWorkMsg(c, content, title, users)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSubTask(t *testing.T) {
+	convey.Convey("SubTask", t, func(ctx convey.C) {
+		var (
+			c  = context.Background()
+			id = int64(32)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := testDao.SubTask(c, id)
+			ctx.Convey("Then err should be nil.subTask should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/dm2/dao/transfer.go

@@ -0,0 +1,69 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/log"
+)
+
+const (
+	_selectTransfer = "SELECT id,from_cid,to_cid,mid,offset,state,ctime FROM dm_transfer_job WHERE state=? limit 1"
+	_updateTransfer = "UPDATE dm_transfer_job SET state=?,dmid=? WHERE id=?"
+	_idxsSQL        = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d FORCE INDEX(ix_oid_state) WHERE type=? AND oid=? AND id >? ORDER BY id limit ?"
+)
+
+// Transfers get all transfer job
+func (d *Dao) Transfers(c context.Context, state int8) (trans []*model.Transfer, err error) {
+	rows, err := d.biliDMWriter.Query(c, _selectTransfer, model.StatInit)
+	if err != nil {
+		log.Error("d.biliDMWriter.Query(sql:%s) error(%v)", _selectTransfer, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		t := &model.Transfer{}
+		if err = rows.Scan(&t.ID, &t.FromCid, &t.ToCid, &t.Mid, &t.Offset, &t.State, &t.Ctime); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		trans = append(trans, t)
+	}
+	return
+}
+
+// UpdateTransfer change transfer job state.
+func (d *Dao) UpdateTransfer(c context.Context, t *model.Transfer) (affect int64, err error) {
+	row, err := d.biliDMWriter.Exec(c, _updateTransfer, t.State, t.Dmid, t.ID)
+	if err != nil {
+		log.Error("d.biliDMWriter.Exec(%+v) error(%v)", t, err)
+		return
+	}
+	return row.RowsAffected()
+}
+
+// DMIndexs get dm indexs info
+func (d *Dao) DMIndexs(c context.Context, tp int32, oid, minID, limit int64) (idxMap map[int64]*model.DM, dmids, special []int64, err error) {
+	query := fmt.Sprintf(_idxsSQL, d.hitIndex(oid))
+	rows, err := d.dmReader.Query(c, query, tp, oid, minID, limit)
+	if err != nil {
+		log.Error("db.Query() error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	idxMap = make(map[int64]*model.DM)
+	for rows.Next() {
+		idx := new(model.DM)
+		if err = rows.Scan(&idx.ID, &idx.Type, &idx.Oid, &idx.Mid, &idx.Progress, &idx.State, &idx.Pool, &idx.Attr, &idx.Ctime, &idx.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			return
+		}
+		idxMap[idx.ID] = idx
+		dmids = append(dmids, idx.ID)
+		if idx.Pool == model.PoolSpecial {
+			special = append(special, idx.ID)
+		}
+	}
+	return
+}

app/job/main/dm2/dao/transfer_test.go

@@ -0,0 +1,52 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm2/conf"
+	"go-common/app/job/main/dm2/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestTransfers(t *testing.T) {
+	var (
+		d = New(conf.Conf)
+		c = context.TODO()
+	)
+	Convey("test transferJob", t, func() {
+		_, err := d.Transfers(c, model.StatInit)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestDmIndexs(t *testing.T) {
+	var (
+		d = New(conf.Conf)
+		c = context.TODO()
+	)
+	Convey("test DmIndexs", t, func() {
+		_, _, _, err := d.DMIndexs(c, 1, 1012, 0, 10)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestUpdateTransfer(t *testing.T) {
+	var (
+		d     = New(conf.Conf)
+		c     = context.TODO()
+		trans = &model.Transfer{
+			ID:      265,
+			FromCid: 233,
+			ToCid:   1221,
+			Dmid:    333,
+			Mid:     1,
+			Offset:  0.00,
+			State:   0,
+		}
+	)
+	Convey("test update job", t, func() {
+		d.UpdateTransfer(c, trans)
+	})
+}

app/job/main/dm2/dao/videoup.go

@@ -0,0 +1,56 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"go-common/app/job/main/dm2/model"
+	"go-common/library/log"
+)
+
+const (
+	_archiveURL = "/videoup/view"
+)
+
+func (d *Dao) archiveURI() string {
+	return d.conf.Host.Videoup + _archiveURL
+}
+
+// Videos 根据aid获取分批信息，包含未开放浏览分批.
+func (d *Dao) Videos(c context.Context, aid int64) (videos []*model.Video, err error) {
+	var (
+		uri    = d.archiveURI()
+		params = url.Values{}
+		res    struct {
+			Code    int64  `json:"code"`
+			Message string `json:"message"`
+			Data    *struct {
+				Archive *model.Archive `json:"archive"`
+				Videos  []*model.Video `json:"videos"`
+			} `json:"data"`
+		}
+	)
+	params.Set("aid", fmt.Sprint(aid))
+	if err = d.httpCli.Get(c, uri, "", params, &res); err != nil {
+		return
+	}
+	if res.Code != 0 {
+		err = fmt.Errorf("aid:%d,res.Code:%d", aid, res.Code)
+		log.Error("url(%s) aid(%d) res(%v)", uri+"?"+params.Encode(), aid, res)
+		return
+	}
+	if res.Data == nil || res.Data.Archive == nil {
+		log.Error("url(%s) aid(%d) res(%v)", uri+"?"+params.Encode(), aid, res)
+		return
+	}
+	if len(res.Data.Videos) == 0 {
+		log.Error("url(%s) aid(%d) videos is empty", uri+"?"+params.Encode(), aid)
+		return
+	}
+	for _, v := range res.Data.Videos {
+		v.Mid = res.Data.Archive.Mid
+		videos = append(videos, v)
+	}
+	return
+}

app/job/main/dm2/dao/videoup_test.go

@@ -0,0 +1,31 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoarchiveURI(t *testing.T) {
+	convey.Convey("archiveURI", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := testDao.archiveURI()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoVideos(t *testing.T) {
+	convey.Convey("Videos", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			aid = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			testDao.Videos(c, aid)
+		})
+	})
+}