Create & Init Project...

app/job/main/dm/dao/BUILD

@@ -0,0 +1,64 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "index_test.go",
+        "memcache_test.go",
+        "redis_test.go",
+        "subject_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "index.go",
+        "memcache.go",
+        "redis.go",
+        "subject.go",
+    ],
+    importpath = "go-common/app/job/main/dm/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/dm/conf:go_default_library",
+        "//app/job/main/dm/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/dm/dao/dao.go

@@ -0,0 +1,81 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_subjectSharding = 100
+	_indexSharding   = 1000
+)
+
+// Dao dao struct.
+type Dao struct {
+	// redis
+	redis       *redis.Pool
+	redisExpire int32
+	// memcache
+	mc       *memcache.Pool
+	mcExpire int32
+	// mysql
+	dmReader *sql.DB
+	dmWriter *sql.DB
+}
+
+// New return dm dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		// redis
+		redis:       redis.NewPool(c.Redis.Config),
+		redisExpire: int32(time.Duration(c.Redis.Expire) / time.Second),
+		// memcache
+		mc:       memcache.NewPool(c.Memcache.Config),
+		mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
+		// mysql
+		dmReader: sql.NewMySQL(c.DB.DMReader),
+		dmWriter: sql.NewMySQL(c.DB.DMWriter),
+	}
+	return
+}
+
+func (d *Dao) hitSubject(oid int64) int64 {
+	return oid % _subjectSharding
+}
+
+func (d *Dao) hitIndex(oid int64) int64 {
+	return oid % _indexSharding
+}
+
+// Ping dm dao ping.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.dmWriter.Ping(c); err != nil {
+		log.Error("dmWriter.Ping() error(%v)", err)
+		return
+	}
+	if err = d.dmReader.Ping(c); err != nil {
+		log.Error("dmReader.Ping() error(%v)", err)
+		return
+	}
+	// mc
+	mconn := d.mc.Get(c)
+	defer mconn.Close()
+	if err = mconn.Set(&memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}); err != nil {
+		log.Error("mc.Set error(%v)", err)
+		return
+	}
+	// dm redis
+	rconn := d.redis.Get(c)
+	defer rconn.Close()
+	if _, err = rconn.Do("SET", "ping", "pong"); err != nil {
+		rconn.Close()
+		log.Error("redis.Set error(%v)", err)
+	}
+	return
+}

app/job/main/dm/dao/dao_test.go

@@ -0,0 +1,24 @@
+package dao
+
+import (
+	"os"
+	"testing"
+
+	"go-common/app/job/main/dm/conf"
+	"go-common/library/log"
+)
+
+var (
+	testDao *Dao
+)
+
+func TestMain(m *testing.M) {
+	conf.ConfPath = "../cmd/dm-job-test.toml"
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Xlog)
+	defer log.Close()
+	testDao = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/dm/dao/index.go

@@ -0,0 +1,76 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_selAllIdxSQL   = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d WHERE type=? AND oid=? AND state IN(0,6)"
+	_selIdxHidesSQL = "SELECT id,type,oid,mid,progress,state,pool,attr,ctime,mtime FROM dm_index_%03d FORCE INDEX(ix_oid_state) WHERE type=? AND oid=? AND state=2 ORDER BY id DESC limit ?" // NOTE slow query
+	_upIdxSQL       = "UPDATE dm_index_%03d SET mid=?,progress=?,state=?,pool=?,attr=? WHERE id=?"
+	_upIdxStatesSQL = "UPDATE dm_index_%03d SET state=? WHERE id IN(%s)"
+)
+
+// DMInfos get indexs of oid.
+func (d *Dao) DMInfos(c context.Context, tp int32, oid int64) (dms []*model.DM, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_selAllIdxSQL, d.hitIndex(oid)), tp, oid)
+	if err != nil {
+		log.Error("db.Query(%d %d) error(%v)", tp, oid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		dm := &model.DM{}
+		if err = rows.Scan(&dm.ID, &dm.Type, &dm.Oid, &dm.Mid, &dm.Progress, &dm.State, &dm.Pool, &dm.Attr, &dm.Ctime, &dm.Mtime); err != nil {
+			log.Error("rows.Scan() error(%v)", err)
+			return
+		}
+		dms = append(dms, dm)
+	}
+	return
+}
+
+// DMHides get hide index info from db by oid and state.
+func (d *Dao) DMHides(c context.Context, typ int32, oid, limit int64) (res []*model.DM, err error) {
+	rows, err := d.dmReader.Query(c, fmt.Sprintf(_selIdxHidesSQL, d.hitIndex(oid)), typ, oid, limit)
+	if err != nil {
+		log.Error("db.Query(%d %d) error(%v)", typ, oid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		dm := &model.DM{}
+		if err = rows.Scan(&dm.ID, &dm.Type, &dm.Oid, &dm.Mid, &dm.Progress, &dm.State, &dm.Pool, &dm.Attr, &dm.Ctime, &dm.Mtime); err != nil {
+			log.Error("row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, dm)
+	}
+	return
+}
+
+// UpdateDM update index of dm.
+func (d *Dao) UpdateDM(c context.Context, m *model.DM) (affect int64, err error) {
+	res, err := d.dmWriter.Exec(c, fmt.Sprintf(_upIdxSQL, d.hitIndex(m.Oid)), m.Mid, m.Progress, m.State, m.Pool, m.Attr, m.ID)
+	if err != nil {
+		log.Error("tx.Exec error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// UpdateDMStates multi update index state of dm.
+func (d *Dao) UpdateDMStates(c context.Context, oid int64, dmids []int64, state int32) (affect int64, err error) {
+	upSQL := fmt.Sprintf(_upIdxStatesSQL, d.hitIndex(oid), xstr.JoinInts(dmids))
+	res, err := d.dmWriter.Exec(c, upSQL, state)
+	if err != nil {
+		log.Error("db.Exec(%s) error(%v)", upSQL, err)
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/dm/dao/index_test.go

@@ -0,0 +1,53 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/dm/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDMInfos(t *testing.T) {
+	Convey("", t, func() {
+		dms, err := testDao.DMInfos(context.TODO(), 1, 1221)
+		if err != nil {
+			fmt.Println(err)
+		}
+		So(err, ShouldBeNil)
+		So(dms, ShouldNotBeEmpty)
+	})
+}
+
+func TestDMHides(t *testing.T) {
+	Convey("", t, func() {
+		dms, err := testDao.DMHides(context.TODO(), 1, 1221, 1)
+		if err != nil {
+			fmt.Println(err)
+		}
+		So(err, ShouldBeNil)
+		So(dms, ShouldNotBeEmpty)
+	})
+}
+
+func TestUpdateDM(t *testing.T) {
+	Convey("", t, func() {
+		dm := &model.DM{
+			ID:    1234555,
+			Oid:   1221,
+			State: 3,
+		}
+		_, err := testDao.UpdateDM(context.TODO(), dm)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestUpdateDMStates(t *testing.T) {
+	Convey("", t, func() {
+		dmids := []int64{709150141, 709150142}
+		_, err := testDao.UpdateDMStates(context.TODO(), 1221, dmids, 1)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/dm/dao/memcache.go

@@ -0,0 +1,107 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_prefixSub   = "s_"
+	_keyDuration = "d_" // video duration
+)
+
+func keySubject(tp int32, oid int64) string {
+	return _prefixSub + fmt.Sprintf("%d_%d", tp, oid)
+}
+
+// keyDuration return video duration key.
+func keyDuration(oid int64) string {
+	return _keyDuration + strconv.FormatInt(oid, 10)
+}
+
+// SubjectCache get subject from memcache.
+func (d *Dao) SubjectCache(c context.Context, tp int32, oid int64) (sub *model.Subject, err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = keySubject(tp, oid)
+		rp   *memcache.Item
+	)
+	defer conn.Close()
+	if rp, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("mc.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	sub = &model.Subject{}
+	if err = conn.Scan(rp, &sub); err != nil {
+		log.Error("conn.Scan() error(%v)", err)
+	}
+	return
+}
+
+// SetSubjectCache add subject cache.
+func (d *Dao) SetSubjectCache(c context.Context, sub *model.Subject) (err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = keySubject(sub.Type, sub.Oid)
+	)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Object:     sub,
+		Flags:      memcache.FlagJSON,
+		Expiration: d.mcExpire,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%v) error(%v)", item, err)
+	}
+	return
+}
+
+// DurationCache return duration of video.
+func (d *Dao) DurationCache(c context.Context, oid int64) (duration int64, err error) {
+	var (
+		key  = keyDuration(oid)
+		conn = d.mc.Get(c)
+		item *memcache.Item
+	)
+	defer conn.Close()
+	if item, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			duration = model.NotFound
+			err = nil
+		} else {
+			log.Error("conn.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	if duration, err = strconv.ParseInt(string(item.Value), 10, 64); err != nil {
+		log.Error("strconv.ParseInt(%s) error(%v)", item.Value, err)
+	}
+	return
+}
+
+// SetDurationCache set video duration to redis.
+func (d *Dao) SetDurationCache(c context.Context, oid, duration int64) (err error) {
+	key := keyDuration(oid)
+	conn := d.mc.Get(c)
+	item := memcache.Item{
+		Key:        key,
+		Value:      []byte(fmt.Sprint(duration)),
+		Expiration: d.mcExpire,
+		Flags:      memcache.FlagRAW,
+	}
+	if err = conn.Set(&item); err != nil {
+		log.Error("mc.Set(%v) error(%v)", item, err)
+	}
+	conn.Close()
+	return
+}

app/job/main/dm/dao/memcache_test.go

@@ -0,0 +1,43 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestSubjectCache(t *testing.T) {
+	Convey("subject cache", t, func() {
+		sub, err := testDao.SubjectCache(context.TODO(), 1, 1221)
+		So(err, ShouldBeNil)
+		So(sub, ShouldNotBeNil)
+	})
+}
+
+func TestSetSubjectCache(t *testing.T) {
+	sub := &model.Subject{
+		Type: 1,
+		Oid:  1221,
+	}
+	Convey("add subject cache", t, func() {
+		err := testDao.SetSubjectCache(context.TODO(), sub)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestDurationCache(t *testing.T) {
+	Convey("", t, func() {
+		_, err := testDao.DurationCache(context.TODO(), 1221)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestSetDurationCachee(t *testing.T) {
+	Convey("", t, func() {
+		err := testDao.SetDurationCache(context.TODO(), 1221, 10000)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/dm/dao/redis.go

@@ -0,0 +1,171 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_keyIdxContent = "c_%d_%d"    // dm content hash(c_type_oid, dmid, xml)
+	_keyTrimQueue  = "tq_n_%d_%d" // trim queue if dm_count > dm_maxlimit
+
+	divide = int64(34359738368) // 2^35
+)
+
+// keyIdxContent return key of different dm.
+func keyIdxContent(typ int32, oid int64) string {
+	return fmt.Sprintf(_keyIdxContent, typ, oid)
+}
+
+// keyIdxQueue return trim queue key.
+func keyTrimQueue(tp int32, oid int64) string {
+	return fmt.Sprintf(_keyTrimQueue, tp, oid)
+}
+
+// ExpireTrimQueue set expire time of index.
+func (d *Dao) ExpireTrimQueue(c context.Context, tp int32, oid int64) (ok bool, err error) {
+	key := keyTrimQueue(tp, oid)
+	conn := d.redis.Get(c)
+	if ok, err = redis.Bool(conn.Do("EXPIRE", key, d.redisExpire)); err != nil {
+		log.Error("conn.Do(EXPIRE %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}
+
+func score(attr int32, id int64) (score float64) {
+	v := id / divide           // 2^63 / 2^35 = 2^28-1 整数部分最大值：268435455
+	k := id % divide           // 精度8位，最后5位可忽略
+	r := int64(attr)&1<<28 | v // NOTE v should less than 2^28
+	score, _ = strconv.ParseFloat(fmt.Sprintf("%d.%d", r, k), 64)
+	return
+}
+
+// AddTrimQueueCache add dm index into trim queue.
+func (d *Dao) AddTrimQueueCache(c context.Context, tp int32, oid int64, trims []*model.Trim) (count int64, err error) {
+	var (
+		key  = keyTrimQueue(tp, oid)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, trim := range trims {
+		if err = conn.Send("ZADD", key, score(trim.Attr, trim.ID), trim.ID); err != nil {
+			log.Error("conn.Send(ZADD %s %v) error(%v)", key, trim, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Send("ZCARD", key); err != nil {
+		log.Error("conn.Send(ZCARD %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() error(%v)", err)
+		return
+	}
+	for i := 0; i < len(trims)+1; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	if count, err = redis.Int64(conn.Receive()); err != nil {
+		log.Error("conn.Receive() error(%v)", err)
+	}
+	return
+}
+
+// FlushTrimCache flush trim queue cache.
+func (d *Dao) FlushTrimCache(c context.Context, tp int32, oid int64, trims []*model.Trim) (err error) {
+	var (
+		key  = keyTrimQueue(tp, oid)
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	for _, trim := range trims {
+		if err = conn.Send("ZADD", key, score(trim.Attr, trim.ID), trim.ID); err != nil {
+			log.Error("conn.Send(ZADD %s %v) error(%v)", key, trim, err)
+			return
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
+		log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush(%s) error(%v)", key, err)
+		return
+	}
+	for i := 0; i < len(trims)+1; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}
+
+// ZRemTrimCache ZREM trim from trim queue.
+func (d *Dao) ZRemTrimCache(c context.Context, tp int32, oid int64, dmID int64) (err error) {
+	var (
+		key  = keyTrimQueue(tp, oid)
+		conn = d.redis.Get(c)
+	)
+	if _, err = conn.Do("ZREM", key, dmID); err != nil {
+		log.Error("conn.Do(ZREM %s %v) error(%v)", key, dmID, err)
+	}
+	conn.Close()
+	return
+}
+
+// TrimCache trim trim queue and return trimed dmid from trim queue.
+func (d *Dao) TrimCache(c context.Context, tp int32, oid, count int64) (dmids []int64, err error) {
+	var (
+		key    = keyTrimQueue(tp, oid)
+		conn   = d.redis.Get(c)
+		replys [][]byte
+		dmID   int64
+	)
+	defer conn.Close()
+	if replys, err = redis.ByteSlices(conn.Do("ZRANGE", key, 0, count-1)); err != nil {
+		log.Error("conn.Do(ZRANGE %s) error(%v)", key, err)
+		return
+	}
+	for _, reply := range replys {
+		if err = json.Unmarshal(reply, &dmID); err != nil {
+			log.Error("json.Unmarshal(%s) error(v)", string(reply), err)
+			return
+		}
+		dmids = append(dmids, dmID)
+	}
+	if len(dmids) > 0 {
+		if _, err = conn.Do("ZREMRANGEBYRANK", key, 0, len(dmids)-1); err != nil {
+			log.Error("conn.Do(ZREMRANGEBYRANK %s) error(%v)", key, err)
+		}
+	}
+	return
+}
+
+// DelIdxContentCaches del index content cache.
+func (d *Dao) DelIdxContentCaches(c context.Context, typ int32, oid int64, dmids ...int64) (err error) {
+	key := keyIdxContent(typ, oid)
+	conn := d.redis.Get(c)
+	args := []interface{}{key}
+	for _, dmid := range dmids {
+		args = append(args, dmid)
+	}
+	if _, err = conn.Do("HDEL", args...); err != nil {
+		log.Error("conn.Do(HDEL %s) error(%v)", key, err)
+	}
+	conn.Close()
+	return
+}

app/job/main/dm/dao/redis_test.go

@@ -0,0 +1,28 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/job/main/dm/conf"
+)
+
+func TestDuration(t *testing.T) {
+	var (
+		err      error
+		res      int64
+		cid      = int64(1)
+		duration = int64(100)
+		c        = context.TODO()
+	)
+	d := New(conf.Conf)
+	if err = d.SetDurationCache(c, cid, duration); err != nil {
+		t.Errorf("d.SetDurationCache(%d,%d) error(%v)", cid, duration, err)
+	}
+	if res, err = d.DurationCache(c, cid); err != nil {
+		t.Errorf("d.DurationCache(%d) error(%v)", cid, err)
+	}
+	if res != duration {
+		t.Errorf("not expect value %d,%d", duration, res)
+	}
+}

app/job/main/dm/dao/subject.go

@@ -0,0 +1,29 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/dm/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_selSubSQL = "SELECT id,oid,type,pid,mid,state,attr,acount,count,mcount,move_count,maxlimit,childpool,ctime,mtime FROM dm_subject_%02d WHERE oid=? AND type=?"
+)
+
+// Subject get subject info from db.
+func (d *Dao) Subject(c context.Context, tp int32, oid int64) (s *model.Subject, err error) {
+	s = &model.Subject{}
+	row := d.dmReader.QueryRow(c, fmt.Sprintf(_selSubSQL, d.hitSubject(oid)), oid, tp)
+	if err = row.Scan(&s.ID, &s.Oid, &s.Type, &s.Pid, &s.Mid, &s.State, &s.Attr, &s.ACount, &s.Count, &s.MCount, &s.MoveCnt, &s.Maxlimit, &s.Childpool, &s.CTime, &s.MTime); err != nil {
+		if err == sql.ErrNoRows {
+			s = nil
+			err = nil
+		} else {
+			log.Error("row.Scan() error(%v)", err)
+		}
+	}
+	return
+}

app/job/main/dm/dao/subject_test.go

@@ -0,0 +1,17 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestSubject d.Subject() unit test.
+func TestSubject(t *testing.T) {
+	Convey("test dao.Subject", t, func() {
+		s, err := testDao.Subject(context.TODO(), 1, 1221)
+		So(err, ShouldBeNil)
+		So(s, ShouldNotBeNil)
+	})
+}