Create & Init Project...

app/job/main/tv/dao/ugc/BUILD

@@ -0,0 +1,85 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "databus_test.go",
+        "del_arc_test.go",
+        "del_upper_test.go",
+        "del_video_test.go",
+        "delete_test.go",
+        "import_test.go",
+        "manual_test.go",
+        "media_cache_test.go",
+        "passed_test.go",
+        "search_con_test.go",
+        "sync_video_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/tv/conf:go_default_library",
+        "//app/job/main/tv/model/ugc:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "databus.go",
+        "del_arc.go",
+        "del_upper.go",
+        "del_video.go",
+        "delete.go",
+        "import.go",
+        "manual.go",
+        "media_cache.go",
+        "passed.go",
+        "report_cid.go",
+        "search_con.go",
+        "sync_arc.go",
+        "sync_video.go",
+    ],
+    importpath = "go-common/app/job/main/tv/dao/ugc",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/tv/conf:go_default_library",
+        "//app/job/main/tv/dao/app:go_default_library",
+        "//app/job/main/tv/model/common:go_default_library",
+        "//app/job/main/tv/model/pgc:go_default_library",
+        "//app/job/main/tv/model/ugc:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/tv/dao/ugc/dao.go

@@ -0,0 +1,49 @@
+package ugc
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/tv/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+	httpx "go-common/library/net/http/blademaster"
+)
+
+// Dao dao.
+type Dao struct {
+	conf     *conf.Config
+	DB       *sql.DB
+	client   *httpx.Client
+	mc       *memcache.Pool
+	mcExpire int32 // expire for ugc cache, same as pgc auth, because it's daily refresh
+	criCID   int64 // critical cid for ugc video sync
+	redis    *redis.Pool
+}
+
+// New create a instance of Dao and return.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		conf:     c,
+		DB:       sql.NewMySQL(c.Mysql),
+		client:   httpx.NewClient(conf.Conf.HTTPClient),
+		mc:       memcache.NewPool(c.Memcache.Config),
+		mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
+		criCID:   c.UgcSync.Cfg.CriticalCid,
+		redis:    redis.NewPool(c.Redis.Config),
+	}
+	return
+}
+
+// Close close the redis and kafka resource.
+func (d *Dao) Close() {
+	if d.DB != nil {
+		d.DB.Close()
+	}
+}
+
+// BeginTran begin mysql transaction
+func (d *Dao) BeginTran(c context.Context) (*sql.Tx, error) {
+	return d.DB.Begin(c)
+}

app/job/main/tv/dao/ugc/dao_test.go

@@ -0,0 +1,70 @@
+package ugc
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/tv/conf"
+	"go-common/app/job/main/tv/model/ugc"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	ctx      = context.TODO()
+	d        *Dao
+	queryMid = "SELECT mid FROM ugc_archive WHERE result=1 AND valid=1 AND deleted=0 LIMIT 1"
+	queryAid = "SELECT aid FROM ugc_archive WHERE result=1 AND valid=1 AND deleted=0 LIMIT 1"
+)
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		dir, _ := filepath.Abs("../../cmd/tv-job-test.toml")
+		flag.Set("conf", dir)
+		conf.Init()
+		if d == nil {
+			d = New(conf.Conf)
+		}
+		f(d)
+	}
+}
+
+func TestDao_UpArcs(t *testing.T) {
+	Convey("TestDao_UpArcs", t, WithDao(func(d *Dao) {
+		var mid int64
+		d.DB.QueryRow(ctx, queryMid).Scan(&mid)
+		if mid == 0 {
+			return
+		}
+		res, err := d.UpArcs(ctx, mid)
+		So(err, ShouldBeNil)
+		So(len(res), ShouldBeGreaterThan, 0)
+	}))
+}
+
+func TestDao_SetArcCMS(t *testing.T) {
+	Convey("TestDao_SetArcCMS", t, WithDao(func(d *Dao) {
+		var aid int64
+		d.DB.QueryRow(ctx, queryAid).Scan(&aid)
+		if aid != 0 {
+			err := d.SetArcCMS(ctx, &ugc.ArcCMS{
+				AID:   aid,
+				Title: "test",
+			})
+			So(err, ShouldBeNil)
+			fmt.Println(aid)
+		}
+	}))
+}
+
+func TestDao_CountArcs(t *testing.T) {
+	Convey("TestDao_CountArcs", t, WithDao(func(d *Dao) {
+		res, err := d.CountUpArcs(ctx, 452156)
+		So(err, ShouldBeNil)
+		So(res, ShouldBeGreaterThan, 0)
+		fmt.Println(res)
+	}))
+}

app/job/main/tv/dao/ugc/databus.go

@@ -0,0 +1,92 @@
+package ugc
+
+import (
+	"context"
+
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+	arccli "go-common/app/service/main/archive/api"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_updateArc = "UPDATE ugc_archive SET title = ?, cover = ?, content = ?, pubtime = ?, " +
+		"typeid = ?, submit = ?, state = ? WHERE aid = ? AND deleted = 0"
+	_updateVideo = "UPDATE ugc_video SET eptitle = ?, index_order = ?, submit = ? WHERE cid = ? AND deleted = 0"
+	_needSubmit  = 1
+	_pickVideos  = "SELECT id, eptitle, cid, index_order FROM ugc_video WHERE aid = ? AND deleted = 0"
+	_upInList    = "SELECT mid FROM ugc_uploader WHERE mid = ? AND deleted = 0"
+	_setUploader = "REPLACE INTO ugc_uploader (mid, state) VALUES (?,?)"
+)
+
+// UpInList checks whether the upper is in our list
+func (d *Dao) UpInList(c context.Context, mid int64) (realID int64, err error) {
+	if err = d.DB.QueryRow(c, _upInList, mid).Scan(&realID); err != nil { // get the qualified aid to sync
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+		log.Error("d.UpInList.Query error(%v)", err)
+	}
+	return
+}
+
+// UpdateArc updates the key fields of an archive, used for databus monitoring
+func (d *Dao) UpdateArc(c context.Context, arc *ugcmdl.ArchDatabus) (err error) {
+	if _, err = d.DB.Exec(c, _updateArc,
+		arc.Title, arc.Cover, arc.Content, arc.PubTime, arc.TypeID, _needSubmit, arc.State, arc.Aid); err != nil {
+		log.Error("UpdateArc, failed to update: (%v), Error: %v", arc, err)
+	}
+	return
+}
+
+// TxUpdateVideo updates the ugc video's status, for databus update
+func (d *Dao) TxUpdateVideo(tx *sql.Tx, video *arccli.Page) (err error) {
+	if _, err = tx.Exec(_updateVideo,
+		video.Part, video.Page, _needSubmit, video.Cid); err != nil {
+		log.Error("TxUpdateVideo, failed to update: (%v), Error: %v", video, err)
+	}
+	return
+}
+
+// PickVideos picks the videos of an archive in one shot
+func (d *Dao) PickVideos(c context.Context, aid int64) (res map[int64]*ugcmdl.SimpleVideo, err error) {
+	var rows *sql.Rows
+	res = make(map[int64]*ugcmdl.SimpleVideo)
+	if rows, err = d.DB.Query(c, _pickVideos, aid); err != nil {
+		log.Error("d.Import.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r = &ugcmdl.SimpleVideo{}
+		if err = rows.Scan(&r.ID, &r.Eptitle, &r.CID, &r.IndexOrder); err != nil {
+			log.Error("PickVideos row.Scan() error(%v)", err)
+			return
+		}
+		res[r.CID] = r
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.PickVideos.Query error(%v)", err)
+	}
+	return
+}
+
+// TxAddVideos add into the db the new videos
+func (d *Dao) TxAddVideos(tx *sql.Tx, pages []*arccli.Page, aid int64) (err error) {
+	for _, v := range pages {
+		if _, err = tx.Exec(_importVideo, aid, v.Cid, v.Part, v.Page, v.Duration, v.Desc); err != nil {
+			log.Error("_importArc, failed to insert: (%v), Error: %v", v, err)
+			return
+		}
+	}
+	return
+}
+
+// TxUpAdd adds the upper
+func (d *Dao) TxUpAdd(tx *sql.Tx, mid int64) (err error) {
+	if _, err = tx.Exec(_setUploader, mid, 1); err != nil {
+		log.Error("UpAdd Error %v", err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/databus_test.go

@@ -0,0 +1,51 @@
+package ugc
+
+import (
+	"fmt"
+	"testing"
+
+	"go-common/app/service/main/archive/api"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_UpInList(t *testing.T) {
+	Convey("TestDao_UpInList", t, WithDao(func(d *Dao) {
+		res, err := d.UpInList(ctx, 27515615)
+		So(err, ShouldBeNil)
+		So(res, ShouldBeGreaterThan, 0)
+		fmt.Println(res)
+		res2, err2 := d.UpInList(ctx, 100997637777)
+		So(err2, ShouldBeNil)
+		So(res2, ShouldEqual, 0)
+	}))
+}
+
+func TestDao_PickVideos(t *testing.T) {
+	Convey("TestDao_PickVideos", t, WithDao(func(d *Dao) {
+		res, err := d.PickVideos(ctx, 10099763)
+		So(err, ShouldBeNil)
+		So(len(res), ShouldBeGreaterThan, 0)
+		for _, v := range res {
+			fmt.Println(v)
+		}
+	}))
+}
+
+func TestDao_InsertVideos(t *testing.T) {
+	Convey("TestDao_InsertVideos", t, WithDao(func(d *Dao) {
+		tx, err := d.BeginTran(ctx)
+		So(err, ShouldBeNil)
+		err = d.TxAddVideos(tx, []*api.Page{
+			{
+				Cid:      10126229,
+				Part:     "test",
+				Duration: 2333,
+				Desc:     "test",
+				Page:     999,
+			},
+		}, 10098693)
+		tx.Commit()
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/tv/dao/ugc/del_arc.go

@@ -0,0 +1,37 @@
+package ugc
+
+import (
+	"context"
+
+	"go-common/library/log"
+	"time"
+)
+
+const (
+	_deletedArc   = "SELECT aid FROM ugc_archive WHERE submit = 1 AND deleted = 1 AND retry < unix_timestamp(now()) LIMIT 1"
+	_finishDelArc = "UPDATE ugc_archive SET submit = 0 WHERE aid = ? AND deleted = 1"
+	_ppDelArc     = "UPDATE ugc_archive SET retry = ? WHERE aid = ? AND deleted = 1"
+)
+
+// DeletedArc picks the deleted archive to sync
+func (d *Dao) DeletedArc(c context.Context) (aid int64, err error) {
+	err = d.DB.QueryRow(c, _deletedArc).Scan(&aid)
+	return
+}
+
+// FinishDelArc updates the submit status from 1 to 0
+func (d *Dao) FinishDelArc(c context.Context, aid int64) (err error) {
+	if _, err = d.DB.Exec(c, _finishDelArc, aid); err != nil {
+		log.Error("FinishVideos Error: %v", aid, err)
+	}
+	return
+}
+
+// PpDelArc postpones the archive's submit in 30 mins
+func (d *Dao) PpDelArc(c context.Context, aid int64) (err error) {
+	var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
+	if _, err = d.DB.Exec(c, _ppDelArc, delay, aid); err != nil {
+		log.Error("PostponeArc, failed to delay: (%v,%v), Error: %v", delay, aid, err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/del_arc_test.go

@@ -0,0 +1,35 @@
+package ugc
+
+import (
+	"fmt"
+	"go-common/library/database/sql"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_DeletedArc(t *testing.T) {
+	Convey("TestDao_DeletedArc", t, WithDao(func(d *Dao) {
+		res, err := d.DeletedArc(ctx)
+		if err == sql.ErrNoRows {
+			fmt.Println("No to delete data")
+			return
+		}
+		So(err, ShouldBeNil)
+		So(res, ShouldBeGreaterThan, 0)
+	}))
+}
+
+func TestDao_PpDelArc(t *testing.T) {
+	Convey("TestDao_PpDelArc", t, WithDao(func(d *Dao) {
+		err := d.PpDelArc(ctx, 333)
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_FinishDelArc(t *testing.T) {
+	Convey("TestDao_FinishDelArc", t, WithDao(func(d *Dao) {
+		err := d.FinishDelArc(ctx, 333)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/tv/dao/ugc/del_upper.go

@@ -0,0 +1,70 @@
+package ugc
+
+import (
+	"context"
+	"time"
+
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_deletedUp   = "SELECT mid FROM ugc_uploader WHERE toinit = 2 AND deleted = 1 AND retry < unix_timestamp(now()) LIMIT 1"
+	_finishDelUp = "UPDATE ugc_uploader SET toinit = 0 WHERE mid = ? AND deleted = 1"
+	_ppDelUp     = "UPDATE ugc_uploader SET retry = ? WHERE mid = ? AND deleted = 1"
+	_upArcs      = "SELECT aid FROM ugc_archive WHERE mid = ? AND deleted = 0 LIMIT 50"
+	_upCountArc  = "SELECT count(1) FROM ugc_archive WHERE mid = ? AND deleted = 0"
+)
+
+// DeletedUp picks the deleted uppers, toinit = 2 and deleted = 1
+func (d *Dao) DeletedUp(c context.Context) (mid int64, err error) {
+	err = d.DB.QueryRow(c, _deletedUp).Scan(&mid)
+	return
+}
+
+// FinishDelUp updates the submit toinit from 2 to 0
+func (d *Dao) FinishDelUp(c context.Context, mid int64) (err error) {
+	if _, err = d.DB.Exec(c, _finishDelUp, mid); err != nil {
+		log.Error("FinishDelUp Error: %v", mid, err)
+	}
+	return
+}
+
+// PpDelUp postpones the upper's videos submit in 30 mins
+func (d *Dao) PpDelUp(c context.Context, mid int64) (err error) {
+	var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
+	if _, err = d.DB.Exec(c, _ppDelUp, delay, mid); err != nil {
+		log.Error("PostponeArc, failed to delay: (%v,%v), Error: %v", delay, mid, err)
+	}
+	return
+}
+
+// CountUpArcs counts the upper's archives
+func (d *Dao) CountUpArcs(c context.Context, mid int64) (count int64, err error) {
+	if err = d.DB.QueryRow(c, _upCountArc, mid).Scan(&count); err != nil {
+		log.Error("d.CountUpArcs.Query error(%v)", err)
+	}
+	return
+}
+
+// UpArcs picks 50 arcs of the upper
+func (d *Dao) UpArcs(c context.Context, mid int64) (aids []int64, err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Query(c, _upArcs, mid); err != nil { // get the qualified aid to sync
+		log.Error("d.UpArcs.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var aid int64
+		if err = rows.Scan(&aid); err != nil {
+			log.Error("ParseVideos row.Scan() error(%v)", err)
+			return
+		}
+		aids = append(aids, aid)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.UpArcs.Query error(%v)", err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/del_upper_test.go

@@ -0,0 +1,80 @@
+package ugc
+
+import (
+	"context"
+	"testing"
+
+	"database/sql"
+	"fmt"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestUgcDeletedUp(t *testing.T) {
+	var c = context.Background()
+	convey.Convey("DeletedUp", t, func(ctx convey.C) {
+		mid, err := d.DeletedUp(c)
+		if err == sql.ErrNoRows {
+			fmt.Println("No to delete data")
+			return
+		}
+		ctx.Convey("Then err should be nil.mid should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(mid, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestUgcFinishDelUp(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(0)
+	)
+	convey.Convey("FinishDelUp", t, func(ctx convey.C) {
+		err := d.FinishDelUp(c, mid)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestUgcPpDelUp(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(0)
+	)
+	convey.Convey("PpDelUp", t, func(ctx convey.C) {
+		err := d.PpDelUp(c, mid)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestUgcCountUpArcs(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(0)
+	)
+	convey.Convey("CountUpArcs", t, func(ctx convey.C) {
+		count, err := d.CountUpArcs(c, mid)
+		ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(count, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestUgcUpArcs(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(0)
+	)
+	convey.Convey("UpArcs", t, func(ctx convey.C) {
+		aids, err := d.UpArcs(c, mid)
+		ctx.Convey("Then err should be nil.aids should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(aids, convey.ShouldNotBeNil)
+		})
+	})
+}

app/job/main/tv/dao/ugc/del_video.go

@@ -0,0 +1,59 @@
+package ugc
+
+import (
+	"context"
+	"time"
+
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_deletedVideos  = "SELECT cid FROM ugc_video WHERE submit = 1 AND deleted = 1 AND retry < unix_timestamp(now()) LIMIT 5"
+	_finishDelVideo = "UPDATE ugc_video SET submit = 0 WHERE cid = ? AND deleted = 1"
+	_ppDelVideos    = "UPDATE ugc_video SET retry = ? WHERE cid = ? AND deleted = 1"
+)
+
+// DeletedVideos picks the deleted videos to sync
+func (d *Dao) DeletedVideos(c context.Context) (delIds []int, err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Query(c, _deletedVideos); err != nil { // get the qualified aid to sync
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var cid int
+		if err = rows.Scan(&cid); err != nil {
+			log.Error("ParseVideos row.Scan() error(%v)", err)
+			return
+		}
+		delIds = append(delIds, cid)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.deletedVideos.Query error(%v)", err)
+	}
+	return
+}
+
+// FinishDelVideos updates the submit status from 1 to 0
+func (d *Dao) FinishDelVideos(c context.Context, delIds []int) (err error) {
+	for _, v := range delIds {
+		if _, err = d.DB.Exec(c, _finishDelVideo, v); err != nil {
+			log.Error("FinishDelVideos Error: %v", v, err)
+			return
+		}
+	}
+	return
+}
+
+// PpDelVideos postpones the archive's videos submit in 30 mins
+func (d *Dao) PpDelVideos(c context.Context, delIds []int) (err error) {
+	var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
+	for _, v := range delIds {
+		if _, err = d.DB.Exec(c, _ppDelVideos, delay, v); err != nil {
+			log.Error("PpDelVideos, failed to delay: (%v,%v), Error: %v", delay, v, err)
+			return
+		}
+	}
+	return
+}

app/job/main/tv/dao/ugc/del_video_test.go

@@ -0,0 +1,23 @@
+package ugc
+
+import (
+	"testing"
+
+	"fmt"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_DeletedVideos(t *testing.T) {
+	Convey("TestDao_DeletedVideos", t, WithDao(func(d *Dao) {
+		res, err := d.DeletedVideos(ctx)
+		if err == nil && len(res) == 0 {
+			fmt.Println("No Delete Data")
+			d.DB.Exec(ctx, "UPDATE ugc_video SET deleted = 1, submit =1 WHERE deleted = 0 LIMIT 1")
+		}
+		res, err = d.DeletedVideos(ctx)
+		So(err, ShouldBeNil)
+		So(len(res), ShouldBeGreaterThan, 0)
+		fmt.Println(res)
+	}))
+}

app/job/main/tv/dao/ugc/delete.go

@@ -0,0 +1,78 @@
+package ugc
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/tv/model/ugc"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_delArc       = "UPDATE ugc_archive SET deleted = 1, submit = 1 WHERE aid = ? AND deleted = 0"
+	_delVideos    = "UPDATE ugc_video SET deleted = 1, submit = 1 WHERE aid = ? AND deleted = 0"
+	_delVideo     = "UPDATE ugc_video SET deleted = 1, submit = 1 WHERE cid = ? AND deleted = 0"
+	_delVideoCids = "UPDATE ugc_video SET deleted = 1, submit = 1 WHERE cid IN (%s) AND deleted = 0"
+	_checkVideos  = "SELECT cid FROM ugc_video WHERE aid = ? AND deleted = 0 LIMIT 1"
+)
+
+// TxDelArc deletes an arc
+func (d *Dao) TxDelArc(tx *sql.Tx, aid int64) (err error) {
+	if _, err = tx.Exec(_delArc, aid); err != nil {
+		log.Error("TxDelArc, failed to update: (%v), Error: %v", aid, err)
+	}
+	return
+}
+
+// DelVideos delete one archive all videos
+func (d *Dao) DelVideos(ctx context.Context, aid int64) (err error) {
+	if _, err = d.DB.Exec(ctx, _delVideos, aid); err != nil {
+		log.Error("DelVideos, failed to update: (%v), Error: %v", aid, err)
+		return
+	}
+	log.Info("Aid %d is deleted, delete its videos", aid)
+	return
+}
+
+// TxDelVideos deletes the videos of an arc
+func (d *Dao) TxDelVideos(tx *sql.Tx, aid int64) (err error) {
+	if _, err = tx.Exec(_delVideos, aid); err != nil {
+		log.Error("TxDelVideos, failed to update: (%v), Error: %v", aid, err)
+	}
+	return
+}
+
+// TxDelVideo deletes a video
+func (d *Dao) TxDelVideo(tx *sql.Tx, cid int64) (err error) {
+	if _, err = tx.Exec(_delVideo, cid); err != nil {
+		log.Error("TxDelVideo, failed to update: (%v), Error: %v", cid, err)
+	}
+	return
+}
+
+// DelVideoArc deletes some videos of an archive, if the archive is empty, also delete it
+func (d *Dao) DelVideoArc(ctx context.Context, req *ugc.DelVideos) (arcValid bool, err error) {
+	var cid int64
+	arcValid = true
+	if _, err = d.DB.Exec(ctx, fmt.Sprintf(_delVideoCids, xstr.JoinInts(req.CIDs))); err != nil {
+		log.Error("DelVideos Cids %v, Aid %d, Error: %v", req.CIDs, req.AID, err)
+		return
+	}
+	if err = d.DB.QueryRow(ctx, _checkVideos, req.AID).Scan(&cid); err != nil { // if no active videos, delete the arc
+		if err == sql.ErrNoRows {
+			err = nil
+			arcValid = false
+			if _, err = d.DB.Exec(ctx, _delArc, req.AID); err != nil {
+				log.Error("DelVideos DelArc Cids %v, Aid %d, Error: %v", req.CIDs, req.AID, err)
+				return
+			}
+			log.Info("DelArc Aid %d Because No Active Video", req.AID)
+		} else {
+			log.Error("DelVideos Cids %v, Aid %d, Error: %v", req.CIDs, req.AID, err)
+			return
+		}
+	}
+	return
+}

app/job/main/tv/dao/ugc/delete_test.go

@@ -0,0 +1,127 @@
+package ugc
+
+import (
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/tv/model/ugc"
+	arccli "go-common/app/service/main/archive/api"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestUgcTxDelArc(t *testing.T) {
+	var (
+		tx, _ = d.DB.Begin(ctx)
+		aid   = int64(0)
+	)
+	Convey("TxDelArc", t, func(ctx C) {
+		err := d.TxDelArc(tx, aid)
+		ctx.Convey("Then err should be nil.", func(ctx C) {
+			ctx.So(err, ShouldBeNil)
+		})
+	})
+}
+
+func TestUgcTxDelVideos(t *testing.T) {
+	var (
+		tx, _ = d.DB.Begin(ctx)
+		aid   = int64(0)
+	)
+	Convey("TxDelVideos", t, func(ctx C) {
+		err := d.TxDelVideos(tx, aid)
+		ctx.Convey("Then err should be nil.", func(ctx C) {
+			ctx.So(err, ShouldBeNil)
+		})
+	})
+}
+
+func TestUgcTxDelVideo(t *testing.T) {
+	var (
+		tx, _ = d.DB.Begin(ctx)
+		cid   = int64(0)
+	)
+	Convey("TxDelVideo", t, func(ctx C) {
+		err := d.TxDelVideo(tx, cid)
+		ctx.Convey("Then err should be nil.", func(ctx C) {
+			ctx.So(err, ShouldBeNil)
+		})
+	})
+}
+
+func TestDao_DelVideoArc(t *testing.T) {
+	Convey("TestDao_DelVideoArc", t, WithDao(func(d *Dao) {
+		var (
+			aid            = int64(99998888)
+			cid1           = int64(999988881)
+			cid2           = int64(999988882)
+			tx, _          = d.DB.Begin(ctx)
+			arc            = &arccli.Arc{Aid: aid}
+			countVQ        = "SELECT COUNT(1) FROM ugc_video WHERE aid = ? AND deleted = 0"
+			countAQ        = "SELECT COUNT(1) FROM ugc_archive WHERE aid = ? AND deleted = 0"
+			countV, countA int
+			arcValid       bool
+		)
+		// add archive and two videos
+		d.TxImportArc(tx, &arccli.Arc{Aid: aid})
+		d.TxMnlVideos(tx, &arccli.ViewReply{
+			Arc: arc,
+			Pages: []*arccli.Page{
+				{
+					Cid: cid1,
+				},
+				{
+					Cid: cid2,
+				},
+			},
+		})
+		tx.Commit()
+		d.DB.QueryRow(ctx, countVQ, aid).Scan(&countV)
+		So(countV, ShouldEqual, 2)
+		// delete one video, still one active video under the archive, we keep the archive
+		_, err := d.DelVideoArc(ctx, &ugc.DelVideos{
+			AID:  aid,
+			CIDs: []int64{cid1},
+		})
+		So(err, ShouldBeNil)
+		d.DB.QueryRow(ctx, countVQ, aid).Scan(&countV)
+		d.DB.QueryRow(ctx, countAQ, aid).Scan(&countA)
+		So(countV, ShouldEqual, 1)
+		So(countA, ShouldEqual, 1)
+		So(err, ShouldBeNil)
+		// delete the last video, the archive should also be deleted
+		arcValid, err = d.DelVideoArc(ctx, &ugc.DelVideos{
+			AID:  aid,
+			CIDs: []int64{cid2},
+		})
+		So(err, ShouldBeNil)
+		So(arcValid, ShouldBeFalse)
+		d.DB.QueryRow(ctx, countVQ, aid).Scan(&countV)
+		d.DB.QueryRow(ctx, countAQ, aid).Scan(&countA)
+		So(countV, ShouldEqual, 0)
+		So(countA, ShouldEqual, 0)
+	}))
+}
+
+func TestDao_DelVideos(t *testing.T) {
+	Convey("TestDao_DelVideos", t, WithDao(func(d *Dao) {
+		var (
+			aid       = int64(99998888)
+			cid1      = 99998887
+			cid2      = 99998886
+			insertSQL = "REPLACE INTO ugc_video (aid, cid) VALUES (%d, %d)"
+		)
+		ress, err2 := d.DB.Exec(ctx, fmt.Sprintf(insertSQL, aid, cid1))
+		fmt.Println(fmt.Sprintf(insertSQL, aid, cid1))
+		fmt.Println(err2)
+		fmt.Println(ress.RowsAffected())
+		d.DB.Exec(ctx, fmt.Sprintf(insertSQL, aid, cid2))
+		var count int
+		d.DB.QueryRow(ctx, "SELECT COUNT(1) FROM ugc_video WHERE aid = ? AND deleted = 0", aid).Scan(&count)
+		So(count, ShouldEqual, 2)
+		err := d.DelVideos(ctx, aid)
+		So(err, ShouldBeNil)
+		d.DB.QueryRow(ctx, "SELECT COUNT(1) FROM ugc_video WHERE aid = ? AND deleted = 0", aid).Scan(&count)
+		So(count, ShouldEqual, 0)
+	}))
+}

app/job/main/tv/dao/ugc/import.go

@@ -0,0 +1,96 @@
+package ugc
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+	"go-common/app/service/main/archive/api"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_import     = "SELECT mid FROM ugc_uploader WHERE toinit = 1 AND retry < UNIX_TIMESTAMP(now()) AND deleted = 0 LIMIT "
+	_postponeUp = "UPDATE ugc_uploader SET retry = ? WHERE mid = ? AND deleted = 0"
+	_finishUp   = "UPDATE ugc_uploader SET toinit = 0 WHERE mid = ? AND deleted = 0"
+	_filterAids = "SELECT aid FROM ugc_archive WHERE aid IN (%s) AND deleted = 0"
+	_importArc  = "REPLACE INTO ugc_archive(aid, videos, mid, typeid, title, cover, content, duration, copyright, pubtime, state) VALUES (?,?,?,?,?,?,?,?,?,?,?)"
+)
+
+// TxImportArc imports an arc
+func (d *Dao) TxImportArc(tx *sql.Tx, arc *api.Arc) (err error) {
+	if _, err = tx.Exec(_importArc, arc.Aid,
+		arc.Videos, arc.Author.Mid, arc.TypeID, arc.Title, arc.Pic, arc.Desc, arc.Duration,
+		arc.Copyright, arc.PubDate, arc.State); err != nil {
+		log.Error("_importArc, failed to update: (%v), Error: %v", arc, err)
+	}
+	return
+}
+
+// Import picks the uppers to init with the RPC data
+func (d *Dao) Import(c context.Context) (res []*ugcmdl.Upper, err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Query(c, _import+fmt.Sprintf("%d", d.conf.UgcSync.Batch.ImportNum)); err != nil {
+		log.Error("d.Import.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r = &ugcmdl.Upper{}
+		if err = rows.Scan(&r.MID); err != nil {
+			log.Error("Manual row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, r)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.Import.Query error(%v)", err)
+	}
+	return
+}
+
+// PpUpper means postpone upper init operation due to some error happened
+func (d *Dao) PpUpper(c context.Context, mid int64) (err error) {
+	var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
+	if _, err = d.DB.Exec(c, _postponeUp, delay, mid); err != nil {
+		log.Error("PpUpper, failed to delay: (%v,%v), Error: %v", delay, mid, err)
+	}
+	return
+}
+
+// FinishUpper updates the upper's to_init status to 0 means we finish the import operation
+func (d *Dao) FinishUpper(c context.Context, mid int64) (err error) {
+	if _, err = d.DB.Exec(c, _finishUp, mid); err != nil {
+		log.Error("FinishUpper, failed to Update: (%v,%v), Error: %v", _finishUp, mid, err)
+	}
+	return
+}
+
+// FilterExist filters the existing archives and remove them from the res, to have only non-existing data to insert
+func (d *Dao) FilterExist(c context.Context, res *map[int64]*api.Arc, aids []int64) (err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Query(c, fmt.Sprintf(_filterAids, xstr.JoinInts(aids))); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil // if non of them exist, it's good, we do nothing
+			return
+		}
+		log.Error("d._filterAids.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var aidEx int64
+		if err = rows.Scan(&aidEx); err != nil {
+			log.Error("Manual row.Scan() error(%v)", err)
+			return
+		}
+		delete(*res, aidEx) // remove existing data from the map
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.FilterExist.Query error(%v)", err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/import_test.go

@@ -0,0 +1,48 @@
+package ugc
+
+import (
+	"go-common/app/service/main/archive/api"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_FinishUpper(t *testing.T) {
+	Convey("TestDao_FinishUpper", t, WithDao(func(d *Dao) {
+		err := d.FinishUpper(ctx, 1)
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_Import(t *testing.T) {
+	Convey("TestDao_Import", t, WithDao(func(d *Dao) {
+		res, err := d.Import(ctx)
+		So(err, ShouldBeNil)
+		So(len(res), ShouldBeGreaterThan, 0)
+	}))
+}
+
+func TestDao_PpUpper(t *testing.T) {
+	Convey("TestDao_PpUpper", t, WithDao(func(d *Dao) {
+		err := d.PpUpper(ctx, 1)
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_BeginTran(t *testing.T) {
+	Convey("TestDao_BeginTran", t, WithDao(func(d *Dao) {
+		tx, err := d.BeginTran(ctx)
+		So(err, ShouldBeNil)
+		tx.Rollback()
+	}))
+}
+
+func TestDao_FilterExist(t *testing.T) {
+	Convey("TestDao_FilterExist", t, WithDao(func(d *Dao) {
+		pMap := make(map[int64]*api.Arc)
+		pAids := []int64{10106351, 10106309, 10106308, 10106307, 10106306, 10106284, 10105807, 10101484, 10100856, 10100855, 10100486, 10100146, 10100145, 10100144,
+			10100044, 10099332, 10099167, 10099150, 10099149, 10098970}
+		err := d.FilterExist(ctx, &pMap, pAids)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/tv/dao/ugc/manual.go

@@ -0,0 +1,93 @@
+package ugc
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+	arccli "go-common/app/service/main/archive/api"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_manual       = "SELECT id,aid FROM ugc_archive WHERE manual = 1 AND retry < UNIX_TIMESTAMP(now()) AND deleted = 0 LIMIT "
+	_postpone     = "UPDATE ugc_archive SET retry = ? WHERE aid = ? AND deleted = 0"
+	_importFinish = "UPDATE ugc_archive SET manual = 0 WHERE aid = ? AND deleted = 0"
+	_manualArc    = "UPDATE ugc_archive SET videos = ?, mid = ?, typeid = ?, title = ?, cover = ?, content = ?, duration = ?, " +
+		"copyright = ?, pubtime = ?, state = ?, submit = ? WHERE aid = ? AND deleted = 0"
+	_autoArc = "REPLACE INTO ugc_archive (videos, mid, typeid, title, cover, content, duration, copyright, pubtime, state, submit, aid) VALUES " +
+		"(?,?,?,?,?,?,?,?,?,?,?,?)"
+	_importVideo = "REPLACE INTO ugc_video (aid,cid,eptitle,index_order,duration,description) VALUES (?,?,?,?,?,?)"
+)
+
+// TxMnlArc updates the db with data from API
+func (d *Dao) TxMnlArc(tx *sql.Tx, arc *ugcmdl.Archive) (err error) {
+	if _, err = tx.Exec(_manualArc,
+		arc.Videos, arc.MID, arc.TypeID, arc.Title, arc.Cover, arc.Content, arc.Duration,
+		arc.Copyright, arc.Pubtime, arc.State, _needSubmit, arc.AID); err != nil {
+		log.Error("_importArc, failed to update: (%v), Error: %v", arc, err)
+	}
+	return
+}
+
+// TxAutoArc imports the db an arc
+func (d *Dao) TxAutoArc(tx *sql.Tx, arc *ugcmdl.Archive) (err error) {
+	if _, err = tx.Exec(_autoArc,
+		arc.Videos, arc.MID, arc.TypeID, arc.Title, arc.Cover, arc.Content, arc.Duration,
+		arc.Copyright, arc.Pubtime, arc.State, _needSubmit, arc.AID); err != nil {
+		log.Error("TxAutoArc, failed to update: (%v), Error: %v", arc, err)
+	}
+	return
+}
+
+// TxMnlVideos updates the db with data from API, if the
+func (d *Dao) TxMnlVideos(tx *sql.Tx, view *arccli.ViewReply) (err error) {
+	for _, v := range view.Pages {
+		if _, err = tx.Exec(_importVideo, view.Arc.Aid, v.Cid, v.Part, v.Page, v.Duration, v.Desc); err != nil {
+			log.Error("_importArc, failed to insert: (%v), Error: %v", v, err)
+			return
+		}
+	}
+	return
+}
+
+// TxMnlStatus updates the aid's manual status to 0
+func (d *Dao) TxMnlStatus(tx *sql.Tx, aid int64) (err error) {
+	if _, err = tx.Exec(_importFinish, aid); err != nil {
+		log.Error("_importFinish, failed to update: (%v), Error: %v", aid, err)
+	}
+	return
+}
+
+// Manual picks the archives that added manually
+func (d *Dao) Manual(c context.Context) (res []*ugcmdl.Archive, err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Query(c, _manual+fmt.Sprintf("%d", d.conf.UgcSync.Batch.ManualNum)); err != nil {
+		log.Error("d.Import.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r = &ugcmdl.Archive{}
+		if err = rows.Scan(&r.ID, &r.AID); err != nil {
+			log.Error("Manual row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, r)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.Manual.Query error(%v)", err)
+	}
+	return
+}
+
+// Ppmnl means postpone manual operation due to some error happened
+func (d *Dao) Ppmnl(c context.Context, aid int64) (err error) {
+	var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
+	if _, err = d.DB.Exec(c, _postpone, delay, aid); err != nil {
+		log.Error("Ppmnl, failed to delay: (%v,%v), Error: %v", delay, aid, err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/manual_test.go

@@ -0,0 +1,47 @@
+package ugc
+
+import (
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/tv/model/ugc"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_Manual(t *testing.T) {
+	Convey("TestDao_Manual", t, WithDao(func(d *Dao) {
+		res, err := d.Manual(ctx)
+		for _, v := range res {
+			fmt.Println(v)
+		}
+		So(err, ShouldBeNil)
+		So(len(res), ShouldBeGreaterThan, 0)
+	}))
+}
+
+func TestDao_Ppmnl(t *testing.T) {
+	Convey("TestDao_Ppmnl", t, WithDao(func(d *Dao) {
+		err := d.Ppmnl(ctx, 10099763)
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_UpdateArc(t *testing.T) {
+	Convey("TestDao_UpdateArc", t, WithDao(func(d *Dao) {
+		err := d.UpdateArc(ctx, &ugc.ArchDatabus{
+			Aid:       10099763,
+			Mid:       452156,
+			Videos:    1,
+			TypeID:    174,
+			Title:     "test",
+			Cover:     "testPic",
+			Content:   "testDesc",
+			Duration:  300,
+			Copyright: 1,
+			PubTime:   "2018-06-05",
+			State:     5,
+		})
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/tv/dao/ugc/media_cache.go

@@ -0,0 +1,176 @@
+package ugc
+
+import (
+	"context"
+	"fmt"
+
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+	"go-common/library/cache/memcache"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_mcArcCMSKey   = "arc_cms_%d"
+	_mcVideoCMSKey = "video_cms_%d"
+	_totalArcs     = "SELECT count(1) FROM ugc_archive WHERE mid = ?"
+	_totalVideos   = "SELECT count(1) FROM ugc_video"
+	_activeVideos  = "SELECT count(1) FROM ugc_video WHERE aid = ? AND deleted = 0"
+	_cntArcVideo   = "SELECT count(1) FROM ugc_video WHERE aid = ?"
+	_pickArcMC     = "SELECT title, aid, content, cover, typeid, pubtime, videos, valid, deleted, result,copyright, state, mid,duration FROM ugc_archive " +
+		"WHERE mid = %d LIMIT %d,%d"
+	_pickArcVideoMC = "SELECT cid, eptitle, aid, index_order, valid, deleted, result FROM ugc_video " +
+		"WHERE aid = ? AND cid > ? ORDER BY cid LIMIT 0,%d"
+	_transFailVideos = "SELECT cid FROM ugc_video WHERE aid = ? AND cid > %d AND transcoded = 2 and deleted = 0"
+)
+
+func arcCMSCacheKey(sid int64) string {
+	return fmt.Sprintf(_mcArcCMSKey, sid)
+}
+
+func videoCMSCacheKey(sid int) string {
+	return fmt.Sprintf(_mcVideoCMSKey, sid)
+}
+
+// SetArcCMS in MC
+func (d *Dao) SetArcCMS(ctx context.Context, res *ugcmdl.ArcCMS) (err error) {
+	var (
+		key  = arcCMSCacheKey(res.AID)
+		conn = d.mc.Get(ctx)
+	)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: key, Object: res, Expiration: d.mcExpire, Flags: memcache.FlagJSON}); err != nil {
+		log.Error("conn.Set(%s,%v) error(%v)", key, res, err)
+	}
+	return
+}
+
+// SetVideoCMS in MC
+func (d *Dao) SetVideoCMS(ctx context.Context, res *ugcmdl.VideoCMS) (err error) {
+	var (
+		key  = videoCMSCacheKey(res.CID)
+		conn = d.mc.Get(ctx)
+	)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: key, Object: res, Expiration: d.mcExpire, Flags: memcache.FlagJSON}); err != nil {
+		log.Error("conn.Set(%s,%v) error(%v)", key, res, err)
+	}
+	return
+}
+
+// UpArcsCnt counts the total number of arcs, including the deleted ones
+func (d *Dao) UpArcsCnt(c context.Context, mid int64) (count int, err error) {
+	if err = d.DB.QueryRow(c, _totalArcs, mid).Scan(&count); err != nil {
+		log.Error("d.UpArcsCnt.Query error(%v)", err)
+	}
+	return
+}
+
+// TotalVideos counts the total number of arcs, including the deleted ones
+func (d *Dao) TotalVideos(c context.Context) (count int, err error) {
+	if err = d.DB.QueryRow(c, _totalVideos).Scan(&count); err != nil {
+		log.Error("d.TotalVideos.Query error(%v)", err)
+	}
+	return
+}
+
+// ActVideos checks whether there is still some active videos
+func (d *Dao) ActVideos(c context.Context, aid int64) (has bool, err error) {
+	var count int
+	if err = d.DB.QueryRow(c, _activeVideos, aid).Scan(&count); err != nil {
+		log.Error("d.TotalVideos.Query error(%v)", err)
+		return
+	}
+	if count == 0 {
+		has = false
+	} else {
+		has = true
+	}
+	return
+}
+
+// ArcVideoCnt counts one archive's video
+func (d *Dao) ArcVideoCnt(c context.Context, aid int64) (count int, err error) {
+	if err = d.DB.QueryRow(c, _cntArcVideo, aid).Scan(&count); err != nil {
+		log.Error("d.TotalVideos.Query error(%v)", err)
+	}
+	return
+}
+
+// PickUpArcs picks data by Piece to sync in MC, attention: nbPiece begins from zero
+func (d *Dao) PickUpArcs(ctx context.Context, mid, nbPiece, nbData int) (res []*ugcmdl.ArcFull, err error) {
+	var (
+		rows  *sql.Rows
+		query = fmt.Sprintf(_pickArcMC, mid, nbPiece*nbData, nbData)
+	)
+	if rows, err = d.DB.Query(ctx, query); err != nil {
+		log.Error("d.PickUpArcs.Query: %s error(%v)", query, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var li = &ugcmdl.ArcFull{}
+		// SELECT title, aid, content, cover, typeid, pubtime, videos, valid, deleted, result,copyright, state, mid,duration
+		if err = rows.Scan(&li.Title, &li.AID, &li.Content, &li.Cover, &li.TypeID, &li.Pubtime,
+			&li.Videos, &li.Valid, &li.Deleted, &li.Result, &li.Copyright, &li.State, &li.MID, &li.Duration); err != nil {
+			log.Error("PickUpArcs row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, li)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.PickUpArcs.Query error(%v)", err)
+	}
+	return
+}
+
+// PickArcVideo picks data by Piece to sync in MC
+func (d *Dao) PickArcVideo(ctx context.Context, aid int64, LastID int, nbData int) (res []*ugcmdl.VideoCMS, myLast int, err error) {
+	var (
+		rows  *sql.Rows
+		query = fmt.Sprintf(_pickArcVideoMC, nbData)
+	)
+	if rows, err = d.DB.Query(ctx, query, aid, LastID); err != nil {
+		log.Error("d._pickArcVideoMC.Query: %s error(%v)", query, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var li = new(ugcmdl.VideoCMS)
+		if err = rows.Scan(&li.CID, &li.Title, &li.AID, &li.IndexOrder, &li.Valid, &li.Deleted, &li.Result); err != nil {
+			log.Error("PickVideoMC row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, li)
+		myLast = li.CID
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.PickArcVideo.Query error(%v)", err)
+	}
+	return
+}
+
+// TransFailVideos picks transcoding failure videos
+func (d *Dao) TransFailVideos(ctx context.Context, aid int64) (cids []int64, err error) {
+	var (
+		rows  *sql.Rows
+		query = fmt.Sprintf(_transFailVideos, d.criCID)
+	)
+	if rows, err = d.DB.Query(ctx, query, aid); err != nil {
+		log.Error("d.TransFailVideos.Query: Cid %d error(%v)", aid, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var li int64
+		if err = rows.Scan(&li); err != nil {
+			log.Error("PickVideoMC row.Scan() error(%v)", err)
+			return
+		}
+		cids = append(cids, li)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.TransFailVideos.Query error(%v)", err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/media_cache_test.go

@@ -0,0 +1,132 @@
+package ugc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_TotalVideos(t *testing.T) {
+	Convey("TestDao_TotalVideos", t, WithDao(func(d *Dao) {
+		res, err := d.TotalVideos(ctx)
+		So(err, ShouldBeNil)
+		So(res, ShouldBeGreaterThan, 0)
+		fmt.Println(res)
+	}))
+}
+
+func TestDao_ArcVideoCnt(t *testing.T) {
+	Convey("TestDao_ArcVideoCnt", t, WithDao(func(d *Dao) {
+		var aid int64
+		d.DB.QueryRow(ctx, "select aid from ugc_video where deleted = 0 limit 1").Scan(&aid)
+		if aid == 0 {
+			fmt.Println("empty arc")
+			return
+		}
+		cnt, errcnt := d.ArcVideoCnt(ctx, aid)
+		So(errcnt, ShouldBeNil)
+		fmt.Println(aid)
+		So(cnt, ShouldBeGreaterThan, 0)
+		fmt.Println(cnt)
+		resVideos, lastIDVideo, errVd := d.PickArcVideo(ctx, aid, 0, 10)
+		So(errVd, ShouldBeNil)
+		So(len(resVideos), ShouldBeGreaterThan, 0)
+		So(lastIDVideo, ShouldBeGreaterThan, 0)
+		str, _ := json.Marshal(resVideos)
+		fmt.Println(string(str))
+	}))
+}
+
+func TestDao_SetArc(t *testing.T) {
+	Convey("TestDao_SetArc", t, WithDao(func(d *Dao) {
+		err := d.SetArcCMS(ctx, &ugcmdl.ArcCMS{
+			Title: "testtest",
+			AID:   777,
+		})
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_UpArcsCnt(t *testing.T) {
+	Convey("TestDao_UpArcsCnt", t, WithDao(func(d *Dao) {
+		var mid int64
+		d.DB.QueryRow(ctx, "select mid from ugc_archive where deleted = 0 limit 1").Scan(&mid)
+		if mid == 0 {
+			fmt.Println("empty arc")
+			return
+		}
+		count, err := d.UpArcsCnt(ctx, mid)
+		So(err, ShouldBeNil)
+		So(count, ShouldBeGreaterThan, 0)
+		fmt.Println("mid ", mid, " cnt", count)
+		if count > 1 {
+			d.DB.Exec(context.Background(), "update ugc_archive set deleted = 1 where mid = ? and deleted = 0 limit 1", mid)
+			cntNonDeleted, err2 := d.CountUpArcs(context.Background(), mid)
+			So(err2, ShouldBeNil)
+			So(count, ShouldBeGreaterThan, cntNonDeleted)
+			fmt.Println("all: ", count, " non-deleted: ", cntNonDeleted)
+		}
+	}))
+}
+
+func TestDao_TransFailVideos(t *testing.T) {
+	Convey("TestDao_TransFailVideos", t, WithDao(func(d *Dao) {
+		query := "SELECT aid FROM ugc_video WHERE  cid > 12780000 AND transcoded = 2 and deleted = 0 limit 1"
+		var aid int64
+		d.DB.QueryRow(context.Background(), query).Scan(&aid)
+		if aid == 0 {
+			fmt.Println("Empty archives")
+			return
+		}
+		cids, err := d.TransFailVideos(ctx, aid)
+		So(err, ShouldBeNil)
+		So(len(cids), ShouldBeGreaterThan, 0)
+		fmt.Println("aid ", aid, " cids ", cids)
+	}))
+}
+
+func TestDao_ActVideos(t *testing.T) {
+	Convey("TestDao_ActVideos", t, WithDao(func(d *Dao) {
+		var (
+			aid = int64(88888888)
+			cid = 99999999
+		)
+		insertSQL := "REPLACE INTO ugc_video (aid, cid, deleted) VALUES (%d, %d, 1)"
+		d.DB.Exec(ctx, fmt.Sprintf(insertSQL, aid, cid))
+		has, err := d.ActVideos(ctx, aid)
+		So(err, ShouldBeNil)
+		So(has, ShouldBeFalse)
+		d.DB.Exec(ctx, "UPDATE ugc_video SET deleted = 0 WHERE cid = ?", cid)
+		has, err = d.ActVideos(ctx, aid)
+		So(err, ShouldBeNil)
+		So(has, ShouldBeTrue)
+	}))
+}
+
+func TestDao_PickArcMC(t *testing.T) {
+	Convey("TestDao_PickArcMC", t, WithDao(func(d *Dao) {
+		pickMid := "select mid from ugc_archive where deleted = 0 group by mid order by count(aid) desc limit 1"
+		var mid = 0
+		d.DB.QueryRow(ctx, pickMid).Scan(&mid)
+		if mid == 0 {
+			fmt.Println("empty archive")
+			return
+		}
+		fmt.Println("mid ", mid)
+		res1, err1 := d.PickUpArcs(ctx, mid, 0, 5)
+		So(err1, ShouldBeNil)
+		So(len(res1), ShouldBeGreaterThan, 0)
+		res2, err2 := d.PickUpArcs(ctx, mid, 7, 5)
+		So(err2, ShouldBeNil)
+		So(len(res2), ShouldBeGreaterThan, 0)
+		str1, _ := json.Marshal(res1)
+		str2, _ := json.Marshal(res2)
+		fmt.Println(string(str1))
+		fmt.Println(string(str2))
+	}))
+}

app/job/main/tv/dao/ugc/passed.go

@@ -0,0 +1,44 @@
+package ugc
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/job/main/tv/model/common"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_passedArc = "SELECT aid, ctime FROM ugc_archive WHERE typeid IN (%s) AND result = 1 AND valid = 1 and deleted = 0"
+)
+
+// PassedArcs picks the passed Arc data for Idx Page
+func (d *Dao) PassedArcs(c context.Context, tids []int64) (res []*common.IdxRank, err error) {
+	if len(tids) == 0 {
+		return
+	}
+
+	var (
+		rows    *sql.Rows
+		tidsStr = xstr.JoinInts(tids)
+	)
+	if rows, err = d.DB.Query(c, fmt.Sprintf(_passedArc, tidsStr)); err != nil {
+		log.Error("d.PassedArcs.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r = &common.IdxRank{}
+		if err = rows.Scan(&r.ID, &r.Ctime); err != nil {
+			log.Error("PassedArcs row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, r)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.PassedArcs.Query error(%v)", err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/passed_test.go

@@ -0,0 +1,19 @@
+package ugc
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_PassedArcs(t *testing.T) {
+	Convey("TestDao_PassedArcs", t, WithDao(func(d *Dao) {
+		res, err := d.PassedArcs(ctx, []int64{76, 75})
+		So(err, ShouldBeNil)
+		So(len(res), ShouldBeGreaterThan, 0)
+		data, _ := json.Marshal(res)
+		fmt.Println(string(data))
+	}))
+}

app/job/main/tv/dao/ugc/report_cid.go

@@ -0,0 +1,54 @@
+package ugc
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	appDao "go-common/app/job/main/tv/dao/app"
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_reported     = 1
+	_finishReport = "UPDATE ugc_video SET mark = %d WHERE cid IN (%s)"
+)
+
+// RepCidBatch reports cid info to VideoCloud api
+func (d *Dao) RepCidBatch(c context.Context, cidReq []*ugcmdl.CidReq) (err error) {
+	var (
+		reportURL = d.conf.UgcSync.Cfg.ReportCidURL
+		resp      = ugcmdl.CidResp{}
+		jsonBody  []byte
+	)
+	if jsonBody, err = json.Marshal(cidReq); err != nil {
+		log.Error("json.Marchal(%v) error(%v)", cidReq, err)
+		return
+	}
+	req, err := http.NewRequest(http.MethodPost, reportURL, bytes.NewBuffer(jsonBody))
+	if err != nil {
+		log.Error("http.NewRequest err - %v", err)
+		return
+	}
+	if err = d.client.Do(c, req, &resp); err != nil {
+		log.Error("ReportCid jsonBody (%s) error(%v)", string(jsonBody), err)
+		return
+	}
+	if resp.Code != 0 {
+		return fmt.Errorf("ReportCid Error Code:%d, Msg:%s", resp.Code, resp.Message)
+	}
+	appDao.PromInfo("ReportCid:Succ")
+	return
+}
+
+// FinishReport change's the ugc_video's state from 0 to 1, means it has already been reported
+func (d *Dao) FinishReport(c context.Context, cids []int64) (err error) {
+	if _, err = d.DB.Exec(c, fmt.Sprintf(_finishReport, _reported, xstr.JoinInts(cids))); err != nil {
+		log.Error("FinishReport Error: %v", cids, err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/search_con.go

@@ -0,0 +1,55 @@
+package ugc
+
+import (
+	"context"
+
+	model "go-common/app/job/main/tv/model/pgc"
+	"go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_UgcFromwhere = "FROM ugc_archive WHERE result=1 AND valid=1 AND deleted=0 "
+	_UgcCont      = "SELECT aid,title,cover,`content`,pubtime,typeid " + _UgcFromwhere + "AND aid > ? ORDER BY aid ASC LIMIT ?"
+	_UgcContCount = " SELECT count(*) " + _UgcFromwhere
+)
+
+// UgcCont is used for getting valid ugc archive data
+func (d *Dao) UgcCont(ctx context.Context, aid int, limit int) (res []*model.SearUgcCon, maxID int, err error) {
+	var (
+		rows *sql.Rows
+	)
+	if rows, err = d.DB.Query(ctx, _UgcCont, aid, limit); err != nil {
+		log.Error("d.UgcCont.Query: %s error(%v)", _UgcCont, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r = &model.SearUgcCon{}
+		if err = rows.Scan(&r.AID, &r.Title, &r.Cover, &r.Content, &r.Pubtime, &r.Typeid); err != nil {
+			log.Error("UgcCont row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, r)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.UgcCont.Query error(%v)", err)
+		return
+	}
+	if len(res) == 0 {
+		err = ecode.NothingFound
+		return
+	}
+	maxID = res[len(res)-1].AID
+	return
+}
+
+// UgcCnt is used for getting valid data count
+func (d *Dao) UgcCnt(ctx context.Context) (upCnt int, err error) {
+	row := d.DB.QueryRow(ctx, _UgcContCount)
+	if err = row.Scan(&upCnt); err != nil {
+		log.Error("d.SeaContCount.Query: %s error(%v)", _UgcContCount, err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/search_con_test.go

@@ -0,0 +1,42 @@
+package ugc
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_UgcCont(t *testing.T) {
+	Convey("TestDao_UgcCont", t, WithDao(func(d *Dao) {
+		var aid int
+		d.DB.QueryRow(ctx, queryAid).Scan(&aid)
+		if aid == 0 {
+			fmt.Println("No ready audit Data")
+			return
+		}
+		if res, maxID, err := d.UgcCont(ctx, aid, 50); err != nil {
+			fmt.Println(err)
+			So(len(res), ShouldEqual, 0)
+			So(maxID, ShouldBeZeroValue)
+		} else {
+			So(len(res), ShouldBeGreaterThan, 0)
+			for _, v := range res {
+				fmt.Println(v)
+			}
+			So(maxID, ShouldBeGreaterThan, 0)
+		}
+	}))
+}
+
+func TestDao_UgcContCount(t *testing.T) {
+	Convey("TestDao_UgcContCount", t, WithDao(func(d *Dao) {
+		var aid int
+		d.DB.QueryRow(ctx, queryAid).Scan(&aid)
+		if aid != 0 {
+			res, _ := d.UgcCnt(ctx)
+			So(res, ShouldBeGreaterThan, 0)
+			fmt.Println(res)
+		}
+	}))
+}

app/job/main/tv/dao/ugc/sync_arc.go

@@ -0,0 +1,15 @@
+package ugc
+
+import (
+	"context"
+
+	"go-common/library/log"
+)
+
+// FinishArc updates the submit status from 1 to 0
+func (d *Dao) FinishArc(c context.Context, aid int64) (err error) {
+	if _, err = d.DB.Exec(c, _finishArc, aid); err != nil {
+		log.Error("FinishVideos Error: %v", aid, err)
+	}
+	return
+}

app/job/main/tv/dao/ugc/sync_video.go

@@ -0,0 +1,115 @@
+package ugc
+
+import (
+	"context"
+	"fmt"
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"time"
+)
+
+const (
+	_videoCond = " AND (v.transcoded = 1 OR v.cid <= %d) " +
+		"AND v.retry < unix_timestamp(now()) " +
+		"AND v.deleted = 0 "
+	_parseVideos = "SELECT id,cid,index_order,eptitle,duration,description FROM ugc_video v " +
+		"WHERE v.aid = ? AND v.submit = 1 " + _videoCond
+	_postArc      = "UPDATE ugc_video SET retry = ? WHERE cid = ? AND deleted = 0"
+	_finishVideos = "UPDATE ugc_video SET submit = 0 WHERE cid = ? AND aid = ? AND deleted = 0"
+	_finishArc    = "UPDATE ugc_archive SET submit = 0 WHERE aid = ? AND deleted = 0"
+	_parseArc     = "SELECT id,aid,mid,typeid,videos,title,cover,content,duration,copyright,pubtime,ctime,mtime,state," +
+		"manual,valid,submit,retry,result,deleted FROM ugc_archive WHERE aid = ?"
+	_shouldAudit = "SELECT COUNT(1) as cnt FROM ugc_video v WHERE v.aid = ? AND v.submit = 1 " + _videoCond
+	_videoSubmit = "SELECT cid FROM ugc_video v WHERE v.aid = ? AND v.submit = 0 " + _videoCond + " LIMIT 1"
+)
+
+// PpVideos postpones the archive's videos submit in 30 mins
+func (d *Dao) PpVideos(c context.Context, cids []int64) (err error) {
+	var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
+	for _, v := range cids {
+		if _, err = d.DB.Exec(c, _postArc, delay, v); err != nil {
+			log.Error("PostponeArc, failed to delay: (%v,%v), Error: %v", delay, v, err)
+			return
+		}
+	}
+	return
+}
+
+// FinishVideos updates the submit status from 1 to 0
+func (d *Dao) FinishVideos(c context.Context, videos []*ugcmdl.SimpleVideo, aid int64) (err error) {
+	for _, v := range videos {
+		if _, err = d.DB.Exec(c, _finishVideos, v.CID, aid); err != nil { // avoid updating the cid under another archive
+			log.Error("FinishVideos Error: %v", v.CID, err)
+			return
+		}
+	}
+	if _, err = d.DB.Exec(c, _finishArc, aid); err != nil {
+		log.Error("FinishVideos Error: %v", aid, err)
+	}
+	return
+}
+
+// ParseArc parses one archive data
+func (d *Dao) ParseArc(c context.Context, aid int64) (res *ugcmdl.Archive, err error) {
+	res = &ugcmdl.Archive{}
+	if err = d.DB.QueryRow(c, _parseArc, aid).Scan(&res.ID, &res.AID, &res.MID, &res.TypeID, &res.Videos, &res.Title,
+		&res.Cover, &res.Content, &res.Duration, &res.Copyright, &res.Pubtime, &res.Ctime, &res.Mtime, &res.State,
+		&res.Manual, &res.Valid, &res.Submit, &res.Retry, &res.Result, &res.Deleted); err != nil { // get the qualified aid to sync
+		log.Warn("d.ParseArc.Query error(%v)", err)
+	}
+	return
+}
+
+// ShouldAudit distinguishes whether the archive should ask for audit or not
+func (d *Dao) ShouldAudit(c context.Context, aid int64) (res bool, err error) {
+	var cnt int
+	if err = d.DB.QueryRow(c, fmt.Sprintf(_shouldAudit, d.criCID), aid).Scan(&cnt); err != nil {
+		log.Error("d.ShouldAudit Aid %d Err %v", aid, err)
+		return
+	}
+	res = cnt > 0
+	return
+}
+
+// VideoSubmit tells whether the archive already has some video submitted
+func (d *Dao) VideoSubmit(c context.Context, aid int64) (cid int64, err error) {
+	if err = d.DB.QueryRow(c, fmt.Sprintf(_videoSubmit, d.criCID), aid).Scan(&cid); err != nil {
+		log.Warn("d.videoSubmit Aid %d, Err %v", aid, err)
+	}
+	return
+}
+
+// ParseVideos picks 20 videos of one qualified archive
+func (d *Dao) ParseVideos(c context.Context, aid int64, ps int) (res [][]*ugcmdl.SimpleVideo, err error) {
+	var (
+		rows   *sql.Rows
+		videos []*ugcmdl.SimpleVideo
+	)
+	if rows, err = d.DB.Query(c, fmt.Sprintf(_parseVideos, d.criCID), aid); err != nil {
+		log.Error("d._parseVideos.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r = ugcmdl.SimpleVideo{}
+		if err = rows.Scan(&r.ID, &r.CID, &r.IndexOrder, &r.Eptitle, &r.Duration, &r.Description); err != nil {
+			log.Error("ParseVideos row.Scan() error(%v)", err)
+			return
+		}
+		videos = append(videos, &r)
+		if len(videos) >= ps {
+			var videoPce = append([]*ugcmdl.SimpleVideo{}, videos...)
+			videos = []*ugcmdl.SimpleVideo{}
+			res = append(res, videoPce)
+		}
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d._parseVideos.Query error(%v)", err)
+		return
+	}
+	if len(videos) > 0 {
+		res = append(res, videos)
+	}
+	return
+}

app/job/main/tv/dao/ugc/sync_video_test.go

@@ -0,0 +1,114 @@
+package ugc
+
+import (
+	"context"
+	ugcmdl "go-common/app/job/main/tv/model/ugc"
+	"testing"
+
+	"fmt"
+
+	"encoding/json"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestUgcPpVideos(t *testing.T) {
+	var (
+		c      = context.Background()
+		videos = []int64{}
+	)
+	convey.Convey("PpVideos", t, func(ctx convey.C) {
+		err := d.PpVideos(c, videos)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestUgcFinishVideos(t *testing.T) {
+	var (
+		c      = context.Background()
+		videos = []*ugcmdl.SimpleVideo{}
+		aid    = int64(0)
+	)
+	convey.Convey("FinishVideos", t, func(ctx convey.C) {
+		err := d.FinishVideos(c, videos, aid)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDao_ParseArc(t *testing.T) {
+	convey.Convey("TestDao_ParseArc", t, WithDao(func(d *Dao) {
+		var (
+			query = "SELECT aid FROM ugc_archive WHERE deleted = 0 LIMIT 1"
+			arc   = &ugcmdl.ArcCMS{}
+		)
+		d.DB.QueryRow(ctx, query).Scan(&arc.AID)
+		res, err := d.ParseArc(ctx, int64(arc.AID))
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(res, convey.ShouldNotBeNil)
+		fmt.Println(res)
+	}))
+}
+
+func TestDao_ParseVideos(t *testing.T) {
+	convey.Convey("TestDao_ParseVideos", t, WithDao(func(d *Dao) {
+		var (
+			query = "SELECT aid FROM ugc_video WHERE deleted = 0 AND submit = 1 AND cid < 12780000 LIMIT 1"
+			aid   int64
+		)
+		if err := d.DB.QueryRow(ctx, query).Scan(&aid); err != nil {
+			fmt.Println("No to submit data")
+			return
+		}
+		fmt.Println(aid)
+		res, err := d.ParseVideos(ctx, int64(aid), 2)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(len(res), convey.ShouldBeGreaterThan, 0)
+		str, _ := json.Marshal(res)
+		fmt.Println(string(str))
+		for _, v := range res {
+			convey.So(len(v), convey.ShouldBeGreaterThan, 0)
+		}
+	}))
+}
+
+func TestDao_ShouldAudit(t *testing.T) {
+	convey.Convey("TestDao_ShouldAudit", t, WithDao(func(d *Dao) {
+		var (
+			query = "SELECT aid FROM ugc_video v WHERE v.submit = 1 " + _videoCond + " LIMIT 1"
+			aid   int64
+		)
+		fmt.Println(query)
+		d.DB.QueryRow(ctx, fmt.Sprintf(query, d.criCID)).Scan(&aid)
+		if aid == 0 {
+			fmt.Println("db empty")
+			return
+		}
+		res, err := d.ShouldAudit(ctx, aid)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(res, convey.ShouldBeTrue)
+		fmt.Println(aid, " ", res)
+	}))
+}
+
+func TestDao_VideoSubmit(t *testing.T) {
+	convey.Convey("TestDao_VideoSubmit", t, WithDao(func(d *Dao) {
+		var (
+			query = "SELECT aid FROM ugc_video v WHERE v.submit = 0 " + _videoCond + " LIMIT 1"
+			aid   int64
+		)
+		fmt.Println(query)
+		d.DB.QueryRow(ctx, fmt.Sprintf(query, d.criCID)).Scan(&aid)
+		if aid == 0 {
+			fmt.Println("db empty")
+			return
+		}
+		res, err := d.VideoSubmit(ctx, aid)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(res, convey.ShouldNotBeNil)
+		fmt.Println(aid, " ", res)
+	}))
+}