Create & Init Project...

app/job/main/tv/dao/upper/BUILD

@@ -0,0 +1,63 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "load_upper_test.go",
+        "refresh_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/tv/conf:go_default_library",
+        "//app/job/main/tv/model/ugc:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "load_upper.go",
+        "refresh.go",
+        "sync.go",
+    ],
+    importpath = "go-common/app/job/main/tv/dao/upper",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/tv/conf:go_default_library",
+        "//app/job/main/tv/model/ugc:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//app/service/main/account/model:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/tv/dao/upper/dao.go

@@ -0,0 +1,57 @@
+package upper
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/tv/conf"
+	acccli "go-common/app/service/main/account/api"
+	accwar "go-common/app/service/main/account/api"
+	account "go-common/app/service/main/account/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/database/sql"
+	"go-common/library/stat/prom"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	missedCount = prom.CacheMiss
+	cachedCount = prom.CacheHit
+)
+
+// Dao is account dao.
+type Dao struct {
+	accClient accwar.AccountClient
+	mc        *memcache.Pool
+	mcExpire  int32
+	DB        *sql.DB
+}
+
+// New account dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		mc:       memcache.NewPool(c.Memcache.Config),
+		mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
+		DB:       sql.NewMySQL(c.Mysql),
+	}
+	var err error
+	if d.accClient, err = acccli.NewClient(c.AccClient); err != nil {
+		panic(err)
+	}
+	return
+}
+
+// Card3 get card info by mid
+func (d *Dao) Card3(c context.Context, mid int64) (res *account.Card, err error) {
+	var (
+		arg  = &accwar.MidReq{Mid: mid}
+		resp *accwar.CardReply
+	)
+	if resp, err = d.accClient.Card3(c, arg); err != nil {
+		err = errors.Wrapf(err, "%v", arg)
+		return
+	}
+	res = resp.Card
+	return
+}

app/job/main/tv/dao/upper/dao_test.go

@@ -0,0 +1,98 @@
+package upper
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/tv/conf"
+	ugcMdl "go-common/app/job/main/tv/model/ugc"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	ctx = context.TODO()
+	d   *Dao
+)
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		dir, _ := filepath.Abs("../../cmd/tv-job-test.toml")
+		flag.Set("conf", dir)
+		conf.Init()
+		if d == nil {
+			d = New(conf.Conf)
+		}
+		f(d)
+	}
+}
+
+func TestDao_LoadUpMeta(t *testing.T) {
+	Convey("TestDao_LoadUpMeta", t, WithDao(func(d *Dao) {
+		res, err := d.LoadUpMeta(ctx, 88895270)
+		So(err, ShouldBeNil)
+		fmt.Println(res)
+	}))
+}
+
+func TestDao_CountUP(t *testing.T) {
+	Convey("TestDao_CountUP", t, WithDao(func(d *Dao) {
+		res, err := d.CountUP(ctx)
+		So(err, ShouldBeNil)
+		fmt.Println(res)
+	}))
+}
+
+func TestDao_PickUppers(t *testing.T) {
+	Convey("TestDao_PickUppers", t, WithDao(func(d *Dao) {
+		res, myLast, err := d.PickUppers(ctx, 0, 50)
+		So(err, ShouldBeNil)
+		So(myLast, ShouldBeGreaterThan, 0)
+		So(len(res), ShouldBeGreaterThan, 0)
+		fmt.Println(myLast)
+	}))
+}
+
+func TestDao_RefreshUp(t *testing.T) {
+	Convey("TestDao_RefreshUp", t, WithDao(func(d *Dao) {
+		res, _, _ := d.PickUppers(ctx, 0, 50)
+		if len(res) > 0 {
+			err := d.RefreshUp(ctx, &ugcMdl.ReqSetUp{
+				MID:    res[0],
+				Value:  "Face666",
+				UpType: _upFace,
+			})
+			So(err, ShouldBeNil)
+		}
+	}))
+}
+
+func TestDao_TosyncUps(t *testing.T) {
+	Convey("TestDao_TosyncUps", t, WithDao(func(d *Dao) {
+		mids, err := d.TosyncUps(ctx)
+		So(err, ShouldBeNil)
+		fmt.Println(mids)
+	}))
+}
+
+func TestDao_FinsyncUps(t *testing.T) {
+	Convey("TestDao_FinsyncUps", t, WithDao(func(d *Dao) {
+		err := d.FinsyncUps(ctx, 88895270)
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_IsSame(t *testing.T) {
+	Convey("TestDao_IsSame", t, WithDao(func(d *Dao) {
+		var u = &ugcMdl.Upper{
+			OriFace: "http://i0.hdslb.com/bfs/face/589568b0b0228d5a38b2e6f14b83bcb174637140.jpg",
+		}
+		var faceSame, _ = u.IsSame("", "http://i1.hdslb.com/bfs/face/589568b0b0228d5a38b2e6f14b83bcb174637140.jpg")
+		So(faceSame, ShouldBeTrue)
+		faceSame, _ = u.IsSame("", "http://i0.hdslb.com/bfs/face/xxx.jpg")
+		So(faceSame, ShouldBeFalse)
+	}))
+}

app/job/main/tv/dao/upper/load_upper.go

@@ -0,0 +1,101 @@
+package upper
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	ugcMdl "go-common/app/job/main/tv/model/ugc"
+	"go-common/library/cache/memcache"
+	xsql "go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_loadUpper = "SELECT mid,toinit,submit,ori_name,cms_name,ori_face,cms_face,valid,deleted FROM ugc_uploader WHERE mid = ?"
+)
+
+func upperMetaKey(MID int64) string {
+	return fmt.Sprintf("up_cms_%d", MID)
+}
+
+// LoadUpMeta loads the upper meta cms data from cache, for missed ones, pick them from the DB
+func (d *Dao) LoadUpMeta(ctx context.Context, mid int64) (upper *ugcMdl.Upper, err error) {
+	if upper, err = d.upMetaCache(ctx, mid); err != nil { // mc error
+		log.Error("LoadUpMeta Get Mid [%d] from CMS Error (%v)", mid, err)
+		return
+	}
+	if upper != nil { // mc found
+		return
+	}
+	if upper, err = d.upMetaDB(ctx, mid); err != nil { // db error
+		log.Error("LoadUpMeta Get Mid ERROR (%d) (%v)", mid, err)
+		return
+	}
+	if upper == nil { // db not found
+		err = ecode.NothingFound
+		return
+	}
+	d.addUpMetaCache(ctx, upper) // db found, re-fill the cache
+	return
+}
+
+// upMetaCache get upper meta cache.
+func (d *Dao) upMetaCache(c context.Context, mid int64) (upper *ugcMdl.Upper, err error) {
+	var (
+		key  = upperMetaKey(mid)
+		conn = d.mc.Get(c)
+		item *memcache.Item
+	)
+	defer conn.Close()
+	if item, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+			missedCount.Add("tv-meta", 1)
+		} else {
+			log.Error("conn.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	if err = conn.Scan(item, &upper); err != nil {
+		log.Error("conn.Get(%s) error(%v)", key, err)
+	}
+	cachedCount.Add("tv-meta", 1)
+	return
+}
+
+// upMetaDB gets upper meta info from DB
+func (d *Dao) upMetaDB(c context.Context, mid int64) (upper *ugcMdl.Upper, err error) {
+	var row *xsql.Row
+	if row = d.DB.QueryRow(c, _loadUpper, mid); err != nil {
+		log.Error("d.db.QueryRow(%d) error(%v)", mid, err)
+		return
+	}
+	upper = &ugcMdl.Upper{}
+	// "SELECT id,mid,to_init,submit,ori_name,cms_name,ori_face,cms_face,valid,deleted FROM ugc_uploader WHERE mid = ?"
+	if err = row.Scan(&upper.MID, &upper.Toinit, &upper.Submit, &upper.OriName,
+		&upper.CMSName, &upper.OriFace, &upper.CMSFace, &upper.Valid, &upper.Deleted); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			upper = nil
+		} else {
+			log.Error("row.Scan(mid %d) error(%v)", mid, err)
+		}
+	}
+	return
+}
+
+// setUpMetaCache save ugcMdl.Upper to memcache
+func (d *Dao) addUpMetaCache(c context.Context, upper *ugcMdl.Upper) (err error) {
+	var (
+		key  = upperMetaKey(upper.MID)
+		conn = d.mc.Get(c)
+	)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: key, Object: upper, Flags: memcache.FlagJSON, Expiration: d.mcExpire}); err != nil {
+		log.Error("conn.Set error(%v)", err)
+		return
+	}
+	return
+}

app/job/main/tv/dao/upper/load_upper_test.go

@@ -0,0 +1,74 @@
+package upper
+
+import (
+	"context"
+	"testing"
+
+	ugcMdl "go-common/app/job/main/tv/model/ugc"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestUpperupperMetaKey(t *testing.T) {
+	var MID = int64(0)
+	convey.Convey("upperMetaKey", t, func(c convey.C) {
+		p1 := upperMetaKey(MID)
+		c.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestUpperLoadUpMeta(t *testing.T) {
+	var (
+		ctx = context.Background()
+		mid = int64(0)
+	)
+	convey.Convey("LoadUpMeta", t, func(c convey.C) {
+		upper, err := d.LoadUpMeta(ctx, mid)
+		c.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(upper, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestUpperupMetaCache(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(0)
+	)
+	convey.Convey("upMetaCache", t, func(ctx convey.C) {
+		upper, err := d.upMetaCache(c, mid)
+		ctx.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(upper, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestUpperupMetaDB(t *testing.T) {
+	var (
+		c   = context.Background()
+		mid = int64(0)
+	)
+	convey.Convey("upMetaDB", t, func(ctx convey.C) {
+		upper, err := d.upMetaDB(c, mid)
+		ctx.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(upper, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestUpperaddUpMetaCache(t *testing.T) {
+	var (
+		upper = &ugcMdl.Upper{}
+	)
+	convey.Convey("addUpMetaCache", t, func(ctx convey.C) {
+		err := d.addUpMetaCache(context.Background(), upper)
+		ctx.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/job/main/tv/dao/upper/refresh.go

@@ -0,0 +1,110 @@
+package upper
+
+import (
+	"context"
+	"fmt"
+
+	ugcMdl "go-common/app/job/main/tv/model/ugc"
+	"go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_countUpper   = "SELECT count(1) FROM ugc_uploader WHERE deleted = 0"
+	_pickUppers   = "SELECT mid FROM ugc_uploader WHERE deleted = 0 AND mid > ? ORDER BY mid LIMIT 0,%d"
+	_setUpperName = "UPDATE ugc_uploader SET ori_name = ?, cms_name = ? WHERE mid = ? AND deleted = 0"
+	_setUpperFace = "UPDATE ugc_uploader SET ori_face = ?, cms_face = ? WHERE mid = ? AND deleted = 0"
+	_sendUpper    = "UPDATE ugc_uploader SET submit = 1 WHERE mid = ? AND deleted = 0"
+	_importUpper  = "REPLACE INTO ugc_uploader (ori_name, cms_name, ori_face, cms_face, mid) VALUES (?,?,?,?,?)"
+	_upName       = 1
+	_upFace       = 2
+)
+
+// CountUP counts the uppers
+func (d *Dao) CountUP(c context.Context) (count int64, err error) {
+	if err = d.DB.QueryRow(c, _countUpper).Scan(&count); err != nil {
+		log.Error("d.CountUP.Query error(%v)", err)
+	}
+	return
+}
+
+// PickUppers picks data by Piece to refresh uppers
+func (d *Dao) PickUppers(ctx context.Context, LastID int64, nbData int) (res []int64, myLast int64, err error) {
+	var (
+		rows  *sql.Rows
+		query = fmt.Sprintf(_pickUppers, nbData)
+	)
+	if rows, err = d.DB.Query(ctx, query, LastID); err != nil {
+		log.Error("d.refreshArcMC.Query: %s error(%v)", query, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var li int64
+		if err = rows.Scan(&li); err != nil {
+			log.Error("refreshArcMC row.Scan() error(%v)", err)
+			return
+		}
+		res = append(res, li)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.PickUppers.Query error(%v)", err)
+		return
+	}
+	// record the max ID in this piece
+	if len(res) > 0 {
+		myLast = res[len(res)-1]
+	}
+	return
+}
+
+// SendUpper updates one upper's submit to 1
+func (d *Dao) SendUpper(ctx context.Context, mid int64) (err error) {
+	if _, err = d.DB.Exec(ctx, _sendUpper, mid); err != nil {
+		log.Error("SendUpper Error: %v", mid, err)
+	}
+	return
+}
+
+// setUpperV updates the upper's name or face value
+func (d *Dao) setUpperV(ctx context.Context, req *ugcMdl.ReqSetUp, oriUp *ugcMdl.Upper) (err error) {
+	var sql string
+	if req.UpType == _upName {
+		sql = _setUpperName
+		oriUp.CMSName = req.Value
+		oriUp.OriName = req.Value
+	} else if req.UpType == _upFace {
+		sql = _setUpperFace
+		oriUp.CMSFace = req.Value
+		oriUp.OriFace = req.Value
+	} else {
+		return ecode.TvDangbeiWrongType
+	}
+	if _, err = d.DB.Exec(ctx, sql, req.Value, req.Value, req.MID); err != nil {
+		log.Error("SetUpperName Error: %v", req.MID, err)
+	}
+	return
+}
+
+// ImportUp is used to import a new up when we manually add an archive
+func (d *Dao) ImportUp(ctx context.Context, up *ugcMdl.EasyUp) (err error) {
+	if _, err = d.DB.Exec(ctx, _importUpper, up.Name, up.Name, up.Face, up.Face, up.MID); err != nil {
+		log.Error("ImportUp Error: %v", up.MID, err)
+	}
+	d.addUpMetaCache(ctx, up.ToUpper(nil)) // add this upper into the cache
+	return
+}
+
+// RefreshUp refreshes the upper's name and face in both DB and cache
+func (d *Dao) RefreshUp(ctx context.Context, req *ugcMdl.ReqSetUp) (err error) {
+	var oriUp *ugcMdl.Upper
+	if oriUp, err = d.LoadUpMeta(ctx, req.MID); err != nil || oriUp == nil {
+		return
+	}
+	if err = d.setUpperV(ctx, req, oriUp); err != nil { // update DB
+		return
+	}
+	d.addUpMetaCache(ctx, oriUp) // update cache
+	return
+}

app/job/main/tv/dao/upper/refresh_test.go

@@ -0,0 +1,72 @@
+package upper
+
+import (
+	"context"
+	"testing"
+
+	ugcMdl "go-common/app/job/main/tv/model/ugc"
+
+	"fmt"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestUpperCountUP(t *testing.T) {
+	var c = context.Background()
+	convey.Convey("CountUP", t, func(ctx convey.C) {
+		count, err := d.CountUP(c)
+		ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(count, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestUpperSendUpper(t *testing.T) {
+	var (
+		mid = int64(0)
+	)
+	convey.Convey("SendUpper", t, func(ctx convey.C) {
+		err := d.SendUpper(context.Background(), mid)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDao_SetUpperV(t *testing.T) {
+	convey.Convey("TestDao_LoadUpMeta", t, WithDao(func(d *Dao) {
+		var (
+			query = "SELECT mid FROM ugc_uploader WHERE deleted = 0 LIMIT 1"
+			upper = &ugcMdl.Upper{}
+		)
+		if err := d.DB.QueryRow(ctx, query).Scan(&upper.MID); err != nil || upper.MID == 0 {
+			fmt.Println("DB Error ", err)
+			return
+		}
+		req := &ugcMdl.ReqSetUp{
+			Value:  "test",
+			MID:    upper.MID,
+			UpType: _upName,
+		}
+		oriUp, err2 := d.LoadUpMeta(ctx, req.MID)
+		if err2 != nil || oriUp == nil {
+			return
+		}
+		fmt.Println(req.MID)
+		err := d.setUpperV(context.Background(), req, oriUp)
+		convey.So(err, convey.ShouldBeNil)
+	}))
+}
+
+func TestUpperImportUp(t *testing.T) {
+	var (
+		up = &ugcMdl.EasyUp{}
+	)
+	convey.Convey("ImportUp", t, func(ctx convey.C) {
+		err := d.ImportUp(context.Background(), up)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/job/main/tv/dao/upper/sync.go

@@ -0,0 +1,42 @@
+package upper
+
+import (
+	"context"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_toSyncUps     = "SELECT mid FROM ugc_uploader WHERE submit = 1 AND deleted = 0"
+	_finishSyncUps = "UPDATE ugc_uploader SET submit = 0 WHERE mid = ? AND deleted = 0"
+)
+
+// TosyncUps gets all the uppers that need to sync ( with archives )
+func (d *Dao) TosyncUps(c context.Context) (mids []int64, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.DB.Query(c, _toSyncUps); err != nil {
+		log.Error("d.TosyncUps.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r int64
+		if err = rows.Scan(&r); err != nil {
+			log.Error("TosyncUps row.Scan() error(%v)", err)
+			return
+		}
+		mids = append(mids, r)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("d.ToSyncUps.Query error(%v)", err)
+	}
+	return
+}
+
+// FinsyncUps is used to update the upper's submit from 1 to 0, after the sync is finished
+func (d *Dao) FinsyncUps(c context.Context, mid int64) (err error) {
+	if _, err = d.DB.Exec(c, _finishSyncUps, mid); err != nil {
+		log.Error("FinsyncUps Error: %v", mid, err)
+	}
+	return
+}