Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,80 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"backsource_test.go",
"dao_test.go",
"db_operate_test.go",
"err_msg_test.go",
"mixed_test.go",
"pgc_auth_test.go",
"pgc_batch_test.go",
"pgc_load_test.go",
"pgc_single_test.go",
"ugc_batch_test.go",
"ugc_load_test.go",
"ugc_single_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/interface/main/tv/conf:go_default_library",
"//app/interface/main/tv/model:go_default_library",
"//library/database/sql:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"backsource.go",
"dao.go",
"db_operate.go",
"err_msg.go",
"mixed.go",
"pgc_auth.go",
"pgc_batch.go",
"pgc_load.go",
"pgc_single.go",
"ugc_batch.go",
"ugc_load.go",
"ugc_single.go",
],
importpath = "go-common/app/interface/main/tv/dao/cms",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/interface/main/tv/conf:go_default_library",
"//app/interface/main/tv/model:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/stat/prom:go_default_library",
"//library/sync/errgroup:go_default_library",
"//library/xstr:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,287 @@
package cms
import (
"context"
"database/sql"
"fmt"
"go-common/app/interface/main/tv/model"
xsql "go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_seasonCols = "SELECT id, cover, title , upinfo, `desc`, category, area, play_time, role, staff, total_num, style,origin_name,alias,status FROM tv_ep_season "
_epCols = "SELECT epid, cover, title, subtitle, pay_status FROM tv_content "
_seasonMetas = _seasonCols + " WHERE id IN (%s)"
_arcMetas = "SELECT title, aid, content, cover, typeid, pubtime, videos, valid, deleted, result FROM ugc_archive WHERE aid IN (%s)"
_videoMeta = "SELECT cid, eptitle, aid, index_order, valid, deleted, result FROM ugc_video WHERE cid = ?"
_videoMetas = "SELECT cid, eptitle, aid, index_order, valid, deleted, result FROM ugc_video WHERE cid IN (%s)"
_epMetas = _epCols + " WHERE epid IN (%s) AND is_deleted = 0 "
_simpleEPC = "SELECT `id`, `epid`, `state`, `is_deleted`, `valid`, `season_id` , `mark` FROM `tv_content` WHERE `epid` = ? LIMIT 1"
_simpleEPCs = "SELECT `id`, `epid`, `state`, `is_deleted`, `valid`, `season_id` , `mark` FROM `tv_content` WHERE `epid` IN (%s)"
_simpleSea = "SELECT `id`, `is_deleted`, `check`, `valid` FROM `tv_ep_season` WHERE `id` = ? LIMIT 1"
_simpleSeas = "SELECT `id`, `is_deleted`, `check`, `valid` FROM `tv_ep_season` WHERE `id` IN (%s)"
_seasonCMS = _seasonCols + "WHERE id = ? AND is_deleted = 0"
_epCMS = _epCols + " WHERE epid = ? AND is_deleted = 0 "
_newestOrder = "SELECT a.epid,b.`order` FROM tv_content a LEFT JOIN tv_ep_content b ON a.epid=b.id " +
"WHERE a.season_id=? AND a.state= ? AND a.valid= ? AND a.is_deleted=0 ORDER by `order` DESC LIMIT 1"
epStatePass = 3
_CMSValid = 1
)
// VideoMetaDB picks video from DB
func (d *Dao) VideoMetaDB(c context.Context, cid int64) (meta *model.VideoCMS, err error) {
rows, err := d.db.Query(c, _videoMeta, cid)
if err != nil {
log.Error("VideoMetaDB d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
li := &model.VideoCMS{}
if err = rows.Scan(&li.CID, &li.Title, &li.AID, &li.IndexOrder, &li.Valid, &li.Deleted, &li.Result); err != nil {
log.Error("VideoMetaDB row.Scan error(%v)", err)
return
}
meta = li
}
return
}
// ArcMetaDB picks arc from DB
func (d *Dao) ArcMetaDB(c context.Context, aid int64) (meta *model.ArcCMS, err error) {
rows, err := d.db.Query(c, fmt.Sprintf(_arcMetas, fmt.Sprintf("%d", aid)))
if err != nil {
log.Error("ArcMetas d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
li := &model.ArcCMS{}
if err = rows.Scan(&li.Title, &li.AID, &li.Content, &li.Cover, &li.TypeID, &li.Pubtime, &li.Videos, &li.Valid, &li.Deleted, &li.Result); err != nil {
log.Error("ArcMetas row.Scan error(%v)", err)
return
}
meta = li
}
return
}
// VideoMetas picks video from DB
func (d *Dao) VideoMetas(c context.Context, cids []int64) (meta map[int64]*model.VideoCMS, err error) {
meta = make(map[int64]*model.VideoCMS)
rows, err := d.db.Query(c, fmt.Sprintf(_videoMetas, xstr.JoinInts(cids)))
if err != nil {
log.Error("VideoMetaDB d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
li := &model.VideoCMS{}
if err = rows.Scan(&li.CID, &li.Title, &li.AID, &li.IndexOrder, &li.Valid, &li.Deleted, &li.Result); err != nil {
log.Error("VideoMetaDB row.Scan error(%v)", err)
return
}
meta[int64(li.CID)] = li
}
return
}
// ArcMetas picks seasons from DB
func (d *Dao) ArcMetas(c context.Context, aids []int64) (metas map[int64]*model.ArcCMS, err error) {
metas = make(map[int64]*model.ArcCMS)
rows, err := d.db.Query(c, fmt.Sprintf(_arcMetas, xstr.JoinInts(aids)))
if err != nil {
log.Error("ArcMetas d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
li := &model.ArcCMS{}
if err = rows.Scan(&li.Title, &li.AID, &li.Content, &li.Cover, &li.TypeID, &li.Pubtime, &li.Videos, &li.Valid, &li.Deleted, &li.Result); err != nil {
log.Error("ArcMetas row.Scan error(%v)", err)
return
}
metas[li.AID] = li
}
return
}
// SeasonMetas picks seasons from DB
func (d *Dao) SeasonMetas(c context.Context, sids []int64) (metas map[int64]*model.SeasonCMS, err error) {
metas = make(map[int64]*model.SeasonCMS)
rows, err := d.db.Query(c, fmt.Sprintf(_seasonMetas, xstr.JoinInts(sids)))
if err != nil {
log.Error("SeasonMetas d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
li := &model.SeasonCMS{}
// SELECT id, cover, title , upinfo, `desc`, category, area, play_time, role, staff, total_num, style, origin_name, status
if err = rows.Scan(&li.SeasonID, &li.Cover, &li.Title, &li.UpInfo, &li.Desc, &li.Category, &li.Area,
&li.Playtime, &li.Role, &li.Staff, &li.TotalNum, &li.Style, &li.OriginName, &li.Alias, &li.PayStatus); err != nil {
log.Error("SeasonMetas row.Scan error(%v)", err)
return
}
metas[int64(li.SeasonID)] = li
}
for _, v := range metas {
v.NewestEPID, v.NewestOrder, _ = d.NewestOrder(c, v.SeasonID)
}
return
}
// NewestOrder picks one season's newest passed ep's order column value
func (d *Dao) NewestOrder(c context.Context, sid int64) (epid int64, newestOrder int, err error) {
if err = d.db.QueryRow(c, _newestOrder, sid, epStatePass, _CMSValid).Scan(&epid, &newestOrder); err != nil { // get the qualified aid to sync
log.Warn("d.NewestOrder(sid %d).Query error(%v)", sid, err)
}
return
}
// EpMetas picks ep info from DB
func (d *Dao) EpMetas(c context.Context, epids []int64) (metas map[int64]*model.EpCMS, err error) {
metas = make(map[int64]*model.EpCMS)
rows, err := d.db.Query(c, fmt.Sprintf(_epMetas, xstr.JoinInts(epids)))
if err != nil {
log.Error("EpMetas d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
li := &model.EpCMS{}
if err = rows.Scan(&li.EPID, &li.Cover, &li.Title, &li.Subtitle, &li.PayStatus); err != nil {
log.Error("EpMetas row.Scan error(%v)", err)
return
}
metas[li.EPID] = li
}
return
}
// EpAuthDB pick ep data from DB for Cache missing case
func (d *Dao) EpAuthDB(c context.Context, cid int64) (ep *model.EpAuth, err error) {
var row *xsql.Row
if row = d.db.QueryRow(c, _simpleEPC, cid); err != nil {
log.Error("d.db.QueryRow(%d) error(%v)", cid, err)
return
}
ep = &model.EpAuth{}
if err = row.Scan(&ep.ID, &ep.EPID, &ep.State, &ep.IsDeleted, &ep.Valid, &ep.SeasonID, &ep.NoMark); err != nil {
if err == sql.ErrNoRows {
err = nil
ep = nil
} else {
log.Error("row.Scan() error(%v)", err)
}
}
return
}
// SnAuthDB .
func (d *Dao) SnAuthDB(c context.Context, cid int64) (s *model.SnAuth, err error) {
var row *xsql.Row
if row = d.db.QueryRow(c, _simpleSea, cid); err != nil {
log.Error("d.db.QueryRow(%d) error(%v)", cid, err)
return
}
s = &model.SnAuth{}
if err = row.Scan(&s.ID, &s.IsDeleted, &s.Check, &s.Valid); err != nil {
if err == sql.ErrNoRows {
err = nil
s = nil
} else {
log.Error("row.Scan() error(%v)", err)
}
}
return
}
// SnsAuthDB .
func (d *Dao) SnsAuthDB(c context.Context, sids []int64) (snsAuth map[int64]*model.SnAuth, err error) {
snsAuth = make(map[int64]*model.SnAuth)
rows, err := d.db.Query(c, fmt.Sprintf(_simpleSeas, xstr.JoinInts(sids)))
if err != nil {
log.Error("ArcMetas d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
s := &model.SnAuth{}
if err = rows.Scan(&s.ID, &s.IsDeleted, &s.Check, &s.Valid); err != nil {
log.Error("SnsAuthDB row.Scan error(%v)", err)
return
}
snsAuth[s.ID] = s
}
return
}
// EpsAuthDB def.
func (d *Dao) EpsAuthDB(c context.Context, epids []int64) (epsAuth map[int64]*model.EpAuth, err error) {
epsAuth = make(map[int64]*model.EpAuth)
rows, err := d.db.Query(c, fmt.Sprintf(_simpleEPCs, xstr.JoinInts(epids)))
if err != nil {
log.Error("ArcMetas d.db.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
ep := &model.EpAuth{}
if err = rows.Scan(&ep.ID, &ep.EPID, &ep.State, &ep.IsDeleted, &ep.Valid, &ep.SeasonID, &ep.NoMark); err != nil {
log.Error("SnsAuthDB row.Scan error(%v)", err)
return
}
epsAuth[ep.EPID] = ep
}
return
}
// SeasonCMS gets the fields that can be changed from tv-cms side to offer the TV APP
func (d *Dao) SeasonCMS(c context.Context, sid int64) (season *model.SeasonCMS, err error) {
var row *xsql.Row
if row = d.db.QueryRow(c, _seasonCMS, sid); err != nil {
log.Error("d.db.QueryRow(%d) error(%v)", sid, err)
return
}
season = &model.SeasonCMS{}
// select id, cover, `desc`, title , upinfo, category, area, play_time, role, staff, total_num, style, status
if err = row.Scan(&season.SeasonID, &season.Cover, &season.Desc, &season.Title, &season.UpInfo,
&season.Category, &season.Area, &season.Playtime, &season.Role, &season.Staff, &season.TotalNum,
&season.Style, &season.OriginName, &season.Alias, &season.PayStatus); err != nil {
if err == sql.ErrNoRows {
err = nil
season = nil
} else {
log.Error("row.Scan(sid %d) error(%v)", sid, err)
}
return
}
// add newest info
season.NewestEPID, season.NewestOrder, _ = d.NewestOrder(c, sid)
return
}
// EpCMS gets the fields that can be changed from tv-cms side to offer the TV APP
func (d *Dao) EpCMS(c context.Context, epid int64) (ep *model.EpCMS, err error) {
var row *xsql.Row
if row = d.db.QueryRow(c, _epCMS, epid); err != nil {
log.Error("d.db.QueryRow(%d) error(%v)", epid, err)
return
}
ep = &model.EpCMS{}
// select id, cover, `desc`, title , upinfo, category, area, play_time, role, staff, total_num, style
if err = row.Scan(&ep.EPID, &ep.Cover, &ep.Title, &ep.Subtitle, &ep.PayStatus); err != nil {
if err == sql.ErrNoRows {
err = nil
ep = nil
} else {
log.Error("row.Scan(sid %d) error(%v)", epid, err)
}
return
}
return
}

View File

@@ -0,0 +1,344 @@
package cms
import (
"context"
"fmt"
"testing"
"go-common/app/interface/main/tv/model"
"go-common/library/database/sql"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsVideoMetaDB(t *testing.T) {
var (
c = context.Background()
cid = int64(0)
)
convey.Convey("VideoMetaDB", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.meta should not be nil.", func(ctx convey.C) {
sids, err := pickIDs(d.db, _pickCids)
if err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
cid = sids[0]
meta, err := d.VideoMetaDB(c, cid)
ctx.So(err, convey.ShouldBeNil)
ctx.So(meta, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.VideoMetaDB(c, cid)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsArcMetaDB(t *testing.T) {
var (
c = context.Background()
aid = int64(0)
)
convey.Convey("ArcMetaDB", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.meta should not be nil.", func(ctx convey.C) {
sids, err := pickIDs(d.db, _pickAids)
if err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
aid = sids[0]
meta, err := d.ArcMetaDB(c, aid)
ctx.So(err, convey.ShouldBeNil)
ctx.So(meta, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.ArcMetaDB(c, aid)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsVideoMetas(t *testing.T) {
var (
c = context.Background()
cids = []int64{}
err error
)
convey.Convey("VideoMetas", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.meta should not be nil.", func(ctx convey.C) {
if cids, err = pickIDs(d.db, _pickCids); err != nil || len(cids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
meta, err := d.VideoMetas(c, cids)
ctx.So(err, convey.ShouldBeNil)
ctx.So(meta, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.VideoMetas(c, cids)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsArcMetas(t *testing.T) {
var (
c = context.Background()
aids = []int64{}
err error
)
convey.Convey("ArcMetas", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.metas should not be nil.", func(ctx convey.C) {
if aids, err = pickIDs(d.db, _pickAids); err != nil || len(aids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
metas, err := d.ArcMetas(c, aids)
ctx.So(err, convey.ShouldBeNil)
ctx.So(metas, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.ArcMetas(c, aids)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsSeasonMetas(t *testing.T) {
var (
c = context.Background()
sids = []int64{}
err error
)
convey.Convey("SeasonMetas", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.metas should not be nil.", func(ctx convey.C) {
if sids, err = pickIDs(d.db, _pickSids); err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
metas, err := d.SeasonMetas(c, sids)
ctx.So(err, convey.ShouldBeNil)
ctx.So(metas, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.SeasonMetas(c, sids)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsNewestOrder(t *testing.T) {
var (
c = context.Background()
sid = int64(0)
)
convey.Convey("NewestOrder", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.epid,newestOrder should not be nil.", func(ctx convey.C) {
epid, newestOrder, err := d.NewestOrder(c, sid)
if err != nil {
ctx.So(err, convey.ShouldEqual, sql.ErrNoRows)
} else {
ctx.So(err, convey.ShouldBeNil)
ctx.So(newestOrder, convey.ShouldNotBeNil)
ctx.So(epid, convey.ShouldNotBeNil)
}
})
})
}
func TestCmsEpMetas(t *testing.T) {
var (
c = context.Background()
epids = []int64{}
err error
)
convey.Convey("EpMetas", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.metas should not be nil.", func(ctx convey.C) {
if epids, err = pickIDs(d.db, _pickEpids); err != nil || len(epids) == 0 {
fmt.Println("Empty epids ", err)
return
}
metas, err := d.EpMetas(c, epids)
ctx.So(err, convey.ShouldBeNil)
ctx.So(metas, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.EpMetas(c, epids)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsEpAuthDB(t *testing.T) {
var (
c = context.Background()
epid = int64(0)
)
convey.Convey("EpAuthDB", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.ep should not be nil.", func(ctx convey.C) {
epids, err := pickIDs(d.db, _pickEpids)
if err != nil || len(epids) == 0 {
fmt.Println("Empty epids ", err)
return
}
epid = epids[0]
ep, err := d.EpAuthDB(c, epid)
ctx.So(err, convey.ShouldBeNil)
ctx.So(ep, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.EpAuthDB(c, epid)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsSnAuthDB(t *testing.T) {
var (
c = context.Background()
sids []int64
sid int64
err error
)
convey.Convey("SnAuthDB", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.s should not be nil.", func(ctx convey.C) {
if sids, err = pickIDs(d.db, _pickSids); err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
sid = sids[0]
s, err := d.SnAuthDB(c, sid)
ctx.So(err, convey.ShouldBeNil)
ctx.So(s, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.SnAuthDB(c, sid)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsSnsAuthDB(t *testing.T) {
var (
c = context.Background()
sids []int64
err error
snsAuth map[int64]*model.SnAuth
)
convey.Convey("SnsAuthDB", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.snsAuth should not be nil.", func(ctx convey.C) {
if sids, err = pickIDs(d.db, _pickSids); err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
snsAuth, err = d.SnsAuthDB(c, sids)
ctx.So(err, convey.ShouldBeNil)
ctx.So(snsAuth, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err = d.SnsAuthDB(c, sids)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsEpsAuthDB(t *testing.T) {
var (
c = context.Background()
epids []int64
err error
epsAuth map[int64]*model.EpAuth
)
convey.Convey("EpsAuthDB", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.epsAuth should not be nil.", func(ctx convey.C) {
epids, err = pickIDs(d.db, _pickEpids)
if err != nil || len(epids) == 0 {
fmt.Println("Empty epids ", err)
return
}
epsAuth, err = d.EpsAuthDB(c, epids)
ctx.So(err, convey.ShouldBeNil)
ctx.So(epsAuth, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err = d.EpsAuthDB(c, epids)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsSeasonCMS(t *testing.T) {
var (
c = context.Background()
sids []int64
sid = int64(0)
err error
season *model.SeasonCMS
)
convey.Convey("SeasonCMS", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.season should not be nil.", func(ctx convey.C) {
sids, err = pickIDs(d.db, _pickSids)
if err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
sid = sids[0]
season, err = d.SeasonCMS(c, sid)
ctx.So(err, convey.ShouldBeNil)
ctx.So(season, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err = d.SeasonCMS(c, sid)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsEpCMS(t *testing.T) {
var (
c = context.Background()
epid = int64(0)
)
convey.Convey("EpCMS", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.ep should not be nil.", func(ctx convey.C) {
epids, err := pickIDs(d.db, _pickEpids)
if err != nil || len(epids) == 0 {
fmt.Println("Empty epids ", err)
return
}
epid = epids[0]
ep, err := d.EpCMS(c, epid)
ctx.So(err, convey.ShouldBeNil)
ctx.So(ep, convey.ShouldNotBeNil)
})
ctx.Convey("Db Error", func(ctx convey.C) {
d.db.Close()
_, err := d.EpCMS(c, epid)
ctx.So(err, convey.ShouldNotBeNil)
d.db = sql.NewMySQL(d.conf.Mysql)
})
})
}

View File

@@ -0,0 +1,72 @@
package cms
import (
"runtime"
"time"
"go-common/app/interface/main/tv/conf"
"go-common/library/cache/memcache"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/stat/prom"
)
// Dao is account dao.
type Dao struct {
mc *memcache.Pool
conf *conf.Config
db *sql.DB
mCh chan func()
expireCMS int32
}
// New account dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
mc: memcache.NewPool(c.Memcache.Config),
db: sql.NewMySQL(c.Mysql),
mCh: make(chan func(), 10240),
expireCMS: int32(time.Duration(c.Memcache.CmsExpire) / time.Second),
}
// video db
for i := 0; i < runtime.NumCPU()*2; i++ {
go d.cacheproc()
}
return
}
// Prom
var (
errorsCount = prom.BusinessErrCount
infosCount = prom.BusinessInfoCount
cachedCount = prom.CacheHit
missedCount = prom.CacheMiss
)
// PromError prom error
func PromError(name string) {
errorsCount.Incr(name)
}
// PromInfo add prom info
func PromInfo(name string) {
infosCount.Incr(name)
}
// addCache add archive to mc or redis
func (d *Dao) addCache(f func()) {
select {
case d.mCh <- f:
default:
log.Warn("cacheproc chan full")
}
}
// cacheproc write memcache and stat redis use goroutine
func (d *Dao) cacheproc() {
for {
f := <-d.mCh
f()
}
}

View File

@@ -0,0 +1,121 @@
package cms
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"testing"
"go-common/app/interface/main/tv/conf"
"go-common/library/database/sql"
. "github.com/smartystreets/goconvey/convey"
)
var (
d *Dao
ctx = context.TODO()
)
const (
_pickSids = 1
_pickEpids = 2
_pickAids = 3
_pickCids = 4
)
func init() {
// dir, _ := filepath.Abs("../../cmd/tv-interface.toml")
// flag.Set("conf", dir)
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.web-svr.tv-interface")
flag.Set("conf_token", "07c1826c1f39df02a1411cdd6f455879")
flag.Set("tree_id", "15326")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
}
func WithDao(f func(d *Dao)) func() {
return func() {
Reset(func() {})
f(d)
}
}
func pickIDs(db *sql.DB, ttype int) (testSids []int64, err error) {
var (
querySQL string
rows *sql.Rows
)
switch ttype {
case _pickSids:
querySQL = "SELECT DISTINCT(id) FROM tv_ep_season WHERE is_deleted = 0 ORDER BY id DESC LIMIT 20"
case _pickEpids:
querySQL = "SELECT DISTINCT(epid) FROM tv_content WHERE is_deleted = 0 ORDER BY id DESC LIMIT 20"
case _pickAids:
querySQL = "SELECT DISTINCT(aid) FROM ugc_archive WHERE deleted = 0 ORDER BY id DESC LIMIT 20"
case _pickCids:
querySQL = "SELECT DISTINCT(cid) FROM ugc_video WHERE deleted = 0 ORDER BY id DESC LIMIT 20"
}
rows, err = db.Query(ctx, querySQL)
if err != nil {
fmt.Println("Query Err ", err)
return
}
defer rows.Close()
for rows.Next() {
var id int64
if err = rows.Scan(&id); err != nil {
fmt.Printf("Scan Id %d, Err %v", id, err)
return
}
testSids = append(testSids, id)
}
return
}
func TestDao_MixedFilter(t *testing.T) {
Convey("TestDao_MixedFilter", t, WithDao(func(d *Dao) {
sids, errDB := pickIDs(d.db, _pickSids)
if errDB != nil {
fmt.Println("PickSids Err ", errDB)
return
}
aids, errDB2 := pickIDs(d.db, _pickAids)
if errDB != nil {
fmt.Println("PickAids Err ", errDB2)
return
}
okSids, okAids := d.MixedFilter(ctx, sids, aids)
fmt.Println(okSids)
fmt.Println(okAids)
So(len(okSids), ShouldBeLessThanOrEqualTo, len(sids))
So(len(okAids), ShouldBeLessThanOrEqualTo, len(aids))
}))
}
func TestDao_LoadVideosMeta(t *testing.T) {
Convey("TestDao_MixedFilter", t, WithDao(func(d *Dao) {
sids, errDB := pickIDs(d.db, _pickCids)
if errDB != nil {
fmt.Println("PickSids Err ", errDB)
return
}
res, err := d.LoadVideosMeta(ctx, sids)
So(err, ShouldBeNil)
data, _ := json.Marshal(res)
fmt.Println(string(data))
}))
}

View File

@@ -0,0 +1,18 @@
package cms
import (
"context"
"fmt"
"go-common/library/xstr"
)
const (
_unshelveArcs = "UPDATE ugc_archive SET valid = 0 WHERE aid IN (%s) AND valid = 1 AND deleted = 0"
)
// UnshelveArcs unshelves the arcs
func (d *Dao) UnshelveArcs(c context.Context, ids []int64) (err error) {
_, err = d.db.Exec(c, fmt.Sprintf(_unshelveArcs, xstr.JoinInts(ids)))
return
}

View File

@@ -0,0 +1,21 @@
package cms
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsUnshelveArcs(t *testing.T) {
var (
c = context.Background()
ids = []int64{123}
)
convey.Convey("UnshelveArcs", t, func(ctx convey.C) {
err := d.UnshelveArcs(c, ids)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

View File

@@ -0,0 +1,77 @@
package cms
import "go-common/app/interface/main/tv/model"
const (
// auth failure cases
_pgcOffline = "pgc_offline"
_cmsInvalid = "cms_invalid"
_licenseReject = "license_reject"
// auth correct values in DB
_contentOnline = 0
_snPass = 1
_epPass = 3
_cmsValid = 1
)
// AuthMsg returns the error message in the config according to the error condition
func (d *Dao) authMsg(cond string) (msg string) {
msgCfg := d.conf.Cfg.AuthMsg
if cond == _licenseReject {
return msgCfg.LicenseReject
}
if cond == _cmsInvalid {
return msgCfg.CMSInvalid
}
if cond == _pgcOffline {
return msgCfg.PGCOffline
}
return "err msg config load error"
}
// SnErrMsg returns the season auth result and the error message in case of auth failure
func (d *Dao) SnErrMsg(season *model.SnAuth) (bool, string) {
if season.IsDeleted == _contentOnline && season.Check == _snPass && season.Valid == _cmsValid {
return true, ""
}
if season.Check != _snPass {
return false, d.authMsg(_licenseReject)
}
if season.IsDeleted != _contentOnline {
return false, d.authMsg(_pgcOffline)
}
return false, d.authMsg(_cmsInvalid) // must be cms valid logically
}
// UgcErrMsg returns the arc auth result and the error message in case of auth failure
func (d *Dao) UgcErrMsg(deleted int, result int, valid int) (bool, string) {
if deleted == _contentOnline && result == _snPass && valid == _cmsValid {
return true, ""
}
if result != _snPass {
return false, d.authMsg(_licenseReject)
}
if deleted != _contentOnline {
return false, d.authMsg(_pgcOffline)
}
return false, d.authMsg(_cmsInvalid) // must be cms valid logically
}
// AuditingMsg returns the msg for the archive whose all videos are being audited
func (d *Dao) AuditingMsg() (bool, string) {
return false, d.authMsg(_licenseReject) // must be cms valid logically
}
// EpErrMsg returns the ep auth result and the error message in case of auth failure
func (d *Dao) EpErrMsg(ep *model.EpAuth) (bool, string) {
if ep.IsDeleted == _contentOnline && ep.State == _epPass && ep.Valid == _cmsValid {
return true, ""
}
if ep.State != _epPass {
return false, d.authMsg(_licenseReject)
}
if ep.IsDeleted != _contentOnline {
return false, d.authMsg(_pgcOffline)
}
return false, d.authMsg(_cmsInvalid) // must be cms valid logically
}

View File

@@ -0,0 +1,71 @@
package cms
import (
"go-common/app/interface/main/tv/model"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsauthMsg(t *testing.T) {
var (
cond = ""
)
convey.Convey("authMsg", t, func(ctx convey.C) {
msg := d.authMsg(cond)
ctx.Convey("Then msg should not be nil.", func(ctx convey.C) {
ctx.So(msg, convey.ShouldNotBeNil)
})
})
}
func TestCmsSnErrMsg(t *testing.T) {
var (
season = &model.SnAuth{}
)
convey.Convey("SnErrMsg", t, func(ctx convey.C) {
p1, p2 := d.SnErrMsg(season)
ctx.Convey("Then p1,p2 should not be nil.", func(ctx convey.C) {
ctx.So(p2, convey.ShouldNotBeNil)
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsUgcErrMsg(t *testing.T) {
var (
deleted = int(0)
result = int(0)
valid = int(0)
)
convey.Convey("UgcErrMsg", t, func(ctx convey.C) {
p1, p2 := d.UgcErrMsg(deleted, result, valid)
ctx.Convey("Then p1,p2 should not be nil.", func(ctx convey.C) {
ctx.So(p2, convey.ShouldNotBeNil)
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsAuditingMsg(t *testing.T) {
convey.Convey("AuditingMsg", t, func(ctx convey.C) {
p1, p2 := d.AuditingMsg()
ctx.Convey("Then p1,p2 should not be nil.", func(ctx convey.C) {
ctx.So(p2, convey.ShouldNotBeNil)
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsEpErrMsg(t *testing.T) {
var (
ep = &model.EpAuth{}
)
convey.Convey("EpErrMsg", t, func(ctx convey.C) {
p1, p2 := d.EpErrMsg(ep)
ctx.Convey("Then p1,p2 should not be nil.", func(ctx convey.C) {
ctx.So(p2, convey.ShouldNotBeNil)
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,67 @@
package cms
import (
"context"
"go-common/app/interface/main/tv/model"
"go-common/library/log"
"go-common/library/sync/errgroup"
)
// MixedFilter filters ugc and pgc data to get the allowed data
func (d *Dao) MixedFilter(ctx context.Context, sids []int64, aids []int64) (okSids map[int64]int, okAids map[int64]int) {
g, _ := errgroup.WithContext(ctx)
g.Go(func() (err error) {
okAids = d.aidsFilter(context.Background(), aids)
return
})
g.Go(func() (err error) {
okSids = d.sidsFilter(context.Background(), sids)
return
})
g.Wait()
return
}
// filter canPlay Aids
func (d *Dao) aidsFilter(ctx context.Context, aids []int64) (okAids map[int64]int) {
var (
arcMetas map[int64]*model.ArcCMS
err error
)
okAids = make(map[int64]int)
if arcMetas, err = d.LoadArcsMediaMap(ctx, aids); err != nil {
log.Error("MixedFilter Aids %v, Err %v", aids, err)
return
}
if len(arcMetas) == 0 {
return
}
for aid, arcMeta := range arcMetas {
if arcMeta.CanPlay() {
okAids[aid] = 1
}
}
return
}
// filter canPlay Sids
func (d *Dao) sidsFilter(ctx context.Context, sids []int64) (okSids map[int64]int) {
var (
snsAuth map[int64]*model.SnAuth
err error
)
okSids = make(map[int64]int)
if snsAuth, err = d.LoadSnsAuthMap(ctx, sids); err != nil {
log.Error("MixedFilter Sids %v, Err %v", sids, err)
}
if len(snsAuth) == 0 {
return
}
for sid, snAuth := range snsAuth {
if snAuth.CanPlay() {
okSids[sid] = 1
}
}
return
}

View File

@@ -0,0 +1,49 @@
package cms
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsMixedFilter(t *testing.T) {
var (
ctx = context.Background()
sids = []int64{}
aids = []int64{}
)
convey.Convey("MixedFilter", t, func(c convey.C) {
okSids, okAids := d.MixedFilter(ctx, sids, aids)
c.Convey("Then okSids,okAids should not be nil.", func(c convey.C) {
c.So(okAids, convey.ShouldNotBeNil)
c.So(okSids, convey.ShouldNotBeNil)
})
})
}
func TestCmsaidsFilter(t *testing.T) {
var (
ctx = context.Background()
aids = []int64{}
)
convey.Convey("aidsFilter", t, func(c convey.C) {
okAids := d.aidsFilter(ctx, aids)
c.Convey("Then okAids should not be nil.", func(c convey.C) {
c.So(okAids, convey.ShouldNotBeNil)
})
})
}
func TestCmssidsFilter(t *testing.T) {
var (
ctx = context.Background()
sids = []int64{}
)
convey.Convey("sidsFilter", t, func(c convey.C) {
okSids := d.sidsFilter(ctx, sids)
c.Convey("Then okSids should not be nil.", func(c convey.C) {
c.So(okSids, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,198 @@
package cms
import (
"context"
"fmt"
"go-common/app/interface/main/tv/model"
"go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_mcEPKey = "ep_%d"
_mcSeasonKey = "sn_%d"
)
// SeaCacheKey .
func (d *Dao) SeaCacheKey(sid int64) string {
return fmt.Sprintf(_mcSeasonKey, sid)
}
// EPCacheKey .
func (d *Dao) EPCacheKey(epid int64) string {
return fmt.Sprintf(_mcEPKey, epid)
}
// GetSeasonCache get SnAuth cache.
func (d *Dao) GetSeasonCache(c context.Context, sid int64) (s *model.SnAuth, err error) {
var (
key = d.SeaCacheKey(sid)
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(item, &s); err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
// GetEPCache get EpAuth cache.
func (d *Dao) GetEPCache(c context.Context, epid int64) (ep *model.EpAuth, err error) {
var (
key = d.EPCacheKey(epid)
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(item, &ep); err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
// AddSnAuthCache save model.SnAuth to memcache
func (d *Dao) AddSnAuthCache(c context.Context, s *model.SnAuth) (err error) {
var (
key = d.SeaCacheKey(s.ID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: s, Flags: memcache.FlagJSON, Expiration: d.expireCMS}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// AddEpAuthCache save model.EpAuth to memcache
func (d *Dao) AddEpAuthCache(c context.Context, ep *model.EpAuth) (err error) {
var (
key = d.EPCacheKey(ep.EPID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: ep, Flags: memcache.FlagJSON, Expiration: d.expireCMS}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// snAuthCache season auth cache
func (d *Dao) snAuthCache(c context.Context, ids []int64) (cached map[int64]*model.SnAuth, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.SnAuth, len(ids))
idmap, allKeys := keysTreat(ids, d.SeaCacheKey)
conn := d.mc.Get(c)
defer conn.Close()
replys, err := conn.GetMulti(allKeys)
if err != nil {
PromError("mc:获取Season信息缓存")
log.Error("conn.Gets(%v) error(%v)", allKeys, err)
return
}
for key, item := range replys {
art := &model.SnAuth{}
if err = conn.Scan(item, art); err != nil {
PromError("mc:获取Season信息缓存json解析")
log.Error("item.Scan(%s) error(%v)", item.Value, err)
err = nil
continue
}
cached[idmap[key]] = art
delete(idmap, key)
}
missed = missedTreat(idmap, len(cached))
return
}
// epAuthCache ep auth cache
func (d *Dao) epAuthCache(c context.Context, ids []int64) (cached map[int64]*model.EpAuth, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.EpAuth, len(ids))
idmap, allKeys := keysTreat(ids, d.EPCacheKey)
conn := d.mc.Get(c)
defer conn.Close()
replys, err := conn.GetMulti(allKeys)
if err != nil {
PromError("mc:获取EP信息缓存")
log.Error("conn.Gets(%v) error(%v)", allKeys, err)
return
}
for key, item := range replys {
art := &model.EpAuth{}
if err = conn.Scan(item, art); err != nil {
PromError("mc:获取EP信息缓存json解析")
log.Error("item.Scan(%s) error(%v)", item.Value, err)
err = nil
continue
}
cached[idmap[key]] = art
delete(idmap, key)
}
missed = missedTreat(idmap, len(cached))
return
}
// SnAuth get's season auth info from Cache & DB
func (d *Dao) SnAuth(c context.Context, sid int64) (sn *model.SnAuth, err error) {
if sn, err = d.GetSeasonCache(c, sid); err != nil { // mc Error
return
} else if sn == nil { // mc not found, go DB
if sn, err = d.SnAuthDB(c, int64(sid)); err != nil { // DB error
log.Error("SnAuthDB (%d) ERROR (%v)", sid, err)
return
}
if sn == nil { // DB not found, build a fake item in MC to avoid checking DB next time
log.Error("SnAuthDB (%d) not found(%v) in DB", sid)
sn = &model.SnAuth{ID: int64(sid), Check: 0, IsDeleted: 1}
}
if err = d.AddSnAuthCache(c, sn); err != nil { // set item in MC ( not found - fake, or true )
log.Error("AddSnAuthCache fail(%v)", err)
}
}
return
}
// EpAuth get's ep auth info from Cache & DB
func (d *Dao) EpAuth(c context.Context, epid int64) (ep *model.EpAuth, err error) {
if ep, err = d.GetEPCache(c, epid); err != nil { // MC error
log.Error("GetEPCache(%d) Error(%v)", epid, err)
return
} else if ep == nil { // MC not found, go DB
if ep, err = d.EpAuthDB(c, epid); err != nil { // DB error
log.Error("DBSimpleEP(%d) ERROR (%v)", epid, err)
return
}
if ep == nil { // DB not found, build a fake item in MC to avoid checking DB next time
log.Error("EPID(%d) not found", epid)
ep = &model.EpAuth{EPID: epid, State: 4, IsDeleted: 1}
}
if err = d.AddEpAuthCache(c, ep); err != nil {
log.Error("AddEpAuthCache fail(%v)", err)
}
}
return
}

View File

@@ -0,0 +1,134 @@
package cms
import (
"context"
"fmt"
"go-common/app/interface/main/tv/model"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsSeaCacheKey(t *testing.T) {
var (
sid = int64(0)
)
convey.Convey("SeaCacheKey", t, func(c convey.C) {
p1 := d.SeaCacheKey(sid)
c.Convey("Then p1 should not be nil.", func(c convey.C) {
c.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsEPCacheKey(t *testing.T) {
var (
epid = int64(0)
)
convey.Convey("EPCacheKey", t, func(c convey.C) {
p1 := d.EPCacheKey(epid)
c.Convey("Then p1 should not be nil.", func(c convey.C) {
c.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsGetSeasonCache(t *testing.T) {
var (
ctx = context.Background()
sid = int64(0)
)
convey.Convey("GetSeasonCache", t, func(c convey.C) {
s, err := d.GetSeasonCache(ctx, sid)
c.Convey("Then err should be nil.s should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(s, convey.ShouldNotBeNil)
})
})
}
func TestCmsGetEPCache(t *testing.T) {
var (
ctx = context.Background()
epid = int64(0)
)
convey.Convey("GetEPCache", t, func(c convey.C) {
ep, err := d.GetEPCache(ctx, epid)
c.Convey("Then err should be nil.ep should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(ep, convey.ShouldNotBeNil)
})
})
}
func TestCmsAddSnAuthCache(t *testing.T) {
var (
ctx = context.Background()
s = &model.SnAuth{}
)
convey.Convey("AddSnAuthCache", t, func(c convey.C) {
err := d.AddSnAuthCache(ctx, s)
c.Convey("Then err should be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
})
})
}
func TestCmsAddEpAuthCache(t *testing.T) {
var (
ctx = context.Background()
ep = &model.EpAuth{}
)
convey.Convey("AddEpAuthCache", t, func(c convey.C) {
err := d.AddEpAuthCache(ctx, ep)
c.Convey("Then err should be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
})
})
}
func TestCmssnAuthCache(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("snAuthCache", t, func(c convey.C) {
sids, errPick := pickIDs(d.db, _pickSids)
if errPick != nil || len(sids) == 0 {
fmt.Println("Empty sids ", errPick)
return
}
cached, missed, err := d.snAuthCache(ctx, sids)
c.Convey("Then err should be nil.cached,missed should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(len(missed)+len(cached), convey.ShouldBeGreaterThan, 0)
})
})
}
func TestCmsSnAuth(t *testing.T) {
var (
ctx = context.Background()
sid = int64(0)
)
convey.Convey("SnAuth", t, func(c convey.C) {
sn, err := d.SnAuth(ctx, sid)
c.Convey("Then err should be nil.sn should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(sn, convey.ShouldNotBeNil)
})
})
}
func TestCmsEpAuth(t *testing.T) {
var (
ctx = context.Background()
epid = int64(0)
)
convey.Convey("EpAuth", t, func(c convey.C) {
ep, err := d.EpAuth(ctx, epid)
c.Convey("Then err should be nil.ep should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(ep, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,152 @@
package cms
import (
"context"
"fmt"
"go-common/app/interface/main/tv/model"
"go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_mcSnCMSKey = "sn_cms_%d"
_mcEPCMSKey = "ep_cms_%d"
)
func snCMSCacheKey(sid int64) string {
return fmt.Sprintf(_mcSnCMSKey, sid)
}
func epCMSCacheKey(epid int64) string {
return fmt.Sprintf(_mcEPCMSKey, epid)
}
func keysTreat(ids []int64, keyFunc func(int64) string) (idmap map[string]int64, allKeys []string) {
idmap = make(map[string]int64, len(ids))
for _, id := range ids {
k := keyFunc(id)
allKeys = append(allKeys, k)
idmap[k] = id
}
return
}
func missedTreat(idmap map[string]int64, lenCached int) (missed []int64) {
missed = make([]int64, 0, len(idmap))
for _, id := range idmap {
missed = append(missed, id)
}
missedCount.Add("tv-meta", int64(len(missed)))
cachedCount.Add("tv-meta", int64(lenCached))
return
}
// SeasonsMetaCache season cms meta cache
func (d *Dao) SeasonsMetaCache(c context.Context, ids []int64) (cached map[int64]*model.SeasonCMS, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.SeasonCMS, len(ids))
idmap, allKeys := keysTreat(ids, snCMSCacheKey)
conn := d.mc.Get(c)
defer conn.Close()
replys, err := conn.GetMulti(allKeys)
if err != nil {
PromError("mc:获取Season信息缓存")
log.Error("conn.Gets(%v) error(%v)", allKeys, err)
err = nil
return
}
for key, item := range replys {
art := &model.SeasonCMS{}
if err = conn.Scan(item, art); err != nil {
PromError("mc:获取Season信息缓存json解析")
log.Error("item.Scan(%s) error(%v)", item.Value, err)
err = nil
continue
}
cached[idmap[key]] = art
delete(idmap, key)
}
missed = missedTreat(idmap, len(cached))
return
}
// EpMetaCache season cms meta cache
func (d *Dao) EpMetaCache(c context.Context, ids []int64) (cached map[int64]*model.EpCMS, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.EpCMS, len(ids))
allKeys := make([]string, 0, len(ids))
idmap := make(map[string]int64, len(ids))
for _, id := range ids {
k := epCMSCacheKey(id)
allKeys = append(allKeys, k)
idmap[k] = id
}
conn := d.mc.Get(c)
defer conn.Close()
replys, err := conn.GetMulti(allKeys)
if err != nil {
PromError("mc:获取EP信息缓存")
log.Error("conn.Gets(%v) error(%v)", allKeys, err)
err = nil
return
}
for key, item := range replys {
art := &model.EpCMS{}
if err = conn.Scan(item, art); err != nil {
PromError("mc:获取EP信息缓存json解析")
log.Error("item.Scan(%s) error(%v)", item.Value, err)
err = nil
continue
}
cached[idmap[key]] = art
delete(idmap, key)
}
missed = make([]int64, 0, len(idmap))
for _, id := range idmap {
missed = append(missed, int64(id))
}
missedCount.Add("tv-meta", int64(len(missed)))
cachedCount.Add("tv-meta", int64(len(cached)))
return
}
//AddSeasonMetaCache add season meta cache
func (d *Dao) AddSeasonMetaCache(c context.Context, vs ...*model.SeasonCMS) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
for _, v := range vs {
if v == nil {
continue
}
item := &memcache.Item{Key: snCMSCacheKey(v.SeasonID), Object: v, Flags: memcache.FlagJSON, Expiration: d.expireCMS}
if err = conn.Set(item); err != nil {
PromError("mc:增加Season信息缓存")
log.Error("conn.Store(%s) error(%v)", snCMSCacheKey(v.SeasonID), err)
return
}
}
return
}
//AddEpMetaCache add ep meta cache
func (d *Dao) AddEpMetaCache(c context.Context, vs ...*model.EpCMS) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
for _, v := range vs {
if v == nil {
continue
}
item := &memcache.Item{Key: epCMSCacheKey(v.EPID), Object: v, Flags: memcache.FlagJSON, Expiration: d.expireCMS}
if err = conn.Set(item); err != nil {
PromError("mc:增加EP信息缓存")
log.Error("conn.Store(%s) error(%v)", epCMSCacheKey(v.EPID), err)
return
}
}
return
}

View File

@@ -0,0 +1,111 @@
package cms
import (
"context"
"fmt"
"go-common/app/interface/main/tv/model"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmssnCMSCacheKey(t *testing.T) {
var (
sid = int64(0)
)
convey.Convey("snCMSCacheKey", t, func(c convey.C) {
p1 := snCMSCacheKey(sid)
c.Convey("Then p1 should not be nil.", func(c convey.C) {
c.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsepCMSCacheKey(t *testing.T) {
var (
epid = int64(0)
)
convey.Convey("epCMSCacheKey", t, func(c convey.C) {
p1 := epCMSCacheKey(epid)
c.Convey("Then p1 should not be nil.", func(c convey.C) {
c.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmskeysTreat(t *testing.T) {
var (
ids = []int64{1, 2, 3}
keyFunc = snCMSCacheKey
)
convey.Convey("keysTreat", t, func(c convey.C) {
idmap, allKeys := keysTreat(ids, keyFunc)
c.Convey("Then idmap,allKeys should not be nil.", func(c convey.C) {
c.So(allKeys, convey.ShouldNotBeNil)
c.So(idmap, convey.ShouldNotBeNil)
})
})
}
func TestCmsmissedTreat(t *testing.T) {
var (
idmap map[string]int64
lenCached = int(0)
)
convey.Convey("missedTreat", t, func(c convey.C) {
missed := missedTreat(idmap, lenCached)
c.Convey("Then missed should not be nil.", func(c convey.C) {
c.So(missed, convey.ShouldNotBeNil)
})
})
}
func TestCmsEpMetaCache(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("EpMetaCache", t, func(c convey.C) {
c.Convey("Empty Input", func(c convey.C) {
cached, missed, err := d.EpMetaCache(ctx, []int64{})
c.So(err, convey.ShouldBeNil)
c.So(len(missed), convey.ShouldBeZeroValue)
c.So(len(cached), convey.ShouldBeZeroValue)
})
c.Convey("Normal Situation", func(c convey.C) {
epids, err := pickIDs(d.db, _pickEpids)
if err != nil || len(epids) == 0 {
fmt.Println("empty epids")
return
}
cached, missed, err := d.EpMetaCache(ctx, epids)
c.So(err, convey.ShouldBeNil)
c.So(len(missed)+len(cached), convey.ShouldNotEqual, 0)
})
})
}
func TestCmsAddSeasonMetaCache(t *testing.T) {
var (
ctx = context.Background()
vs = &model.SeasonCMS{}
)
convey.Convey("AddSeasonMetaCache", t, func(c convey.C) {
err := d.AddSeasonMetaCache(ctx, vs)
c.Convey("Then err should be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
})
})
}
func TestCmsAddEpMetaCache(t *testing.T) {
var (
ctx = context.Background()
vs = &model.EpCMS{}
)
convey.Convey("AddEpMetaCache", t, func(c convey.C) {
err := d.AddEpMetaCache(ctx, vs)
c.Convey("Then err should be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
})
})
}

View File

@@ -0,0 +1,226 @@
package cms
import (
"context"
"go-common/app/interface/main/tv/model"
"go-common/library/ecode"
"go-common/library/log"
)
// LoadSnsAuthMap loads a batch of arc meta
func (d *Dao) LoadSnsAuthMap(ctx context.Context, sids []int64) (resMetas map[int64]*model.SnAuth, err error) {
var (
cachedMetas map[int64]*model.SnAuth // cache hit seasons
missedMetas map[int64]*model.SnAuth // cache miss seasons, pick from DB
missed []int64 // cache miss seasons
addCache = true // whether we need to fill DB data in MC
)
resMetas = make(map[int64]*model.SnAuth) // merge info from MC and from DB
if cachedMetas, missed, err = d.snAuthCache(ctx, sids); err != nil {
log.Error("LoadSnsAuthMap snAuthCache Sids:%v, Error:%v", sids, err)
err = nil
addCache = false // mc error, we don't add
}
if len(missed) > 0 {
if missedMetas, err = d.SnsAuthDB(ctx, missed); err != nil {
log.Error("LoadSnsAuthMap SnsAuthDB Sids:%v, Error:%v", missed, err)
return
}
}
// merge info from DB and the info from MC
for sid, v := range cachedMetas {
resMetas[sid] = v
}
for sid, v := range missedMetas {
resMetas[sid] = v
}
if addCache && len(missedMetas) > 0 {
for _, snAuth := range missedMetas {
d.addCache(func() {
d.AddSnAuthCache(ctx, snAuth)
})
}
}
return
}
// LoadEpsAuthMap loads a batch of arc meta
func (d *Dao) LoadEpsAuthMap(ctx context.Context, epids []int64) (resMetas map[int64]*model.EpAuth, err error) {
var (
cachedMetas map[int64]*model.EpAuth // cache hit seasons
missedMetas map[int64]*model.EpAuth // cache miss seasons, pick from DB
missed []int64 // cache miss seasons
addCache = true // whether we need to fill DB data in MC
)
resMetas = make(map[int64]*model.EpAuth) // merge info from MC and from DB
if cachedMetas, missed, err = d.epAuthCache(ctx, epids); err != nil {
log.Error("LoadEpsAuthMap epAuthCache epids:%v, Error:%v", epids, err)
err = nil
addCache = false // mc error, we don't add
}
if len(missed) > 0 {
if missedMetas, err = d.EpsAuthDB(ctx, missed); err != nil {
log.Error("LoadEpsAuthMap EpsAuthDB epids:%v, Error:%v", missed, err)
return
}
}
// merge info from DB and the info from MC
for sid, v := range cachedMetas {
resMetas[sid] = v
}
for sid, v := range missedMetas {
resMetas[sid] = v
}
if addCache && len(missedMetas) > 0 {
for _, epAuth := range missedMetas {
d.addCache(func() {
d.AddEpAuthCache(ctx, epAuth)
})
}
}
return
}
// LoadSnsCMSMap loads season cms meta data from cache and db
func (d *Dao) LoadSnsCMSMap(ctx context.Context, sids []int64) (resMetas map[int64]*model.SeasonCMS, err error) {
var (
cachedMetas, missedMetas map[int64]*model.SeasonCMS // cache hit seasons
missed []int64 // cache miss seasons
addCache = true // whether we need to fill DB data in MC
)
resMetas = make(map[int64]*model.SeasonCMS)
// pick up the information for these season ids
if cachedMetas, missed, err = d.SeasonsMetaCache(ctx, sids); err != nil {
log.Error("LoadSnsCMS SeasonMetaCache Sids:%v, Error:%v", sids, err)
err = nil
addCache = false // mc error, we don't add
}
if len(missed) > 0 {
if missedMetas, err = d.SeasonMetas(ctx, missed); err != nil {
log.Error("LoadSnsCMS SeasonMetas Sids:%v, Error:%v", sids, err)
return
}
}
log.Info("Set Sids [%d], HitMetas [%d], MissedMetas [%d][%d] Data in MC", len(sids), len(cachedMetas), len(missed), len(missedMetas))
// merge info from DB and the info from MC
for sid, v := range cachedMetas {
resMetas[sid] = v
}
for sid, v := range missedMetas {
resMetas[sid] = v
}
// async Reset the DB data in MC for next time
if addCache && len(missedMetas) > 0 {
for _, art := range missedMetas {
d.addCache(func() {
d.AddSeasonMetaCache(ctx, art)
})
}
}
return
}
// LoadSnsCMS loads the seasons meta cms data from cache, for missed ones, pick them from the DB
func (d *Dao) LoadSnsCMS(ctx context.Context, sids []int64) (seasons []*model.SeasonCMS, newestEpids []int64, err error) {
var (
resMetas map[int64]*model.SeasonCMS // merge info from MC and from DB
)
if resMetas, err = d.LoadSnsCMSMap(ctx, sids); err != nil {
log.Error("LoadSnsCMS Sids %v, Err %v", sids, err)
return
}
// re-arrange the info, according to the order got from Redis
for _, v := range sids {
if SnCMS, ok := resMetas[v]; !ok {
log.Error("LoadSnsCMS Miss Info for Sid: %d", v)
continue
} else {
seasons = append(seasons, SnCMS)
newestEpids = append(newestEpids, SnCMS.NewestEPID)
}
}
return
}
// LoadSnCMS loads the sn meta cms data from cache, for missed ones, pick them from the DB
func (d *Dao) LoadSnCMS(ctx context.Context, sid int64) (sn *model.SeasonCMS, err error) {
if sn, err = d.GetSnCMSCache(ctx, sid); err != nil {
log.Error("LoadSnsCMS Get Season[%d] from CMS Error (%v)", sid, err) // cache set/get error
return
}
if sn != nil { // if cache hit, return
return
}
if sn, err = d.SeasonCMS(ctx, sid); err != nil {
log.Error("[LoadSnCMS] SeasonCMS SeasonID ERROR (%d) (%v)", sid, err)
return
} else if sn == nil {
err = ecode.NothingFound
return
}
d.addCache(func() {
d.AddSeasonMetaCache(ctx, sn)
})
return
}
// LoadEpCMS loads the sn meta cms data from cache, for missed ones, pick them from the DB
func (d *Dao) LoadEpCMS(ctx context.Context, epid int64) (ep *model.EpCMS, err error) {
if ep, err = d.GetEpCMSCache(ctx, epid); err != nil {
log.Error("LoadEpCMS Get EP[%d] from CMS Error (%v)", epid, err) // cache set/get error
return
} else if ep == nil {
if ep, err = d.EpCMS(ctx, epid); err != nil {
log.Error("[LoadEpCMS] EpCMS Epid ERROR (%d) (%v)", epid, err)
return
} else if ep == nil {
err = ecode.NothingFound
return
}
}
d.addCache(func() {
d.SetEpCMSCache(ctx, ep)
})
return
}
// LoadEpsCMS picks ep meta information from Cache & DB
func (d *Dao) LoadEpsCMS(ctx context.Context, epids []int64) (resMetas map[int64]*model.EpCMS, err error) {
var (
cachedMetas, missedMetas map[int64]*model.EpCMS
missed []int64
addCache = true
)
resMetas = make(map[int64]*model.EpCMS)
// pick up the information for these season ids
if cachedMetas, missed, err = d.EpMetaCache(ctx, epids); err != nil {
log.Error("loadEpCMS EpMetaCache Sids:%v, Error:%v", epids, err)
err = nil
addCache = false // mc error, we don't add
}
if len(missed) > 0 {
if missedMetas, err = d.EpMetas(ctx, missed); err != nil {
log.Error("loadEpCMS EpMetas Sids:%v, Error:%v", epids, err)
return
}
}
// merge info from DB and the info from MC
resMetas = make(map[int64]*model.EpCMS, len(epids))
for sid, v := range cachedMetas {
resMetas[sid] = v
}
for sid, v := range missedMetas {
resMetas[sid] = v
}
log.Info("Combine Info for %d Epids, Origin Length %d", len(epids), len(resMetas))
if addCache && len(missedMetas) > 0 { // async Reset the DB data in MC for next time
log.Info("Set MissedMetas %d Data in MC", missedMetas)
for _, art := range missedMetas {
d.addCache(func() {
d.AddEpMetaCache(ctx, art)
})
}
}
return
}

View File

@@ -0,0 +1,113 @@
package cms
import (
"context"
"fmt"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsLoadSnsAuthMap(t *testing.T) {
var (
ctx = context.Background()
sids = []int64{}
)
convey.Convey("LoadSnsAuthMap", t, func(c convey.C) {
resMetas, err := d.LoadSnsAuthMap(ctx, sids)
c.Convey("Then err should be nil.resMetas should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(resMetas, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadEpsAuthMap(t *testing.T) {
var (
ctx = context.Background()
epids = []int64{}
)
convey.Convey("LoadEpsAuthMap", t, func(c convey.C) {
resMetas, err := d.LoadEpsAuthMap(ctx, epids)
c.Convey("Then err should be nil.resMetas should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(resMetas, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadSnsCMSMap(t *testing.T) {
var (
ctx = context.Background()
sids = []int64{}
)
convey.Convey("LoadSnsCMSMap", t, func(c convey.C) {
resMetas, err := d.LoadSnsCMSMap(ctx, sids)
c.Convey("Then err should be nil.resMetas should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(resMetas, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadSnsCMS(t *testing.T) {
var (
ctx = context.Background()
sids = []int64{}
err error
)
convey.Convey("LoadSnsCMS", t, func(c convey.C) {
c.Convey("Then err should be nil.seasons,newestEpids should not be nil.", func(cx convey.C) {
if sids, err = pickIDs(d.db, _pickSids); err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
seasons, newestEpids, err := d.LoadSnsCMS(ctx, sids)
cx.So(err, convey.ShouldBeNil)
cx.So(newestEpids, convey.ShouldNotBeNil)
cx.So(seasons, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadSnCMS(t *testing.T) {
var (
ctx = context.Background()
sid = int64(0)
)
convey.Convey("LoadSnCMS", t, func(c convey.C) {
sn, err := d.LoadSnCMS(ctx, sid)
c.Convey("Then err should be nil.sn should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(sn, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadEpCMS(t *testing.T) {
var (
ctx = context.Background()
epid = int64(0)
)
convey.Convey("LoadEpCMS", t, func(c convey.C) {
ep, err := d.LoadEpCMS(ctx, epid)
c.Convey("Then err should be nil.ep should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(ep, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadEpsCMS(t *testing.T) {
var (
ctx = context.Background()
epids = []int64{}
)
convey.Convey("LoadEpsCMS", t, func(c convey.C) {
resMetas, err := d.LoadEpsCMS(ctx, epids)
c.Convey("Then err should be nil.resMetas should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(resMetas, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,85 @@
package cms
import (
"context"
"go-common/app/interface/main/tv/model"
"go-common/library/cache/memcache"
"go-common/library/log"
)
// GetSnCMSCache get SeasonCMS cache.
func (d *Dao) GetSnCMSCache(c context.Context, sid int64) (s *model.SeasonCMS, err error) {
var (
key = snCMSCacheKey(sid)
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
missedCount.Add("tv-meta", 1)
} else {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(item, &s); err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
}
cachedCount.Add("tv-meta", 1)
return
}
// SetSnCMSCache save model.SeasonCMS to memcache
func (d *Dao) SetSnCMSCache(c context.Context, s *model.SeasonCMS) (err error) {
var (
key = snCMSCacheKey(s.SeasonID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: s, Flags: memcache.FlagJSON, Expiration: d.expireCMS}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// GetEpCMSCache get EpCMS cache.
func (d *Dao) GetEpCMSCache(c context.Context, epid int64) (s *model.EpCMS, err error) {
var (
key = epCMSCacheKey(epid)
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
missedCount.Add("tv-meta", 1)
} else {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(item, &s); err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
}
cachedCount.Add("tv-meta", 1)
return
}
// SetEpCMSCache save model.EpCMS to memcache
func (d *Dao) SetEpCMSCache(c context.Context, s *model.EpCMS) (err error) {
var (
key = epCMSCacheKey(s.EPID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: s, Flags: memcache.FlagJSON, Expiration: d.expireCMS}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}

View File

@@ -0,0 +1,77 @@
package cms
import (
"fmt"
"testing"
"go-common/app/interface/main/tv/model"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_SetSnCMSCache(t *testing.T) {
Convey("TestDao_SetSnCMSCache Test", t, WithDao(func(d *Dao) {
err := d.SetSnCMSCache(ctx, &model.SeasonCMS{
SeasonID: 6462,
Cover: "Test_cover1",
Title: "Test_title1",
Desc: "Test_desc1",
})
So(err, ShouldBeNil)
}))
}
func TestDao_GetSnCMSCache(t *testing.T) {
Convey("TestDao_GetSnCMSCache Test", t, WithDao(func(d *Dao) {
sids, errPick := pickIDs(d.db, _pickSids)
if errPick != nil || len(sids) == 0 {
fmt.Println("Empty sids ", errPick)
return
}
sid := sids[0]
d.LoadSnCMS(ctx, sid)
season, err := d.GetSnCMSCache(ctx, sid)
So(err, ShouldBeNil)
So(season, ShouldNotBeNil)
fmt.Println(*season)
}))
}
func TestDao_GetEpCMSCache(t *testing.T) {
Convey("TestDao_GetEpCMSCache Test", t, WithDao(func(d *Dao) {
sids, errPick := pickIDs(d.db, _pickEpids)
if errPick != nil || len(sids) == 0 {
fmt.Println("Empty sids ", errPick)
return
}
epid := sids[1]
ep, err := d.GetEpCMSCache(ctx, epid)
So(err, ShouldBeNil)
So(ep, ShouldNotBeNil)
fmt.Println(*ep)
}))
}
func TestDao_SnCMSCacheKey(t *testing.T) {
Convey("TestDao_SnCMSCacheKey Test", t, WithDao(func(d *Dao) {
key := snCMSCacheKey(177)
So(key, ShouldNotBeBlank)
fmt.Println(key)
}))
}
func TestDao_EPCMSCacheKey(t *testing.T) {
Convey("TestDao_EPCMSCacheKey Test", t, WithDao(func(d *Dao) {
key := epCMSCacheKey(1)
So(key, ShouldNotBeBlank)
fmt.Println(key)
}))
}
func TestDao_ArcCMSCacheKey(t *testing.T) {
Convey("TestDao_ArcCMSCacheKey Test", t, WithDao(func(d *Dao) {
key := d.ArcCMSCacheKey(177)
So(key, ShouldNotBeBlank)
fmt.Println(key)
}))
}

View File

@@ -0,0 +1,91 @@
package cms
import (
"context"
"fmt"
"go-common/app/interface/main/tv/model"
"go-common/library/log"
)
const (
_arcCMSDeleted = 1
_mcArcCMSKey = "arc_cms_%d"
_mcVideoCMSKey = "video_cms_%d"
)
// ArcCMSCacheKey .
func (d *Dao) ArcCMSCacheKey(aid int64) string {
return fmt.Sprintf(_mcArcCMSKey, aid)
}
// VideoCMSCacheKey .
func (d *Dao) VideoCMSCacheKey(cid int64) string {
return fmt.Sprintf(_mcVideoCMSKey, cid)
}
// ArcsMetaCache pick archive cms meta cache
func (d *Dao) ArcsMetaCache(c context.Context, ids []int64) (cached map[int64]*model.ArcCMS, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.ArcCMS, len(ids))
idmap, allKeys := keysTreat(ids, d.ArcCMSCacheKey)
conn := d.mc.Get(c)
defer conn.Close()
replys, err := conn.GetMulti(allKeys)
if err != nil {
PromError("mc:获取Archive信息缓存")
log.Error("conn.Gets(%v) error(%v)", allKeys, err)
err = nil
return
}
for key, item := range replys {
art := &model.ArcCMS{}
if err = conn.Scan(item, art); err != nil {
PromError("mc:获取Archive信息缓存json解析")
log.Error("item.Scan(%s) error(%v)", item.Value, err)
err = nil
continue
}
cached[idmap[key]] = art
delete(idmap, key)
}
missed = missedTreat(idmap, len(cached))
return
}
// VideosMetaCache pick video cms meta cache
func (d *Dao) VideosMetaCache(c context.Context, ids []int64) (cached map[int64]*model.VideoCMS, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.VideoCMS, len(ids))
idmap, allKeys := keysTreat(ids, d.VideoCMSCacheKey)
conn := d.mc.Get(c)
defer conn.Close()
replys, err := conn.GetMulti(allKeys)
if err != nil {
PromError("mc:获取Video信息缓存")
log.Error("conn.Gets(%v) error(%v)", allKeys, err)
err = nil
return
}
for key, item := range replys {
art := &model.VideoCMS{}
if err = conn.Scan(item, art); err != nil {
PromError("mc:获取Video信息缓存缓存json解析")
log.Error("item.Scan(%s) error(%v)", item.Value, err)
err = nil
continue
}
if art.Deleted == _arcCMSDeleted { // if it's deleted, we ignore it
log.Info("ArcCMS deleted, %v, %v", item, art)
continue
}
cached[idmap[key]] = art
delete(idmap, key)
}
missed = missedTreat(idmap, len(cached))
return
}

View File

@@ -0,0 +1,69 @@
package cms
import (
"context"
"fmt"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsArcCMSCacheKey(t *testing.T) {
var (
aid = int64(0)
)
convey.Convey("ArcCMSCacheKey", t, func(c convey.C) {
p1 := d.ArcCMSCacheKey(aid)
c.Convey("Then p1 should not be nil.", func(c convey.C) {
c.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsVideoCMSCacheKey(t *testing.T) {
var (
cid = int64(0)
)
convey.Convey("VideoCMSCacheKey", t, func(c convey.C) {
p1 := d.VideoCMSCacheKey(cid)
c.Convey("Then p1 should not be nil.", func(c convey.C) {
c.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestCmsArcsMetaCache(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("ArcsMetaCache", t, func(c convey.C) {
sids, errPick := pickIDs(d.db, _pickAids)
if errPick != nil || len(sids) == 0 {
fmt.Println("Empty sids ", errPick)
return
}
cached, missed, err := d.ArcsMetaCache(ctx, sids)
c.Convey("Then err should be nil.cached,missed should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(len(missed)+len(cached), convey.ShouldEqual, len(sids))
})
})
}
func TestCmsVideosMetaCache(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("VideosMetaCache", t, func(c convey.C) {
sids, errPick := pickIDs(d.db, _pickCids)
if errPick != nil || len(sids) == 0 {
fmt.Println("Empty sids ", errPick)
return
}
cached, missed, err := d.VideosMetaCache(ctx, sids)
c.Convey("Then err should be nil.cached,missed should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(len(missed)+len(cached), convey.ShouldEqual, len(sids))
})
})
}

View File

@@ -0,0 +1,148 @@
package cms
import (
"context"
"go-common/app/interface/main/tv/model"
"go-common/library/ecode"
"go-common/library/log"
)
// LoadArcsMediaMap loads a batch of arc meta
func (d *Dao) LoadArcsMediaMap(ctx context.Context, aids []int64) (resMetas map[int64]*model.ArcCMS, err error) {
var (
cachedMetas map[int64]*model.ArcCMS // cache hit seasons
missedMetas map[int64]*model.ArcCMS // cache miss seasons, pick from DB
missed []int64 // cache miss seasons
addCache = true // whether we need to fill DB data in MC
)
resMetas = make(map[int64]*model.ArcCMS) // merge info from MC and from DB
// pick up the information for these season ids
if cachedMetas, missed, err = d.ArcsMetaCache(ctx, aids); err != nil {
log.Error("LoadArcsMedia ArcsMetaCache Aids:%v, Error:%v", aids, err)
err = nil
addCache = false // mc error, we don't add
}
if len(missed) > 0 {
if missedMetas, err = d.ArcMetas(ctx, missed); err != nil {
log.Error("LoadArcsMedia ArcMetas Sids:%v, Error:%v", missed, err)
return
}
}
// merge info from DB and the info from MC
for sid, v := range cachedMetas {
resMetas[sid] = v
}
for sid, v := range missedMetas {
resMetas[sid] = v
}
// async Reset the DB data in MC for next time
log.Info("Set Sids [%d], MissedMetas [%d] Data in MC", len(aids), len(missedMetas))
if addCache && len(missedMetas) > 0 {
for _, art := range missedMetas {
d.AddArcMetaCache(art)
}
}
return
}
// LoadVideosMeta picks the videos meta info
func (d *Dao) LoadVideosMeta(ctx context.Context, cids []int64) (resMetas map[int64]*model.VideoCMS, err error) {
var (
cachedMetas map[int64]*model.VideoCMS // cache hit seasons
missedMetas map[int64]*model.VideoCMS // cache miss seasons, pick from DB
missed []int64 // cache miss seasons
addCache = true // whether we need to fill DB data in MC
)
resMetas = make(map[int64]*model.VideoCMS) // merge info from MC and from DB
// pick up the information for these season ids
if cachedMetas, missed, err = d.VideosMetaCache(ctx, cids); err != nil {
log.Error("LoadVideosMeta VideosMetaCache Aids:%v, Error:%v", cids, err)
err = nil
addCache = false // mc error, we don't add
}
if len(missed) > 0 {
if missedMetas, err = d.VideoMetas(ctx, missed); err != nil {
log.Error("LoadVideosMeta VideoMetas Sids:%v, Error:%v", missed, err)
return
}
}
// merge info from DB and the info from MC
for sid, v := range cachedMetas {
resMetas[sid] = v
}
for sid, v := range missedMetas {
resMetas[sid] = v
}
// async Reset the DB data in MC for next time
log.Info("Set Sids [%d], MissedMetas [%d] Data in MC", len(cids), len(missedMetas))
if addCache && len(missedMetas) > 0 {
for _, art := range missedMetas {
d.AddVideoMetaCache(art)
}
}
return
}
// LoadArcsMedia loads the arc meta cms data from cache, for missed ones, pick them from the DB
func (d *Dao) LoadArcsMedia(ctx context.Context, aids []int64) (arcs []*model.ArcCMS, err error) {
var (
resMetas map[int64]*model.ArcCMS // merge info from MC and from DB
)
if resMetas, err = d.LoadArcsMediaMap(ctx, aids); err != nil {
log.Error("LoadArcsMedia LoadArcsMediaMap Aids: %v, Err: %v", aids, err)
return
}
// re-arrange the info, according to the order got from Redis
for _, v := range aids {
if arcCMS, ok := resMetas[v]; !ok {
log.Error("PickDBeiPage LoadArcsMedia Miss Info for Sid: %d", v)
continue
} else {
arcs = append(arcs, arcCMS)
}
}
return
}
// LoadArcMeta loads the arc meta cms data from cache, for missed ones, pick them from the DB
func (d *Dao) LoadArcMeta(ctx context.Context, aid int64) (arcMeta *model.ArcCMS, err error) {
if arcMeta, err = d.ArcMetaCache(ctx, aid); err != nil { // mc error
log.Error("LoadArcMedia Get Aid [%d] from CMS Error (%v)", aid, err)
return
}
if arcMeta != nil { // mc found
return
}
if arcMeta, err = d.ArcMetaDB(ctx, aid); err != nil { // db error
log.Error("LoadArcMedia ArcMetaDB Aid ERROR (%d) (%v)", aid, err)
return
}
if arcMeta == nil { // db not found
err = ecode.NothingFound
return
}
d.AddArcMetaCache(arcMeta) // db found, re-fill the cache
return
}
// LoadVideoMeta loads the video meta cms data from cache, for missed ones, pick them from the DB
func (d *Dao) LoadVideoMeta(ctx context.Context, cid int64) (videoMeta *model.VideoCMS, err error) {
if videoMeta, err = d.VideoMetaCache(ctx, cid); err != nil { // mc error
log.Error("LoadVideoMeta Get Cid [%d] from CMS Error (%v)", cid, err)
return
}
if videoMeta != nil { // mc found
return
}
if videoMeta, err = d.VideoMetaDB(ctx, cid); err != nil { // db error
log.Error("LoadArcMedia ArcMetaDB Aid ERROR (%d) (%v)", cid, err)
return
}
if videoMeta == nil { // db not found
err = ecode.NothingFound
return
}
d.AddVideoMetaCache(videoMeta) // db found, re-fill the cache
return
}

View File

@@ -0,0 +1,97 @@
package cms
import (
"context"
"fmt"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsLoadArcsMediaMap(t *testing.T) {
var ctx = context.Background()
convey.Convey("LoadArcsMediaMap", t, func(c convey.C) {
aids, errPick := pickIDs(d.db, _pickAids)
if errPick != nil || len(aids) == 0 {
fmt.Println("Empty aids ", errPick)
return
}
resMetas, err := d.LoadArcsMediaMap(ctx, aids)
c.Convey("Then err should be nil.resMetas should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(resMetas, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadVideosMeta(t *testing.T) {
var (
ctx = context.Background()
cids = []int64{}
)
convey.Convey("LoadVideosMeta", t, func(c convey.C) {
resMetas, err := d.LoadVideosMeta(ctx, cids)
c.Convey("Then err should be nil.resMetas should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(resMetas, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadArcsMedia(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("LoadArcsMedia", t, func(c convey.C) {
aids, errPick := pickIDs(d.db, _pickAids)
if errPick != nil || len(aids) == 0 {
fmt.Println("Empty aids ", errPick)
return
}
arcs, err := d.LoadArcsMedia(ctx, aids)
c.Convey("Then err should be nil.arcs should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(arcs, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadArcMeta(t *testing.T) {
var (
ctx = context.Background()
aid = int64(0)
)
convey.Convey("LoadArcMeta", t, func(c convey.C) {
aids, errPick := pickIDs(d.db, _pickAids)
if errPick != nil || len(aids) == 0 {
fmt.Println("Empty aids ", errPick)
return
}
aid = aids[0]
arcMeta, err := d.LoadArcMeta(ctx, aid)
c.Convey("Then err should be nil.arcMeta should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(arcMeta, convey.ShouldNotBeNil)
})
})
}
func TestCmsLoadVideoMeta(t *testing.T) {
var (
ctx = context.Background()
cid = int64(0)
)
convey.Convey("LoadVideoMeta", t, func(c convey.C) {
aids, errPick := pickIDs(d.db, _pickCids)
if errPick != nil || len(aids) == 0 {
fmt.Println("Empty aids ", errPick)
return
}
cid = aids[0]
videoMeta, err := d.LoadVideoMeta(ctx, cid)
c.Convey("Then err should be nil.videoMeta should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(videoMeta, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,99 @@
package cms
import (
"context"
"go-common/app/interface/main/tv/model"
"go-common/library/cache/memcache"
"go-common/library/log"
)
// ArcMetaCache get arc cms cache.
func (d *Dao) ArcMetaCache(c context.Context, aid int64) (s *model.ArcCMS, err error) {
var (
key = d.ArcCMSCacheKey(aid)
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
missedCount.Add("tv-meta", 1)
} else {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(item, &s); err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
}
cachedCount.Add("tv-meta", 1)
return
}
// SetArcMetaCache save model.ArcCMS to memcache
func (d *Dao) SetArcMetaCache(c context.Context, s *model.ArcCMS) (err error) {
var (
key = d.ArcCMSCacheKey(s.AID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: s, Flags: memcache.FlagJSON, Expiration: d.expireCMS}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// AddArcMetaCache add view relates
func (d *Dao) AddArcMetaCache(arc *model.ArcCMS) {
d.addCache(func() {
d.SetArcMetaCache(context.TODO(), arc)
})
}
// VideoMetaCache get video cms cache.
func (d *Dao) VideoMetaCache(c context.Context, cid int64) (s *model.VideoCMS, err error) {
var (
key = d.VideoCMSCacheKey(cid)
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
missedCount.Add("tv-meta", 1)
} else {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(item, &s); err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
}
cachedCount.Add("tv-meta", 1)
return
}
// SetVideoMetaCache save model.VideoCMS to memcache
func (d *Dao) SetVideoMetaCache(c context.Context, s *model.VideoCMS) (err error) {
var (
key = d.VideoCMSCacheKey(s.CID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: s, Flags: memcache.FlagJSON, Expiration: d.expireCMS}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// AddVideoMetaCache add view relates
func (d *Dao) AddVideoMetaCache(video *model.VideoCMS) {
d.addCache(func() {
d.SetVideoMetaCache(context.TODO(), video)
})
}

View File

@@ -0,0 +1,105 @@
package cms
import (
"context"
"fmt"
"testing"
"go-common/app/interface/main/tv/model"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsArcMetaCache(t *testing.T) {
var (
c = context.Background()
aid = int64(0)
)
convey.Convey("ArcMetaCache", t, func(ctx convey.C) {
ctx.Convey("Then err should be nil.s should not be nil.", func(ctx convey.C) {
sids, err := pickIDs(d.db, _pickAids)
if err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
aid = sids[0]
d.LoadArcMeta(c, aid)
s, err := d.ArcMetaCache(c, aid)
ctx.So(err, convey.ShouldBeNil)
ctx.So(s, convey.ShouldNotBeNil)
})
ctx.Convey("mc not found Error", func(ctx convey.C) {
_, err := d.ArcMetaCache(c, 0)
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestCmsSetArcMetaCache(t *testing.T) {
var (
c = context.Background()
s = &model.ArcCMS{}
)
convey.Convey("SetArcMetaCache", t, func(ctx convey.C) {
err := d.SetArcMetaCache(c, s)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestCmsAddArcMetaCache(t *testing.T) {
var (
arc = &model.ArcCMS{}
)
convey.Convey("AddArcMetaCache", t, func(ctx convey.C) {
d.AddArcMetaCache(arc)
ctx.Convey("No return values", func(ctx convey.C) {
})
})
}
func TestCmsVideoMetaCache(t *testing.T) {
var (
c = context.Background()
cid = int64(0)
)
convey.Convey("VideoMetaCache", t, func(ctx convey.C) {
sids, err := pickIDs(d.db, _pickCids)
if err != nil || len(sids) == 0 {
fmt.Println("Empty Sids ", err)
return
}
cid = sids[0]
d.LoadVideoMeta(c, cid)
s, err := d.VideoMetaCache(c, cid)
ctx.Convey("Then err should be nil.s should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(s, convey.ShouldNotBeNil)
})
})
}
func TestCmsSetVideoMetaCache(t *testing.T) {
var (
c = context.Background()
s = &model.VideoCMS{}
)
convey.Convey("SetVideoMetaCache", t, func(ctx convey.C) {
err := d.SetVideoMetaCache(c, s)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestCmsAddVideoMetaCache(t *testing.T) {
var (
video = &model.VideoCMS{}
)
convey.Convey("AddVideoMetaCache", t, func(ctx convey.C) {
d.AddVideoMetaCache(video)
ctx.Convey("No return values", func(ctx convey.C) {
})
})
}