openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
app/job/main/tv/dao/BUILD Normal file
View File

@@ -0,0 +1,24 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/tv/dao/app:all-srcs",
"//app/job/main/tv/dao/archive:all-srcs",
"//app/job/main/tv/dao/cms:all-srcs",
"//app/job/main/tv/dao/ftp:all-srcs",
"//app/job/main/tv/dao/lic:all-srcs",
"//app/job/main/tv/dao/playurl:all-srcs",
"//app/job/main/tv/dao/report:all-srcs",
"//app/job/main/tv/dao/ugc:all-srcs",
"//app/job/main/tv/dao/upper:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

86
app/job/main/tv/dao/app/BUILD Normal file
View File

@@ -0,0 +1,86 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"cont_del_test.go",
"cont_sync_test.go",
"dao_test.go",
"media_cache_test.go",
"new_test.go",
"playurl_test.go",
"redis_test.go",
"search_con_test.go",
"search_sug_test.go",
"season_del_test.go",
"season_sync_test.go",
"sync_mc_test.go",
"sync_retry_test.go",
"zone_index_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/common:go_default_library",
"//app/job/main/tv/model/pgc:go_default_library",
"//library/database/sql:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
"//vendor/gopkg.in/h2non/gock.v1:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"cont_del.go",
"cont_sync.go",
"dao.go",
"media_cache.go",
"playurl.go",
"redis.go",
"search_con.go",
"search_sug.go",
"season_del.go",
"season_sync.go",
"sync_mc.go",
"sync_retry.go",
"zone_index.go",
],
importpath = "go-common/app/job/main/tv/dao/app",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/common:go_default_library",
"//app/job/main/tv/model/pgc:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/stat/prom:go_default_library",
"//library/xstr:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

71
app/job/main/tv/dao/app/cont_del.go Normal file
View File

@@ -0,0 +1,71 @@
package app
import (
"context"
dsql "database/sql"
"fmt"
"time"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_deletedContSQL = "SELECT id, title, subtitle, `desc`, cover, epid, season_id, cid FROM tv_content WHERE state = ? AND is_deleted = 1 AND audit_time < UNIX_TIMESTAMP(now()) ORDER BY season_id, epid LIMIT 0,"
_delaySyncSQL = "UPDATE tv_content SET audit_time = ? WHERE epid IN (%s)"
_delSyncContSQL = "UPDATE tv_content SET state = ? WHERE is_deleted = 1 AND epid = ?"
)
// DelCont picks the deleted content data to sync
func (d *Dao) DelCont(c context.Context) (res []*model.Content, err error) {
var (
rows *sql.Rows
nbData = d.conf.Sync.LConf.SizeMsg
)
if rows, err = d.DB.Query(c, _deletedContSQL+fmt.Sprintf("%d", nbData), EPToAudit); err != nil {
log.Error("d._deletedEPSQL.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.Content{}
if err = rows.Scan(&r.ID, &r.Title, &r.Subtitle, &r.Desc, &r.Cover, &r.EPID, &r.SeasonID, &r.CID); err != nil {
log.Error("DelCont row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.DelCont.Query error(%v)", err)
}
return
}
// SyncCont updates the delete content's state to not pass, avoid being selected again
func (d *Dao) SyncCont(c context.Context, cid int) (nbRows int64, err error) {
var res dsql.Result
if res, err = d.DB.Exec(c, _delSyncContSQL, EPNotPass, cid); err != nil {
log.Error("_delSyncContSQL, failed to update to auditing: (%v), Error: %v", cid, err)
return
}
return res.RowsAffected()
}
// DelaySync postpones the sync of the deleted content, in case of there is an error happenning in the interface
func (d *Dao) DelaySync(c context.Context, conts []*model.Content) (nbRows int64, err error) {
var (
res dsql.Result
xstrIds []int64
delay = time.Now().Unix() + int64(d.conf.Sync.Frequency.AuditDelay) // postpone the season's eps auditing to when
)
for _, v := range conts {
xstrIds = append(xstrIds, int64(v.EPID))
}
if res, err = d.DB.Exec(c, fmt.Sprintf(_delaySyncSQL, xstr.JoinInts(xstrIds)), delay); err != nil {
log.Error("_delSyncContSQL, failed to delay: (%v,%v), Error: %v", delay, xstrIds, err)
return
}
return res.RowsAffected()
}

89
app/job/main/tv/dao/app/cont_del_test.go Normal file
View File

@@ -0,0 +1,89 @@
package app
import (
"context"
"fmt"
"testing"
model "go-common/app/job/main/tv/model/pgc"
"github.com/smartystreets/goconvey/convey"
)
func pickDelEpid() (epid int64, err error) {
if err := d.DB.QueryRow(context.Background(), "select epid from tv_content where is_deleted = 1 limit 1").Scan(&epid); err != nil {
fmt.Println("Pick EPid Err ", err)
}
return
}
func pickDelSid() (sid int64, err error) {
if err := d.DB.QueryRow(context.Background(), "select id from tv_ep_season where is_deleted = 1 limit 1").Scan(&sid); err != nil {
fmt.Println("Pick sid Err ", err)
}
return
}
func TestAppDelCont(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("DelCont", t, func(ctx convey.C) {
res, err := d.DelCont(c)
if err == nil && len(res) == 0 {
fmt.Println("No deleted data, let me create one")
epid, errPick := pickDelEpid()
if errPick != nil {
fmt.Println("pick err ", errPick)
return
}
d.DB.Exec(c, "update tv_content set state = 1,audit_time = 0 where epid = ?", epid)
res, err = d.DelCont(c)
}
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
convey.Convey("DelCont", t, func(ctx convey.C) {
d.conf.Sync.LConf.SizeMsg = -1
_, err := d.DelCont(c)
ctx.So(err, convey.ShouldNotBeNil)
fmt.Println(err)
})
}
func TestAppSyncCont(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("SyncCont", t, func(ctx convey.C) {
epid, errPick := pickDelEpid()
if errPick != nil {
fmt.Println("pick err ", errPick)
return
}
nbRows, err := d.SyncCont(c, int(epid))
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}
func TestAppDelaySync(t *testing.T) {
var c = context.Background()
convey.Convey("DelaySync", t, func(ctx convey.C) {
epid, errPick := pickDelEpid()
if errPick != nil {
fmt.Println("pick err ", errPick)
return
}
conts := []*model.Content{{EPID: int(epid)}}
nbRows, err := d.DelaySync(c, conts)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}

196
app/job/main/tv/dao/app/cont_sync.go Normal file
View File

@@ -0,0 +1,196 @@
package app
import (
"context"
dsql "database/sql"
"fmt"
"time"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_pickDataSQL = "SELECT id, title, subtitle, `desc`, cover, epid, season_id, cid FROM tv_content " +
"WHERE season_id = ? AND state = 1 AND audit_time < UNIX_TIMESTAMP(now()) AND is_deleted = 0"
_pickSeasonSQL = "SELECT id,origin_name,title,alias,category,`desc`,style,area,play_time,info,state,total_num,upinfo,staff,role,copyright,`check`,is_deleted,cover,producer,version,status FROM tv_ep_season WHERE id = ?"
_pickEPSQL = "SELECT a.id,a.season_id,a.title,a.long_title,a.cover,a.length,a.cid,b.pay_status FROM tv_ep_content AS a LEFT JOIN tv_content AS b ON a.id=b.epid WHERE a.id=? AND a.is_deleted=0"
_waitCallSQL = "UPDATE tv_content SET audit_time = ? WHERE epid = ? AND state = ? AND is_deleted = 0"
_deleteEPSQL = "UPDATE tv_ep_content SET is_deleted = 1 WHERE season_id = ? AND is_deleted = 0"
_deleteContSQL = "UPDATE tv_content SET is_deleted = 1 WHERE season_id = ? AND is_deleted = 0"
_rejectContSQL = "UPDATE tv_content SET audit_time = ? WHERE season_id = ? AND state = ? AND is_deleted = 0"
_auditingContSQL = "UPDATE tv_content SET state = ? WHERE state = ? AND is_deleted = 0 AND epid IN (%s)"
_removeContSQL = "UPDATE tv_content SET state = ?,is_deleted = 1 WHERE state = ? AND is_deleted = 0 AND epid = ?"
_ContSQL = "SELECT id, title, subtitle, `desc`, cover, epid, season_id, cid FROM tv_content WHERE epid = ?"
_readySns = "SELECT DISTINCT a.season_id FROM tv_content a LEFT JOIN tv_ep_season b ON a.season_id = b.id " +
"WHERE a.state = 1 AND a.is_deleted = 0 AND a.audit_time < UNIX_TIMESTAMP(now()) " +
"AND b.`check` != 0 AND b.is_deleted = 0"
)
// RemoveCont is used to treat invalid ep data's content
func (d *Dao) RemoveCont(c context.Context, epid int) (nbRows int64, err error) {
var (
res dsql.Result
)
if res, err = d.DB.Exec(c, _removeContSQL, EPNotPass, EPToAudit, epid); err != nil {
log.Error("_removeContSQL, failed to delay: (%v,%v), Error: %v", err)
return
}
return res.RowsAffected()
}
// PickData picks the source content data to sync
func (d *Dao) PickData(c context.Context, sid int64) (res [][]*model.Content, err error) {
var (
rows *sql.Rows
nbData = d.conf.Sync.LConf.SizeMsg
conts []*model.Content
)
if rows, err = d.DB.Query(c, _pickDataSQL, sid); err != nil {
log.Error("d._pickDataSQL error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.Content{}
if err = rows.Scan(&r.ID, &r.Title, &r.Subtitle, &r.Desc, &r.Cover, &r.EPID, &r.SeasonID, &r.CID); err != nil {
log.Error("Conts row.Scan() error(%v)", err)
return
}
conts = append(conts, r)
if len(conts) >= nbData {
res = append(res, conts)
conts = []*model.Content{}
}
}
if err = rows.Err(); err != nil {
log.Error("d.PickData.Query error(%v)", err)
}
if len(conts) > 0 {
res = append(res, conts)
}
return
}
// Season gets one season data
func (d *Dao) Season(c context.Context, sid int) (r *model.TVEpSeason, err error) {
row := d.DB.QueryRow(c, _pickSeasonSQL, sid)
r = &model.TVEpSeason{}
if err = row.Scan(&r.ID, &r.OriginName, &r.Title, &r.Alias, &r.Category, &r.Desc, &r.Style, &r.Area, &r.PlayTime, &r.Info,
&r.State, &r.TotalNum, &r.Upinfo, &r.Staff, &r.Role, &r.Copyright, &r.Check, &r.IsDeleted, &r.Cover, &r.Producer, &r.Version, &r.Status); err != nil {
return
}
return
}
// EP gets one EP data
func (d *Dao) EP(c context.Context, epid int) (r *model.TVEpContent, err error) {
row := d.DB.QueryRow(c, _pickEPSQL, epid)
r = &model.TVEpContent{}
if err = row.Scan(&r.ID, &r.SeasonID, &r.Title, &r.LongTitle, &r.Cover, &r.Length, &r.CID, &r.PayStatus); err != nil {
if err != sql.ErrNoRows {
log.Error("row.Scan error(%v)", err)
}
return
}
return
}
// Cont picks the content data
func (d *Dao) Cont(c context.Context, epid int) (res *model.Content, err error) {
row := d.DB.QueryRow(c, _ContSQL, epid)
res = &model.Content{}
if err = row.Scan(&res.ID, &res.Title, &res.Subtitle, &res.Desc, &res.Cover, &res.EPID, &res.SeasonID, &res.CID); err != nil {
if err != sql.ErrNoRows {
log.Error("row.Scan error(%v)", err)
}
return
}
return
}
// WaitCall updates the audit time ( wait for caliing also )
func (d *Dao) WaitCall(c context.Context, epid int) (nbRows int64, err error) {
var (
res dsql.Result
delay = time.Now().Unix() + int64(d.conf.Sync.Frequency.WaitCall)
)
if res, err = d.DB.Exec(c, _waitCallSQL, delay, epid, EPToAudit); err != nil {
log.Error("_waitCallSQL, failed to delay: (%v,%v), Error: %v", err)
return
}
return res.RowsAffected()
}
// DeleteEP deletes ep data of a deleted season
func (d *Dao) DeleteEP(c context.Context, sid int) (nbRows int64, err error) {
var res dsql.Result
if res, err = d.DB.Exec(c, _deleteEPSQL, sid); err != nil {
log.Error("_deleteEPSQL, failed to delete: (%v), Error: %v", sid, err)
return
}
return res.RowsAffected()
}
// DeleteCont deletes cont data of a deleted season
func (d *Dao) DeleteCont(c context.Context, sid int) (nbRows int64, err error) {
var res dsql.Result
if res, err = d.DB.Exec(c, _deleteContSQL, sid); err != nil {
log.Error("_deleteContSQL, failed to delete: (%v), Error: %v", sid, err)
return
}
return res.RowsAffected()
}
// RejectCont postpones its content in 1 day
func (d *Dao) RejectCont(c context.Context, sid int) (nbRows int64, err error) {
var (
res dsql.Result
delay = time.Now().Unix() + int64(d.conf.Sync.Frequency.RejectWait)
)
if res, err = d.DB.Exec(c, _rejectContSQL, delay, sid, EPToAudit); err != nil {
log.Error("_rejectContSQL, failed to reject: (%v), Error: %v", sid, err)
return
}
return res.RowsAffected()
}
// AuditingCont updates the content state from 1 ( auditing ) to 2
func (d *Dao) AuditingCont(c context.Context, conts []*model.Content) (nbRows int64, err error) {
var (
res dsql.Result
epids []int64
)
for _, v := range conts {
epids = append(epids, int64(v.EPID))
}
if res, err = d.DB.Exec(c, fmt.Sprintf(_auditingContSQL, xstr.JoinInts(epids)), EPAuditing, EPToAudit); err != nil {
log.Error("_auditingContSQL, failed to update to auditing: (%v), Error: %v", epids, err)
return
}
return res.RowsAffected()
}
// ReadySns picks ready to sync seasons
func (d *Dao) ReadySns(c context.Context) (res []int64, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(c, _readySns); err != nil {
log.Error("d.ReadySns.Query: %s error(%v)", _readySns, err)
return
}
defer rows.Close()
for rows.Next() {
var r int64
if err = rows.Scan(&r); err != nil {
log.Error("ReadySns row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("ReadySns rows.Err() error(%v)", err)
}
return
}

175
app/job/main/tv/dao/app/cont_sync_test.go Normal file
View File

@@ -0,0 +1,175 @@
package app
import (
"context"
"testing"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"github.com/smartystreets/goconvey/convey"
)
func TestAppRemoveCont(t *testing.T) {
var (
c = context.Background()
epid = int(12373)
)
convey.Convey("RemoveCont", t, func(ctx convey.C) {
nbRows, err := d.RemoveCont(c, epid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}
func TestAppPickData(t *testing.T) {
var (
c = context.Background()
sid = int64(296)
)
convey.Convey("PickData", t, func(ctx convey.C) {
_, err := d.PickData(c, sid)
if err == sql.ErrNoRows {
println(err)
} else {
ctx.So(err, convey.ShouldBeNil)
}
})
}
func TestAppSeason(t *testing.T) {
var (
c = context.Background()
sid = int(296)
)
convey.Convey("Season", t, func(ctx convey.C) {
r, err := d.Season(c, sid)
if err == sql.ErrNoRows {
println(err)
} else {
ctx.So(err, convey.ShouldBeNil)
ctx.So(r, convey.ShouldNotBeNil)
}
})
}
func TestAppEP(t *testing.T) {
var (
c = context.Background()
epid = int(12373)
)
convey.Convey("EP", t, func(ctx convey.C) {
r, err := d.EP(c, epid)
if err == sql.ErrNoRows {
println(err)
} else {
ctx.So(err, convey.ShouldBeNil)
ctx.So(r, convey.ShouldNotBeNil)
}
})
}
func TestAppCont(t *testing.T) {
var (
c = context.Background()
epid = int(12373)
)
convey.Convey("Cont", t, func(ctx convey.C) {
res, err := d.Cont(c, epid)
if err == sql.ErrNoRows {
println(err)
} else {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
}
})
}
func TestAppWaitCall(t *testing.T) {
var (
c = context.Background()
epid = int(12373)
)
convey.Convey("WaitCall", t, func(ctx convey.C) {
nbRows, err := d.WaitCall(c, epid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}
func TestAppDeleteEP(t *testing.T) {
var (
c = context.Background()
sid = int(296)
)
convey.Convey("DeleteEP", t, func(ctx convey.C) {
nbRows, err := d.DeleteEP(c, sid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}
func TestAppDeleteCont(t *testing.T) {
var (
c = context.Background()
sid = int(296)
)
convey.Convey("DeleteCont", t, func(ctx convey.C) {
nbRows, err := d.DeleteCont(c, sid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}
func TestAppRejectCont(t *testing.T) {
var (
c = context.Background()
sid = int(296)
)
convey.Convey("RejectCont", t, func(ctx convey.C) {
nbRows, err := d.RejectCont(c, sid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}
func TestAppAuditingCont(t *testing.T) {
var (
c = context.Background()
conts []*model.Content
)
a := &model.Content{
EPID: 12373,
}
conts = append(conts, a)
convey.Convey("AuditingCont", t, func(ctx convey.C) {
_, err := d.AuditingCont(c, conts)
if err == sql.ErrNoRows {
println(err)
} else {
ctx.So(err, convey.ShouldBeNil)
}
})
}
func TestAppReadySns(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("ReadySns", t, func(ctx convey.C) {
_, err := d.ReadySns(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

83
app/job/main/tv/dao/app/dao.go Normal file
View File

@@ -0,0 +1,83 @@
package app
import (
"time"
"go-common/app/job/main/tv/conf"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/database/sql"
httpx "go-common/library/net/http/blademaster"
"go-common/library/stat/prom"
)
// Dao dao.
type Dao struct {
conf *conf.Config
// DB
DB *sql.DB
// Memcache
mc *memcache.Pool
mcExpire int32
mcMediaExpire int32
// Http client
client *httpx.Client
// redis
redis *redis.Pool
redisExpire int32
}
var (
errorsCount = prom.BusinessErrCount
infosCount = prom.BusinessInfoCount
)
// PromError prometheus error count.
func PromError(name string) {
errorsCount.Incr(name)
}
// PromInfo prometheus info count.
func PromInfo(name string) {
infosCount.Incr(name)
}
// New create a instance of Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
// conf
conf: c,
// db
DB: sql.NewMySQL(c.Mysql),
// mc
mc: memcache.NewPool(c.Memcache.Config),
mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
mcMediaExpire: int32(time.Duration(c.Memcache.ExpireMedia) / time.Second),
client: httpx.NewClient(conf.Conf.HTTPClient),
redis: redis.NewPool(c.Redis.Config),
redisExpire: int32(time.Duration(c.Redis.Expire) / time.Second),
}
return
}
// Close close the redis and kafka resource.
func (d *Dao) Close() {
if d.DB != nil {
d.DB.Close()
}
if d.mc != nil {
d.mc.Close()
}
if d.redis != nil {
d.redis.Close()
}
}
// NumPce calculates number of piece
func NumPce(count int, pagesize int) (numPce int) {
if count%pagesize == 0 {
numPce = count / pagesize
return
}
return count/pagesize + 1
}

40
app/job/main/tv/dao/app/dao_test.go Normal file
View File

@@ -0,0 +1,40 @@
package app
import (
"flag"
"os"
"testing"
"go-common/app/job/main/tv/conf"
"gopkg.in/h2non/gock.v1"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
// os.Setenv("DEPLOY_ENV", "")
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.web-svr.tv-job")
flag.Set("conf_token", "ab3e9801a77c076b997de0ac5cb21775")
flag.Set("tree_id", "15260")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../../cmd/tv-job-test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
d.client.SetTransport(gock.DefaultTransport)
m.Run()
os.Exit(0)
}

94
app/job/main/tv/dao/app/media_cache.go Normal file
View File

@@ -0,0 +1,94 @@
package app
import (
"context"
"fmt"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/cache/memcache"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_mcSnCMSKey = "sn_cms_%d"
_mcEPCMSKey = "ep_cms_%d"
_newestOrder = "SELECT a.epid,b.`order` FROM tv_content a LEFT JOIN tv_ep_content b ON a.epid=b.id " +
"WHERE a.season_id=? AND a.state= ? AND a.valid= ? AND a.is_deleted=0 ORDER BY b.`order` DESC LIMIT 1"
_AllEPs = "SELECT subtitle, epid FROM tv_content WHERE season_id = ?"
_lessStrategy = 1
)
// SnCMSCacheKey .
func (d *Dao) SnCMSCacheKey(sid int) string {
return fmt.Sprintf(_mcSnCMSKey, sid)
}
// EpCMSCacheKey .
func (d *Dao) EpCMSCacheKey(epid int) string {
return fmt.Sprintf(_mcEPCMSKey, epid)
}
// SetSnCMSCache save model.SeasonCMS to memcache
func (d *Dao) SetSnCMSCache(c context.Context, s *model.SeasonCMS) (err error) {
var (
key = d.SnCMSCacheKey(s.SeasonID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: s, Flags: memcache.FlagJSON, Expiration: d.mcMediaExpire}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// SetEpCMSCache save model.EpCMS to memcache
func (d *Dao) SetEpCMSCache(c context.Context, s *model.EpCMS) (err error) {
var (
key = d.EpCMSCacheKey(s.EPID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: s, Flags: memcache.FlagJSON, Expiration: d.mcMediaExpire}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// NewestOrder picks one season's newest passed ep's order column value
func (d *Dao) NewestOrder(c context.Context, sid int64) (epid, newestOrder int, err error) {
if err = d.DB.QueryRow(c, _newestOrder, sid, EPPassed, _CMSValid).Scan(&epid, &newestOrder); err != nil { // get the qualified aid to sync
log.Info("d.NewestOrder(sid %d).Query error(%v)", sid, err)
}
return
}
// AllEP picks all the not deleted ep of a season
func (d *Dao) AllEP(c context.Context, sid int, strategy int) (eps []*model.EpCMS, err error) {
var (
rows *sql.Rows
query = _AllEPs
)
if strategy == _lessStrategy {
query = _AllEPs + " AND is_deleted = 0" // less strategy
}
if rows, err = d.DB.Query(c, query, sid); err != nil {
log.Error("d.AllEP.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.EpCMS{}
if err = rows.Scan(&r.Title, &r.EPID); err != nil {
log.Error("AllEP row.Scan() error(%v)", err)
return
}
eps = append(eps, r)
}
if err = rows.Err(); err != nil {
log.Error("d.AllEp.Query error(%v)", err)
}
return
}

102
app/job/main/tv/dao/app/media_cache_test.go Normal file
View File

@@ -0,0 +1,102 @@
package app
import (
"context"
"fmt"
model "go-common/app/job/main/tv/model/pgc"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppSnCMSCacheKey(t *testing.T) {
var (
sid = int(0)
)
convey.Convey("SnCMSCacheKey", t, func(ctx convey.C) {
p1 := d.SnCMSCacheKey(sid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestAppEpCMSCacheKey(t *testing.T) {
var (
epid = int(0)
)
convey.Convey("EpCMSCacheKey", t, func(ctx convey.C) {
p1 := d.EpCMSCacheKey(epid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestAppSetSnCMSCache(t *testing.T) {
var (
c = context.Background()
s = &model.SeasonCMS{}
)
convey.Convey("SetSnCMSCache", t, func(ctx convey.C) {
err := d.SetSnCMSCache(c, s)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestAppSetEpCMSCache(t *testing.T) {
var (
c = context.Background()
s = &model.EpCMS{}
)
convey.Convey("SetEpCMSCache", t, func(ctx convey.C) {
err := d.SetEpCMSCache(c, s)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func pickEpSid() (sid int64, err error) {
if err = d.DB.QueryRow(context.Background(), "select season_id from tv_content where is_deleted = 0 "+
"and valid = 1 and state = 3 limit 1").Scan(&sid); err != nil {
fmt.Println(err)
}
return
}
func TestAppNewestOrder(t *testing.T) {
var c = context.Background()
convey.Convey("NewestOrder", t, func(ctx convey.C) {
sid, errPick := pickEpSid()
if errPick != nil {
return
}
epid, newestOrder, err := d.NewestOrder(c, sid)
ctx.Convey("Then err should be nil.epid,newestOrder should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(newestOrder, convey.ShouldNotBeNil)
ctx.So(epid, convey.ShouldNotBeNil)
})
})
}
func TestAppAllEP(t *testing.T) {
var (
c = context.Background()
strategy = int(0)
)
convey.Convey("AllEP", t, func(ctx convey.C) {
sid, errPick := pickEpSid()
if errPick != nil {
return
}
eps, err := d.AllEP(c, int(sid), strategy)
ctx.Convey("Then err should be nil.eps should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(eps, convey.ShouldNotBeNil)
})
})
}

42
app/job/main/tv/dao/app/new_test.go Normal file
View File

@@ -0,0 +1,42 @@
package app
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppPromError(t *testing.T) {
var (
name = "test"
)
convey.Convey("PromError", t, func(ctx convey.C) {
PromError(name)
ctx.Convey("No return values", func(ctx convey.C) {
})
})
}
func TestAppPromInfo(t *testing.T) {
var (
name = "test"
)
convey.Convey("PromInfo", t, func(ctx convey.C) {
PromInfo(name)
ctx.Convey("No return values", func(ctx convey.C) {
})
})
}
func TestAppNumPce(t *testing.T) {
var (
count = int(50)
pagesize = int(10)
)
convey.Convey("NumPce", t, func(ctx convey.C) {
numPce := NumPce(count, pagesize)
ctx.Convey("Then numPce should not be nil.", func(ctx convey.C) {
ctx.So(numPce, convey.ShouldNotBeNil)
})
})
}

42
app/job/main/tv/dao/app/playurl.go Normal file
View File

@@ -0,0 +1,42 @@
package app
import (
"context"
"fmt"
"net/url"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/log"
)
const _type = "mp4"
const _maxBackup = 0
const _otype = "json"
// Playurl calls the api of playurl to get the url to play the video
func (d *Dao) Playurl(ctx context.Context, cid int) (playurl string, err error) {
var (
result = model.PlayurlResp{}
params = url.Values{}
api = d.conf.Sync.PlayURL.API
)
params.Set("cid", fmt.Sprintf("%d", cid))
params.Set("type", _type) // to get one piece
params.Set("max_backup", fmt.Sprintf("%d", _maxBackup)) // no backup url needed
params.Set("otype", _otype) // json format response
params.Set("qn", d.conf.Sync.PlayURL.Qn) // quality fix to 16
if err = d.client.Get(ctx, api, "", params, &result); err != nil {
log.Error("ClientGet error[%v]", err)
return
}
if result.Code != 0 { // logic error
err = fmt.Errorf("Resp Code:[%v], Message:[%v]", result.Code, result.Message)
return
}
if len(result.Durl) < 1 { // result empty
err = fmt.Errorf("Playurl Result is Empty! Resp (%v)", result)
return
}
playurl = result.Durl[0].URL
return
}

22
app/job/main/tv/dao/app/playurl_test.go Normal file
View File

@@ -0,0 +1,22 @@
package app
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppPlayurl(t *testing.T) {
var (
c = context.Background()
cid = int(2578706)
)
convey.Convey("Playurl", t, func(ctx convey.C) {
playurl, err := d.Playurl(c, cid)
ctx.Convey("Then err should be nil.playurl should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(playurl, convey.ShouldNotBeNil)
})
})
}

124
app/job/main/tv/dao/app/redis.go Normal file
View File

@@ -0,0 +1,124 @@
package app
import (
"context"
"fmt"
"time"
commonMdl "go-common/app/job/main/tv/model/common"
"go-common/library/cache/redis"
"go-common/library/log"
)
// keyZone gets the key of the zone in Redis
func keyZone(category int) string {
return fmt.Sprintf("zone_idx_%d", category)
}
// Flush it flushes the list of one zone
func (d *Dao) Flush(c context.Context, category int, idxRanks []*commonMdl.IdxRank) (err error) {
var (
ctime int64
length int64
conn = d.redis.Get(c)
key = keyZone(category)
)
// remove the previous list
if err = conn.Send("EXPIRE", key, 0); err != nil {
log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
return
}
// add new ids inside
for _, v := range idxRanks {
ctime = int64(v.Ctime)
if err = conn.Send("ZADD", key, ctime, v.ID); err != nil {
log.Error("conn.Send(ZADD %s %v) error(%v)", key, v.ID, ctime)
return
}
}
// set expiration
if err = conn.Send("EXPIRE", key, d.redisExpire); err != nil {
log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
return
}
// check result
if err = conn.Send("ZCARD", key); err != nil {
log.Error("conn.Send(ZCARD %s) error(%v)", key, err)
return
}
// flush result
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for i := 0; i < len(idxRanks)+2; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
if length, err = redis.Int64(conn.Receive()); err != nil {
log.Error("conn.Receive() error(%v)", err)
}
log.Info("Flush Succ! Zone %d, Items: %d", category, length)
conn.Close()
return
}
// TimeTrans transform the time format to Unix Timestamp
func TimeTrans(stimeStr string) (stime int64, err error) {
local, _ := time.LoadLocation("Local")
var (
timeValue time.Time
)
timeValue, err = time.ParseInLocation("2006-01-02 15:04:05", stimeStr, local)
if err != nil {
log.Warn("TimeTrans %s, Error %v", stimeStr, err)
return
}
if stime = timeValue.Unix(); stime < 1 {
err = fmt.Errorf("time %s transform %d error", stimeStr, stime)
return
}
return
}
// ZAddIdx adds one valid season into the zone list
func (d *Dao) ZAddIdx(c context.Context, category int, ctimeStr string, id int64) (err error) {
var (
conn = d.redis.Get(c)
key = keyZone(category)
ctime int64
)
defer conn.Close()
if ctime, err = TimeTrans(ctimeStr); err != nil {
log.Warn("ZAddIdx Ctime %s Error %v", ctimeStr, err)
return
}
if err = conn.Send("ZADD", key, ctime, id); err != nil {
log.Error("conn.Send(ZADD %s - %v) error(%v)", key, id, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
return
}
// ZRemIdx ZREM trim from trim queue.
func (d *Dao) ZRemIdx(c context.Context, category int, id int64) (err error) {
var (
conn = d.redis.Get(c)
key = keyZone(category)
)
if _, err = conn.Do("ZREM", key, id); err != nil {
log.Error("conn.Send(ZADD %s - %v) error(%v)", key, id, err)
}
conn.Close()
return
}

77
app/job/main/tv/dao/app/redis_test.go Normal file
View File

@@ -0,0 +1,77 @@
package app
import (
"context"
commonMdl "go-common/app/job/main/tv/model/common"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppkeyZone(t *testing.T) {
var (
category = int(0)
)
convey.Convey("keyZone", t, func(ctx convey.C) {
p1 := keyZone(category)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestAppFlush(t *testing.T) {
var (
c = context.Background()
category = int(0)
idxRanks = []*commonMdl.IdxRank{}
)
convey.Convey("Flush", t, func(ctx convey.C) {
err := d.Flush(c, category, idxRanks)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestAppTimeTrans(t *testing.T) {
var (
stimeStr = "2018-09-16 08:00:01"
)
convey.Convey("TimeTrans", t, func(ctx convey.C) {
stime, err := TimeTrans(stimeStr)
ctx.Convey("Then err should be nil.stime should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(stime, convey.ShouldNotBeNil)
})
})
}
func TestAppZAddIdx(t *testing.T) {
var (
c = context.Background()
category = int(0)
ctimeStr = "2018-09-16 08:00:01"
id = int64(0)
)
convey.Convey("ZAddIdx", t, func(ctx convey.C) {
err := d.ZAddIdx(c, category, ctimeStr, id)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestAppZRemIdx(t *testing.T) {
var (
c = context.Background()
category = int(0)
id = int64(0)
)
convey.Convey("ZRemIdx", t, func(ctx convey.C) {
err := d.ZRemIdx(c, category, id)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

48
app/job/main/tv/dao/app/search_con.go Normal file
View File

@@ -0,0 +1,48 @@
package app
import (
"context"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_PgcFromwhere = "FROM tv_ep_season WHERE `check` = 1 AND valid = 1 AND is_deleted = 0 "
_PgcCont = "SELECT id,category,cover,title,play_time,role,staff,`desc` " + _PgcFromwhere + "AND id > ? limit ?"
_PgcContCount = " SELECT count(*) " + _PgcFromwhere
)
// PgcCont is used for getting valid pgc season data
func (d *Dao) PgcCont(ctx context.Context, id int, limit int) (res []*model.SearPgcCon, err error) {
var (
rows *sql.Rows
)
if rows, err = d.DB.Query(ctx, _PgcCont, id, limit); err != nil {
log.Error("d.PgcCont.Query: %s error(%v)", _PgcCont, err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.SearPgcCon{}
if err = rows.Scan(&r.ID, &r.Category, &r.Cover, &r.Title, &r.PlayTime, &r.Role, &r.Staff, &r.Desc); err != nil {
log.Error("PgcCont row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.PgcCont.Query error(%v)", err)
}
return
}
// PgcContCount is used for getting valid data count
func (d *Dao) PgcContCount(ctx context.Context) (upCnt int, err error) {
row := d.DB.QueryRow(ctx, _PgcContCount)
if err = row.Scan(&upCnt); err != nil {
log.Error("d.SeaContCount.Query: %s error(%v)", _PgcContCount, err)
}
return
}

36
app/job/main/tv/dao/app/search_con_test.go Normal file
View File

@@ -0,0 +1,36 @@
package app
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppPgcCont(t *testing.T) {
var (
c = context.Background()
id = int(0)
limit = int(10)
)
convey.Convey("PgcCont", t, func(ctx convey.C) {
res, err := d.PgcCont(c, id, limit)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestAppPgcContCount(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("PgcContCount", t, func(ctx convey.C) {
upCnt, err := d.PgcContCount(c)
ctx.Convey("Then err should be nil.upCnt should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(upCnt, convey.ShouldNotBeNil)
})
})
}

68
app/job/main/tv/dao/app/search_sug.go Normal file
View File

@@ -0,0 +1,68 @@
package app
import (
"context"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_pgcSeaSug = "SELECT id,title FROM tv_ep_season WHERE `check` = ? AND valid = ? AND is_deleted = ?"
_ugcSeaSug = "SELECT aid,title FROM ugc_archive WHERE result=1 AND valid=1 AND deleted=0 "
_pgcType = "pgc"
_ugcType = "ugc"
)
// PgcSeaSug is used for getting pgc search suggest content
func (d *Dao) PgcSeaSug(ctx context.Context) (res []*model.SearchSug, err error) {
var (
rows *sql.Rows
)
if rows, err = d.DB.Query(ctx, _pgcSeaSug, SeasonPassed, _CMSValid, _NotDeleted); err != nil {
log.Error("d.PgcSeaSug.Query: %s error(%v)", _pgcSeaSug, err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.SearchSug{
Type: _pgcType,
}
if err = rows.Scan(&r.ID, &r.Term); err != nil {
log.Error("PgcSeaSug row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.PgcSeaSug.Query error(%v)", err)
}
return
}
// UgcSeaSug is used for getting ugc search suggest content
func (d *Dao) UgcSeaSug(ctx context.Context) (res []*model.SearchSug, err error) {
var (
rows *sql.Rows
)
if rows, err = d.DB.Query(ctx, _ugcSeaSug); err != nil {
log.Error("d.UgcSeaSug.Query: %s error(%v)", _ugcSeaSug, err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.SearchSug{
Type: _ugcType,
}
if err = rows.Scan(&r.ID, &r.Term); err != nil {
log.Error("UgcSeaSug row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.UgcSeaSug.Query error(%v)", err)
}
return
}

34
app/job/main/tv/dao/app/search_sug_test.go Normal file
View File

@@ -0,0 +1,34 @@
package app
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppPgcSeaSug(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("PgcSeaSug", t, func(ctx convey.C) {
res, err := d.PgcSeaSug(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestAppUgcSeaSug(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("UgcSeaSug", t, func(ctx convey.C) {
res, err := d.UgcSeaSug(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}

52
app/job/main/tv/dao/app/season_del.go Normal file
View File

@@ -0,0 +1,52 @@
package app
import (
"context"
dsql "database/sql"
"fmt"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_deletedSeasonSQL = "SELECT id,origin_name,title,alias,category,`desc`,style,area,play_time,info,state,total_num,upinfo,staff,role,copyright,`check`,is_deleted " +
"FROM tv_ep_season WHERE is_deleted = 1 AND `check` = ? AND audit_time < UNIX_TIMESTAMP(now()) LIMIT 0,"
_delSyncSeasonSQL = "UPDATE tv_ep_season SET `check` = ? WHERE is_deleted = 1 AND id = ?"
)
// DelSeason picks the modified season data to sync
func (d *Dao) DelSeason(c context.Context) (res []*model.TVEpSeason, err error) {
var (
rows *sql.Rows
)
if rows, err = d.DB.Query(c, _deletedSeasonSQL+fmt.Sprintf("%d", d.conf.Sync.LConf.NbSeason), SeasonToReAudit); err != nil {
log.Error("d._deletedSeasonSQL.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.TVEpSeason{}
if err = rows.Scan(&r.ID, &r.OriginName, &r.Title, &r.Alias, &r.Category, &r.Desc, &r.Style, &r.Area, &r.PlayTime, &r.Info,
&r.State, &r.TotalNum, &r.Upinfo, &r.Staff, &r.Role, &r.Copyright, &r.Check, &r.IsDeleted); err != nil {
log.Error("modSeason row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.DelSeason.Query error(%v)", err)
}
return
}
// RejectSeason updates the indicated season to rejected status
func (d *Dao) RejectSeason(c context.Context, sid int) (nbRows int64, err error) {
var res dsql.Result
if res, err = d.DB.Exec(c, _delSyncSeasonSQL, SeasonRejected, sid); err != nil {
log.Error("_delSyncSeason, failed to update to auditing: (%v), Error: %v", sid, err)
return
}
return res.RowsAffected()
}

45
app/job/main/tv/dao/app/season_del_test.go Normal file
View File

@@ -0,0 +1,45 @@
package app
import (
"context"
"fmt"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppDelSeason(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("DelSeason", t, func(ctx convey.C) {
res, err := d.DelSeason(c)
if err == nil && len(res) == 0 {
sid, errPick := pickDelSid()
if errPick != nil {
return
}
fmt.Println(sid)
d.DB.Exec(c, "update tv_ep_season set `check` = 2,audit_time = 0 where id = ?", sid)
res, err = d.DelSeason(c)
}
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestAppRejectSeason(t *testing.T) {
var (
c = context.Background()
sid = int(0)
)
convey.Convey("RejectSeason", t, func(ctx convey.C) {
nbRows, err := d.RejectSeason(c, sid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}

109
app/job/main/tv/dao/app/season_sync.go Normal file
View File

@@ -0,0 +1,109 @@
package app
import (
"context"
dsql "database/sql"
"fmt"
"time"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/log"
)
// season `check` meaning
const (
SeasonRejected = 0
SeasonPassed = 1
SeasonToReAudit = 2
SeasonAuditing = 3
SeasonAlreadyPassed = 7
EPToAudit = 1
EPAuditing = 2
EPPassed = 3
EPNotPass = 4
_CMSValid = 1
_NotDeleted = 0
)
const (
_modifiedSeason = "SELECT id,origin_name,title,alias,category,`desc`,style,area,play_time,info,state,total_num,upinfo,staff,role,copyright,`check`,is_deleted,cover,producer,version,status" +
" FROM tv_ep_season WHERE `check` = ? AND audit_time < UNIX_TIMESTAMP(now()) AND is_deleted = 0 LIMIT 0,"
_snEmpty = "SELECT id FROM tv_content WHERE season_id = ? AND is_deleted = 0 LIMIT 1"
_auditingSeason = "UPDATE tv_ep_season SET `check` = ? WHERE is_deleted = 0 AND id = ?"
_delaySeason = "UPDATE tv_ep_season SET audit_time = ? WHERE id = ?"
)
// SnEmpty determines whether the
func (d *Dao) SnEmpty(c context.Context, sid int64) (res bool, err error) {
var epid int
if err = d.DB.QueryRow(c, _snEmpty, sid).Scan(&epid); err != nil {
if err == sql.ErrNoRows {
err = nil
res = true
return
}
log.Error("d.SnEmpty Error (%v)", err)
return
}
res = false
return
}
// ModSeason picks the modified season data to sync
func (d *Dao) ModSeason(c context.Context) (res []*model.TVEpSeason, err error) {
var (
rows *sql.Rows
isEmpty bool
)
if rows, err = d.DB.Query(c, _modifiedSeason+fmt.Sprintf("%d", d.conf.Sync.LConf.NbSeason), SeasonToReAudit); err != nil {
log.Error("d._modifiedSeason.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.TVEpSeason{}
if err = rows.Scan(&r.ID, &r.OriginName, &r.Title, &r.Alias, &r.Category, &r.Desc, &r.Style, &r.Area, &r.PlayTime, &r.Info,
&r.State, &r.TotalNum, &r.Upinfo, &r.Staff, &r.Role, &r.Copyright, &r.Check, &r.IsDeleted, &r.Cover,
&r.Producer, &r.Version, &r.Status); err != nil {
log.Error("modSeason row.Scan() error(%v)", err)
return
}
if isEmpty, err = d.SnEmpty(c, r.ID); err != nil {
log.Error("modSeason SnEmpty Error (%v)", err)
return
}
if !isEmpty { // we don't submit empty season to audit
res = append(res, r)
} else {
d.DelaySeason(c, r.ID)
}
}
if err = rows.Err(); err != nil {
log.Error("d.ModSeason.Query error(%v)", err)
}
return
}
// AuditSeason updates the indicated season to Auditing Status
func (d *Dao) AuditSeason(c context.Context, sid int) (nbRows int64, err error) {
var res dsql.Result
if res, err = d.DB.Exec(c, _auditingSeason, SeasonAuditing, sid); err != nil {
log.Error("_auditingSeason, failed to update to auditing: (%v), Error: %v", sid, err)
return
}
return res.RowsAffected()
}
// DelaySeason postpones the season to sync in 30 minutes
func (d *Dao) DelaySeason(c context.Context, sid int64) (nbRows int64, err error) {
var (
res dsql.Result
delay = time.Now().Unix() + d.conf.Sync.Frequency.AuditDelay
)
if res, err = d.DB.Exec(c, _delaySeason, delay, sid); err != nil {
log.Error("_delaySeason, failed to delay: (%v,%v), Error: %v", err)
return
}
return res.RowsAffected()
}

71
app/job/main/tv/dao/app/season_sync_test.go Normal file
View File

@@ -0,0 +1,71 @@
package app
import (
"context"
"fmt"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppSnEmpty(t *testing.T) {
var (
c = context.Background()
sid = int64(12373)
)
convey.Convey("SnEmpty", t, func(ctx convey.C) {
res, err := d.SnEmpty(c, sid)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestAppModSeason(t *testing.T) {
var c = context.Background()
convey.Convey("ModSeason", t, func(ctx convey.C) {
res, err := d.ModSeason(c)
if len(res) == 0 && err == nil {
sid, errPick := pickEpSid()
if errPick != nil {
return
}
fmt.Println(sid)
d.DB.Exec(c, "update tv_ep_season set `check` = 2,audit_time=0 where id = ?", sid)
res, err = d.ModSeason(c)
}
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestAppAuditSeason(t *testing.T) {
var (
c = context.Background()
sid = int(12373)
)
convey.Convey("AuditSeason", t, func(ctx convey.C) {
nbRows, err := d.AuditSeason(c, sid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}
func TestAppDelaySeason(t *testing.T) {
var (
c = context.Background()
sid = int64(12373)
)
convey.Convey("DelaySeason", t, func(ctx convey.C) {
nbRows, err := d.DelaySeason(c, sid)
ctx.Convey("Then err should be nil.nbRows should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(nbRows, convey.ShouldNotBeNil)
})
})
}

162
app/job/main/tv/dao/app/sync_mc.go Normal file
View File

@@ -0,0 +1,162 @@
package app
import (
"context"
"fmt"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/cache/memcache"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_seasonKey = "sn_%d"
_epKey = "ep_%d"
_countEP = "SELECT COUNT(1) AS cnt FROM tv_content"
_countSeason = "SELECT COUNT(1) AS cnt FROM tv_ep_season"
_pickEPMC = "SELECT is_deleted, state,valid, season_id, epid,id, mark, cover, title, subtitle, pay_status FROM tv_content " +
"WHERE id > ? ORDER BY id LIMIT 0,"
_pickSeasonMC = "SELECT is_deleted, `check`,valid, id, cover, `desc`, title, upinfo, category, area, play_time, role, staff, total_num, style, alias, origin_name, status FROM tv_ep_season " +
"WHERE id > ? ORDER BY id LIMIT 0,"
_singleSn = "SELECT id, cover, `desc`, title, upinfo, category, area, play_time, role, staff, total_num, style, alias, origin_name, status FROM tv_ep_season WHERE id = ?"
)
// EpCacheKey is used to generate the key of ep
func EpCacheKey(epid int) string {
return fmt.Sprintf(_epKey, epid)
}
// SeasonCacheKey is used to generate the key of season
func SeasonCacheKey(sid int) string {
return fmt.Sprintf(_seasonKey, sid)
}
// SetEP in MC
func (d *Dao) SetEP(ctx context.Context, res *model.SimpleEP) (err error) {
var (
key = EpCacheKey(res.EPID)
conn = d.mc.Get(ctx)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: res, Expiration: d.mcExpire, Flags: memcache.FlagJSON}); err != nil {
log.Error("conn.Set(%s,%v) error(%v)", key, res, err)
}
return
}
// SetSeason in MC
func (d *Dao) SetSeason(ctx context.Context, res *model.SimpleSeason) (err error) {
var (
key = SeasonCacheKey(int(res.ID))
conn = d.mc.Get(ctx)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: res, Expiration: d.mcExpire, Flags: memcache.FlagJSON}); err != nil {
log.Error("conn.Set(%s,%v) error(%v)", key, res, err)
}
return
}
// CountEP counts number of ep in DB
func (d *Dao) CountEP(ctx context.Context) (count int, err error) {
row := d.DB.QueryRow(ctx, _countEP)
err = row.Scan(&count)
return
}
// CountSeason counts number of ep in DB
func (d *Dao) CountSeason(ctx context.Context) (count int, err error) {
row := d.DB.QueryRow(ctx, _countSeason)
err = row.Scan(&count)
return
}
// RefreshEPMC picks data by Piece to sync in MC
func (d *Dao) RefreshEPMC(ctx context.Context, LastID int, nbData int) (myLast int, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(ctx, _pickEPMC+fmt.Sprintf("%d", nbData), LastID); err != nil {
log.Error("d._pickEPMC.Query: %s error(%v)", _pickEPMC+fmt.Sprintf("%d", nbData), err)
return
}
defer rows.Close()
for rows.Next() {
var (
r = &model.SimpleEP{}
rMeta = &model.EpCMS{}
)
if err = rows.Scan(&r.IsDeleted, &r.State, &r.Valid, &r.SeasonID, &r.EPID, &r.ID,
&r.NoMark, &rMeta.Cover, &rMeta.Title, &rMeta.Subtitle, &rMeta.PayStatus); err != nil {
log.Error("RefreshEPMC row.Scan() error(%v)", err)
return
}
rMeta.EPID = r.EPID
myLast = int(r.ID)
if err = d.SetEP(ctx, r); err != nil {
log.Warn("RefreshEPMC Auth Set EPID (%d), error (%v)", r.EPID, err)
}
if err = d.SetEpCMSCache(ctx, rMeta); err != nil {
log.Warn("RefreshEPMC Meta Set EPID (%d), error (%v)", r.EPID, err)
}
}
if err = rows.Err(); err != nil {
log.Error("d.RefreshEpMC.Query error(%v)", err)
}
return
}
// RefreshSnMC picks data by Piece to sync into MC
func (d *Dao) RefreshSnMC(ctx context.Context, LastID int, nbData int) (myLast int, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(ctx, _pickSeasonMC+fmt.Sprintf("%d", nbData), LastID); err != nil {
log.Error("d._pickSeasonMC.Query: %s error(%v)", _pickSeasonMC+fmt.Sprintf("%d", nbData), err)
return
}
defer rows.Close()
for rows.Next() {
var (
auth = &model.SimpleSeason{}
media = &model.SeasonCMS{}
)
// SELECT is_deleted, `check`,valid, id, cover, `desc`, title, upinfo, category, area, play_time, role, staff, total_num, style FROM tv_ep_season
if err = rows.Scan(&auth.IsDeleted, &auth.Check, &auth.Valid, &auth.ID, &media.Cover,
&media.Desc, &media.Title, &media.UpInfo, &media.Category, &media.Area, &media.Playtime,
&media.Role, &media.Staff, &media.TotalNum, &media.Style, &media.Alias, &media.OriginName,
&media.PayStatus); err != nil { // refresh cache sn logic
log.Error("RefreshSnMC row.Scan() error(%v)", err)
return
}
media.SeasonID = int(auth.ID) // season_id
media.NewestEPID, media.NewestOrder, _ = d.NewestOrder(ctx, auth.ID) // newest info
myLast = int(auth.ID)
if err = d.SetSeason(ctx, auth); err != nil {
log.Warn("RefreshSnMC Auth Set Sid (%d), error (%v)", auth.ID, err)
}
if err = d.SetSnCMSCache(ctx, media); err != nil {
log.Warn("RefreshSnMC Meta Set Sid (%d), error (%v)", auth.ID, err)
}
}
if err = rows.Err(); err != nil {
log.Error("d.RefreshSnMC.Query error(%v)", err)
}
return
}
// PickSeason picks one season CMS struct
func (d *Dao) PickSeason(ctx context.Context, sid int) (media *model.SeasonCMS, err error) {
media = &model.SeasonCMS{}
if err = d.DB.QueryRow(ctx, _singleSn, sid).Scan(&media.SeasonID, &media.Cover,
&media.Desc, &media.Title, &media.UpInfo, &media.Category, &media.Area, &media.Playtime,
&media.Role, &media.Staff, &media.TotalNum, &media.Style, &media.Alias, &media.OriginName,
&media.PayStatus); err != nil { // databus ep logic, with sn
log.Error("d.PickSeason Sid %d Error %v", sid, err)
if err == sql.ErrNoRows {
err = nil
media = nil
return
}
return
}
media.NewestEPID, media.NewestOrder, _ = d.NewestOrder(ctx, int64(media.SeasonID)) // newest info
return
}

130
app/job/main/tv/dao/app/sync_mc_test.go Normal file
View File

@@ -0,0 +1,130 @@
package app
import (
"context"
model "go-common/app/job/main/tv/model/pgc"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppEpCacheKey(t *testing.T) {
var (
epid = int(0)
)
convey.Convey("EpCacheKey", t, func(ctx convey.C) {
p1 := EpCacheKey(epid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestAppSeasonCacheKey(t *testing.T) {
var (
sid = int(0)
)
convey.Convey("SeasonCacheKey", t, func(ctx convey.C) {
p1 := SeasonCacheKey(sid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestAppSetEP(t *testing.T) {
var (
ctx = context.Background()
res = &model.SimpleEP{}
)
convey.Convey("SetEP", t, func(cx convey.C) {
err := d.SetEP(ctx, res)
cx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestAppSetSeason(t *testing.T) {
var (
ctx = context.Background()
res = &model.SimpleSeason{}
)
convey.Convey("SetSeason", t, func(cx convey.C) {
err := d.SetSeason(ctx, res)
cx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestAppCountEP(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("CountEP", t, func(cx convey.C) {
count, err := d.CountEP(ctx)
cx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestAppCountSeason(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("CountSeason", t, func(cx convey.C) {
count, err := d.CountSeason(ctx)
cx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestAppRefreshEPMC(t *testing.T) {
var (
ctx = context.Background()
LastID = int(0)
nbData = int(0)
)
convey.Convey("RefreshEPMC", t, func(cx convey.C) {
myLast, err := d.RefreshEPMC(ctx, LastID, nbData)
cx.Convey("Then err should be nil.myLast should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(myLast, convey.ShouldNotBeNil)
})
})
}
func TestAppRefreshSnMC(t *testing.T) {
var (
ctx = context.Background()
LastID = int(0)
nbData = int(0)
)
convey.Convey("RefreshSnMC", t, func(cx convey.C) {
myLast, err := d.RefreshSnMC(ctx, LastID, nbData)
cx.Convey("Then err should be nil.myLast should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(myLast, convey.ShouldNotBeNil)
})
})
}
func TestAppPickSeason(t *testing.T) {
var ctx = context.Background()
convey.Convey("PickSeason", t, func(cx convey.C) {
sid, errPick := pickEpSid()
if errPick != nil {
return
}
media, err := d.PickSeason(ctx, int(sid))
cx.Convey("Then err should be nil.media should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(media, convey.ShouldNotBeNil)
})
})
}

58
app/job/main/tv/dao/app/sync_retry.go Normal file
View File

@@ -0,0 +1,58 @@
package app
import (
"context"
"encoding/json"
"go-common/app/job/main/tv/model/common"
"go-common/library/cache/memcache"
"go-common/library/log"
)
// SetRetry save retry model to memcache
func (d *Dao) SetRetry(c context.Context, retry *common.SyncRetry) (err error) {
var conn = d.mc.Get(c)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: retry.MCKey(), Object: retry, Flags: memcache.FlagJSON, Expiration: d.mcMediaExpire}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}
// GetRetry gets retry times
func (d *Dao) GetRetry(c context.Context, req *common.SyncRetry) (times int, err error) {
var (
conn memcache.Conn
item *memcache.Item
)
conn = d.mc.Get(c)
defer conn.Close()
if item, err = conn.Get(req.MCKey()); err != nil {
if err == memcache.ErrNotFound {
err = nil
return // 0
}
log.Error("GetRetry Req %s, Err %v", req.MCKey(), err)
return
}
if err = json.Unmarshal(item.Value, &req); err != nil {
log.Error("GetRetry Req %s, Json Err %v", req.MCKey(), err)
return
}
times = req.Retry
return
}
// DelRetry deletes the cache from MC
func (d *Dao) DelRetry(c context.Context, req *common.SyncRetry) (err error) {
var (
key = req.MCKey()
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Delete(key); err != nil {
log.Error("conn.Set error(%v)", err)
}
return
}

49
app/job/main/tv/dao/app/sync_retry_test.go Normal file
View File

@@ -0,0 +1,49 @@
package app
import (
"context"
"go-common/app/job/main/tv/model/common"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppSetRetry(t *testing.T) {
var (
c = context.Background()
retry = &common.SyncRetry{}
)
convey.Convey("SetRetry", t, func(ctx convey.C) {
err := d.SetRetry(c, retry)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestAppGetRetry(t *testing.T) {
var (
c = context.Background()
req = &common.SyncRetry{}
)
convey.Convey("GetRetry", t, func(ctx convey.C) {
times, err := d.GetRetry(c, req)
ctx.Convey("Then err should be nil.times should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(times, convey.ShouldNotBeNil)
})
})
}
func TestAppDelRetry(t *testing.T) {
var (
c = context.Background()
req = &common.SyncRetry{}
)
convey.Convey("DelRetry", t, func(ctx convey.C) {
err := d.DelRetry(c, req)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

36
app/job/main/tv/dao/app/zone_index.go Normal file
View File

@@ -0,0 +1,36 @@
package app
import (
"context"
"go-common/app/job/main/tv/model/common"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_passedSn = "SELECT id,ctime FROM tv_ep_season WHERE `check` = ? AND valid = ? AND category = ? AND is_deleted = 0"
)
// PassedSn picks the passed seasons data to sync to search
func (d *Dao) PassedSn(c context.Context, category int) (res []*common.IdxRank, err error) {
var (
rows *sql.Rows
)
if rows, err = d.DB.Query(c, _passedSn, SeasonPassed, _CMSValid, category); err != nil {
log.Error("d.PassedSn.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &common.IdxRank{}
if err = rows.Scan(&r.ID, &r.Ctime); err != nil {
log.Error("PassedSn row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.PassedSn.Query error(%v)", err)
}
return
}

22
app/job/main/tv/dao/app/zone_index_test.go Normal file
View File

@@ -0,0 +1,22 @@
package app
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestAppPassedSn(t *testing.T) {
var (
c = context.Background()
category = int(1)
)
convey.Convey("PassedSn", t, func(ctx convey.C) {
res, err := d.PassedSn(c, category)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}

53
app/job/main/tv/dao/archive/BUILD Normal file
View File

@@ -0,0 +1,53 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"archive.go",
"dao.go",
],
importpath = "go-common/app/job/main/tv/dao/archive",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/service/main/archive/api:go_default_library",
"//library/cache/memcache:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"archive_test.go",
"dao_test.go",
],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/service/main/archive/api:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

46
app/job/main/tv/dao/archive/archive.go Normal file
View File

@@ -0,0 +1,46 @@
package archive
import (
"context"
"strconv"
arccli "go-common/app/service/main/archive/api"
"go-common/library/cache/memcache"
"github.com/pkg/errors"
)
const (
_prefixArc = "a3p_"
_prefixView = "avp_"
)
func keyArc(aid int64) string {
return _prefixArc + strconv.FormatInt(aid, 10)
}
func keyView(aid int64) string {
return _prefixView + strconv.FormatInt(aid, 10)
}
// UpArcCache update archive cache
func (d *Dao) UpArcCache(c context.Context, a *arccli.Arc) (err error) {
conn := d.mc.Get(c)
item := &memcache.Item{Key: keyArc(a.Aid), Object: a, Flags: memcache.FlagJSON, Expiration: 0}
if err = conn.Set(item); err != nil {
err = errors.Wrapf(err, "conn.Set(%v)", item)
}
conn.Close()
return
}
// UpViewCache up all app cache .
func (d *Dao) UpViewCache(c context.Context, v *arccli.ViewReply) (err error) {
conn := d.mc.Get(c)
item := &memcache.Item{Key: keyView(v.Arc.Aid), Object: v, Flags: memcache.FlagJSON, Expiration: 0}
if err = conn.Set(item); err != nil {
err = errors.Wrapf(err, "conn.Set(%v)", item)
}
conn.Close()
return
}

64
app/job/main/tv/dao/archive/archive_test.go Normal file
View File

@@ -0,0 +1,64 @@
package archive
import (
"context"
"testing"
arccli "go-common/app/service/main/archive/api"
"github.com/smartystreets/goconvey/convey"
)
func TestArchivekeyArc(t *testing.T) {
var (
aid = int64(0)
)
convey.Convey("keyArc", t, func(ctx convey.C) {
p1 := keyArc(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchivekeyView(t *testing.T) {
var (
aid = int64(0)
)
convey.Convey("keyView", t, func(ctx convey.C) {
p1 := keyView(aid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestArchiveUpArcCache(t *testing.T) {
var (
c = context.Background()
a = &arccli.Arc{}
)
convey.Convey("UpArcCache", t, func(ctx convey.C) {
err := d.UpArcCache(c, a)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestArchiveUpViewCache(t *testing.T) {
var (
c = context.Background()
v = &arccli.ViewReply{
Arc: &arccli.Arc{
Aid: 123,
},
}
)
convey.Convey("UpViewCache", t, func(ctx convey.C) {
err := d.UpViewCache(c, v)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

25
app/job/main/tv/dao/archive/dao.go Normal file
View File

@@ -0,0 +1,25 @@
package archive
import (
"time"
"go-common/app/job/main/tv/conf"
"go-common/library/cache/memcache"
)
// Dao is archive dao.
type Dao struct {
// memcache
mc *memcache.Pool
expireView int32
}
// New new a archive dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
// memcache
mc: memcache.NewPool(c.Memcache.Config),
expireView: int32(time.Duration(c.Memcache.ExpireMedia) / time.Second),
}
return
}

34
app/job/main/tv/dao/archive/dao_test.go Normal file
View File

@@ -0,0 +1,34 @@
package archive
import (
"flag"
"go-common/app/job/main/tv/conf"
"os"
"testing"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.web-svr.tv-job")
flag.Set("conf_token", "ab3e9801a77c076b997de0ac5cb21775")
flag.Set("tree_id", "15260")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../../cmd/tv-job-test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
os.Exit(m.Run())
}

58
app/job/main/tv/dao/cms/BUILD Normal file
View File

@@ -0,0 +1,58 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"merak_test.go",
"shelve_test.go",
],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//library/database/sql:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
"//vendor/gopkg.in/h2non/gock.v1:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"merak.go",
"shelve.go",
],
importpath = "go-common/app/job/main/tv/dao/cms",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/xstr:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

24
app/job/main/tv/dao/cms/dao.go Normal file
View File

@@ -0,0 +1,24 @@
package cms
import (
"go-common/app/job/main/tv/conf"
"go-common/library/database/sql"
httpx "go-common/library/net/http/blademaster"
)
// Dao dao.
type Dao struct {
conf *conf.Config
DB *sql.DB
client *httpx.Client
}
// New create a instance of Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
DB: sql.NewMySQL(c.Mysql),
client: httpx.NewClient(conf.Conf.HTTPClient),
}
return
}

37
app/job/main/tv/dao/cms/dao_test.go Normal file
View File

@@ -0,0 +1,37 @@
package cms
import (
"flag"
"os"
"testing"
"go-common/app/job/main/tv/conf"
"gopkg.in/h2non/gock.v1"
)
var d *Dao
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.web-svr.tv-job")
flag.Set("conf_token", "ab3e9801a77c076b997de0ac5cb21775")
flag.Set("tree_id", "15260")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../../cmd/tv-job-test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
d.client.SetTransport(gock.DefaultTransport)
m.Run()
os.Exit(0)
}

84
app/job/main/tv/dao/cms/merak.go Normal file
View File

@@ -0,0 +1,84 @@
package cms
import (
"bytes"
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"net/http"
"sort"
"strings"
"go-common/library/log"
"github.com/pkg/errors"
)
// MerakNotify send notify
func (d *Dao) MerakNotify(ctx context.Context, title, content string) (err error) {
var (
cfg = d.conf.Cfg.Merak
sign string
req *http.Request
body []byte
)
params := map[string]string{
"Action": "CreateWechatMessage",
"PublicKey": cfg.Key,
"UserName": strings.Join(cfg.Names, ","),
"Title": title,
"Content": content,
"TreeId": "",
}
if sign, err = MerakSign(params, cfg.Secret); err != nil {
log.Error("MerakNotify Failed to sign params: %+v: %+v", params, err)
return err
}
params["Signature"] = sign
if body, err = json.Marshal(params); err != nil {
log.Error("MerakNotify Json %v, Err %v", params, err)
return
}
if req, err = http.NewRequest(http.MethodPost, cfg.Host, bytes.NewReader(body)); err != nil {
log.Error("MerakNotify NewRequest Err %v, Host %v", err, cfg.Host)
return
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
res := struct {
Action string `json:"Action"`
RetCode int `json:"RetCode"`
Data []string `json:"Data"`
}{}
if err = d.client.Do(ctx, req, &res); err != nil {
return
}
if res.RetCode != 0 {
err = errors.Errorf("Merak error: %d", res.RetCode)
log.Error("Failed to send notify by merak with params: %+v: %+v", string(body), err)
return
}
return
}
// MerakSign is used to sign for merak wechat msg
func MerakSign(params map[string]string, secret string) (string, error) {
keys := make([]string, 0, len(params))
for k := range params {
keys = append(keys, k)
}
sort.Strings(keys)
buf := bytes.Buffer{}
for _, k := range keys {
buf.WriteString(k + params[k])
}
h := sha1.New()
if _, err := h.Write(buf.Bytes()); err != nil {
return "", errors.WithStack(err)
}
if _, err := h.Write([]byte(secret)); err != nil {
return "", errors.WithStack(err)
}
sum := h.Sum(nil)
return hex.EncodeToString(sum), nil
}

50
app/job/main/tv/dao/cms/merak_test.go Normal file
View File

@@ -0,0 +1,50 @@
package cms
import (
"context"
"fmt"
"strings"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoSendWechat(t *testing.T) {
convey.Convey("SendWechat", t, func(ctx convey.C) {
var (
content = "测试内容"
title = "测试标题"
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.MerakNotify(context.Background(), content, title)
fmt.Println(err)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoGetMerakSign(t *testing.T) {
convey.Convey("sign", t, func(ctx convey.C) {
var (
params = map[string]string{
"Action": "CreateWechatMessage",
"PublicKey": "1",
"UserName": strings.Join([]string{"user1", "user2"}, ","),
"Title": "测试标题",
"Content": "测试内容",
"TreeId": "",
}
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
p1, err := MerakSign(params, "secret")
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldEqual, "59cd4e74b225a7d326ee7d6c89bf27cf2f6015dc")
})
})
})
}

117
app/job/main/tv/dao/cms/shelve.go Normal file
View File

@@ -0,0 +1,117 @@
package cms
import (
"context"
"fmt"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_validSns = "SELECT DISTINCT a.id FROM tv_ep_season a LEFT JOIN tv_content b ON a.id = b.season_id " +
"WHERE a.is_deleted = 0 AND a.`check` = ? AND b.is_deleted = 0 AND b.valid = ? AND b.state = ?"
_allPassedSns = "SELECT id, valid FROM tv_ep_season WHERE is_deleted = 0 AND `check` = 1"
_actSns = "UPDATE tv_ep_season SET valid = ? WHERE id IN (%s)"
_offArcs = "SELECT aid FROM ugc_archive WHERE aid IN (%s) AND valid = 0 AND deleted = 0 AND result = 1 "
_reshelfArcs = "UPDATE ugc_archive SET valid = 1 WHERE aid IN (%s)"
_cmsOnline = 1
_cmsOffline = 0
_epPassed = 3
)
// ValidSns gets all the seasons that should be on the shelves, which includes free and audited episodes.
func (d *Dao) ValidSns(ctx context.Context, onlyfree bool) (res map[int64]int, err error) {
var (
rows *sql.Rows
validSql = _validSns
)
res = make(map[int64]int)
if onlyfree {
validSql = validSql + " AND b.pay_status = 2" // free episode
}
if rows, err = d.DB.Query(ctx, validSql, _cmsOnline, _cmsOnline, _epPassed); err != nil {
log.Error("d.ValidSns.Query: %s error(%v)", validSql, err)
return
}
defer rows.Close()
for rows.Next() {
var sid int64
if err = rows.Scan(&sid); err != nil {
log.Error("ValidSns row.Scan() error(%v)", err)
return
}
res[sid] = 1
}
if err = rows.Err(); err != nil {
log.Error("d.PgcCont.Query error(%v)", err)
}
return
}
// ShelveOp gets the status of all audited seasons on and off shelves, and compare the results with the "ValidSns" method above to determine which episodes need to be on or off shelves.
func (d *Dao) ShelveOp(ctx context.Context, validSns map[int64]int) (onIDs, offIDs []int64, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(ctx, _allPassedSns); err != nil {
log.Error("d.ShelveOp.Query: %s error(%v)", _allPassedSns, err)
return
}
defer rows.Close()
for rows.Next() {
var sid, valid int64
if err = rows.Scan(&sid, &valid); err != nil {
log.Error("ValidSns row.Scan() error(%v)", err)
return
}
_, ok := validSns[sid]
if ok && valid == 0 {
onIDs = append(onIDs, sid)
}
if !ok && valid == 1 {
offIDs = append(offIDs, sid)
}
}
if err = rows.Err(); err != nil {
log.Error("d.PgcCont.Query error(%v)", err)
}
return
}
// ActOps carries out the action on the season which need to be on/off shelves
func (d *Dao) ActOps(ctx context.Context, ids []int64, on bool) (err error) {
var action int
if on {
action = _cmsOnline
} else {
action = _cmsOffline
}
if _, err = d.DB.Exec(ctx, fmt.Sprintf(_actSns, xstr.JoinInts(ids)), action); err != nil {
log.Error("ActOps, Ids %v, Err %v", ids, err)
}
return
}
// OffArcs takes the archives that passed but cms invalid archives
func (d *Dao) OffArcs(ctx context.Context, aids []int64) (offAids []int64, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(ctx, fmt.Sprintf(_offArcs, xstr.JoinInts(aids))); err != nil {
return
}
defer rows.Close()
for rows.Next() {
var aid int64
if err = rows.Scan(&aid); err != nil {
return
}
offAids = append(offAids, aid)
}
err = rows.Err()
return
}
// ReshelfArcs re-put the arcs onshelf ( CMS valid )
func (d *Dao) ReshelfArcs(ctx context.Context, aids []int64) (err error) {
_, err = d.DB.Exec(ctx, fmt.Sprintf(_reshelfArcs, xstr.JoinInts(aids)))
return
}

117
app/job/main/tv/dao/cms/shelve_test.go Normal file
View File

@@ -0,0 +1,117 @@
package cms
import (
"context"
"fmt"
"testing"
"go-common/library/database/sql"
"github.com/smartystreets/goconvey/convey"
)
func TestCmsValidSns(t *testing.T) {
var (
ctx = context.Background()
)
convey.Convey("ValidSns", t, func(cx convey.C) {
res, err := d.ValidSns(ctx, false)
cx.Convey("Consider Audited, Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
fmt.Println(len(res))
})
res, err = d.ValidSns(ctx, true)
cx.Convey("Consider Audited and Free, Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
fmt.Println(len(res))
})
})
}
func TestCmsShelveOp(t *testing.T) {
var (
ctx = context.Background()
validSns, _ = d.ValidSns(ctx, true)
)
convey.Convey("ShelveOp", t, func(cx convey.C) {
onIDs, offIDs, err := d.ShelveOp(ctx, validSns)
cx.Convey("Then err should be nil.onIDs,offIDs should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(offIDs, convey.ShouldNotBeNil)
ctx.So(onIDs, convey.ShouldNotBeNil)
fmt.Println(offIDs)
fmt.Println(onIDs)
})
})
}
func TestCmsActOps(t *testing.T) {
var (
ctx = context.Background()
sid int64
)
d.DB.QueryRow(ctx, "SELECT id FROM tv_ep_season WHERE valid = 1 and is_deleted = 0 AND `check` = 1 LIMIT 1").Scan(&sid)
convey.Convey("ActOps", t, func(cx convey.C) {
err := d.ActOps(ctx, []int64{sid}, false)
cx.Convey("Action 0 Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
err = d.ActOps(ctx, []int64{sid}, true)
cx.Convey("Action 1 Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
fmt.Println(sid)
})
}
func TestCmsOffArcs(t *testing.T) {
var (
ctx = context.Background()
aid int64
)
convey.Convey("OffArcs", t, func(cx convey.C) {
cx.Convey("Then err should be nil.offAids should not be nil.", func(cx convey.C) {
if err := d.DB.QueryRow(ctx, "select aid from ugc_archive where deleted = 0 and valid = 0 and result = 1 limit 1").Scan(&aid); err != nil {
offAids, err := d.OffArcs(context.Background(), []int64{1, 2, 3})
cx.So(err, convey.ShouldBeNil)
cx.So(offAids, convey.ShouldBeNil)
} else {
fmt.Println("Have Aid ", aid)
offAids, err := d.OffArcs(context.Background(), []int64{1, 2, 3, aid})
cx.So(err, convey.ShouldBeNil)
cx.So(offAids, convey.ShouldNotBeNil)
}
})
cx.Convey("Arg Error", func(cx convey.C) {
_, err := d.OffArcs(context.Background(), []int64{})
cx.So(err, convey.ShouldNotBeNil)
})
cx.Convey("DB Error", func(cx convey.C) {
d.DB.Close()
_, err := d.OffArcs(context.Background(), []int64{1, 2, 3})
cx.So(err, convey.ShouldNotBeNil)
d.DB = sql.NewMySQL(d.conf.Mysql)
})
})
}
func TestCmsReshelfArcs(t *testing.T) {
convey.Convey("OffArcs", t, func(cx convey.C) {
cx.Convey("Then err should be nil.offAids should not be nil.", func(cx convey.C) {
err := d.ReshelfArcs(context.Background(), []int64{1, 2, 3})
cx.So(err, convey.ShouldBeNil)
})
cx.Convey("Arg Error", func(cx convey.C) {
err := d.ReshelfArcs(context.Background(), []int64{})
cx.So(err, convey.ShouldNotBeNil)
})
cx.Convey("DB Error", func(cx convey.C) {
d.DB.Close()
err := d.ReshelfArcs(context.Background(), []int64{1, 2, 3})
cx.So(err, convey.ShouldNotBeNil)
d.DB = sql.NewMySQL(d.conf.Mysql)
})
})
}

51
app/job/main/tv/dao/ftp/BUILD Normal file
View File

@@ -0,0 +1,51 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"ftp.go",
],
importpath = "go-common/app/job/main/tv/dao/ftp",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/ftp-master:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"ftp_test.go",
],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

18
app/job/main/tv/dao/ftp/dao.go Normal file
View File

@@ -0,0 +1,18 @@
package ftp
import (
"go-common/app/job/main/tv/conf"
)
// Dao dao.
type Dao struct {
conf *conf.Config
}
// New create a instance of Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
}
return
}

35
app/job/main/tv/dao/ftp/dao_test.go Normal file
View File

@@ -0,0 +1,35 @@
package ftp
import (
"flag"
"os"
"testing"
"go-common/app/job/main/tv/conf"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.web-svr.tv-job")
flag.Set("conf_token", "ab3e9801a77c076b997de0ac5cb21775")
flag.Set("tree_id", "15260")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../../cmd/tv-job-test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
os.Exit(m.Run())
}

112
app/job/main/tv/dao/ftp/ftp.go Normal file
View File

@@ -0,0 +1,112 @@
package ftp
import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
"time"
"go-common/library/log"
"github.com/ftp-master"
)
const (
_ftpRetry = 3
errFormat = "Func:[%s] - Step:[%s] - Error:[%v]"
_sleep = 100 * time.Millisecond
)
// Retry . retry one function until no error
func Retry(callback func() error, retry int, sleep time.Duration) (err error) {
for i := 0; i < retry; i++ {
if err = callback(); err == nil {
return
}
time.Sleep(sleep)
}
return
}
// FileMd5 calculates the local file's md5 and store it in a file
func (d *Dao) FileMd5(path string, md5Path string) (err error) {
var (
content []byte
)
if content, err = ioutil.ReadFile(path); err != nil {
log.Error(errFormat+" FilePath: %s", "fileMd5", "ReadFile", err, path)
return
}
md5hash := md5.New()
if _, err = io.Copy(md5hash, bytes.NewReader(content)); err != nil {
log.Error(errFormat, "fileMd5", "CopyContent", err)
return
}
md5 := md5hash.Sum(nil)
fMd5 := hex.EncodeToString(md5[:])
file, error := os.OpenFile(md5Path, os.O_RDWR|os.O_CREATE, 0766)
if error != nil {
log.Error(errFormat, "fileMd5", "OpenFile", err)
return
}
file.WriteString(fMd5)
file.Close()
return
}
// UploadFile the file to remote frp server and update the md5 file
func (d *Dao) UploadFile(localPath string, remotePath string, url string) (err error) {
var (
ftpInfo = d.conf.Search.FTP
c *ftp.ServerConn
content []byte // file's content
fileSize int64
)
// Dial
if c, err = ftp.DialTimeout(ftpInfo.Host, time.Duration(ftpInfo.Timeout)); err != nil {
log.Error(errFormat, "uploadFile", "DialTimeout", err)
return
}
// use EPSV or not
if !ftpInfo.UseEPSV {
c.DisableEPSV = true
}
// Login
if err = c.Login(ftpInfo.User, ftpInfo.Pass); err != nil {
log.Error(errFormat, "uploadFile", "Login", err)
return
}
// Change dir
if err = c.ChangeDir(url); err != nil {
log.Error(errFormat, "uploadFile", "ChangeDir", err)
return
}
// Upload the file
if content, err = ioutil.ReadFile(localPath); err != nil {
log.Error(errFormat, "uploadFile", "ReadFile", err)
return
}
data := bytes.NewBuffer(content)
if err = Retry(func() (err error) {
return c.Stor(remotePath, data)
}, _ftpRetry, _sleep); err != nil {
log.Error("upArchives Error %+v", err)
return
}
// Calculate the file size to check it's ok
if fileSize, err = c.FileSize(remotePath); err != nil {
log.Error(errFormat, "uploadFile", "FileSize", err)
return
}
if localSize := int64(len(content)); localSize != fileSize {
err = fmt.Errorf("LocalSize is %d, RemoteSize is %d", localSize, fileSize)
log.Error(errFormat, "uploadFile", "FileSize", err)
return
}
log.Info("File %s is uploaded successfully, size: %d", remotePath, fileSize)
return
}

78
app/job/main/tv/dao/ftp/ftp_test.go Normal file
View File

@@ -0,0 +1,78 @@
package ftp
import (
"fmt"
"os"
"testing"
"time"
"github.com/smartystreets/goconvey/convey"
)
func fileExist(path string) bool {
_, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false
}
}
return true
}
func createFile(path string) {
if !fileExist(path) {
// If the file doesn't exist, create it, or append to the file
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println(err)
}
_, err = f.Write([]byte("Hello"))
if err != nil {
fmt.Println(err)
}
f.Close()
}
}
func TestFtpRetry(t *testing.T) {
var (
callback func() error
retry = int(0)
sleep time.Duration
)
convey.Convey("Retry", t, func(ctx convey.C) {
err := Retry(callback, retry, sleep)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestFtpFileMd5(t *testing.T) {
var (
path = "/tmp/testMd5.source"
md5Path = "/tmp/testMd5.target"
)
convey.Convey("FileMd5", t, func(ctx convey.C) {
createFile(path)
err := d.FileMd5(path, md5Path)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestFtpUploadFile(t *testing.T) {
var (
cfg = d.conf.Search
path = "/tmp/testMd5.source"
)
convey.Convey("UploadFile", t, func(ctx convey.C) {
createFile(path)
err := d.UploadFile(path, "testMd5.remote", cfg.FTP.RemotePgcURL)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

58
app/job/main/tv/dao/lic/BUILD Normal file
View File

@@ -0,0 +1,58 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"call.go",
"dao.go",
"lic.go",
"xml.go",
],
importpath = "go-common/app/job/main/tv/dao/lic",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/pgc:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"lic_test.go",
"xml_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/pgc:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

63
app/job/main/tv/dao/lic/call.go Normal file
View File

@@ -0,0 +1,63 @@
package lic
import (
"bytes"
"context"
"encoding/xml"
"fmt"
"net/http"
"time"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/log"
"github.com/pkg/errors"
)
const (
_normalCode = "0000"
)
func (d *Dao) callLic(c context.Context, url string, xmlBody string) (result *model.Document, err error) {
var resp []byte
result = &model.Document{}
req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer([]byte(xmlBody)))
if err != nil {
log.Error("http.NewRequest err - %v", err)
return
}
req.Header.Add("Content-Type", "text/xml; charset=utf-8")
if resp, err = d.client.Raw(c, req); err != nil {
log.Error("ClientGet error[%v]", err)
return
}
if err = xml.Unmarshal(resp, result); err != nil {
log.Error("XML Unmarshal %s, Error %v", string(resp), err)
return
}
if result == nil || result.Response == nil {
err = errors.Wrap(err, "Response Empty Error")
return
}
if result.Response.ResponseCode != _normalCode {
err = fmt.Errorf("Response Code Error %s", result.Response.ResponseCode)
return
}
if result.Response.ErrorList != nil && result.Response.ErrorList.Error != nil {
log.Warn("Response Error %v", result.Response.ErrorList.Error)
}
return
}
// CallRetry retries the xml call
func (d *Dao) CallRetry(c context.Context, url string, xmlBody string) (result *model.Document, err error) {
log.Info("callLic URL: %s, Body %s", url, xmlBody)
for i := 0; i < 3; i++ {
result, err = d.callLic(c, url, xmlBody)
if err == nil {
break
}
time.Sleep(10 * time.Second) // 5 seconds gap for each retrial
}
return
}

21
app/job/main/tv/dao/lic/dao.go Normal file
View File

@@ -0,0 +1,21 @@
package lic
import (
"go-common/app/job/main/tv/conf"
httpx "go-common/library/net/http/blademaster"
)
// Dao dao.
type Dao struct {
conf *conf.Config
client *httpx.Client
}
// New create a instance of Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
client: httpx.NewClient(conf.Conf.HTTPClient),
}
return
}

56
app/job/main/tv/dao/lic/dao_test.go Normal file
View File

@@ -0,0 +1,56 @@
package lic
import (
"context"
"flag"
"path/filepath"
"testing"
"go-common/app/job/main/tv/conf"
"fmt"
. "github.com/smartystreets/goconvey/convey"
)
var (
ctx = context.Background()
d *Dao
xmlBody = `inputTime=20180606&sign=timer-import_BILIBILI&tId=UHZFmufgweRpWhqAzToFYMWtYuZhMKCU&xmlData=<?xmlversion="1.0"encoding="UTF-8"?><Serviceid="dataSync"><Head><TradeId>UHZFmufgweRpWhqAzToFYMWtYuZhMKCU</TradeId><Date>2018-06-06</Date><Count>1</Count></Head><Body><programSetList><programSet><programSetId>ugc10100044</programSetId><programSetName>drm</programSetName><programSetClass></programSetClass><programSetType></programSetType><programSetPoster>http://i1.hdslb.com/bfs/archive/diuren.png</programSetPoster><publishDate>2018-01-30</publishDate><copyright>bilibili</copyright><programCount>1</programCount><cREndDate>1970-01-01</cREndDate><definitionType>SD</definitionType><cpCode>BILIBILI</cpCode><payStatus>0</payStatus><primitiveName></primitiveName><alias></alias><zone></zone><leadingRole></leadingRole><programSetDesc>drm</programSetDesc><Staff></Staff><programList><program><programId>ugc10114149</programId><programName>1</programName><programPoster></programPoster><programLength>1448</programLength><publishDate>1970-01-01</publishDate><ifPreview>0</ifPreview><number>1</number><definitionType>SD</definitionType><playCount>0</playCount><drm>0</drm><programMediaList><programMedia><mediaId>ugc10114149</mediaId><playUrl>http://upos-hz-tvshenhe.acgvideo.com/upgcxcode/87/75/41057587/41057587-1-6.mp4</playUrl><definition>SD</definition><htmlUrl>http://upos-hz-tvshenhe.acgvideo.com/upgcxcode/87/75/41057587/41057587-1-6.mp4</htmlUrl></programMedia></programMediaList></program></programList></programSet></programSetList></Body></Service>`
)
func WithDao(f func(d *Dao)) func() {
return func() {
dir, _ := filepath.Abs("../../cmd/tv-job-test.toml")
flag.Set("conf", dir)
conf.Init()
if d == nil {
d = New(conf.Conf)
}
f(d)
}
}
func TestDelEpLic(t *testing.T) {
Convey("TestDao_CallRetry", t, WithDao(func(d *Dao) {
res := DelEpLic("ugc", "timer-import_BILIBILI", []int{10109083, 10109084})
So(len(res), ShouldBeGreaterThan, 0)
fmt.Println(res)
}))
}
func TestDao_CallRetry(t *testing.T) {
Convey("TestDao_CallRetry", t, WithDao(func(d *Dao) {
res, err := d.CallRetry(ctx, d.conf.Sync.API.AddURL, xmlBody)
So(err, ShouldBeNil)
So(res, ShouldNotBeNil)
}))
}
func TestDao_CallLic(t *testing.T) {
Convey("TestDao_CallLic", t, WithDao(func(d *Dao) {
result, err := d.callLic(ctx, d.conf.Sync.API.AddURL, xmlBody)
So(err, ShouldBeNil)
So(result, ShouldNotBeNil)
}))
}

101
app/job/main/tv/dao/lic/lic.go Normal file
View File

@@ -0,0 +1,101 @@
package lic
import (
"encoding/xml"
"fmt"
model "go-common/app/job/main/tv/model/pgc"
"math/rand"
"net/url"
"time"
)
const (
letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-"
_serviceID = "dataSync"
)
// BuildLic builds the skeleton of a license
func BuildLic(sign string, ps []*model.PS, count int) *model.License {
var (
tid = RandStringBytesRmndr(32)
now = time.Now()
)
return &model.License{
TId: tid,
InputTime: now.Format("20060102"),
Sign: sign,
XMLData: &model.XMLData{
Service: &model.Service{
ID: _serviceID,
Head: &model.Head{
TradeID: tid,
Date: now.Format("2006-01-02"),
Count: count,
},
Body: &model.Body{
ProgramSetList: &model.PSList{
ProgramSet: ps,
},
},
},
},
}
}
// RandStringBytesRmndr generates an random string
func RandStringBytesRmndr(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = letterBytes[rand.Int63()%int64(len(letterBytes))]
}
return string(b)
}
// DelLic creates the license message with only the Season ID, for deletion
func DelLic(sign string, prefix string, sid int64) *model.License {
var (
ps []*model.PS
programS = &model.PS{
ProgramSetID: fmt.Sprintf("%s%d", prefix, sid),
}
)
ps = append(ps, programS)
return BuildLic(sign, ps, 1)
}
// DelEpLic creates the license message with only the Ep IDs, for deletion
func DelEpLic(prefix string, sign string, delEps []int) string {
// message skeleton
var tid = RandStringBytesRmndr(32)
type Service struct {
ID string `xml:"id,attr"`
Head *model.Head
Body *model.DelBody `xml:"Body"`
}
Msg := &Service{
ID: _serviceID,
Head: &model.Head{
TradeID: tid,
Date: time.Now().Format("2006-01-02"),
Count: len(delEps),
},
Body: &model.DelBody{
ProgramList: &model.ProgramList{},
},
}
for _, v := range delEps {
pm := &model.Program{
ProgramID: fmt.Sprintf("%s%d", prefix, v),
}
Msg.Body.ProgramList.Program = append(Msg.Body.ProgramList.Program, pm)
}
// combine the xml message
xmlRes, _ := xml.MarshalIndent(Msg, " ", " ")
params := url.Values{}
params.Set("tId", tid)
params.Set("inputTime", time.Now().Format("20060102"))
params.Set("sign", sign)
body := params.Encode()
body = body + "&xmlData=<?xml version=\"1.0\" encoding=\"UTF-8\"?> " + string(xmlRes)
return body
}

62
app/job/main/tv/dao/lic/lic_test.go Normal file
View File

@@ -0,0 +1,62 @@
package lic
import (
model "go-common/app/job/main/tv/model/pgc"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestLicBuildLic(t *testing.T) {
var (
sign = ""
ps = []*model.PS{}
count = int(0)
)
convey.Convey("BuildLic", t, func(ctx convey.C) {
p1 := BuildLic(sign, ps, count)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestLicRandStringBytesRmndr(t *testing.T) {
var (
n = int(0)
)
convey.Convey("RandStringBytesRmndr", t, func(ctx convey.C) {
p1 := RandStringBytesRmndr(n)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestLicDelLic(t *testing.T) {
var (
sign = ""
prefix = ""
sid = int64(0)
)
convey.Convey("DelLic", t, func(ctx convey.C) {
p1 := DelLic(sign, prefix, sid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestLicDelEpLic(t *testing.T) {
var (
prefix = ""
sign = ""
delEps = []int{}
)
convey.Convey("DelEpLic", t, func(ctx convey.C) {
p1 := DelEpLic(prefix, sign, delEps)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}

20
app/job/main/tv/dao/lic/xml.go Normal file
View File

@@ -0,0 +1,20 @@
package lic
import (
"encoding/xml"
"net/url"
model "go-common/app/job/main/tv/model/pgc"
)
// PrepareXML combine the xml data to sync
func PrepareXML(v *model.License) (body string) {
xmlRes, _ := xml.MarshalIndent(v.XMLData.Service, " ", " ")
params := url.Values{}
params.Set("tId", v.TId)
params.Set("inputTime", v.InputTime)
params.Set("sign", v.Sign)
body = params.Encode()
body = body + "&xmlData=<?xml version=\"1.0\" encoding=\"UTF-8\"?> " + string(xmlRes)
return body
}

22
app/job/main/tv/dao/lic/xml_test.go Normal file
View File

@@ -0,0 +1,22 @@
package lic
import (
"encoding/json"
"testing"
model "go-common/app/job/main/tv/model/pgc"
"github.com/smartystreets/goconvey/convey"
)
func TestLicPrepareXML(t *testing.T) {
licStr := `{"TId":"cd-IqonHtgOndMAywoPLezvXwYmUxcHJ","InputTime":"20180904","Sign":"timer-import_BILIBILI","XMLData":{"Service":{"ID":"dataSync","Head":{"TradeID":"cd-IqonHtgOndMAywoPLezvXwYmUxcHJ","Date":"2018-09-04","Count":0},"Body":{"ProgramSetList":{"ProgramSet":[{"ProgramSetID":"xds296","ProgramSetName":"我家浴缸的二三事-Test","ProgramSetClass":"基腐,日常,泡面,治愈","ProgramSetType":"电影","ProgramSetPoster":"http://i0.hdslb.com/bfs/bangumi/23fbb5ece1d3adb8700988c02e8e97f30bbfbf33.jpg","Portrait":"","Producer":"","PublishDate":"2014-10-06","Copyright":"bilibili","ProgramCount":13,"CREndData":"1970-01-01","DefinitionType":"SD","CpCode":"BILIBILI","PayStatus":0,"PrimitiveName":"オレん家のフロ事情","Alias":"我家浴室的二三事,我家浴室的现况,オレん家のフロ事情","Zone":"日本","LeadingRole":"若狭：梅原裕一郎\n龙己：岛崎信长\n鹰巢：铃木达央\n真木：津田健次郎\n三国：花江夏树\n霞：木户衣吹\n阿比留：川原庆久","ProgramSetDesc":"一个人无忧无虑独自生活的男子高中生龙己，某日救下了一位倒在河边的美青年，没想到这位青年竟然是人鱼！于是这位人鱼先生似乎很中意龙己的浴缸，变住了下来！基友和卖萌人鱼同居的故事开始上演！","Staff":"原作：いときち\n监督：青井小夜\n演出：青井小夜\n脚本：绫奈由仁子\n系列构成：绫奈由仁子\n角色设计：羽田浩二\n色彩设计：小鹿绘里\n摄影监督：藤坂めぐみ\n美术监督：永吉幸树\n音响监督：小泉纪介\n编辑：斋藤朱里\n动画制作：旭PRODUCTION","ProgramList":{"Program":null}}]}}}}}`
license := &model.License{}
json.Unmarshal([]byte(licStr), &license)
convey.Convey("PrepareXML", t, func(ctx convey.C) {
body := PrepareXML(license)
ctx.Convey("Then body should not be nil.", func(ctx convey.C) {
ctx.So(body, convey.ShouldNotBeNil)
})
})
}

53
app/job/main/tv/dao/playurl/BUILD Normal file
View File

@@ -0,0 +1,53 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"playurl_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"playurl.go",
],
importpath = "go-common/app/job/main/tv/dao/playurl",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/pgc:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

21
app/job/main/tv/dao/playurl/dao.go Normal file
View File

@@ -0,0 +1,21 @@
package playurl
import (
"go-common/app/job/main/tv/conf"
httpx "go-common/library/net/http/blademaster"
)
// Dao dao.
type Dao struct {
conf *conf.Config
client *httpx.Client
}
// New create a instance of Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
client: httpx.NewClient(conf.Conf.HTTPClient),
}
return
}

25
app/job/main/tv/dao/playurl/dao_test.go Normal file
View File

@@ -0,0 +1,25 @@
package playurl
import (
"context"
"flag"
"go-common/app/job/main/tv/conf"
"path/filepath"
)
var (
ctx = context.TODO()
d *Dao
)
func WithDao(f func(d *Dao)) func() {
return func() {
dir, _ := filepath.Abs("../../cmd/tv-job-test.toml")
flag.Set("conf", dir)
conf.Init()
if d == nil {
d = New(conf.Conf)
}
f(d)
}
}

69
app/job/main/tv/dao/playurl/playurl.go Normal file
View File

@@ -0,0 +1,69 @@
package playurl
import (
"context"
"fmt"
"net/url"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/log"
)
const (
_type = "mp4"
_maxBackup = 0
_otype = "json"
)
// Playurl calls the api of playurl to get the url to play the video
func (d *Dao) Playurl(ctx context.Context, cid int) (playurl string, hitDead bool, err error) {
var (
result = model.PlayurlResp{}
params = url.Values{}
api = d.conf.Sync.PlayURL.API
originURL string
)
params.Set("cid", fmt.Sprintf("%d", cid))
params.Set("type", _type) // to get one piece
params.Set("max_backup", fmt.Sprintf("%d", _maxBackup)) // no backup url needed
params.Set("otype", _otype) // json format response
params.Set("qn", d.conf.Sync.PlayURL.Qn) // quality fix to 16
if err = d.client.Get(ctx, api, "", params, &result); err != nil {
log.Error("ClientGet error[%v]", err)
return
}
if result.Code != 0 { // logic error
for _, v := range d.conf.Sync.PlayURL.Deadcodes {
if result.Code == v { // hit dead code
hitDead = true
return
}
}
err = fmt.Errorf("Resp Code:[%v], Message:[%v]", result.Code, result.Message)
return
}
if len(result.Durl) < 1 { // result empty
err = fmt.Errorf("Playurl Result is Empty! Resp (%v)", result)
return
}
originURL = result.Durl[0].URL
if playurl, err = d.hostChange(originURL); err != nil { // replace the host of the playurl
log.Error("HostChange Origin: %s, Error: %v", originURL, err)
}
return
}
// replace the url's host by TV's dedicated host
func (d *Dao) hostChange(playurl string) (replaced string, err error) {
var host = d.conf.Sync.PlayURL.PlayPath
u, err := url.Parse(playurl)
if err != nil {
log.Error("hostChange ParseURL error (%v)", err)
return
}
log.Info("[hostChange] for URL: %s, Original Host: %s, Now we change it to: %s", playurl, u.Host, host)
u.Host = host // replace the host
u.RawQuery = "" // remove useless query
replaced = u.String()
return
}

16
app/job/main/tv/dao/playurl/playurl_test.go Normal file
View File

@@ -0,0 +1,16 @@
package playurl
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_Playurl(t *testing.T) {
Convey("TestDao_Playurl", t, WithDao(func(d *Dao) {
url, hitDead, err := d.Playurl(ctx, 41057587)
So(err, ShouldBeNil)
So(hitDead, ShouldBeTrue)
So(url, ShouldNotBeEmpty)
}))
}

62
app/job/main/tv/dao/report/BUILD Normal file
View File

@@ -0,0 +1,62 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"mc_test.go",
"report_test.go",
"style_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"mc.go",
"report.go",
"style.go",
],
importpath = "go-common/app/job/main/tv/dao/report",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/pgc:go_default_library",
"//app/job/main/tv/model/report:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/conf/env:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/metadata:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

27
app/job/main/tv/dao/report/dao.go Normal file
View File

@@ -0,0 +1,27 @@
package report
import (
"go-common/app/job/main/tv/conf"
"go-common/library/cache/memcache"
"go-common/library/database/sql"
bm "go-common/library/net/http/blademaster"
)
// Dao dao .
type Dao struct {
conf *conf.Config
httpR *bm.Client
mc *memcache.Pool
DB *sql.DB
}
// New create a instance of Dao and return .
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
httpR: bm.NewClient(c.DpClient),
mc: memcache.NewPool(c.Memcache.Config),
DB: sql.NewMySQL(c.Mysql),
}
return
}

35
app/job/main/tv/dao/report/dao_test.go Normal file
View File

@@ -0,0 +1,35 @@
package report
import (
"flag"
"os"
"testing"
"go-common/app/job/main/tv/conf"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.web-svr.tv-job")
flag.Set("conf_token", "ab3e9801a77c076b997de0ac5cb21775")
flag.Set("tree_id", "15260")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../../cmd/tv-job-test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
os.Exit(m.Run())
}

119
app/job/main/tv/dao/report/mc.go Normal file
View File

@@ -0,0 +1,119 @@
package report
import (
"context"
"time"
mdlpgc "go-common/app/job/main/tv/model/pgc"
"go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_report = "_report"
_style = "style_label"
_label = "label_data"
)
// SetReportCache set report cache .
func (d *Dao) SetReportCache(c context.Context, val map[string]interface{}) (err error) {
var (
conn = d.mc.Get(c)
key = _report
)
defer conn.Close()
item := &memcache.Item{
Key: key,
Object: val,
Flags: memcache.FlagJSON,
Expiration: int32(time.Duration(d.conf.Report.Expire) / time.Second),
}
if err = conn.Set(item); err != nil {
log.Error("conn.Set(%v) error(%v)", item, err)
}
return
}
// GetReportCache get report all data .
func (d *Dao) GetReportCache(c context.Context) (res map[string]interface{}, err error) {
var (
conn = d.mc.Get(c)
key = _report
rp *memcache.Item
)
res = make(map[string]interface{})
defer conn.Close()
if rp, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
log.Error("mc.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(rp, &res); err != nil {
log.Error("conn.Scan error(%v)", err)
}
return
}
// SetStyleCache style show .
func (d *Dao) SetStyleCache(c context.Context, val map[int][]*mdlpgc.ParamStyle) (err error) {
var (
conn = d.mc.Get(c)
key = _style
)
defer conn.Close()
item := &memcache.Item{
Key: key,
Object: val,
Flags: memcache.FlagJSON,
Expiration: 0,
}
if err = conn.Set(item); err != nil {
log.Error("conn.Set(%v) error(%v)", item, err)
}
return
}
// SetLabelCache label show .
func (d *Dao) SetLabelCache(c context.Context, val map[int]map[string]int) (err error) {
var (
conn = d.mc.Get(c)
key = _label
)
defer conn.Close()
item := &memcache.Item{
Key: key,
Object: val,
Flags: memcache.FlagJSON,
Expiration: 0,
}
if err = conn.Set(item); err != nil {
log.Error("conn.Set(%v) error(%v)", item, err)
}
return
}
//
// GetLabelCache get label all data .
func (d *Dao) GetLabelCache(c context.Context) (res map[int]map[string]int, err error) {
var (
conn = d.mc.Get(c)
key = _label
rp *memcache.Item
)
defer conn.Close()
if rp, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
log.Error("mc.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(rp, &res); err != nil {
log.Error("conn.Scan error(%v)", err)
}
return
}

38
app/job/main/tv/dao/report/mc_test.go Normal file
View File

@@ -0,0 +1,38 @@
package report
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestReportSetReportCache(t *testing.T) {
convey.Convey("SetReportCache", t, func(ctx convey.C) {
var (
c = context.Background()
val = make(map[string]interface{})
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
err := d.SetReportCache(c, val)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestReportGetReportCache(t *testing.T) {
convey.Convey("GetReportCache", t, func(ctx convey.C) {
var (
c = context.Background()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := d.GetReportCache(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}

227
app/job/main/tv/dao/report/report.go Normal file
View File

@@ -0,0 +1,227 @@
package report
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
xhttp "net/http"
"net/url"
"sort"
"strings"
"time"
mdlrep "go-common/app/job/main/tv/model/report"
"go-common/library/conf/env"
"go-common/library/log"
"go-common/library/net/metadata"
)
const (
_httpOK = "0"
_pid = "73" // platform: 73 is tv
_plat = 5 // platform: 5 is tv
_tv = "android_tv_yst"
_actDur = `select request_uri,time_iso,ip,version,buvid,fts,proid,chid,pid,brand,deviceid,model,osver,ctime,mid,ver,net,oid,eid,start_time,end_time,duration,openudid,idfa,mac,is_coldstart,session_id,buvid_ext from ods.app_active_duration where unix_timestamp(ctime, 'yyyyMMddHHmmss')>=%d and unix_timestamp(ctime, 'yyyyMMddHHmmss')<%d and log_date=%s and pid=%s`
_playDur = `select stime,build,buvid,mobi_app,platform,session,mid,aid,cid,sid,epid,type,sub_type,quality,total_time,paused_time,played_time,video_duration,play_type,network_type,last_play_progress_time,max_play_progress_time,device,epid_status,play_status,user_status,actual_played_time,auto_play,detail_play_time,list_play_time from ods.app_play_duration where stime>=%d and stime<%d and log_date=%s and mobi_app=%s`
_visitEvent = `select request_uri,time_iso,ip,version,buvid,fts,proid,chid,pid,brand,deviceid,model,osver,mid,ctime,ver,net,oid,page_name,page_arg,ua,h5_chid,unix_timestamp(ctime, 'yyyyMMddHHmmss') from ods.app_visit_event where unix_timestamp(ctime, 'yyyyMMddHHmmss')>=%d and unix_timestamp(ctime, 'yyyyMMddHHmmss')<%d and log_date=%s and pid=%s`
_arcClick = `select r_type,avid,cid,part,mid,stime,did,ip,ua,buvid,cookie_sid,refer,type,sub_type,sid,epid,platform,device from ods.ods_archive_click where stime>=%d and stime<%d and log_date=%s and plat=%d`
)
var (
signParams = []string{"appKey", "timestamp", "version"}
_userAgent = "User-Agent"
)
// Report .
func (d *Dao) Report(ctx context.Context, table string) (info string, err error) {
var (
v = url.Values{}
ip = metadata.String(ctx, metadata.RemoteIP)
logdata = d.queryfmt()
start, end = d.dealDate()
query string
res struct {
Code int `json:"code"`
Msg string `json:"msg"`
StatusURL string `json:"jobStatusUrl"`
}
)
logdata = "'" + logdata + "'"
switch table {
case mdlrep.ArchiveClick:
query = fmt.Sprintf(_arcClick, start, end, logdata, _plat)
case mdlrep.ActiveDuration:
query = fmt.Sprintf(_actDur, start, end, logdata, `"`+_pid+`"`)
case mdlrep.PlayDuration:
query = fmt.Sprintf(_playDur, start, end, logdata, "'"+_tv+"'")
case mdlrep.VisitEvent:
query = fmt.Sprintf(_visitEvent, start, end, logdata, `"`+_pid+`"`)
default:
err = errors.New("table is nill")
return
}
v.Set("appKey", d.conf.DpClient.Key)
v.Set("signMethod", "md5")
v.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
v.Set("version", "1.0")
v.Set("query", query)
if err = d.newRequest(ctx, d.conf.Report.ReportURI, ip, v, &res); err != nil {
log.Error("newRequest url(%v), err(%v)", d.conf.Report.ReportURI+"?"+v.Encode(), err)
return
}
if res.Code == 200 && res.StatusURL != "" {
info = res.StatusURL
}
return
}
// CheckJob .
func (d *Dao) CheckJob(ctx context.Context, urls string) (res *mdlrep.DpCheckJobResult, err error) {
var (
v = url.Values{}
ip = metadata.String(ctx, metadata.RemoteIP)
)
res = &mdlrep.DpCheckJobResult{}
v.Set("appKey", d.conf.DpClient.Key)
v.Set("signMethod", "md5")
v.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
v.Set("version", "1.0")
if err = d.newRequest(ctx, urls, ip, v, &res); err != nil {
log.Error("d.newRequest error(%v)", err)
return
}
if res.Code != xhttp.StatusOK {
log.Error("d.CheckJob newRequest error code:%d ; url(%s) ", res.Code, urls+"?"+v.Encode())
err = fmt.Errorf("code(%d) msg(%s) statusID(%d) statusID(%s)", res.Code, res.Msg, res.StatusID, res.StatusMsg)
}
return
}
// PostRequest .
func (d *Dao) PostRequest(ctx context.Context, body string) (err error) {
var (
req *xhttp.Request
res struct {
Code string `json:"code"`
Message string `json:"message"`
}
)
if req, err = xhttp.NewRequest(xhttp.MethodPost, d.conf.Report.UpDataURI, strings.NewReader(body)); err != nil {
log.Error("xhttp.NewRequest url(%s) error (%v)", d.conf.Report.UpDataURI, err)
return
}
req.Header.Add("Content-Type", "text/plain; charset=utf-8")
if err = d.httpR.Do(ctx, req, &res); err != nil {
log.Error("d.httpReq.Do error(%v) url(%s)", err, d.conf.Report.UpDataURI)
return
}
if res.Code != _httpOK {
log.Error("PostRequest error code:%s ; url(%s) ", res.Code, d.conf.Report.UpDataURI)
err = fmt.Errorf("code(%s)", res.Code)
}
return
}
// NewRequest new http request with method, url, ip, values and headers .
func (d *Dao) newRequest(c context.Context, url, realIP string, params url.Values, res interface{}) (err error) {
enc, err := d.sign(params)
if err != nil {
log.Error("url:%s,params:%v", url, params)
return
}
if enc != "" {
url = url + "?" + enc
}
req, err := xhttp.NewRequest(xhttp.MethodGet, url, nil)
if err != nil {
log.Error("xhttp.NewRequest method:%s,url:%s", xhttp.MethodGet, url)
return
}
req.Header.Set(_userAgent, "haoguanwei@bilibili.com "+env.AppID)
if err != nil {
return
}
return d.httpR.Do(c, req, res)
}
// Sign calc appkey and appsecret sign .
func (d *Dao) sign(params url.Values) (query string, err error) {
tmp := params.Encode()
signTmp := d.encode(params)
if strings.IndexByte(tmp, '+') > -1 {
tmp = strings.Replace(tmp, "+", "%20", -1)
}
var b bytes.Buffer
b.WriteString(d.conf.DpClient.Secret)
b.WriteString(signTmp)
b.WriteString(d.conf.DpClient.Secret)
mh := md5.Sum(b.Bytes())
// query
var qb bytes.Buffer
qb.WriteString(tmp)
qb.WriteString("&sign=")
qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
query = qb.String()
return
}
// Encode encodes the values into ``URL encoded'' form .
// ("bar=baz&foo=quux") sorted by key .
func (d *Dao) encode(v url.Values) string {
if v == nil {
return ""
}
var buf bytes.Buffer
keys := make([]string, 0, len(v))
for k := range v {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
found := false
for _, p := range signParams {
if p == k {
found = true
break
}
}
if !found {
continue
}
vs := v[k]
prefix := k
for _, v := range vs {
buf.WriteString(prefix)
buf.WriteString(v)
}
}
return buf.String()
}
// delDate create start and end time .
func (d *Dao) dealDate() (start, end int64) {
var (
timeDelay = "-" + d.conf.Report.TimeDelay
timeSpan = "-" + time.Duration(d.conf.Report.SeTimeSpan).String()
)
endTime := time.Now()
et, _ := time.ParseDuration(timeDelay)
endTmp := endTime.Add(et)
end = endTime.Add(et).Unix()
st, _ := time.ParseDuration(timeSpan)
start = endTmp.Add(st).Unix()
return
}
func (d *Dao) queryfmt() (logdata string) {
var (
dtime = time.Now()
timeData = "-" + d.conf.Report.TimeDelay
)
et, _ := time.ParseDuration(timeData)
logdata = dtime.Add(et).Format("20060102")
return
}

32
app/job/main/tv/dao/report/report_test.go Normal file
View File

@@ -0,0 +1,32 @@
package report
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestReportReport(t *testing.T) {
convey.Convey("Report", t, func(ctx convey.C) {
var (
c = context.Background()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
_, err := d.Report(c, "archive_click")
println(err)
})
})
}
func TestReportCheckJob(t *testing.T) {
convey.Convey("CheckJob", t, func(ctx convey.C) {
var (
c = context.Background()
urls = ""
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
d.CheckJob(c, urls)
})
})
}

69
app/job/main/tv/dao/report/style.go Normal file
View File

@@ -0,0 +1,69 @@
package report
import (
"context"
mdlpgc "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_styleSQL = "SELECT id,style,category FROM tv_ep_season WHERE is_deleted=0 AND `check`=1 AND valid=1"
_labelSQL = `SELECT name,value,category FROM tv_label WHERE deleted=0 AND param="style_id" AND valid=1`
)
// FindStyle style all .
func (d *Dao) FindStyle(ctx context.Context) (res []*mdlpgc.StyleRes, err error) {
var (
rows *sql.Rows
)
if rows, err = d.DB.Query(ctx, _styleSQL); err != nil {
log.Error("d.DB.Query sql(%s) error(%v)", _styleSQL, err)
return
}
defer rows.Close()
for rows.Next() {
r := &mdlpgc.StyleRes{}
if err = rows.Scan(&r.ID, &r.Style, &r.Category); err != nil {
log.Error("d.DB.QueryRow error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("rows.Err() error(%v)", err)
}
return
}
// FindLabelID label all .
func (d *Dao) FindLabelID(ctx context.Context) (res map[int]map[string]int, err error) {
var (
rows *sql.Rows
m map[string]int
)
res = make(map[int]map[string]int)
if rows, err = d.DB.Query(ctx, _labelSQL); err != nil {
log.Error("d.DB.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
r := &mdlpgc.LabelRes{}
if err = rows.Scan(&r.Name, &r.Value, &r.Category); err != nil {
log.Error("d.DB.Query Scan error(%v)", err)
return
}
if _, ok := res[r.Category]; ok {
m[r.Name] = r.Value
} else {
m = make(map[string]int)
}
res[r.Category] = m
}
if err = rows.Err(); err != nil {
log.Error("rows.Err() error(%v)", err)
}
return
}

38
app/job/main/tv/dao/report/style_test.go Normal file
View File

@@ -0,0 +1,38 @@
package report
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestReportFindStyle(t *testing.T) {
convey.Convey("FindStyle", t, func(ctx convey.C) {
var (
c = context.Background()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := d.FindStyle(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}
func TestReportFindLabelID(t *testing.T) {
convey.Convey("FindLabelID", t, func(ctx convey.C) {
var (
c = context.Background()
)
ctx.Convey("When everything goes positive", func(ctx convey.C) {
res, err := d.FindLabelID(c)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
})
}

85
app/job/main/tv/dao/ugc/BUILD Normal file
View File

@@ -0,0 +1,85 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"databus_test.go",
"del_arc_test.go",
"del_upper_test.go",
"del_video_test.go",
"delete_test.go",
"import_test.go",
"manual_test.go",
"media_cache_test.go",
"passed_test.go",
"search_con_test.go",
"sync_video_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/ugc:go_default_library",
"//app/service/main/archive/api:go_default_library",
"//library/database/sql:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"databus.go",
"del_arc.go",
"del_upper.go",
"del_video.go",
"delete.go",
"import.go",
"manual.go",
"media_cache.go",
"passed.go",
"report_cid.go",
"search_con.go",
"sync_arc.go",
"sync_video.go",
],
importpath = "go-common/app/job/main/tv/dao/ugc",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/dao/app:go_default_library",
"//app/job/main/tv/model/common:go_default_library",
"//app/job/main/tv/model/pgc:go_default_library",
"//app/job/main/tv/model/ugc:go_default_library",
"//app/service/main/archive/api:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/xstr:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

49
app/job/main/tv/dao/ugc/dao.go Normal file
View File

@@ -0,0 +1,49 @@
package ugc
import (
"context"
"time"
"go-common/app/job/main/tv/conf"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/database/sql"
httpx "go-common/library/net/http/blademaster"
)
// Dao dao.
type Dao struct {
conf *conf.Config
DB *sql.DB
client *httpx.Client
mc *memcache.Pool
mcExpire int32 // expire for ugc cache, same as pgc auth, because it's daily refresh
criCID int64 // critical cid for ugc video sync
redis *redis.Pool
}
// New create a instance of Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
conf: c,
DB: sql.NewMySQL(c.Mysql),
client: httpx.NewClient(conf.Conf.HTTPClient),
mc: memcache.NewPool(c.Memcache.Config),
mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
criCID: c.UgcSync.Cfg.CriticalCid,
redis: redis.NewPool(c.Redis.Config),
}
return
}
// Close close the redis and kafka resource.
func (d *Dao) Close() {
if d.DB != nil {
d.DB.Close()
}
}
// BeginTran begin mysql transaction
func (d *Dao) BeginTran(c context.Context) (*sql.Tx, error) {
return d.DB.Begin(c)
}

70
app/job/main/tv/dao/ugc/dao_test.go Normal file
View File

@@ -0,0 +1,70 @@
package ugc
import (
"context"
"flag"
"fmt"
"path/filepath"
"testing"
"go-common/app/job/main/tv/conf"
"go-common/app/job/main/tv/model/ugc"
. "github.com/smartystreets/goconvey/convey"
)
var (
ctx = context.TODO()
d *Dao
queryMid = "SELECT mid FROM ugc_archive WHERE result=1 AND valid=1 AND deleted=0 LIMIT 1"
queryAid = "SELECT aid FROM ugc_archive WHERE result=1 AND valid=1 AND deleted=0 LIMIT 1"
)
func WithDao(f func(d *Dao)) func() {
return func() {
dir, _ := filepath.Abs("../../cmd/tv-job-test.toml")
flag.Set("conf", dir)
conf.Init()
if d == nil {
d = New(conf.Conf)
}
f(d)
}
}
func TestDao_UpArcs(t *testing.T) {
Convey("TestDao_UpArcs", t, WithDao(func(d *Dao) {
var mid int64
d.DB.QueryRow(ctx, queryMid).Scan(&mid)
if mid == 0 {
return
}
res, err := d.UpArcs(ctx, mid)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
}))
}
func TestDao_SetArcCMS(t *testing.T) {
Convey("TestDao_SetArcCMS", t, WithDao(func(d *Dao) {
var aid int64
d.DB.QueryRow(ctx, queryAid).Scan(&aid)
if aid != 0 {
err := d.SetArcCMS(ctx, &ugc.ArcCMS{
AID: aid,
Title: "test",
})
So(err, ShouldBeNil)
fmt.Println(aid)
}
}))
}
func TestDao_CountArcs(t *testing.T) {
Convey("TestDao_CountArcs", t, WithDao(func(d *Dao) {
res, err := d.CountUpArcs(ctx, 452156)
So(err, ShouldBeNil)
So(res, ShouldBeGreaterThan, 0)
fmt.Println(res)
}))
}

92
app/job/main/tv/dao/ugc/databus.go Normal file
View File

@@ -0,0 +1,92 @@
package ugc
import (
"context"
ugcmdl "go-common/app/job/main/tv/model/ugc"
arccli "go-common/app/service/main/archive/api"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_updateArc = "UPDATE ugc_archive SET title = ?, cover = ?, content = ?, pubtime = ?, " +
"typeid = ?, submit = ?, state = ? WHERE aid = ? AND deleted = 0"
_updateVideo = "UPDATE ugc_video SET eptitle = ?, index_order = ?, submit = ? WHERE cid = ? AND deleted = 0"
_needSubmit = 1
_pickVideos = "SELECT id, eptitle, cid, index_order FROM ugc_video WHERE aid = ? AND deleted = 0"
_upInList = "SELECT mid FROM ugc_uploader WHERE mid = ? AND deleted = 0"
_setUploader = "REPLACE INTO ugc_uploader (mid, state) VALUES (?,?)"
)
// UpInList checks whether the upper is in our list
func (d *Dao) UpInList(c context.Context, mid int64) (realID int64, err error) {
if err = d.DB.QueryRow(c, _upInList, mid).Scan(&realID); err != nil { // get the qualified aid to sync
if err == sql.ErrNoRows {
err = nil
return
}
log.Error("d.UpInList.Query error(%v)", err)
}
return
}
// UpdateArc updates the key fields of an archive, used for databus monitoring
func (d *Dao) UpdateArc(c context.Context, arc *ugcmdl.ArchDatabus) (err error) {
if _, err = d.DB.Exec(c, _updateArc,
arc.Title, arc.Cover, arc.Content, arc.PubTime, arc.TypeID, _needSubmit, arc.State, arc.Aid); err != nil {
log.Error("UpdateArc, failed to update: (%v), Error: %v", arc, err)
}
return
}
// TxUpdateVideo updates the ugc video's status, for databus update
func (d *Dao) TxUpdateVideo(tx *sql.Tx, video *arccli.Page) (err error) {
if _, err = tx.Exec(_updateVideo,
video.Part, video.Page, _needSubmit, video.Cid); err != nil {
log.Error("TxUpdateVideo, failed to update: (%v), Error: %v", video, err)
}
return
}
// PickVideos picks the videos of an archive in one shot
func (d *Dao) PickVideos(c context.Context, aid int64) (res map[int64]*ugcmdl.SimpleVideo, err error) {
var rows *sql.Rows
res = make(map[int64]*ugcmdl.SimpleVideo)
if rows, err = d.DB.Query(c, _pickVideos, aid); err != nil {
log.Error("d.Import.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &ugcmdl.SimpleVideo{}
if err = rows.Scan(&r.ID, &r.Eptitle, &r.CID, &r.IndexOrder); err != nil {
log.Error("PickVideos row.Scan() error(%v)", err)
return
}
res[r.CID] = r
}
if err = rows.Err(); err != nil {
log.Error("d.PickVideos.Query error(%v)", err)
}
return
}
// TxAddVideos add into the db the new videos
func (d *Dao) TxAddVideos(tx *sql.Tx, pages []*arccli.Page, aid int64) (err error) {
for _, v := range pages {
if _, err = tx.Exec(_importVideo, aid, v.Cid, v.Part, v.Page, v.Duration, v.Desc); err != nil {
log.Error("_importArc, failed to insert: (%v), Error: %v", v, err)
return
}
}
return
}
// TxUpAdd adds the upper
func (d *Dao) TxUpAdd(tx *sql.Tx, mid int64) (err error) {
if _, err = tx.Exec(_setUploader, mid, 1); err != nil {
log.Error("UpAdd Error %v", err)
}
return
}

51
app/job/main/tv/dao/ugc/databus_test.go Normal file
View File

@@ -0,0 +1,51 @@
package ugc
import (
"fmt"
"testing"
"go-common/app/service/main/archive/api"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_UpInList(t *testing.T) {
Convey("TestDao_UpInList", t, WithDao(func(d *Dao) {
res, err := d.UpInList(ctx, 27515615)
So(err, ShouldBeNil)
So(res, ShouldBeGreaterThan, 0)
fmt.Println(res)
res2, err2 := d.UpInList(ctx, 100997637777)
So(err2, ShouldBeNil)
So(res2, ShouldEqual, 0)
}))
}
func TestDao_PickVideos(t *testing.T) {
Convey("TestDao_PickVideos", t, WithDao(func(d *Dao) {
res, err := d.PickVideos(ctx, 10099763)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
for _, v := range res {
fmt.Println(v)
}
}))
}
func TestDao_InsertVideos(t *testing.T) {
Convey("TestDao_InsertVideos", t, WithDao(func(d *Dao) {
tx, err := d.BeginTran(ctx)
So(err, ShouldBeNil)
err = d.TxAddVideos(tx, []*api.Page{
{
Cid: 10126229,
Part: "test",
Duration: 2333,
Desc: "test",
Page: 999,
},
}, 10098693)
tx.Commit()
So(err, ShouldBeNil)
}))
}

37
app/job/main/tv/dao/ugc/del_arc.go Normal file
View File

@@ -0,0 +1,37 @@
package ugc
import (
"context"
"go-common/library/log"
"time"
)
const (
_deletedArc = "SELECT aid FROM ugc_archive WHERE submit = 1 AND deleted = 1 AND retry < unix_timestamp(now()) LIMIT 1"
_finishDelArc = "UPDATE ugc_archive SET submit = 0 WHERE aid = ? AND deleted = 1"
_ppDelArc = "UPDATE ugc_archive SET retry = ? WHERE aid = ? AND deleted = 1"
)
// DeletedArc picks the deleted archive to sync
func (d *Dao) DeletedArc(c context.Context) (aid int64, err error) {
err = d.DB.QueryRow(c, _deletedArc).Scan(&aid)
return
}
// FinishDelArc updates the submit status from 1 to 0
func (d *Dao) FinishDelArc(c context.Context, aid int64) (err error) {
if _, err = d.DB.Exec(c, _finishDelArc, aid); err != nil {
log.Error("FinishVideos Error: %v", aid, err)
}
return
}
// PpDelArc postpones the archive's submit in 30 mins
func (d *Dao) PpDelArc(c context.Context, aid int64) (err error) {
var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
if _, err = d.DB.Exec(c, _ppDelArc, delay, aid); err != nil {
log.Error("PostponeArc, failed to delay: (%v,%v), Error: %v", delay, aid, err)
}
return
}

35
app/job/main/tv/dao/ugc/del_arc_test.go Normal file
View File

@@ -0,0 +1,35 @@
package ugc
import (
"fmt"
"go-common/library/database/sql"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_DeletedArc(t *testing.T) {
Convey("TestDao_DeletedArc", t, WithDao(func(d *Dao) {
res, err := d.DeletedArc(ctx)
if err == sql.ErrNoRows {
fmt.Println("No to delete data")
return
}
So(err, ShouldBeNil)
So(res, ShouldBeGreaterThan, 0)
}))
}
func TestDao_PpDelArc(t *testing.T) {
Convey("TestDao_PpDelArc", t, WithDao(func(d *Dao) {
err := d.PpDelArc(ctx, 333)
So(err, ShouldBeNil)
}))
}
func TestDao_FinishDelArc(t *testing.T) {
Convey("TestDao_FinishDelArc", t, WithDao(func(d *Dao) {
err := d.FinishDelArc(ctx, 333)
So(err, ShouldBeNil)
}))
}

70
app/job/main/tv/dao/ugc/del_upper.go Normal file
View File

@@ -0,0 +1,70 @@
package ugc
import (
"context"
"time"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_deletedUp = "SELECT mid FROM ugc_uploader WHERE toinit = 2 AND deleted = 1 AND retry < unix_timestamp(now()) LIMIT 1"
_finishDelUp = "UPDATE ugc_uploader SET toinit = 0 WHERE mid = ? AND deleted = 1"
_ppDelUp = "UPDATE ugc_uploader SET retry = ? WHERE mid = ? AND deleted = 1"
_upArcs = "SELECT aid FROM ugc_archive WHERE mid = ? AND deleted = 0 LIMIT 50"
_upCountArc = "SELECT count(1) FROM ugc_archive WHERE mid = ? AND deleted = 0"
)
// DeletedUp picks the deleted uppers, toinit = 2 and deleted = 1
func (d *Dao) DeletedUp(c context.Context) (mid int64, err error) {
err = d.DB.QueryRow(c, _deletedUp).Scan(&mid)
return
}
// FinishDelUp updates the submit toinit from 2 to 0
func (d *Dao) FinishDelUp(c context.Context, mid int64) (err error) {
if _, err = d.DB.Exec(c, _finishDelUp, mid); err != nil {
log.Error("FinishDelUp Error: %v", mid, err)
}
return
}
// PpDelUp postpones the upper's videos submit in 30 mins
func (d *Dao) PpDelUp(c context.Context, mid int64) (err error) {
var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
if _, err = d.DB.Exec(c, _ppDelUp, delay, mid); err != nil {
log.Error("PostponeArc, failed to delay: (%v,%v), Error: %v", delay, mid, err)
}
return
}
// CountUpArcs counts the upper's archives
func (d *Dao) CountUpArcs(c context.Context, mid int64) (count int64, err error) {
if err = d.DB.QueryRow(c, _upCountArc, mid).Scan(&count); err != nil {
log.Error("d.CountUpArcs.Query error(%v)", err)
}
return
}
// UpArcs picks 50 arcs of the upper
func (d *Dao) UpArcs(c context.Context, mid int64) (aids []int64, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(c, _upArcs, mid); err != nil { // get the qualified aid to sync
log.Error("d.UpArcs.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var aid int64
if err = rows.Scan(&aid); err != nil {
log.Error("ParseVideos row.Scan() error(%v)", err)
return
}
aids = append(aids, aid)
}
if err = rows.Err(); err != nil {
log.Error("d.UpArcs.Query error(%v)", err)
}
return
}

80
app/job/main/tv/dao/ugc/del_upper_test.go Normal file
View File

@@ -0,0 +1,80 @@
package ugc
import (
"context"
"testing"
"database/sql"
"fmt"
"github.com/smartystreets/goconvey/convey"
)
func TestUgcDeletedUp(t *testing.T) {
var c = context.Background()
convey.Convey("DeletedUp", t, func(ctx convey.C) {
mid, err := d.DeletedUp(c)
if err == sql.ErrNoRows {
fmt.Println("No to delete data")
return
}
ctx.Convey("Then err should be nil.mid should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(mid, convey.ShouldNotBeNil)
})
})
}
func TestUgcFinishDelUp(t *testing.T) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("FinishDelUp", t, func(ctx convey.C) {
err := d.FinishDelUp(c, mid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestUgcPpDelUp(t *testing.T) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("PpDelUp", t, func(ctx convey.C) {
err := d.PpDelUp(c, mid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestUgcCountUpArcs(t *testing.T) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("CountUpArcs", t, func(ctx convey.C) {
count, err := d.CountUpArcs(c, mid)
ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(count, convey.ShouldNotBeNil)
})
})
}
func TestUgcUpArcs(t *testing.T) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("UpArcs", t, func(ctx convey.C) {
aids, err := d.UpArcs(c, mid)
ctx.Convey("Then err should be nil.aids should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(aids, convey.ShouldNotBeNil)
})
})
}

59
app/job/main/tv/dao/ugc/del_video.go Normal file
View File

@@ -0,0 +1,59 @@
package ugc
import (
"context"
"time"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_deletedVideos = "SELECT cid FROM ugc_video WHERE submit = 1 AND deleted = 1 AND retry < unix_timestamp(now()) LIMIT 5"
_finishDelVideo = "UPDATE ugc_video SET submit = 0 WHERE cid = ? AND deleted = 1"
_ppDelVideos = "UPDATE ugc_video SET retry = ? WHERE cid = ? AND deleted = 1"
)
// DeletedVideos picks the deleted videos to sync
func (d *Dao) DeletedVideos(c context.Context) (delIds []int, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(c, _deletedVideos); err != nil { // get the qualified aid to sync
return
}
defer rows.Close()
for rows.Next() {
var cid int
if err = rows.Scan(&cid); err != nil {
log.Error("ParseVideos row.Scan() error(%v)", err)
return
}
delIds = append(delIds, cid)
}
if err = rows.Err(); err != nil {
log.Error("d.deletedVideos.Query error(%v)", err)
}
return
}
// FinishDelVideos updates the submit status from 1 to 0
func (d *Dao) FinishDelVideos(c context.Context, delIds []int) (err error) {
for _, v := range delIds {
if _, err = d.DB.Exec(c, _finishDelVideo, v); err != nil {
log.Error("FinishDelVideos Error: %v", v, err)
return
}
}
return
}
// PpDelVideos postpones the archive's videos submit in 30 mins
func (d *Dao) PpDelVideos(c context.Context, delIds []int) (err error) {
var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
for _, v := range delIds {
if _, err = d.DB.Exec(c, _ppDelVideos, delay, v); err != nil {
log.Error("PpDelVideos, failed to delay: (%v,%v), Error: %v", delay, v, err)
return
}
}
return
}

23
app/job/main/tv/dao/ugc/del_video_test.go Normal file
View File

@@ -0,0 +1,23 @@
package ugc
import (
"testing"
"fmt"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_DeletedVideos(t *testing.T) {
Convey("TestDao_DeletedVideos", t, WithDao(func(d *Dao) {
res, err := d.DeletedVideos(ctx)
if err == nil && len(res) == 0 {
fmt.Println("No Delete Data")
d.DB.Exec(ctx, "UPDATE ugc_video SET deleted = 1, submit =1 WHERE deleted = 0 LIMIT 1")
}
res, err = d.DeletedVideos(ctx)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
fmt.Println(res)
}))
}

78
app/job/main/tv/dao/ugc/delete.go Normal file
View File

@@ -0,0 +1,78 @@
package ugc
import (
"context"
"fmt"
"go-common/app/job/main/tv/model/ugc"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_delArc = "UPDATE ugc_archive SET deleted = 1, submit = 1 WHERE aid = ? AND deleted = 0"
_delVideos = "UPDATE ugc_video SET deleted = 1, submit = 1 WHERE aid = ? AND deleted = 0"
_delVideo = "UPDATE ugc_video SET deleted = 1, submit = 1 WHERE cid = ? AND deleted = 0"
_delVideoCids = "UPDATE ugc_video SET deleted = 1, submit = 1 WHERE cid IN (%s) AND deleted = 0"
_checkVideos = "SELECT cid FROM ugc_video WHERE aid = ? AND deleted = 0 LIMIT 1"
)
// TxDelArc deletes an arc
func (d *Dao) TxDelArc(tx *sql.Tx, aid int64) (err error) {
if _, err = tx.Exec(_delArc, aid); err != nil {
log.Error("TxDelArc, failed to update: (%v), Error: %v", aid, err)
}
return
}
// DelVideos delete one archive all videos
func (d *Dao) DelVideos(ctx context.Context, aid int64) (err error) {
if _, err = d.DB.Exec(ctx, _delVideos, aid); err != nil {
log.Error("DelVideos, failed to update: (%v), Error: %v", aid, err)
return
}
log.Info("Aid %d is deleted, delete its videos", aid)
return
}
// TxDelVideos deletes the videos of an arc
func (d *Dao) TxDelVideos(tx *sql.Tx, aid int64) (err error) {
if _, err = tx.Exec(_delVideos, aid); err != nil {
log.Error("TxDelVideos, failed to update: (%v), Error: %v", aid, err)
}
return
}
// TxDelVideo deletes a video
func (d *Dao) TxDelVideo(tx *sql.Tx, cid int64) (err error) {
if _, err = tx.Exec(_delVideo, cid); err != nil {
log.Error("TxDelVideo, failed to update: (%v), Error: %v", cid, err)
}
return
}
// DelVideoArc deletes some videos of an archive, if the archive is empty, also delete it
func (d *Dao) DelVideoArc(ctx context.Context, req *ugc.DelVideos) (arcValid bool, err error) {
var cid int64
arcValid = true
if _, err = d.DB.Exec(ctx, fmt.Sprintf(_delVideoCids, xstr.JoinInts(req.CIDs))); err != nil {
log.Error("DelVideos Cids %v, Aid %d, Error: %v", req.CIDs, req.AID, err)
return
}
if err = d.DB.QueryRow(ctx, _checkVideos, req.AID).Scan(&cid); err != nil { // if no active videos, delete the arc
if err == sql.ErrNoRows {
err = nil
arcValid = false
if _, err = d.DB.Exec(ctx, _delArc, req.AID); err != nil {
log.Error("DelVideos DelArc Cids %v, Aid %d, Error: %v", req.CIDs, req.AID, err)
return
}
log.Info("DelArc Aid %d Because No Active Video", req.AID)
} else {
log.Error("DelVideos Cids %v, Aid %d, Error: %v", req.CIDs, req.AID, err)
return
}
}
return
}

127
app/job/main/tv/dao/ugc/delete_test.go Normal file
View File

@@ -0,0 +1,127 @@
package ugc
import (
"fmt"
"testing"
"go-common/app/job/main/tv/model/ugc"
arccli "go-common/app/service/main/archive/api"
. "github.com/smartystreets/goconvey/convey"
)
func TestUgcTxDelArc(t *testing.T) {
var (
tx, _ = d.DB.Begin(ctx)
aid = int64(0)
)
Convey("TxDelArc", t, func(ctx C) {
err := d.TxDelArc(tx, aid)
ctx.Convey("Then err should be nil.", func(ctx C) {
ctx.So(err, ShouldBeNil)
})
})
}
func TestUgcTxDelVideos(t *testing.T) {
var (
tx, _ = d.DB.Begin(ctx)
aid = int64(0)
)
Convey("TxDelVideos", t, func(ctx C) {
err := d.TxDelVideos(tx, aid)
ctx.Convey("Then err should be nil.", func(ctx C) {
ctx.So(err, ShouldBeNil)
})
})
}
func TestUgcTxDelVideo(t *testing.T) {
var (
tx, _ = d.DB.Begin(ctx)
cid = int64(0)
)
Convey("TxDelVideo", t, func(ctx C) {
err := d.TxDelVideo(tx, cid)
ctx.Convey("Then err should be nil.", func(ctx C) {
ctx.So(err, ShouldBeNil)
})
})
}
func TestDao_DelVideoArc(t *testing.T) {
Convey("TestDao_DelVideoArc", t, WithDao(func(d *Dao) {
var (
aid = int64(99998888)
cid1 = int64(999988881)
cid2 = int64(999988882)
tx, _ = d.DB.Begin(ctx)
arc = &arccli.Arc{Aid: aid}
countVQ = "SELECT COUNT(1) FROM ugc_video WHERE aid = ? AND deleted = 0"
countAQ = "SELECT COUNT(1) FROM ugc_archive WHERE aid = ? AND deleted = 0"
countV, countA int
arcValid bool
)
// add archive and two videos
d.TxImportArc(tx, &arccli.Arc{Aid: aid})
d.TxMnlVideos(tx, &arccli.ViewReply{
Arc: arc,
Pages: []*arccli.Page{
{
Cid: cid1,
},
{
Cid: cid2,
},
},
})
tx.Commit()
d.DB.QueryRow(ctx, countVQ, aid).Scan(&countV)
So(countV, ShouldEqual, 2)
// delete one video, still one active video under the archive, we keep the archive
_, err := d.DelVideoArc(ctx, &ugc.DelVideos{
AID: aid,
CIDs: []int64{cid1},
})
So(err, ShouldBeNil)
d.DB.QueryRow(ctx, countVQ, aid).Scan(&countV)
d.DB.QueryRow(ctx, countAQ, aid).Scan(&countA)
So(countV, ShouldEqual, 1)
So(countA, ShouldEqual, 1)
So(err, ShouldBeNil)
// delete the last video, the archive should also be deleted
arcValid, err = d.DelVideoArc(ctx, &ugc.DelVideos{
AID: aid,
CIDs: []int64{cid2},
})
So(err, ShouldBeNil)
So(arcValid, ShouldBeFalse)
d.DB.QueryRow(ctx, countVQ, aid).Scan(&countV)
d.DB.QueryRow(ctx, countAQ, aid).Scan(&countA)
So(countV, ShouldEqual, 0)
So(countA, ShouldEqual, 0)
}))
}
func TestDao_DelVideos(t *testing.T) {
Convey("TestDao_DelVideos", t, WithDao(func(d *Dao) {
var (
aid = int64(99998888)
cid1 = 99998887
cid2 = 99998886
insertSQL = "REPLACE INTO ugc_video (aid, cid) VALUES (%d, %d)"
)
ress, err2 := d.DB.Exec(ctx, fmt.Sprintf(insertSQL, aid, cid1))
fmt.Println(fmt.Sprintf(insertSQL, aid, cid1))
fmt.Println(err2)
fmt.Println(ress.RowsAffected())
d.DB.Exec(ctx, fmt.Sprintf(insertSQL, aid, cid2))
var count int
d.DB.QueryRow(ctx, "SELECT COUNT(1) FROM ugc_video WHERE aid = ? AND deleted = 0", aid).Scan(&count)
So(count, ShouldEqual, 2)
err := d.DelVideos(ctx, aid)
So(err, ShouldBeNil)
d.DB.QueryRow(ctx, "SELECT COUNT(1) FROM ugc_video WHERE aid = ? AND deleted = 0", aid).Scan(&count)
So(count, ShouldEqual, 0)
}))
}

96
app/job/main/tv/dao/ugc/import.go Normal file
View File

@@ -0,0 +1,96 @@
package ugc
import (
"context"
"fmt"
"time"
ugcmdl "go-common/app/job/main/tv/model/ugc"
"go-common/app/service/main/archive/api"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_import = "SELECT mid FROM ugc_uploader WHERE toinit = 1 AND retry < UNIX_TIMESTAMP(now()) AND deleted = 0 LIMIT "
_postponeUp = "UPDATE ugc_uploader SET retry = ? WHERE mid = ? AND deleted = 0"
_finishUp = "UPDATE ugc_uploader SET toinit = 0 WHERE mid = ? AND deleted = 0"
_filterAids = "SELECT aid FROM ugc_archive WHERE aid IN (%s) AND deleted = 0"
_importArc = "REPLACE INTO ugc_archive(aid, videos, mid, typeid, title, cover, content, duration, copyright, pubtime, state) VALUES (?,?,?,?,?,?,?,?,?,?,?)"
)
// TxImportArc imports an arc
func (d *Dao) TxImportArc(tx *sql.Tx, arc *api.Arc) (err error) {
if _, err = tx.Exec(_importArc, arc.Aid,
arc.Videos, arc.Author.Mid, arc.TypeID, arc.Title, arc.Pic, arc.Desc, arc.Duration,
arc.Copyright, arc.PubDate, arc.State); err != nil {
log.Error("_importArc, failed to update: (%v), Error: %v", arc, err)
}
return
}
// Import picks the uppers to init with the RPC data
func (d *Dao) Import(c context.Context) (res []*ugcmdl.Upper, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(c, _import+fmt.Sprintf("%d", d.conf.UgcSync.Batch.ImportNum)); err != nil {
log.Error("d.Import.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &ugcmdl.Upper{}
if err = rows.Scan(&r.MID); err != nil {
log.Error("Manual row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.Import.Query error(%v)", err)
}
return
}
// PpUpper means postpone upper init operation due to some error happened
func (d *Dao) PpUpper(c context.Context, mid int64) (err error) {
var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
if _, err = d.DB.Exec(c, _postponeUp, delay, mid); err != nil {
log.Error("PpUpper, failed to delay: (%v,%v), Error: %v", delay, mid, err)
}
return
}
// FinishUpper updates the upper's to_init status to 0 means we finish the import operation
func (d *Dao) FinishUpper(c context.Context, mid int64) (err error) {
if _, err = d.DB.Exec(c, _finishUp, mid); err != nil {
log.Error("FinishUpper, failed to Update: (%v,%v), Error: %v", _finishUp, mid, err)
}
return
}
// FilterExist filters the existing archives and remove them from the res, to have only non-existing data to insert
func (d *Dao) FilterExist(c context.Context, res *map[int64]*api.Arc, aids []int64) (err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(c, fmt.Sprintf(_filterAids, xstr.JoinInts(aids))); err != nil {
if err == sql.ErrNoRows {
err = nil // if non of them exist, it's good, we do nothing
return
}
log.Error("d._filterAids.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var aidEx int64
if err = rows.Scan(&aidEx); err != nil {
log.Error("Manual row.Scan() error(%v)", err)
return
}
delete(*res, aidEx) // remove existing data from the map
}
if err = rows.Err(); err != nil {
log.Error("d.FilterExist.Query error(%v)", err)
}
return
}

48
app/job/main/tv/dao/ugc/import_test.go Normal file
View File

@@ -0,0 +1,48 @@
package ugc
import (
"go-common/app/service/main/archive/api"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_FinishUpper(t *testing.T) {
Convey("TestDao_FinishUpper", t, WithDao(func(d *Dao) {
err := d.FinishUpper(ctx, 1)
So(err, ShouldBeNil)
}))
}
func TestDao_Import(t *testing.T) {
Convey("TestDao_Import", t, WithDao(func(d *Dao) {
res, err := d.Import(ctx)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
}))
}
func TestDao_PpUpper(t *testing.T) {
Convey("TestDao_PpUpper", t, WithDao(func(d *Dao) {
err := d.PpUpper(ctx, 1)
So(err, ShouldBeNil)
}))
}
func TestDao_BeginTran(t *testing.T) {
Convey("TestDao_BeginTran", t, WithDao(func(d *Dao) {
tx, err := d.BeginTran(ctx)
So(err, ShouldBeNil)
tx.Rollback()
}))
}
func TestDao_FilterExist(t *testing.T) {
Convey("TestDao_FilterExist", t, WithDao(func(d *Dao) {
pMap := make(map[int64]*api.Arc)
pAids := []int64{10106351, 10106309, 10106308, 10106307, 10106306, 10106284, 10105807, 10101484, 10100856, 10100855, 10100486, 10100146, 10100145, 10100144,
10100044, 10099332, 10099167, 10099150, 10099149, 10098970}
err := d.FilterExist(ctx, &pMap, pAids)
So(err, ShouldBeNil)
}))
}

93
app/job/main/tv/dao/ugc/manual.go Normal file
View File

@@ -0,0 +1,93 @@
package ugc
import (
"context"
"fmt"
"time"
ugcmdl "go-common/app/job/main/tv/model/ugc"
arccli "go-common/app/service/main/archive/api"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_manual = "SELECT id,aid FROM ugc_archive WHERE manual = 1 AND retry < UNIX_TIMESTAMP(now()) AND deleted = 0 LIMIT "
_postpone = "UPDATE ugc_archive SET retry = ? WHERE aid = ? AND deleted = 0"
_importFinish = "UPDATE ugc_archive SET manual = 0 WHERE aid = ? AND deleted = 0"
_manualArc = "UPDATE ugc_archive SET videos = ?, mid = ?, typeid = ?, title = ?, cover = ?, content = ?, duration = ?, " +
"copyright = ?, pubtime = ?, state = ?, submit = ? WHERE aid = ? AND deleted = 0"
_autoArc = "REPLACE INTO ugc_archive (videos, mid, typeid, title, cover, content, duration, copyright, pubtime, state, submit, aid) VALUES " +
"(?,?,?,?,?,?,?,?,?,?,?,?)"
_importVideo = "REPLACE INTO ugc_video (aid,cid,eptitle,index_order,duration,description) VALUES (?,?,?,?,?,?)"
)
// TxMnlArc updates the db with data from API
func (d *Dao) TxMnlArc(tx *sql.Tx, arc *ugcmdl.Archive) (err error) {
if _, err = tx.Exec(_manualArc,
arc.Videos, arc.MID, arc.TypeID, arc.Title, arc.Cover, arc.Content, arc.Duration,
arc.Copyright, arc.Pubtime, arc.State, _needSubmit, arc.AID); err != nil {
log.Error("_importArc, failed to update: (%v), Error: %v", arc, err)
}
return
}
// TxAutoArc imports the db an arc
func (d *Dao) TxAutoArc(tx *sql.Tx, arc *ugcmdl.Archive) (err error) {
if _, err = tx.Exec(_autoArc,
arc.Videos, arc.MID, arc.TypeID, arc.Title, arc.Cover, arc.Content, arc.Duration,
arc.Copyright, arc.Pubtime, arc.State, _needSubmit, arc.AID); err != nil {
log.Error("TxAutoArc, failed to update: (%v), Error: %v", arc, err)
}
return
}
// TxMnlVideos updates the db with data from API, if the
func (d *Dao) TxMnlVideos(tx *sql.Tx, view *arccli.ViewReply) (err error) {
for _, v := range view.Pages {
if _, err = tx.Exec(_importVideo, view.Arc.Aid, v.Cid, v.Part, v.Page, v.Duration, v.Desc); err != nil {
log.Error("_importArc, failed to insert: (%v), Error: %v", v, err)
return
}
}
return
}
// TxMnlStatus updates the aid's manual status to 0
func (d *Dao) TxMnlStatus(tx *sql.Tx, aid int64) (err error) {
if _, err = tx.Exec(_importFinish, aid); err != nil {
log.Error("_importFinish, failed to update: (%v), Error: %v", aid, err)
}
return
}
// Manual picks the archives that added manually
func (d *Dao) Manual(c context.Context) (res []*ugcmdl.Archive, err error) {
var rows *sql.Rows
if rows, err = d.DB.Query(c, _manual+fmt.Sprintf("%d", d.conf.UgcSync.Batch.ManualNum)); err != nil {
log.Error("d.Import.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &ugcmdl.Archive{}
if err = rows.Scan(&r.ID, &r.AID); err != nil {
log.Error("Manual row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.Manual.Query error(%v)", err)
}
return
}
// Ppmnl means postpone manual operation due to some error happened
func (d *Dao) Ppmnl(c context.Context, aid int64) (err error) {
var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
if _, err = d.DB.Exec(c, _postpone, delay, aid); err != nil {
log.Error("Ppmnl, failed to delay: (%v,%v), Error: %v", delay, aid, err)
}
return
}

47
app/job/main/tv/dao/ugc/manual_test.go Normal file
View File

@@ -0,0 +1,47 @@
package ugc
import (
"fmt"
"testing"
"go-common/app/job/main/tv/model/ugc"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_Manual(t *testing.T) {
Convey("TestDao_Manual", t, WithDao(func(d *Dao) {
res, err := d.Manual(ctx)
for _, v := range res {
fmt.Println(v)
}
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
}))
}
func TestDao_Ppmnl(t *testing.T) {
Convey("TestDao_Ppmnl", t, WithDao(func(d *Dao) {
err := d.Ppmnl(ctx, 10099763)
So(err, ShouldBeNil)
}))
}
func TestDao_UpdateArc(t *testing.T) {
Convey("TestDao_UpdateArc", t, WithDao(func(d *Dao) {
err := d.UpdateArc(ctx, &ugc.ArchDatabus{
Aid: 10099763,
Mid: 452156,
Videos: 1,
TypeID: 174,
Title: "test",
Cover: "testPic",
Content: "testDesc",
Duration: 300,
Copyright: 1,
PubTime: "2018-06-05",
State: 5,
})
So(err, ShouldBeNil)
}))
}

176
app/job/main/tv/dao/ugc/media_cache.go Normal file
View File

@@ -0,0 +1,176 @@
package ugc
import (
"context"
"fmt"
ugcmdl "go-common/app/job/main/tv/model/ugc"
"go-common/library/cache/memcache"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_mcArcCMSKey = "arc_cms_%d"
_mcVideoCMSKey = "video_cms_%d"
_totalArcs = "SELECT count(1) FROM ugc_archive WHERE mid = ?"
_totalVideos = "SELECT count(1) FROM ugc_video"
_activeVideos = "SELECT count(1) FROM ugc_video WHERE aid = ? AND deleted = 0"
_cntArcVideo = "SELECT count(1) FROM ugc_video WHERE aid = ?"
_pickArcMC = "SELECT title, aid, content, cover, typeid, pubtime, videos, valid, deleted, result,copyright, state, mid,duration FROM ugc_archive " +
"WHERE mid = %d LIMIT %d,%d"
_pickArcVideoMC = "SELECT cid, eptitle, aid, index_order, valid, deleted, result FROM ugc_video " +
"WHERE aid = ? AND cid > ? ORDER BY cid LIMIT 0,%d"
_transFailVideos = "SELECT cid FROM ugc_video WHERE aid = ? AND cid > %d AND transcoded = 2 and deleted = 0"
)
func arcCMSCacheKey(sid int64) string {
return fmt.Sprintf(_mcArcCMSKey, sid)
}
func videoCMSCacheKey(sid int) string {
return fmt.Sprintf(_mcVideoCMSKey, sid)
}
// SetArcCMS in MC
func (d *Dao) SetArcCMS(ctx context.Context, res *ugcmdl.ArcCMS) (err error) {
var (
key = arcCMSCacheKey(res.AID)
conn = d.mc.Get(ctx)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: res, Expiration: d.mcExpire, Flags: memcache.FlagJSON}); err != nil {
log.Error("conn.Set(%s,%v) error(%v)", key, res, err)
}
return
}
// SetVideoCMS in MC
func (d *Dao) SetVideoCMS(ctx context.Context, res *ugcmdl.VideoCMS) (err error) {
var (
key = videoCMSCacheKey(res.CID)
conn = d.mc.Get(ctx)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: res, Expiration: d.mcExpire, Flags: memcache.FlagJSON}); err != nil {
log.Error("conn.Set(%s,%v) error(%v)", key, res, err)
}
return
}
// UpArcsCnt counts the total number of arcs, including the deleted ones
func (d *Dao) UpArcsCnt(c context.Context, mid int64) (count int, err error) {
if err = d.DB.QueryRow(c, _totalArcs, mid).Scan(&count); err != nil {
log.Error("d.UpArcsCnt.Query error(%v)", err)
}
return
}
// TotalVideos counts the total number of arcs, including the deleted ones
func (d *Dao) TotalVideos(c context.Context) (count int, err error) {
if err = d.DB.QueryRow(c, _totalVideos).Scan(&count); err != nil {
log.Error("d.TotalVideos.Query error(%v)", err)
}
return
}
// ActVideos checks whether there is still some active videos
func (d *Dao) ActVideos(c context.Context, aid int64) (has bool, err error) {
var count int
if err = d.DB.QueryRow(c, _activeVideos, aid).Scan(&count); err != nil {
log.Error("d.TotalVideos.Query error(%v)", err)
return
}
if count == 0 {
has = false
} else {
has = true
}
return
}
// ArcVideoCnt counts one archive's video
func (d *Dao) ArcVideoCnt(c context.Context, aid int64) (count int, err error) {
if err = d.DB.QueryRow(c, _cntArcVideo, aid).Scan(&count); err != nil {
log.Error("d.TotalVideos.Query error(%v)", err)
}
return
}
// PickUpArcs picks data by Piece to sync in MC, attention: nbPiece begins from zero
func (d *Dao) PickUpArcs(ctx context.Context, mid, nbPiece, nbData int) (res []*ugcmdl.ArcFull, err error) {
var (
rows *sql.Rows
query = fmt.Sprintf(_pickArcMC, mid, nbPiece*nbData, nbData)
)
if rows, err = d.DB.Query(ctx, query); err != nil {
log.Error("d.PickUpArcs.Query: %s error(%v)", query, err)
return
}
defer rows.Close()
for rows.Next() {
var li = &ugcmdl.ArcFull{}
// SELECT title, aid, content, cover, typeid, pubtime, videos, valid, deleted, result,copyright, state, mid,duration
if err = rows.Scan(&li.Title, &li.AID, &li.Content, &li.Cover, &li.TypeID, &li.Pubtime,
&li.Videos, &li.Valid, &li.Deleted, &li.Result, &li.Copyright, &li.State, &li.MID, &li.Duration); err != nil {
log.Error("PickUpArcs row.Scan() error(%v)", err)
return
}
res = append(res, li)
}
if err = rows.Err(); err != nil {
log.Error("d.PickUpArcs.Query error(%v)", err)
}
return
}
// PickArcVideo picks data by Piece to sync in MC
func (d *Dao) PickArcVideo(ctx context.Context, aid int64, LastID int, nbData int) (res []*ugcmdl.VideoCMS, myLast int, err error) {
var (
rows *sql.Rows
query = fmt.Sprintf(_pickArcVideoMC, nbData)
)
if rows, err = d.DB.Query(ctx, query, aid, LastID); err != nil {
log.Error("d._pickArcVideoMC.Query: %s error(%v)", query, err)
return
}
defer rows.Close()
for rows.Next() {
var li = new(ugcmdl.VideoCMS)
if err = rows.Scan(&li.CID, &li.Title, &li.AID, &li.IndexOrder, &li.Valid, &li.Deleted, &li.Result); err != nil {
log.Error("PickVideoMC row.Scan() error(%v)", err)
return
}
res = append(res, li)
myLast = li.CID
}
if err = rows.Err(); err != nil {
log.Error("d.PickArcVideo.Query error(%v)", err)
}
return
}
// TransFailVideos picks transcoding failure videos
func (d *Dao) TransFailVideos(ctx context.Context, aid int64) (cids []int64, err error) {
var (
rows *sql.Rows
query = fmt.Sprintf(_transFailVideos, d.criCID)
)
if rows, err = d.DB.Query(ctx, query, aid); err != nil {
log.Error("d.TransFailVideos.Query: Cid %d error(%v)", aid, err)
return
}
defer rows.Close()
for rows.Next() {
var li int64
if err = rows.Scan(&li); err != nil {
log.Error("PickVideoMC row.Scan() error(%v)", err)
return
}
cids = append(cids, li)
}
if err = rows.Err(); err != nil {
log.Error("d.TransFailVideos.Query error(%v)", err)
}
return
}

132
app/job/main/tv/dao/ugc/media_cache_test.go Normal file
View File

@@ -0,0 +1,132 @@
package ugc
import (
"context"
"encoding/json"
"fmt"
"testing"
ugcmdl "go-common/app/job/main/tv/model/ugc"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_TotalVideos(t *testing.T) {
Convey("TestDao_TotalVideos", t, WithDao(func(d *Dao) {
res, err := d.TotalVideos(ctx)
So(err, ShouldBeNil)
So(res, ShouldBeGreaterThan, 0)
fmt.Println(res)
}))
}
func TestDao_ArcVideoCnt(t *testing.T) {
Convey("TestDao_ArcVideoCnt", t, WithDao(func(d *Dao) {
var aid int64
d.DB.QueryRow(ctx, "select aid from ugc_video where deleted = 0 limit 1").Scan(&aid)
if aid == 0 {
fmt.Println("empty arc")
return
}
cnt, errcnt := d.ArcVideoCnt(ctx, aid)
So(errcnt, ShouldBeNil)
fmt.Println(aid)
So(cnt, ShouldBeGreaterThan, 0)
fmt.Println(cnt)
resVideos, lastIDVideo, errVd := d.PickArcVideo(ctx, aid, 0, 10)
So(errVd, ShouldBeNil)
So(len(resVideos), ShouldBeGreaterThan, 0)
So(lastIDVideo, ShouldBeGreaterThan, 0)
str, _ := json.Marshal(resVideos)
fmt.Println(string(str))
}))
}
func TestDao_SetArc(t *testing.T) {
Convey("TestDao_SetArc", t, WithDao(func(d *Dao) {
err := d.SetArcCMS(ctx, &ugcmdl.ArcCMS{
Title: "testtest",
AID: 777,
})
So(err, ShouldBeNil)
}))
}
func TestDao_UpArcsCnt(t *testing.T) {
Convey("TestDao_UpArcsCnt", t, WithDao(func(d *Dao) {
var mid int64
d.DB.QueryRow(ctx, "select mid from ugc_archive where deleted = 0 limit 1").Scan(&mid)
if mid == 0 {
fmt.Println("empty arc")
return
}
count, err := d.UpArcsCnt(ctx, mid)
So(err, ShouldBeNil)
So(count, ShouldBeGreaterThan, 0)
fmt.Println("mid ", mid, " cnt", count)
if count > 1 {
d.DB.Exec(context.Background(), "update ugc_archive set deleted = 1 where mid = ? and deleted = 0 limit 1", mid)
cntNonDeleted, err2 := d.CountUpArcs(context.Background(), mid)
So(err2, ShouldBeNil)
So(count, ShouldBeGreaterThan, cntNonDeleted)
fmt.Println("all: ", count, " non-deleted: ", cntNonDeleted)
}
}))
}
func TestDao_TransFailVideos(t *testing.T) {
Convey("TestDao_TransFailVideos", t, WithDao(func(d *Dao) {
query := "SELECT aid FROM ugc_video WHERE cid > 12780000 AND transcoded = 2 and deleted = 0 limit 1"
var aid int64
d.DB.QueryRow(context.Background(), query).Scan(&aid)
if aid == 0 {
fmt.Println("Empty archives")
return
}
cids, err := d.TransFailVideos(ctx, aid)
So(err, ShouldBeNil)
So(len(cids), ShouldBeGreaterThan, 0)
fmt.Println("aid ", aid, " cids ", cids)
}))
}
func TestDao_ActVideos(t *testing.T) {
Convey("TestDao_ActVideos", t, WithDao(func(d *Dao) {
var (
aid = int64(88888888)
cid = 99999999
)
insertSQL := "REPLACE INTO ugc_video (aid, cid, deleted) VALUES (%d, %d, 1)"
d.DB.Exec(ctx, fmt.Sprintf(insertSQL, aid, cid))
has, err := d.ActVideos(ctx, aid)
So(err, ShouldBeNil)
So(has, ShouldBeFalse)
d.DB.Exec(ctx, "UPDATE ugc_video SET deleted = 0 WHERE cid = ?", cid)
has, err = d.ActVideos(ctx, aid)
So(err, ShouldBeNil)
So(has, ShouldBeTrue)
}))
}
func TestDao_PickArcMC(t *testing.T) {
Convey("TestDao_PickArcMC", t, WithDao(func(d *Dao) {
pickMid := "select mid from ugc_archive where deleted = 0 group by mid order by count(aid) desc limit 1"
var mid = 0
d.DB.QueryRow(ctx, pickMid).Scan(&mid)
if mid == 0 {
fmt.Println("empty archive")
return
}
fmt.Println("mid ", mid)
res1, err1 := d.PickUpArcs(ctx, mid, 0, 5)
So(err1, ShouldBeNil)
So(len(res1), ShouldBeGreaterThan, 0)
res2, err2 := d.PickUpArcs(ctx, mid, 7, 5)
So(err2, ShouldBeNil)
So(len(res2), ShouldBeGreaterThan, 0)
str1, _ := json.Marshal(res1)
str2, _ := json.Marshal(res2)
fmt.Println(string(str1))
fmt.Println(string(str2))
}))
}

44
app/job/main/tv/dao/ugc/passed.go Normal file
View File

@@ -0,0 +1,44 @@
package ugc
import (
"context"
"fmt"
"go-common/app/job/main/tv/model/common"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_passedArc = "SELECT aid, ctime FROM ugc_archive WHERE typeid IN (%s) AND result = 1 AND valid = 1 and deleted = 0"
)
// PassedArcs picks the passed Arc data for Idx Page
func (d *Dao) PassedArcs(c context.Context, tids []int64) (res []*common.IdxRank, err error) {
if len(tids) == 0 {
return
}
var (
rows *sql.Rows
tidsStr = xstr.JoinInts(tids)
)
if rows, err = d.DB.Query(c, fmt.Sprintf(_passedArc, tidsStr)); err != nil {
log.Error("d.PassedArcs.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = &common.IdxRank{}
if err = rows.Scan(&r.ID, &r.Ctime); err != nil {
log.Error("PassedArcs row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.PassedArcs.Query error(%v)", err)
}
return
}

19
app/job/main/tv/dao/ugc/passed_test.go Normal file
View File

@@ -0,0 +1,19 @@
package ugc
import (
"encoding/json"
"fmt"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_PassedArcs(t *testing.T) {
Convey("TestDao_PassedArcs", t, WithDao(func(d *Dao) {
res, err := d.PassedArcs(ctx, []int64{76, 75})
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
data, _ := json.Marshal(res)
fmt.Println(string(data))
}))
}

54
app/job/main/tv/dao/ugc/report_cid.go Normal file
View File

@@ -0,0 +1,54 @@
package ugc
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
appDao "go-common/app/job/main/tv/dao/app"
ugcmdl "go-common/app/job/main/tv/model/ugc"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_reported = 1
_finishReport = "UPDATE ugc_video SET mark = %d WHERE cid IN (%s)"
)
// RepCidBatch reports cid info to VideoCloud api
func (d *Dao) RepCidBatch(c context.Context, cidReq []*ugcmdl.CidReq) (err error) {
var (
reportURL = d.conf.UgcSync.Cfg.ReportCidURL
resp = ugcmdl.CidResp{}
jsonBody []byte
)
if jsonBody, err = json.Marshal(cidReq); err != nil {
log.Error("json.Marchal(%v) error(%v)", cidReq, err)
return
}
req, err := http.NewRequest(http.MethodPost, reportURL, bytes.NewBuffer(jsonBody))
if err != nil {
log.Error("http.NewRequest err - %v", err)
return
}
if err = d.client.Do(c, req, &resp); err != nil {
log.Error("ReportCid jsonBody (%s) error(%v)", string(jsonBody), err)
return
}
if resp.Code != 0 {
return fmt.Errorf("ReportCid Error Code:%d, Msg:%s", resp.Code, resp.Message)
}
appDao.PromInfo("ReportCid:Succ")
return
}
// FinishReport change's the ugc_video's state from 0 to 1, means it has already been reported
func (d *Dao) FinishReport(c context.Context, cids []int64) (err error) {
if _, err = d.DB.Exec(c, fmt.Sprintf(_finishReport, _reported, xstr.JoinInts(cids))); err != nil {
log.Error("FinishReport Error: %v", cids, err)
}
return
}

55
app/job/main/tv/dao/ugc/search_con.go Normal file
View File

@@ -0,0 +1,55 @@
package ugc
import (
"context"
model "go-common/app/job/main/tv/model/pgc"
"go-common/library/database/sql"
"go-common/library/ecode"
"go-common/library/log"
)
const (
_UgcFromwhere = "FROM ugc_archive WHERE result=1 AND valid=1 AND deleted=0 "
_UgcCont = "SELECT aid,title,cover,`content`,pubtime,typeid " + _UgcFromwhere + "AND aid > ? ORDER BY aid ASC LIMIT ?"
_UgcContCount = " SELECT count(*) " + _UgcFromwhere
)
// UgcCont is used for getting valid ugc archive data
func (d *Dao) UgcCont(ctx context.Context, aid int, limit int) (res []*model.SearUgcCon, maxID int, err error) {
var (
rows *sql.Rows
)
if rows, err = d.DB.Query(ctx, _UgcCont, aid, limit); err != nil {
log.Error("d.UgcCont.Query: %s error(%v)", _UgcCont, err)
return
}
defer rows.Close()
for rows.Next() {
var r = &model.SearUgcCon{}
if err = rows.Scan(&r.AID, &r.Title, &r.Cover, &r.Content, &r.Pubtime, &r.Typeid); err != nil {
log.Error("UgcCont row.Scan() error(%v)", err)
return
}
res = append(res, r)
}
if err = rows.Err(); err != nil {
log.Error("d.UgcCont.Query error(%v)", err)
return
}
if len(res) == 0 {
err = ecode.NothingFound
return
}
maxID = res[len(res)-1].AID
return
}
// UgcCnt is used for getting valid data count
func (d *Dao) UgcCnt(ctx context.Context) (upCnt int, err error) {
row := d.DB.QueryRow(ctx, _UgcContCount)
if err = row.Scan(&upCnt); err != nil {
log.Error("d.SeaContCount.Query: %s error(%v)", _UgcContCount, err)
}
return
}

42
app/job/main/tv/dao/ugc/search_con_test.go Normal file
View File

@@ -0,0 +1,42 @@
package ugc
import (
"fmt"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_UgcCont(t *testing.T) {
Convey("TestDao_UgcCont", t, WithDao(func(d *Dao) {
var aid int
d.DB.QueryRow(ctx, queryAid).Scan(&aid)
if aid == 0 {
fmt.Println("No ready audit Data")
return
}
if res, maxID, err := d.UgcCont(ctx, aid, 50); err != nil {
fmt.Println(err)
So(len(res), ShouldEqual, 0)
So(maxID, ShouldBeZeroValue)
} else {
So(len(res), ShouldBeGreaterThan, 0)
for _, v := range res {
fmt.Println(v)
}
So(maxID, ShouldBeGreaterThan, 0)
}
}))
}
func TestDao_UgcContCount(t *testing.T) {
Convey("TestDao_UgcContCount", t, WithDao(func(d *Dao) {
var aid int
d.DB.QueryRow(ctx, queryAid).Scan(&aid)
if aid != 0 {
res, _ := d.UgcCnt(ctx)
So(res, ShouldBeGreaterThan, 0)
fmt.Println(res)
}
}))
}

15
app/job/main/tv/dao/ugc/sync_arc.go Normal file
View File

@@ -0,0 +1,15 @@
package ugc
import (
"context"
"go-common/library/log"
)
// FinishArc updates the submit status from 1 to 0
func (d *Dao) FinishArc(c context.Context, aid int64) (err error) {
if _, err = d.DB.Exec(c, _finishArc, aid); err != nil {
log.Error("FinishVideos Error: %v", aid, err)
}
return
}

115
app/job/main/tv/dao/ugc/sync_video.go Normal file
View File

@@ -0,0 +1,115 @@
package ugc
import (
"context"
"fmt"
ugcmdl "go-common/app/job/main/tv/model/ugc"
"go-common/library/database/sql"
"go-common/library/log"
"time"
)
const (
_videoCond = " AND (v.transcoded = 1 OR v.cid <= %d) " +
"AND v.retry < unix_timestamp(now()) " +
"AND v.deleted = 0 "
_parseVideos = "SELECT id,cid,index_order,eptitle,duration,description FROM ugc_video v " +
"WHERE v.aid = ? AND v.submit = 1 " + _videoCond
_postArc = "UPDATE ugc_video SET retry = ? WHERE cid = ? AND deleted = 0"
_finishVideos = "UPDATE ugc_video SET submit = 0 WHERE cid = ? AND aid = ? AND deleted = 0"
_finishArc = "UPDATE ugc_archive SET submit = 0 WHERE aid = ? AND deleted = 0"
_parseArc = "SELECT id,aid,mid,typeid,videos,title,cover,content,duration,copyright,pubtime,ctime,mtime,state," +
"manual,valid,submit,retry,result,deleted FROM ugc_archive WHERE aid = ?"
_shouldAudit = "SELECT COUNT(1) as cnt FROM ugc_video v WHERE v.aid = ? AND v.submit = 1 " + _videoCond
_videoSubmit = "SELECT cid FROM ugc_video v WHERE v.aid = ? AND v.submit = 0 " + _videoCond + " LIMIT 1"
)
// PpVideos postpones the archive's videos submit in 30 mins
func (d *Dao) PpVideos(c context.Context, cids []int64) (err error) {
var delay = time.Now().Unix() + int64(d.conf.UgcSync.Frequency.ErrorWait)
for _, v := range cids {
if _, err = d.DB.Exec(c, _postArc, delay, v); err != nil {
log.Error("PostponeArc, failed to delay: (%v,%v), Error: %v", delay, v, err)
return
}
}
return
}
// FinishVideos updates the submit status from 1 to 0
func (d *Dao) FinishVideos(c context.Context, videos []*ugcmdl.SimpleVideo, aid int64) (err error) {
for _, v := range videos {
if _, err = d.DB.Exec(c, _finishVideos, v.CID, aid); err != nil { // avoid updating the cid under another archive
log.Error("FinishVideos Error: %v", v.CID, err)
return
}
}
if _, err = d.DB.Exec(c, _finishArc, aid); err != nil {
log.Error("FinishVideos Error: %v", aid, err)
}
return
}
// ParseArc parses one archive data
func (d *Dao) ParseArc(c context.Context, aid int64) (res *ugcmdl.Archive, err error) {
res = &ugcmdl.Archive{}
if err = d.DB.QueryRow(c, _parseArc, aid).Scan(&res.ID, &res.AID, &res.MID, &res.TypeID, &res.Videos, &res.Title,
&res.Cover, &res.Content, &res.Duration, &res.Copyright, &res.Pubtime, &res.Ctime, &res.Mtime, &res.State,
&res.Manual, &res.Valid, &res.Submit, &res.Retry, &res.Result, &res.Deleted); err != nil { // get the qualified aid to sync
log.Warn("d.ParseArc.Query error(%v)", err)
}
return
}
// ShouldAudit distinguishes whether the archive should ask for audit or not
func (d *Dao) ShouldAudit(c context.Context, aid int64) (res bool, err error) {
var cnt int
if err = d.DB.QueryRow(c, fmt.Sprintf(_shouldAudit, d.criCID), aid).Scan(&cnt); err != nil {
log.Error("d.ShouldAudit Aid %d Err %v", aid, err)
return
}
res = cnt > 0
return
}
// VideoSubmit tells whether the archive already has some video submitted
func (d *Dao) VideoSubmit(c context.Context, aid int64) (cid int64, err error) {
if err = d.DB.QueryRow(c, fmt.Sprintf(_videoSubmit, d.criCID), aid).Scan(&cid); err != nil {
log.Warn("d.videoSubmit Aid %d, Err %v", aid, err)
}
return
}
// ParseVideos picks 20 videos of one qualified archive
func (d *Dao) ParseVideos(c context.Context, aid int64, ps int) (res [][]*ugcmdl.SimpleVideo, err error) {
var (
rows *sql.Rows
videos []*ugcmdl.SimpleVideo
)
if rows, err = d.DB.Query(c, fmt.Sprintf(_parseVideos, d.criCID), aid); err != nil {
log.Error("d._parseVideos.Query error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
var r = ugcmdl.SimpleVideo{}
if err = rows.Scan(&r.ID, &r.CID, &r.IndexOrder, &r.Eptitle, &r.Duration, &r.Description); err != nil {
log.Error("ParseVideos row.Scan() error(%v)", err)
return
}
videos = append(videos, &r)
if len(videos) >= ps {
var videoPce = append([]*ugcmdl.SimpleVideo{}, videos...)
videos = []*ugcmdl.SimpleVideo{}
res = append(res, videoPce)
}
}
if err = rows.Err(); err != nil {
log.Error("d._parseVideos.Query error(%v)", err)
return
}
if len(videos) > 0 {
res = append(res, videos)
}
return
}

114
app/job/main/tv/dao/ugc/sync_video_test.go Normal file
View File

@@ -0,0 +1,114 @@
package ugc
import (
"context"
ugcmdl "go-common/app/job/main/tv/model/ugc"
"testing"
"fmt"
"encoding/json"
"github.com/smartystreets/goconvey/convey"
)
func TestUgcPpVideos(t *testing.T) {
var (
c = context.Background()
videos = []int64{}
)
convey.Convey("PpVideos", t, func(ctx convey.C) {
err := d.PpVideos(c, videos)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestUgcFinishVideos(t *testing.T) {
var (
c = context.Background()
videos = []*ugcmdl.SimpleVideo{}
aid = int64(0)
)
convey.Convey("FinishVideos", t, func(ctx convey.C) {
err := d.FinishVideos(c, videos, aid)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestDao_ParseArc(t *testing.T) {
convey.Convey("TestDao_ParseArc", t, WithDao(func(d *Dao) {
var (
query = "SELECT aid FROM ugc_archive WHERE deleted = 0 LIMIT 1"
arc = &ugcmdl.ArcCMS{}
)
d.DB.QueryRow(ctx, query).Scan(&arc.AID)
res, err := d.ParseArc(ctx, int64(arc.AID))
convey.So(err, convey.ShouldBeNil)
convey.So(res, convey.ShouldNotBeNil)
fmt.Println(res)
}))
}
func TestDao_ParseVideos(t *testing.T) {
convey.Convey("TestDao_ParseVideos", t, WithDao(func(d *Dao) {
var (
query = "SELECT aid FROM ugc_video WHERE deleted = 0 AND submit = 1 AND cid < 12780000 LIMIT 1"
aid int64
)
if err := d.DB.QueryRow(ctx, query).Scan(&aid); err != nil {
fmt.Println("No to submit data")
return
}
fmt.Println(aid)
res, err := d.ParseVideos(ctx, int64(aid), 2)
convey.So(err, convey.ShouldBeNil)
convey.So(len(res), convey.ShouldBeGreaterThan, 0)
str, _ := json.Marshal(res)
fmt.Println(string(str))
for _, v := range res {
convey.So(len(v), convey.ShouldBeGreaterThan, 0)
}
}))
}
func TestDao_ShouldAudit(t *testing.T) {
convey.Convey("TestDao_ShouldAudit", t, WithDao(func(d *Dao) {
var (
query = "SELECT aid FROM ugc_video v WHERE v.submit = 1 " + _videoCond + " LIMIT 1"
aid int64
)
fmt.Println(query)
d.DB.QueryRow(ctx, fmt.Sprintf(query, d.criCID)).Scan(&aid)
if aid == 0 {
fmt.Println("db empty")
return
}
res, err := d.ShouldAudit(ctx, aid)
convey.So(err, convey.ShouldBeNil)
convey.So(res, convey.ShouldBeTrue)
fmt.Println(aid, " ", res)
}))
}
func TestDao_VideoSubmit(t *testing.T) {
convey.Convey("TestDao_VideoSubmit", t, WithDao(func(d *Dao) {
var (
query = "SELECT aid FROM ugc_video v WHERE v.submit = 0 " + _videoCond + " LIMIT 1"
aid int64
)
fmt.Println(query)
d.DB.QueryRow(ctx, fmt.Sprintf(query, d.criCID)).Scan(&aid)
if aid == 0 {
fmt.Println("db empty")
return
}
res, err := d.VideoSubmit(ctx, aid)
convey.So(err, convey.ShouldBeNil)
convey.So(res, convey.ShouldNotBeNil)
fmt.Println(aid, " ", res)
}))
}

63
app/job/main/tv/dao/upper/BUILD Normal file
View File

@@ -0,0 +1,63 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"load_upper_test.go",
"refresh_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/ugc:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"load_upper.go",
"refresh.go",
"sync.go",
],
importpath = "go-common/app/job/main/tv/dao/upper",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/tv/conf:go_default_library",
"//app/job/main/tv/model/ugc:go_default_library",
"//app/service/main/account/api:go_default_library",
"//app/service/main/account/model:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/stat/prom:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

57
app/job/main/tv/dao/upper/dao.go Normal file
View File

@@ -0,0 +1,57 @@
package upper
import (
"context"
"time"
"go-common/app/job/main/tv/conf"
acccli "go-common/app/service/main/account/api"
accwar "go-common/app/service/main/account/api"
account "go-common/app/service/main/account/model"
"go-common/library/cache/memcache"
"go-common/library/database/sql"
"go-common/library/stat/prom"
"github.com/pkg/errors"
)
var (
missedCount = prom.CacheMiss
cachedCount = prom.CacheHit
)
// Dao is account dao.
type Dao struct {
accClient accwar.AccountClient
mc *memcache.Pool
mcExpire int32
DB *sql.DB
}
// New account dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
mc: memcache.NewPool(c.Memcache.Config),
mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
DB: sql.NewMySQL(c.Mysql),
}
var err error
if d.accClient, err = acccli.NewClient(c.AccClient); err != nil {
panic(err)
}
return
}
// Card3 get card info by mid
func (d *Dao) Card3(c context.Context, mid int64) (res *account.Card, err error) {
var (
arg = &accwar.MidReq{Mid: mid}
resp *accwar.CardReply
)
if resp, err = d.accClient.Card3(c, arg); err != nil {
err = errors.Wrapf(err, "%v", arg)
return
}
res = resp.Card
return
}

98
app/job/main/tv/dao/upper/dao_test.go Normal file
View File

@@ -0,0 +1,98 @@
package upper
import (
"context"
"flag"
"fmt"
"path/filepath"
"testing"
"go-common/app/job/main/tv/conf"
ugcMdl "go-common/app/job/main/tv/model/ugc"
. "github.com/smartystreets/goconvey/convey"
)
var (
ctx = context.TODO()
d *Dao
)
func WithDao(f func(d *Dao)) func() {
return func() {
dir, _ := filepath.Abs("../../cmd/tv-job-test.toml")
flag.Set("conf", dir)
conf.Init()
if d == nil {
d = New(conf.Conf)
}
f(d)
}
}
func TestDao_LoadUpMeta(t *testing.T) {
Convey("TestDao_LoadUpMeta", t, WithDao(func(d *Dao) {
res, err := d.LoadUpMeta(ctx, 88895270)
So(err, ShouldBeNil)
fmt.Println(res)
}))
}
func TestDao_CountUP(t *testing.T) {
Convey("TestDao_CountUP", t, WithDao(func(d *Dao) {
res, err := d.CountUP(ctx)
So(err, ShouldBeNil)
fmt.Println(res)
}))
}
func TestDao_PickUppers(t *testing.T) {
Convey("TestDao_PickUppers", t, WithDao(func(d *Dao) {
res, myLast, err := d.PickUppers(ctx, 0, 50)
So(err, ShouldBeNil)
So(myLast, ShouldBeGreaterThan, 0)
So(len(res), ShouldBeGreaterThan, 0)
fmt.Println(myLast)
}))
}
func TestDao_RefreshUp(t *testing.T) {
Convey("TestDao_RefreshUp", t, WithDao(func(d *Dao) {
res, _, _ := d.PickUppers(ctx, 0, 50)
if len(res) > 0 {
err := d.RefreshUp(ctx, &ugcMdl.ReqSetUp{
MID: res[0],
Value: "Face666",
UpType: _upFace,
})
So(err, ShouldBeNil)
}
}))
}
func TestDao_TosyncUps(t *testing.T) {
Convey("TestDao_TosyncUps", t, WithDao(func(d *Dao) {
mids, err := d.TosyncUps(ctx)
So(err, ShouldBeNil)
fmt.Println(mids)
}))
}
func TestDao_FinsyncUps(t *testing.T) {
Convey("TestDao_FinsyncUps", t, WithDao(func(d *Dao) {
err := d.FinsyncUps(ctx, 88895270)
So(err, ShouldBeNil)
}))
}
func TestDao_IsSame(t *testing.T) {
Convey("TestDao_IsSame", t, WithDao(func(d *Dao) {
var u = &ugcMdl.Upper{
OriFace: "http://i0.hdslb.com/bfs/face/589568b0b0228d5a38b2e6f14b83bcb174637140.jpg",
}
var faceSame, _ = u.IsSame("", "http://i1.hdslb.com/bfs/face/589568b0b0228d5a38b2e6f14b83bcb174637140.jpg")
So(faceSame, ShouldBeTrue)
faceSame, _ = u.IsSame("", "http://i0.hdslb.com/bfs/face/xxx.jpg")
So(faceSame, ShouldBeFalse)
}))
}

101
app/job/main/tv/dao/upper/load_upper.go Normal file
View File

@@ -0,0 +1,101 @@
package upper
import (
"context"
"database/sql"
"fmt"
ugcMdl "go-common/app/job/main/tv/model/ugc"
"go-common/library/cache/memcache"
xsql "go-common/library/database/sql"
"go-common/library/ecode"
"go-common/library/log"
)
const (
_loadUpper = "SELECT mid,toinit,submit,ori_name,cms_name,ori_face,cms_face,valid,deleted FROM ugc_uploader WHERE mid = ?"
)
func upperMetaKey(MID int64) string {
return fmt.Sprintf("up_cms_%d", MID)
}
// LoadUpMeta loads the upper meta cms data from cache, for missed ones, pick them from the DB
func (d *Dao) LoadUpMeta(ctx context.Context, mid int64) (upper *ugcMdl.Upper, err error) {
if upper, err = d.upMetaCache(ctx, mid); err != nil { // mc error
log.Error("LoadUpMeta Get Mid [%d] from CMS Error (%v)", mid, err)
return
}
if upper != nil { // mc found
return
}
if upper, err = d.upMetaDB(ctx, mid); err != nil { // db error
log.Error("LoadUpMeta Get Mid ERROR (%d) (%v)", mid, err)
return
}
if upper == nil { // db not found
err = ecode.NothingFound
return
}
d.addUpMetaCache(ctx, upper) // db found, re-fill the cache
return
}
// upMetaCache get upper meta cache.
func (d *Dao) upMetaCache(c context.Context, mid int64) (upper *ugcMdl.Upper, err error) {
var (
key = upperMetaKey(mid)
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
missedCount.Add("tv-meta", 1)
} else {
log.Error("conn.Get(%s) error(%v)", key, err)
}
return
}
if err = conn.Scan(item, &upper); err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
}
cachedCount.Add("tv-meta", 1)
return
}
// upMetaDB gets upper meta info from DB
func (d *Dao) upMetaDB(c context.Context, mid int64) (upper *ugcMdl.Upper, err error) {
var row *xsql.Row
if row = d.DB.QueryRow(c, _loadUpper, mid); err != nil {
log.Error("d.db.QueryRow(%d) error(%v)", mid, err)
return
}
upper = &ugcMdl.Upper{}
// "SELECT id,mid,to_init,submit,ori_name,cms_name,ori_face,cms_face,valid,deleted FROM ugc_uploader WHERE mid = ?"
if err = row.Scan(&upper.MID, &upper.Toinit, &upper.Submit, &upper.OriName,
&upper.CMSName, &upper.OriFace, &upper.CMSFace, &upper.Valid, &upper.Deleted); err != nil {
if err == sql.ErrNoRows {
err = nil
upper = nil
} else {
log.Error("row.Scan(mid %d) error(%v)", mid, err)
}
}
return
}
// setUpMetaCache save ugcMdl.Upper to memcache
func (d *Dao) addUpMetaCache(c context.Context, upper *ugcMdl.Upper) (err error) {
var (
key = upperMetaKey(upper.MID)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: upper, Flags: memcache.FlagJSON, Expiration: d.mcExpire}); err != nil {
log.Error("conn.Set error(%v)", err)
return
}
return
}

74
app/job/main/tv/dao/upper/load_upper_test.go Normal file
View File

@@ -0,0 +1,74 @@
package upper
import (
"context"
"testing"
ugcMdl "go-common/app/job/main/tv/model/ugc"
"github.com/smartystreets/goconvey/convey"
)
func TestUpperupperMetaKey(t *testing.T) {
var MID = int64(0)
convey.Convey("upperMetaKey", t, func(c convey.C) {
p1 := upperMetaKey(MID)
c.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestUpperLoadUpMeta(t *testing.T) {
var (
ctx = context.Background()
mid = int64(0)
)
convey.Convey("LoadUpMeta", t, func(c convey.C) {
upper, err := d.LoadUpMeta(ctx, mid)
c.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(upper, convey.ShouldNotBeNil)
})
})
}
func TestUpperupMetaCache(t *testing.T) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("upMetaCache", t, func(ctx convey.C) {
upper, err := d.upMetaCache(c, mid)
ctx.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(upper, convey.ShouldNotBeNil)
})
})
}
func TestUpperupMetaDB(t *testing.T) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("upMetaDB", t, func(ctx convey.C) {
upper, err := d.upMetaDB(c, mid)
ctx.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(upper, convey.ShouldNotBeNil)
})
})
}
func TestUpperaddUpMetaCache(t *testing.T) {
var (
upper = &ugcMdl.Upper{}
)
convey.Convey("addUpMetaCache", t, func(ctx convey.C) {
err := d.addUpMetaCache(context.Background(), upper)
ctx.Convey("Then err should be nil.upper should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

Some files were not shown because too many files have changed in this diff Show More