Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,78 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"activity_test.go",
"dao_test.go",
"game_test.go",
"mysql_test.go",
"redis_test.go",
"reply_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/interface/openplatform/article/model:go_default_library",
"//app/job/openplatform/article/conf:go_default_library",
"//library/cache/redis:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"activity.go",
"art_redis.go",
"dao.cache.go",
"dao.go",
"dynamic.go",
"flow.go",
"game.go",
"media.go",
"mysql.go",
"purge.go",
"recommend.go",
"redis.go",
"reply.go",
],
importpath = "go-common/app/job/openplatform/article/dao",
tags = ["automanaged"],
deps = [
"//app/interface/openplatform/article/model:go_default_library",
"//app/job/openplatform/article/conf:go_default_library",
"//app/job/openplatform/article/model:go_default_library",
"//library/cache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/metadata:go_default_library",
"//library/queue/databus:go_default_library",
"//library/stat/prom:go_default_library",
"//library/xstr:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,43 @@
package dao
import (
"context"
"net/url"
"strconv"
"go-common/library/ecode"
"go-common/library/log"
)
const (
_typeArticle = "12"
)
// LikeSync like sync
func (d *Dao) LikeSync(c context.Context, aid, likes int64) (err error) {
params := url.Values{}
params.Set("oid", strconv.FormatInt(aid, 10))
params.Set("likes", strconv.FormatInt(likes, 10))
params.Set("type", _typeArticle)
resp := struct {
Code int
Data interface{}
}{}
if err = d.httpClient.Get(c, d.c.Job.ActLikeURL, "", params, &resp); err != nil {
log.Error("activity: d.LikeSync.Get(%s) error(%+v)", d.c.Job.ActLikeURL+params.Encode(), err)
PromError("activity:同步点赞数")
return
}
if resp.Code != 0 {
// 未参与活动
if resp.Code == -403 {
return
}
err = ecode.Int(resp.Code)
log.Error("activity: d.LikeSync.Get(%s) error(%+v)", d.c.Job.ActLikeURL+"?"+params.Encode(), resp)
PromError("activity:同步点赞数")
return
}
log.Info("activity: dao.LikeSync success aid: %v count: %v ", aid, likes)
return
}

View File

@@ -0,0 +1,15 @@
package dao
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_LikeSync(t *testing.T) {
Convey("work", t, WithDao(func(d *Dao) {
err := d.LikeSync(context.Background(), 1, 20)
So(err, ShouldBeNil)
}))
}

View File

@@ -0,0 +1,78 @@
package dao
import (
"context"
"fmt"
"go-common/library/cache/redis"
"go-common/library/log"
)
func sortedKey(categoryID int64, field int) string {
return fmt.Sprintf("art_sort_%d_%d", categoryID, field)
}
// ExpireSortCache expire sort cache
func (d *Dao) ExpireSortCache(c context.Context, categoryID int64, field int) (ok bool, err error) {
key := sortedKey(categoryID, field)
conn := d.artRedis.Get(c)
defer conn.Close()
var ttl int64
if ttl, err = redis.Int64(conn.Do("TTL", key)); err != nil {
PromError("redis:排序缓存ttl")
log.Error("conn.Do(TTL, %s) error(%+v)", key, err)
}
if ttl > (d.redisSortTTL - d.redisSortExpire) {
ok = true
}
return
}
// AddSortCaches add sort articles cache
func (d *Dao) AddSortCaches(c context.Context, categoryID int64, field int, arts [][2]int64, maxLength int64) (err error) {
var (
id, score int64
key = sortedKey(categoryID, field)
conn = d.artRedis.Get(c)
count int
)
defer conn.Close()
if len(arts) == 0 {
return
}
if err = conn.Send("DEL", key); err != nil {
PromError("redis:删除排序缓存")
log.Error("conn.Send(DEL, %s) error(%+v)", key, err)
return
}
count++
for _, art := range arts {
id = art[0]
score = art[1]
if err = conn.Send("ZADD", key, "CH", score, id); err != nil {
PromError("redis:增加排序缓存")
log.Error("conn.Send(ZADD, %s, %d, %v) error(%+v)", key, score, id, err)
return
}
count++
}
if err = conn.Send("EXPIRE", key, d.redisSortTTL); err != nil {
PromError("redis:排序缓存设定过期")
log.Error("conn.Send(EXPIRE, %s, %d) error(%+v)", key, d.redisSortTTL, err)
return
}
count++
if err = conn.Flush(); err != nil {
PromError("redis:增加排序缓存flush")
log.Error("conn.Flush error(%+v)", err)
return
}
for i := 0; i < count; i++ {
if _, err = conn.Receive(); err != nil {
PromError("redis:增加排序缓存receive")
log.Error("conn.Receive error(%+v)", err)
return
}
}
return
}

View File

@@ -0,0 +1,47 @@
// Code generated by $GOPATH/src/go-common/app/tool/cache/gen. DO NOT EDIT.
/*
Package dao is a generated cache proxy package.
It is generated from:
type _cache interface {
GameList(c context.Context) (res []int64, err error)
}
*/
package dao
import (
"context"
"go-common/library/net/metadata"
"go-common/library/stat/prom"
)
var _ _cache
// GameList get data from cache if miss will call source method, then add to cache.
func (d *Dao) GameList(c context.Context) (res []int64, err error) {
addCache := true
res, err = d.CacheGameList(c)
if err != nil {
addCache = false
err = nil
}
if len(res) != 0 {
prom.CacheHit.Incr("GameList")
return
}
prom.CacheMiss.Incr("GameList")
res, err = d.RawGameList(c)
if err != nil {
return
}
var miss = res
if !addCache {
return
}
d.cache.Save(func() {
d.AddCacheGameList(metadata.WithContext(c), miss)
})
return
}

View File

@@ -0,0 +1,106 @@
package dao
import (
"context"
"time"
"go-common/app/job/openplatform/article/conf"
"go-common/library/cache"
"go-common/library/cache/redis"
xsql "go-common/library/database/sql"
bm "go-common/library/net/http/blademaster"
"go-common/library/queue/databus"
"go-common/library/stat/prom"
)
// Dao .
type Dao struct {
c *conf.Config
db *xsql.DB
redis *redis.Pool
artRedis *redis.Pool
httpClient *bm.Client
gameHTTPClient *bm.Client
viewCacheTTL, gameCacheExpire int64
dupViewCacheTTL int64
redisSortExpire int64
redisSortTTL int64
// stmt
updateSearchStmt *xsql.Stmt
delSearchStmt *xsql.Stmt
updateSearchStatsStmt *xsql.Stmt
gameStmt *xsql.Stmt
cheatStmt *xsql.Stmt
newestArtsMetaStmt *xsql.Stmt
searchArtsStmt *xsql.Stmt
updateRecheckStmt *xsql.Stmt
getRecheckStmt *xsql.Stmt
settingsStmt *xsql.Stmt
midByPubtimeStmt *xsql.Stmt
statByMidStmt *xsql.Stmt
dynamicDbus *databus.Databus
cache *cache.Cache
}
var (
errorsCount = prom.BusinessErrCount
cacheLen = prom.BusinessInfoCount
infosCount = prom.BusinessInfoCount
)
// New creates a dao instance.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
c: c,
db: xsql.NewMySQL(c.DB),
redis: redis.NewPool(c.Redis),
artRedis: redis.NewPool(c.ArtRedis),
httpClient: bm.NewClient(c.HTTPClient),
gameHTTPClient: bm.NewClient(c.GameHTTPClient),
viewCacheTTL: int64(time.Duration(c.Job.ViewCacheTTL) / time.Second),
dupViewCacheTTL: int64(time.Duration(c.Job.DupViewCacheTTL) / time.Second),
gameCacheExpire: int64(time.Duration(c.Job.GameCacheExpire) / time.Second),
redisSortExpire: int64(time.Duration(c.Job.ExpireSortArts) / time.Second),
redisSortTTL: int64(time.Duration(c.Job.TTLSortArts) / time.Second),
dynamicDbus: databus.New(c.DynamicDbus),
cache: cache.New(1, 1024),
}
d.updateSearchStmt = d.db.Prepared(_updateSearch)
d.delSearchStmt = d.db.Prepared(_delSearch)
d.updateSearchStatsStmt = d.db.Prepared(_updateSearchStats)
d.newestArtsMetaStmt = d.db.Prepared(_newestArtsMetaSQL)
d.gameStmt = d.db.Prepared(_gameList)
d.cheatStmt = d.db.Prepared(_allCheat)
d.searchArtsStmt = d.db.Prepared(_searchArticles)
d.updateRecheckStmt = d.db.Prepared(_updateCheckState)
d.getRecheckStmt = d.db.Prepared(_checkStateSQL)
d.settingsStmt = d.db.Prepared(_settingsSQL)
d.midByPubtimeStmt = d.db.Prepared(_midsByPublishTimeSQL)
d.statByMidStmt = d.db.Prepared(_statByMidSQL)
return
}
//go:generate $GOPATH/src/go-common/app/tool/cache/gen
type _cache interface {
GameList(c context.Context) (res []int64, err error)
}
// PromError prometheus error count.
func PromError(name string) {
errorsCount.Incr(name)
}
// PromInfo prometheus info count.
func PromInfo(name string) {
infosCount.Incr(name)
}
// Ping reports the health of the db/cache etc.
func (d *Dao) Ping(c context.Context) (err error) {
if err = d.db.Ping(c); err != nil {
PromError("db:Ping")
return
}
err = d.pingRedis(c)
return
}

View File

@@ -0,0 +1,34 @@
package dao
import (
"context"
"flag"
"path/filepath"
"testing"
"go-common/app/job/openplatform/article/conf"
. "github.com/smartystreets/goconvey/convey"
)
func WithDao(f func(d *Dao)) func() {
return func() {
dir, _ := filepath.Abs("../cmd/goconvey.toml")
flag.Set("conf", dir)
flag.Parse()
conf.Init()
d := New(conf.Conf)
f(d)
}
}
func Test_Reply(t *testing.T) {
Convey("open reply", t, WithDao(func(d *Dao) {
var (
err error
c = context.TODO()
)
err = d.OpenReply(c, 88, 88)
So(err, ShouldBeNil)
}))
}

View File

@@ -0,0 +1,33 @@
package dao
import (
"context"
"strconv"
"go-common/app/job/openplatform/article/model"
"go-common/library/log"
)
const _dynamicArt = 64
// PubDynamic pub dynamic
func (d *Dao) PubDynamic(c context.Context, mid int64, aid int64, show bool, comment string, ts int64, dynamicIntro string) (err error) {
msg := &model.DynamicMsg{}
msg.Card.Type = _dynamicArt
msg.Card.Rid = aid
msg.Card.OwnerID = mid
if show {
msg.Card.Show = 1
}
msg.Card.Comment = comment
msg.Card.Ts = ts
msg.Card.Dynamic = dynamicIntro
if err = d.dynamicDbus.Send(c, strconv.FormatInt(aid, 10), msg); err != nil {
PromError("dynamic:发送动态消息")
log.Error("dynamic: d.SendPubDynamic(%+v) error(%+v)", msg, err)
return
}
PromInfo("databus:发送动态消息")
log.Info("dynamic: dao.PubDynamic(%+v)", msg)
return
}

View File

@@ -0,0 +1,39 @@
package dao
import (
"context"
"net/url"
"strconv"
"go-common/library/ecode"
"go-common/library/log"
)
const (
_typeArticleFlow = "3"
)
// FlowSync 流量管理同步过审文章
func (d *Dao) FlowSync(c context.Context, mid, aid int64) (err error) {
params := url.Values{}
params.Set("oid", strconv.FormatInt(aid, 10))
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("business", _typeArticleFlow)
resp := struct {
Code int
Data interface{}
}{}
if err = d.httpClient.Post(c, d.c.Job.FlowURL, "", params, &resp); err != nil {
log.Error("flow: d.FlowSync.Post(%s) error(%+v)", d.c.Job.FlowURL+params.Encode(), err)
PromError("flow:文章过审")
return
}
if resp.Code != 0 {
err = ecode.Int(resp.Code)
log.Error("flow: d.FlowSync.Post(%s) error(%+v)", d.c.Job.FlowURL+"?"+params.Encode(), resp)
PromError("flow:文章过审")
return
}
log.Info("flow: dao.FlowSync success aid: %v mid: %v ", aid, mid)
return
}

View File

@@ -0,0 +1,111 @@
package dao
import (
"context"
"net/url"
"strconv"
"time"
"go-common/app/job/openplatform/article/model"
"go-common/library/cache/redis"
"go-common/library/ecode"
"go-common/library/log"
)
const (
_gameSyncURL = "http://line3-h5-mobile-api.biligame.com/h5/internal/article/sync"
_typeAdd = "1"
_typeUpdate = "2"
_typeDel = "3"
_gameKey = "artjob_game_mids"
)
// GameSync game sync
func (d *Dao) GameSync(c context.Context, action string, cvid int64) (err error) {
params := url.Values{}
params.Set("timestamp", strconv.FormatInt(time.Now().Unix()*1000, 10))
var tp string
if action == model.ActInsert {
tp = _typeAdd
} else if action == model.ActUpdate {
tp = _typeUpdate
} else {
tp = _typeDel
}
params.Set("type", tp)
params.Set("article_id", strconv.FormatInt(cvid, 10))
resp := struct {
Code int
}{}
if err = d.gameHTTPClient.Post(c, _gameSyncURL, "", params, &resp); err != nil {
log.Error("game: d.gameHTTPClient.Post(%s) error(%+v)", _gameSyncURL+params.Encode(), err)
PromError("game:同步数据")
return
}
if resp.Code != 0 {
err = ecode.Int(resp.Code)
log.Error("game: d.gameHTTPClient.Get(%s) code: %v error(%+v)", _gameSyncURL, resp.Code, err)
PromError("game:同步数据")
return
}
log.Info("game: dao.GameSync success action: %v cvid: %v", action, cvid)
return
}
// CacheGameList .
func (d *Dao) CacheGameList(c context.Context) (mids []int64, err error) {
conn := d.redis.Get(c)
defer conn.Close()
if mids, err = redis.Int64s(conn.Do("ZRANGE", _gameKey, 0, -1)); err != nil {
PromError("redis:游戏列表缓存")
log.Error("conn.Zrange(%s) error(%+v)", _gameKey, err)
}
return
}
// AddCacheGameList .
func (d *Dao) AddCacheGameList(c context.Context, mids []int64) (err error) {
var (
key = _gameKey
conn = d.redis.Get(c)
count int
)
defer conn.Close()
if len(mids) == 0 {
return
}
if err = conn.Send("DEL", key); err != nil {
PromError("redis:删除游戏列表缓存")
log.Error("conn.Send(DEL, %s) error(%+v)", key, err)
return
}
count++
for i, mid := range mids {
score := i
if err = conn.Send("ZADD", key, "CH", score, mid); err != nil {
PromError("redis:增加游戏列表缓存")
log.Error("conn.Send(ZADD, %s, %d, %v) error(%+v)", key, score, mid, err)
return
}
count++
}
if err = conn.Send("EXPIRE", key, d.gameCacheExpire); err != nil {
PromError("redis:游戏列表缓存设定过期")
log.Error("conn.Send(EXPIRE, %s, %d) error(%+v)", key, d.gameCacheExpire, err)
return
}
count++
if err = conn.Flush(); err != nil {
PromError("redis:增加游戏列表缓存flush")
log.Error("conn.Flush error(%+v)", err)
return
}
for i := 0; i < count; i++ {
if _, err = conn.Receive(); err != nil {
PromError("redis:增加游戏列表缓存receive")
log.Error("conn.Receive error(%+v)", err)
return
}
}
return
}

View File

@@ -0,0 +1,23 @@
package dao
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_GameSync(t *testing.T) {
Convey("work", t, WithDao(func(d *Dao) {
err := d.GameSync(context.Background(), "add", 1)
So(err, ShouldBeNil)
}))
}
func Test_NewGameCache(t *testing.T) {
Convey("work", t, WithDao(func(d *Dao) {
res, err := d.GameList(context.TODO())
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}

View File

@@ -0,0 +1,41 @@
package dao
import (
"context"
"net/url"
"strconv"
"go-common/library/ecode"
"go-common/library/log"
)
const _delScoreURL = "http://api.bilibili.co/pgc/internal/review/score/delete"
// DelScore .
func (d *Dao) DelScore(c context.Context, aid, mediaID, mid int64) (err error) {
if mediaID == 0 || mid == 0 || aid == 0 {
return
}
params := url.Values{}
params.Set("media_id", strconv.FormatInt(mediaID, 10))
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("oid", strconv.FormatInt(aid, 10))
params.Set("from", "1")
var resp struct {
Code int `json:"code"`
Message string `json:"message"`
}
err = d.httpClient.Post(c, _delScoreURL, "", params, &resp)
if err != nil {
PromError("media:番剧删除评分接口")
log.Error("media: d.client.Post(%s, data:%s) error(%+v)", _delScoreURL, params.Encode(), err)
return
}
if resp.Code != 0 {
PromError("media:番剧删除评分接口")
log.Error("media: d.client.Post(%s, data:%s) res: %+v", _delScoreURL, params.Encode(), resp)
err = ecode.Int(resp.Code)
}
log.Info("media: del score success(media_id: %d, mid: %d, oid: %d)", mediaID, mid, aid)
return
}

View File

@@ -0,0 +1,411 @@
package dao
import (
"context"
"fmt"
"time"
artmdl "go-common/app/interface/openplatform/article/model"
"go-common/app/job/openplatform/article/model"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_sharding = 100
// stat
_statSQL = "SELECT article_id,favorite,reply,share,likes,dislike,view,coin FROM article_stats_%s WHERE article_id=%d and deleted_time =0"
_upStatSQL = `INSERT INTO article_stats_%s (article_id,favorite,reply,share,likes,dislike,view,coin) VALUES (?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE favorite=?,reply=?,share=?,likes=?,dislike=?,view=?,coin=?`
_updateSearch = `INSERT INTO search_articles(ctime, article_id, category_id, title, summary, template_id, mid, image_urls, publish_time, content, tags, stats_view, stats_favorite, stats_likes, stats_dislike, stats_reply, stats_share, stats_coin, origin_image_urls, attributes, keywords)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE ctime = ?, category_id =?, title=?, summary=?, template_id=?, mid=?, image_urls=?, publish_time=?, content=?, tags=?, stats_view=?, stats_favorite=?, stats_likes=?, stats_dislike=?, stats_reply=?, stats_share=?, stats_coin = ?, origin_image_urls = ?, attributes = ?, keywords = ?`
_delSearch = "DELETE FROM search_articles where article_id = ?"
_articleContentSQL = "SELECT content FROM filtered_article_contents_%s WHERE article_id = ?"
_updateSearchStats = "UPDATE search_articles SET stats_view=?, stats_favorite=?, stats_likes=?, stats_dislike=?, stats_reply=?, stats_share=?, stats_coin =? where article_id = ?"
_gameList = "SELECT mid FROM white_list_users WHERE group_id = 4 AND deleted = 0"
_allCheat = "SELECT article_id, lv FROM stats_filters WHERE deleted_time = 0"
_newestArtsMetaSQL = "SELECT article_id, publish_time, attributes FROM filtered_articles ORDER BY publish_time DESC LIMIT ?"
_newestArtCategorysMetaSQL = "SELECT article_id, publish_time, attributes FROM filtered_articles where category_id in (%s) ORDER BY publish_time DESC LIMIT %d"
_searchArticles = "select article_id, category_id, attributes, stats_view, stats_reply, stats_favorite, stats_likes, stats_coin from search_articles where publish_time >= ? and publish_time < ?"
_checkStateSQL = "SELECT publish_time,check_state FROM articles WHERE id = ? and deleted_time = 0"
_updateCheckState = "UPDATE articles SET check_state = 3 WHERE id = ?"
_settingsSQL = "SELECT name,value FROM article_settings WHERE deleted_time=0"
_midsByPublishTimeSQL = "SELECT mid FROM articles WHERE publish_time > ? and state = 0 and deleted_time = 0 group by mid"
_statByMidSQL = "select count(*), sum(words), category_id from articles where mid = ? and state = 0 and deleted_time = 0 group by category_id"
_keywordsSQL = "select tags from article_contents_%s where article_id = ?"
_actIDSQL = "select act_id from articles where id = ?"
_lastModsArtsSQL = "select id from articles where state in (0,5,6,7) and deleted_time = 0 order by mtime desc limit ?"
)
var _searchInterval = int64(3 * 24 * 3600)
func (d *Dao) hit(id int64) string {
return fmt.Sprintf("%02d", id%_sharding)
}
// Stat returns stat info.
func (d *Dao) Stat(c context.Context, aid int64) (stat *artmdl.StatMsg, err error) {
stat = &artmdl.StatMsg{}
err = d.db.QueryRow(c, fmt.Sprintf(_statSQL, d.hit(aid), aid)).Scan(&stat.Aid, &stat.Favorite, &stat.Reply, &stat.Share, &stat.Like, &stat.Dislike, &stat.View, &stat.Coin)
if err == sql.ErrNoRows {
err = nil
stat = nil
} else if err != nil {
log.Error("Stat(%v) error(%+v)", aid, err)
PromError("db:读取计数")
}
return
}
// Update updates stat in db.
func (d *Dao) Update(c context.Context, stat *artmdl.StatMsg) (rows int64, err error) {
res, err := d.db.Exec(c, fmt.Sprintf(_upStatSQL, d.hit(stat.Aid)), stat.Aid, *stat.Favorite, *stat.Reply, *stat.Share, *stat.Like, *stat.Dislike, *stat.View, *stat.Coin,
*stat.Favorite, *stat.Reply, *stat.Share, *stat.Like, *stat.Dislike, *stat.View, *stat.Coin)
if err != nil {
log.Error("Update(%d,%+v) error(%+v)", stat.Aid, stat, err)
PromError("db:更新计数")
return
}
rows, err = res.RowsAffected()
return
}
// UpdateSearch update search article table
func (d *Dao) UpdateSearch(c context.Context, a *model.SearchArticle) (err error) {
_, err = d.updateSearchStmt.Exec(c, a.CTime, a.ID, a.CategoryID, a.Title, a.Summary, a.TemplateID, a.Mid, a.ImageURLs, a.PublishTime, a.Content, a.Tags, a.StatsView, a.StatsFavorite, a.StatsLikes, a.StatsDisLike, a.StatsReply, a.StatsShare, a.StatsCoin, a.OriginImageURLs, a.Attributes, a.Keywords,
a.CTime, a.CategoryID, a.Title, a.Summary, a.TemplateID, a.Mid, a.ImageURLs, a.PublishTime, a.Content, a.Tags, a.StatsView, a.StatsFavorite, a.StatsLikes, a.StatsDisLike, a.StatsReply, a.StatsShare, a.StatsCoin, a.OriginImageURLs, a.Attributes, a.Keywords)
if err != nil {
PromError("db:更新搜索表")
log.Error("UpdateSearch(%+v) error(%+v)", a, err)
}
return
}
// DelSearch del search article table
func (d *Dao) DelSearch(c context.Context, aid int64) (err error) {
_, err = d.delSearchStmt.Exec(c, aid)
if err != nil {
PromError("db:删除搜索表")
log.Error("DelSearch(%v) error(%+v)", aid, err)
return
}
return
}
// UpdateRecheck update recheck table
func (d *Dao) UpdateRecheck(c context.Context, aid int64) (err error) {
_, err = d.updateRecheckStmt.Exec(c, aid)
if err != nil {
PromError("db:修改回查状态")
log.Error("UpdateRecheck(%v) error(%+v)", aid, err)
}
return
}
// ArticleContent get article content
func (d *Dao) ArticleContent(c context.Context, id int64) (res string, err error) {
contentSQL := fmt.Sprintf(_articleContentSQL, d.hit(id))
if err = d.db.QueryRow(c, contentSQL, id).Scan(&res); err != nil {
if err == sql.ErrNoRows {
err = nil
return
}
PromError("db:ArticleContent")
log.Error("dao.ArticleContent(%s) error(%+v)", contentSQL, err)
}
return
}
// GetRecheckInfo get article recheck
func (d *Dao) GetRecheckInfo(c context.Context, id int64) (publishTime int64, checkState int, err error) {
if err = d.getRecheckStmt.QueryRow(c, id).Scan(&publishTime, &checkState); err != nil {
if err == sql.ErrNoRows {
err = nil
return
}
PromError("db:ReCheckQuery")
log.Error("dao.GetRecheckInfo(%d) error(%+v)", id, err)
}
return
}
// UpdateSearchStats update search stats
func (d *Dao) UpdateSearchStats(c context.Context, stat *artmdl.StatMsg) (err error) {
_, err = d.updateSearchStatsStmt.Exec(c, *stat.View, *stat.Favorite, *stat.Like, *stat.Dislike, *stat.Reply, *stat.Share, *stat.Coin, stat.Aid)
if err != nil {
log.Error("updateSearchStatsStmt(%d,%+v) error(%+v)", stat.Aid, stat, err)
PromError("db:更新搜索计数")
}
return
}
// RawGameList game list
func (d *Dao) RawGameList(c context.Context) (mids []int64, err error) {
rows, err := d.gameStmt.Query(c)
if err != nil {
return
}
defer rows.Close()
for rows.Next() {
var id int64
if rows.Scan(&id); err != nil {
PromError("db:GameList")
log.Error("dao.GameList() error(%+v)", err)
return
}
mids = append(mids, id)
}
return
}
// CheatArts cheat list
func (d *Dao) CheatArts(c context.Context) (res map[int64]int, err error) {
rows, err := d.cheatStmt.Query(c)
if err != nil {
return
}
res = make(map[int64]int)
defer rows.Close()
defer func() {
if err != nil {
PromError("db:CheatArts")
log.Error("dao.CheatArts() error(%+v)", err)
}
}()
for rows.Next() {
var id int64
var lv int
if rows.Scan(&id, &lv); err != nil {
return
}
res[id] = lv
}
err = rows.Err()
return
}
// NewestArtIDs find newest article's id
func (d *Dao) NewestArtIDs(c context.Context, limit int64) (res [][2]int64, err error) {
rows, err := d.newestArtsMetaStmt.Query(c, limit)
if err != nil {
PromError("db:最新文章")
log.Error("dao.newestArtsMetaStmt.Query error(%+v)", err)
return
}
defer rows.Close()
for rows.Next() {
var (
id int64
ptime int64
attribute int32
)
if err = rows.Scan(&id, &ptime, &attribute); err != nil {
PromError("db:最新文章scan")
log.Error("dao.NewestArtIDs.rows.Scan error(%+v)", err)
return
}
if artmdl.NoDistributeAttr(attribute) || artmdl.NoRegionAttr(attribute) {
continue
}
res = append(res, [2]int64{id, ptime})
}
if err = rows.Err(); err != nil {
PromError("db:最新文章")
log.Error("dao.NewestArtIDs.rows error(%+v)", err)
}
return
}
// NewestArtIDByCategory find newest article's id
func (d *Dao) NewestArtIDByCategory(c context.Context, cids []int64, limit int64) (res [][2]int64, err error) {
sql := fmt.Sprintf(_newestArtCategorysMetaSQL, xstr.JoinInts(cids), limit)
rows, err := d.db.Query(c, sql)
if err != nil {
PromError("db:最新文章")
log.Error("dao.NewestArtIDByCategorys.Query error(%+v)", err)
return
}
defer rows.Close()
for rows.Next() {
var (
id int64
ptime int64
attribute int32
)
if err = rows.Scan(&id, &ptime, &attribute); err != nil {
PromError("db:最新文章scan")
log.Error("dao.NewestArtIDByCategory.rows.Scan error(%+v)", err)
return
}
if artmdl.NoDistributeAttr(attribute) || artmdl.NoRegionAttr(attribute) {
continue
}
res = append(res, [2]int64{id, ptime})
}
if err = rows.Err(); err != nil {
PromError("db:最新文章")
log.Error("dao.NewestArtIDByCategory.Scan error(%+v)", err)
}
return
}
// SearchArts get articles publish time after ptime
func (d *Dao) SearchArts(c context.Context, ptime int64) (res []*model.SearchArticle, err error) {
var rows *sql.Rows
now := time.Now().Unix()
for ; ptime < now; ptime += _searchInterval {
if rows, err = d.searchArtsStmt.Query(c, ptime, ptime+_searchInterval); err != nil {
PromError("db:searchArts")
log.Error("mysql: search arts Query() error(%+v)", err)
return
}
defer rows.Close()
for rows.Next() {
ba := &model.SearchArticle{}
if err = rows.Scan(&ba.ID, &ba.CategoryID, &ba.Attributes, &ba.StatsView, &ba.StatsReply, &ba.StatsFavorite, &ba.StatsLikes, &ba.StatsCoin); err != nil {
PromError("db:searchArts")
log.Error("mysql: search arts Scan() error(%+v)", err)
return
}
res = append(res, ba)
}
if err = rows.Err(); err != nil {
PromError("db:searchArts")
log.Error("mysql: search arts Query() error(%+v)", err)
return
}
}
return
}
// Settings gets article settings.
func (d *Dao) Settings(c context.Context) (res map[string]string, err error) {
var rows *sql.Rows
if rows, err = d.settingsStmt.Query(c); err != nil {
PromError("db:文章配置查询")
log.Error("mysql: db.settingsStmt.Query error(%+v)", err)
return
}
defer rows.Close()
res = make(map[string]string)
for rows.Next() {
var name, value string
if err = rows.Scan(&name, &value); err != nil {
PromError("文章配置scan")
log.Error("mysql: rows.Scan error(%+v)", err)
return
}
res[name] = value
}
err = rows.Err()
if err = rows.Err(); err != nil {
PromError("db:loadSettings")
log.Error("mysql: load settings Query() error(%+v)", err)
}
return
}
//MidsByPublishTime get mids by publish time
func (d *Dao) MidsByPublishTime(c context.Context, pubTime int64) (mids []int64, err error) {
var rows *sql.Rows
if rows, err = d.midByPubtimeStmt.Query(c, pubTime); err != nil {
PromError("db:查询近7天mid")
log.Error("mysql: db.MidsByPublishTime.Query error(%+v)", err)
return
}
defer rows.Close()
for rows.Next() {
var mid int64
if err = rows.Scan(&mid); err != nil {
PromError("查询近7天mid scan")
log.Error("mysql: rows.Scan error(%+v)", err)
return
}
mids = append(mids, mid)
}
err = rows.Err()
if err = rows.Err(); err != nil {
PromError("db:MidsByPublishTime")
log.Error("mysql: get mids by publish time Query() error(%+v)", err)
}
return
}
//StatByMid get author info by mid
func (d *Dao) StatByMid(c context.Context, mid int64) (res map[int64][2]int64, err error) {
res = make(map[int64][2]int64)
var rows *sql.Rows
if rows, err = d.statByMidStmt.Query(c, mid); err != nil {
PromError("db:作者分区数据")
log.Error("mysql: db.statByMidStmt.Query error(%+v)", err)
return
}
defer rows.Close()
for rows.Next() {
var (
data [2]int64
cate int64
)
if err = rows.Scan(&data[0], &data[1], &cate); err != nil {
PromError("作者分区数据 scan")
log.Error("mysql: rows.Scan error(%+v)", err)
return
}
res[cate] = data
}
err = rows.Err()
if err = rows.Err(); err != nil {
PromError("db:StatByMid")
log.Error("mysql: get stat infos by mid Query() error(%+v)", err)
}
return
}
// Keywords .
func (d *Dao) Keywords(c context.Context, id int64) (keywords string, err error) {
var sqlStr = fmt.Sprintf(_keywordsSQL, d.hit(id))
row := d.db.QueryRow(c, sqlStr, id)
if err = row.Scan(&keywords); err != nil {
PromError("db:Keywords")
log.Error("d.keywords scan error(%+v)", err)
}
return
}
// IsAct .
func (d *Dao) IsAct(c context.Context, id int64) (res bool) {
var actID int64
if err := d.db.QueryRow(c, _actIDSQL, id).Scan(&actID); err != nil {
PromError("db:IsAct")
log.Error("d.IsAct scan error(%+v)", err)
return
}
if actID > 0 {
res = true
}
return
}
// LastModIDs .
func (d *Dao) LastModIDs(c context.Context, size int) (aids []int64, err error) {
var (
id int64
rows *sql.Rows
)
if rows, err = d.db.Query(c, _lastModsArtsSQL, size); err != nil {
PromError("db:LastModIDs")
log.Error("d.LastModIDs query error(%+v) size(%d)", err, size)
return
}
for rows.Next() {
if err = rows.Scan(&id); err != nil {
PromError("db:LastModIDs")
log.Error("d.LastModIDs scan error(%+v) size(%d)", err, size)
return
}
aids = append(aids, id)
}
return
}

View File

@@ -0,0 +1,96 @@
package dao
import (
"context"
"testing"
artmdl "go-common/app/interface/openplatform/article/model"
. "github.com/smartystreets/goconvey/convey"
)
func Test_Update(t *testing.T) {
var (
c = context.TODO()
cnt1 = int64(1)
cnt2 = int64(2)
st1 = &artmdl.StatMsg{
Aid: 888,
View: &cnt1,
Favorite: &cnt1,
Like: &cnt1,
Dislike: &cnt1,
Reply: &cnt1,
Share: &cnt1,
}
st2 = &artmdl.StatMsg{
Aid: 888,
View: &cnt2,
Favorite: &cnt2,
Like: &cnt2,
Dislike: &cnt2,
Reply: &cnt2,
Share: &cnt2,
}
)
Convey("update stats", t, WithDao(func(d *Dao) {
rows, err := d.Update(c, st1)
So(err, ShouldBeNil)
So(rows, ShouldBeGreaterThan, 0)
Convey("get st1", func() {
stat, err1 := d.Stat(c, 888)
So(err1, ShouldBeNil)
So(stat, ShouldResemble, st1)
})
rows, err = d.Update(c, st2)
So(err, ShouldBeNil)
So(rows, ShouldBeGreaterThan, 0)
Convey("get st2", func() {
stat, err1 := d.Stat(c, 888)
So(err1, ShouldBeNil)
So(stat, ShouldResemble, st2)
})
}))
}
func Test_GameList(t *testing.T) {
Convey("work", t, WithDao(func(d *Dao) {
mids, err := d.GameList(context.Background())
So(err, ShouldBeNil)
So(mids, ShouldNotBeEmpty)
}))
}
func Test_NewestArtIDByCategory(t *testing.T) {
var _dataCategory = int64(6)
Convey("get data", t, WithDao(func(d *Dao) {
res, err := d.NewestArtIDByCategory(context.TODO(), []int64{_dataCategory}, 100)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
Convey("no data", t, WithDao(func(d *Dao) {
res, err := d.NewestArtIDByCategory(context.TODO(), []int64{1000}, 100)
So(err, ShouldBeNil)
So(res, ShouldBeEmpty)
}))
}
func Test_NewestArtIDs(t *testing.T) {
Convey("get data", t, WithDao(func(d *Dao) {
res, err := d.NewestArtIDs(context.TODO(), 100)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}
func Test_SearchArts(t *testing.T) {
Convey("should get data", t, WithDao(func(d *Dao) {
_searchInterval = 24 * 3600 * 365
res, err := d.SearchArts(context.TODO(), 0)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}

View File

@@ -0,0 +1,32 @@
package dao
import (
"context"
"net/url"
"time"
"go-common/library/log"
)
const _purgeURL = "http://cp.bilibili.co/api_purge.php"
// PurgeCDN purges cdn.
func (d *Dao) PurgeCDN(c context.Context, file string) (err error) {
defer func() {
if err == nil {
return
}
time.Sleep(time.Second)
if e := d.PushCDN(c, file); e != nil {
log.Error("d.PushCDN(%s) error(%+v)", file, e)
}
}()
params := url.Values{}
params.Set("sid", "2d0586d2c63fb82a69b20c8992811055")
params.Set("file", file)
if err = d.httpClient.Get(c, _purgeURL, "", params, nil); err != nil {
log.Error("d.httpClient.Get() error(%+v)", err)
PromError("purge:刷新CDN")
}
return
}

View File

@@ -0,0 +1,95 @@
package dao
import (
"context"
"fmt"
"go-common/library/cache/redis"
"go-common/library/log"
)
// AddcategoriesAuthors .
func (d *Dao) AddcategoriesAuthors(c context.Context, data map[int64][]int64) (err error) {
if len(data) == 0 {
return
}
conn := d.artRedis.Get(c)
defer conn.Close()
for k, x := range data {
key := fmt.Sprintf("recommends:authors:%d", k)
args := redis.Args{}.Add(key)
for _, v := range x {
args = args.Add(fmt.Sprintf("%d", v))
}
if err = conn.Send("DEL", key); err != nil {
log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
PromError("redis:分区作者数据")
return
}
if err = conn.Send("SADD", args...); err != nil {
log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
PromError("redis:分区作者数据")
return
}
if err = conn.Send("EXPIRE", key, d.c.Job.RecommendExpire); err != nil {
log.Error("conn.Send(EXPIRE, %s) error(%v)", d.c.Job.RecommendExpire, err)
PromError("redis:分区作者数据")
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
PromError("redis:分区作者数据")
return
}
for i := 0; i < 3; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
PromError("redis:分区作者数据")
return
}
}
}
return
}
// AddAuthorMostCategories .
func (d *Dao) AddAuthorMostCategories(c context.Context, mid int64, categories []int64) (err error) {
if len(categories) == 0 {
return
}
conn := d.artRedis.Get(c)
defer conn.Close()
key := fmt.Sprintf("author:categories:%d", mid)
args := redis.Args{}.Add(key)
for _, v := range categories {
args = args.Add(fmt.Sprintf("%d", v))
}
if err = conn.Send("DEL", key); err != nil {
log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
PromError("redis:作者分区数据")
return
}
if err = conn.Send("SADD", args...); err != nil {
log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
PromError("redis:作者分区数据")
return
}
if err = conn.Send("EXPIRE", key, d.c.Job.RecommendExpire); err != nil {
log.Error("conn.Send(EXPIRE, %s) error(%v)", d.c.Job.RecommendExpire, err)
PromError("redis:作者分区数据")
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%v)", err)
PromError("redis:作者分区数据")
return
}
for i := 0; i < 3; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive error(%v)", err)
PromError("redis:作者分区数据")
return
}
}
return
}

View File

@@ -0,0 +1,631 @@
package dao
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
artmdl "go-common/app/interface/openplatform/article/model"
"go-common/app/job/openplatform/article/model"
"go-common/library/cache/redis"
"go-common/library/log"
)
const (
// view
_viewPrefix = "v_"
// retry stat
_retryStatKey = "retry_stat"
// RetryUpdateStatCache .
RetryUpdateStatCache = 1
// RetryUpdateStatDB .
RetryUpdateStatDB = 2
// RetryStatCount is retry upper limit.
RetryStatCount = 10
// retry cache
_retryArtCacheKey = "retry_art_cache"
_retryGameCacheKey = "artj_retry_game_cache"
_retryFlowCacheKey = "artj_retry_flow_cache"
_retryDynamicCacheKey = "artj_retry_dynamic_cache"
// RetryAddArtCache .
RetryAddArtCache = 1
// RetryUpdateArtCache .
RetryUpdateArtCache = 2
// RetryDeleteArtCache .
RetryDeleteArtCache = 3
// RetryDeleteArtRecCache .
RetryDeleteArtRecCache = 4
// retry reply
_retryReplyKey = "retry_reply"
// retry purge cdn
_retryCDNKey = "retry_cdn"
_recheckArtKey = "recheck_lock_%d"
// reading start set
_readPingSet = "art:readping"
// reading during on some device for some article
_prefixReadPing = "art:readping:%s:%d"
)
// StatRetry .
type StatRetry struct {
Action int `json:"action"`
Count int `json:"count"`
Data *artmdl.StatMsg `json:"data"`
}
func viewKey(aid, mid int64, ip string) (key string) {
if ip == "" {
// let it pass if ip is empty.
return
}
key = _viewPrefix + strconv.FormatInt(aid, 10) + ip
if mid != 0 {
key += strconv.FormatInt(mid, 10)
}
return
}
func dupViewKey(aid, mid int64) (key string) {
return fmt.Sprintf("dv_%v_%v", aid, mid)
}
func recheckKey(aid int64) (key string) {
return fmt.Sprintf(_recheckArtKey, aid)
}
// pingRedis checks redis healthy.
func (d *Dao) pingRedis(c context.Context) (err error) {
conn := d.redis.Get(c)
defer conn.Close()
if _, err = conn.Do("SET", "PING", "PONG"); err != nil {
PromError("redis:Ping")
log.Error("redis: conn.Do(SET,PING,PONG) error(%+v)", err)
}
return
}
// Intercept intercepts illegal views.
func (d *Dao) Intercept(c context.Context, aid, mid int64, ip string) (ban bool) {
var (
err error
exist bool
key = viewKey(aid, mid, ip)
conn = d.redis.Get(c)
)
defer conn.Close()
if key == "" {
return
}
if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
log.Error("conn.Do(EXISTS, %s) error(%+v)", key, err)
PromError("redis:EXISTS阅读数")
return
}
if exist {
ban = true
return
}
if err = conn.Send("SET", key, "1"); err != nil {
log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
PromError("redis:SET阅读数")
return
}
if err = conn.Send("EXPIRE", key, d.viewCacheTTL); err != nil {
log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
PromError("redis:EXPIRE阅读数")
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%+v)", err)
PromError("redis:阅读数缓存Flush")
return
}
for i := 0; i < 2; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%+v)", err)
PromError("redis:阅读数缓存Receive")
return
}
}
return
}
// DupViewIntercept intercepts illegal views.
func (d *Dao) DupViewIntercept(c context.Context, aid, mid int64) (ban bool) {
if mid == 0 {
return
}
var (
err error
exist bool
key = dupViewKey(aid, mid)
conn = d.redis.Get(c)
)
defer conn.Close()
if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
log.Error("conn.Do(EXISTS, %s) error(%+v)", key, err)
PromError("redis:EXISTS连续阅读数")
return
}
if exist {
ban = true
return
}
if err = conn.Send("SET", key, "1"); err != nil {
log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
PromError("redis:SET连续阅读数")
return
}
if err = conn.Send("EXPIRE", key, d.dupViewCacheTTL); err != nil {
log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
PromError("redis:EXPIRE连续阅读数")
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%+v)", err)
PromError("redis:连续阅读数缓存Flush")
return
}
for i := 0; i < 2; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%+v)", err)
PromError("redis:连续阅读数缓存Receive")
return
}
}
return
}
// PushStat pushs failed item to redis.
func (d *Dao) PushStat(c context.Context, retry *StatRetry) (err error) {
var (
length int64
bs []byte
conn = d.redis.Get(c)
)
defer conn.Close()
if length, err = redis.Int64(conn.Do("LLEN", _retryStatKey)); err != nil {
log.Error("conn.Do(%s) error(%+v)", _retryStatKey, err)
PromError("redis:计数重试LLEN")
return
}
cacheLen.State("redis:retry_stat_length", length)
if bs, err = json.Marshal(retry); err != nil {
log.Error("json.Marshal(%v) error(%+v)", retry, err)
PromError("redis:计数重试消息Marshal")
return
}
if _, err = conn.Do("RPUSH", _retryStatKey, bs); err != nil {
log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
PromError("redis:计数重试RPUSH")
}
return
}
// PopStat pops failed item from redis.
func (d *Dao) PopStat(c context.Context) (bs []byte, err error) {
var conn = d.redis.Get(c)
defer conn.Close()
if bs, err = redis.Bytes(conn.Do("LPOP", _retryStatKey)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryStatKey, err)
PromError("redis:计数重试LPOP")
}
return
}
// PushReply opens article's reply.
func (d *Dao) PushReply(c context.Context, aid, mid int64) (err error) {
var (
length int64
conn = d.redis.Get(c)
bs = []byte(strconv.FormatInt(aid, 10) + "_" + strconv.FormatInt(mid, 10))
)
defer conn.Close()
if length, err = redis.Int64(conn.Do("LLEN", _retryReplyKey)); err != nil {
log.Error("conn.Do(%s) error(%+v)", _retryReplyKey, err)
PromError("redis:打开评论重试LLEN")
return
}
cacheLen.State("redis:retry_reply_length", length)
if _, err = conn.Do("RPUSH", _retryReplyKey, bs); err != nil {
log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
PromError("redis:打开评论重试RPUSH")
}
return
}
// PopReply consume reply's job.
func (d *Dao) PopReply(c context.Context) (aid, mid int64, err error) {
var (
conn = d.redis.Get(c)
bs []byte
v string
arr []string
)
defer conn.Close()
if bs, err = redis.Bytes(conn.Do("LPOP", _retryReplyKey)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryReplyKey, err)
PromError("redis:打开评论重试LPOP")
return
}
if v = string(bs); v == "" {
return
}
if arr = strings.Split(v, "_"); len(arr) < 2 {
log.Error("reply retry param error (%s)", v)
PromError("redis:打开评论重试消息内容错误")
return
}
aid, _ = strconv.ParseInt(arr[0], 10, 64)
mid, _ = strconv.ParseInt(arr[1], 10, 64)
return
}
// PushCDN .
func (d *Dao) PushCDN(c context.Context, file string) (err error) {
var (
length int64
conn = d.redis.Get(c)
bs = []byte(file)
)
defer conn.Close()
if length, err = redis.Int64(conn.Do("LLEN", _retryCDNKey)); err != nil {
log.Error("conn.Do(%s) error(%+v)", _retryCDNKey, err)
PromError("redis:重试刷新CDN LLEN")
return
}
cacheLen.State("redis:retry_cdn_length", length)
if _, err = conn.Do("RPUSH", _retryCDNKey, bs); err != nil {
log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
PromError("redis:重试刷新CDN RPUSH")
}
return
}
// PopCDN .
func (d *Dao) PopCDN(c context.Context) (file string, err error) {
var (
conn = d.redis.Get(c)
bs []byte
)
defer conn.Close()
if bs, err = redis.Bytes(conn.Do("LPOP", _retryCDNKey)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryCDNKey, err)
PromError("redis:重试刷新CDN LPOP")
return
}
file = string(bs)
return
}
// CacheRetry struct of retry cache info.
type CacheRetry struct {
Action int `json:"action"`
Aid int64 `json:"aid"`
Mid int64 `json:"mid"`
Cid int64 `json:"cid"`
}
// PushArtCache .
func (d *Dao) PushArtCache(c context.Context, info *CacheRetry) (err error) {
var (
length int64
bs []byte
conn = d.redis.Get(c)
)
defer conn.Close()
if length, err = redis.Int64(conn.Do("LLEN", _retryArtCacheKey)); err != nil {
log.Error("conn.Do(%s) error(%+v)", _retryArtCacheKey, err)
PromError("redis:重试文章缓存LLEN")
return
}
cacheLen.State("redis:retry_art_cache_length", length)
if bs, err = json.Marshal(info); err != nil {
log.Error("json.Marshal(%v) error(%+v)", info, err)
PromError("redis:重试文章缓存Marshal")
return
}
if _, err = conn.Do("RPUSH", _retryArtCacheKey, bs); err != nil {
log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
PromError("redis:重试文章缓存RPUSH")
}
return
}
// PopArtCache .
func (d *Dao) PopArtCache(c context.Context) (bs []byte, err error) {
var conn = d.redis.Get(c)
defer conn.Close()
if bs, err = redis.Bytes(conn.Do("LPOP", _retryArtCacheKey)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryArtCacheKey, err)
PromError("redis:重试文章缓存LPOP")
}
return
}
// PushGameCache .
func (d *Dao) PushGameCache(c context.Context, info *model.GameCacheRetry) (err error) {
var (
length int64
bs []byte
conn = d.redis.Get(c)
)
defer conn.Close()
if length, err = redis.Int64(conn.Do("LLEN", _retryGameCacheKey)); err != nil {
log.Error("conn.Do(%s) error(%+v)", _retryGameCacheKey, err)
PromError("redis:重试游戏缓存LLEN")
return
}
cacheLen.State("redis:retry_game_cache_length", length)
if bs, err = json.Marshal(info); err != nil {
log.Error("json.Marshal(%v) error(%+v)", info, err)
PromError("redis:重试游戏缓存Marshal")
return
}
if _, err = conn.Do("RPUSH", _retryGameCacheKey, bs); err != nil {
log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
PromError("redis:重试游戏缓存RPUSH")
}
return
}
// PopGameCache .
func (d *Dao) PopGameCache(c context.Context) (res *model.GameCacheRetry, err error) {
var conn = d.redis.Get(c)
defer conn.Close()
var bs []byte
if bs, err = redis.Bytes(conn.Do("LPOP", _retryGameCacheKey)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryGameCacheKey, err)
PromError("redis:重试游戏缓存LPOP")
return
}
res = new(model.GameCacheRetry)
if err = json.Unmarshal(bs, res); err != nil {
log.Error("redis.Unmarshal(%s) error(%+v)", bs, err)
PromError("redis:解析游戏缓存")
}
return
}
// PushFlowCache .
func (d *Dao) PushFlowCache(c context.Context, info *model.FlowCacheRetry) (err error) {
var (
length int64
bs []byte
conn = d.redis.Get(c)
)
defer conn.Close()
if length, err = redis.Int64(conn.Do("LLEN", _retryFlowCacheKey)); err != nil {
log.Error("conn.Do(%s) error(%+v)", _retryFlowCacheKey, err)
PromError("redis:重试flow缓存LLEN")
return
}
cacheLen.State("redis:retry_flow_cache_length", length)
if bs, err = json.Marshal(info); err != nil {
log.Error("json.Marshal(%v) error(%+v)", info, err)
PromError("redis:重试flow缓存Marshal")
return
}
if _, err = conn.Do("RPUSH", _retryFlowCacheKey, bs); err != nil {
log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
PromError("redis:重试flow缓存RPUSH")
}
return
}
// PopFlowCache .
func (d *Dao) PopFlowCache(c context.Context) (res *model.FlowCacheRetry, err error) {
var conn = d.redis.Get(c)
defer conn.Close()
var bs []byte
if bs, err = redis.Bytes(conn.Do("LPOP", _retryFlowCacheKey)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryFlowCacheKey, err)
PromError("redis:重试flow缓存LPOP")
return
}
res = new(model.FlowCacheRetry)
if err = json.Unmarshal(bs, res); err != nil {
log.Error("redis.Unmarshal(%s) error(%+v)", bs, err)
PromError("redis:解析flow缓存")
}
return
}
// PushDynamicCache put dynamic to redis
func (d *Dao) PushDynamicCache(c context.Context, info *model.DynamicCacheRetry) (err error) {
var (
length int64
bs []byte
conn = d.redis.Get(c)
)
defer conn.Close()
if length, err = redis.Int64(conn.Do("LLEN", _retryDynamicCacheKey)); err != nil {
log.Error("conn.Do(%s) error(%+v)", _retryDynamicCacheKey, err)
PromError("redis:重试dynamic缓存LLEN")
return
}
cacheLen.State("redis:retry_dynamic_cache_length", length)
if bs, err = json.Marshal(info); err != nil {
log.Error("json.Marshal(%v) error(%+v)", info, err)
PromError("redis:重试dynamic缓存Marshal")
return
}
if _, err = conn.Do("RPUSH", _retryDynamicCacheKey, bs); err != nil {
log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
PromError("redis:重试dynamic缓存RPUSH")
}
return
}
// PopDynamicCache .
func (d *Dao) PopDynamicCache(c context.Context) (res *model.DynamicCacheRetry, err error) {
var conn = d.redis.Get(c)
defer conn.Close()
var bs []byte
if bs, err = redis.Bytes(conn.Do("LPOP", _retryDynamicCacheKey)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryDynamicCacheKey, err)
PromError("redis:重试dynamic缓存LPOP")
return
}
res = new(model.DynamicCacheRetry)
if err = json.Unmarshal(bs, res); err != nil {
log.Error("redis.Unmarshal(%s) error(%+v)", bs, err)
PromError("redis:解析dynamic缓存")
}
return
}
// GetRecheckCache get recheck info from redis
func (d *Dao) GetRecheckCache(c context.Context, aid int64) (isRecheck bool, err error) {
var conn = d.redis.Get(c)
defer conn.Close()
key := recheckKey(aid)
if isRecheck, err = redis.Bool(conn.Do("GET", key)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
log.Error("redis.BOOL(conn.Do(GET, %s)) error(%+v)", key, err)
PromError("redis:获取回查缓存")
return
}
return
}
// SetRecheckCache set recheck info to redis
func (d *Dao) SetRecheckCache(c context.Context, aid int64) (isRecheck bool, err error) {
var conn = d.redis.Get(c)
defer conn.Close()
key := recheckKey(aid)
if _, err = conn.Do("SETEX", key, 604800, true); err != nil {
log.Error("redis.BOOL(conn.Do(SETEX, %s)) error(%+v)", key, err)
PromError("redis:设置回查缓存")
return
}
return
}
func readPingSetKey() string {
return _readPingSet
}
func readPingKey(buvid string, aid int64) string {
return fmt.Sprintf(_prefixReadPing, buvid, aid)
}
// ReadPingSet 获取所有阅读记录(不删除)
func (d *Dao) ReadPingSet(c context.Context) (res []*model.Read, err error) {
var (
key = readPingSetKey()
conn = d.artRedis.Get(c)
tmpRes []string
tmpArr []string
)
defer conn.Close()
if err = conn.Send("SMEMBERS", key); err != nil {
log.Error("conn.Send(SMEMBERS, %s) error(%+v)", key, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush error(%+v)", err)
return
}
if tmpRes, err = redis.Strings(conn.Receive()); err != nil {
log.Error("conn.Receive error(%+v)", err)
return
}
for _, tmp := range tmpRes {
tmpArr = strings.Split(tmp, "|")
if len(tmpArr) != 6 {
log.Error("redis key(%s)存在脏数据(%s)", key, tmp)
if _, err = conn.Do("SREM", key, tmp); err != nil {
log.Error("d.Redis.SREM error(%+v), set(%s), key(%s)", err, key, tmp)
}
continue
}
read := &model.Read{
Buvid: tmpArr[0],
EndTime: 0,
}
read.Aid, _ = strconv.ParseInt(tmpArr[1], 10, 64)
read.Mid, _ = strconv.ParseInt(tmpArr[2], 10, 64)
read.IP = tmpArr[3]
read.StartTime, _ = strconv.ParseInt(tmpArr[4], 10, 64)
read.From = tmpArr[5]
res = append(res, read)
}
return
}
// ReadPing 获取上次阅读心跳时间不存在则返回0
func (d *Dao) ReadPing(c context.Context, buvid string, aid int64) (last int64, err error) {
var (
key = readPingKey(buvid, aid)
conn = d.artRedis.Get(c)
)
defer conn.Close()
if last, err = redis.Int64(conn.Do("GET", key)); err != nil && err != redis.ErrNil {
log.Error("conn.Do(GET, %s) error(%+v)", key, err)
return
}
err = nil
return
}
// DelReadPingSet 删除阅读记录缓存
func (d *Dao) DelReadPingSet(c context.Context, read *model.Read) (err error) {
if read == nil {
return
}
var (
elemKey = readPingKey(read.Buvid, read.Aid)
setKey = readPingSetKey()
value = fmt.Sprintf("%s|%d|%d|%s|%d|%s", read.Buvid, read.Aid, read.Mid, read.IP, read.StartTime, read.From)
conn = d.artRedis.Get(c)
)
defer conn.Close()
if _, err = conn.Do("DEL", elemKey); err != nil {
log.Error("conn.Do(DEL, %s) error(%+v)", elemKey, err)
return
}
if _, err = conn.Do("SREM", setKey, value); err != nil {
log.Error("conn.Do(SREM, %s, %s) error(%+v)", setKey, value, err)
return
}
return
}

View File

@@ -0,0 +1,69 @@
package dao
import (
"context"
"fmt"
"testing"
"go-common/library/cache/redis"
. "github.com/smartystreets/goconvey/convey"
)
func Test_Redis(t *testing.T) {
Convey("test redis", t, WithDao(func(d *Dao) {
c := context.TODO()
conn := d.redis.Get(c)
defer conn.Close()
conn.Do("SET", "name", "echo")
if t, err := redis.Bool(conn.Do("EXISTS", "name")); err != nil {
fmt.Println(t)
fmt.Println(err)
_ = t
} else {
fmt.Println(t)
fmt.Println(err)
_ = t
}
fmt.Println("done")
err := d.PushStat(c, &StatRetry{})
So(err, ShouldBeNil)
res, err := d.PopStat(c)
So(err, ShouldBeNil)
So(res, ShouldNotBeNil)
So(d.Intercept(c, 1, 2, ""), ShouldBeFalse)
So(d.DupViewIntercept(c, 1, 2), ShouldBeTrue)
So(d.PushStat(c, nil), ShouldBeNil)
_, err = d.PopStat(c)
So(err, ShouldBeNil)
err = d.PushReply(c, 1, 2)
So(err, ShouldBeNil)
_, _, err = d.PopReply(c)
So(err, ShouldBeNil)
err = d.PushCDN(c, "")
So(err, ShouldBeNil)
_, err = d.PopCDN(c)
So(err, ShouldBeNil)
err = d.PushArtCache(c, nil)
So(err, ShouldBeNil)
_, err = d.PopArtCache(c)
So(err, ShouldBeNil)
err = d.PushGameCache(c, nil)
So(err, ShouldBeNil)
_, err = d.PopGameCache(c)
So(err, ShouldBeNil)
err = d.PushFlowCache(c, nil)
So(err, ShouldBeNil)
_, err = d.PopFlowCache(c)
So(err, ShouldBeNil)
err = d.PushDynamicCache(c, nil)
So(err, ShouldBeNil)
_, err = d.PopDynamicCache(c)
So(err, ShouldBeNil)
}))
}

View File

@@ -0,0 +1,82 @@
package dao
import (
"context"
"net/url"
"strconv"
"time"
"go-common/library/ecode"
"go-common/library/log"
)
const (
_replyType = "12"
_replyStateOpen = "0" // 0: open, 1: close
_replyStateClose = "1"
_replyURL = "http://api.bilibili.co/x/internal/v2/reply/subject/regist"
)
// OpenReply opens article's reply.
func (d *Dao) OpenReply(c context.Context, aid, mid int64) (err error) {
defer func() {
if err == nil {
return
}
time.Sleep(time.Second)
if e := d.PushReply(c, aid, mid); e != nil {
log.Error("d.PushReply(%d,%d) error(%+v)", aid, mid, e)
}
}()
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("oid", strconv.FormatInt(aid, 10))
params.Set("type", _replyType)
params.Set("state", _replyStateOpen)
var res struct {
Code int `json:"code"`
}
if err = d.httpClient.Post(c, _replyURL, "", params, &res); err != nil {
log.Error("d.httpClient.Post(%s) error(%+v)", _replyURL+"?"+params.Encode(), err)
PromError("reply:打开评论")
return
}
if res.Code != ecode.OK.Code() {
log.Error("d.httpClient.Post(%s) code(%d)", _replyURL+"?"+params.Encode(), res.Code)
PromError("reply:打开评论状态码异常")
err = ecode.Int(res.Code)
}
return
}
// CloseReply close article's reply.
func (d *Dao) CloseReply(c context.Context, aid, mid int64) (err error) {
defer func() {
if err == nil {
return
}
time.Sleep(time.Second)
if e := d.PushReply(c, aid, mid); e != nil {
log.Error("d.PushReply(%d,%d) error(%+v)", aid, mid, e)
}
}()
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("oid", strconv.FormatInt(aid, 10))
params.Set("type", _replyType)
params.Set("state", _replyStateClose)
var res struct {
Code int `json:"code"`
}
if err = d.httpClient.Post(c, _replyURL, "", params, &res); err != nil {
log.Error("d.httpClient.Post(%s) error(%+v)", _replyURL+"?"+params.Encode(), err)
PromError("reply:打开评论")
return
}
if res.Code != ecode.OK.Code() {
log.Error("d.httpClient.Post(%s) code(%d)", _replyURL+"?"+params.Encode(), res.Code)
PromError("reply:打开评论状态码异常")
err = ecode.Int(res.Code)
}
return
}

View File

@@ -0,0 +1,15 @@
package dao
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_OpenReply(t *testing.T) {
Convey("work", t, WithDao(func(d *Dao) {
err := d.OpenReply(context.Background(), 1, 2)
So(err, ShouldBeNil)
}))
}