go-common/app/interface/openplatform/article/dao/memcached.go
2019-04-22 18:49:16 +08:00

664 lines
17 KiB
Go

package dao
import (
"context"
"fmt"
"strconv"
"strings"
"sync"
"time"
"go-common/app/interface/openplatform/article/model"
"go-common/library/cache/memcache"
"go-common/library/ecode"
"go-common/library/log"
"go-common/library/xstr"
"go-common/library/sync/errgroup"
)
const (
_prefixArtMeta = "art_mp_%d"
_prefixArtContent = "art_c_%d"
_prefixArtKeywords = "art_kw_%d"
_prefixArtStat = "art_s_%d"
_prefixCard = "art_cards_"
_bulkSize = 50
)
func artMetaKey(id int64) string {
return fmt.Sprintf(_prefixArtMeta, id)
}
func artContentKey(id int64) string {
return fmt.Sprintf(_prefixArtContent, id)
}
func artKeywordsKey(id int64) string {
return fmt.Sprintf(_prefixArtKeywords, id)
}
func artStatsKey(id int64) string {
return fmt.Sprintf(_prefixArtStat, id)
}
func cardKey(id string) string {
return _prefixCard + id
}
func hotspotsKey() string {
return fmt.Sprintf("art_hotspots")
}
func mcHotspotKey(id int64) string {
return fmt.Sprintf("art_hotspot_%d", id)
}
func mcAuthorKey(mid int64) string {
return fmt.Sprintf("art_author_%d", mid)
}
func mcTagKey(tag int64) string {
return fmt.Sprintf("tag_aids_%d", tag)
}
func mcUpStatKey(mid int64) string {
var (
hour int
day int
)
now := time.Now()
hour = now.Hour()
if hour < 7 {
day = now.Add(time.Hour * -24).Day()
} else {
day = now.Day()
}
return fmt.Sprintf("up_stat_daily_%d_%d", mid, day)
}
// statsValue convert stats to string, format: "view,favorite,like,unlike,reply..."
func statsValue(s *model.Stats) string {
if s == nil {
return ",,,,,,"
}
ids := []int64{s.View, s.Favorite, s.Like, s.Dislike, s.Reply, s.Share, s.Coin}
return xstr.JoinInts(ids)
}
func revoverStatsValue(c context.Context, s string) (res *model.Stats) {
var (
vs []int64
err error
)
res = new(model.Stats)
if s == "" {
return
}
if vs, err = xstr.SplitInts(s); err != nil || len(vs) < 7 {
PromError("mc:stats解析")
log.Error("dao.revoverStatsValue(%s) err: %+v", s, err)
return
}
res = &model.Stats{
View: vs[0],
Favorite: vs[1],
Like: vs[2],
Dislike: vs[3],
Reply: vs[4],
Share: vs[5],
Coin: vs[6],
}
return
}
// pingMc ping memcache
func (d *Dao) pingMC(c context.Context) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
item := memcache.Item{Key: "ping", Value: []byte{1}, Expiration: d.mcArticleExpire}
err = conn.Set(&item)
return
}
//AddArticlesMetaCache add articles meta cache
func (d *Dao) AddArticlesMetaCache(c context.Context, vs ...*model.Meta) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
for _, v := range vs {
if v == nil {
continue
}
item := &memcache.Item{Key: artMetaKey(v.ID), Object: v, Flags: memcache.FlagProtobuf, Expiration: d.mcArticleExpire}
if err = conn.Set(item); err != nil {
PromError("mc:增加文章meta缓存")
log.Error("conn.Store(%s) error(%+v)", artMetaKey(v.ID), err)
return
}
}
return
}
// ArticleMetaCache gets article's meta cache.
func (d *Dao) ArticleMetaCache(c context.Context, aid int64) (res *model.Meta, err error) {
var (
conn = d.mc.Get(c)
key = artMetaKey(aid)
)
defer conn.Close()
reply, err := conn.Get(key)
if err != nil {
if err == memcache.ErrNotFound {
missedCount.Incr("article-meta")
err = nil
return
}
PromError("mc:获取文章meta缓存")
log.Error("conn.Get(%v) error(%+v)", key, err)
return
}
res = &model.Meta{}
if err = conn.Scan(reply, res); err != nil {
PromError("mc:文章meta缓存json解析")
log.Error("reply.Scan(%s) error(%+v)", reply.Value, err)
return
}
res.Strong()
cachedCount.Incr("article-meta")
return
}
//ArticlesMetaCache articles meta cache
func (d *Dao) ArticlesMetaCache(c context.Context, ids []int64) (cached map[int64]*model.Meta, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.Meta, len(ids))
allKeys := make([]string, 0, len(ids))
idmap := make(map[string]int64, len(ids))
for _, id := range ids {
k := artMetaKey(id)
allKeys = append(allKeys, k)
idmap[k] = id
}
group, errCtx := errgroup.WithContext(c)
mutex := sync.Mutex{}
keysLen := len(allKeys)
for i := 0; i < keysLen; i += _bulkSize {
var keys []string
if (i + _bulkSize) > keysLen {
keys = allKeys[i:]
} else {
keys = allKeys[i : i+_bulkSize]
}
group.Go(func() (err error) {
conn := d.mc.Get(errCtx)
defer conn.Close()
replys, err := conn.GetMulti(keys)
if err != nil {
PromError("mc:获取文章meta缓存")
log.Error("conn.Gets(%v) error(%+v)", keys, err)
err = nil
return
}
for key, item := range replys {
art := &model.Meta{}
if err = conn.Scan(item, art); err != nil {
PromError("mc:文章meta缓存json解析")
log.Error("item.Scan(%s) error(%+v)", item.Value, err)
err = nil
continue
}
mutex.Lock()
cached[idmap[key]] = art.Strong()
delete(idmap, key)
mutex.Unlock()
}
return
})
}
group.Wait()
missed = make([]int64, 0, len(idmap))
for _, id := range idmap {
missed = append(missed, id)
}
missedCount.Add("article-meta", int64(len(missed)))
cachedCount.Add("article-meta", int64(len(cached)))
return
}
// AddArticleStatsCache batch set article cache.
func (d *Dao) AddArticleStatsCache(c context.Context, id int64, v *model.Stats) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
bs := []byte(statsValue(v))
item := &memcache.Item{Key: artStatsKey(id), Value: bs, Expiration: d.mcStatsExpire}
if err = conn.Set(item); err != nil {
PromError("mc:增加文章统计缓存")
log.Error("conn.Store(%s) error(%+v)", artStatsKey(id), err)
}
return
}
//AddArticleContentCache add article content cache
func (d *Dao) AddArticleContentCache(c context.Context, id int64, content string) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
var bs = []byte(content)
item := &memcache.Item{Key: artContentKey(id), Value: bs, Expiration: d.mcArticleExpire, Flags: memcache.FlagGzip}
if err = conn.Set(item); err != nil {
PromError("mc:增加文章内容缓存")
log.Error("conn.Store(%s) error(%+v)", artContentKey(id), err)
}
return
}
// AddArticleKeywordsCache add article keywords cache.
func (d *Dao) AddArticleKeywordsCache(c context.Context, id int64, keywords string) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
var bs = []byte(keywords)
item := &memcache.Item{Key: artKeywordsKey(id), Value: bs, Expiration: d.mcArticleExpire, Flags: memcache.FlagGzip}
if err = conn.Set(item); err != nil {
PromError("mc:增加文章关键字缓存")
log.Error("conn.Store(%s) error(%+v)", artKeywordsKey(id), err)
}
return
}
// ArticleContentCache article content cache
func (d *Dao) ArticleContentCache(c context.Context, id int64) (res string, err error) {
conn := d.mc.Get(c)
defer conn.Close()
reply, err := conn.Get(artContentKey(id))
if err != nil {
if err == memcache.ErrNotFound {
missedCount.Incr("article-content")
err = nil
return
}
PromError("mc:获取文章内容缓存")
log.Error("conn.Get(%v) error(%+v)", artContentKey(id), err)
return
}
err = conn.Scan(reply, &res)
return
}
// ArticleKeywordsCache article Keywords cache
func (d *Dao) ArticleKeywordsCache(c context.Context, id int64) (res string, err error) {
conn := d.mc.Get(c)
defer conn.Close()
reply, err := conn.Get(artKeywordsKey(id))
if err != nil {
if err == memcache.ErrNotFound {
missedCount.Incr("article-keywords")
err = nil
return
}
PromError("mc:获取文章关键字缓存")
log.Error("conn.Get(%v) error(%+v)", artKeywordsKey(id), err)
return
}
err = conn.Scan(reply, &res)
return
}
//DelArticleMetaCache delete article meta cache
func (d *Dao) DelArticleMetaCache(c context.Context, id int64) (err error) {
var (
key = artMetaKey(id)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Delete(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
PromError("mc:删除文章meta缓存")
log.Error("key(%v) error(%+v)", key, err)
}
}
return
}
// DelArticleStatsCache delete article stats cache
func (d *Dao) DelArticleStatsCache(c context.Context, id int64) (err error) {
var (
key = artStatsKey(id)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Delete(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
PromError("mc:删除文章stats缓存")
log.Error("key(%v) error(%+v)", key, err)
}
}
return
}
//DelArticleContentCache delete article content cache
func (d *Dao) DelArticleContentCache(c context.Context, id int64) (err error) {
var (
key = artContentKey(id)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Delete(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
PromError("mc:删除文章content缓存")
log.Error("key(%v) error(%+v)", key, err)
}
}
return
}
// ArticleStatsCache article stats cache
func (d *Dao) ArticleStatsCache(c context.Context, id int64) (res *model.Stats, err error) {
if id == 0 {
err = ecode.NothingFound
return
}
var (
conn = d.mc.Get(c)
key = artStatsKey(id)
statsStr string
)
defer conn.Close()
reply, err := conn.Get(key)
if err != nil {
if err == memcache.ErrNotFound {
res = nil
err = nil
return
}
PromError("mc:获取文章计数缓存")
log.Error("conn.Get(%v) error(%+v)", key, err)
return
}
if err = conn.Scan(reply, &statsStr); err == nil {
res = revoverStatsValue(c, statsStr)
} else {
PromError("mc:获取文章计数缓存")
log.Error("dao.ArticleStatsCache.reply.Scan(%v, %v) error(%+v)", key, statsStr, err)
}
return
}
// ArticlesStatsCache articles stats cache
func (d *Dao) ArticlesStatsCache(c context.Context, ids []int64) (cached map[int64]*model.Stats, missed []int64, err error) {
if len(ids) == 0 {
return
}
cached = make(map[int64]*model.Stats, len(ids))
allKeys := make([]string, 0, len(ids))
idmap := make(map[string]int64, len(ids))
for _, id := range ids {
k := artStatsKey(id)
allKeys = append(allKeys, k)
idmap[k] = id
}
group, errCtx := errgroup.WithContext(c)
mutex := sync.Mutex{}
keysLen := len(allKeys)
for i := 0; i < keysLen; i += _bulkSize {
var keys []string
if (i + _bulkSize) > keysLen {
keys = allKeys[i:]
} else {
keys = allKeys[i : i+_bulkSize]
}
group.Go(func() (err error) {
conn := d.mc.Get(errCtx)
defer conn.Close()
replys, err := conn.GetMulti(keys)
if err != nil {
PromError("mc:获取文章计数缓存")
log.Error("conn.Gets(%v) error(%+v)", keys, err)
err = nil
return
}
for _, reply := range replys {
var info string
if e := conn.Scan(reply, &info); e != nil {
PromError("mc:获取文章计数缓存scan")
continue
}
art := revoverStatsValue(c, info)
mutex.Lock()
cached[idmap[reply.Key]] = art
delete(idmap, reply.Key)
mutex.Unlock()
}
return
})
}
group.Wait()
missed = make([]int64, 0, len(idmap))
for _, id := range idmap {
missed = append(missed, id)
}
missedCount.Add("article-stats", int64(len(missed)))
cachedCount.Add("article-stats", int64(len(cached)))
return
}
// AddCardsCache .
func (d *Dao) addCardsCache(c context.Context, vs ...*model.Cards) (err error) {
if len(vs) == 0 {
return
}
conn := d.mc.Get(c)
defer conn.Close()
for _, v := range vs {
if v == nil {
continue
}
key := cardKey(v.Key())
item := memcache.Item{Key: key, Object: v, Expiration: d.mcCardsExpire, Flags: memcache.FlagJSON}
if err = conn.Set(&item); err != nil {
PromError("mc:增加卡片缓存")
log.Error("conn.Set(%s) error(%+v)", key, err)
return
}
}
return
}
// CardsCache ids like cv123 av123 au123
func (d *Dao) cardsCache(c context.Context, ids []string) (res map[string]*model.Cards, err error) {
if len(ids) == 0 {
return
}
res = make(map[string]*model.Cards, len(ids))
var keys []string
for _, id := range ids {
keys = append(keys, cardKey(id))
}
conn := d.mc.Get(c)
replys, err := conn.GetMulti(keys)
defer conn.Close()
if err != nil {
PromError("mc:获取cards缓存")
log.Error("conn.Gets(%v) error(%+v)", keys, err)
err = nil
return
}
for _, reply := range replys {
s := model.Cards{}
if err = conn.Scan(reply, &s); err != nil {
PromError("获取cards缓存json解析")
log.Error("json.Unmarshal(%v) error(%+v)", reply.Value, err)
err = nil
continue
}
res[strings.TrimPrefix(reply.Key, _prefixCard)] = &s
}
return
}
// AddBangumiCardsCache .
func (d *Dao) AddBangumiCardsCache(c context.Context, vs map[int64]*model.BangumiCard) (err error) {
var cards []*model.Cards
for _, v := range vs {
cards = append(cards, &model.Cards{Type: model.CardPrefixBangumi, BangumiCard: v})
}
err = d.addCardsCache(c, cards...)
return
}
// BangumiCardsCache .
func (d *Dao) BangumiCardsCache(c context.Context, ids []int64) (vs map[int64]*model.BangumiCard, err error) {
var cards map[string]*model.Cards
var idsStr []string
for _, id := range ids {
idsStr = append(idsStr, model.CardPrefixBangumi+strconv.FormatInt(id, 10))
}
if cards, err = d.cardsCache(c, idsStr); err != nil {
return
}
vs = make(map[int64]*model.BangumiCard)
for _, card := range cards {
if (card != nil) && (card.BangumiCard != nil) {
vs[card.BangumiCard.ID] = card.BangumiCard
}
}
return
}
// AddBangumiEpCardsCache .
func (d *Dao) AddBangumiEpCardsCache(c context.Context, vs map[int64]*model.BangumiCard) (err error) {
var cards []*model.Cards
for _, v := range vs {
cards = append(cards, &model.Cards{Type: model.CardPrefixBangumiEp, BangumiCard: v})
}
err = d.addCardsCache(c, cards...)
return
}
// BangumiEpCardsCache .
func (d *Dao) BangumiEpCardsCache(c context.Context, ids []int64) (vs map[int64]*model.BangumiCard, err error) {
var cards map[string]*model.Cards
var idsStr []string
for _, id := range ids {
idsStr = append(idsStr, model.CardPrefixBangumiEp+strconv.FormatInt(id, 10))
}
if cards, err = d.cardsCache(c, idsStr); err != nil {
return
}
vs = make(map[int64]*model.BangumiCard)
for _, card := range cards {
if (card != nil) && (card.BangumiCard != nil) {
vs[card.BangumiCard.ID] = card.BangumiCard
}
}
return
}
// AddAudioCardsCache .
func (d *Dao) AddAudioCardsCache(c context.Context, vs map[int64]*model.AudioCard) (err error) {
var cards []*model.Cards
for _, v := range vs {
cards = append(cards, &model.Cards{Type: model.CardPrefixAudio, AudioCard: v})
}
err = d.addCardsCache(c, cards...)
return
}
// AudioCardsCache .
func (d *Dao) AudioCardsCache(c context.Context, ids []int64) (vs map[int64]*model.AudioCard, err error) {
var cards map[string]*model.Cards
var idsStr []string
for _, id := range ids {
idsStr = append(idsStr, model.CardPrefixAudio+strconv.FormatInt(id, 10))
}
if cards, err = d.cardsCache(c, idsStr); err != nil {
return
}
vs = make(map[int64]*model.AudioCard)
for _, card := range cards {
if (card != nil) && (card.AudioCard != nil) {
vs[card.AudioCard.ID] = card.AudioCard
}
}
return
}
// AddMallCardsCache .
func (d *Dao) AddMallCardsCache(c context.Context, vs map[int64]*model.MallCard) (err error) {
var cards []*model.Cards
for _, v := range vs {
cards = append(cards, &model.Cards{Type: model.CardPrefixMall, MallCard: v})
}
err = d.addCardsCache(c, cards...)
return
}
// MallCardsCache .
func (d *Dao) MallCardsCache(c context.Context, ids []int64) (vs map[int64]*model.MallCard, err error) {
var cards map[string]*model.Cards
var idsStr []string
for _, id := range ids {
idsStr = append(idsStr, model.CardPrefixMall+strconv.FormatInt(id, 10))
}
if cards, err = d.cardsCache(c, idsStr); err != nil {
return
}
vs = make(map[int64]*model.MallCard)
for _, card := range cards {
if (card != nil) && (card.MallCard != nil) {
vs[card.MallCard.ID] = card.MallCard
}
}
return
}
// AddTicketCardsCache .
func (d *Dao) AddTicketCardsCache(c context.Context, vs map[int64]*model.TicketCard) (err error) {
var cards []*model.Cards
for _, v := range vs {
cards = append(cards, &model.Cards{Type: model.CardPrefixTicket, TicketCard: v})
}
err = d.addCardsCache(c, cards...)
return
}
// TicketCardsCache .
func (d *Dao) TicketCardsCache(c context.Context, ids []int64) (vs map[int64]*model.TicketCard, err error) {
var cards map[string]*model.Cards
var idsStr []string
for _, id := range ids {
idsStr = append(idsStr, model.CardPrefixTicket+strconv.FormatInt(id, 10))
}
if cards, err = d.cardsCache(c, idsStr); err != nil {
return
}
vs = make(map[int64]*model.TicketCard)
for _, card := range cards {
if (card != nil) && (card.TicketCard != nil) {
vs[card.TicketCard.ID] = card.TicketCard
}
}
return
}
// CacheHotspots .
func (d *Dao) CacheHotspots(c context.Context) (res []*model.Hotspot, err error) {
res, err = d.cacheHotspots(c)
for _, r := range res {
if r.TopArticles == nil {
r.TopArticles = []int64{}
}
}
return
}