144 lines
3.0 KiB
Go
144 lines
3.0 KiB
Go
package income
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"fmt"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
model "go-common/app/job/main/growup/model/income"
|
||
task "go-common/app/job/main/growup/service"
|
||
|
||
"go-common/library/log"
|
||
)
|
||
|
||
// BGMs map[avid][]*model.BGM,
|
||
func (s *Service) BGMs(c context.Context, limit int64) (bm map[int64][]*model.BGM, err error) {
|
||
var id int64
|
||
bm = make(map[int64][]*model.BGM)
|
||
for {
|
||
var bs []*model.BGM
|
||
bs, id, err = s.dao.GetBGM(c, id, limit)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if len(bs) == 0 {
|
||
break
|
||
}
|
||
for _, b := range bs {
|
||
if _, ok := bm[b.AID]; ok {
|
||
bm[b.AID] = append(bm[b.AID], b)
|
||
} else {
|
||
bm[b.AID] = []*model.BGM{b}
|
||
}
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func (s *Service) batchInsertBGMs(c context.Context, bs []*model.BGM) (err error) {
|
||
var (
|
||
buff = make([]*model.BGM, 2000)
|
||
buffEnd = 0
|
||
)
|
||
for _, b := range bs {
|
||
buff[buffEnd] = b
|
||
buffEnd++
|
||
if buffEnd >= 2000 {
|
||
values := bgmValues(buff[:buffEnd])
|
||
buffEnd = 0
|
||
_, err = s.dao.InsertBGM(c, values)
|
||
if err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
if buffEnd > 0 {
|
||
values := bgmValues(buff[:buffEnd])
|
||
buffEnd = 0
|
||
_, err = s.dao.InsertBGM(c, values)
|
||
}
|
||
return
|
||
}
|
||
|
||
func bgmValues(bs []*model.BGM) (values string) {
|
||
var buf bytes.Buffer
|
||
for _, b := range bs {
|
||
buf.WriteString("(")
|
||
buf.WriteString(strconv.FormatInt(b.MID, 10))
|
||
buf.WriteByte(',')
|
||
buf.WriteString(strconv.FormatInt(b.SID, 10))
|
||
buf.WriteByte(',')
|
||
buf.WriteString(strconv.FormatInt(b.AID, 10))
|
||
buf.WriteByte(',')
|
||
buf.WriteString(strconv.FormatInt(b.CID, 10))
|
||
buf.WriteByte(',')
|
||
buf.WriteString("'" + b.JoinAt + "'")
|
||
buf.WriteByte(',')
|
||
buf.WriteString("\"" + strings.Replace(b.Title, "\"", "\\\"", -1) + "\"")
|
||
buf.WriteString(")")
|
||
buf.WriteByte(',')
|
||
}
|
||
if buf.Len() > 0 {
|
||
buf.Truncate(buf.Len() - 1)
|
||
}
|
||
values = buf.String()
|
||
buf.Reset()
|
||
return
|
||
}
|
||
|
||
const query = "{\"select\": [],\"page\": {\"skip\": %d,\"limit\": %d}}"
|
||
|
||
func (s *Service) getBGM(c context.Context) (bgms []*model.BGM, err error) {
|
||
from, limit := 0, 1000
|
||
var info []*model.BGM
|
||
for {
|
||
info, err = s.dp.SendBGMRequest(c, fmt.Sprintf(query, from, limit))
|
||
if err != nil {
|
||
log.Error("s.getBGM error(%v)", err)
|
||
return
|
||
}
|
||
if len(info) == 0 {
|
||
break
|
||
}
|
||
bgms = append(bgms, info...)
|
||
from += len(info)
|
||
time.Sleep(15 * time.Second) // 数据平台限流，api 10s内只能请求一次
|
||
}
|
||
log.Info("get bgm data total (%d) rows", from)
|
||
return
|
||
}
|
||
|
||
// SyncBgmInfo sync bgm info from data platform
|
||
func (s *Service) SyncBgmInfo(c context.Context) (err error) {
|
||
defer func() {
|
||
task.GetTaskService().SetTaskStatus(c, task.TaskBgmSync, time.Now().AddDate(0, 0, -1).Format(_layout), err)
|
||
}()
|
||
err = s.delBGM(c)
|
||
if err != nil {
|
||
return
|
||
}
|
||
bgms, err := s.getBGM(c)
|
||
if err != nil {
|
||
return
|
||
}
|
||
return s.batchInsertBGMs(c, bgms)
|
||
}
|
||
|
||
func (s *Service) delBGM(c context.Context) (err error) {
|
||
var limit int64 = 2000
|
||
for {
|
||
var rows int64
|
||
rows, err = s.dao.DelBGM(c, limit)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if rows < limit {
|
||
break
|
||
}
|
||
}
|
||
return
|
||
}
|