486 lines
12 KiB
Go
486 lines
12 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"fmt"
|
||
|
|
"sort"
|
||
|
|
"strconv"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"go-common/app/job/main/growup/model"
|
||
|
|
"go-common/library/database/sql"
|
||
|
|
"go-common/library/log"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
_layout = "2006-01-02"
|
||
|
|
)
|
||
|
|
|
||
|
|
// InsertTagIncome insert up_tag_income.
|
||
|
|
func (s *Service) InsertTagIncome(c context.Context, date time.Time) (err error) {
|
||
|
|
infos, err := s.getTagAvInfo(c, date)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.InsertTagIncome getTagAVInfo error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
tx, err := s.dao.BeginTran(c)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.InsertTagIncome dao.BeginTran error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if err = s.insertTagIncome(c, tx, infos); err != nil {
|
||
|
|
tx.Rollback()
|
||
|
|
log.Error("s.InsertTagIncome insertTagIncome error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if err = s.updateTagInfo(tx, infos); err != nil {
|
||
|
|
tx.Rollback()
|
||
|
|
log.Error("s.InsertTagIncome updateTagInfo error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if err = s.updateTagUpInfo(tx, infos); err != nil {
|
||
|
|
tx.Rollback()
|
||
|
|
log.Error("s.InsertTagIncome updateTagUpInfo error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if err = tx.Commit(); err != nil {
|
||
|
|
log.Error("s.InsertTagIncome tx.Commit error")
|
||
|
|
return
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) getTagAvInfo(c context.Context, date time.Time) (infos []*model.TagAvIncome, err error) {
|
||
|
|
var (
|
||
|
|
from, limit int64
|
||
|
|
av, avs []*model.ActivityAVInfo
|
||
|
|
)
|
||
|
|
from, limit = 0, 3000
|
||
|
|
for {
|
||
|
|
av, err = s.dao.GetAvTagRatio(c, from, limit)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.getTagAvInfo dao.GetAvTagRatio error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
avs = append(avs, av...)
|
||
|
|
if int64(len(av)) < limit {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
from = av[len(av)-1].MID
|
||
|
|
}
|
||
|
|
for _, a := range avs {
|
||
|
|
var income *model.TagAvIncome
|
||
|
|
income, err = s.dao.GetAvIncomeInfo(c, a.AVID, date)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.GetAvIncomes dao.GetAvIncomeInfo error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if income == nil {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
income.TagID = a.TagID
|
||
|
|
infos = append(infos, income)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) getTagAVLatestTotalIncome(c context.Context, avID, tagID int64) (totalIncome int, err error) {
|
||
|
|
infos, err := s.dao.GetTagAvTotalIncome(c, tagID, avID)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.getTagAVLatestTotalIncome dao.GetTagAvTotalIncome error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, info := range infos {
|
||
|
|
if int(info.TotalIncome) > totalIncome {
|
||
|
|
totalIncome = int(info.TotalIncome)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) insertTagIncome(c context.Context, tx *sql.Tx, infos []*model.TagAvIncome) (err error) {
|
||
|
|
var buf bytes.Buffer
|
||
|
|
var cnt, totalIncome int
|
||
|
|
var rows, totalRows int64
|
||
|
|
for _, info := range infos {
|
||
|
|
totalIncome, err = s.getTagAVLatestTotalIncome(c, info.TagID, info.AVID)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.insertTagIncome dao.GetTagAvTotalIncome error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
buf.WriteString("(")
|
||
|
|
buf.WriteString(strconv.FormatInt(info.TagID, 10))
|
||
|
|
buf.WriteString(",")
|
||
|
|
buf.WriteString(strconv.FormatInt(info.MID, 10))
|
||
|
|
buf.WriteString(",")
|
||
|
|
buf.WriteString(strconv.FormatInt(info.AVID, 10))
|
||
|
|
buf.WriteString(",")
|
||
|
|
buf.WriteString(strconv.Itoa(info.Income))
|
||
|
|
buf.WriteString(",")
|
||
|
|
buf.WriteString(strconv.Itoa(totalIncome + info.Income))
|
||
|
|
buf.WriteString(",")
|
||
|
|
buf.WriteString("'")
|
||
|
|
buf.WriteString(strconv.Itoa(info.Date.Year()))
|
||
|
|
buf.WriteString("-")
|
||
|
|
if int(info.Date.Month()) < 10 {
|
||
|
|
buf.WriteString("0")
|
||
|
|
}
|
||
|
|
buf.WriteString(strconv.Itoa(int(info.Date.Month())))
|
||
|
|
buf.WriteString("-")
|
||
|
|
if info.Date.Day() < 10 {
|
||
|
|
buf.WriteString("0")
|
||
|
|
}
|
||
|
|
buf.WriteString(strconv.Itoa(info.Date.Day()))
|
||
|
|
buf.WriteString("'")
|
||
|
|
buf.WriteString("),")
|
||
|
|
cnt++
|
||
|
|
if cnt%1000 == 0 {
|
||
|
|
buf.Truncate(buf.Len() - 1)
|
||
|
|
rows, err = s.dao.TxInsertTagIncome(tx, buf.String())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.InsertTagIncome dao.TxInsertTagIncome error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
totalRows += rows
|
||
|
|
buf.Reset()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if buf.Len() > 0 {
|
||
|
|
buf.Truncate(buf.Len() - 1)
|
||
|
|
rows, err = s.dao.TxInsertTagIncome(tx, buf.String())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.InsertTagIncome dao.TxInsertTagIncome error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
totalRows += rows
|
||
|
|
}
|
||
|
|
log.Info("s.InsertTagIncome insert up_tag_income (%d) rows", totalRows)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// updateTagInfo update tag_info total_income.
|
||
|
|
func (s *Service) updateTagInfo(tx *sql.Tx, infos []*model.TagAvIncome) (err error) {
|
||
|
|
tim := make(map[int64]int) // key-value: tag_id-total av income.
|
||
|
|
for _, info := range infos {
|
||
|
|
tim[info.TagID] += info.Income
|
||
|
|
}
|
||
|
|
for k, v := range tim {
|
||
|
|
query := "total_income = total_income + "
|
||
|
|
query += strconv.Itoa(v)
|
||
|
|
_, err = s.dao.TxUpdateTagInfo(tx, k, query)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.updateTagInfo dao.UpdateTagInfo error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// updateTagUpInfo update tag_up_info totalIncome.
|
||
|
|
func (s *Service) updateTagUpInfo(tx *sql.Tx, infos []*model.TagAvIncome) (err error) {
|
||
|
|
utm := make(map[int64]*model.TagAvIncome) // key-value: mid-totalIncome
|
||
|
|
for _, info := range infos {
|
||
|
|
_, ok := utm[info.MID]
|
||
|
|
if !ok {
|
||
|
|
a := &model.TagAvIncome{TagID: info.TagID, MID: info.MID, TotalIncome: info.Income}
|
||
|
|
utm[info.MID] = a
|
||
|
|
} else {
|
||
|
|
utm[info.MID].TotalIncome += info.Income
|
||
|
|
}
|
||
|
|
}
|
||
|
|
cnt := 0
|
||
|
|
var buf bytes.Buffer
|
||
|
|
for _, v := range utm {
|
||
|
|
buf.WriteString("(")
|
||
|
|
buf.WriteString(strconv.FormatInt(v.TagID, 10))
|
||
|
|
buf.WriteString(",")
|
||
|
|
buf.WriteString(strconv.FormatInt(v.MID, 10))
|
||
|
|
buf.WriteString(",")
|
||
|
|
buf.WriteString(strconv.Itoa(v.TotalIncome))
|
||
|
|
buf.WriteString("),")
|
||
|
|
cnt++
|
||
|
|
if cnt%2000 == 0 {
|
||
|
|
buf.Truncate(buf.Len() - 1)
|
||
|
|
_, err = s.dao.TxUpdateTagUpInfo(tx, buf.String())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.updateTagUpInfo dao.UpdateTagUpInfo error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
buf.Reset()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if buf.Len() > 0 {
|
||
|
|
buf.Truncate(buf.Len() - 1)
|
||
|
|
_, err = s.dao.TxUpdateTagUpInfo(tx, buf.String())
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.updateTagUpInfo dao.UpdateTagUpInfo error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetAvIncomeStatis get av monthly income
|
||
|
|
func (s *Service) GetAvIncomeStatis(c context.Context, date string) error {
|
||
|
|
d, _ := time.Parse(_layout, date)
|
||
|
|
endTime := d.AddDate(0, 1, 0).Format(_layout)
|
||
|
|
avs, err := s.GetAvIncome(c)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.GetAvIncome error(%v)", err)
|
||
|
|
return err
|
||
|
|
|
||
|
|
}
|
||
|
|
log.Info("GetAvIncomeStatis get %d avs", len(avs))
|
||
|
|
|
||
|
|
avsMap := make(map[int64]*model.AvIncome)
|
||
|
|
avIncomeStatis(avsMap, avs, date, endTime)
|
||
|
|
|
||
|
|
data := make([]*model.AvIncome, 0)
|
||
|
|
for _, av := range avsMap {
|
||
|
|
data = append(data, av)
|
||
|
|
}
|
||
|
|
sort.Slice(data, func(i, j int) bool {
|
||
|
|
return data[i].Income > data[j].Income
|
||
|
|
})
|
||
|
|
log.Info("GetAvIncomeStatis calculate success: %d", len(data))
|
||
|
|
return s.batchSend(data)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) batchSend(data []*model.AvIncome) error {
|
||
|
|
fileNo, start, batchSize := 0, 0, 50000
|
||
|
|
for {
|
||
|
|
if start+batchSize >= len(data) {
|
||
|
|
batchSize = len(data) - start
|
||
|
|
}
|
||
|
|
if batchSize <= 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
|
||
|
|
records := formatAvIncome(data[start : start+batchSize])
|
||
|
|
filename := fmt.Sprintf("av_statis0%d", fileNo)
|
||
|
|
err := WriteCSV(records, filename)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
err = s.email.SendMailAttach(filename, "稿件月收入", []string{"shaozhenyu@bilibili.com"})
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
fileNo++
|
||
|
|
start += batchSize
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func avIncomeStatis(avsMap map[int64]*model.AvIncome, avs []*model.AvIncome, fromTime, toTime string) {
|
||
|
|
for _, av := range avs {
|
||
|
|
d := av.Date.Time().Format(_layout)
|
||
|
|
if d < fromTime || d >= toTime {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if _, ok := avsMap[av.AvID]; ok {
|
||
|
|
avsMap[av.AvID].Income += av.Income
|
||
|
|
if avsMap[av.AvID].Date < av.Date {
|
||
|
|
avsMap[av.AvID].TotalIncome = av.TotalIncome
|
||
|
|
avsMap[av.AvID].Date = av.Date
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
avsMap[av.AvID] = av
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetAvIncome get av_income
|
||
|
|
func (s *Service) GetAvIncome(c context.Context) (avs []*model.AvIncome, err error) {
|
||
|
|
limit := 2000
|
||
|
|
var id int64
|
||
|
|
for {
|
||
|
|
av, err := s.dao.ListAvIncome(c, id, limit)
|
||
|
|
if err != nil {
|
||
|
|
return avs, err
|
||
|
|
}
|
||
|
|
avs = append(avs, av...)
|
||
|
|
if len(av) < limit {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
id = av[len(av)-1].ID
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetUpIncomeStatis get up statis
|
||
|
|
func (s *Service) GetUpIncomeStatis(c context.Context, date string, hasWithdraw int) (err error) {
|
||
|
|
var upAccount []*model.UpAccount
|
||
|
|
if hasWithdraw == 1 {
|
||
|
|
upAccount, err = s.getUpIncomeStatisAfterWithdraw(c, date)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.getUpIncomeStatisAfterWithdraw error(%v)", err)
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
upAccount, err = s.getUpIncomeStatisBeforeWithdraw(c, date)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.getUpIncomeStatisBeforeWithdraw error(%v)", err)
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
upa := make(map[int64]*model.UpAccount)
|
||
|
|
mids := make([]int64, len(upAccount))
|
||
|
|
for i := 0; i < len(upAccount); i++ {
|
||
|
|
mids[i] = upAccount[i].MID
|
||
|
|
upa[upAccount[i].MID] = upAccount[i]
|
||
|
|
}
|
||
|
|
|
||
|
|
upNick, err := s.GetUpNickname(c, mids)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.GetUpNickname error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
upIncome, err := s.GetUpIncome(c, "up_income_monthly", date)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.GetUpIncome error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
upIncomeStatis(upa, upIncome, upNick)
|
||
|
|
|
||
|
|
data := []*model.UpAccount{}
|
||
|
|
for _, up := range upa {
|
||
|
|
data = append(data, up)
|
||
|
|
}
|
||
|
|
|
||
|
|
sort.Slice(data, func(i, j int) bool {
|
||
|
|
return data[i].MonthIncome > data[j].MonthIncome
|
||
|
|
})
|
||
|
|
|
||
|
|
d, _ := time.Parse(_layout, date)
|
||
|
|
records := formatUpAccount(data, int(d.Month()))
|
||
|
|
err = WriteCSV(records, "up_statis.csv")
|
||
|
|
if err != nil {
|
||
|
|
log.Error("WriteCSV error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
return s.email.SendMailAttach("up_statis.csv", "up主月结算", []string{"shaozhenyu@bilibili.com"})
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) getUpIncomeStatisAfterWithdraw(c context.Context, date string) (upAccount []*model.UpAccount, err error) {
|
||
|
|
d, _ := time.Parse(_layout, date)
|
||
|
|
withdrawDateStr := d.Format("2006-01")
|
||
|
|
ctime := d.AddDate(0, 1, 1).Format(_layout)
|
||
|
|
upAccount, err = s.GetUpAccount(c, withdrawDateStr, ctime)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.GetUpAccount error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
upWithdraw, err := s.GetUpWithdraw(c, withdrawDateStr)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.GetUpWithdraw error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, up := range upAccount {
|
||
|
|
up.TotalUnwithdrawIncome = upWithdraw[up.MID]
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) getUpIncomeStatisBeforeWithdraw(c context.Context, date string) (upAccount []*model.UpAccount, err error) {
|
||
|
|
d, _ := time.Parse(_layout, date)
|
||
|
|
withdrawDateStr := d.AddDate(0, -1, 0).Format("2006-01")
|
||
|
|
ctime := d.AddDate(0, 1, 1).Format(_layout)
|
||
|
|
return s.GetUpAccount(c, withdrawDateStr, ctime)
|
||
|
|
}
|
||
|
|
|
||
|
|
func upIncomeStatis(upa map[int64]*model.UpAccount, upIncome []*model.UpIncome, upNick map[int64]string) {
|
||
|
|
for _, income := range upIncome {
|
||
|
|
if _, ok := upa[income.MID]; ok {
|
||
|
|
upa[income.MID].AvCount = income.AvCount
|
||
|
|
upa[income.MID].MonthIncome = income.Income
|
||
|
|
upa[income.MID].Nickname = upNick[income.MID]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetUpAccount get up_account
|
||
|
|
func (s *Service) GetUpAccount(c context.Context, date, ctime string) (ups []*model.UpAccount, err error) {
|
||
|
|
offset, limit := 0, 2000
|
||
|
|
for {
|
||
|
|
up, err := s.dao.ListUpAccount(c, date, ctime, offset, limit)
|
||
|
|
if err != nil {
|
||
|
|
return ups, err
|
||
|
|
}
|
||
|
|
ups = append(ups, up...)
|
||
|
|
if len(up) < limit {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
offset += limit
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetUpIncome get up_income
|
||
|
|
func (s *Service) GetUpIncome(c context.Context, table, date string) (ups []*model.UpIncome, err error) {
|
||
|
|
ups = make([]*model.UpIncome, 0)
|
||
|
|
var id int64
|
||
|
|
limit := 2000
|
||
|
|
for {
|
||
|
|
var up []*model.UpIncome
|
||
|
|
up, err = s.dao.ListUpIncome(c, table, date, id, limit)
|
||
|
|
if err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
ups = append(ups, up...)
|
||
|
|
if len(up) < limit {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
id = up[len(up)-1].ID
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetUpWithdraw get up_income_withdraw
|
||
|
|
func (s *Service) GetUpWithdraw(c context.Context, date string) (ups map[int64]int64, err error) {
|
||
|
|
ups = make(map[int64]int64)
|
||
|
|
offset, limit := 0, 2000
|
||
|
|
for {
|
||
|
|
up, err := s.dao.ListUpWithdraw(c, date, offset, limit)
|
||
|
|
if err != nil {
|
||
|
|
return ups, err
|
||
|
|
}
|
||
|
|
for mid, income := range up {
|
||
|
|
ups[mid] = income
|
||
|
|
}
|
||
|
|
if len(up) < limit {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
offset += limit
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// GetUpNickname get up nickname
|
||
|
|
func (s *Service) GetUpNickname(c context.Context, mids []int64) (upNick map[int64]string, err error) {
|
||
|
|
upNick = make(map[int64]string)
|
||
|
|
offset, limit := 0, 2000
|
||
|
|
for {
|
||
|
|
if offset+limit > len(mids) {
|
||
|
|
limit = len(mids) - offset
|
||
|
|
}
|
||
|
|
if limit <= 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
err = s.dao.ListUpNickname(c, mids[offset:offset+limit], upNick)
|
||
|
|
if err != nil {
|
||
|
|
log.Error("s.dao.ListUpNickname error(%v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
offset += limit
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|