539 lines
13 KiB
Go
539 lines
13 KiB
Go
package income
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
model "go-common/app/job/main/growup/model/income"
|
|
task "go-common/app/job/main/growup/service"
|
|
"go-common/library/log"
|
|
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
// RunAndSendMail run and send email
|
|
func (s *Service) RunAndSendMail(c context.Context, date time.Time) (err error) {
|
|
var mailReceivers []string
|
|
var msg string
|
|
for _, v := range s.conf.Mail.Send {
|
|
if v.Type == 3 {
|
|
mailReceivers = v.Addr
|
|
}
|
|
}
|
|
startTime := time.Now().Unix()
|
|
err = s.run(c, date)
|
|
if err != nil {
|
|
msg = err.Error()
|
|
mailReceivers = []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"}
|
|
} else {
|
|
msg = fmt.Sprintf("%s 计算完成，耗时%ds", date.Format("2006-01-02"), time.Now().Unix()-startTime)
|
|
}
|
|
emailErr := s.email.SendMail(date, msg, "创作激励每日计算%d年%d月%d日", mailReceivers...)
|
|
if emailErr != nil {
|
|
log.Error("s.email.SendMail error(%v)", emailErr)
|
|
}
|
|
return
|
|
}
|
|
|
|
func (s *Service) run(c context.Context, date time.Time) (err error) {
|
|
defer func() {
|
|
task.GetTaskService().SetTaskStatus(c, task.TaskCreativeIncome, date.Format(_layout), err)
|
|
}()
|
|
|
|
err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskAvCharge, task.TaskCmCharge, task.TaskTagRatio, task.TaskBubbleMeta, task.TaskBlacklist, task.TaskBgmSync)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
startWeeklyDate = getStartWeeklyDate(date)
|
|
startMonthlyDate = getStartMonthlyDate(date)
|
|
|
|
/*################ Serializable Begin ################*/
|
|
|
|
// av charge ratio
|
|
ratios, err := s.ratio.ArchiveChargeRatio(c, int64(_limitSize))
|
|
if err != nil {
|
|
return
|
|
}
|
|
// up charge ratio
|
|
urs, err := s.ratio.UpChargeRatio(c, int64(_limitSize))
|
|
if err != nil {
|
|
return
|
|
}
|
|
// av income statistics
|
|
astat, err := s.income.avIncomeStatSvr.AvIncomeStat(c, int64(_limitSize))
|
|
if err != nil {
|
|
log.Error("s.income.avIncomeStatSvr.AvIncomeStat error(%v) ", err)
|
|
return
|
|
}
|
|
log.Info("get av_income_statis : %d", len(astat))
|
|
|
|
// bgm income statistics
|
|
bstat, err := s.income.bgmIncomeStatSvr.BgmIncomeStat(c, int64(_limitSize))
|
|
if err != nil {
|
|
log.Error("s.income.bgmIncomeStatSvr.BgmIncomeStat error(%v) ", err)
|
|
return
|
|
}
|
|
log.Info("get bgm_income_statis : %d", len(bstat))
|
|
|
|
// column income statistics
|
|
cstat, err := s.income.columnIncomeStatSvr.ColumnIncomeStat(c, int64(_limitSize))
|
|
if err != nil {
|
|
log.Error("s.income.columnIncomeStatSvr.ColumnIncomeStat error(%v) ", err)
|
|
return
|
|
}
|
|
log.Info("get column_income_statis : %d", len(cstat))
|
|
|
|
// up income statistics
|
|
ustat, err := s.income.upIncomeStatSvr.UpIncomeStat(c, int64(_limitSize))
|
|
if err != nil {
|
|
log.Error("s.income.upIncomeStatSvr.UpIncomeStat error(%v) ", err)
|
|
return
|
|
}
|
|
log.Info("get up_income_statis : %d", len(ustat))
|
|
|
|
// up accounts
|
|
accs, err := s.income.upAccountSvr.UpAccount(c, int64(_limitSize))
|
|
if err != nil {
|
|
log.Error("s.income.upAccountSvr.UpAccount error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("get up_account : %d", len(accs))
|
|
|
|
// bubble meta
|
|
bubbleMeta, err := s.GetBubbleMeta(c)
|
|
if err != nil {
|
|
log.Error("s.GetBubbleMeta error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("get lottery_av_info avids: %d", len(bubbleMeta))
|
|
bubbleRatio := s.avToBubbleRatio(bubbleMeta)
|
|
|
|
// av signed ups
|
|
var (
|
|
avFilters []AvFilter
|
|
columnFilters []ColumnFilter
|
|
bgmFilter BgmFilter
|
|
)
|
|
|
|
//black list
|
|
blacks, err := s.Blacklist(c, 2000)
|
|
if err != nil {
|
|
return
|
|
}
|
|
// black ctype 0: av
|
|
avBlackFilter := avFilter(blacks[0])
|
|
|
|
// black ctype 2: column
|
|
columnBlackFilter := columnFilter(blacks[2])
|
|
|
|
// business orders
|
|
bos, err := s.GetBusinessOrders(c, 2000)
|
|
if err != nil {
|
|
return
|
|
}
|
|
bosFilter := avFilter(bos)
|
|
|
|
// signed up
|
|
signed := make(map[int64]bool)
|
|
|
|
signedAv, err := s.Signed(c, "video", 2000)
|
|
if err != nil {
|
|
return
|
|
}
|
|
savf := signedAvFilter(signedAv, date)
|
|
for mid := range signedAv {
|
|
signed[mid] = true
|
|
}
|
|
|
|
signedColumn, err := s.Signed(c, "column", 2000)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for mid := range signedColumn {
|
|
signed[mid] = true
|
|
}
|
|
|
|
signedBgm, err := s.Signed(c, "bgm", 2000)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for mid := range signedBgm {
|
|
signed[mid] = true
|
|
}
|
|
|
|
bgms, err := s.BGMs(c, 2000)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
{
|
|
avFilters = append(avFilters, avBlackFilter)
|
|
avFilters = append(avFilters, bosFilter)
|
|
avFilters = append(avFilters, savf)
|
|
|
|
bgmFilter = signedBgmFilter(signedBgm, date)
|
|
|
|
columnFilters = append(columnFilters, signedColumnFilter(signedColumn, date))
|
|
columnFilters = append(columnFilters, columnBlackFilter)
|
|
}
|
|
|
|
/*################ Serializable End ##################*/
|
|
|
|
var (
|
|
readGroup errgroup.Group
|
|
sourceCh = make(chan []*model.AvCharge, 1000)
|
|
// av
|
|
incomeCh = make(chan []*model.AvCharge, 1000)
|
|
// bgm
|
|
bgmCh = make(chan []*model.AvCharge, 1000)
|
|
// column
|
|
columnSourceCh = make(chan []*model.ColumnCharge, 1000)
|
|
// business income
|
|
businessCh = make(chan map[int64]*model.UpBusinessIncome, 10)
|
|
)
|
|
|
|
// get av daily charge and repost to other channels
|
|
readGroup.Go(func() (err error) {
|
|
err = s.avCharge.AvCharges(c, date, sourceCh, bubbleRatio)
|
|
if err != nil {
|
|
log.Error("s.avCharge.AvCharges error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("av_daily_charge finished")
|
|
return
|
|
})
|
|
|
|
// get column daily charge
|
|
readGroup.Go(func() (err error) {
|
|
err = s.columnCharges(c, date, columnSourceCh)
|
|
if err != nil {
|
|
log.Error("s.columnCharges error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("column_daily_charge finished")
|
|
return
|
|
})
|
|
|
|
readGroup.Go(func() (err error) {
|
|
defer func() {
|
|
close(incomeCh)
|
|
close(bgmCh)
|
|
}()
|
|
|
|
for charges := range sourceCh {
|
|
incomeCh <- charges
|
|
bgmCh <- charges
|
|
}
|
|
return
|
|
})
|
|
|
|
// up and av income compute
|
|
var (
|
|
um map[int64]*model.UpIncome
|
|
am map[int64][]*model.AvIncome
|
|
bm map[int64]map[int64]map[int64]*model.BgmIncome
|
|
cm map[int64][]*model.ColumnIncome
|
|
)
|
|
|
|
readGroup.Go(func() (err error) {
|
|
//um, am = s.income.Compute(c, date, incomeCh, urs, ars, ustat, astat, accs, filters, signed)
|
|
var business map[int64]*model.UpBusinessIncome
|
|
am, business = s.income.CalAvIncome(incomeCh, urs[1], ratios[1], avFilters, signed)
|
|
businessCh <- business
|
|
return
|
|
})
|
|
|
|
readGroup.Go(func() (err error) {
|
|
var business map[int64]*model.UpBusinessIncome
|
|
bm, business = s.income.CalBgmIncome(bgmCh, bgms, urs[3], ratios[3], avFilters, bgmFilter, blacks[3], signed)
|
|
businessCh <- business
|
|
return
|
|
})
|
|
|
|
readGroup.Go(func() (err error) {
|
|
var business map[int64]*model.UpBusinessIncome
|
|
cm, business = s.income.CalColumnIncome(columnSourceCh, urs[2], ratios[2], columnFilters, signed)
|
|
businessCh <- business
|
|
return
|
|
})
|
|
|
|
readGroup.Go(func() (err error) {
|
|
um = s.income.CalUpIncome(businessCh, date)
|
|
s.income.IncomeStat(um, am, bm, cm, ustat, astat, bstat, cstat)
|
|
s.income.PurgeUpAccount(date, accs, um)
|
|
return
|
|
})
|
|
|
|
if err = readGroup.Wait(); err != nil {
|
|
log.Error("run readGroup.Wait error(%v)", err)
|
|
return
|
|
}
|
|
|
|
// security verification
|
|
{
|
|
if len(am) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 av_income")
|
|
return
|
|
}
|
|
if len(bm) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 bgm_income")
|
|
return
|
|
}
|
|
if len(cm) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 column_income")
|
|
return
|
|
}
|
|
if len(um) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 up_income")
|
|
return
|
|
}
|
|
if len(astat) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 av_income_statis")
|
|
return
|
|
}
|
|
if len(bstat) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 bgm_income_statis")
|
|
return
|
|
}
|
|
if len(cstat) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 column_income_statis")
|
|
return
|
|
}
|
|
if len(ustat) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 up_income_statis")
|
|
return
|
|
}
|
|
if len(accs) == 0 {
|
|
err = fmt.Errorf("Error: insert 0 up_account")
|
|
return
|
|
}
|
|
}
|
|
|
|
// persistent
|
|
var writeGroup errgroup.Group
|
|
|
|
// av_income
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.avIncomeSvr.BatchInsertAvIncome(c, am)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertAvIncome error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert av_income : %d", len(am))
|
|
return
|
|
})
|
|
|
|
// column_income
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.columnIncomeSvr.BatchInsertColumnIncome(c, cm)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertColumnIncome error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert column_income : %d", len(cm))
|
|
return
|
|
})
|
|
|
|
// bgm_income
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.bgmIncomeSvr.BatchInsertBgmIncome(c, bm)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertBgmIncome error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert bgm_income : %d", len(bm))
|
|
return
|
|
})
|
|
|
|
// up income
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.upIncomeSvr.BatchInsertUpIncome(c, um)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertUpIncome error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert up_income : %d", len(um))
|
|
return
|
|
})
|
|
|
|
// av_income_statis
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.avIncomeStatSvr.BatchInsertAvIncomeStat(c, astat)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertAvIncomeStat error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert av_income_statis : %d", len(astat))
|
|
return
|
|
})
|
|
|
|
// column_income_statis
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.columnIncomeStatSvr.BatchInsertColumnIncomeStat(c, cstat)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertColumnIncomeStat error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert column_income_statis : %d", len(cstat))
|
|
return
|
|
})
|
|
|
|
// bgm_income_statis
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.bgmIncomeStatSvr.BatchInsertBgmIncomeStat(c, bstat)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertBgmIncomeStat error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert bgm_income_statis : %d", len(bstat))
|
|
return
|
|
})
|
|
|
|
// up_income_statis
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.upIncomeStatSvr.BatchInsertUpIncomeStat(c, ustat)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertUpIncomeStat error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert up_income_statis : %d", len(ustat))
|
|
return
|
|
})
|
|
|
|
// up_account batch insert
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.upAccountSvr.BatchInsertUpAccount(c, accs)
|
|
if err != nil {
|
|
log.Error("s.income.BatchInsertUpAccount error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("insert up_account : %d", len(accs))
|
|
return
|
|
})
|
|
|
|
// up account single update
|
|
writeGroup.Go(func() (err error) {
|
|
err = s.income.upAccountSvr.UpdateUpAccount(c, accs)
|
|
if err != nil {
|
|
log.Error("s.income.UpdateUpAccount error(%v)", err)
|
|
return
|
|
}
|
|
log.Info("update up_account : %d", len(accs))
|
|
return
|
|
})
|
|
|
|
if err = writeGroup.Wait(); err != nil {
|
|
log.Error("run writeGroup.Wait error(%v)", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
func signedBgmFilter(m map[int64]*model.Signed, date time.Time) BgmFilter {
|
|
return func(charge *model.AvCharge, bgm *model.BGM) bool {
|
|
if up, ok := m[bgm.MID]; ok {
|
|
if charge.Date.Time().Before(up.SignedAt.Time()) {
|
|
return true
|
|
}
|
|
if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
|
|
return true
|
|
}
|
|
} else {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
}
|
|
|
|
func signedAvFilter(m map[int64]*model.Signed, date time.Time) AvFilter {
|
|
return func(charge *model.AvCharge) bool {
|
|
if up, ok := m[charge.MID]; ok {
|
|
if charge.UploadTime.Time().Before(up.SignedAt.Time()) {
|
|
return true
|
|
}
|
|
if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
|
|
return true
|
|
}
|
|
} else {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
}
|
|
|
|
func signedColumnFilter(m map[int64]*model.Signed, date time.Time) ColumnFilter {
|
|
return func(charge *model.ColumnCharge) bool {
|
|
if up, ok := m[charge.MID]; ok {
|
|
if charge.UploadTime.Time().Before(up.SignedAt.Time()) {
|
|
return true
|
|
}
|
|
if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
|
|
return true
|
|
}
|
|
} else {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
}
|
|
|
|
func avFilter(m map[int64]bool) AvFilter {
|
|
return func(charge *model.AvCharge) bool {
|
|
return m[charge.AvID]
|
|
}
|
|
}
|
|
|
|
func columnFilter(m map[int64]bool) ColumnFilter {
|
|
return func(charge *model.ColumnCharge) bool {
|
|
return m[charge.ArticleID]
|
|
}
|
|
}
|
|
|
|
func (s *Service) avToBubbleRatio(bubbleMeta map[int64][]int) map[int64]float64 {
|
|
var (
|
|
res = make(map[int64]float64)
|
|
typeToRatio = make(map[int]float64)
|
|
chooseBType = func(bTypes []int) (bType int) {
|
|
if len(bTypes) == 1 {
|
|
return bTypes[0]
|
|
}
|
|
sort.Slice(bTypes, func(i, j int) bool {
|
|
bti, btj := bTypes[i], bTypes[j]
|
|
if typeToRatio[bti] == typeToRatio[btj] {
|
|
return bti < btj
|
|
}
|
|
return typeToRatio[bti] < typeToRatio[btj]
|
|
})
|
|
return bTypes[0]
|
|
}
|
|
)
|
|
|
|
for _, v := range s.conf.Bubble.BRatio {
|
|
typeToRatio[v.BType] = v.Ratio
|
|
}
|
|
for avID, bTypes := range bubbleMeta {
|
|
bType := chooseBType(bTypes)
|
|
res[avID] = typeToRatio[bType]
|
|
}
|
|
return res
|
|
}
|
|
|
|
// BgmFilter av charge filter
|
|
type BgmFilter func(*model.AvCharge, *model.BGM) bool
|
|
|
|
// AvFilter av charge filter
|
|
type AvFilter func(*model.AvCharge) bool
|
|
|
|
// ColumnFilter column charge filter
|
|
type ColumnFilter func(*model.ColumnCharge) bool
|
|
|
|
// ChargeRegulator regulates av charge
|
|
type ChargeRegulator func(*model.AvCharge)
|
|
|
|
// UpdateBusinessIncome ..
|
|
func (s *Service) UpdateBusinessIncome(c context.Context, date string) (err error) {
|
|
return s.income.UpdateBusinessIncomeByDate(c, date)
|
|
}
|