openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
app/job/main/growup/service/BUILD Normal file
View File

@@ -0,0 +1,105 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"blacklist_test.go",
"cheat_test.go",
"combinemails_test.go",
"data_test.go",
"httpClient_test.go",
"income_test.go",
"recompute_test.go",
"service_test.go",
"tag_test.go",
"tagincome_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"auto.go",
"blacklist.go",
"cheat.go",
"combinemails.go",
"creative_activity.go",
"creative_budget.go",
"data.go",
"databus.go",
"format.go",
"httpClient.go",
"income.go",
"lottery.go",
"recompute.go",
"sendincome.go",
"service.go",
"signedups.go",
"tag.go",
"tagincome.go",
"task_status.go",
"tax.go",
"tool.go",
"topten.go",
"up.go",
"up_bill.go",
"upload.go",
"video.go",
],
importpath = "go-common/app/job/main/growup/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/dao:go_default_library",
"//app/job/main/growup/dao/charge:go_default_library",
"//app/job/main/growup/dao/dataplatform:go_default_library",
"//app/job/main/growup/dao/email:go_default_library",
"//app/job/main/growup/dao/income:go_default_library",
"//app/job/main/growup/dao/tag:go_default_library",
"//app/job/main/growup/model:go_default_library",
"//app/job/main/growup/model/income:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//library/xstr:go_default_library",
"//vendor/golang.org/x/sync/errgroup:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/growup/service/charge:all-srcs",
"//app/job/main/growup/service/ctrl:all-srcs",
"//app/job/main/growup/service/income:all-srcs",
"//app/job/main/growup/service/tag:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

228
app/job/main/growup/service/auto.go Normal file
View File

@@ -0,0 +1,228 @@
package service
import (
"context"
"fmt"
"time"
"go-common/library/log"
"go-common/library/xstr"
)
var (
_video = 0
_forbidDay = 10
_breachReason = "非自制"
_forbidReason = "投递非本人自制稿件2次"
_dismissReason = "投递非本人自制稿件3次及以上"
)
// AutoBreach auto breach
func (s *Service) AutoBreach(c context.Context, date string) (msg string, err error) {
return s.autoAvBreach(c, date)
}
func (s *Service) autoAvBreach(c context.Context, date string) (msg string, err error) {
avs, err := s.dao.GetAvBreachPre(c, _video, 1, date)
if err != nil {
log.Error("s.dao.GetAvBreachPre error(%v)", err)
return
}
needBreach := make(map[int64]bool)
for _, av := range avs {
err = s.dao.DoAvBreach(c, av.MID, av.AvID, _video, _breachReason)
if err != nil {
log.Error("s.dao.DoAvBreach error(%v)", err)
return
}
needBreach[av.AvID] = true
}
breach := make([]int64, 0)
breachAvs, err := s.dao.GetAvBreachPre(c, _video, 2, date)
if err != nil {
log.Error("s.dao.GetAvBreachPre error(%v)", err)
return
}
for _, av := range breachAvs {
if needBreach[av.AvID] {
breach = append(breach, av.AvID)
}
}
msg = fmt.Sprintf("%s 自制转转载违规扣除稿件:%s", date, xstr.JoinInts(breach))
return
}
// AutoPunish auto punish
func (s *Service) AutoPunish(c context.Context) (msg string, err error) {
return s.autoUpPunish(c)
}
// 每周一检查并处罚 只检查非自制
func (s *Service) autoUpPunish(c context.Context) (msg string, err error) {
now := time.Now()
if now.Format(_layout) != getStartWeeklyDate(now).Format(_layout) {
return
}
lastWeek := getStartWeeklyDate(getStartWeeklyDate(now).AddDate(0, 0, -1))
avs, err := s.dao.GetAvBreach(c, "2018-10-15", now.AddDate(0, 0, -1).Format(_layout)) // 从2018-10-15上线开始计算扣除
if err != nil {
log.Error("s.dao.GetAvBreach error(%v)", err)
return
}
blacks, err := s.listBlacklist(c, "")
if err != nil {
log.Error("s.listBlacklist error(%v)", err)
return
}
mBlacks := make(map[int64]bool)
for _, b := range blacks {
mBlacks[b.AvID] = true
}
mids := make(map[int64]map[string]struct{})
for _, av := range avs {
if av.Reason != _breachReason {
continue
}
if !mBlacks[av.AvID] {
continue
}
date := getStartWeeklyDate(av.Date.Time()).Format(_layout)
if _, ok := mids[av.MID]; !ok {
mids[av.MID] = make(map[string]struct{})
}
mids[av.MID][date] = struct{}{}
}
forbidMIDs, dismissMIDs := make([]int64, 0), make([]int64, 0)
for mid, times := range mids {
// 当且仅当上周有发生过扣除才处罚up主，防止多次处罚
if _, ok := times[lastWeek.Format(_layout)]; !ok {
continue
}
if len(times) == 2 {
err = s.autoForbid(c, mid)
if err != nil {
log.Error("s.autoForbid(%d) error(%v)", mid, err)
return
}
forbidMIDs = append(forbidMIDs, mid)
}
if len(times) >= 3 {
err = s.autoDismiss(c, mid)
if err != nil {
log.Error("s.autoDismiss(%d) error(%v)", mid, err)
return
}
dismissMIDs = append(dismissMIDs, mid)
}
}
msg = fmt.Sprintf("%s 自制转转载处罚up主: 封禁(%s), 清退(%s)", time.Now().Format(_layout), xstr.JoinInts(forbidMIDs), xstr.JoinInts(dismissMIDs))
return
}
func (s *Service) autoForbid(c context.Context, mid int64) (err error) {
accState, err := s.dao.GetUpStateByMID(c, mid)
if err != nil {
log.Error(" s.dao.GetUpStateByMID(%d) error(%v)", mid, err)
return
}
if !(accState == 3 || accState == 7) {
return
}
err = s.dao.DoUpForbid(c, mid, _forbidDay, _video, _forbidReason)
if err != nil {
log.Error("s.dao.DoUpForbid error(%v)", err)
}
return
}
func (s *Service) autoDismiss(c context.Context, mid int64) (err error) {
accState, err := s.dao.GetUpStateByMID(c, mid)
if err != nil {
log.Error(" s.dao.GetUpStateByMID(%d) error(%v)", mid, err)
return
}
if accState != 3 && accState != 7 {
return
}
err = s.dao.DoUpDismiss(c, mid, _video, _dismissReason)
if err != nil {
log.Error("s.dao.DoUpDismiss error(%v)", err)
}
return
}
// AutoExamination auto examination
func (s *Service) AutoExamination(c context.Context) (msg string, err error) {
return s.autoExamination(c)
}
func (s *Service) autoExamination(c context.Context) (msg string, err error) {
ups, err := s.getAllUps(c, 2000)
if err != nil {
log.Error("s.getAllUps error(%v)", err)
return
}
mids := make([]int64, 0, len(ups))
for mid, up := range ups {
if up.AccountState != 2 || up.IsDeleted == 1 {
continue
}
if up.Fans < 10000 && up.TotalPlayCount < 500000 {
continue
}
mids = append(mids, mid)
}
scores, err := s.dao.GetUpCreditScore(c, mids)
if err != nil {
log.Error("s.dao.GetUpCreditScore error(%v)", err)
return
}
passMID := make([]int64, 0, len(mids))
for _, mid := range mids {
if score, ok := scores[mid]; ok {
if score < 100 {
continue
}
}
passMID = append(passMID, mid)
}
err = s.doUpPass(c, passMID, _video)
if err != nil {
log.Error("s.doUpPass error(%v)", err)
return
}
msg = fmt.Sprintf("%s 自动过审up主: %s", time.Now().Format(_layout), xstr.JoinInts(passMID))
return
}
func (s *Service) doUpPass(c context.Context, mids []int64, ctype int) (err error) {
start, end := 0, 100
for {
if start >= len(mids) {
break
}
if end > len(mids) {
end = len(mids)
}
err = s.dao.DoUpPass(c, mids[start:end], _video)
if err != nil {
log.Error("s.dao.DoUpPass error(%v)", err)
return
}
start = end
end += 100
}
return
}
func getStartWeeklyDate(date time.Time) time.Time {
for date.Weekday() != time.Monday {
date = date.AddDate(0, 0, -1)
}
return date
}

268
app/job/main/growup/service/blacklist.go Normal file
View File

@@ -0,0 +1,268 @@
package service
import (
"context"
"encoding/json"
"math"
"strconv"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/log"
"go-common/library/xstr"
)
// add to blacklist reason
const (
// _all = 0
// _stopIncome = 1
// _breachRecord = 2
_porder = 3
_executeOrder = 4
)
// InitBlacklistMID init av_black_list mid
func (s *Service) InitBlacklistMID(c context.Context) (err error) {
blacklist, err := s.listBlacklist(c, "mid = 0")
if err != nil {
log.Error("s.listBlacklist error(%v)", err)
return
}
avIDs := make([]int64, 0)
for _, b := range blacklist {
if b.MID == 0 {
avIDs = append(avIDs, b.AvID)
}
}
if len(avIDs) == 0 {
return
}
m, err := s.GetAvsMID(c, avIDs)
if err != nil {
log.Error("GetAvsMID error(%v)", err)
return
}
for i := 0; i < len(blacklist); i++ {
blacklist[i].MID = m[blacklist[i].AvID]
}
_, err = s.updateBlacklistBatch(c, blacklist)
return
}
func (s *Service) listBlacklist(c context.Context, query string) (list []*model.Blacklist, err error) {
from, limit := 0, 2000
var b []*model.Blacklist
for {
b, err = s.dao.ListBlacklist(c, query, from, limit)
if err != nil {
return
}
list = append(list, b...)
if len(b) < limit {
break
}
from += len(b)
}
return
}
// UpdateBlacklist update blacklist
func (s *Service) UpdateBlacklist(c context.Context) (err error) {
defer func() {
GetTaskService().SetTaskStatus(c, TaskBlacklist, time.Now().AddDate(0, 0, -1).Format(_layout), err)
}()
blacklist := make([]*model.Blacklist, 0)
porders, err := s.getNewPorder(c)
if err != nil {
log.Error("s.getNewPorder error(%v)", err)
return
}
log.Info("Get new porder %d", len(porders))
blacklist = append(blacklist, porders...)
executeOrders, err := s.getNewExecuteOrder(c)
if err != nil {
log.Error("s.getNewExecuteOrder error(%v)", err)
return
}
log.Info("Get new execute order %d", len(executeOrders))
blacklist = append(blacklist, executeOrders...)
count, err := s.updateBlacklistBatch(c, blacklist)
if err != nil {
log.Error("s.updateBlacklistBatch error(%v)", err)
return
}
log.Info("Add %d list into blacklist", count)
return
}
func (s *Service) updateBlacklistBatch(c context.Context, blacklist []*model.Blacklist) (count int64, err error) {
ups, err := s.getHasSignUpInfo(c)
if err != nil {
log.Error("s.dao.GetHasSignUpInfo error(%v)", err)
return
}
for i := 0; i < len(blacklist); i++ {
if nickname, ok := ups[blacklist[i].MID]; ok {
blacklist[i].HasSigned = 1
blacklist[i].Nickname = nickname
}
}
return s.dao.AddBlacklistBatch(c, blacklist)
}
func (s *Service) getHasSignUpInfo(c context.Context) (m map[int64]string, err error) {
m = make(map[int64]string)
offset, limit := 0, 2000
for {
err = s.dao.GetHasSignUpInfo(c, offset, limit, m)
if err != nil {
log.Error("s.dao.GetHasSignUpInfo error(%v)", err)
return
}
offset += limit
if len(m) < offset {
break
}
}
return
}
func (s *Service) getNewPorder(c context.Context) (blacklist []*model.Blacklist, err error) {
beginTime, err := s.dao.GetLastCtime(c, _porder)
if err != nil {
log.Error("s.dao.GetLastCtime error(%v)", err)
return
}
if beginTime != 0 {
beginTime -= 10 * 60 // pre 10min
}
endTime := time.Now().Unix()
porders, err := s.getPorder(beginTime, endTime)
if err != nil {
log.Error("get Porder error(%v)", err)
return
}
// get porder mid
avIds := []int64{}
for _, b := range porders {
avIds = append(avIds, b.AID)
}
m, err := s.GetAvsMID(c, avIds)
if err != nil {
log.Error("s.dao.GetAvsMID error(%v)", err)
return
}
blacklist = make([]*model.Blacklist, len(porders))
for i := 0; i < len(porders); i++ {
blacklist[i] = &model.Blacklist{
AvID: porders[i].AID,
MID: m[porders[i].AID],
Reason: _porder,
}
}
return
}
// GetAvsMID get avs mid from api
func (s *Service) GetAvsMID(c context.Context, avs []int64) (avsMap map[int64]int64, err error) {
avsMap = make(map[int64]int64)
if len(avs) == 0 {
return
}
start, limit := 0, 10
if limit > len(avs) {
limit = len(avs)
}
for start+limit <= len(avs) {
if err = s.getAvsMID(c, avs[start:start+limit], avsMap); err != nil {
return
}
start += limit
if start < len(avs) && start+limit > len(avs) {
limit = len(avs) - start
}
}
log.Info("Get avs(%d) from archiveURL", len(avsMap))
return
}
func (s *Service) getAvsMID(c context.Context, avs []int64, avsMap map[int64]int64) (err error) {
params := map[string]string{"aids": xstr.JoinInts(avs)}
body, err := s.HTTPClient("GET", s.conf.Host.Archives, params, time.Now().Unix())
if err != nil {
log.Error("s.HTTPClient error(%v)", err)
return
}
res := model.ArchiveRes{}
err = json.Unmarshal(body, &res)
if err != nil {
log.Error("json.Unmarshal body %s error(%v)", string(body), err)
return
}
for _, archive := range res.Data {
avsMap[archive.AID] = archive.Owner.MID
}
return
}
func (s *Service) getPorder(begin, end int64) (porders []*model.Porder, err error) {
params := map[string]string{
"begin": strconv.FormatInt(begin, 10),
"end": strconv.FormatInt(end, 10),
}
body, err := s.HTTPClient("GET", s.conf.Host.Porder, params, time.Now().UnixNano()/int64(math.Pow(10, 6)))
if err != nil {
log.Error("s.HTTPClient error(%v)", err)
return
}
res := model.PorderRes{}
err = json.Unmarshal(body, &res)
if err != nil {
log.Error("json.Unmarshal body %s error(%v)", string(body), err)
return
}
porders = res.Data
return
}
func (s *Service) getNewExecuteOrder(c context.Context) (blacklist []*model.Blacklist, err error) {
beginTime, err := s.dao.GetLastCtime(c, _executeOrder)
if err != nil {
log.Error("s.dao.GetLastCtime error(%v)", err)
return
}
if beginTime != 0 {
beginTime -= 10 * 60 // pre 10min
}
executeOrders, err := s.dao.GetExecuteOrder(c, time.Unix(beginTime, 0), time.Now())
if err != nil {
log.Error("s.dao.GetExecuteOrder error(%v)", err)
return
}
blacklist = make([]*model.Blacklist, len(executeOrders))
for i := 0; i < len(executeOrders); i++ {
blacklist[i] = &model.Blacklist{
AvID: executeOrders[i].AvID,
MID: executeOrders[i].MID,
Reason: _executeOrder,
}
}
return
}

32
app/job/main/growup/service/blacklist_test.go Normal file
View File

@@ -0,0 +1,32 @@
package service
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_InitBlacklistMID(t *testing.T) {
Convey("growup-job InitBlacklistMID", t, WithService(func(s *Service) {
err := s.InitBlacklistMID(context.Background())
So(err, ShouldBeNil)
}))
}
func Test_UpdateBlacklist(t *testing.T) {
Convey("growup-job UpdateBlacklist", t, WithService(func(s *Service) {
err := s.UpdateBlacklist(context.Background())
So(err, ShouldBeNil)
}))
}
func Test_GetAvsMID(t *testing.T) {
Convey("growup-job GetAvsMID", t, WithService(func(s *Service) {
var (
avs = []int64{int64(1), int64(2)}
)
_, err := s.GetAvsMID(context.Background(), avs)
So(err, ShouldBeNil)
}))
}

72
app/job/main/growup/service/charge/BUILD Normal file
View File

@@ -0,0 +1,72 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"av_charge_statis_test.go",
"run_test.go",
"service_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/model/charge:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"av_charge.go",
"av_charge_statis.go",
"bgm_charge.go",
"column_charge.go",
"date_statis.go",
"run.go",
"run_bgm.go",
"run_column.go",
"run_video.go",
"service.go",
"trans.go",
"up_charge.go",
],
importpath = "go-common/app/job/main/growup/service/charge",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/dao/charge:go_default_library",
"//app/job/main/growup/dao/email:go_default_library",
"//app/job/main/growup/model/charge:go_default_library",
"//app/job/main/growup/service:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//library/log:go_default_library",
"//library/time:go_default_library",
"//vendor/golang.org/x/sync/errgroup:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

308
app/job/main/growup/service/charge/av_charge.go Normal file
View File

@@ -0,0 +1,308 @@
package charge
import (
"bytes"
"context"
"strconv"
"time"
dao "go-common/app/job/main/growup/dao/charge"
model "go-common/app/job/main/growup/model/charge"
"go-common/library/log"
xtime "go-common/library/time"
"golang.org/x/sync/errgroup"
)
func (s *Service) handleAvCharge(c context.Context, date time.Time,
dailyChannel chan []*model.AvCharge) (weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
var eg errgroup.Group
weeklyChargeMap = make(map[int64]*model.AvCharge)
monthlyChargeMap = make(map[int64]*model.AvCharge)
eg.Go(func() (err error) {
avWeeklyCharge, err := s.GetAvCharge(c, getStartWeeklyDate(date), s.dao.AvWeeklyCharge)
if err != nil {
log.Error("s.GetAvCharge(av_weekly_charge) error(%v)", err)
return
}
for _, weeklyCharge := range avWeeklyCharge {
weeklyChargeMap[weeklyCharge.AvID] = weeklyCharge
}
return
})
eg.Go(func() (err error) {
avMonthlyCharge, err := s.GetAvCharge(c, getStartMonthlyDate(date), s.dao.AvMonthlyCharge)
if err != nil {
log.Error("s.GetAvCharge(av_monthly_charge) error(%v)", err)
return
}
for _, monthlyCharge := range avMonthlyCharge {
monthlyChargeMap[monthlyCharge.AvID] = monthlyCharge
}
return
})
eg.Go(func() (err error) {
chargeStatisMap, err = s.GetAvChargeStatisMap(c)
if err != nil {
log.Error("s.GetAvChargeStatisMap error(%v)", err)
}
return
})
if err = eg.Wait(); err != nil {
log.Error("HandleAvCharge eg.Wait error(%v)", err)
return
}
s.calAvCharges(date, weeklyChargeMap, monthlyChargeMap, chargeStatisMap, dailyChannel)
return
}
func (s *Service) avDailyCharges(c context.Context, date time.Time, ch chan []*model.AvCharge) (err error) {
defer func() {
close(ch)
}()
var id int64
for {
var charges []*model.AvCharge
charges, err = s.dao.AvDailyCharge(c, date, id, _limitSize)
if err != nil {
return
}
ch <- charges
if len(charges) < _limitSize {
break
}
id = charges[len(charges)-1].ID
}
return
}
// GetAvCharge get av charge
func (s *Service) GetAvCharge(c context.Context, date time.Time, fn dao.IAvCharge) (avCharges []*model.AvCharge, err error) {
var id int64
for {
var avCharge []*model.AvCharge
avCharge, err = fn(c, date, id, _limitSize)
if err != nil {
return
}
avCharges = append(avCharges, avCharge...)
if len(avCharge) < _limitSize {
break
}
id = avCharge[len(avCharge)-1].ID
}
return
}
func (s *Service) calAvCharges(date time.Time, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, dailyChannel chan []*model.AvCharge) {
for avDailyCharge := range dailyChannel {
s.calAvCharge(date, avDailyCharge, weeklyChargeMap, monthlyChargeMap, chargeStatisMap)
}
}
func (s *Service) calAvCharge(date time.Time, avDailyCharge []*model.AvCharge, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
for _, dailyCharge := range avDailyCharge {
if weeklyCharge, ok := weeklyChargeMap[dailyCharge.AvID]; ok {
updateAvCharge(weeklyCharge, dailyCharge)
} else {
weeklyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startWeeklyDate)
}
if monthlyCharge, ok := monthlyChargeMap[dailyCharge.AvID]; ok {
updateAvCharge(monthlyCharge, dailyCharge)
} else {
monthlyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startMonthlyDate)
}
s.CalAvChargeStatis(dailyCharge, chargeStatisMap)
}
}
func addAvCharge(daily *model.AvCharge, fixDate time.Time) *model.AvCharge {
return &model.AvCharge{
AvID: daily.AvID,
MID: daily.MID,
TagID: daily.TagID,
IsOriginal: daily.IsOriginal,
DanmakuCount: daily.DanmakuCount,
CommentCount: daily.CommentCount,
CollectCount: daily.CollectCount,
CoinCount: daily.CoinCount,
ShareCount: daily.ShareCount,
ElecPayCount: daily.ElecPayCount,
TotalPlayCount: daily.TotalPlayCount,
WebPlayCount: daily.WebPlayCount,
AppPlayCount: daily.AppPlayCount,
H5PlayCount: daily.H5PlayCount,
LvUnknown: daily.LvUnknown,
Lv0: daily.Lv0,
Lv1: daily.Lv1,
Lv2: daily.Lv2,
Lv3: daily.Lv3,
Lv4: daily.Lv4,
Lv5: daily.Lv5,
Lv6: daily.Lv6,
VScore: daily.VScore,
IncCharge: daily.IncCharge,
TotalCharge: daily.IncCharge,
Date: xtime.Time(fixDate.Unix()),
UploadTime: daily.UploadTime,
DBState: _dbInsert,
}
}
func updateAvCharge(origin, daily *model.AvCharge) {
origin.DanmakuCount += daily.DanmakuCount
origin.CommentCount += daily.CommentCount
origin.CollectCount += daily.CollectCount
origin.CoinCount += daily.CoinCount
origin.ShareCount += daily.ShareCount
origin.ElecPayCount += daily.ElecPayCount
origin.TotalPlayCount += daily.TotalPlayCount
origin.WebPlayCount += daily.WebPlayCount
origin.AppPlayCount += daily.AppPlayCount
origin.H5PlayCount += daily.H5PlayCount
origin.LvUnknown += daily.LvUnknown
origin.Lv0 += daily.Lv0
origin.Lv1 += daily.Lv1
origin.Lv2 += daily.Lv2
origin.Lv3 += daily.Lv3
origin.Lv4 += daily.Lv4
origin.Lv5 += daily.Lv5
origin.Lv6 += daily.Lv6
origin.VScore += daily.VScore
origin.IncCharge += daily.IncCharge
origin.TotalCharge += daily.IncCharge
origin.DBState = _dbUpdate
}
// AvChargeDBStore store data
func (s *Service) AvChargeDBStore(c context.Context, table string, avChargeMap map[int64]*model.AvCharge) error {
insert, update := make([]*model.AvCharge, _batchSize), make([]*model.AvCharge, _batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range avChargeMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= _batchSize {
_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= _batchSize {
_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleAvCharge(avCharge []*model.AvCharge) (vals string) {
var buf bytes.Buffer
for _, row := range avCharge {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(row.IsOriginal))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.DanmakuCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CommentCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CollectCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CoinCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.ShareCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.ElecPayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalPlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.WebPlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.AppPlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.H5PlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.LvUnknown, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv0, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv1, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv2, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv3, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv4, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv5, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv6, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.VScore, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *Service) avChargeBatchInsert(c context.Context, avCharge []*model.AvCharge, table string) (rows int64, err error) {
vals := assembleAvCharge(avCharge)
rows, err = s.dao.InsertAvChargeTable(c, vals, table)
return
}

155
app/job/main/growup/service/charge/av_charge_statis.go Normal file
View File

@@ -0,0 +1,155 @@
package charge
import (
"bytes"
"context"
"strconv"
model "go-common/app/job/main/growup/model/charge"
"go-common/library/log"
)
// GetAvChargeStatisMap get av charge statistics map
func (s *Service) GetAvChargeStatisMap(c context.Context) (chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
avChargeStatis, err := s.GetAvChargeStatis(c)
if err != nil {
log.Error("s.GetAvChargeStatis error(%v)", err)
return
}
chargeStatisMap = make(map[int64]*model.AvChargeStatis)
for _, chargeStatis := range avChargeStatis {
chargeStatisMap[chargeStatis.AvID] = chargeStatis
}
return
}
// GetAvChargeStatis get av charge statistics
func (s *Service) GetAvChargeStatis(c context.Context) (avChargeStatis []*model.AvChargeStatis, err error) {
var id int64
for {
statis, err1 := s.dao.AvChargeStatis(c, id, _limitSize)
if err1 != nil {
err = err1
return
}
avChargeStatis = append(avChargeStatis, statis...)
if len(statis) < _limitSize {
break
}
id = statis[len(statis)-1].ID
}
return
}
// CalAvChargeStatis cal av charge statis
func (s *Service) CalAvChargeStatis(dailyCharge *model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
if statisCharge, ok := chargeStatisMap[dailyCharge.AvID]; ok {
updateAvChargeStatis(statisCharge, dailyCharge)
} else {
chargeStatisMap[dailyCharge.AvID] = addAvChargeStatis(dailyCharge)
}
}
func addAvChargeStatis(daily *model.AvCharge) *model.AvChargeStatis {
return &model.AvChargeStatis{
AvID: daily.AvID,
MID: daily.MID,
TagID: daily.TagID,
IsOriginal: daily.IsOriginal,
UploadTime: daily.UploadTime,
TotalCharge: daily.IncCharge,
DBState: _dbInsert,
}
}
func updateAvChargeStatis(avChargeStatis *model.AvChargeStatis, daily *model.AvCharge) {
avChargeStatis.TotalCharge += daily.IncCharge
avChargeStatis.DBState = _dbUpdate
}
// AvChargeStatisDBStore store data
func (s *Service) AvChargeStatisDBStore(c context.Context, chargeStatisMap map[int64]*model.AvChargeStatis) error {
insert, update := make([]*model.AvChargeStatis, _batchSize), make([]*model.AvChargeStatis, _batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range chargeStatisMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= _batchSize {
_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= _batchSize {
_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleAvChargeStatis(avChargeStatis []*model.AvChargeStatis) (vals string) {
var buf bytes.Buffer
for _, row := range avChargeStatis {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(row.IsOriginal))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *Service) avChargeStatisBatchInsert(c context.Context, avChargeStatis []*model.AvChargeStatis) (rows int64, err error) {
vals := assembleAvChargeStatis(avChargeStatis)
rows, err = s.dao.InsertAvChargeStatisBatch(c, vals)
return
}

39
app/job/main/growup/service/charge/av_charge_statis_test.go Normal file
View File

@@ -0,0 +1,39 @@
package charge
import (
"context"
"testing"
model "go-common/app/job/main/growup/model/charge"
. "github.com/smartystreets/goconvey/convey"
)
func Test_GetAvChargeStatisMap(t *testing.T) {
Convey("GetAvChargeStatisMap", t, func() {
_, err := s.GetAvChargeStatisMap(context.Background())
So(err, ShouldBeNil)
})
}
func Test_GetAvChargeStatis(t *testing.T) {
Convey("GetAvChargeStatis", t, func() {
_, err := s.GetAvChargeStatis(context.Background())
So(err, ShouldBeNil)
})
}
func Test_AvChargeStatisDBStore(t *testing.T) {
Convey("AvChargeStatisDBStore", t, func() {
chargeStatisMap := make(map[int64]*model.AvChargeStatis)
value := &model.AvChargeStatis{
AvID: 11,
MID: 11,
TagID: 11,
DBState: 1,
}
chargeStatisMap[11] = value
err := s.AvChargeStatisDBStore(context.Background(), chargeStatisMap)
So(err, ShouldBeNil)
})
}

431
app/job/main/growup/service/charge/bgm_charge.go Normal file
View File

@@ -0,0 +1,431 @@
package charge
import (
"bytes"
"context"
"fmt"
"math"
"math/big"
"strconv"
"strings"
"time"
model "go-common/app/job/main/growup/model/charge"
"go-common/library/log"
xtime "go-common/library/time"
"golang.org/x/sync/errgroup"
)
func (s *Service) handleBgm(c context.Context, date time.Time,
dailyChannel chan []*model.BgmCharge) (weekly, monthly map[string]*model.BgmCharge, statis map[string]*model.BgmStatis, err error) {
var eg errgroup.Group
weekly = make(map[string]*model.BgmCharge)
monthly = make(map[string]*model.BgmCharge)
statis = make(map[string]*model.BgmStatis)
eg.Go(func() (err error) {
weekly, err = s.getBgmCharge(c, getStartWeeklyDate(date), "bgm_weekly_charge")
if err != nil {
log.Error("s.GetBgmCharge(bgm_weekly_charge) error(%v)", err)
return
}
return
})
eg.Go(func() (err error) {
monthly, err = s.getBgmCharge(c, getStartMonthlyDate(date), "bgm_monthly_charge")
if err != nil {
log.Error("s.GetBgmCharge(bgm_monthly_charge) error(%v)", err)
return
}
return
})
eg.Go(func() (err error) {
statis, err = s.getBgmStatis(c)
if err != nil {
log.Error("s.getBgmStatisMap error(%v)", err)
}
return
})
if err = eg.Wait(); err != nil {
log.Error("handleBgm eg.Wait error(%v)", err)
return
}
s.calBgms(date, weekly, monthly, statis, dailyChannel)
return
}
func (s *Service) calBgms(date time.Time, weeklyMap, monthlyMap map[string]*model.BgmCharge, statisMap map[string]*model.BgmStatis, dailyChannel chan []*model.BgmCharge) {
for daily := range dailyChannel {
s.calBgm(date, daily, weeklyMap, monthlyMap, statisMap)
}
}
func (s *Service) calBgm(date time.Time, daily []*model.BgmCharge, weeklyMap, monthlyMap map[string]*model.BgmCharge, statisMap map[string]*model.BgmStatis) {
for _, charge := range daily {
key := fmt.Sprintf("%d+%d", charge.SID, charge.AID)
if weekly, ok := weeklyMap[key]; ok {
updateBgmCharge(weekly, charge)
} else {
weeklyMap[key] = addBgmCharge(charge, startWeeklyDate)
}
if monthly, ok := monthlyMap[key]; ok {
updateBgmCharge(monthly, charge)
} else {
monthlyMap[key] = addBgmCharge(charge, startMonthlyDate)
}
if statis, ok := statisMap[key]; ok {
updateBgmStatis(statis, charge)
} else {
statisMap[key] = addBgmStatis(charge)
}
}
}
func addBgmCharge(daily *model.BgmCharge, fixDate time.Time) *model.BgmCharge {
return &model.BgmCharge{
SID: daily.SID,
AID: daily.AID,
MID: daily.MID,
CID: daily.CID,
JoinAt: daily.JoinAt,
Title: daily.Title,
IncCharge: daily.IncCharge,
Date: xtime.Time(fixDate.Unix()),
DBState: _dbInsert,
}
}
func updateBgmCharge(origin, daily *model.BgmCharge) {
origin.IncCharge += daily.IncCharge
origin.DBState = _dbUpdate
}
func addBgmStatis(daily *model.BgmCharge) *model.BgmStatis {
return &model.BgmStatis{
SID: daily.SID,
AID: daily.AID,
MID: daily.MID,
CID: daily.CID,
JoinAt: daily.JoinAt,
Title: daily.Title,
TotalCharge: daily.IncCharge,
DBState: _dbInsert,
}
}
func updateBgmStatis(statis *model.BgmStatis, daily *model.BgmCharge) {
statis.TotalCharge += daily.IncCharge
statis.DBState = _dbUpdate
}
func (s *Service) getBgmCharge(c context.Context, date time.Time, table string) (bgms map[string]*model.BgmCharge, err error) {
bgms = make(map[string]*model.BgmCharge)
var id int64
for {
var bgm []*model.BgmCharge
bgm, err = s.dao.BgmCharge(c, date, id, _limitSize, table)
if err != nil {
return
}
for _, b := range bgm {
key := fmt.Sprintf("%d+%d", b.SID, b.AID)
bgms[key] = b
}
if len(bgm) < _limitSize {
break
}
id = bgm[len(bgm)-1].ID
}
return
}
func (s *Service) getBgmStatis(c context.Context) (bgms map[string]*model.BgmStatis, err error) {
bgms = make(map[string]*model.BgmStatis)
var id int64
for {
var bgm []*model.BgmStatis
bgm, err = s.dao.BgmStatis(c, id, _limitSize)
if err != nil {
return
}
for _, b := range bgm {
key := fmt.Sprintf("%d+%d", b.SID, b.AID)
bgms[key] = b
}
if len(bgm) < _limitSize {
break
}
id = bgm[len(bgm)-1].ID
}
return
}
func (s *Service) bgmCharges(c context.Context, date time.Time, ch chan []*model.BgmCharge, avBgmCharge chan []*model.AvCharge) (dailyMap map[string]*model.BgmCharge, err error) {
dailyMap = make(map[string]*model.BgmCharge)
bgms, err := s.GetBgms(c, int64(_limitSize))
if err != nil {
log.Error("s.GetBgms error(%v)", err)
return
}
defer func() {
close(ch)
}()
for avs := range avBgmCharge {
for _, av := range avs {
bgm, ok := bgms[av.AvID]
if !ok {
continue
}
incCharge := int64(round(div(mul(float64(av.IncCharge), float64(0.3)), float64(len(bgm))), 0))
if incCharge == 0 {
continue
}
charges := make([]*model.BgmCharge, 0, len(bgm))
for _, b := range bgm {
if b.MID == av.MID {
continue
}
c := &model.BgmCharge{
SID: b.SID,
MID: b.MID,
AID: b.AID,
CID: b.CID,
IncCharge: incCharge,
JoinAt: b.JoinAt,
Date: av.Date,
Title: b.Title,
DBState: _dbInsert,
}
dailyMap[fmt.Sprintf("%d+%d", c.SID, c.AID)] = c
charges = append(charges, c)
}
ch <- charges
}
}
return
}
// GetBgms map[av_id][]*model.Bgm
func (s *Service) GetBgms(c context.Context, limit int64) (bm map[int64][]*model.Bgm, err error) {
var id int64
bm = make(map[int64][]*model.Bgm)
for {
var bs []*model.Bgm
bs, id, err = s.dao.GetBgm(c, id, limit)
if err != nil {
return
}
if len(bs) == 0 {
break
}
for _, b := range bs {
if _, ok := bm[b.AID]; ok {
bm[b.AID] = append(bm[b.AID], b)
} else {
bm[b.AID] = []*model.Bgm{b}
}
}
}
return
}
func round(val float64, places int) float64 {
var round float64
pow := math.Pow(10, float64(places))
digit := pow * val
_, div := math.Modf(digit)
if div >= 0.5 {
round = math.Ceil(digit)
} else {
round = math.Floor(digit)
}
return round / pow
}
func div(x, y float64) float64 {
a := big.NewFloat(x)
b := big.NewFloat(y)
c := new(big.Float).Quo(a, b)
d, _ := c.Float64()
return d
}
func mul(x, y float64) float64 {
a := big.NewFloat(x)
b := big.NewFloat(y)
c := new(big.Float).Mul(a, b)
d, _ := c.Float64()
return d
}
func (s *Service) bgmDBStore(c context.Context, table string, bgmMap map[string]*model.BgmCharge) error {
insert, update := make([]*model.BgmCharge, _batchSize), make([]*model.BgmCharge, _batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range bgmMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= _batchSize {
_, err := s.bgmBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.bgmBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= _batchSize {
_, err := s.bgmBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.bgmBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.bgmBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.bgmBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.bgmBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.bgmBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleBgmCharge(charges []*model.BgmCharge) (vals string) {
var buf bytes.Buffer
for _, row := range charges {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.SID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.AID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CID, 10))
buf.WriteByte(',')
buf.WriteString("\"" + strings.Replace(row.Title, "\"", "\\\"", -1) + "\"")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString("'" + row.JoinAt.Time().Format(_layoutSec) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *Service) bgmBatchInsert(c context.Context, charges []*model.BgmCharge, table string) (rows int64, err error) {
vals := assembleBgmCharge(charges)
rows, err = s.dao.InsertBgmChargeTable(c, vals, table)
return
}
func (s *Service) bgmStatisDBStore(c context.Context, statisMap map[string]*model.BgmStatis) error {
insert, update := make([]*model.BgmStatis, _batchSize), make([]*model.BgmStatis, _batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range statisMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= _batchSize {
_, err := s.bgmStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.bgmStatisBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= _batchSize {
_, err := s.bgmStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.bgmStatisBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.bgmStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.bgmStatisBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.bgmStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.bgmStatisBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleBgmStatis(statis []*model.BgmStatis) (vals string) {
var buf bytes.Buffer
for _, row := range statis {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.SID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.AID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CID, 10))
buf.WriteByte(',')
buf.WriteString("\"" + strings.Replace(row.Title, "\"", "\\\"", -1) + "\"")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.JoinAt.Time().Format(_layoutSec) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *Service) bgmStatisBatchInsert(c context.Context, statis []*model.BgmStatis) (rows int64, err error) {
vals := assembleBgmStatis(statis)
rows, err = s.dao.InsertBgmStatisBatch(c, vals)
return
}

357
app/job/main/growup/service/charge/column_charge.go Normal file
View File

@@ -0,0 +1,357 @@
package charge
import (
"bytes"
"context"
"fmt"
"strconv"
"time"
model "go-common/app/job/main/growup/model/charge"
task "go-common/app/job/main/growup/service"
"go-common/library/log"
xtime "go-common/library/time"
"golang.org/x/sync/errgroup"
)
func (s *Service) columnCharges(c context.Context, date time.Time, ch chan []*model.Column) (err error) {
defer func() {
close(ch)
}()
var id int64
for {
var charges []*model.Column
charges, err = s.dao.ColumnCharge(c, date, id, _limitSize, "column_daily_charge")
if err != nil {
return
}
ch <- charges
if len(charges) < _limitSize {
break
}
id = charges[len(charges)-1].ID
}
return
}
func (s *Service) handleColumn(c context.Context, date time.Time,
dailyChannel chan []*model.Column) (weeklyMap, monthlyMap map[int64]*model.Column, statisMap map[int64]*model.ColumnStatis, err error) {
var eg errgroup.Group
weeklyMap = make(map[int64]*model.Column)
monthlyMap = make(map[int64]*model.Column)
statisMap = make(map[int64]*model.ColumnStatis)
eg.Go(func() (err error) {
weekly, err := s.getColumnCharge(c, getStartWeeklyDate(date), "column_weekly_charge")
if err != nil {
log.Error("s.GetColumnCharge(column_weekly_charge) error(%v)", err)
return
}
for _, cm := range weekly {
weeklyMap[cm.AID] = cm
}
return
})
eg.Go(func() (err error) {
monthly, err := s.getColumnCharge(c, getStartMonthlyDate(date), "column_monthly_charge")
if err != nil {
log.Error("s.GetColumnCharge(column_monthly_charge) error(%v)", err)
return
}
for _, cm := range monthly {
monthlyMap[cm.AID] = cm
}
return
})
eg.Go(func() (err error) {
statis, err := s.getCmStatisMap(c)
if err != nil {
log.Error("s.getCmStatisMap error(%v)", err)
}
for _, cm := range statis {
statisMap[cm.AID] = cm
}
return
})
if err = eg.Wait(); err != nil {
log.Error("handleColumn eg.Wait error(%v)", err)
return
}
s.calColumns(date, weeklyMap, monthlyMap, statisMap, dailyChannel)
return
}
func (s *Service) calColumns(date time.Time, weeklyMap, monthlyMap map[int64]*model.Column, statisMap map[int64]*model.ColumnStatis, dailyChannel chan []*model.Column) {
for daily := range dailyChannel {
s.calColumn(date, daily, weeklyMap, monthlyMap, statisMap)
}
}
func (s *Service) calColumn(date time.Time, daily []*model.Column, weeklyMap, monthlyMap map[int64]*model.Column, statisMap map[int64]*model.ColumnStatis) {
for _, charge := range daily {
if weekly, ok := weeklyMap[charge.AID]; ok {
updateColumnCharge(weekly, charge)
} else {
weeklyMap[charge.AID] = addColumnCharge(charge, startWeeklyDate)
}
if monthly, ok := monthlyMap[charge.AID]; ok {
updateColumnCharge(monthly, charge)
} else {
monthlyMap[charge.AID] = addColumnCharge(charge, startMonthlyDate)
}
if statis, ok := statisMap[charge.AID]; ok {
updateColumnStatis(statis, charge)
} else {
statisMap[charge.AID] = addColumnStatis(charge)
}
}
}
func addColumnCharge(daily *model.Column, fixDate time.Time) *model.Column {
return &model.Column{
AID: daily.AID,
MID: daily.MID,
Title: daily.Title,
TagID: daily.TagID,
Words: daily.Words,
UploadTime: daily.UploadTime,
IncCharge: daily.IncCharge,
Date: xtime.Time(fixDate.Unix()),
DBState: _dbInsert,
}
}
func updateColumnCharge(origin, daily *model.Column) {
origin.IncCharge += daily.IncCharge
origin.DBState = _dbUpdate
}
func addColumnStatis(daily *model.Column) *model.ColumnStatis {
return &model.ColumnStatis{
AID: daily.AID,
MID: daily.MID,
Title: daily.Title,
TagID: daily.TagID,
UploadTime: daily.UploadTime,
TotalCharge: daily.IncCharge,
DBState: _dbInsert,
}
}
func updateColumnStatis(statis *model.ColumnStatis, daily *model.Column) {
statis.TotalCharge += daily.IncCharge
statis.DBState = _dbUpdate
}
func (s *Service) getColumnCharge(c context.Context, date time.Time, table string) (cms []*model.Column, err error) {
cms = make([]*model.Column, 0)
var id int64
for {
var cm []*model.Column
cm, err = s.dao.ColumnCharge(c, date, id, _limitSize, table)
if err != nil {
return
}
cms = append(cms, cm...)
if len(cm) < _limitSize {
break
}
id = cm[len(cm)-1].ID
}
return
}
func (s *Service) getCmStatisMap(c context.Context) (cms []*model.ColumnStatis, err error) {
cms = make([]*model.ColumnStatis, 0)
var id int64
for {
var cm []*model.ColumnStatis
cm, err = s.dao.CmStatis(c, id, _limitSize)
if err != nil {
return
}
cms = append(cms, cm...)
if len(cm) < _limitSize {
break
}
id = cm[len(cm)-1].ID
}
return
}
func (s *Service) cmDBStore(c context.Context, table string, cmMap map[int64]*model.Column) error {
insert, update := make([]*model.Column, _batchSize), make([]*model.Column, _batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range cmMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= _batchSize {
_, err := s.cmBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.cmBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= _batchSize {
_, err := s.cmBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.cmBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.cmBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.cmBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.cmBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.cmBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleCmCharge(charges []*model.Column) (vals string) {
var buf bytes.Buffer
for _, row := range charges {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.AID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.UploadTime, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *Service) cmBatchInsert(c context.Context, charges []*model.Column, table string) (rows int64, err error) {
vals := assembleCmCharge(charges)
rows, err = s.dao.InsertCmChargeTable(c, vals, table)
return
}
func (s *Service) cmStatisDBStore(c context.Context, statisMap map[int64]*model.ColumnStatis) error {
insert, update := make([]*model.ColumnStatis, _batchSize), make([]*model.ColumnStatis, _batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range statisMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= _batchSize {
_, err := s.cmStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.cmStatisBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= _batchSize {
_, err := s.cmStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.cmStatisBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.cmStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.cmStatisBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.cmStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.cmStatisBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleCmStatis(statis []*model.ColumnStatis) (vals string) {
var buf bytes.Buffer
for _, row := range statis {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.AID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + time.Unix(row.UploadTime, 0).Format(_layoutSec) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *Service) cmStatisBatchInsert(c context.Context, statis []*model.ColumnStatis) (rows int64, err error) {
vals := assembleCmStatis(statis)
rows, err = s.dao.InsertCmStatisBatch(c, vals)
return
}
// CheckTaskColumn check column count by date
func (s *Service) CheckTaskColumn(c context.Context, date string) (err error) {
defer func() {
task.GetTaskService().SetTaskStatus(c, task.TaskCmCharge, date, err)
}()
count, err := s.dao.CountCmDailyCharge(c, date)
if count == 0 {
err = fmt.Errorf("date(%s) column_daily_charge = 0", date)
}
return
}

143
app/job/main/growup/service/charge/date_statis.go Normal file
View File

@@ -0,0 +1,143 @@
package charge
import (
"bytes"
"context"
"fmt"
"math"
"strconv"
"time"
model "go-common/app/job/main/growup/model/charge"
"go-common/library/log"
xtime "go-common/library/time"
)
// SectionEntries section entries
type SectionEntries struct {
daily []*model.DateStatis
weekly []*model.DateStatis
monthly []*model.DateStatis
}
func initChargeSections(charge, tagID int64, date xtime.Time) []*model.DateStatis {
chargeSections := make([]*model.DateStatis, 12)
chargeSections[0] = initChargeSection(0, 1, 0, charge, tagID, date)
chargeSections[1] = initChargeSection(1, 5, 1, charge, tagID, date)
chargeSections[2] = initChargeSection(5, 10, 2, charge, tagID, date)
chargeSections[3] = initChargeSection(10, 20, 3, charge, tagID, date)
chargeSections[4] = initChargeSection(20, 50, 4, charge, tagID, date)
chargeSections[5] = initChargeSection(50, 100, 5, charge, tagID, date)
chargeSections[6] = initChargeSection(100, 200, 6, charge, tagID, date)
chargeSections[7] = initChargeSection(200, 500, 7, charge, tagID, date)
chargeSections[8] = initChargeSection(500, 1000, 8, charge, tagID, date)
chargeSections[9] = initChargeSection(1000, 3000, 9, charge, tagID, date)
chargeSections[10] = initChargeSection(3000, 5000, 10, charge, tagID, date)
chargeSections[11] = initChargeSection(5000, math.MaxInt32, 11, charge, tagID, date)
return chargeSections
}
func initChargeSection(min, max, section, charge, tagID int64, date xtime.Time) *model.DateStatis {
var tips string
if max == math.MaxInt32 {
tips = fmt.Sprintf("\"%d+\"", min)
} else {
tips = fmt.Sprintf("\"%d~%d\"", min, max)
}
return &model.DateStatis{
MinCharge: min,
MaxCharge: max,
MoneySection: section,
MoneyTips: tips,
Charge: charge,
CategoryID: tagID,
CDate: date,
}
}
func (s *Service) handleDateStatis(c context.Context, sourceCh chan []*model.Archive, date time.Time, table string) (sections []*model.DateStatis, err error) {
// delete
if table != "" {
_, err = s.dao.DelStatisTable(c, table, date.Format(_layout))
if err != nil {
log.Error("s.dao.DelChargeStatisTable error(%v)", err)
return
}
}
// add
sections = s.handleArchives(c, sourceCh, date)
return
}
// HandleAv handle archive_charge_daily_statis, archive_charge_weekly_statis, archive_charge_monthly_statis
func (s *Service) handleArchives(c context.Context, archiveCh chan []*model.Archive, date time.Time) (sections []*model.DateStatis) {
archiveTagMap := make(map[int64]map[int64]int64) // key TagID, value map[int64]int64 -> key avId, value charge
tagChargeMap := make(map[int64]int64) // key TagID, value TagID total Charge
handleArchive(archiveCh, archiveTagMap, tagChargeMap, date)
sections = make([]*model.DateStatis, 0)
for tagID, archives := range archiveTagMap {
section := countDateStatis(archives, tagChargeMap[tagID], tagID, date)
sections = append(sections, section...)
}
return
}
func handleArchive(archiveCh chan []*model.Archive, archiveTagMap map[int64]map[int64]int64, tagChargeMap map[int64]int64, startDate time.Time) {
for archives := range archiveCh {
for _, ac := range archives {
if !startDate.After(ac.Date.Time()) {
tagChargeMap[ac.TagID] += ac.IncCharge
if _, ok := archiveTagMap[ac.TagID]; !ok {
archiveTagMap[ac.TagID] = make(map[int64]int64)
}
archiveTagMap[ac.TagID][ac.ID] += ac.IncCharge
}
}
}
}
func countDateStatis(charges map[int64]int64, totalCharge, tagID int64, date time.Time) (sections []*model.DateStatis) {
if len(charges) == 0 {
return
}
sections = initChargeSections(totalCharge, tagID, xtime.Time(date.Unix()))
for _, charge := range charges {
for _, section := range sections {
min, max := section.MinCharge*100, section.MaxCharge*100
if charge >= min && charge < max {
section.Count++
}
}
}
return
}
func (s *Service) dateStatisInsert(c context.Context, avChargeSection []*model.DateStatis, table string) (rows int64, err error) {
var buf bytes.Buffer
for _, row := range avChargeSection {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.Count, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MoneySection, 10))
buf.WriteByte(',')
buf.WriteString(row.MoneyTips)
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Charge, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CategoryID, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.CDate.Time().Format(_layout) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals := buf.String()
buf.Reset()
rows, err = s.dao.InsertStatisTable(c, table, vals)
return
}

99
app/job/main/growup/service/charge/run.go Normal file
View File

@@ -0,0 +1,99 @@
package charge
import (
"context"
"fmt"
"time"
model "go-common/app/job/main/growup/model/charge"
task "go-common/app/job/main/growup/service"
"go-common/library/log"
"golang.org/x/sync/errgroup"
)
const (
_layout = "2006-01-02"
_layoutSec = "2006-01-02 15:04:05"
)
var (
_dbInsert = 1
_dbUpdate = 2
_batchSize = 2000
_limitSize = 2000
startWeeklyDate time.Time
startMonthlyDate time.Time
)
// RunAndSendMail run and send mail
func (s *Service) RunAndSendMail(c context.Context, date time.Time) (err error) {
var msg, msgVideo, msgColumn, msgBgm string
mailReceivers := []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"}
var (
eg errgroup.Group
avBgmCharge = make(chan []*model.AvCharge, 1000)
)
// check task
defer func() {
task.GetTaskService().SetTaskStatus(c, task.TaskCreativeCharge, date.Format(_layout), err)
if err != nil {
msg = err.Error()
}
msgErr := s.email.SendMail(date, msg, "创作激励每日补贴%d年%d月%d日", mailReceivers...)
if msgErr != nil {
log.Error("s.email.SendMail error(%v)", msgErr)
}
}()
err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskAvCharge, task.TaskCmCharge, task.TaskBgmSync)
if err != nil {
return
}
eg.Go(func() (err error) {
startTime := time.Now().Unix()
if err = s.runVideo(c, date, avBgmCharge); err != nil {
log.Error("s.runVideo error(%v)", err)
} else {
msgVideo = fmt.Sprintf("%s 视频补贴计算完成，耗时%ds\n", date.Format("2006-01-02"), time.Now().Unix()-startTime)
}
return
})
eg.Go(func() (err error) {
startTime := time.Now().Unix()
if err = s.runColumn(c, date); err != nil {
log.Error("s.runColumn error(%v)", err)
} else {
msgColumn = fmt.Sprintf("%s 专栏补贴计算完成，耗时%ds\n", date.Format("2006-01-02"), time.Now().Unix()-startTime)
}
return
})
eg.Go(func() (err error) {
startTime := time.Now().Unix()
if err = s.runBgm(c, date, avBgmCharge); err != nil {
log.Error("s.runBgm error(%v)", err)
} else {
msgBgm = fmt.Sprintf("%s 素材补贴计算完成，耗时%ds\n", date.Format("2006-01-02"), time.Now().Unix()-startTime)
}
return
})
if err = eg.Wait(); err != nil {
log.Error("run readGroup.Wait error(%v)", err)
}
msg = fmt.Sprintf("%s,%s,%s", msgVideo, msgColumn, msgBgm)
return
}
func getStartWeeklyDate(date time.Time) time.Time {
for date.Weekday() != time.Monday {
date = date.AddDate(0, 0, -1)
}
return date
}
func getStartMonthlyDate(date time.Time) time.Time {
return time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
}

251
app/job/main/growup/service/charge/run_bgm.go Normal file
View File

@@ -0,0 +1,251 @@
package charge
import (
"context"
"fmt"
"time"
model "go-common/app/job/main/growup/model/charge"
"go-common/library/log"
"golang.org/x/sync/errgroup"
)
var (
_bgmDailyCharge = "bgm_daily_charge"
_bgmWeeklyCharge = "bgm_weekly_charge"
_bgmMonthlyCharge = "bgm_monthly_charge"
_bgmDailyStatis = "bgm_charge_daily_statis"
_bgmWeeklyStatis = "bgm_charge_weekly_statis"
_bgmMonthlyStatis = "bgm_charge_monthly_statis"
)
func (s *Service) runBgm(c context.Context, date time.Time, avBgmCharge chan []*model.AvCharge) (err error) {
startWeeklyDate = getStartWeeklyDate(date)
startMonthlyDate = getStartMonthlyDate(date)
var (
readGroup errgroup.Group
dailyMap = make(map[string]*model.BgmCharge)
sourceCh = make(chan []*model.BgmCharge, 1000)
dailyStatisCh = make(chan []*model.BgmCharge, 1000)
bgmCh = make(chan []*model.BgmCharge, 1000)
)
readGroup.Go(func() (err error) {
dailyMap, err = s.bgmCharges(c, date, sourceCh, avBgmCharge)
if err != nil {
log.Error("s.bgmCharges error(%v)", err)
return
}
log.Info("get bgm_daily_charge finished")
return
})
readGroup.Go(func() (err error) {
defer func() {
close(bgmCh)
close(dailyStatisCh)
}()
for charges := range sourceCh {
bgmCh <- charges
dailyStatisCh <- charges
}
return
})
var (
weeklyMap map[string]*model.BgmCharge
monthlyMap map[string]*model.BgmCharge
statisMap map[string]*model.BgmStatis
weeklyCh = make(chan map[string]*model.BgmCharge, 1)
monthlyCh = make(chan map[string]*model.BgmCharge, 1)
)
readGroup.Go(func() (err error) {
defer func() {
close(weeklyCh)
close(monthlyCh)
}()
weeklyMap, monthlyMap, statisMap, err = s.handleBgm(c, date, bgmCh)
if err != nil {
log.Error("s.handleBgm error(%v)", err)
return
}
weeklyCh <- weeklyMap
monthlyCh <- monthlyMap
log.Info("handleBgm finished")
return
})
var (
dateStatis = &SectionEntries{}
bgmDaily = make(chan []*model.Archive, 2000)
bgmWeekly = make(chan []*model.Archive, 1)
bgmMonthly = make(chan []*model.Archive, 1)
)
readGroup.Go(func() (err error) {
defer close(bgmDaily)
for bgms := range dailyStatisCh {
bgmDaily <- transBgm2Archive(bgms)
}
return
})
readGroup.Go(func() (err error) {
defer close(bgmWeekly)
bgmWeekly <- transBgmMap2Archive(<-weeklyCh)
return
})
readGroup.Go(func() (err error) {
defer close(bgmMonthly)
bgmMonthly <- transBgmMap2Archive(<-monthlyCh)
return
})
readGroup.Go(func() (err error) {
dateStatis.daily, err = s.handleDateStatis(c, bgmDaily, date, _bgmDailyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _bgmDailyStatis, err)
}
return
})
readGroup.Go(func() (err error) {
dateStatis.weekly, err = s.handleDateStatis(c, bgmWeekly, startWeeklyDate, _bgmWeeklyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _bgmWeeklyStatis, err)
}
return
})
readGroup.Go(func() (err error) {
dateStatis.monthly, err = s.handleDateStatis(c, bgmMonthly, startMonthlyDate, _bgmMonthlyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _bgmMonthlyStatis, err)
}
return
})
if err = readGroup.Wait(); err != nil {
log.Error("run readGroup.Wait error(%v)", err)
return
}
{
if len(dailyMap) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_daily_charge")
return
}
if len(weeklyMap) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_weekly_charge")
return
}
if len(monthlyMap) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_monthly_charge")
return
}
if len(statisMap) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_charge_statis")
return
}
if len(dateStatis.daily) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_charge_daily_statis")
return
}
if len(dateStatis.weekly) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_charge_weekly_statis")
return
}
if len(dateStatis.monthly) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_charge_monthly_statis")
return
}
}
// persist
var writeGroup errgroup.Group
// bgm_daily_charge
writeGroup.Go(func() (err error) {
err = s.bgmDBStore(c, _bgmDailyCharge, dailyMap)
if err != nil {
log.Error("s.bgmDBStore bgm_daily_charge error(%v)", err)
return
}
log.Info("insert bgm_daily_charge : %d", len(dailyMap))
// bgm_weekly_charge
err = s.bgmDBStore(c, _bgmWeeklyCharge, weeklyMap)
if err != nil {
log.Error("s.bgmDBStore bgm_weekly_charge error(%v)", err)
return
}
log.Info("insert bgm_weekly_charge : %d", len(weeklyMap))
// bgm_monthly_charge
err = s.bgmDBStore(c, _bgmMonthlyCharge, monthlyMap)
if err != nil {
log.Error("s.bgmDBStore bgm_monthly_charge error(%v)", err)
return
}
log.Info("insert bgm_monthly_charge : %d", len(monthlyMap))
// bgm_charge_statis
err = s.bgmStatisDBStore(c, statisMap)
if err != nil {
log.Error("s.bgmStatisDBStore error(%v)", err)
return
}
log.Info("insert bgm_charge_statis : %d", len(statisMap))
// bgm_charge_daily_statis
_, err = s.dateStatisInsert(c, dateStatis.daily, _bgmDailyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert bgm_charge_daily_statis : %d", len(dateStatis.daily))
// bgm_charge_weekly_statis
_, err = s.dateStatisInsert(c, dateStatis.weekly, _bgmWeeklyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert bgm_charge_weekly_statis : %d", len(dateStatis.weekly))
// bgm_charge_monthly_statis
_, err = s.dateStatisInsert(c, dateStatis.monthly, _bgmMonthlyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert bgm_charge_monthly_statis : %d", len(dateStatis.monthly))
return
})
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
if err = writeGroup.Wait(); err != nil {
log.Error("run writeGroup.Wait error(%v)", err)
}
return
}

232
app/job/main/growup/service/charge/run_column.go Normal file
View File

@@ -0,0 +1,232 @@
package charge
import (
"context"
"fmt"
"time"
model "go-common/app/job/main/growup/model/charge"
"go-common/library/log"
"golang.org/x/sync/errgroup"
)
var (
_cmWeeklyCharge = "column_weekly_charge"
_cmMonthlyCharge = "column_monthly_charge"
_cmDailyStatis = "column_charge_daily_statis"
_cmWeeklyStatis = "column_charge_weekly_statis"
_cmMonthlyStatis = "column_charge_monthly_statis"
)
func (s *Service) runColumn(c context.Context, date time.Time) (err error) {
startWeeklyDate = getStartWeeklyDate(date)
startMonthlyDate = getStartMonthlyDate(date)
var (
readGroup errgroup.Group
sourceCh = make(chan []*model.Column, 1000)
cmCh = make(chan []*model.Column, 1000)
dailyStatisCh = make(chan []*model.Column, 1000)
)
readGroup.Go(func() (err error) {
err = s.columnCharges(c, date, sourceCh)
if err != nil {
log.Error("s.columnCharges error(%v)", err)
return
}
log.Info("read column_daily_charge finished")
return
})
readGroup.Go(func() (err error) {
defer func() {
close(cmCh)
close(dailyStatisCh)
}()
for charges := range sourceCh {
cmCh <- charges
dailyStatisCh <- charges
}
return
})
var (
weeklyMap map[int64]*model.Column
monthlyMap map[int64]*model.Column
statisMap map[int64]*model.ColumnStatis
weeklyCh = make(chan map[int64]*model.Column, 1)
monthlyCh = make(chan map[int64]*model.Column, 1)
)
readGroup.Go(func() (err error) {
defer func() {
close(weeklyCh)
close(monthlyCh)
}()
weeklyMap, monthlyMap, statisMap, err = s.handleColumn(c, date, cmCh)
if err != nil {
log.Error("s.handleColumn error(%v)", err)
return
}
weeklyCh <- weeklyMap
monthlyCh <- monthlyMap
log.Info("handleColumn finished")
return
})
var (
dateStatis = &SectionEntries{}
cmDaily = make(chan []*model.Archive, 2000)
cmWeekly = make(chan []*model.Archive, 1)
cmMonthly = make(chan []*model.Archive, 1)
)
readGroup.Go(func() (err error) {
defer close(cmDaily)
for cms := range dailyStatisCh {
cmDaily <- transCm2Archive(cms)
}
return
})
readGroup.Go(func() (err error) {
defer close(cmWeekly)
cmWeekly <- transCmMap2Archive(<-weeklyCh)
return
})
readGroup.Go(func() (err error) {
defer close(cmMonthly)
cmMonthly <- transCmMap2Archive(<-monthlyCh)
return
})
readGroup.Go(func() (err error) {
dateStatis.daily, err = s.handleDateStatis(c, cmDaily, date, _cmDailyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _cmDailyStatis, err)
}
return
})
readGroup.Go(func() (err error) {
dateStatis.weekly, err = s.handleDateStatis(c, cmWeekly, startWeeklyDate, _cmWeeklyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _cmWeeklyStatis, err)
}
return
})
readGroup.Go(func() (err error) {
dateStatis.monthly, err = s.handleDateStatis(c, cmMonthly, startMonthlyDate, _cmMonthlyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _cmMonthlyStatis, err)
}
return
})
if err = readGroup.Wait(); err != nil {
log.Error("run readGroup.Wait error(%v)", err)
return
}
{
if len(weeklyMap) == 0 {
err = fmt.Errorf("Error: insert 0 column_weekly_charge")
return
}
if len(monthlyMap) == 0 {
err = fmt.Errorf("Error: insert 0 column_monthly_charge")
return
}
if len(statisMap) == 0 {
err = fmt.Errorf("Error: insert 0 column_charge_statis")
return
}
if len(dateStatis.daily) == 0 {
err = fmt.Errorf("Error: insert 0 column_charge_daily_statis")
return
}
if len(dateStatis.weekly) == 0 {
err = fmt.Errorf("Error: insert 0 column_charge_weekly_statis")
return
}
if len(dateStatis.monthly) == 0 {
err = fmt.Errorf("Error: insert 0 column_charge_monthly_statis")
return
}
}
// persist
var writeGroup errgroup.Group
// column_weekly_charge
writeGroup.Go(func() (err error) {
err = s.cmDBStore(c, _cmWeeklyCharge, weeklyMap)
if err != nil {
log.Error("s.cmDBStore column_weekly_charge error(%v)", err)
return
}
log.Info("insert column_weekly_charge : %d", len(weeklyMap))
// column_monthly_charge
err = s.cmDBStore(c, _cmMonthlyCharge, monthlyMap)
if err != nil {
log.Error("s.cmDBStore column_monthly_charge error(%v)", err)
return
}
log.Info("insert column_monthly_charge : %d", len(monthlyMap))
// column_charge_statis
err = s.cmStatisDBStore(c, statisMap)
if err != nil {
log.Error("s.cmStatisDBStore error(%v)", err)
return
}
log.Info("insert column_charge_statis : %d", len(statisMap))
// column_charge_daily_statis
_, err = s.dateStatisInsert(c, dateStatis.daily, _cmDailyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert column_charge_daily_statis : %d", len(dateStatis.daily))
// column_charge_weekly_statis
_, err = s.dateStatisInsert(c, dateStatis.weekly, _cmWeeklyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert column_charge_weekly_statis : %d", len(dateStatis.weekly))
// column_charge_monthly_statis
_, err = s.dateStatisInsert(c, dateStatis.monthly, _cmMonthlyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert column_charge_monthly_statis : %d", len(dateStatis.monthly))
return
})
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
if err = writeGroup.Wait(); err != nil {
log.Error("run writeGroup.Wait error(%v)", err)
}
return
}

369
app/job/main/growup/service/charge/run_test.go Normal file
View File

@@ -0,0 +1,369 @@
package charge
import (
"context"
"fmt"
"strings"
"testing"
"time"
model "go-common/app/job/main/growup/model/charge"
xtime "go-common/library/time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_AvCharge(t *testing.T) {
Convey("Test av weekly and monthly charge\n", t, func() {
c := context.Background()
deleteAll(c)
ac1 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1101, 110, 10)
ac2 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1102, 110, 10)
d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
err := s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvWeeklyCharge(c, ac1, 30, 10, 10)
checkAvWeeklyCharge(c, ac2, 30, 10, 10)
checkAvMonthlyCharge(c, ac1, 30, 10, 10)
checkAvMonthlyCharge(c, ac2, 30, 10, 10)
ac1 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1101, 110, 10)
ac2 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1102, 110, 20)
d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvWeeklyCharge(c, ac1, 60, 20, 20)
checkAvWeeklyCharge(c, ac2, 90, 30, 30)
checkAvMonthlyCharge(c, ac1, 60, 20, 20)
checkAvMonthlyCharge(c, ac2, 90, 30, 30)
ac1 = insertAvDailyCharge(c, "2018-06-05", "2018-06-01 15:02:03", 1101, 110, 15)
d, _ = time.ParseInLocation("2006-01-02", "2018-06-05", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvWeeklyCharge(c, ac1, 45, 15, 15)
checkAvMonthlyCharge(c, ac1, 105, 35, 35)
})
}
func Test_AvChargeDateStatis(t *testing.T) {
Convey("Test av charge date statis\n", t, func() {
c := context.Background()
deleteAll(c)
insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1101, 110, 10)
insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1102, 110, 10)
insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1103, 110, 3500)
d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
err := s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvChargeDateStatis(c, 0, 2, 1, 3520, 12, d, "av_charge_daily_statis")
checkAvChargeDateStatis(c, 1, 1, 1, 3520, 12, d, "av_charge_daily_statis")
weekD := getStartWeeklyDate(d)
checkAvChargeDateStatis(c, 0, 2, 1, 3520, 12, weekD, "av_charge_weekly_statis")
checkAvChargeDateStatis(c, 1, 1, 1, 3520, 12, weekD, "av_charge_weekly_statis")
monthD := getStartMonthlyDate(d)
checkAvChargeDateStatis(c, 0, 2, 1, 3520, 12, monthD, "av_charge_monthly_statis")
checkAvChargeDateStatis(c, 1, 1, 1, 3520, 12, monthD, "av_charge_monthly_statis")
insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1101, 110, 1000)
insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1102, 110, 1000)
insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1103, 110, 4000)
d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvChargeDateStatis(c, 0, 2, 1, 6000, 12, d, "av_charge_daily_statis")
checkAvChargeDateStatis(c, 1, 1, 1, 6000, 12, d, "av_charge_daily_statis")
weekD = getStartWeeklyDate(d)
charge := 6000 + 3520
checkAvChargeDateStatis(c, 0, 2, 1, charge, 12, weekD, "av_charge_weekly_statis")
checkAvChargeDateStatis(c, 1, 0, 1, charge, 12, weekD, "av_charge_weekly_statis")
checkAvChargeDateStatis(c, 2, 1, 1, charge, 12, weekD, "av_charge_weekly_statis")
monthD = getStartMonthlyDate(d)
checkAvChargeDateStatis(c, 0, 2, 1, charge, 12, monthD, "av_charge_monthly_statis")
checkAvChargeDateStatis(c, 1, 0, 1, charge, 12, monthD, "av_charge_monthly_statis")
checkAvChargeDateStatis(c, 2, 1, 1, charge, 12, monthD, "av_charge_monthly_statis")
insertAvDailyCharge(c, "2018-06-10", "2018-06-01 15:02:03", 1101, 110, 15000)
insertAvDailyCharge(c, "2018-06-10", "2018-06-01 15:02:03", 1102, 110, 15000)
insertAvDailyCharge(c, "2018-06-10", "2018-06-01 15:02:03", 1103, 110, 40000)
d, _ = time.ParseInLocation("2006-01-02", "2018-06-10", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvChargeDateStatis(c, 0, 0, 1, 70000, 12, d, "av_charge_daily_statis")
checkAvChargeDateStatis(c, 1, 0, 1, 70000, 12, d, "av_charge_daily_statis")
checkAvChargeDateStatis(c, 3, 2, 1, 70000, 12, d, "av_charge_daily_statis")
checkAvChargeDateStatis(c, 4, 1, 1, 70000, 12, d, "av_charge_daily_statis")
weekD = getStartWeeklyDate(d)
checkAvChargeDateStatis(c, 0, 0, 1, 70000, 12, weekD, "av_charge_weekly_statis")
checkAvChargeDateStatis(c, 1, 0, 1, 70000, 12, weekD, "av_charge_weekly_statis")
checkAvChargeDateStatis(c, 3, 2, 1, 70000, 12, weekD, "av_charge_weekly_statis")
checkAvChargeDateStatis(c, 4, 1, 1, 70000, 12, weekD, "av_charge_weekly_statis")
monthD = getStartMonthlyDate(d)
checkAvChargeDateStatis(c, 0, 0, 1, 79520, 12, monthD, "av_charge_monthly_statis")
checkAvChargeDateStatis(c, 1, 0, 1, 79520, 12, monthD, "av_charge_monthly_statis")
checkAvChargeDateStatis(c, 3, 2, 1, 79520, 12, monthD, "av_charge_monthly_statis")
checkAvChargeDateStatis(c, 4, 1, 1, 79520, 12, monthD, "av_charge_monthly_statis")
insertAvDailyCharge(c, "2018-07-10", "2018-06-01 15:02:03", 1101, 110, 100)
d, _ = time.ParseInLocation("2006-01-02", "2018-07-10", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
monthD = getStartMonthlyDate(d)
checkAvChargeDateStatis(c, 0, 1, 1, 100, 12, monthD, "av_charge_monthly_statis")
checkAvChargeDateStatis(c, 1, 0, 1, 100, 12, monthD, "av_charge_monthly_statis")
})
}
func Test_AvChargeStatis(t *testing.T) {
Convey("Test av charge statis\n", t, func() {
c := context.Background()
deleteAll(c)
ac1 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1101, 110, 1000)
ac2 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1102, 110, 1000)
d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
err := s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvChargeStatis(c, ac1, 1000)
checkAvChargeStatis(c, ac2, 1000)
ac1 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1101, 110, 1000)
ac2 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1102, 110, 1000)
ac3 := insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1103, 110, 1000)
d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvChargeStatis(c, ac1, 2000)
checkAvChargeStatis(c, ac2, 2000)
checkAvChargeStatis(c, ac3, 1000)
ac2 = insertAvDailyCharge(c, "2018-07-03", "2018-06-01 15:02:03", 1102, 110, 1000)
ac3 = insertAvDailyCharge(c, "2018-07-03", "2018-06-01 15:02:03", 1103, 110, 1000)
ac4 := insertAvDailyCharge(c, "2018-07-03", "2018-06-01 15:02:03", 1104, 110, 1000)
d, _ = time.ParseInLocation("2006-01-02", "2018-07-03", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkAvChargeStatis(c, ac1, 2000)
checkAvChargeStatis(c, ac2, 3000)
checkAvChargeStatis(c, ac3, 2000)
checkAvChargeStatis(c, ac4, 1000)
})
}
// up_charge
func Test_UpCharge(t *testing.T) {
Convey("Test up daily weekly monthly charge\n", t, func() {
c := context.Background()
deleteAll(c)
ac1 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 20:02:03", 1101, 110, 10)
ac2 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 20:02:03", 1102, 110, 10)
d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
err := s.runVideo(c, d, nil)
So(err, ShouldBeNil)
acs := []*model.AvCharge{ac1, ac2}
checkUpDailyCharge(c, acs, 2, 20, "up_daily_charge", "2018-06-01")
checkUpWeeklyCharge(c, acs, 2, 20)
checkUpMonthlyCharge(c, acs, 2, 20)
ac11 := insertAvDailyCharge(c, "2018-06-02", "2018-06-01 20:02:03", 1101, 110, 10)
ac21 := insertAvDailyCharge(c, "2018-06-02", "2018-06-01 20:02:03", 1102, 110, 10)
d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
acs = append(acs, []*model.AvCharge{ac11, ac21}...)
checkUpDailyCharge(c, []*model.AvCharge{ac11, ac21}, 2, 20, "up_daily_charge", "2018-06-01")
checkUpWeeklyCharge(c, acs, 2, 40)
checkUpMonthlyCharge(c, acs, 2, 40)
ac12 := insertAvDailyCharge(c, "2018-06-05", "2018-06-01 20:02:03", 1101, 110, 10)
d, _ = time.ParseInLocation("2006-01-02", "2018-06-05", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
acs = append(acs, []*model.AvCharge{ac12}...)
checkUpDailyCharge(c, []*model.AvCharge{ac12}, 1, 10, "up_daily_charge", "2018-06-05")
checkUpWeeklyCharge(c, []*model.AvCharge{ac12}, 1, 10)
checkUpMonthlyCharge(c, acs, 2, 50)
ac13 := insertAvDailyCharge(c, "2018-07-05", "2018-06-01 20:02:03", 1101, 110, 10)
d, _ = time.ParseInLocation("2006-01-02", "2018-07-05", time.Local)
err = s.runVideo(c, d, nil)
So(err, ShouldBeNil)
checkUpDailyCharge(c, []*model.AvCharge{ac13}, 1, 10, "up_daily_charge", "2018-07-05")
checkUpWeeklyCharge(c, []*model.AvCharge{ac13}, 1, 10)
checkUpMonthlyCharge(c, []*model.AvCharge{ac13}, 1, 10)
})
}
func checkAvWeeklyCharge(c context.Context, ac *model.AvCharge, totalPlayCount, incCharge, totalCharge int64) {
avWeeklyCharge, err := s.GetAvCharge(c, startWeeklyDate, s.dao.AvWeeklyCharge)
So(err, ShouldBeNil)
var aw *model.AvCharge
for _, av := range avWeeklyCharge {
if av.AvID == ac.AvID {
aw = av
break
}
}
So(ac.AvID, ShouldEqual, aw.AvID)
So(ac.MID, ShouldEqual, aw.MID)
So(ac.TagID, ShouldEqual, aw.TagID)
So(ac.IsOriginal, ShouldEqual, aw.IsOriginal)
So(ac.UploadTime, ShouldEqual, aw.UploadTime)
So(totalPlayCount, ShouldEqual, aw.TotalPlayCount)
So(totalCharge, ShouldEqual, aw.TotalCharge)
So(incCharge, ShouldEqual, aw.IncCharge)
So(xtime.Time(getStartWeeklyDate(ac.Date.Time()).Unix()), ShouldEqual, aw.Date)
}
func checkAvMonthlyCharge(c context.Context, ac *model.AvCharge, totalPlayCount, incCharge, totalCharge int64) {
avCharge, err := s.GetAvCharge(c, startMonthlyDate, s.dao.AvMonthlyCharge)
So(err, ShouldBeNil)
var aw *model.AvCharge
for _, av := range avCharge {
if av.AvID == ac.AvID {
aw = av
break
}
}
So(ac.AvID, ShouldEqual, aw.AvID)
So(ac.MID, ShouldEqual, aw.MID)
So(ac.TagID, ShouldEqual, aw.TagID)
So(ac.IsOriginal, ShouldEqual, aw.IsOriginal)
So(ac.UploadTime, ShouldEqual, aw.UploadTime)
So(totalPlayCount, ShouldEqual, aw.TotalPlayCount)
So(totalCharge, ShouldEqual, aw.TotalCharge)
So(incCharge, ShouldEqual, aw.IncCharge)
So(xtime.Time(getStartMonthlyDate(ac.Date.Time()).Unix()), ShouldEqual, aw.Date)
}
func checkAvChargeStatis(c context.Context, ac *model.AvCharge, totalCharge int64) {
ai := &model.AvChargeStatis{}
err := s.dao.QueryRow(c, fmt.Sprintf("select av_id,mid,tag_id,is_original,upload_time,total_charge from av_charge_statis where av_id = %d", ac.AvID)).Scan(
&ai.AvID, &ai.MID, &ai.TagID, &ai.IsOriginal, &ai.UploadTime, &ai.TotalCharge)
So(err, ShouldBeNil)
So(ac.AvID, ShouldEqual, ai.AvID)
So(ac.MID, ShouldEqual, ai.MID)
So(ac.TagID, ShouldEqual, ai.TagID)
So(ac.IsOriginal, ShouldEqual, ai.IsOriginal)
So(ac.UploadTime, ShouldEqual, ai.UploadTime)
So(totalCharge, ShouldEqual, ai.TotalCharge)
}
func checkUpDailyCharge(c context.Context, acs []*model.AvCharge, count, total int64, table, date string) {
if len(acs) == 0 {
return
}
if date == "" {
date = acs[0].Date.Time().Format(_layout)
}
var incCharge int64
for _, ac := range acs {
incCharge += ac.IncCharge
}
up := &model.UpCharge{}
err := s.dao.QueryRow(c, fmt.Sprintf("select av_count,inc_charge, total_charge from %s where mid = %d and date = '%s'", table, acs[0].MID, date)).Scan(
&up.AvCount, &up.IncCharge, &up.TotalCharge)
So(err, ShouldBeNil)
So(total, ShouldEqual, up.TotalCharge)
So(incCharge, ShouldEqual, up.IncCharge)
}
func checkUpWeeklyCharge(c context.Context, avs []*model.AvCharge, count, total int64) {
m := make(map[string][]*model.AvCharge)
for _, av := range avs {
d := getStartWeeklyDate(av.Date.Time()).Format(_layout)
if _, ok := m[d]; !ok {
m[d] = make([]*model.AvCharge, 0)
}
m[d] = append(m[d], av)
}
for date, avs := range m {
checkUpDailyCharge(c, avs, count, total, "up_weekly_charge", date)
}
}
func checkUpMonthlyCharge(c context.Context, avs []*model.AvCharge, count, total int64) {
m := make(map[string][]*model.AvCharge)
for _, av := range avs {
d := getStartMonthlyDate(av.Date.Time()).Format(_layout)
if _, ok := m[d]; !ok {
m[d] = make([]*model.AvCharge, 0)
}
m[d] = append(m[d], av)
}
for date, avs := range m {
checkUpDailyCharge(c, avs, count, total, "up_monthly_charge", date)
}
}
func checkAvChargeDateStatis(c context.Context, section, avs, categoryID, charge, count int, d time.Time, table string) {
xd := xtime.Time(d.Unix())
ads := &model.DateStatis{}
err := s.dao.QueryRow(c, fmt.Sprintf("select avs,charge,cdate from %s where money_section = %d and category_id = %d and cdate = '%s'", table, section, categoryID, d.Format(_layout))).Scan(
&ads.Count, &ads.Charge, &ads.CDate)
So(err, ShouldBeNil)
So(charge, ShouldEqual, ads.Charge)
So(xd, ShouldEqual, ads.CDate)
So(avs, ShouldEqual, ads.Count)
var ccount int64
err = s.dao.QueryRow(c, fmt.Sprintf("select count(*) from %s where category_id = %d and cdate = '%s'", table, categoryID, d.Format(_layout))).Scan(&ccount)
So(err, ShouldBeNil)
So(count, ShouldEqual, ccount)
}
func deleteAll(c context.Context) {
s.dao.Exec(c, "truncate av_daily_charge_06")
s.dao.Exec(c, "truncate av_daily_charge_07")
s.dao.Exec(c, "truncate av_weekly_charge")
s.dao.Exec(c, "truncate av_monthly_charge")
s.dao.Exec(c, "truncate av_charge_statis")
s.dao.Exec(c, "truncate up_daily_charge")
s.dao.Exec(c, "truncate up_weekly_charge")
s.dao.Exec(c, "truncate up_monthly_charge")
s.dao.Exec(c, "truncate av_charge_daily_statis")
s.dao.Exec(c, "truncate av_charge_weekly_statis")
s.dao.Exec(c, "truncate av_charge_monthly_statis")
}
func getAvDailyChargeStruct(date string, uploadDate string, avID, mid, charge int64) *model.AvCharge {
d, _ := time.ParseInLocation(_layout, date, time.Local)
upD, _ := time.ParseInLocation(_layoutSec, uploadDate, time.Local)
ac := &model.AvCharge{
AvID: avID,
MID: mid,
TagID: 1,
IsOriginal: 1,
DanmakuCount: charge,
CommentCount: charge,
CollectCount: charge,
CoinCount: charge,
ShareCount: charge,
ElecPayCount: charge,
TotalPlayCount: charge * int64(3),
WebPlayCount: charge,
AppPlayCount: charge,
H5PlayCount: charge,
LvUnknown: charge,
Lv0: charge,
Lv1: charge,
Lv2: charge,
Lv3: charge,
Lv4: charge,
Lv5: charge,
Lv6: charge,
VScore: charge,
IncCharge: charge,
TotalCharge: 0,
Date: xtime.Time(d.Unix()),
UploadTime: xtime.Time(upD.Unix()),
DBState: _dbInsert,
}
return ac
}
func insertAvDailyCharge(c context.Context, date string, uploadDate string, avID, mid, charge int64) *model.AvCharge {
s.dao.Exec(c, fmt.Sprintf("insert into task_status(type,date,status) values(1,'%s',1) on duplicate key update date=values(date)", date))
ac := getAvDailyChargeStruct(date, uploadDate, avID, mid, charge)
_, err := s.avChargeBatchInsert(c, []*model.AvCharge{ac}, fmt.Sprintf("av_daily_charge_%s", strings.Split(date, "-")[1]))
So(err, ShouldBeNil)
return ac
}

299
app/job/main/growup/service/charge/run_video.go Normal file
View File

@@ -0,0 +1,299 @@
package charge
import (
"context"
"fmt"
"time"
model "go-common/app/job/main/growup/model/charge"
"go-common/library/log"
"golang.org/x/sync/errgroup"
)
var (
_avChargeDailyStatis = "av_charge_daily_statis"
_avChargeWeeklyStatis = "av_charge_weekly_statis"
_avChargeMonthlyStatis = "av_charge_monthly_statis"
_avWeeklyCharge = "av_weekly_charge"
_avMonthlyCharge = "av_monthly_charge"
)
func (s *Service) runVideo(c context.Context, date time.Time, avBgmCharge chan []*model.AvCharge) (err error) {
startWeeklyDate = getStartWeeklyDate(date)
startMonthlyDate = getStartMonthlyDate(date)
var (
readGroup errgroup.Group
sourceCh = make(chan []*model.AvCharge, 1000)
avChargeCh = make(chan []*model.AvCharge, 1000)
upChargeCh = make(chan []*model.AvCharge, 1000)
dailyStatisCh = make(chan []*model.AvCharge, 1000)
)
readGroup.Go(func() (err error) {
err = s.avDailyCharges(c, date, sourceCh)
if err != nil {
log.Error("s.avCharge.AvCharges error(%v)", err)
return
}
log.Info("read av_daily_charge finished")
return
})
readGroup.Go(func() (err error) {
defer func() {
close(avChargeCh)
close(upChargeCh)
close(dailyStatisCh)
close(avBgmCharge)
}()
for charges := range sourceCh {
avChargeCh <- charges
upChargeCh <- charges
dailyStatisCh <- charges
avBgmCharge <- charges
}
return
})
var (
weeklyChargeMap map[int64]*model.AvCharge
monthlyChargeMap map[int64]*model.AvCharge
chargeStatisMap map[int64]*model.AvChargeStatis
weeklyCh = make(chan map[int64]*model.AvCharge, 1)
monthlyCh = make(chan map[int64]*model.AvCharge, 1)
)
readGroup.Go(func() (err error) {
defer func() {
close(weeklyCh)
close(monthlyCh)
}()
weeklyChargeMap, monthlyChargeMap, chargeStatisMap, err = s.handleAvCharge(c, date, avChargeCh)
if err != nil {
log.Error("s.handleAvCharge error(%v)", err)
return
}
weeklyCh <- weeklyChargeMap
monthlyCh <- monthlyChargeMap
log.Info("handleAvCharge finished")
return
})
var (
dateStatis = &SectionEntries{}
avDaily = make(chan []*model.Archive, 2000)
avWeekly = make(chan []*model.Archive, 1)
avMonthly = make(chan []*model.Archive, 1)
)
readGroup.Go(func() (err error) {
defer close(avDaily)
for avs := range dailyStatisCh {
avDaily <- transAv2Archive(avs)
}
return
})
readGroup.Go(func() (err error) {
defer close(avWeekly)
avWeekly <- transAvMap2Archive(<-weeklyCh)
return
})
readGroup.Go(func() (err error) {
defer close(avMonthly)
avMonthly <- transAvMap2Archive(<-monthlyCh)
return
})
readGroup.Go(func() (err error) {
dateStatis.daily, err = s.handleDateStatis(c, avDaily, date, _avChargeDailyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _avChargeDailyStatis, err)
}
return
})
readGroup.Go(func() (err error) {
dateStatis.weekly, err = s.handleDateStatis(c, avWeekly, startWeeklyDate, _avChargeWeeklyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _avChargeWeeklyStatis, err)
}
return
})
readGroup.Go(func() (err error) {
dateStatis.monthly, err = s.handleDateStatis(c, avMonthly, startMonthlyDate, _avChargeMonthlyStatis)
if err != nil {
log.Error("s.handleDateStatis(%s) error(%v)", _avChargeMonthlyStatis, err)
}
return
})
var (
daily map[int64]*model.UpCharge
weekly map[int64]*model.UpCharge
monthly map[int64]*model.UpCharge
)
readGroup.Go(func() (err error) {
daily, weekly, monthly, err = s.calUpCharge(c, date, upChargeCh)
if err != nil {
log.Error("s.calUpCharge error(%v)", err)
return
}
log.Info("s.calUpCharge finished")
return
})
if err = readGroup.Wait(); err != nil {
log.Error("run readGroup.Wait error(%v)", err)
return
}
{
if len(weeklyChargeMap) == 0 {
err = fmt.Errorf("Error: insert 0 av_weekly_charge")
return
}
if len(monthlyChargeMap) == 0 {
err = fmt.Errorf("Error: insert 0 av_monthly_charge")
return
}
if len(chargeStatisMap) == 0 {
err = fmt.Errorf("Error: insert 0 av_charge_statis")
return
}
if len(daily) == 0 {
err = fmt.Errorf("Error: insert 0 up_daily_charge")
return
}
if len(weekly) == 0 {
err = fmt.Errorf("Error: insert 0 up_weekly_charge")
return
}
if len(monthly) == 0 {
err = fmt.Errorf("Error: insert 0 up_monthly_charge")
return
}
if len(dateStatis.daily) == 0 {
err = fmt.Errorf("Error: insert 0 av_charge_daily_statis")
return
}
if len(dateStatis.weekly) == 0 {
err = fmt.Errorf("Error: insert 0 av_charge_weekly_statis")
return
}
if len(dateStatis.monthly) == 0 {
err = fmt.Errorf("Error: insert 0 av_charge_monthly_statis")
return
}
}
// persist
var writeGroup errgroup.Group
// av_weekly_charge
writeGroup.Go(func() (err error) {
err = s.AvChargeDBStore(c, _avWeeklyCharge, weeklyChargeMap)
if err != nil {
log.Error("s.AvChargeDBStore av_weekly_charge error(%v)", err)
return
}
log.Info("insert av_weekly_charge : %d", len(weeklyChargeMap))
return
})
// av_monthly_charge
writeGroup.Go(func() (err error) {
err = s.AvChargeDBStore(c, _avMonthlyCharge, monthlyChargeMap)
if err != nil {
log.Error("s.AvChargeDBStore av_monthly_charge error(%v)", err)
return
}
log.Info("insert av_monthly_charge : %d", len(monthlyChargeMap))
// av_charge_statis
err = s.AvChargeStatisDBStore(c, chargeStatisMap)
if err != nil {
log.Error("s.AvChargeStatisDBStore error(%v)", err)
return
}
log.Info("insert av_charge_statis : %d", len(chargeStatisMap))
// up_daily_charge
err = s.BatchInsertUpCharge(c, "up_daily_charge", daily)
if err != nil {
log.Error("s.BatchInsertUpDailyCharge error(%v)", err)
return
}
log.Info("insert up_daily_charge : %d", len(daily))
// up_weekly_charge
err = s.BatchInsertUpCharge(c, "up_weekly_charge", weekly)
if err != nil {
log.Error("s.BatchInsertUpWeeklyCharge error(%v)", err)
return
}
log.Info("insert up_weekly_charge : %d", len(weekly))
// up_monthly_charge
err = s.BatchInsertUpCharge(c, "up_monthly_charge", monthly)
if err != nil {
log.Error("s.BatchInsertUpMonthlyCharge error(%v)", err)
return
}
log.Info("insert up_monthly_charge : %d", len(monthly))
// av_charge_daily_statis
_, err = s.dateStatisInsert(c, dateStatis.daily, _avChargeDailyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert av_charge_daily_statis : %d", len(dateStatis.daily))
// av_charge_weekly_statis
_, err = s.dateStatisInsert(c, dateStatis.weekly, _avChargeWeeklyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert av_charge_weekly_statis : %d", len(dateStatis.weekly))
// av_charge_monthly_statis
_, err = s.dateStatisInsert(c, dateStatis.monthly, _avChargeMonthlyStatis)
if err != nil {
log.Error("s.dateStatisInsert error(%v)", err)
return
}
log.Info("insert av_charge_monthly_statis : %d", len(dateStatis.monthly))
return
})
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
//
// writeGroup.Go(func() (err error) {
// return
// })
if err = writeGroup.Wait(); err != nil {
log.Error("run writeGroup.Wait error(%v)", err)
}
return
}

79
app/job/main/growup/service/charge/service.go Normal file
View File

@@ -0,0 +1,79 @@
package charge
import (
"context"
"fmt"
"time"
"go-common/app/job/main/growup/conf"
chargeD "go-common/app/job/main/growup/dao/charge"
"go-common/app/job/main/growup/dao/email"
"go-common/app/job/main/growup/service"
"go-common/app/job/main/growup/service/ctrl"
"go-common/library/log"
)
// Service struct
type Service struct {
conf *conf.Config
dao *chargeD.Dao
email *email.Dao
}
// New fn
func New(c *conf.Config, executor ctrl.Executor) (s *Service) {
s = &Service{
conf: c,
dao: chargeD.New(c),
email: email.New(c),
}
log.Info("charge service start")
executor.Submit(
s.calDailyCreativeCharge,
s.checkColumnDailyCharge,
)
return s
}
func (s *Service) calDailyCreativeCharge(ctx context.Context) {
for {
time.Sleep(service.NextDay(17, 0, 0))
log.Info("calDailyCreativeCharge begin:%v", time.Now().Format("2006-01-02 15:04:05"))
now := time.Now()
date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local)
err := s.RunAndSendMail(context.TODO(), date)
if err != nil {
log.Error("s.RunAndSendMail error(%v)", err)
}
log.Info("calDailyCreativeCharge end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) checkColumnDailyCharge(ctx context.Context) {
for {
time.Sleep(service.NextDay(13, 0, 0))
msg := ""
log.Info("checkColumnDailyCharge begin:%v", time.Now().Format("2006-01-02 15:04:05"))
now := time.Now()
date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local)
err := s.CheckTaskColumn(context.TODO(), date.Format(_layout))
if err != nil {
log.Error("s.CheckTaskColumn error(%v)", err)
}
if err != nil {
msg = fmt.Sprintf("CheckTaskColumn error(%v)", err)
} else {
msg = "Success"
}
err = s.email.SendMail(date, msg, "创作激励专栏补贴数据%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("checkColumnDailyCharge end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
// Ping check dao health.
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}

33
app/job/main/growup/service/charge/service_test.go Normal file
View File

@@ -0,0 +1,33 @@
package charge
import (
"context"
"flag"
"path/filepath"
"testing"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/service/ctrl"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *Service
)
func init() {
dir, _ := filepath.Abs("../../cmd/growup-job.toml")
flag.Set("conf", dir)
conf.Init()
if s == nil {
s = New(conf.Conf, ctrl.NewUnboundedExecutor())
}
}
func TestPing(t *testing.T) {
Convey("Test_Ping", t, func() {
err := s.Ping(context.Background())
So(err, ShouldBeNil)
})
}

83
app/job/main/growup/service/charge/trans.go Normal file
View File

@@ -0,0 +1,83 @@
package charge
import (
model "go-common/app/job/main/growup/model/charge"
)
func transAv2Archive(avs []*model.AvCharge) (archs []*model.Archive) {
archs = make([]*model.Archive, 0, len(avs))
for _, av := range avs {
archs = append(archs, &model.Archive{
ID: av.AvID,
IncCharge: av.IncCharge,
TagID: av.TagID,
Date: av.Date,
})
}
return
}
func transAvMap2Archive(avs map[int64]*model.AvCharge) (archs []*model.Archive) {
archs = make([]*model.Archive, 0, len(avs))
for _, av := range avs {
archs = append(archs, &model.Archive{
ID: av.AvID,
IncCharge: av.IncCharge,
TagID: av.TagID,
Date: av.Date,
})
}
return
}
func transCm2Archive(cms []*model.Column) (archs []*model.Archive) {
archs = make([]*model.Archive, 0, len(cms))
for _, cm := range cms {
archs = append(archs, &model.Archive{
ID: cm.AID,
IncCharge: cm.IncCharge,
TagID: cm.TagID,
Date: cm.Date,
})
}
return
}
func transCmMap2Archive(cms map[int64]*model.Column) (archs []*model.Archive) {
archs = make([]*model.Archive, 0, len(cms))
for _, cm := range cms {
archs = append(archs, &model.Archive{
ID: cm.AID,
IncCharge: cm.IncCharge,
TagID: cm.TagID,
Date: cm.Date,
})
}
return
}
func transBgm2Archive(bgms []*model.BgmCharge) (archs []*model.Archive) {
archs = make([]*model.Archive, 0, len(bgms))
for _, bgm := range bgms {
archs = append(archs, &model.Archive{
ID: bgm.SID,
IncCharge: bgm.IncCharge,
TagID: 0,
Date: bgm.Date,
})
}
return
}
func transBgmMap2Archive(bgms map[string]*model.BgmCharge) (archs []*model.Archive) {
archs = make([]*model.Archive, 0, len(bgms))
for _, bgm := range bgms {
archs = append(archs, &model.Archive{
ID: bgm.SID,
IncCharge: bgm.IncCharge,
TagID: 0,
Date: bgm.Date,
})
}
return
}

170
app/job/main/growup/service/charge/up_charge.go Normal file
View File

@@ -0,0 +1,170 @@
package charge
import (
"bytes"
"context"
"fmt"
"strconv"
"time"
model "go-common/app/job/main/growup/model/charge"
xtime "go-common/library/time"
)
func (s *Service) calUpCharge(c context.Context, t time.Time, avc chan []*model.AvCharge) (daily, weekly, monthly map[int64]*model.UpCharge, err error) {
od, err := s.getUpDailyCharge(c, t)
if err != nil {
return
}
daily = calUpDailyCharge(od, avc)
weekly, err = s.getUpWeeklyCharge(c)
if err != nil {
return
}
// group by weekly
upChargeStat(startWeeklyDate, weekly, daily)
monthly, err = s.getUpMonthlyCharge(c)
if err != nil {
return
}
// group by monthly
upChargeStat(startMonthlyDate, monthly, daily)
return
}
// 1 od: old daily up charges
func (s *Service) getUpDailyCharge(c context.Context, t time.Time) (od map[int64]*model.UpCharge, err error) {
// old: yesterday
old := t.AddDate(0, 0, -1)
return s.getUpCharges(c, "up_daily_charge", old.Format("2006-01-02"))
}
// 2 ow: old weekly up charges
func (s *Service) getUpWeeklyCharge(c context.Context) (ow map[int64]*model.UpCharge, err error) {
return s.getUpCharges(c, "up_weekly_charge", startWeeklyDate.Format("2006-01-02"))
}
// 3 om: old monthly up charges
func (s *Service) getUpMonthlyCharge(c context.Context) (om map[int64]*model.UpCharge, err error) {
return s.getUpCharges(c, "up_monthly_charge", startMonthlyDate.Format("2006-01-02"))
}
func calUpDailyCharge(od map[int64]*model.UpCharge, avc chan []*model.AvCharge) (mu map[int64]*model.UpCharge) {
mu = make(map[int64]*model.UpCharge)
for charges := range avc {
for _, charge := range charges {
if charge.IncCharge <= 0 {
continue
}
// udc: up daily charge
if udc, ok := mu[charge.MID]; ok {
udc.IncCharge += charge.IncCharge
udc.TotalCharge += charge.IncCharge
} else {
var total int64
if o, ok := od[charge.MID]; ok {
total = o.TotalCharge
}
mu[charge.MID] = &model.UpCharge{
MID: charge.MID,
IncCharge: charge.IncCharge,
TotalCharge: total + charge.IncCharge,
Date: charge.Date,
}
}
}
}
return
}
// os: old weekly/month charge chan, empty maybe
func upChargeStat(t time.Time, os, daily map[int64]*model.UpCharge) {
for mid, ucd := range daily {
if charge, ok := os[mid]; ok {
// update
charge.IncCharge += ucd.IncCharge
charge.TotalCharge += ucd.IncCharge
charge.Date = xtime.Time(t.Unix())
} else {
// new
os[mid] = &model.UpCharge{
MID: mid,
IncCharge: ucd.IncCharge,
TotalCharge: ucd.TotalCharge,
Date: xtime.Time(t.Unix()),
}
}
}
}
// get up charges by date
func (s *Service) getUpCharges(c context.Context, table, date string) (m map[int64]*model.UpCharge, err error) {
var id int64
m = make(map[int64]*model.UpCharge)
for {
var charges map[int64]*model.UpCharge
id, charges, err = s.dao.GetUpCharges(c, table, date, id, 2000)
if err != nil {
return
}
if len(charges) == 0 {
break
}
for k, v := range charges {
m[k] = v
}
}
return
}
// BatchInsertUpCharge batch insert up charge
func (s *Service) BatchInsertUpCharge(c context.Context, table string, us map[int64]*model.UpCharge) (err error) {
var (
buff = make([]*model.UpCharge, _batchSize)
buffEnd = 0
)
for _, u := range us {
buff[buffEnd] = u
buffEnd++
if buffEnd >= _batchSize {
values := upChargeValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.InsertUpCharge(c, table, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := upChargeValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.InsertUpCharge(c, table, values)
}
return
}
func upChargeValues(us []*model.UpCharge) (values string) {
var buf bytes.Buffer
for _, charge := range us {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(charge.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(charge.IncCharge, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(charge.TotalCharge, 10))
buf.WriteByte(',')
buf.WriteString(fmt.Sprintf("'%s'", charge.Date.Time().Format("2006-01-02")))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

397
app/job/main/growup/service/cheat.go Normal file
View File

@@ -0,0 +1,397 @@
package service
import (
"bytes"
"context"
"fmt"
"strconv"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/log"
)
const (
_spyArchiveCoin = 26 //稿件硬币
_spyArchiveFavorite = 27 //稿件收藏
_spyArchivePlay = 28 //稿件播放
_spyUpFans = 29 //异常粉丝数
query = "{\"select\": [{\"name\": \"id\", \"as\": \"id\"}," +
"{\"name\": \"log_date\",\"as\": \"log_date\"},{\"name\": \"target_mid\",\"as\": \"target_mid\"},{\"name\": \"target_id\",\"as\": \"target_id\"}," +
"{\"name\": \"event_id\",\"as\": \"event_id\"},{\"name\": \"state\",\"as\": \"state\"},{\"name\": \"type\",\"as\": \"type\"},{\"name\": \"quantity\",\"as\": \"quantity\"}," +
"{\"name\": \"isdel\",\"as\": \"isdel\"}],\"where\": {\"log_date\": {\"in\": [\"%s\"]}},\"sort\": {\"id\": -1},\"page\": {\"skip\": %d,\"limit\": %d}}"
)
// UpdateCheatHTTP update cheat by http
func (s *Service) UpdateCheatHTTP(c context.Context, date time.Time) (err error) {
err = s.CheatStatistics(c, date)
if err != nil {
log.Error("s.UpdateSpy UpdateSpyData error(%v)", err)
}
return
}
// CheatStatistics task update cheat
func (s *Service) CheatStatistics(c context.Context, date time.Time) (err error) {
spies, err := s.getSpy(c, date)
if err != nil {
return
}
// first filter
cs := cheats(spies)
// aggregation by mid
cm := aggreByMID(cs)
log.Info("agreegation by mid spies:", len(cm))
// aggregation by av_id
am := aggreByAvID(cs)
log.Info("agreegation by av_id spies:", len(cm))
// get up base info
upm, err := s.getUps(c, cm)
if err != nil {
return
}
// get av base info
avm, err := s.getAvs(c, am, time.Now().Add(-30*24*time.Hour))
if err != nil {
return
}
pcs, err := s.playCount(c, cm)
if err != nil {
return
}
deducted, err := s.breachRecord(c)
if err != nil {
return
}
var ups []*model.Cheating
for mid, cheat := range cm {
if up, ok := upm[mid]; ok {
cheat.Nickname = up.Nickname
cheat.Fans = up.Fans
cheat.SignedAt = up.SignedAt
cheat.AccountState = 3
cheat.PlayCount = pcs[mid]
ups = append(ups, cheat)
}
}
var avs []*model.Cheating
for avID, cheat := range am {
if av, ok := avm[avID]; ok {
cheat.UploadTime = av.UploadTime
cheat.TotalIncome = av.TotalIncome
cheat.Nickname = cm[cheat.MID].Nickname
if deducted[avID] {
cheat.Deducted = 1
} else {
cheat.Deducted = 0
}
avs = append(avs, cheat)
}
}
log.Info("signed cheat up count:", len(ups))
err = s.batchInsertCheats(c, ups, s.batchInsertCheatUps)
if err != nil {
log.Error("batchInsertCheatUps error(%v)", err)
return
}
log.Info("signed cheat av count:", len(avs))
err = s.batchInsertCheats(c, avs, s.batchInsertCheatArchives)
if err != nil {
log.Error("batchInsertCheatArchives error(%v)", err)
return
}
return
}
func (s *Service) breachRecord(c context.Context) (deducted map[int64]bool, err error) {
deducted = make(map[int64]bool)
var id int64
for {
var ds map[int64]bool
id, ds, err = s.dao.AvBreachRecord(c, id, 2000)
if err != nil {
return
}
if len(ds) == 0 {
break
}
for k, v := range ds {
deducted[k] = v
}
}
return
}
func (s *Service) getSpy(c context.Context, date time.Time) (spies []*model.Spy, err error) {
from, limit := 0, 500
var info []*model.Spy
for {
dateStr := date.Format("20060102")
info, err = s.dp.SendSpyRequest(c, fmt.Sprintf(query, dateStr, from, limit))
if err != nil {
log.Error("s.getSpyData error(%v)", err)
return
}
if len(info) == 0 {
break
}
spies = append(spies, info...)
from += len(info)
}
log.Info("get spy data total (%d) rows", from)
return
}
func cheats(spies []*model.Spy) (cs []*model.Cheating) {
for _, spy := range spies {
c := &model.Cheating{}
c.MID = spy.TargetMID
switch spy.EventID {
case _spyArchiveCoin:
c.CheatCoin = spy.Quantity
c.AvID = spy.TargetID
case _spyArchiveFavorite:
c.CheatFavorite = spy.Quantity
c.AvID = spy.TargetID
case _spyArchivePlay:
c.CheatPlayCount = spy.Quantity
c.AvID = spy.TargetID
case _spyUpFans:
c.CheatFans = spy.Quantity
}
cs = append(cs, c)
}
return
}
func aggreByMID(source []*model.Cheating) (cheats map[int64]*model.Cheating) {
cheats = make(map[int64]*model.Cheating)
for _, c := range source {
if cheat, ok := cheats[c.MID]; !ok {
cheats[c.MID] = &model.Cheating{
MID: c.MID,
CheatFans: c.CheatFans,
CheatPlayCount: c.CheatPlayCount,
}
} else {
cheat.CheatFans += c.CheatFans
cheat.CheatPlayCount += c.CheatPlayCount
}
}
return
}
func aggreByAvID(source []*model.Cheating) (cheats map[int64]*model.Cheating) {
cheats = make(map[int64]*model.Cheating)
for _, c := range source {
if c.AvID == 0 {
continue
}
if cheat, ok := cheats[c.AvID]; !ok {
cheats[c.AvID] = &model.Cheating{
MID: c.MID,
AvID: c.AvID,
CheatPlayCount: c.CheatPlayCount,
CheatCoin: c.CheatCoin,
CheatFavorite: c.CheatFavorite,
}
} else {
cheat.CheatPlayCount += c.CheatPlayCount
cheat.CheatCoin += c.CheatCoin
cheat.CheatFavorite += c.CheatFavorite
}
}
return
}
// Ups get ups in up_info_video
func (s *Service) getUps(c context.Context, cheats map[int64]*model.Cheating) (ups map[int64]*model.Cheating, err error) {
ups = make(map[int64]*model.Cheating)
var mids []int64
for mid := range cheats {
mids = append(mids, mid)
if len(mids) == 200 {
var nc map[int64]*model.Cheating
nc, err = s.dao.Ups(c, mids)
if err != nil {
return
}
for k, v := range nc {
if v.IsDeleted == 0 {
ups[k] = v
}
}
mids = make([]int64, 0)
time.Sleep(200 * time.Millisecond)
}
}
if len(mids) > 0 {
var nc map[int64]*model.Cheating
nc, err = s.dao.Ups(c, mids)
if err != nil {
return
}
for k, v := range nc {
ups[k] = v
}
}
return
}
// avs result key: av_id, value: cheating with total_income, upload_time
func (s *Service) getAvs(c context.Context, cheats map[int64]*model.Cheating, mtime time.Time) (avs map[int64]*model.Cheating, err error) {
avs = make(map[int64]*model.Cheating)
var avIds []int64
for avID := range cheats {
avIds = append(avIds, avID)
if len(avIds) == 200 {
var nc map[int64]*model.Cheating
nc, err = s.dao.Avs(c, mtime, avIds)
if err != nil {
return
}
for k, v := range nc {
avs[k] = v
}
avIds = make([]int64, 0)
time.Sleep(200 * time.Millisecond)
}
}
if len(avIds) > 0 {
var na map[int64]*model.Cheating
na, err = s.dao.Avs(c, mtime, avIds)
if err != nil {
return
}
for k, v := range na {
avs[k] = v
}
}
return
}
// key: mid, value: play_count
func (s *Service) playCount(c context.Context, cheats map[int64]*model.Cheating) (pcs map[int64]int64, err error) {
var mids []int64
pcs = make(map[int64]int64)
for mid := range cheats {
mids = append(mids, mid)
if len(mids) == 200 {
var pc map[int64]int64
pc, err = s.dao.PlayCount(c, mids)
if err != nil {
return
}
for k, v := range pc {
pcs[k] = v
}
mids = make([]int64, 200)
}
}
if len(mids) > 0 {
var pc map[int64]int64
pc, err = s.dao.PlayCount(c, mids)
if err != nil {
return
}
for k, v := range pc {
pcs[k] = v
}
}
return
}
type insertCheats func(c context.Context, cheats []*model.Cheating) (err error)
func (s *Service) batchInsertCheats(c context.Context, cheats []*model.Cheating, insert insertCheats) (err error) {
var end int
for range cheats {
end++
if end%2000 == 0 {
err = insert(c, cheats[:end])
if err != nil {
return
}
cheats = cheats[end:]
end = 0
}
}
if end > 0 {
err = insert(c, cheats)
}
return
}
func (s *Service) batchInsertCheatUps(c context.Context, cheats []*model.Cheating) (err error) {
var buf bytes.Buffer
for _, cheat := range cheats {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(cheat.MID, 10))
buf.WriteByte(',')
buf.WriteString("\"" + cheat.SignedAt.Time().Format("2006-01-02 15:04:05") + "\"")
buf.WriteByte(',')
buf.WriteString("\"" + cheat.Nickname + "\"")
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.Fans))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.CheatFans))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(cheat.PlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.CheatPlayCount))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.AccountState))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values := buf.String()
buf.Reset()
_, err = s.dao.InsertCheatUps(c, values)
return
}
func (s *Service) batchInsertCheatArchives(c context.Context, cheats []*model.Cheating) (err error) {
var buf bytes.Buffer
for _, cheat := range cheats {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(cheat.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(cheat.MID, 10))
buf.WriteByte(',')
buf.WriteString("\"" + cheat.Nickname + "\"")
buf.WriteByte(',')
buf.WriteString("\"" + cheat.UploadTime.Time().Format("2006-01-02 15:04:05") + "\"")
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.TotalIncome))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.CheatPlayCount))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.CheatFavorite))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.CheatCoin))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(cheat.Deducted))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values := buf.String()
buf.Reset()
_, err = s.dao.InsertCheatArchives(c, values)
return
}

25
app/job/main/growup/service/cheat_test.go Normal file
View File

@@ -0,0 +1,25 @@
package service
import (
"context"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_UpdateCheatHTTP(t *testing.T) {
Convey("growup-job UpdateCheatHTTP", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
err := s.UpdateCheatHTTP(context.Background(), date)
So(err, ShouldBeNil)
}))
}
func Test_CheatStatistics(t *testing.T) {
Convey("growup-job CheatStatistics", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
err := s.CheatStatistics(context.Background(), date)
So(err, ShouldBeNil)
}))
}

76
app/job/main/growup/service/combinemails.go Normal file
View File

@@ -0,0 +1,76 @@
package service
import (
"context"
"fmt"
"time"
"go-common/library/log"
)
const _combine = "%d年%d月%d日统计信息邮件"
// CombineMailsByHTTP send income mail by http.
func (s *Service) CombineMailsByHTTP(c context.Context, year int, month int, day int) (err error) {
t := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local)
err = s.execCombineMails(c, t)
if err != nil {
log.Error("s.CombineMailsByHTTP error(%v)", err)
}
return
}
// CombineMails combine mails.
func (s *Service) CombineMails() (err error) {
c := context.TODO()
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
err = s.execCombineMails(c, t)
if err != nil {
log.Error("s.CombineMails execCombineMails error(%v)", err)
}
return
}
func (s *Service) execCombineMails(c context.Context, date time.Time) (err error) {
var body string
singedUpBody, err := s.execSignedUps(c, date)
if err != nil {
log.Error("s.CombineMails s.execSignedUps error(%v)", err)
return
}
body += singedUpBody
incomeBody, err := s.execIncome(c, date.Add(-24*time.Hour))
if err != nil {
log.Error("s.CombineMails s.execIncome error(%v)", err)
return
}
body += incomeBody
uploadBody, err := s.execSendUpload(c, date.Add(-24*time.Hour))
if err != nil {
log.Error("s.CombineMails s.execSendUpload error(%v)", err)
return
}
body += uploadBody
topTenBody, err := s.execSendTopTen(c, date.Add(-24*time.Hour))
if err != nil {
log.Error("s.CombineMails s.execSendTopTen error(%v)", err)
return
}
body += topTenBody
var send []string
for _, v := range s.conf.Mail.Send {
if v.Type == 2 {
send = v.Addr
}
}
err = s.email.SendMail(date, fmt.Sprintf("<table border='1'>%s</table>", body), _combine, send...)
if err != nil {
log.Error("s.execSendUpload send upload.csv error(%v)", err)
return
}
return
}

27
app/job/main/growup/service/combinemails_test.go Normal file
View File

@@ -0,0 +1,27 @@
package service
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_CombineMailsByHTTP(t *testing.T) {
Convey("growup-job", t, WithService(func(s *Service) {
var (
year = 2018
month = 5
day = 16
)
err := s.CombineMailsByHTTP(context.Background(), year, month, day)
So(err, ShouldBeNil)
}))
}
func Test_CombineMails(t *testing.T) {
Convey("growup-job", t, WithService(func(s *Service) {
err := s.CombineMails()
So(err, ShouldBeNil)
}))
}

448
app/job/main/growup/service/creative_activity.go Normal file
View File

@@ -0,0 +1,448 @@
package service
import (
"bytes"
"context"
"fmt"
"sort"
"strconv"
"strings"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/log"
xtime "go-common/library/time"
"go-common/library/xstr"
)
const (
_like = "1"
_share = "2"
_play = "3"
_reply = "4"
_dm = "5"
)
// CreativeActivity creative activity job
func (s *Service) CreativeActivity(c context.Context, date time.Time) (err error) {
activities, err := s.dao.GetCActivities(c)
if err != nil {
log.Error("s.dao.GetCActivities error(%v)", err)
return
}
// get signed up
ups, err := s.signed(c, int64(_dbLimit))
if err != nil {
log.Error("s.signed error(%v)", err)
return
}
now := xtime.Time(date.Unix())
// calculate activity within the statistical period
for _, ac := range activities {
if now >= ac.StatisticsStart && now <= ac.StatisticsEnd {
log.Info("calculate ac: %d", ac.ID)
err = s.handleActivity(c, ac, ups, date)
if err != nil {
log.Error("s.handleActivity error(%v)", err)
return
}
}
}
return
}
func (s *Service) handleActivity(c context.Context, ac *model.CActivity, upInfo map[int64]*model.UpInfoVideo, date time.Time) (err error) {
// 获取已报名并且在签约时间内的up主，
signedUps, err := s.getSignedUps(c, ac, upInfo)
if err != nil {
log.Error("s.getSignedUps error(%v)", err)
return
}
// 获取活动稿件点赞、分享、播放、评论、弹幕
archiveInfo, err := s.getArchiveInfo(c, ac.ID)
if err != nil {
log.Error("s.getArchiveInfo error(%v)", err)
return
}
if len(archiveInfo) == 0 {
log.Info("activity(%d) get 0 archiveInfo", ac.ID)
return
}
log.Info("get %d archiveInfo", len(archiveInfo))
fmt.Printf("get %d archiveInfo\n", len(archiveInfo))
// 获取在投稿时间内的稿件
avs, err := s.getAvsUpload(c, signedUps)
if err != nil {
log.Error("s.getAvsByMID error(%v)", err)
return
}
if len(avs) == 0 {
log.Info("activity(%d) get 0 av", ac.ID)
return
}
log.Info("get %d archives", len(avs))
fmt.Printf("get %d archives\n", len(avs))
// 筛选在签约时间内的稿件
handleActivityAvs(signedUps, avs, ac)
for _, u := range signedUps {
// 获取up主对应稿件要求的和
err = s.getUpStatisData(c, u, ac, date, archiveInfo)
if err != nil {
log.Error("s.getUpData error(%v)", err)
return
}
}
// 计算中奖up主
upBonus, err := s.calUpWin(c, signedUps, ac)
if err != nil {
log.Error("s.calUpBonus error(%v)", err)
return
}
err = s.updateUpActivity(c, ac.ID, upBonus)
if err != nil {
log.Error("s.updateUpActivity error(%v)", err)
}
return
}
func (s *Service) calUpWin(c context.Context, ups map[int64]*model.UpActivity, ac *model.CActivity) (bonusUp []*model.UpActivity, err error) {
activityBonus, err := s.dao.GetActivityBonus(c, ac.ID)
if err != nil {
log.Error("s.dao.GetActivityBonus error(%v)", err)
return
}
bonusUp = make([]*model.UpActivity, 0)
var (
upList = make([]*model.UpActivity, len(ups))
index int
)
for _, up := range ups {
upList[index] = up
index++
}
sort.Slice(upList, func(i, j int) bool {
return upList[i].ItemVal > upList[j].ItemVal
})
// 1:达标型 2:排序型
if ac.WinType == 1 {
for _, up := range upList {
// uplist经过ItemVal,所以如果有ItemVal<RequireValue,可以直接break
if up.ItemVal < ac.RequireValue {
break
}
bonusUp = append(bonusUp, up)
}
} else if ac.WinType == 2 {
// 去掉item为0的up
for i := 0; i < len(upList); i++ {
if upList[i].ItemVal == 0 {
upList = upList[:i]
break
}
}
if len(upList) > int(ac.RequireValue) {
upList = upList[:int(ac.RequireValue)]
}
bonusUp = upList
}
if len(bonusUp) == 0 {
return
}
// 计算up主奖金
calUpBonus(bonusUp, activityBonus, ac.BonusType, ac.WinType)
return
}
// cal ups bonus
func calUpBonus(bonusUp []*model.UpActivity, activityBonus map[int64]int64, bonusType, winType int) (err error) {
// 中奖类型 1:达标型 2:排序型
// 奖金类型 1:平分 2:各得
if winType == 1 {
money, ok := activityBonus[0]
if !ok {
err = fmt.Errorf("活动奖金设置错误:达标型未设置金额")
return
}
if bonusType == 1 {
money = money / int64(len(bonusUp))
}
for _, up := range bonusUp {
up.Bonus = money
up.Rank = 0
if up.State < 2 {
// 中奖
up.State = 2
up.SuccessTime = xtime.Time(time.Now().Unix())
}
}
} else if winType == 2 {
other := len(activityBonus)
otherMoney, ok := activityBonus[int64(other)]
if !ok {
err = fmt.Errorf("活动奖金设置错误:排序型没有其他金额")
return
}
for i := 0; i < len(bonusUp); i++ {
var (
rank = i + 1
money int64
ok bool
)
if rank >= other {
money = otherMoney
} else {
money, ok = activityBonus[int64(rank)]
if !ok {
err = fmt.Errorf("活动奖金设置错误:没有名次金额")
return
}
}
if money == 0 {
continue
}
bonusUp[i].Bonus = money
bonusUp[i].Rank = i + 1
if bonusUp[i].State < 2 {
// 中奖
bonusUp[i].State = 2
bonusUp[i].SuccessTime = xtime.Time(time.Now().Unix())
}
}
}
return
}
func (s *Service) getUpStatisData(c context.Context, up *model.UpActivity, ac *model.CActivity, date time.Time, archiveInfo map[int64]map[int]*model.ArchiveStat) (err error) {
if len(up.AIDs) == 0 {
return
}
avItem := make([]*model.AvItem, 0)
aIDs := up.AIDs
for _, avID := range aIDs {
// 稿件统计数据: 计算当天的和 - 统计开始前一天的和
var itemSumStart, itemSumEnd int64
itemSumEnd, err = s.getAvStatisState(c, avID, ac, archiveInfo, 2)
if err != nil {
log.Error("s.getAvStatisState error(%v)", err)
return
}
itemSumStart, err = s.getAvStatisState(c, avID, ac, archiveInfo, 1)
if err != nil {
log.Error("s.getAvStatisState error(%v)", err)
return
}
itemSum := itemSumEnd - itemSumStart
if itemSum > 0 {
avItem = append(avItem, &model.AvItem{
AvID: avID,
Value: itemSum,
})
}
}
if len(avItem) == 0 {
return
}
sort.Slice(avItem, func(i, j int) bool {
return avItem[i].Value > avItem[j].Value
})
// 1:uid 2 avid
if ac.Object == 1 {
up.AIDs = make([]int64, 0)
for _, av := range avItem {
if len(up.AIDs) < 5 {
up.AIDs = append(up.AIDs, av.AvID)
}
up.ItemVal += av.Value
}
up.AIDNum = int64(len(avItem))
} else if ac.Object == 2 {
up.AIDs = []int64{avItem[0].AvID}
up.ItemVal = avItem[0].Value
up.AIDNum = 1
}
return
}
func (s *Service) getAvStatisState(c context.Context, avID int64, ac *model.CActivity, archiveInfo map[int64]map[int]*model.ArchiveStat, state int) (sum int64, err error) {
if _, ok := archiveInfo[avID]; !ok {
return
}
stat, ok := archiveInfo[avID][state]
if !ok {
return
}
requireItems := strings.Split(ac.RequireItems, ",")
for _, item := range requireItems {
switch item {
case _like:
sum += stat.Like
case _share:
sum += stat.Share
case _play:
sum += stat.Play
case _reply:
sum += stat.Reply
case _dm:
sum += stat.Dm
}
}
return
}
func handleActivityAvs(ups map[int64]*model.UpActivity, avs []*model.AvUpload, ac *model.CActivity) {
for _, av := range avs {
if !(av.UploadTime >= ac.UploadStart && av.UploadTime <= ac.UploadEnd) {
continue
}
if _, ok := ups[av.MID]; ok {
if len(ups[av.MID].AIDs) == 0 {
ups[av.MID].AIDs = make([]int64, 0)
}
ups[av.MID].AIDs = append(ups[av.MID].AIDs, av.AvID)
}
}
}
func (s *Service) getAvsUpload(c context.Context, ups map[int64]*model.UpActivity) (avs []*model.AvUpload, err error) {
avs = make([]*model.AvUpload, 0)
var id int64
for {
var av []*model.AvUpload
av, err = s.dao.GetAvUploadByMID(c, id, _dbLimit)
if err != nil {
return
}
avs = append(avs, av...)
if len(av) < _dbLimit {
break
}
id = av[len(av)-1].ID
}
return
}
func (s *Service) getArchiveInfo(c context.Context, activityID int64) (archives map[int64]map[int]*model.ArchiveStat, err error) {
archives = make(map[int64]map[int]*model.ArchiveStat) // map[av_id][state]*model.ArchiveStat
var id int64
for {
var arch []*model.ArchiveStat
arch, err = s.dao.GetArchiveInfo(c, activityID, id, _dbLimit)
if err != nil {
return
}
for _, a := range arch {
if _, ok := archives[a.AvID]; !ok {
archives[a.AvID] = make(map[int]*model.ArchiveStat)
}
archives[a.AvID][a.State] = a
}
if len(arch) < _dbLimit {
break
}
id = arch[len(arch)-1].ID
}
return
}
func (s *Service) getSignedUps(c context.Context, ac *model.CActivity, upInfo map[int64]*model.UpInfoVideo) (signedUps map[int64]*model.UpActivity, err error) {
signedUps = make(map[int64]*model.UpActivity)
// if need sign up
if ac.SignUp == 1 {
var ups []*model.UpActivity
ups, err = s.dao.ListUpActivity(c, ac.ID)
if err != nil {
return
}
for _, up := range ups {
if info, ok := upInfo[up.MID]; ok && info.SignedAt >= ac.SignedStart && info.SignedAt <= ac.SignedEnd {
signedUps[up.MID] = up
}
}
} else {
for _, info := range upInfo {
if info.SignedAt >= ac.SignedStart && info.SignedAt <= ac.SignedEnd {
signedUps[info.MID] = &model.UpActivity{MID: info.MID, ActivityID: ac.ID, State: 1, Nickname: info.Nickname}
}
}
}
return
}
func (s *Service) updateUpActivity(c context.Context, id int64, upBonus []*model.UpActivity) (err error) {
// 更新所有之前获奖up主为已报名
// update state 2->1 将所有状态设置为已报名
_, err = s.dao.UpdateUpActivityState(c, id, 2, 1)
if err != nil {
log.Error("s.dao.UpdateUpActivityState error(%v)", err)
return
}
if len(upBonus) > 0 {
_, err = s.insertUpActivityBatch(c, upBonus)
if err != nil {
log.Error("s.insertUpActivity error(%v)", err)
}
}
return
}
func (s *Service) insertUpActivityBatch(c context.Context, ups []*model.UpActivity) (rows int64, err error) {
insert := make([]*model.UpActivity, _dbBatchSize)
insertIndex := 0
for _, up := range ups {
insert[insertIndex] = up
insertIndex++
if insertIndex >= _dbBatchSize {
_, err = s.insertUpActivity(c, insert[:insertIndex])
if err != nil {
return
}
insertIndex = 0
}
}
if insertIndex > 0 {
_, err = s.insertUpActivity(c, insert[:insertIndex])
}
return
}
func assembleUpActivity(ups []*model.UpActivity) (vals string) {
var buf bytes.Buffer
for _, row := range ups {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString("\"" + row.Nickname + "\"")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.ActivityID, 10))
buf.WriteByte(',')
buf.WriteString("'" + xstr.JoinInts(row.AIDs) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.AIDNum, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(row.Rank))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Bonus, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(row.State))
buf.WriteByte(',')
buf.WriteString("'" + row.SuccessTime.Time().Format("2006-01-02 15:04:05") + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *Service) insertUpActivity(c context.Context, ups []*model.UpActivity) (rows int64, err error) {
vals := assembleUpActivity(ups)
rows, err = s.dao.InsertUpActivityBatch(c, vals)
return
}

160
app/job/main/growup/service/creative_budget.go Normal file
View File

@@ -0,0 +1,160 @@
package service
import (
"context"
"fmt"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/log"
)
// CreativeBudget creative budget
func (s *Service) CreativeBudget(c context.Context, date time.Time) (err error) {
defer func() {
GetTaskService().SetTaskStatus(c, TaskBudget, date.Format("2006-01-02"), err)
}()
err = GetTaskService().TaskReady(c, date.Format("2006-01-02"), TaskCreativeIncome)
if err != nil {
return
}
ups, err := s.GetUpIncome(c, "up_income", date.Format(_layout))
if err != nil {
log.Error("s.GetUpIncome(up_income) error(%v)", err)
return
}
if len(ups) == 0 {
err = fmt.Errorf("get 0 record from up_income")
return
}
log.Info("daily ups(%d)", len(ups))
monthStart := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
monthUps, err := s.GetUpIncome(c, "up_income_monthly", monthStart.Format(_layout))
if err != nil {
log.Error("s.GetUpIncome(up_income_monthly) error(%v)", err)
return
}
if len(monthUps) == 0 {
err = fmt.Errorf("get 0 record from up_income_monthly")
return
}
log.Info("monthly ups(%d)", len(ups))
preTotalExpense, err := s.dao.GetTotalExpenseByDate(c, date.AddDate(0, 0, -1).Format(_layout))
if err != nil {
log.Error("s.dao.GetTotalExpenseByDate error(%v)", err)
return
}
log.Info("CreativeBudget ready date ok(%s)", date.Format(_layout))
avDaily, cmDaily, bgmDaily := calCreativebudget(ups, date, preTotalExpense)
avMonthly, cmMonthly, bgmMonthly := calCreativebudget(monthUps, date, preTotalExpense)
avMonthly.TotalExpense = avDaily.TotalExpense
cmMonthly.TotalExpense = cmDaily.TotalExpense
bgmMonthly.TotalExpense = bgmDaily.TotalExpense
// insert
if avDaily.Expense > 0 {
_, err = s.dao.InsertDailyExpense(c, avDaily)
if err != nil {
log.Error("s.dao.InsertDailyExpense error(%v)", err)
return
}
_, err = s.dao.InsertMonthlyExpense(c, avMonthly)
if err != nil {
log.Error("s.dao.InsertMonthlyExpense error(%v)", err)
return
}
}
if cmDaily.Expense > 0 {
_, err = s.dao.InsertDailyExpense(c, cmDaily)
if err != nil {
log.Error("s.dao.InsertDailyExpense error(%v)", err)
return
}
_, err = s.dao.InsertMonthlyExpense(c, cmMonthly)
if err != nil {
log.Error("s.dao.InsertMonthlyExpense error(%v)", err)
return
}
}
if bgmDaily.Expense > 0 {
_, err = s.dao.InsertDailyExpense(c, bgmDaily)
if err != nil {
log.Error("s.dao.InsertDailyExpense error(%v)", err)
return
}
_, err = s.dao.InsertMonthlyExpense(c, bgmMonthly)
if err != nil {
log.Error("s.dao.InsertMonthlyExpense error(%v)", err)
}
}
return
}
func calCreativebudget(ups []*model.UpIncome, date time.Time, preTotalExpense map[int64]int64) (avBudget, cmBudget, bgmBudget *model.BudgetExpense) {
avBudget, cmBudget, bgmBudget = &model.BudgetExpense{}, &model.BudgetExpense{}, &model.BudgetExpense{}
var (
avExpense, cmExpense, bgmExpense int64
avCount, cmCount, bgmCount int64
upAvCount, upCmCount, upBgmCount int64
)
for _, up := range ups {
if up.AvIncome > 0 {
avCount += up.AvCount
upAvCount++
}
// add up fix adjust to av
avExpense += up.Income - up.ColumnIncome - up.BgmIncome // TODO up.AvIncome
if up.ColumnIncome > 0 {
cmExpense += up.ColumnIncome
cmCount += up.ColumnCount
upCmCount++
}
if up.BgmIncome > 0 {
bgmExpense += up.BgmIncome
bgmCount += up.BgmCount
upBgmCount++
}
}
if avCount > 0 && upAvCount > 0 {
avBudget = &model.BudgetExpense{
Expense: avExpense,
AvCount: avCount,
UpCount: upAvCount,
UpAvgExpense: avExpense / upAvCount,
AvAvgExpense: avExpense / avCount,
Date: date,
TotalExpense: preTotalExpense[0] + avExpense,
CType: 0,
}
}
if cmCount > 0 && upCmCount > 0 {
cmBudget = &model.BudgetExpense{
Expense: cmExpense,
AvCount: cmCount,
UpCount: upCmCount,
UpAvgExpense: cmExpense / upCmCount,
AvAvgExpense: cmExpense / cmCount,
Date: date,
TotalExpense: preTotalExpense[2] + cmExpense,
CType: 2,
}
}
if bgmCount > 0 && upBgmCount > 0 {
bgmBudget = &model.BudgetExpense{
Expense: bgmExpense,
AvCount: bgmCount,
UpCount: upBgmCount,
UpAvgExpense: bgmExpense / upBgmCount,
AvAvgExpense: bgmExpense / bgmCount,
Date: date,
TotalExpense: preTotalExpense[3] + bgmExpense,
CType: 3,
}
}
return
}

29
app/job/main/growup/service/ctrl/BUILD Normal file
View File

@@ -0,0 +1,29 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["executor.go"],
importpath = "go-common/app/job/main/growup/service/ctrl",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["//library/log:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

40
app/job/main/growup/service/ctrl/executor.go Normal file
View File

@@ -0,0 +1,40 @@
package ctrl
import (
"context"
"reflect"
"runtime"
"sync"
"go-common/library/log"
)
func NewUnboundedExecutor() *UnboundedExecutor {
ctx := context.Background()
return &UnboundedExecutor{
ctx: ctx,
}
}
type UnboundedExecutor struct {
wg sync.WaitGroup
// for future extension
ctx context.Context
}
type Executor interface {
Submit(bizFunc ...func(ctx context.Context))
}
func (executor *UnboundedExecutor) Submit(bizFunc ...func(c context.Context)) {
for _, biz := range bizFunc {
pc := reflect.ValueOf(biz).Pointer()
funcName := runtime.FuncForPC(pc).Name()
executor.wg.Add(1)
go func(funcName string, biz func(ctx context.Context)) {
defer executor.wg.Done()
log.Info("Exec Task %s", funcName)
biz(executor.ctx)
}(funcName, biz)
}
}

887
app/job/main/growup/service/data.go Normal file
View File

@@ -0,0 +1,887 @@
package service
import (
"bytes"
"context"
"fmt"
"strconv"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_updateUpAccount = "UPDATE up_account SET withdraw_date_version = '%s', total_unwithdraw_income=total_income - total_withdraw_income - exchange_income where withdraw_date_version = '%s' AND is_deleted = 0"
_updateTagAdjust = "UPDATE tag_info SET adjust_type = 1 WHERE id = %d AND adjust_type = 0"
// fix data
_txUpdateAvIncome = "UPDATE av_income SET total_income = total_income %s, income = income %s WHERE av_id = %d AND mid = %d AND date = '%s'"
_txUpdateAvIncomeStatis = "UPDATE av_income_statis SET total_income = total_income %s WHERE av_id = %d AND mid = %d"
_txUpdateAvIncomeStatisIncome = "UPDATE av_income_%s_statis SET income = income %s WHERE category_id = %d AND cdate = '%s'"
_txUpAccount = "UPDATE up_account SET total_income = total_income %s, total_unwithdraw_income = total_unwithdraw_income %s WHERE mid = %d AND is_deleted = 0"
_txUpIncomeTable = "UPDATE %s SET av_income = av_income %s, total_income = total_income %s, income = income %s WHERE mid = %d AND date = '%s'"
_txUpIncomeStatis = "UPDATE up_income_statis SET total_income = total_income %s WHERE mid = %d"
_txUpIncomeDailyStatis = "UPDATE up_income_daily_statis set income = income %s WHERE cdate = '%s'"
_txTagInsertUpIncomeDate = "INSERT INTO %s(mid,income,total_income,date) VALUES(%d,%d,%d,'%s') ON DUPLICATE KEY UPDATE total_income=VALUES(total_income),income=VALUES(income)"
_txTagInsertUpIncomeStatis = "INSERT INTO up_income_statis(mid, total_income) VALUES(%d,%d) ON DUPLICATE KEY UPDATE total_income=VALUES(total_income)"
_txTagInsertUpAccount = "INSERT INTO up_account(mid,has_sign_contract,state,total_income,total_unwithdraw_income,withdraw_date_version) VALUES(%d,1,1,%d,%d,'%s') ON DUPLICATE KEY UPDATE total_income=VALUES(total_income),total_unwithdraw_income=VALUES(total_unwithdraw_income)"
_txTagUpIncomeDailyStatis = "UPDATE up_income_daily_statis SET ups = ups + %d WHERE cdate = '%s' AND money_section = %d"
_txUpAvStatis = "INSERT INTO up_av_statis(mid,weekly_date,weekly_av_ids,monthly_date,monthly_av_ids) VALUES(%d,'2018-05-28','%s', '2018-05-01', '%s') ON DUPLICATE KEY UPDATE mid=VALUES(mid)"
_txAddUpIncomeStatis = "UPDATE up_income_statis SET total_income = total_income + %d WHERE mid = %d"
_txAddAvIncomeStatis = "UPDATE av_income_statis SET total_income = total_income + %d WHERE av_id = %d"
_txAddUpIncome = "UPDATE %s SET total_income = total_income + %d, income = income + %d, av_income = av_income + %d WHERE mid = %d AND date = '%s'"
_txAddAvIncome = "UPDATE av_income SET total_income = total_income + %d, income = income + %d WHERE av_id = %d AND date = '%s'"
_txAddUpAccount = "UPDATE up_account SET total_income = total_income + %d, total_unwithdraw_income = total_unwithdraw_income + %d WHERE mid = %d"
_txUpdateAccountType = "UPDATE up_info_video SET account_type=%d WHERE mid=%d"
_txUpdateUpAccountMoney = "UPDATE up_account SET total_income = total_income + %d, total_unwithdraw_income = total_unwithdraw_income + %d WHERE mid = %d LIMIT 1"
_txUpdateUpBaseIncome = "UPDATE up_income SET base_income=%d WHERE mid=%d AND date='%s'"
_txInUpInfoPGCSQL = "INSERT INTO up_info_bgm(mid,nickname,fans,account_type,account_state,sign_type) values(%d,%s,%d,%d,%d,%d) ON DUPLICATE KEY UPDATE account_type=VALUES(account_type)"
_txDelAvBreachSQL = "DELETE FROM av_breach_record WHERE id = %d LIMIT 1"
_txUpTotalIncomeSQL = "UPDATE %s SET total_income = total_income - %d WHERE mid = %d AND date = '%s' LIMIT 1"
_txColumnTagSQL = "UPDATE %s SET tag_id = %d WHERE date = '2018-08-19' AND tag_id in (%s) AND inc_charge > 0"
_txUpdateBgmBaseIncome = "UPDATE up_income SET bgm_base_income=bgm_income WHERE mid=%d AND date='%s' AND bgm_base_income=0"
_txDelData = "DELETE FROM %s LIMIT %d"
)
// DelDataLimit del up_bill
func (s *Service) DelDataLimit(c context.Context, table string, count int64) (err error) {
if table == "" {
return
}
return s.txUpdateSQL(c, fmt.Sprintf(_txDelData, table, count), count)
}
// FixBgmBaseIncome fix bgm base income
func (s *Service) FixBgmBaseIncome(c context.Context, mid int64, date string) (err error) {
sql := fmt.Sprintf(_txUpdateBgmBaseIncome, mid, date)
return s.txUpdateSQL(c, sql, 1)
}
// FixBaseIncome fix income
func (s *Service) FixBaseIncome(c context.Context, base int64, mid int64, date string) (err error) {
sql := fmt.Sprintf(_txUpdateUpBaseIncome, base, mid, date)
return s.txUpdateSQL(c, sql, 1)
}
// FixIncome fix income
func (s *Service) FixIncome(c context.Context) (err error) {
date := time.Date(2018, 9, 10, 0, 0, 0, 0, time.Local)
total, err := s.getAvIncome(c, date)
if err != nil {
log.Error("s.getAvIncome error(%v)", err)
return
}
avIncomes := make([]*model.IncomeInfo, 0)
for _, av := range total {
if av.UploadTime.Unix() >= date.Unix() {
avIncomes = append(avIncomes, av)
}
}
if len(avIncomes) != 1768 {
err = fmt.Errorf("get av_income(%d) != 1768", len(avIncomes))
return
}
var tx *sql.Tx
tx, err = s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
for _, av := range avIncomes {
// av_income_statis
err = s.updateSQL(tx, fmt.Sprintf(_txAddAvIncomeStatis, av.Income, av.AVID), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txAddAvIncomeStatis, err)
return
}
// up_income_statis
err = s.updateSQL(tx, fmt.Sprintf(_txAddUpIncomeStatis, av.Income, av.MID), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txAddUpIncomeStatis, err)
return
}
// av_income
err = s.updateSQL(tx, fmt.Sprintf(_txAddAvIncome, av.Income, av.Income, av.AVID, "2018-09-10"), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txAddAvIncome, err)
return
}
// up_income
err = s.updateSQL(tx, fmt.Sprintf(_txAddUpIncome, "up_income", av.Income, av.Income, av.Income, av.MID, "2018-09-10"), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txAddUpIncome, err)
return
}
// up_income_weekly
err = s.updateSQL(tx, fmt.Sprintf(_txAddUpIncome, "up_income_weekly", av.Income, av.Income, av.Income, av.MID, "2018-09-10"), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txAddUpIncome, err)
return
}
// up_income_monthly
err = s.updateSQL(tx, fmt.Sprintf(_txAddUpIncome, "up_income_monthly", av.Income, av.Income, av.Income, av.MID, "2018-09-01"), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txAddUpIncome, err)
return
}
// up_account
err = s.updateSQL(tx, fmt.Sprintf(_txAddUpAccount, av.Income, av.Income, av.MID), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txAddUpAccount, err)
return
}
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
// FixUpAvStatis fix up_av_statis
func (s *Service) FixUpAvStatis(c context.Context, count int) (err error) {
upIncome, err := s.GetUpIncome(c, "up_income", "2018-05-31")
if err != nil {
log.Error("s.GetUpIncome error(%v)", err)
return
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
// 301
addCount := 0
for _, up := range upIncome {
if up.TotalIncome == 12700 && up.Income == 12700 {
err = s.updateSQL(tx, fmt.Sprintf(_txUpAvStatis, up.MID, "", ""), 0)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
addCount++
}
}
if count != addCount {
err = fmt.Errorf("需要添加的record不匹配 %d:%d", count, addCount)
tx.Rollback()
return
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
// FixUpIncome fix up_income
func (s *Service) FixUpIncome(c context.Context, date string, tagID int64, addCount, needAddIncome int) (err error) {
upIncome, err := s.GetUpIncome(c, "up_income", date)
if err != nil {
log.Error("s.GetUpIncome error(%v)", err)
return
}
upChargeRatio, err := s.dao.GetUpChargeRatio(c, tagID)
if err != nil {
log.Error("s.dao.GetUpChargeRatio error(%v)", err)
return
}
for _, up := range upIncome {
if _, ok := upChargeRatio[up.MID]; ok {
delete(upChargeRatio, up.MID)
}
}
if len(upChargeRatio) != addCount {
err = fmt.Errorf("需要调节的up主数量不匹配 %d:%d", len(upChargeRatio), addCount)
return
}
mids := make([]int64, 0)
for mid := range upChargeRatio {
mids = append(mids, mid)
}
upIncomeStatis, err := s.dao.GetUpIncomeStatis(c, mids)
if err != nil {
log.Error("s.dao.GetUpIncomeStatis error(%v)", err)
return
}
upIncomeWeek, err := s.dao.GetUpIncomeDate(c, mids, "up_income_weekly", "2018-05-28")
if err != nil {
log.Error("s.dao.GetUpIncomeDate error(%v)", err)
return
}
upIncomeMonth, err := s.dao.GetUpIncomeDate(c, mids, "up_income_monthly", "2018-05-01")
if err != nil {
log.Error("s.dao.GetUpIncomeDate error(%v)", err)
return
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
// start add
var totalAddIncome int64
for mid, ratio := range upChargeRatio {
totalIncome := upIncomeStatis[mid]
weekIncome := upIncomeWeek[mid]
monthIncome := upIncomeMonth[mid]
// up_income
err = s.insertIntoSQL(tx, fmt.Sprintf(_txTagInsertUpIncomeDate, "up_income", mid, ratio, totalIncome+ratio, date), 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income_weekly todo
weekDate := "2018-05-28"
err = s.insertIntoSQL(tx, fmt.Sprintf(_txTagInsertUpIncomeDate, "up_income_weekly", mid, ratio+weekIncome, totalIncome+ratio, weekDate), 0)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income_monthly todo
monthDate := "2018-05-01"
err = s.insertIntoSQL(tx, fmt.Sprintf(_txTagInsertUpIncomeDate, "up_income_monthly", mid, ratio+monthIncome, totalIncome+ratio, monthDate), 0)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income_statis
err = s.insertIntoSQL(tx, fmt.Sprintf(_txTagInsertUpIncomeStatis, mid, ratio+totalIncome), 0)
if err != nil {
log.Error("s.insertIntoSQL error(%v)", err)
return
}
// up_account
err = s.insertIntoSQL(tx, fmt.Sprintf(_txTagInsertUpAccount, mid, ratio+totalIncome, ratio+totalIncome, "2018-04"), 0)
if err != nil {
log.Error("s.insertIntoSQL error(%v)", err)
return
}
totalAddIncome += ratio
}
// up_income_daily_statis
if totalAddIncome != int64(needAddIncome) {
err = fmt.Errorf("需要调节的up主总收入不匹配 %d:%d", totalAddIncome, needAddIncome)
tx.Rollback()
return
}
add := fmt.Sprintf(" + %d", totalAddIncome)
err = s.updateSQL(tx, fmt.Sprintf(_txUpIncomeDailyStatis, add, date), 12)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
err = s.updateSQL(tx, fmt.Sprintf(_txTagUpIncomeDailyStatis, len(upChargeRatio), date, 3), 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
func (s *Service) getAllAvRatio(c context.Context, limit int64) (rs map[int64]*model.AvChargeRatio, err error) {
rs = make(map[int64]*model.AvChargeRatio)
var id int64
for {
var ros map[int64]*model.AvChargeRatio
ros, id, err = s.dao.AvChargeRatio(c, id, limit)
if err != nil {
return
}
if len(ros) == 0 {
break
}
for k, v := range ros {
rs[k] = v
}
}
return
}
// UpdateTagIncome update tag_Info income
func (s *Service) UpdateTagIncome(c context.Context, date string) (err error) {
avRatio, err := s.getAllAvRatio(c, 2000)
if err != nil {
log.Error("s.getAllAvRatio error(%v)", err)
return
}
log.Info("get avratios:%d", len(avRatio))
updateDate := time.Date(2018, 6, 24, 0, 0, 0, 0, time.Local)
avIncome, err := s.getAvIncome(c, updateDate)
if err != nil {
log.Error("s.getAvIncome error(%v)", err)
return
}
log.Info("get av_income:%d ", len(avIncome))
err = s.updateIncome(c, avIncome, avRatio)
if err != nil {
log.Error("s.updateIncome error(%v)", err)
return
}
return
}
// GetTrueAvsIncome get true av_income
func (s *Service) GetTrueAvsIncome(c context.Context, mids []int64, date string) (avs map[int64]*model.Patch, err error) {
avs = make(map[int64]*model.Patch)
for _, mid := range mids {
var av map[int64]*model.Patch
av, err = s.AvIncomes(c, mid, date)
if err != nil {
return
}
for key, val := range av {
avs[key] = val
}
}
return
}
func (s *Service) updateIncome(c context.Context, avIncome []*model.IncomeInfo, avRatio map[int64]*model.AvChargeRatio) (err error) {
trueAvs := make([]*model.IncomeInfo, 0)
uploadTime := time.Date(2018, 6, 24, 0, 0, 0, 0, time.Local)
for _, av := range avIncome {
if !uploadTime.After(av.UploadTime) {
if _, ok := avRatio[av.AVID]; !ok {
trueAvs = append(trueAvs, av)
}
}
}
if len(trueAvs) != 1856 {
err = fmt.Errorf("实际被作用稿件(%d) != 1856", len(trueAvs))
return
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
for _, av := range trueAvs {
var incIncome, categoryID int64
incIncome, categoryID, err = s.dao.AvDailyIncCharge(c, av.AVID)
if err != nil {
log.Error("s.dao.AvDailyIncCharge avid(%d) error(%v)", av.MID, err)
return
}
err = s.TxUpdateIncome(tx, av.AVID, av.MID, categoryID, incIncome)
if err != nil {
log.Error("ERROR(%v) avid(%d), mid(%d)", err, av.AVID, av.MID)
return
}
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
// TxUpdateIncome update creative income
func (s *Service) TxUpdateIncome(tx *sql.Tx, avID, mid, categoryID int64, incIncome int64) (err error) {
if incIncome <= 0 {
return
}
var incIncomeStr string
if incIncome > 0 {
incIncomeStr = fmt.Sprintf("+ %d", incIncome)
}
date := "2018-06-24"
// av_income
avIncomeSQL := fmt.Sprintf(_txUpdateAvIncome, incIncomeStr, incIncomeStr, avID, mid, date)
err = s.updateSQL(tx, avIncomeSQL, 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// av_income_statis
avIncomeStatisSQL := fmt.Sprintf(_txUpdateAvIncomeStatis, incIncomeStr, avID, mid)
err = s.updateSQL(tx, avIncomeStatisSQL, 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// av_income_daily_statis
avIncomeDailyStatisIncome := fmt.Sprintf(_txUpdateAvIncomeStatisIncome, "daily", incIncomeStr, categoryID, date)
err = s.updateSQL(tx, avIncomeDailyStatisIncome, 12)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// av_income_weekly_statis
avIncomeWeeklyStatisIncome := fmt.Sprintf(_txUpdateAvIncomeStatisIncome, "weekly", incIncomeStr, categoryID, "2018-06-18")
err = s.updateSQL(tx, avIncomeWeeklyStatisIncome, 12)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// av_income_monthly_statis
avIncomeMonthlyStatisIncome := fmt.Sprintf(_txUpdateAvIncomeStatisIncome, "monthly", incIncomeStr, categoryID, "2018-06-01")
err = s.updateSQL(tx, avIncomeMonthlyStatisIncome, 12)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_account
upAcccountSQL := fmt.Sprintf(_txUpAccount, incIncomeStr, incIncomeStr, mid)
err = s.updateSQL(tx, upAcccountSQL, 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income_statis
upIncomeStatisSQL := fmt.Sprintf(_txUpIncomeStatis, incIncomeStr, mid)
err = s.updateSQL(tx, upIncomeStatisSQL, 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income_daily_statis
upIncomeDailyStatisSQL := fmt.Sprintf(_txUpIncomeDailyStatis, incIncomeStr, date)
err = s.updateSQL(tx, upIncomeDailyStatisSQL, 12)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income
upIncomeSQL := fmt.Sprintf(_txUpIncomeTable, "up_income", incIncomeStr, incIncomeStr, incIncomeStr, mid, date)
err = s.updateSQL(tx, upIncomeSQL, 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income_weekly
upIncomeWeeklySQL := fmt.Sprintf(_txUpIncomeTable, "up_income_weekly", incIncomeStr, incIncomeStr, incIncomeStr, mid, "2018-06-18")
err = s.updateSQL(tx, upIncomeWeeklySQL, 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_income_monthly
upIncomeMonthlySQL := fmt.Sprintf(_txUpIncomeTable, "up_income_monthly", incIncomeStr, incIncomeStr, incIncomeStr, mid, "2018-06-01")
err = s.updateSQL(tx, upIncomeMonthlySQL, 1)
if err != nil {
log.Error("s.UpdateSQL error(%v)", err)
return
}
// up_av_statis 不需要修改
// up_income_withdraw 不需要修改
return
}
// UpdateWithdraw update up_account withdraw
func (s *Service) UpdateWithdraw(c context.Context, oldDate, newDate string, count int64) (err error) {
sql := fmt.Sprintf(_updateUpAccount, newDate, oldDate)
return s.txUpdateSQL(c, sql, count)
}
// UpdateTagAdjust update tag adjust_type
func (s *Service) UpdateTagAdjust(c context.Context, id int64) (err error) {
sql := fmt.Sprintf(_updateTagAdjust, id)
return s.txUpdateSQL(c, sql, 1)
}
// UpdateAccountType update account type
func (s *Service) UpdateAccountType(c context.Context, mid int64, accType int) (err error) {
sql := fmt.Sprintf(_txUpdateAccountType, accType, mid)
return s.txUpdateSQL(c, sql, 1)
}
// UpdateUpAccountMoney update up_account
func (s *Service) UpdateUpAccountMoney(c context.Context, mid int64, total, unwithdraw int64) (err error) {
sql := fmt.Sprintf(_txUpdateUpAccountMoney, total, unwithdraw, mid)
return s.txUpdateSQL(c, sql, 1)
}
// SyncUpPGC sync pgc up from up_info_video to up_info_column
func (s *Service) SyncUpPGC(c context.Context) (err error) {
ups, err := s.getAllUps(c, 2000)
if err != nil {
log.Error("s.getAllUps error(%v)", err)
return
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
for _, up := range ups {
if up.AccountType == 2 {
sql := fmt.Sprintf(_txInUpInfoPGCSQL, up.MID, "\""+up.Nickname+"\"", up.Fans, 2, 1, 2)
err = s.insertIntoSQL(tx, sql, 0)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txInUpInfoPGCSQL, err)
return
}
}
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
// FixAvBreach fix av_breach_record data
func (s *Service) FixAvBreach(c context.Context, mid int64, date string, count int) (err error) {
breachs, err := s.dao.GetAvBreach(c, date, date)
if err != nil {
log.Error("s.dao.GetAvBreach error(%v)", err)
return
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
avMap := make(map[int64]bool)
var delCount int
for _, b := range breachs {
if b.MID != mid {
continue
}
if avMap[b.AvID] {
err = s.updateSQL(tx, fmt.Sprintf(_txDelAvBreachSQL, b.ID), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txDelAvBreachSQL, err)
return
}
delCount++
} else {
avMap[b.AvID] = true
}
}
if count != delCount {
tx.Rollback()
log.Error("delete count error %d %d", count, delCount)
err = fmt.Errorf("delete count error")
return
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
// FixUpTotalIncome fix up_income total income
func (s *Service) FixUpTotalIncome(c context.Context, table, date string, count int) (err error) {
upIncome, err := s.GetUpIncome(c, "up_income", date)
if err != nil {
log.Error("s.GetUpIncome error(%v)", err)
return
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
trueCount := 0
for _, up := range upIncome {
err = s.updateSQL(tx, fmt.Sprintf(_txUpTotalIncomeSQL, table, up.Income, up.MID, date), 1)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", _txUpTotalIncomeSQL, err)
return
}
trueCount++
}
if count != trueCount {
tx.Rollback()
log.Error("count error %d %d", count, trueCount)
err = fmt.Errorf(" count error")
return
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
// UpdateColumnTag update column tag
func (s *Service) UpdateColumnTag(c context.Context, table string, oldTag string, newTag int, count int64) (err error) {
sql := fmt.Sprintf(_txColumnTagSQL, table, newTag, oldTag)
return s.txUpdateSQL(c, sql, count)
}
func (s *Service) txUpdateSQL(c context.Context, sql string, count int64) (err error) {
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
err = s.updateSQL(tx, sql, count)
if err != nil {
log.Error("s.UpdateSQL(%s) error(%v)", sql, err)
return
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
}
return
}
func (s *Service) updateSQL(tx *sql.Tx, stmt string, count int64) error {
rows, err := s.dao.UpdateDate(tx, stmt)
if err != nil {
tx.Rollback()
log.Error("s.dao.UpdateDate (%s) error(%v)", stmt, err)
return err
}
if count == 0 && rows <= 1 {
return nil
}
if rows != count {
tx.Rollback()
return fmt.Errorf("%s : rows(%d) != count(%d) error", stmt, rows, count)
}
return nil
}
func (s *Service) insertIntoSQL(tx *sql.Tx, stmt string, count int) error {
rows, err := s.dao.UpdateDate(tx, stmt)
if err != nil {
tx.Rollback()
log.Error("s.dao.UpdateDate (%s) error(%v)", stmt, err)
return err
}
if count == 0 {
if rows > 2 {
tx.Rollback()
return fmt.Errorf("rows(%d) error", rows)
}
} else if rows != int64(count) {
tx.Rollback()
return fmt.Errorf("rows(%d) != count(%d) error", rows, count)
}
return nil
}
// SyncAvBaseIncome sync base_income to av_base_income by mid_date
func (s *Service) SyncAvBaseIncome(c context.Context, table string) (err error) {
data, err := s.avBaseIncomes(c, table)
if err != nil {
return
}
err = s.batchUpdateUpIncome(c, data, table)
if err != nil {
log.Error("batch update av base income error(%v)", err)
}
return
}
func (s *Service) avBaseIncomes(c context.Context, table string) (data []*model.AvBaseIncome, err error) {
var id int64
for {
var abs []*model.AvBaseIncome
abs, id, err = s.dao.GetAvBaseIncome(c, table, id, 2000)
if err != nil {
return
}
if len(abs) == 0 {
break
}
for _, ab := range abs {
if ab.AvBaseIncome > 0 {
data = append(data, ab)
}
}
}
return
}
func (s *Service) batchUpdateUpIncome(c context.Context, us []*model.AvBaseIncome, table string) (err error) {
var (
buff = make([]*model.AvBaseIncome, 2000)
buffEnd = 0
)
for _, u := range us {
buff[buffEnd] = u
buffEnd++
if buffEnd >= 2000 {
values := avBaseIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.BatchUpdateUpIncome(c, table, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := avBaseIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.BatchUpdateUpIncome(c, table, values)
}
return
}
func avBaseIncomeValues(us []*model.AvBaseIncome) (values string) {
var buf bytes.Buffer
for _, u := range us {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(u.MID, 10))
buf.WriteByte(',')
buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvBaseIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}
// SyncCreditScore sync credit score
func (s *Service) SyncCreditScore(c context.Context) (err error) {
m := make(map[int64]int)
am, err := s.getCreditScore(c, "video")
if err != nil {
return
}
for mid, score := range am {
m[mid] = score
}
cm, err := s.getCreditScore(c, "column")
if err != nil {
return
}
for mid, score := range cm {
m[mid] = score
}
return s.batchInsertCreditScore(c, m)
}
func (s *Service) batchInsertCreditScore(c context.Context, m map[int64]int) (err error) {
batch := make(map[int64]int)
for mid, score := range m {
batch[mid] = score
if len(batch) == 2000 {
values := creditScoreValues(batch)
_, err = s.dao.SyncCreditScore(c, values)
if err != nil {
return
}
batch = make(map[int64]int)
}
}
if len(batch) > 0 {
values := creditScoreValues(batch)
_, err = s.dao.SyncCreditScore(c, values)
}
return
}
func creditScoreValues(m map[int64]int) (values string) {
var buf bytes.Buffer
for mid, score := range m {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(mid, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(score))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}
func (s *Service) getCreditScore(c context.Context, table string) (m map[int64]int, err error) {
m = make(map[int64]int)
var id int64
for {
var sm map[int64]int
sm, id, err = s.dao.GetCreditScore(c, table, id, 2000)
if err != nil {
return
}
if len(sm) == 0 {
break
}
for k, v := range sm {
m[k] = v
}
}
return
}
// FixBgmIncomeStatis fix bgm income statis
func (s *Service) FixBgmIncomeStatis(c context.Context) (err error) {
total, err := s.dao.GetBGMIncome(c)
if err != nil {
return
}
for sid, income := range total {
_, err = s.dao.InsertBGMIncomeStatis(c, sid, income)
if err != nil {
return
}
}
return
}

18
app/job/main/growup/service/data_test.go Normal file
View File

@@ -0,0 +1,18 @@
package service
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_UpdateWithdraw(t *testing.T) {
Convey("growup-job UpdateWithdraw", t, WithService(func(s *Service) {
newDate, oldDate := "2018-01", "2018-02"
count := int64(0)
err := s.UpdateWithdraw(context.Background(), oldDate, newDate, count)
So(err, ShouldBeNil)
}))
}

123
app/job/main/growup/service/databus.go Normal file
View File

@@ -0,0 +1,123 @@
package service
import (
"context"
"encoding/json"
"fmt"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/log"
)
var (
_archiveTable = "archive"
_musicTable = "music"
_actionUpdate = "update"
_actionInsert = "insert"
)
func (s *Service) archiveConsume(ctx context.Context) {
var (
msgs = s.arcSub.Messages()
err error
)
log.Info("archiveConsume start")
for {
msg, ok := <-msgs
if !ok {
log.Error("s.arcSub.Messages closed", err)
return
}
msg.Commit()
archive := &model.ArchiveMsg{}
if err = json.Unmarshal(msg.Value, archive); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
continue
}
if archive.Table == _musicTable {
go s.addBgmWhiteList(archive.Action, archive.New, archive.Old)
}
if archive.Table == _archiveTable && archive.Action == _actionUpdate {
go s.checkArchiveState(archive.New, archive.Old)
}
}
}
// (action == insert && new.state = 0) || (action = update && new.state = 0 && old.state < 0)
func (s *Service) addBgmWhiteList(action string, newMsg, oldMsg []byte) {
nw := &model.BgmSub{}
if err := json.Unmarshal(newMsg, nw); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", newMsg, err)
return
}
old := &model.BgmSub{}
if action == _actionUpdate {
if err := json.Unmarshal(oldMsg, old); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", oldMsg, err)
return
}
}
if (action == _actionInsert && nw.State == 0) || (action == _actionUpdate && nw.State == 0 && old.State < 0) {
log.Info("addBgmWhiteList mid(%d)", nw.MID)
_, err := s.dao.InsertBgmWhiteList(context.Background(), nw.MID)
if err != nil {
log.Error(" s.dao.InsertBgmWhiteList(%d) error(%v)", nw.MID, err)
}
}
}
// new.state>=0 && new.Copyright =2 && old.Copyright == 1 原创变转载
// new.state>=0 && new.Copyright =1 && old.Copyright == 2 转载变原创
func (s *Service) checkArchiveState(newMsg, oldMsg []byte) {
nw := &model.ArchiveSub{}
if err := json.Unmarshal(newMsg, nw); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", newMsg, err)
return
}
old := &model.ArchiveSub{}
if err := json.Unmarshal(oldMsg, old); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", oldMsg, err)
return
}
// 原创变转载
if nw.State >= 0 && nw.Copyright == 2 && old.Copyright == 1 {
log.Info("checkArchiveState get 1 avid(%d) mid(%d)", nw.ID, nw.MID)
s.avBreachPre(context.Background(), nw.ID, nw.MID, 1)
}
// 转载变原创
if nw.State >= 0 && nw.Copyright == 1 && old.Copyright == 2 {
log.Info("checkArchiveState get 0 avid(%d) mid(%d)", nw.ID, nw.MID)
s.avBreachPre(context.Background(), nw.ID, nw.MID, 0)
}
}
func (s *Service) avBreachPre(c context.Context, aid, mid int64, state int) (err error) {
if aid == 0 || mid == 0 {
return
}
accState, err := s.dao.GetUpStateByMID(c, mid)
if err != nil {
log.Error(" s.dao.GetUpStateByMID(%d) error(%v)", mid, err)
return
}
if accState != 3 {
return
}
// status == 1 insert av_breach_pre, status == 0 update state if exist
if state == 1 {
val := fmt.Sprintf("%d,%d,'%s',0,1", aid, mid, time.Now().Format(_layout))
_, err = s.dao.InsertAvBreachPre(c, val)
if err != nil {
log.Error("s.dao.InsertAvBreachPre error(%v)", err)
}
} else if state == 0 {
_, err = s.dao.UpdateAvBreachPre(c, aid, 0, time.Now().AddDate(0, 0, -1).Format(_layout), state)
if err != nil {
log.Error("s.dao.UpdateAvBreachPre error(%v)", err)
}
}
return
}

82
app/job/main/growup/service/format.go Normal file
View File

@@ -0,0 +1,82 @@
package service
import (
"bytes"
"encoding/csv"
"fmt"
"os"
"strconv"
"go-common/app/job/main/growup/model"
)
// WriteCSV write data to csv
func WriteCSV(records [][]string, filename string) (err error) {
f, err := os.Create(filename)
if err != nil {
return
}
defer f.Close()
w := csv.NewWriter(f)
w.WriteAll(records)
w.Flush()
return
}
// FormatCSV format to csv data
func FormatCSV(records [][]string) (data []byte, err error) {
buf := new(bytes.Buffer)
// add utf bom
buf.WriteString("\xEF\xBB\xBF")
w := csv.NewWriter(buf)
err = w.WriteAll(records)
if err != nil {
return
}
data = buf.Bytes()
return
}
func formatAvIncome(list []*model.AvIncome) (data [][]string) {
if len(list) <= 0 {
return
}
data = make([][]string, len(list)+1)
data[0] = []string{"稿件id", "UP主UID", "稿件月收入", "累计收入", "分区id", "最后收入时间"}
for i := 0; i < len(list); i++ {
l := list[i]
data[i+1] = []string{
strconv.FormatInt(l.AvID, 10),
strconv.FormatInt(l.MID, 10),
fmt.Sprintf("%.2f", float64(l.Income)/float64(100)),
fmt.Sprintf("%.2f", float64(l.TotalIncome)/float64(100)),
strconv.FormatInt(l.TagID, 10),
l.Date.Time().Format(_layout),
}
}
return
}
func formatUpAccount(list []*model.UpAccount, month int) (data [][]string) {
if len(list) <= 0 {
return
}
data = make([][]string, len(list)+1)
data[0] = []string{"UP主UID", "昵称", fmt.Sprintf("%d月有收入稿件数", month), fmt.Sprintf("%d月收入", month), "累计收入", "待结算收入", fmt.Sprintf("%d月收入-待结算", month)}
for i := 0; i < len(list); i++ {
l := list[i]
data[i+1] = []string{
strconv.FormatInt(l.MID, 10),
l.Nickname,
strconv.FormatInt(l.AvCount, 10),
fmt.Sprintf("%.2f", float64(l.MonthIncome)/float64(100)),
fmt.Sprintf("%.2f", float64(l.TotalIncome)/float64(100)),
fmt.Sprintf("%.2f", float64(l.TotalUnwithdrawIncome)/float64(100)),
fmt.Sprintf("%.2f", float64(l.MonthIncome-l.TotalUnwithdrawIncome)/float64(100)),
}
}
return
}

44
app/job/main/growup/service/httpClient.go Normal file
View File

@@ -0,0 +1,44 @@
package service
import (
"crypto/md5"
"fmt"
"io/ioutil"
"net/http"
"strconv"
"go-common/library/log"
)
// HTTPClient http client handle
func (s *Service) HTTPClient(method, url string, params map[string]string, nowTime int64) (body []byte, err error) {
req, err := http.NewRequest(method, url, nil)
if err != nil {
log.Error("http.NewRequest error(%v)", err)
return
}
q := req.URL.Query()
for key, value := range params {
q.Add(key, value)
}
q.Add("appkey", s.conf.KeySecret.Key)
q.Add("ts", strconv.FormatInt(nowTime, 10))
sign := q.Encode() + s.conf.KeySecret.Secret
q.Add("sign", fmt.Sprintf("%x", md5.Sum([]byte(sign))))
req.URL.RawQuery = q.Encode()
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Error("http.DefaultClient.Do error(%v)", err)
return
}
defer resp.Body.Close()
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
log.Error("ioutil.ReadAll error(%v)", err)
}
return
}

22
app/job/main/growup/service/httpClient_test.go Normal file
View File

@@ -0,0 +1,22 @@
package service
import (
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_HTTPClient(t *testing.T) {
Convey("growup-job HttpClient", t, WithService(func(s *Service) {
var (
method = "GET"
url = ""
params = make(map[string]string)
nowTime = time.Now().Add(-24 * time.Hour).Unix()
)
res, err := s.HTTPClient(method, url, params, nowTime)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}

485
app/job/main/growup/service/income.go Normal file
View File

@@ -0,0 +1,485 @@
package service
import (
"bytes"
"context"
"fmt"
"sort"
"strconv"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/database/sql"
"go-common/library/log"
)
const (
_layout = "2006-01-02"
)
// InsertTagIncome insert up_tag_income.
func (s *Service) InsertTagIncome(c context.Context, date time.Time) (err error) {
infos, err := s.getTagAvInfo(c, date)
if err != nil {
log.Error("s.InsertTagIncome getTagAVInfo error(%v)", err)
return
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.InsertTagIncome dao.BeginTran error(%v)", err)
return
}
if err = s.insertTagIncome(c, tx, infos); err != nil {
tx.Rollback()
log.Error("s.InsertTagIncome insertTagIncome error(%v)", err)
return
}
if err = s.updateTagInfo(tx, infos); err != nil {
tx.Rollback()
log.Error("s.InsertTagIncome updateTagInfo error(%v)", err)
return
}
if err = s.updateTagUpInfo(tx, infos); err != nil {
tx.Rollback()
log.Error("s.InsertTagIncome updateTagUpInfo error(%v)", err)
return
}
if err = tx.Commit(); err != nil {
log.Error("s.InsertTagIncome tx.Commit error")
return
}
return
}
func (s *Service) getTagAvInfo(c context.Context, date time.Time) (infos []*model.TagAvIncome, err error) {
var (
from, limit int64
av, avs []*model.ActivityAVInfo
)
from, limit = 0, 3000
for {
av, err = s.dao.GetAvTagRatio(c, from, limit)
if err != nil {
log.Error("s.getTagAvInfo dao.GetAvTagRatio error(%v)", err)
return
}
avs = append(avs, av...)
if int64(len(av)) < limit {
break
}
from = av[len(av)-1].MID
}
for _, a := range avs {
var income *model.TagAvIncome
income, err = s.dao.GetAvIncomeInfo(c, a.AVID, date)
if err != nil {
log.Error("s.GetAvIncomes dao.GetAvIncomeInfo error(%v)", err)
return
}
if income == nil {
continue
}
income.TagID = a.TagID
infos = append(infos, income)
}
return
}
func (s *Service) getTagAVLatestTotalIncome(c context.Context, avID, tagID int64) (totalIncome int, err error) {
infos, err := s.dao.GetTagAvTotalIncome(c, tagID, avID)
if err != nil {
log.Error("s.getTagAVLatestTotalIncome dao.GetTagAvTotalIncome error(%v)", err)
return
}
for _, info := range infos {
if int(info.TotalIncome) > totalIncome {
totalIncome = int(info.TotalIncome)
}
}
return
}
func (s *Service) insertTagIncome(c context.Context, tx *sql.Tx, infos []*model.TagAvIncome) (err error) {
var buf bytes.Buffer
var cnt, totalIncome int
var rows, totalRows int64
for _, info := range infos {
totalIncome, err = s.getTagAVLatestTotalIncome(c, info.TagID, info.AVID)
if err != nil {
log.Error("s.insertTagIncome dao.GetTagAvTotalIncome error(%v)", err)
return
}
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(info.TagID, 10))
buf.WriteString(",")
buf.WriteString(strconv.FormatInt(info.MID, 10))
buf.WriteString(",")
buf.WriteString(strconv.FormatInt(info.AVID, 10))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(info.Income))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(totalIncome + info.Income))
buf.WriteString(",")
buf.WriteString("'")
buf.WriteString(strconv.Itoa(info.Date.Year()))
buf.WriteString("-")
if int(info.Date.Month()) < 10 {
buf.WriteString("0")
}
buf.WriteString(strconv.Itoa(int(info.Date.Month())))
buf.WriteString("-")
if info.Date.Day() < 10 {
buf.WriteString("0")
}
buf.WriteString(strconv.Itoa(info.Date.Day()))
buf.WriteString("'")
buf.WriteString("),")
cnt++
if cnt%1000 == 0 {
buf.Truncate(buf.Len() - 1)
rows, err = s.dao.TxInsertTagIncome(tx, buf.String())
if err != nil {
log.Error("s.InsertTagIncome dao.TxInsertTagIncome error(%v)", err)
return
}
totalRows += rows
buf.Reset()
}
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
rows, err = s.dao.TxInsertTagIncome(tx, buf.String())
if err != nil {
log.Error("s.InsertTagIncome dao.TxInsertTagIncome error(%v)", err)
return
}
totalRows += rows
}
log.Info("s.InsertTagIncome insert up_tag_income (%d) rows", totalRows)
return
}
// updateTagInfo update tag_info total_income.
func (s *Service) updateTagInfo(tx *sql.Tx, infos []*model.TagAvIncome) (err error) {
tim := make(map[int64]int) // key-value: tag_id-total av income.
for _, info := range infos {
tim[info.TagID] += info.Income
}
for k, v := range tim {
query := "total_income = total_income + "
query += strconv.Itoa(v)
_, err = s.dao.TxUpdateTagInfo(tx, k, query)
if err != nil {
log.Error("s.updateTagInfo dao.UpdateTagInfo error(%v)", err)
return
}
}
return
}
// updateTagUpInfo update tag_up_info totalIncome.
func (s *Service) updateTagUpInfo(tx *sql.Tx, infos []*model.TagAvIncome) (err error) {
utm := make(map[int64]*model.TagAvIncome) // key-value: mid-totalIncome
for _, info := range infos {
_, ok := utm[info.MID]
if !ok {
a := &model.TagAvIncome{TagID: info.TagID, MID: info.MID, TotalIncome: info.Income}
utm[info.MID] = a
} else {
utm[info.MID].TotalIncome += info.Income
}
}
cnt := 0
var buf bytes.Buffer
for _, v := range utm {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(v.TagID, 10))
buf.WriteString(",")
buf.WriteString(strconv.FormatInt(v.MID, 10))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(v.TotalIncome))
buf.WriteString("),")
cnt++
if cnt%2000 == 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.TxUpdateTagUpInfo(tx, buf.String())
if err != nil {
log.Error("s.updateTagUpInfo dao.UpdateTagUpInfo error(%v)", err)
return
}
buf.Reset()
}
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.TxUpdateTagUpInfo(tx, buf.String())
if err != nil {
log.Error("s.updateTagUpInfo dao.UpdateTagUpInfo error(%v)", err)
return
}
}
return
}
// GetAvIncomeStatis get av monthly income
func (s *Service) GetAvIncomeStatis(c context.Context, date string) error {
d, _ := time.Parse(_layout, date)
endTime := d.AddDate(0, 1, 0).Format(_layout)
avs, err := s.GetAvIncome(c)
if err != nil {
log.Error("s.GetAvIncome error(%v)", err)
return err
}
log.Info("GetAvIncomeStatis get %d avs", len(avs))
avsMap := make(map[int64]*model.AvIncome)
avIncomeStatis(avsMap, avs, date, endTime)
data := make([]*model.AvIncome, 0)
for _, av := range avsMap {
data = append(data, av)
}
sort.Slice(data, func(i, j int) bool {
return data[i].Income > data[j].Income
})
log.Info("GetAvIncomeStatis calculate success: %d", len(data))
return s.batchSend(data)
}
func (s *Service) batchSend(data []*model.AvIncome) error {
fileNo, start, batchSize := 0, 0, 50000
for {
if start+batchSize >= len(data) {
batchSize = len(data) - start
}
if batchSize <= 0 {
break
}
records := formatAvIncome(data[start : start+batchSize])
filename := fmt.Sprintf("av_statis0%d", fileNo)
err := WriteCSV(records, filename)
if err != nil {
return err
}
err = s.email.SendMailAttach(filename, "稿件月收入", []string{"shaozhenyu@bilibili.com"})
if err != nil {
return err
}
fileNo++
start += batchSize
}
return nil
}
func avIncomeStatis(avsMap map[int64]*model.AvIncome, avs []*model.AvIncome, fromTime, toTime string) {
for _, av := range avs {
d := av.Date.Time().Format(_layout)
if d < fromTime || d >= toTime {
continue
}
if _, ok := avsMap[av.AvID]; ok {
avsMap[av.AvID].Income += av.Income
if avsMap[av.AvID].Date < av.Date {
avsMap[av.AvID].TotalIncome = av.TotalIncome
avsMap[av.AvID].Date = av.Date
}
} else {
avsMap[av.AvID] = av
}
}
}
// GetAvIncome get av_income
func (s *Service) GetAvIncome(c context.Context) (avs []*model.AvIncome, err error) {
limit := 2000
var id int64
for {
av, err := s.dao.ListAvIncome(c, id, limit)
if err != nil {
return avs, err
}
avs = append(avs, av...)
if len(av) < limit {
break
}
id = av[len(av)-1].ID
}
return
}
// GetUpIncomeStatis get up statis
func (s *Service) GetUpIncomeStatis(c context.Context, date string, hasWithdraw int) (err error) {
var upAccount []*model.UpAccount
if hasWithdraw == 1 {
upAccount, err = s.getUpIncomeStatisAfterWithdraw(c, date)
if err != nil {
log.Error("s.getUpIncomeStatisAfterWithdraw error(%v)", err)
return err
}
} else {
upAccount, err = s.getUpIncomeStatisBeforeWithdraw(c, date)
if err != nil {
log.Error("s.getUpIncomeStatisBeforeWithdraw error(%v)", err)
return err
}
}
upa := make(map[int64]*model.UpAccount)
mids := make([]int64, len(upAccount))
for i := 0; i < len(upAccount); i++ {
mids[i] = upAccount[i].MID
upa[upAccount[i].MID] = upAccount[i]
}
upNick, err := s.GetUpNickname(c, mids)
if err != nil {
log.Error("s.GetUpNickname error(%v)", err)
return
}
upIncome, err := s.GetUpIncome(c, "up_income_monthly", date)
if err != nil {
log.Error("s.GetUpIncome error(%v)", err)
return
}
upIncomeStatis(upa, upIncome, upNick)
data := []*model.UpAccount{}
for _, up := range upa {
data = append(data, up)
}
sort.Slice(data, func(i, j int) bool {
return data[i].MonthIncome > data[j].MonthIncome
})
d, _ := time.Parse(_layout, date)
records := formatUpAccount(data, int(d.Month()))
err = WriteCSV(records, "up_statis.csv")
if err != nil {
log.Error("WriteCSV error(%v)", err)
return
}
return s.email.SendMailAttach("up_statis.csv", "up主月结算", []string{"shaozhenyu@bilibili.com"})
}
func (s *Service) getUpIncomeStatisAfterWithdraw(c context.Context, date string) (upAccount []*model.UpAccount, err error) {
d, _ := time.Parse(_layout, date)
withdrawDateStr := d.Format("2006-01")
ctime := d.AddDate(0, 1, 1).Format(_layout)
upAccount, err = s.GetUpAccount(c, withdrawDateStr, ctime)
if err != nil {
log.Error("s.GetUpAccount error(%v)", err)
return
}
upWithdraw, err := s.GetUpWithdraw(c, withdrawDateStr)
if err != nil {
log.Error("s.GetUpWithdraw error(%v)", err)
return
}
for _, up := range upAccount {
up.TotalUnwithdrawIncome = upWithdraw[up.MID]
}
return
}
func (s *Service) getUpIncomeStatisBeforeWithdraw(c context.Context, date string) (upAccount []*model.UpAccount, err error) {
d, _ := time.Parse(_layout, date)
withdrawDateStr := d.AddDate(0, -1, 0).Format("2006-01")
ctime := d.AddDate(0, 1, 1).Format(_layout)
return s.GetUpAccount(c, withdrawDateStr, ctime)
}
func upIncomeStatis(upa map[int64]*model.UpAccount, upIncome []*model.UpIncome, upNick map[int64]string) {
for _, income := range upIncome {
if _, ok := upa[income.MID]; ok {
upa[income.MID].AvCount = income.AvCount
upa[income.MID].MonthIncome = income.Income
upa[income.MID].Nickname = upNick[income.MID]
}
}
}
// GetUpAccount get up_account
func (s *Service) GetUpAccount(c context.Context, date, ctime string) (ups []*model.UpAccount, err error) {
offset, limit := 0, 2000
for {
up, err := s.dao.ListUpAccount(c, date, ctime, offset, limit)
if err != nil {
return ups, err
}
ups = append(ups, up...)
if len(up) < limit {
break
}
offset += limit
}
return
}
// GetUpIncome get up_income
func (s *Service) GetUpIncome(c context.Context, table, date string) (ups []*model.UpIncome, err error) {
ups = make([]*model.UpIncome, 0)
var id int64
limit := 2000
for {
var up []*model.UpIncome
up, err = s.dao.ListUpIncome(c, table, date, id, limit)
if err != nil {
return
}
ups = append(ups, up...)
if len(up) < limit {
break
}
id = up[len(up)-1].ID
}
return
}
// GetUpWithdraw get up_income_withdraw
func (s *Service) GetUpWithdraw(c context.Context, date string) (ups map[int64]int64, err error) {
ups = make(map[int64]int64)
offset, limit := 0, 2000
for {
up, err := s.dao.ListUpWithdraw(c, date, offset, limit)
if err != nil {
return ups, err
}
for mid, income := range up {
ups[mid] = income
}
if len(up) < limit {
break
}
offset += limit
}
return
}
// GetUpNickname get up nickname
func (s *Service) GetUpNickname(c context.Context, mids []int64) (upNick map[int64]string, err error) {
upNick = make(map[int64]string)
offset, limit := 0, 2000
for {
if offset+limit > len(mids) {
limit = len(mids) - offset
}
if limit <= 0 {
break
}
err = s.dao.ListUpNickname(c, mids[offset:offset+limit], upNick)
if err != nil {
log.Error("s.dao.ListUpNickname error(%v)", err)
return
}
offset += limit
}
return
}

96
app/job/main/growup/service/income/BUILD Normal file
View File

@@ -0,0 +1,96 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"av_charge_statis_test.go",
"av_charge_test.go",
"blacklist_test.go",
"charge_ratio_test.go",
"run_test.go",
"service_test.go",
"up_account_test.go",
"up_income_date_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/model/income:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"archive.go",
"av_charge.go",
"av_charge_statis.go",
"av_income.go",
"av_income_stat.go",
"bgm.go",
"bgm_income.go",
"bgm_income_stat.go",
"blacklist.go",
"business_order.go",
"charge.go",
"charge_ratio.go",
"column_charge.go",
"column_income.go",
"column_income_stat.go",
"common.go",
"income.go",
"income_date_statis.go",
"lottery.go",
"run.go",
"run_statis.go",
"service.go",
"tax.go",
"tools.go",
"up_account.go",
"up_archive_statis.go",
"up_income.go",
"up_income_date.go",
"up_income_stat.go",
"up_info_video.go",
],
importpath = "go-common/app/job/main/growup/service/income",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/dao/dataplatform:go_default_library",
"//app/job/main/growup/dao/email:go_default_library",
"//app/job/main/growup/dao/income:go_default_library",
"//app/job/main/growup/model/income:go_default_library",
"//app/job/main/growup/service:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//library/log:go_default_library",
"//library/time:go_default_library",
"//vendor/golang.org/x/sync/errgroup:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

53
app/job/main/growup/service/income/archive.go Normal file
View File

@@ -0,0 +1,53 @@
package income
import (
"context"
"time"
model "go-common/app/job/main/growup/model/income"
)
func (s *DateStatis) getArchiveByDate(c context.Context, archiveCh chan []*model.ArchiveIncome, startDate, endDate time.Time, typ, limit int) (err error) {
defer close(archiveCh)
var aid, table string
switch typ {
case _video:
aid, table = "av_id", "av_income"
case _column:
aid, table = "aid", "column_income"
case _bgm:
aid, table = "sid", "bgm_income"
default:
return
}
endDate = endDate.AddDate(0, 0, 1)
for startDate.Before(endDate) {
err = s.getArchiveIncome(c, archiveCh, aid, table, startDate.Format(_layout), limit)
if err != nil {
return
}
startDate = startDate.AddDate(0, 0, 1)
}
return
}
func (s *DateStatis) getArchiveIncome(c context.Context, archiveCh chan []*model.ArchiveIncome, aid, table, date string, limit int) (err error) {
var id int64
for {
var archive []*model.ArchiveIncome
if aid == "sid" {
archive, err = s.dao.GetBgmIncomeByDate(c, date, id, limit)
} else {
archive, err = s.dao.GetArchiveByDate(c, aid, table, date, id, limit)
}
if err != nil {
return
}
archiveCh <- archive
if len(archive) < limit {
break
}
id = archive[len(archive)-1].ID
}
return
}

375
app/job/main/growup/service/income/av_charge.go Normal file
View File

@@ -0,0 +1,375 @@
package income
import (
"bytes"
"context"
"strconv"
"strings"
"time"
dao "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
"go-common/library/log"
xtime "go-common/library/time"
"golang.org/x/sync/errgroup"
)
var (
_avWeeklyCharge = "av_weekly_charge"
_avMonthlyCharge = "av_monthly_charge"
)
// HandleAvCharge handle av daily charge
func (s *AvChargeSvr) HandleAvCharge(c context.Context, date time.Time,
dailyChannel chan []*model.AvCharge) (weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
var eg errgroup.Group
weeklyChargeMap = make(map[int64]*model.AvCharge)
monthlyChargeMap = make(map[int64]*model.AvCharge)
eg.Go(func() (err error) {
avWeeklyCharge, err := s.GetAvCharge(c, getStartWeeklyDate(date), s.dao.AvWeeklyCharge)
if err != nil {
log.Error("s.GetAvCharge(av_weekly_charge) error(%v)", err)
return
}
for _, weeklyCharge := range avWeeklyCharge {
weeklyChargeMap[weeklyCharge.AvID] = weeklyCharge
}
return
})
eg.Go(func() (err error) {
avMonthlyCharge, err := s.GetAvCharge(c, getStartMonthlyDate(date), s.dao.AvMonthlyCharge)
if err != nil {
log.Error("s.GetAvCharge(av_monthly_charge) error(%v)", err)
return
}
for _, monthlyCharge := range avMonthlyCharge {
monthlyChargeMap[monthlyCharge.AvID] = monthlyCharge
}
return
})
eg.Go(func() (err error) {
chargeStatisMap, err = s.GetAvChargeStatisMap(c)
if err != nil {
log.Error("s.GetAvChargeStatisMap error(%v)", err)
}
return
})
if err = eg.Wait(); err != nil {
log.Error("HandleAvCharge eg.Wait error(%v)", err)
return
}
s.CalAvCharge(date, weeklyChargeMap, monthlyChargeMap, chargeStatisMap, dailyChannel)
return
}
// AvCharges get av charges
func (s *AvChargeSvr) AvCharges(c context.Context, date time.Time, ch chan []*model.AvCharge, bubbleRatio map[int64]float64) (err error) {
defer func() {
close(ch)
}()
var id int64
for {
var charges []*model.AvCharge
charges, err = s.dao.AvDailyCharge(c, date, id, _limitSize)
if err != nil {
return
}
for _, charge := range charges {
if ratio, ok := bubbleRatio[charge.AvID]; ok {
charge.IncCharge = int64(Round(Mul(float64(charge.IncCharge), ratio), 0))
}
}
ch <- charges
if len(charges) < _limitSize {
break
}
id = charges[len(charges)-1].ID
}
return
}
// GetAvCharge get av charge
func (s *AvChargeSvr) GetAvCharge(c context.Context, date time.Time, fn dao.IAvCharge) (avCharges []*model.AvCharge, err error) {
var id int64
for {
avCharge, err1 := fn(c, date, id, _limitSize)
if err1 != nil {
err = err1
return
}
avCharges = append(avCharges, avCharge...)
if len(avCharge) < _limitSize {
break
}
id = avCharge[len(avCharge)-1].ID
}
return
}
// CalAvCharge cal av charge
func (s *AvChargeSvr) CalAvCharge(date time.Time, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, dailyChannel chan []*model.AvCharge) {
for avDailyCharge := range dailyChannel {
s.calAvCharge(date, avDailyCharge, weeklyChargeMap, monthlyChargeMap, chargeStatisMap)
}
}
func (s *AvChargeSvr) calAvCharge(date time.Time, avDailyCharge []*model.AvCharge, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
startWeeklyDate = getStartWeeklyDate(date)
startMonthlyDate = getStartMonthlyDate(date)
for _, dailyCharge := range avDailyCharge {
if weeklyCharge, ok := weeklyChargeMap[dailyCharge.AvID]; ok {
updateAvCharge(weeklyCharge, dailyCharge)
} else {
weeklyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startWeeklyDate)
}
if monthlyCharge, ok := monthlyChargeMap[dailyCharge.AvID]; ok {
updateAvCharge(monthlyCharge, dailyCharge)
} else {
monthlyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startMonthlyDate)
}
s.CalAvChargeStatis(dailyCharge, chargeStatisMap)
}
}
func addAvCharge(daily *model.AvCharge, fixDate time.Time) *model.AvCharge {
return &model.AvCharge{
AvID: daily.AvID,
MID: daily.MID,
TagID: daily.TagID,
IsOriginal: daily.IsOriginal,
DanmakuCount: daily.DanmakuCount,
CommentCount: daily.CommentCount,
CollectCount: daily.CollectCount,
CoinCount: daily.CoinCount,
ShareCount: daily.ShareCount,
ElecPayCount: daily.ElecPayCount,
TotalPlayCount: daily.TotalPlayCount,
WebPlayCount: daily.WebPlayCount,
AppPlayCount: daily.AppPlayCount,
H5PlayCount: daily.H5PlayCount,
LvUnknown: daily.LvUnknown,
Lv0: daily.Lv0,
Lv1: daily.Lv1,
Lv2: daily.Lv2,
Lv3: daily.Lv3,
Lv4: daily.Lv4,
Lv5: daily.Lv5,
Lv6: daily.Lv6,
VScore: daily.VScore,
IncCharge: daily.IncCharge,
TotalCharge: daily.IncCharge,
Date: xtime.Time(fixDate.Unix()),
UploadTime: daily.UploadTime,
DBState: _dbInsert,
}
}
func updateAvCharge(origin, daily *model.AvCharge) {
origin.DanmakuCount += daily.DanmakuCount
origin.CommentCount += daily.CommentCount
origin.CollectCount += daily.CollectCount
origin.CoinCount += daily.CoinCount
origin.ShareCount += daily.ShareCount
origin.ElecPayCount += daily.ElecPayCount
origin.TotalPlayCount += daily.TotalPlayCount
origin.WebPlayCount += daily.WebPlayCount
origin.AppPlayCount += daily.AppPlayCount
origin.H5PlayCount += daily.H5PlayCount
origin.LvUnknown += daily.LvUnknown
origin.Lv0 += daily.Lv0
origin.Lv1 += daily.Lv1
origin.Lv2 += daily.Lv2
origin.Lv3 += daily.Lv3
origin.Lv4 += daily.Lv4
origin.Lv5 += daily.Lv5
origin.Lv6 += daily.Lv6
origin.VScore += daily.VScore
origin.IncCharge += daily.IncCharge
origin.TotalCharge += daily.IncCharge
origin.DBState = _dbUpdate
}
// AvChargeDBStore av charge db store
func (s *AvChargeSvr) AvChargeDBStore(c context.Context, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge) (err error) {
err = s.avChargeDBStoreBatch(c, _avWeeklyCharge, weeklyChargeMap)
if err != nil {
log.Error("s.avChargeDBStoreBatch av_weekly_charge error(%v)", err)
return
}
err = s.avChargeDBStoreBatch(c, _avMonthlyCharge, monthlyChargeMap)
if err != nil {
log.Error("s.avChargeDBStoreBatch av_monthly_charge error(%v)", err)
return
}
return
}
func (s *AvChargeSvr) avChargeDBStoreBatch(c context.Context, table string, avChargeMap map[int64]*model.AvCharge) error {
insert, update := make([]*model.AvCharge, batchSize), make([]*model.AvCharge, batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range avChargeMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= batchSize {
_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= batchSize {
_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
if err != nil {
log.Error("s.avChargeBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleAvCharge(avCharge []*model.AvCharge) (vals string) {
var buf bytes.Buffer
for _, row := range avCharge {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(row.IsOriginal))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.DanmakuCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CommentCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CollectCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CoinCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.ShareCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.ElecPayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalPlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.WebPlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.AppPlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.H5PlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.LvUnknown, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv0, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv1, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv2, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv3, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv4, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv5, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Lv6, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.VScore, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *AvChargeSvr) avChargeBatchInsert(c context.Context, avCharge []*model.AvCharge, table string) (rows int64, err error) {
vals := assembleAvCharge(avCharge)
rows, err = s.dao.InsertAvChargeTable(c, vals, table)
return
}
// for test
func assembleColumnCharge(cols []*model.ColumnCharge) (vals string) {
var buf bytes.Buffer
for _, row := range cols {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.ArticleID, 10))
buf.WriteByte(',')
buf.WriteString("\"" + strings.Replace(row.Title, "\"", "\\\"", -1) + "\"")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.UploadTime.Time().Unix(), 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *AvChargeSvr) columnChargeBatchInsert(c context.Context, colCharge []*model.ColumnCharge, table string) (rows int64, err error) {
vals := assembleColumnCharge(colCharge)
rows, err = s.dao.InsertColumnChargeTable(c, vals, table)
return
}

155
app/job/main/growup/service/income/av_charge_statis.go Normal file
View File

@@ -0,0 +1,155 @@
package income
import (
"bytes"
"context"
"strconv"
model "go-common/app/job/main/growup/model/income"
"go-common/library/log"
)
// GetAvChargeStatisMap get av charge statis map
func (s *AvChargeSvr) GetAvChargeStatisMap(c context.Context) (chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
avChargeStatis, err := s.GetAvChargeStatis(c)
if err != nil {
log.Error("s.GetAvChargeStatis error(%v)", err)
return
}
chargeStatisMap = make(map[int64]*model.AvChargeStatis)
for _, chargeStatis := range avChargeStatis {
chargeStatisMap[chargeStatis.AvID] = chargeStatis
}
return
}
// GetAvChargeStatis get av charge statis
func (s *AvChargeSvr) GetAvChargeStatis(c context.Context) (avChargeStatis []*model.AvChargeStatis, err error) {
var id int64
for {
statis, err1 := s.dao.AvChargeStatis(c, id, _limitSize)
if err1 != nil {
err = err1
return
}
avChargeStatis = append(avChargeStatis, statis...)
if len(statis) < _limitSize {
break
}
id = statis[len(statis)-1].ID
}
return
}
// CalAvChargeStatis cal av charge statis
func (s *AvChargeSvr) CalAvChargeStatis(dailyCharge *model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
if statisCharge, ok := chargeStatisMap[dailyCharge.AvID]; ok {
updateAvChargeStatis(statisCharge, dailyCharge)
} else {
chargeStatisMap[dailyCharge.AvID] = addAvChargeStatis(dailyCharge)
}
}
func addAvChargeStatis(daily *model.AvCharge) *model.AvChargeStatis {
return &model.AvChargeStatis{
AvID: daily.AvID,
MID: daily.MID,
TagID: daily.TagID,
IsOriginal: daily.IsOriginal,
UploadTime: daily.UploadTime,
TotalCharge: daily.IncCharge,
DBState: _dbInsert,
}
}
func updateAvChargeStatis(avChargeStatis *model.AvChargeStatis, daily *model.AvCharge) {
avChargeStatis.TotalCharge += daily.IncCharge
avChargeStatis.DBState = _dbUpdate
}
// AvChargeStatisDBStore store charge statis
func (s *AvChargeSvr) AvChargeStatisDBStore(c context.Context, chargeStatisMap map[int64]*model.AvChargeStatis) error {
insert, update := make([]*model.AvChargeStatis, batchSize), make([]*model.AvChargeStatis, batchSize)
insertIndex, updateIndex := 0, 0
for _, charge := range chargeStatisMap {
if charge.DBState == _dbInsert {
insert[insertIndex] = charge
insertIndex++
} else if charge.DBState == _dbUpdate {
update[updateIndex] = charge
updateIndex++
}
if insertIndex >= batchSize {
_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= batchSize {
_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
if err != nil {
log.Error("s.avChargeStatisBatchInsert error(%v)", err)
return err
}
}
return nil
}
func assembleAvChargeStatis(avChargeStatis []*model.AvChargeStatis) (vals string) {
var buf bytes.Buffer
for _, row := range avChargeStatis {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(row.IsOriginal))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}
func (s *AvChargeSvr) avChargeStatisBatchInsert(c context.Context, avChargeStatis []*model.AvChargeStatis) (rows int64, err error) {
vals := assembleAvChargeStatis(avChargeStatis)
rows, err = s.dao.InsertAvChargeStatisBatch(c, vals)
return
}

75
app/job/main/growup/service/income/av_charge_statis_test.go Normal file
View File

@@ -0,0 +1,75 @@
package income
import (
"context"
"testing"
"time"
model "go-common/app/job/main/growup/model/income"
xtime "go-common/library/time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_GetAvChargeStatisMap(t *testing.T) {
Convey("GetAvChargeStatisMap", t, func() {
_, err := charge.GetAvChargeStatisMap(context.Background())
So(err, ShouldBeNil)
})
}
func Test_GetAvChargeStatis(t *testing.T) {
Convey("GetAvChargeStatis", t, func() {
_, err := charge.GetAvChargeStatis(context.Background())
So(err, ShouldBeNil)
})
}
func Test_AvChargeStatisDBStore(t *testing.T) {
Convey("AvChargeStatisDBStore", t, func() {
chargeStatisMap := make(map[int64]*model.AvChargeStatis)
value := &model.AvChargeStatis{
AvID: 11,
MID: 11,
TagID: 11,
DBState: 1,
}
chargeStatisMap[11] = value
err := charge.AvChargeStatisDBStore(context.Background(), chargeStatisMap)
So(err, ShouldBeNil)
})
}
func benchmarkAvChargeStatisDBStore(bsize int, size int64, b *testing.B) {
batchSize = bsize
var i int64
chargeStatisMap := make(map[int64]*model.AvChargeStatis)
for i = 0; i < size; i++ {
chargeStatisMap[i] = &model.AvChargeStatis{
AvID: i,
MID: i,
TagID: i,
IsOriginal: int(i),
UploadTime: xtime.Time(time.Now().Unix()),
TotalCharge: i,
DBState: int(i % 2),
}
}
for n := 0; n < b.N; n++ {
charge.AvChargeStatisDBStore(context.Background(), chargeStatisMap)
}
}
func BenchmarkAvChargeStatisDBStore100(b *testing.B) {
benchmarkAvChargeStatisDBStore(100, 100000, b)
}
func BenchmarkAvChargeStatisDBStore1000(b *testing.B) {
benchmarkAvChargeStatisDBStore(1000, 100000, b)
}
func BenchmarkAvChargeStatisDBStore2000(b *testing.B) {
benchmarkAvChargeStatisDBStore(2000, 100000, b)
}
func BenchmarkAvChargeStatisDBStore10000(b *testing.B) {
benchmarkAvChargeStatisDBStore(10000, 100000, b)
}

93
app/job/main/growup/service/income/av_charge_test.go Normal file
View File

@@ -0,0 +1,93 @@
package income
import (
"context"
"testing"
"time"
model "go-common/app/job/main/growup/model/income"
xtime "go-common/library/time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_GetAvCharge(t *testing.T) {
Convey("GetAvCharge", t, func() {
_, err := charge.GetAvCharge(context.Background(), time.Now(), charge.dao.AvMonthlyCharge)
So(err, ShouldBeNil)
})
}
func Test_AvChargeDBStore(t *testing.T) {
Convey("AvChargeDBStore", t, func() {
monthlyChargeMap := make(map[int64]*model.AvCharge)
value := &model.AvCharge{
AvID: 11,
MID: 11,
TagID: 11,
DBState: 1,
}
monthlyChargeMap[11] = value
err := charge.AvChargeDBStore(context.Background(), monthlyChargeMap, monthlyChargeMap)
So(err, ShouldBeNil)
})
}
func benchmarkAvChargeDBStore(bsize int, size int64, b *testing.B) {
batchSize = bsize
for n := 0; n < b.N; n++ {
var i int64
weeklyChargeMap := make(map[int64]*model.AvCharge, size)
for i = 0; i < size; i++ {
v := int64(n+1) * i
weeklyChargeMap[v] = &model.AvCharge{
AvID: v,
MID: v,
TagID: v,
IsOriginal: int(v),
DanmakuCount: v,
CommentCount: v,
CollectCount: v,
CoinCount: v,
ShareCount: v,
ElecPayCount: v,
TotalPlayCount: v,
WebPlayCount: v,
AppPlayCount: v,
H5PlayCount: v,
LvUnknown: v,
Lv0: v,
Lv1: v,
Lv2: v,
Lv3: v,
Lv4: v,
Lv5: v,
Lv6: v,
VScore: v,
IncCharge: v,
TotalCharge: v,
UploadTime: xtime.Time(time.Now().Unix()),
Date: xtime.Time(time.Now().Unix()),
DBState: 1,
}
}
charge.AvChargeDBStore(context.Background(), weeklyChargeMap, weeklyChargeMap)
}
}
func BenchmarkAvChargeDBStore100(b *testing.B) {
benchmarkAvChargeDBStore(100, 100000, b)
}
func BenchmarkAvChargeDBStore1000(b *testing.B) {
benchmarkAvChargeDBStore(1000, 100000, b)
}
func BenchmarkAvChargeDBStore2000(b *testing.B) {
benchmarkAvChargeDBStore(2000, 100000, b)
}
func BenchmarkAvChargeDBStore10000(b *testing.B) {
benchmarkAvChargeDBStore(10000, 100000, b)
}

90
app/job/main/growup/service/income/av_income.go Normal file
View File

@@ -0,0 +1,90 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// AvIncomeSvr Av income service
type AvIncomeSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewAvIncomeSvr new av income service
func NewAvIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *AvIncomeSvr) {
return &AvIncomeSvr{
batchSize: batchSize,
dao: dao,
}
}
// BatchInsertAvIncome batch insert av income
func (p *AvIncomeSvr) BatchInsertAvIncome(c context.Context, am map[int64][]*model.AvIncome) (err error) {
var (
buff = make([]*model.AvIncome, p.batchSize)
buffEnd = 0
)
for _, as := range am {
for _, a := range as {
buff[buffEnd] = a
buffEnd++
if buffEnd >= p.batchSize {
values := avIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertAvIncome(c, values)
if err != nil {
return
}
}
}
}
if buffEnd > 0 {
values := avIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertAvIncome(c, values)
}
return
}
func avIncomeValues(as []*model.AvIncome) (values string) {
var buf bytes.Buffer
for _, a := range as {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(a.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(a.IsOriginal))
buf.WriteByte(',')
buf.WriteString("'" + a.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.PlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TaxMoney, 10))
buf.WriteByte(',')
buf.WriteString("'" + a.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.BaseIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

100
app/job/main/growup/service/income/av_income_stat.go Normal file
View File

@@ -0,0 +1,100 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// AvIncomeStatSvr av income stat service
type AvIncomeStatSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewAvIncomeStatSvr new av income service
func NewAvIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *AvIncomeStatSvr) {
return &AvIncomeStatSvr{
batchSize: batchSize,
dao: dao,
}
}
// AvIncomeStat get av income stat
func (p *AvIncomeStatSvr) AvIncomeStat(c context.Context, limit int64) (m map[int64]*model.AvIncomeStat, err error) {
m = make(map[int64]*model.AvIncomeStat)
var id int64
for {
var am map[int64]*model.AvIncomeStat
am, id, err = p.dao.AvIncomeStat(c, id, limit)
if err != nil {
return
}
if len(am) == 0 {
break
}
for avID, stat := range am {
m[avID] = stat
}
}
return
}
// BatchInsertAvIncomeStat batch insert av income statistics
func (p *AvIncomeStatSvr) BatchInsertAvIncomeStat(c context.Context, as map[int64]*model.AvIncomeStat) (err error) {
var (
buff = make([]*model.AvIncomeStat, p.batchSize)
buffEnd = 0
)
for _, a := range as {
if a.DataState == 0 {
continue
}
buff[buffEnd] = a
buffEnd++
if buffEnd >= p.batchSize {
values := avIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertAvIncomeStat(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := avIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertAvIncomeStat(c, values)
}
return
}
func avIncomeStatValues(as []*model.AvIncomeStat) (values string) {
var buf bytes.Buffer
for _, a := range as {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(a.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(a.IsOriginal))
buf.WriteByte(',')
buf.WriteString("'" + a.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

143
app/job/main/growup/service/income/bgm.go Normal file
View File

@@ -0,0 +1,143 @@
package income
import (
"bytes"
"context"
"fmt"
"strconv"
"strings"
"time"
model "go-common/app/job/main/growup/model/income"
task "go-common/app/job/main/growup/service"
"go-common/library/log"
)
// BGMs map[avid][]*model.BGM,
func (s *Service) BGMs(c context.Context, limit int64) (bm map[int64][]*model.BGM, err error) {
var id int64
bm = make(map[int64][]*model.BGM)
for {
var bs []*model.BGM
bs, id, err = s.dao.GetBGM(c, id, limit)
if err != nil {
return
}
if len(bs) == 0 {
break
}
for _, b := range bs {
if _, ok := bm[b.AID]; ok {
bm[b.AID] = append(bm[b.AID], b)
} else {
bm[b.AID] = []*model.BGM{b}
}
}
}
return
}
func (s *Service) batchInsertBGMs(c context.Context, bs []*model.BGM) (err error) {
var (
buff = make([]*model.BGM, 2000)
buffEnd = 0
)
for _, b := range bs {
buff[buffEnd] = b
buffEnd++
if buffEnd >= 2000 {
values := bgmValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.InsertBGM(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := bgmValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.InsertBGM(c, values)
}
return
}
func bgmValues(bs []*model.BGM) (values string) {
var buf bytes.Buffer
for _, b := range bs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(b.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.SID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.AID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.CID, 10))
buf.WriteByte(',')
buf.WriteString("'" + b.JoinAt + "'")
buf.WriteByte(',')
buf.WriteString("\"" + strings.Replace(b.Title, "\"", "\\\"", -1) + "\"")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}
const query = "{\"select\": [],\"page\": {\"skip\": %d,\"limit\": %d}}"
func (s *Service) getBGM(c context.Context) (bgms []*model.BGM, err error) {
from, limit := 0, 1000
var info []*model.BGM
for {
info, err = s.dp.SendBGMRequest(c, fmt.Sprintf(query, from, limit))
if err != nil {
log.Error("s.getBGM error(%v)", err)
return
}
if len(info) == 0 {
break
}
bgms = append(bgms, info...)
from += len(info)
time.Sleep(15 * time.Second) // 数据平台限流，api 10s内只能请求一次
}
log.Info("get bgm data total (%d) rows", from)
return
}
// SyncBgmInfo sync bgm info from data platform
func (s *Service) SyncBgmInfo(c context.Context) (err error) {
defer func() {
task.GetTaskService().SetTaskStatus(c, task.TaskBgmSync, time.Now().AddDate(0, 0, -1).Format(_layout), err)
}()
err = s.delBGM(c)
if err != nil {
return
}
bgms, err := s.getBGM(c)
if err != nil {
return
}
return s.batchInsertBGMs(c, bgms)
}
func (s *Service) delBGM(c context.Context) (err error) {
var limit int64 = 2000
for {
var rows int64
rows, err = s.dao.DelBGM(c, limit)
if err != nil {
return
}
if rows < limit {
break
}
}
return
}

90
app/job/main/growup/service/income/bgm_income.go Normal file
View File

@@ -0,0 +1,90 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// BgmIncomeSvr bgm income service
type BgmIncomeSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewBgmIncomeSvr new bgm income service
func NewBgmIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *BgmIncomeSvr) {
return &BgmIncomeSvr{
batchSize: batchSize,
dao: dao,
}
}
// BatchInsertBgmIncome batch insert bgm income
func (p *BgmIncomeSvr) BatchInsertBgmIncome(ctx context.Context, bm map[int64]map[int64]map[int64]*model.BgmIncome) (err error) {
var (
buff = make([]*model.BgmIncome, p.batchSize)
buffEnd = 0
)
for _, sm := range bm {
for _, bs := range sm {
for _, b := range bs {
buff[buffEnd] = b
buffEnd++
if buffEnd >= p.batchSize {
values := bgmIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertBgmIncome(ctx, values)
if err != nil {
return
}
}
}
}
}
if buffEnd > 0 {
values := bgmIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertBgmIncome(ctx, values)
}
return
}
func bgmIncomeValues(bs []*model.BgmIncome) (values string) {
var buf bytes.Buffer
for _, b := range bs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(b.AID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.SID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.CID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.TaxMoney, 10))
buf.WriteByte(',')
buf.WriteString("'" + b.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.BaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.DailyTotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

92
app/job/main/growup/service/income/bgm_income_stat.go Normal file
View File

@@ -0,0 +1,92 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// BgmIncomeStatSvr bgm income stat service
type BgmIncomeStatSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewBgmIncomeStatSvr new bgm income stat svr
func NewBgmIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *BgmIncomeStatSvr) {
return &BgmIncomeStatSvr{
batchSize: batchSize,
dao: dao,
}
}
// BgmIncomeStat get bgm income stat
func (p *BgmIncomeStatSvr) BgmIncomeStat(c context.Context, limit int64) (m map[int64]*model.BgmIncomeStat, err error) {
m = make(map[int64]*model.BgmIncomeStat)
var id int64
for {
var bm map[int64]*model.BgmIncomeStat
bm, id, err = p.dao.BgmIncomeStat(c, id, limit)
if err != nil {
return
}
if len(bm) == 0 {
break
}
for sid, stat := range bm {
m[sid] = stat
}
}
return
}
// BatchInsertBgmIncomeStat batch insert bgm income stat
func (p *BgmIncomeStatSvr) BatchInsertBgmIncomeStat(c context.Context, bs map[int64]*model.BgmIncomeStat) (err error) {
var (
buff = make([]*model.BgmIncomeStat, p.batchSize)
buffEnd = 0
)
for _, b := range bs {
if b.DataState == 0 {
continue
}
buff[buffEnd] = b
buffEnd++
if buffEnd >= p.batchSize {
values := bgmIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertBgmIncomeStat(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := bgmIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertBgmIncomeStat(c, values)
}
return
}
func bgmIncomeStatValues(bs []*model.BgmIncomeStat) (values string) {
var buf bytes.Buffer
for _, b := range bs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(b.SID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(b.TotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

33
app/job/main/growup/service/income/blacklist.go Normal file
View File

@@ -0,0 +1,33 @@
package income
import (
"context"
)
// Blacklist map[ctype]map[int64]bool
func (s *Service) Blacklist(c context.Context, limit int64) (m map[int]map[int64]bool, err error) {
var id int64
m = make(map[int]map[int64]bool)
for {
var ab map[int][]int64
ab, id, err = s.dao.Blacklist(c, id, limit)
if err != nil {
return
}
if len(ab) == 0 {
break
}
for ctype, avIDs := range ab {
for _, avID := range avIDs {
if _, ok := m[ctype]; ok {
m[ctype][avID] = true
} else {
m[ctype] = map[int64]bool{
avID: true,
}
}
}
}
}
return
}

15
app/job/main/growup/service/income/blacklist_test.go Normal file
View File

@@ -0,0 +1,15 @@
package income
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_Blacklist(t *testing.T) {
Convey("Blacklist", t, func() {
_, err := s.Blacklist(context.Background(), 10)
So(err, ShouldBeNil)
})
}

23
app/job/main/growup/service/income/business_order.go Normal file
View File

@@ -0,0 +1,23 @@
package income
import "context"
// GetBusinessOrders get business orders
func (s *Service) GetBusinessOrders(c context.Context, limit int64) (result map[int64]bool, err error) {
var id int64
result = make(map[int64]bool)
for {
var m map[int64]bool
id, m, err = s.dao.BusinessOrders(c, id, limit)
if err != nil {
return
}
if len(m) == 0 {
break
}
for k, v := range m {
result[k] = v
}
}
return
}

17
app/job/main/growup/service/income/charge.go Normal file
View File

@@ -0,0 +1,17 @@
package income
import (
incomeD "go-common/app/job/main/growup/dao/income"
)
// AvChargeSvr av charge service
type AvChargeSvr struct {
dao *incomeD.Dao
}
// NewAvChargeSvr new av charge service
func NewAvChargeSvr(dao *incomeD.Dao) (svr *AvChargeSvr) {
return &AvChargeSvr{
dao: dao,
}
}

70
app/job/main/growup/service/income/charge_ratio.go Normal file
View File

@@ -0,0 +1,70 @@
package income
import (
"context"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// ChargeRatioSvr charge ratio service
type ChargeRatioSvr struct {
dao *incomeD.Dao
}
// NewChargeRatioSvr new charge ratio service
func NewChargeRatioSvr(dao *incomeD.Dao) *ChargeRatioSvr {
return &ChargeRatioSvr{dao: dao}
}
// ArchiveChargeRatio get av charge ratio
func (p *ChargeRatioSvr) ArchiveChargeRatio(c context.Context, limit int64) (rs map[int]map[int64]*model.ArchiveChargeRatio, err error) {
rs = make(map[int]map[int64]*model.ArchiveChargeRatio)
var id int64
for {
var ros map[int]map[int64]*model.ArchiveChargeRatio
ros, id, err = p.dao.ArchiveChargeRatio(c, id, limit)
if err != nil {
return
}
if len(ros) == 0 {
break
}
for ctype, m := range ros {
if _, ok := rs[ctype]; ok {
for aid, ratio := range m {
rs[ctype][aid] = ratio
}
} else {
rs[ctype] = m
}
}
}
return
}
// UpChargeRatio get up charge ratio
func (p *ChargeRatioSvr) UpChargeRatio(c context.Context, limit int64) (rs map[int]map[int64]*model.UpChargeRatio, err error) {
rs = make(map[int]map[int64]*model.UpChargeRatio)
var id int64
for {
var ros map[int]map[int64]*model.UpChargeRatio
ros, id, err = p.dao.UpChargeRatio(c, id, limit)
if err != nil {
return
}
if len(ros) == 0 {
break
}
for ctype, m := range ros {
if _, ok := rs[ctype]; ok {
for mid, ratio := range m {
rs[ctype][mid] = ratio
}
} else {
rs[ctype] = m
}
}
}
return
}

22
app/job/main/growup/service/income/charge_ratio_test.go Normal file
View File

@@ -0,0 +1,22 @@
package income
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_AvChargeRatio(t *testing.T) {
Convey("AvChargeRatio", t, func() {
_, err := s.ratio.ArchiveChargeRatio(context.Background(), 10)
So(err, ShouldBeNil)
})
}
func Test_UpChargeRatio(t *testing.T) {
Convey("UpChargeRatio", t, func() {
_, err := s.ratio.UpChargeRatio(context.Background(), 10)
So(err, ShouldBeNil)
})
}

28
app/job/main/growup/service/income/column_charge.go Normal file
View File

@@ -0,0 +1,28 @@
package income
import (
"context"
"time"
model "go-common/app/job/main/growup/model/income"
)
func (s *Service) columnCharges(c context.Context, date time.Time, ch chan []*model.ColumnCharge) (err error) {
defer func() {
close(ch)
}()
var id int64
for {
var charges []*model.ColumnCharge
charges, err = s.dao.ColumnDailyCharge(c, date, id, _limitSize)
if err != nil {
return
}
ch <- charges
if len(charges) < _limitSize {
break
}
id = charges[len(charges)-1].ID
}
return
}

88
app/job/main/growup/service/income/column_income.go Normal file
View File

@@ -0,0 +1,88 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// ColumnIncomeSvr column income service
type ColumnIncomeSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewColumnIncomeSvr new income service
func NewColumnIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *ColumnIncomeSvr) {
return &ColumnIncomeSvr{
batchSize: batchSize,
dao: dao,
}
}
// BatchInsertColumnIncome batch insert column income
func (p *ColumnIncomeSvr) BatchInsertColumnIncome(ctx context.Context, cm map[int64][]*model.ColumnIncome) (err error) {
var (
buff = make([]*model.ColumnIncome, p.batchSize)
buffEnd = 0
)
for _, cs := range cm {
for _, c := range cs {
buff[buffEnd] = c
buffEnd++
if buffEnd >= p.batchSize {
values := columnIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertColumnIncome(ctx, values)
if err != nil {
return
}
}
}
}
if buffEnd > 0 {
values := columnIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertColumnIncome(ctx, values)
}
return
}
func columnIncomeValues(cs []*model.ColumnIncome) (values string) {
var buf bytes.Buffer
for _, c := range cs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(c.ArticleID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.TagID, 10))
buf.WriteByte(',')
buf.WriteString("'" + c.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.ViewCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.TaxMoney, 10))
buf.WriteByte(',')
buf.WriteString("'" + c.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.BaseIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

101
app/job/main/growup/service/income/column_income_stat.go Normal file
View File

@@ -0,0 +1,101 @@
package income
import (
"bytes"
"context"
"strconv"
"strings"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// ColumnIncomeStatSvr column income statistics service
type ColumnIncomeStatSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewColumnIncomeStatSvr new column income stat service
func NewColumnIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *ColumnIncomeStatSvr) {
return &ColumnIncomeStatSvr{
batchSize: batchSize,
dao: dao,
}
}
// ColumnIncomeStat column income statistisc
func (p *ColumnIncomeStatSvr) ColumnIncomeStat(c context.Context, limit int64) (m map[int64]*model.ColumnIncomeStat, err error) {
m = make(map[int64]*model.ColumnIncomeStat)
var id int64
for {
var cm map[int64]*model.ColumnIncomeStat
cm, id, err = p.dao.ColumnIncomeStat(c, id, limit)
if err != nil {
return
}
if len(cm) == 0 {
break
}
for sid, stat := range cm {
m[sid] = stat
}
}
return
}
// BatchInsertColumnIncomeStat batch insert column income stat
func (p *ColumnIncomeStatSvr) BatchInsertColumnIncomeStat(ctx context.Context, cs map[int64]*model.ColumnIncomeStat) (err error) {
var (
buff = make([]*model.ColumnIncomeStat, p.batchSize)
buffEnd = 0
)
for _, c := range cs {
if c.DataState == 0 {
continue
}
buff[buffEnd] = c
buffEnd++
if buffEnd >= p.batchSize {
values := columnIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertColumnIncomeStat(ctx, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := columnIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertColumnIncomeStat(ctx, values)
}
return
}
func columnIncomeStatValues(cs []*model.ColumnIncomeStat) (values string) {
var buf bytes.Buffer
for _, c := range cs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(c.ArticleID, 10))
buf.WriteByte(',')
buf.WriteString("\"" + strings.Replace(c.Title, "\"", "\\\"", -1) + "\"")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.TagID, 10))
buf.WriteByte(',')
buf.WriteString("'" + c.UploadTime.Time().Format(_layoutSec) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(c.TotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

38
app/job/main/growup/service/income/common.go Normal file
View File

@@ -0,0 +1,38 @@
package income
import (
"time"
)
const (
_layout = "2006-01-02"
_layoutSec = "2006-01-02 15:04:05"
)
var (
startWeeklyDate time.Time
startMonthlyDate time.Time
)
var (
batchSize = 2000
_dbInsert = 1
_dbUpdate = 2
_video = 0
_column = 2
_bgm = 3
_limitSize = 2000
)
func getStartWeeklyDate(date time.Time) time.Time {
for date.Weekday() != time.Monday {
date = date.AddDate(0, 0, -1)
}
return date
}
func getStartMonthlyDate(date time.Time) time.Time {
return time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
}

963
app/job/main/growup/service/income/income.go Normal file
View File

@@ -0,0 +1,963 @@
package income
import (
"bytes"
"context"
"strconv"
"time"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
xtime "go-common/library/time"
)
// Income income service
type Income struct {
avIncomeSvr *AvIncomeSvr
upIncomeSvr *UpIncomeSvr
avIncomeStatSvr *AvIncomeStatSvr
bgmIncomeStatSvr *BgmIncomeStatSvr
columnIncomeStatSvr *ColumnIncomeStatSvr
upIncomeStatSvr *UpIncomeStatSvr
upAccountSvr *UpAccountSvr
dateStatisSvr *DateStatis
bgmIncomeSvr *BgmIncomeSvr
columnIncomeSvr *ColumnIncomeSvr
}
// NewIncome new income service
func NewIncome(batchSize int, dao *incomeD.Dao) *Income {
return &Income{
avIncomeSvr: NewAvIncomeSvr(dao, batchSize),
bgmIncomeSvr: NewBgmIncomeSvr(dao, batchSize),
columnIncomeSvr: NewColumnIncomeSvr(dao, batchSize),
upIncomeSvr: NewUpIncomeSvr(dao, batchSize),
avIncomeStatSvr: NewAvIncomeStatSvr(dao, batchSize),
bgmIncomeStatSvr: NewBgmIncomeStatSvr(dao, batchSize),
columnIncomeStatSvr: NewColumnIncomeStatSvr(dao, batchSize),
upIncomeStatSvr: NewUpIncomeStatSvr(dao, batchSize),
upAccountSvr: NewUpAccountSvr(dao, batchSize),
dateStatisSvr: NewDateStatis(dao),
}
}
// CalAvIncome cal av income
func (p *Income) CalAvIncome(ch chan []*model.AvCharge, urs map[int64]*model.UpChargeRatio, ars map[int64]*model.ArchiveChargeRatio, filters []AvFilter, signed map[int64]bool) (am map[int64][]*model.AvIncome, total map[int64]*model.UpBusinessIncome) {
am, total = getClassifyAvIncome(ch, ars, filters)
classifyAvIncome(am, urs, ars, total, signed)
return
}
// CalColumnIncome cal column income
func (p *Income) CalColumnIncome(ch chan []*model.ColumnCharge, urs map[int64]*model.UpChargeRatio, ars map[int64]*model.ArchiveChargeRatio, filters []ColumnFilter, signed map[int64]bool) (cm map[int64][]*model.ColumnIncome, total map[int64]*model.UpBusinessIncome) {
cm, total = getClassifyColumnIncome(ch, ars, filters)
classifyColumnIncome(cm, urs, ars, total, signed)
return
}
// CalBgmIncome cal bgm income
func (p *Income) CalBgmIncome(ch chan []*model.AvCharge, bgms map[int64][]*model.BGM, urs map[int64]*model.UpChargeRatio, ars map[int64]*model.ArchiveChargeRatio, avf []AvFilter, exclude BgmFilter, black map[int64]bool, signed map[int64]bool) (bm map[int64]map[int64]map[int64]*model.BgmIncome, total map[int64]*model.UpBusinessIncome) {
bm, total = getClassifyBgmIncome(ch, bgms, avf, exclude, black, ars)
classifyBgmIncome(bm, urs, ars, total, signed)
return
}
func trans2AvIncome(charge *model.AvCharge, incr int64) *model.AvIncome {
return &model.AvIncome{
AvID: charge.AvID,
MID: charge.MID,
TagID: charge.TagID,
IsOriginal: charge.IsOriginal,
UploadTime: charge.UploadTime,
PlayCount: charge.TotalPlayCount,
Date: charge.Date,
Income: incr,
BaseIncome: charge.IncCharge,
}
}
func av2BusinessIncome(a *model.AvIncome) *model.UpBusinessIncome {
return &model.UpBusinessIncome{
MID: a.MID,
Income: a.Income,
PlayCount: a.PlayCount,
AvCount: 1,
Business: 1,
}
}
// av income before tax
// total key:mid, value:total_income
func getClassifyAvIncome(ch chan []*model.AvCharge, ratio map[int64]*model.ArchiveChargeRatio, filters []AvFilter) (am map[int64][]*model.AvIncome, total map[int64]*model.UpBusinessIncome) {
am = make(map[int64][]*model.AvIncome)
total = make(map[int64]*model.UpBusinessIncome)
for charges := range ch {
CHARGE:
for _, charge := range charges {
for _, filter := range filters {
if filter(charge) {
continue CHARGE
}
}
var incr int64
if r, ok := ratio[charge.AvID]; ok {
if r.AdjustType == 0 {
incr = int64(Round(float64(charge.IncCharge)*Div(float64(r.Ratio), float64(100)), 0))
} else if r.AdjustType == 1 {
incr = charge.IncCharge
}
} else {
incr = charge.IncCharge
}
avIncome := trans2AvIncome(charge, incr)
if _, ok := am[charge.MID]; ok {
am[charge.MID] = append(am[charge.MID], avIncome)
} else {
am[charge.MID] = []*model.AvIncome{avIncome}
}
if business, ok := total[charge.MID]; ok {
business.PlayCount += avIncome.PlayCount
business.AvCount++
business.Income += avIncome.Income
} else {
business := av2BusinessIncome(avIncome)
total[charge.MID] = business
}
}
}
return
}
func classifyAvIncome(am map[int64][]*model.AvIncome,
urs map[int64]*model.UpChargeRatio,
ars map[int64]*model.ArchiveChargeRatio,
total map[int64]*model.UpBusinessIncome,
signed map[int64]bool) {
for mid, business := range total {
realIncome := business.Income
if r, ok := urs[mid]; ok {
if r.AdjustType == 0 {
// up主浮动调节,分配到视频业务
realIncome = int64(Round(float64(business.Income)*Div(float64(r.Ratio), float64(100)), 0))
}
}
// up主浮动调节后视频业务的税
tax := int64(Round(Tax(Div(float64(realIncome), 100))*100, 0))
// 税后收入
netIncome := realIncome - tax
// update up income
business.Percent = Div(float64(netIncome), float64(business.Income))
business.Income = netIncome
business.Tax = tax
}
// 计算每个视频浮动调节后收入
for mid, as := range am {
business := total[mid]
// up主的固定调节收入
var patchIncome int64
var c bool
for _, a := range as {
avIncome := int64(Round(Mul(float64(a.Income), business.Percent), 0))
a.Income = avIncome
a.TaxMoney = int64(Round(Mul(float64(business.Tax), Div(float64(avIncome), float64(business.Income))), 0))
// 以下是计算up主基础收入
// original raito: 100
var o int64 = 100
if upRatio, ok := urs[mid]; ok {
if upRatio.AdjustType == 0 {
o = upRatio.Ratio
c = true
}
}
avBaseIncome := int64(Round(Div(float64(avIncome), Div(float64(o), 100)), 0))
a.BaseIncome = avBaseIncome
if r, ok := ars[a.AvID]; ok {
if r.AdjustType == 0 {
// 如果视频收入被加倍过,还原
originIncome := int64(Round(Div(float64(avBaseIncome), Div(float64(r.Ratio), 100)), 0))
a.BaseIncome = originIncome
business.BaseIncome += originIncome
c = true
}
if r.AdjustType == 1 {
// av 固定调节, 更新av的收入和up主的基础收入
a.Income += r.Ratio
// 更新up主的av收入, Income在计算BaseIncome后更新
// upIncome.AvIncome += r.Ratio
// business.Income += r.Ratio
patchIncome += r.Ratio
business.BaseIncome += avBaseIncome
}
} else {
business.BaseIncome += avBaseIncome
}
}
// 如果没有被加倍过, 那么避免误差将BaseIncome直接置为Income
if !c {
business.BaseIncome = business.Income
}
// 最后加上该up主av的总固定调节收入
business.Income += patchIncome
}
// + up主的固定调节
for mid, ratio := range urs {
if _, ok := signed[mid]; !ok {
continue
}
if ratio.AdjustType == 0 {
continue
}
if business, ok := total[mid]; ok {
business.Income += ratio.Ratio
} else {
total[mid] = &model.UpBusinessIncome{
MID: mid,
Income: ratio.Ratio,
Business: 1,
}
}
}
}
/*################################################################## BGM ########################################################################*/
func trans2BgmIncome(b *model.BGM, charge int64, date xtime.Time) *model.BgmIncome {
return &model.BgmIncome{
AID: b.AID,
SID: b.SID,
MID: b.MID,
CID: b.CID,
Income: charge,
Date: date,
}
}
func bgm2BusinessIncome(b *model.BgmIncome) *model.UpBusinessIncome {
return &model.UpBusinessIncome{
MID: b.MID,
Income: b.Income,
Business: 3,
BgmCount: map[int64]bool{
b.SID: true,
},
}
}
// bgms map[avid][sid]*model.BGM | bm map[mid]map[sid]map[avid]*model.BgmIncome
func getClassifyBgmIncome(ch chan []*model.AvCharge, bgms map[int64][]*model.BGM, filters []AvFilter, exclude BgmFilter, black map[int64]bool, ratio map[int64]*model.ArchiveChargeRatio) (bm map[int64]map[int64]map[int64]*model.BgmIncome, total map[int64]*model.UpBusinessIncome) {
bm = make(map[int64]map[int64]map[int64]*model.BgmIncome)
total = make(map[int64]*model.UpBusinessIncome)
for charges := range ch {
CHARGE:
for _, charge := range charges {
for _, filter := range filters {
if filter(charge) {
continue CHARGE
}
}
if bs, ok := bgms[charge.AvID]; ok {
bgmCharge := int64(Round(Div(Mul(float64(charge.IncCharge), float64(0.3)), float64(len(bs))), 0))
for _, b := range bs {
// if av's bgm is own, continue
if b.MID == charge.MID {
continue
}
if _, ok := black[b.SID]; ok {
continue
}
if exclude(charge, b) {
continue
}
var incr int64
if r, ok := ratio[b.SID]; ok {
if r.AdjustType == 0 {
incr = int64(Round(float64(bgmCharge)*Div(float64(r.Ratio), float64(100)), 0))
} else if r.AdjustType == 1 {
incr = bgmCharge
}
} else {
incr = bgmCharge
}
// bm map[mid]map[sid]map[avid]*model.BgmIncome
var bgmIncome *model.BgmIncome
if sm, ok := bm[b.MID]; ok {
if am, ok := sm[b.SID]; ok {
if bgmIncome, ok = am[b.AID]; ok {
bgmIncome.Income += incr
} else {
bgmIncome = trans2BgmIncome(b, incr, charge.Date)
am[b.AID] = bgmIncome
}
} else {
bgmIncome = trans2BgmIncome(b, incr, charge.Date)
sm[b.SID] = map[int64]*model.BgmIncome{
b.AID: bgmIncome,
}
}
} else {
bgmIncome = trans2BgmIncome(b, incr, charge.Date)
// am map[avid]*model.BgmIncome
am := map[int64]*model.BgmIncome{
b.AID: bgmIncome,
}
// sm map[sid]map[avid]*model.BgmIncome
sm := map[int64]map[int64]*model.BgmIncome{
b.SID: am,
}
// bm map[mid]map[sid]map[avid]*model.BgmIncome
bm[b.MID] = sm
}
if business, ok := total[b.MID]; ok {
business.Income += incr
business.BgmCount[b.SID] = true
} else {
business := bgm2BusinessIncome(bgmIncome)
total[b.MID] = business
}
}
}
}
}
return
}
// 算税并分配收入
func classifyBgmIncome(bm map[int64]map[int64]map[int64]*model.BgmIncome,
urs map[int64]*model.UpChargeRatio,
ars map[int64]*model.ArchiveChargeRatio,
total map[int64]*model.UpBusinessIncome,
signed map[int64]bool) {
for mid, business := range total {
if business.Income == 0 {
delete(total, mid)
delete(bm, mid)
continue
}
realIncome := business.Income
if r, ok := urs[mid]; ok {
if r.AdjustType == 0 {
// up主浮动调节,分配到视频业务
realIncome = int64(Round(float64(business.Income)*Div(float64(r.Ratio), float64(100)), 0))
}
}
// up主浮动调节后视频业务的税
tax := int64(Round(Tax(Div(float64(realIncome), 100))*100, 0))
// 税后收入
netIncome := realIncome - tax
// update up income
business.Percent = Div(float64(netIncome), float64(business.Income))
business.Income = netIncome
business.Tax = tax
}
// bm map[mid]map[sid]map[avid]*model.BgmIncome
for mid, sm := range bm {
business := total[mid]
var c bool
var patchIncome int64
for sid, bs := range sm {
var dailyTotalIncome int64
for _, b := range bs {
income := int64(Round(Mul(float64(b.Income), business.Percent), 0))
b.Income = income
b.TaxMoney = int64(Round(Mul(float64(business.Tax), Div(float64(income), float64(business.Income))), 0))
dailyTotalIncome += b.Income
// 以下是计算up主基础收入
// original raito: 100
var o int64 = 100
if upRatio, ok := urs[mid]; ok {
if upRatio.AdjustType == 0 {
o = upRatio.Ratio
c = true
}
}
bgmBaseIncome := int64(Round(Div(float64(income), Div(float64(o), 100)), 0))
b.BaseIncome = bgmBaseIncome
if r, ok := ars[sid]; ok {
if r.AdjustType == 0 {
// 如果bgm收入被加倍过,还原
originIncome := int64(Round(Div(float64(bgmBaseIncome), Div(float64(r.Ratio), 100)), 0))
b.BaseIncome = originIncome
business.BaseIncome += originIncome
c = true
} else {
business.BaseIncome += bgmBaseIncome
}
} else {
business.BaseIncome += bgmBaseIncome
}
}
if r, ok := ars[sid]; ok {
if r.AdjustType == 1 {
patchIncome += r.Ratio
dailyTotalIncome += r.Ratio
}
}
// update bgm daily total income
for _, b := range bs {
b.DailyTotalIncome = dailyTotalIncome
}
}
if !c {
business.BaseIncome = business.Income
}
// 最后加上该up主bgm的总固定调节收入
business.Income += patchIncome
}
// + up主的固定调节
for mid, ratio := range urs {
if _, ok := signed[mid]; !ok {
continue
}
if ratio.AdjustType == 0 {
continue
}
if business, ok := total[mid]; ok {
business.Income += ratio.Ratio
} else {
total[mid] = &model.UpBusinessIncome{
MID: mid,
Income: ratio.Ratio,
Business: 3,
}
}
}
}
/*###################################################### Column #######################################################*/
func trans2ColumnIncome(charge *model.ColumnCharge, incr int64) (c *model.ColumnIncome) {
return &model.ColumnIncome{
ArticleID: charge.ArticleID,
Title: charge.Title,
MID: charge.MID,
TagID: charge.TagID,
UploadTime: charge.UploadTime,
ViewCount: charge.IncViewCount,
Date: charge.Date,
Income: incr,
BaseIncome: charge.IncCharge,
}
}
// business type 2: column
func column2BusinessIncome(c *model.ColumnIncome) *model.UpBusinessIncome {
return &model.UpBusinessIncome{
MID: c.MID,
Income: c.Income,
ViewCount: c.ViewCount,
ColumnCount: 1,
Business: 2,
}
}
func getClassifyColumnIncome(ch chan []*model.ColumnCharge, ratio map[int64]*model.ArchiveChargeRatio, filters []ColumnFilter) (cm map[int64][]*model.ColumnIncome, total map[int64]*model.UpBusinessIncome) {
cm = make(map[int64][]*model.ColumnIncome)
total = make(map[int64]*model.UpBusinessIncome)
for charges := range ch {
CHARGE:
for _, charge := range charges {
for _, filter := range filters {
if filter(charge) {
continue CHARGE
}
}
var incr int64
if r, ok := ratio[charge.ArticleID]; ok {
if r.AdjustType == 0 {
incr = int64(Round(float64(charge.IncCharge)*Div(float64(r.Ratio), float64(100)), 0))
} else if r.AdjustType == 1 {
incr = charge.IncCharge
}
} else {
incr = charge.IncCharge
}
columnIncome := trans2ColumnIncome(charge, incr)
if _, ok := cm[charge.MID]; ok {
cm[charge.MID] = append(cm[charge.MID], columnIncome)
} else {
cm[charge.MID] = []*model.ColumnIncome{columnIncome}
}
if business, ok := total[charge.MID]; ok {
business.ViewCount += columnIncome.ViewCount
business.ColumnCount++
business.Income += columnIncome.Income
} else {
business := column2BusinessIncome(columnIncome)
total[charge.MID] = business
}
}
}
return
}
// calculate column income and archive income
func classifyColumnIncome(cm map[int64][]*model.ColumnIncome,
urs map[int64]*model.UpChargeRatio,
ars map[int64]*model.ArchiveChargeRatio,
total map[int64]*model.UpBusinessIncome,
signed map[int64]bool) {
for mid, business := range total {
realIncome := business.Income
if r, ok := urs[mid]; ok {
if r.AdjustType == 0 {
// up主浮动调节,分配到视频业务
realIncome = int64(Round(float64(business.Income)*Div(float64(r.Ratio), float64(100)), 0))
}
}
// up主浮动调节后视频业务的税
tax := int64(Round(Tax(Div(float64(realIncome), 100))*100, 0))
// 税后收入
netIncome := realIncome - tax
// update up income
business.Percent = Div(float64(netIncome), float64(business.Income))
business.Income = netIncome
business.Tax = tax
}
for mid, cs := range cm {
business := total[mid]
var x bool
var patchIncome int64
for _, c := range cs {
columnIncome := int64(Round(Mul(float64(c.Income), business.Percent), 0))
c.Income = columnIncome
c.TaxMoney = int64(Round(Mul(float64(business.Tax), Div(float64(columnIncome), float64(business.Income))), 0))
// 以下是计算up主基础收入
// original raito: 100
var o int64 = 100
if upRatio, ok := urs[mid]; ok {
if upRatio.AdjustType == 0 {
o = upRatio.Ratio
x = true
}
}
columnBaseIncome := int64(Round(Div(float64(columnIncome), Div(float64(o), 100)), 0))
c.BaseIncome = columnBaseIncome
if r, ok := ars[c.ArticleID]; ok {
if r.AdjustType == 0 {
// 如果视频收入被加倍过,还原
originIncome := int64(Round(Div(float64(columnBaseIncome), Div(float64(r.Ratio), 100)), 0))
c.BaseIncome = originIncome
business.BaseIncome += originIncome
x = true
}
if r.AdjustType == 1 {
// column 固定调节, 更新column 的收入和up主的基础收入
c.Income += r.Ratio
// 更新up主的column 收入, Income在计算BaseIncome后更新
patchIncome += r.Ratio
business.BaseIncome += columnBaseIncome
}
} else {
business.BaseIncome += columnBaseIncome
}
}
if !x {
business.BaseIncome = business.Income
}
// 最后加上该up主column的总固定调节收入
business.Income += patchIncome
}
// + up主的固定调节
for mid, ratio := range urs {
if _, ok := signed[mid]; !ok {
continue
}
if ratio.AdjustType == 0 {
continue
}
if business, ok := total[mid]; ok {
business.Income += ratio.Ratio
} else {
total[mid] = &model.UpBusinessIncome{
MID: mid,
Income: ratio.Ratio,
Business: 2,
}
}
}
}
/**######################################################## UpIncome ###################################################################**/
// CalUpIncome calculate upIncome
func (p *Income) CalUpIncome(ch chan map[int64]*model.UpBusinessIncome, date time.Time) (m map[int64]*model.UpIncome) {
defer close(ch)
m = make(map[int64]*model.UpIncome)
var finished int
for bm := range ch {
for mid, business := range bm {
if upIncome, ok := m[mid]; ok {
if business.Business == 1 {
upIncome.AvCount = business.AvCount
upIncome.PlayCount = business.PlayCount
upIncome.AvIncome = business.Income
upIncome.AvBaseIncome = business.BaseIncome
upIncome.AvTax = business.Tax
}
if business.Business == 2 {
upIncome.ColumnCount = business.ColumnCount
upIncome.ColumnIncome = business.Income
upIncome.ColumnBaseIncome = business.BaseIncome
upIncome.ColumnTax = business.Tax
}
if business.Business == 3 {
upIncome.BgmIncome = business.Income
upIncome.BgmBaseIncome = business.BaseIncome
upIncome.BgmTax = business.Tax
upIncome.BgmCount = int64(len(business.BgmCount))
}
upIncome.TaxMoney += business.Tax
upIncome.BaseIncome += business.BaseIncome
upIncome.Income += business.Income
} else {
var upIncome *model.UpIncome
if business.Business == 1 {
upIncome = &model.UpIncome{
AvCount: business.AvCount,
PlayCount: business.PlayCount,
AvIncome: business.Income,
AvBaseIncome: business.BaseIncome,
AvTax: business.Tax,
}
}
if business.Business == 2 {
upIncome = &model.UpIncome{
ColumnCount: business.ColumnCount,
ColumnIncome: business.Income,
ColumnBaseIncome: business.BaseIncome,
ColumnTax: business.Tax,
}
}
if business.Business == 3 {
upIncome = &model.UpIncome{
BgmIncome: business.Income,
BgmBaseIncome: business.BaseIncome,
BgmTax: business.Tax,
BgmCount: int64(len(business.BgmCount)),
}
}
upIncome.MID = mid
upIncome.TaxMoney = business.Tax
upIncome.BaseIncome = business.BaseIncome
upIncome.Income = business.Income
upIncome.Date = xtime.Time(date.Unix())
m[mid] = upIncome
}
}
finished++
if finished == 3 {
break
}
}
return
}
// IncomeStat income statistics
func (p *Income) IncomeStat(
um map[int64]*model.UpIncome,
am map[int64][]*model.AvIncome,
bm map[int64]map[int64]map[int64]*model.BgmIncome,
cm map[int64][]*model.ColumnIncome,
ustat map[int64]*model.UpIncomeStat,
astat map[int64]*model.AvIncomeStat,
bstat map[int64]*model.BgmIncomeStat,
cstat map[int64]*model.ColumnIncomeStat) {
for mid, upIncome := range um {
// update up income total income
ut, ok := ustat[mid]
if !ok {
ut = &model.UpIncomeStat{
MID: upIncome.MID,
DataState: 1, // insert
}
ustat[mid] = ut
} else {
ut.DataState = 2 // update
}
// update up income statis up total income
// up av total income
// up column total income
// up bgm total income
ut.TotalIncome += upIncome.Income
upIncome.TotalIncome = ut.TotalIncome
ut.AvTotalIncome += upIncome.AvIncome
upIncome.AvTotalIncome = ut.AvTotalIncome
ut.ColumnTotalIncome += upIncome.ColumnIncome
upIncome.ColumnTotalIncome = ut.ColumnTotalIncome
ut.BgmTotalIncome += upIncome.BgmIncome
upIncome.BgmTotalIncome = ut.BgmTotalIncome
// update av income total income
for _, a := range am[mid] {
at, ok := astat[a.AvID]
if !ok {
at = &model.AvIncomeStat{
AvID: a.AvID,
MID: a.MID,
TagID: a.TagID,
IsOriginal: a.IsOriginal,
UploadTime: a.UploadTime,
DataState: 1, // insert
}
astat[a.AvID] = at
} else {
at.DataState = 2 // update
}
at.TotalIncome += a.Income
a.TotalIncome = at.TotalIncome
}
// update bgmIncome total income
for sid, am := range bm[mid] {
bt, ok := bstat[sid]
if !ok {
bt = &model.BgmIncomeStat{
SID: sid,
DataState: 1, // insert
}
bstat[sid] = bt
} else {
bt.DataState = 2 // update
}
for _, b := range am {
bt.TotalIncome += b.DailyTotalIncome
break
}
for _, b := range am {
b.TotalIncome = bt.TotalIncome
}
}
// update columnIncome total income
for _, c := range cm[mid] {
ct, ok := cstat[c.ArticleID]
if !ok {
ct = &model.ColumnIncomeStat{
ArticleID: c.ArticleID,
Title: c.Title,
TagID: c.TagID,
MID: c.MID,
UploadTime: c.UploadTime,
DataState: 1, // insert
}
cstat[c.ArticleID] = ct
} else {
ct.DataState = 2 // update
}
ct.TotalIncome += c.Income
c.TotalIncome = ct.TotalIncome
}
}
}
// PurgeUpAccount purge up account
func (p *Income) PurgeUpAccount(date time.Time, accs map[int64]*model.UpAccount, um map[int64]*model.UpIncome) {
fd := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
last := fd.AddDate(0, 0, -1).Format("2006-01")
// TODO check
for mid, upIncome := range um {
// update old up account
if upAccount, ok := accs[mid]; ok {
upAccount.TotalIncome += upIncome.Income
if upAccount.WithdrawDateVersion == last {
upAccount.TotalUnwithdrawIncome += upIncome.Income
}
upAccount.DataState = 2
} else { // append new up account
accs[mid] = &model.UpAccount{
MID: mid,
HasSignContract: 1,
TotalIncome: upIncome.Income,
TotalUnwithdrawIncome: upIncome.Income,
WithdrawDateVersion: last,
DataState: 1,
}
}
}
}
/************************************************************ FOR HISTORY ****************************************************************/
// UpdateBusinessIncomeByDate update business income by date
func (p *Income) UpdateBusinessIncomeByDate(c context.Context, date string) (err error) {
ustat, err := p.upIncomeStatSvr.UpIncomeStat(c, int64(_limitSize))
if err != nil {
return
}
return p.updateBusinessIncomeByDate(c, date, ustat)
}
func (p *Income) updateBusinessIncomeByDate(c context.Context, date string, ustat map[int64]*model.UpIncomeStat) (err error) {
us, err := p.businessTotalIncome(c, ustat, date)
if err != nil {
return
}
err = p.batchUpdateUpIncome(c, us)
if err != nil {
return
}
return p.batchUpdateUpIncomeStat(c, ustat)
}
// for history data m: map[mid]map[date]*model.UpIncome
func (p *Income) businessTotalIncome(c context.Context, ustat map[int64]*model.UpIncomeStat, date string) (m []*model.UpIncome, err error) {
var id int64
for {
var ups []*model.UpIncome
ups, err = p.upIncomeSvr.dao.GetUpIncomeTable(c, "up_income", date, id, 2000)
if err != nil {
return
}
for _, up := range ups {
if ut, ok := ustat[up.MID]; ok {
ut.AvTotalIncome += up.AvIncome
up.AvTotalIncome = ut.AvTotalIncome
ut.ColumnTotalIncome += up.ColumnIncome
up.ColumnTotalIncome = ut.ColumnTotalIncome
ut.BgmTotalIncome += up.BgmIncome
up.BgmTotalIncome = ut.BgmTotalIncome
m = append(m, up)
}
}
if len(ups) < 2000 {
break
}
id = ups[len(ups)-1].ID
}
return
}
// BatchUpdateUpIncome insert up_income batch
func (p *Income) batchUpdateUpIncome(c context.Context, us []*model.UpIncome) (err error) {
var (
buff = make([]*model.UpIncome, p.upIncomeSvr.batchSize)
buffEnd = 0
)
for _, u := range us {
buff[buffEnd] = u
buffEnd++
if buffEnd >= p.upIncomeSvr.batchSize {
values := businessValues(buff[:buffEnd])
buffEnd = 0
_, err = p.upIncomeSvr.dao.FixInsertUpIncome(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := businessValues(buff[:buffEnd])
buffEnd = 0
_, err = p.upIncomeSvr.dao.FixInsertUpIncome(c, values)
}
return
}
func businessValues(us []*model.UpIncome) (values string) {
var buf bytes.Buffer
for _, u := range us {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(u.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}
// BatchUpdateUpIncome insert up_income batch
func (p *Income) batchUpdateUpIncomeStat(c context.Context, us map[int64]*model.UpIncomeStat) (err error) {
var (
buff = make([]*model.UpIncomeStat, p.upIncomeStatSvr.batchSize)
buffEnd = 0
)
for _, u := range us {
buff[buffEnd] = u
buffEnd++
if buffEnd >= p.upIncomeStatSvr.batchSize {
values := businessStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.upIncomeStatSvr.dao.FixInsertUpIncomeStat(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := businessStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.upIncomeStatSvr.dao.FixInsertUpIncomeStat(c, values)
}
return
}
func businessStatValues(ustat []*model.UpIncomeStat) (values string) {
var buf bytes.Buffer
for _, u := range ustat {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(u.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

272
app/job/main/growup/service/income/income_date_statis.go Normal file
View File

@@ -0,0 +1,272 @@
package income
import (
"bytes"
"context"
"fmt"
"math"
"strconv"
"time"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
"go-common/library/log"
xtime "go-common/library/time"
)
var (
_avIncomeDailyStatis = "av_income_daily_statis"
_avIncomeWeeklyStatis = "av_income_weekly_statis"
_avIncomeMonthlyStatis = "av_income_monthly_statis"
_cmIncomeDailyStatis = "column_income_daily_statis"
_cmIncomeWeeklyStatis = "column_income_weekly_statis"
_cmIncomeMonthlyStatis = "column_income_monthly_statis"
_bgmIncomeDailyStatis = "bgm_income_daily_statis"
_bgmIncomeWeeklyStatis = "bgm_income_weekly_statis"
_bgmIncomeMonthlyStatis = "bgm_income_monthly_statis"
)
// SectionEntries section entries
type SectionEntries struct {
avDaily []*model.DateStatis
avWeekly []*model.DateStatis
avMonthly []*model.DateStatis
}
// DateStatis income date statistics
type DateStatis struct {
dao *incomeD.Dao
}
// NewDateStatis new income date statistics service
func NewDateStatis(dao *incomeD.Dao) *DateStatis {
return &DateStatis{dao: dao}
}
func initIncomeSections(income, tagID int64, date xtime.Time) []*model.DateStatis {
incomeSections := make([]*model.DateStatis, 12)
incomeSections[0] = initIncomeSection(0, 1, 0, income, tagID, date)
incomeSections[1] = initIncomeSection(1, 5, 1, income, tagID, date)
incomeSections[2] = initIncomeSection(5, 10, 2, income, tagID, date)
incomeSections[3] = initIncomeSection(10, 30, 3, income, tagID, date)
incomeSections[4] = initIncomeSection(30, 50, 4, income, tagID, date)
incomeSections[5] = initIncomeSection(50, 100, 5, income, tagID, date)
incomeSections[6] = initIncomeSection(100, 200, 6, income, tagID, date)
incomeSections[7] = initIncomeSection(200, 500, 7, income, tagID, date)
incomeSections[8] = initIncomeSection(500, 1000, 8, income, tagID, date)
incomeSections[9] = initIncomeSection(1000, 3000, 9, income, tagID, date)
incomeSections[10] = initIncomeSection(3000, 5000, 10, income, tagID, date)
incomeSections[11] = initIncomeSection(5000, math.MaxInt32, 11, income, tagID, date)
return incomeSections
}
func initIncomeSection(min, max, section, income, tagID int64, date xtime.Time) *model.DateStatis {
var tips string
if max == math.MaxInt32 {
tips = fmt.Sprintf("\"%d+\"", min)
} else {
tips = fmt.Sprintf("\"%d~%d\"", min, max)
}
return &model.DateStatis{
MinIncome: min,
MaxIncome: max,
MoneySection: section,
MoneyTips: tips,
Income: income,
CategoryID: tagID,
CDate: date,
}
}
func (s *DateStatis) handleDateStatis(c context.Context, archiveCh chan []*model.ArchiveIncome, date time.Time, table string) (incomeSections []*model.DateStatis, err error) {
// delete
if table != "" {
_, err = s.dao.DelIncomeStatisTable(c, table, date.Format(_layout))
if err != nil {
log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
return
}
}
// add
incomeSections = s.handleArchives(c, archiveCh, date)
return
}
// handleArchives handle archive_income_daily_statis, archive_income_weekly_statis, archive_income_monthly_statis
func (s *DateStatis) handleArchives(c context.Context, archiveCh chan []*model.ArchiveIncome, date time.Time) (incomeSections []*model.DateStatis) {
archTagMap := make(map[int64]map[int64]int64) // key TagID, value map[int64]int64 -> key aid, value income
tagIncomeMap := make(map[int64]int64) // key TagID, value TagID total income
for archive := range archiveCh {
handleArchive(archive, archTagMap, tagIncomeMap, date)
}
incomeSections = make([]*model.DateStatis, 0)
for tagID, avMap := range archTagMap {
incomeSection := countIncomeDailyStatis(avMap, tagIncomeMap[tagID], tagID, date)
incomeSections = append(incomeSections, incomeSection...)
}
return
}
func handleArchive(archives []*model.ArchiveIncome, archTagMap map[int64]map[int64]int64, tagIncomeMap map[int64]int64, startDate time.Time) {
if archives == nil {
return
}
if archTagMap == nil {
archTagMap = make(map[int64]map[int64]int64)
}
if tagIncomeMap == nil {
tagIncomeMap = make(map[int64]int64)
}
for _, archive := range archives {
if !startDate.After(archive.Date.Time()) {
tagIncomeMap[archive.TagID] += archive.Income
if _, ok := archTagMap[archive.TagID]; !ok {
archTagMap[archive.TagID] = make(map[int64]int64)
}
archTagMap[archive.TagID][archive.AID] += archive.Income
}
}
}
func (s *DateStatis) handleDateUp(c context.Context, upStatisCh chan []*model.UpIncome, date time.Time) (upSections, upAvSections, upCmSections, upBgmSections []*model.DateStatis, err error) {
_, err = s.dao.DelIncomeStatisTable(c, "up_income_daily_statis", date.Format(_layout))
if err != nil {
log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
return
}
_, err = s.dao.DelIncomeStatisTable(c, "up_av_daily_statis", date.Format(_layout))
if err != nil {
log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
return
}
_, err = s.dao.DelIncomeStatisTable(c, "up_column_daily_statis", date.Format(_layout))
if err != nil {
log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
return
}
_, err = s.dao.DelIncomeStatisTable(c, "up_bgm_daily_statis", date.Format(_layout))
if err != nil {
log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
return
}
upMap := make(map[int64]int64)
upAvMap := make(map[int64]int64)
upCmMap := make(map[int64]int64)
upBgmMap := make(map[int64]int64)
var upTotal, avTotal, cmTotal, bgmTotal int64
for up := range upStatisCh {
up, av, cm, bgm := handleUp(up, upMap, upAvMap, upCmMap, upBgmMap, date)
upTotal += up
avTotal += av
cmTotal += cm
bgmTotal += bgm
}
upSections = countIncomeDailyStatis(upMap, upTotal, 0, date)
upAvSections = countIncomeDailyStatis(upAvMap, avTotal, 0, date)
upCmSections = countIncomeDailyStatis(upCmMap, cmTotal, 0, date)
upBgmSections = countIncomeDailyStatis(upBgmMap, bgmTotal, 0, date)
return
}
func handleUp(upIncomes []*model.UpIncome, upMap, upAvMap, upCmMap, upBgmMap map[int64]int64, startDate time.Time) (income, avIncome, cmIncome, bgmIncome int64) {
if len(upIncomes) == 0 {
return
}
for _, upIncome := range upIncomes {
if startDate.Equal(upIncome.Date.Time()) {
income += upIncome.Income
avIncome += upIncome.AvIncome
cmIncome += upIncome.ColumnIncome
bgmIncome += upIncome.BgmIncome
upMap[upIncome.MID] += upIncome.Income
if upIncome.AvIncome > 0 {
upAvMap[upIncome.MID] += upIncome.AvIncome
}
if upIncome.ColumnIncome > 0 {
upCmMap[upIncome.MID] += upIncome.ColumnIncome
}
if upIncome.BgmIncome > 0 {
upBgmMap[upIncome.MID] += upIncome.BgmIncome
}
}
}
return
}
func (s *DateStatis) incomeDateStatisInsert(c context.Context, incomeSection []*model.DateStatis, table string) (rows int64, err error) {
var buf bytes.Buffer
for _, row := range incomeSection {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.Count, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MoneySection, 10))
buf.WriteByte(',')
buf.WriteString(row.MoneyTips)
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.CategoryID, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.CDate.Time().Format(_layout) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals := buf.String()
buf.Reset()
rows, err = s.dao.InsertIncomeStatisTable(c, table, vals)
return
}
func countIncomeDailyStatis(incomes map[int64]int64, totalIncome, tagID int64, date time.Time) (incomeSections []*model.DateStatis) {
if len(incomes) == 0 {
return
}
incomeSections = initIncomeSections(totalIncome, tagID, xtime.Time(date.Unix()))
for _, income := range incomes {
for _, section := range incomeSections {
min, max := section.MinIncome*100, section.MaxIncome*100
if income >= min && income < max {
section.Count++
}
}
}
return
}
func (s *DateStatis) upIncomeDailyStatisInsert(c context.Context, upIncomeSection []*model.DateStatis, table string) (rows int64, err error) {
var buf bytes.Buffer
for _, row := range upIncomeSection {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(row.Count, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.MoneySection, 10))
buf.WriteByte(',')
buf.WriteString(row.MoneyTips)
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(row.Income, 10))
buf.WriteByte(',')
buf.WriteString("'" + row.CDate.Time().Format(_layout) + "'")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals := buf.String()
buf.Reset()
rows, err = s.dao.InsertUpIncomeDailyStatis(c, table, vals)
return
}

25
app/job/main/growup/service/income/lottery.go Normal file
View File

@@ -0,0 +1,25 @@
package income
import (
"context"
)
// GetBubbleMeta .
func (s *Service) GetBubbleMeta(c context.Context) (data map[int64][]int, err error) {
var id int64
data = make(map[int64][]int)
for {
var meta map[int64][]int
meta, id, err = s.dao.GetBubbleMeta(c, id, int64(_limitSize))
if err != nil {
return
}
if len(meta) == 0 {
break
}
for avID, bTypes := range meta {
data[avID] = append(data[avID], bTypes...)
}
}
return
}

538
app/job/main/growup/service/income/run.go Normal file
View File

@@ -0,0 +1,538 @@
package income
import (
"context"
"fmt"
"sort"
"time"
model "go-common/app/job/main/growup/model/income"
task "go-common/app/job/main/growup/service"
"go-common/library/log"
"golang.org/x/sync/errgroup"
)
// RunAndSendMail run and send email
func (s *Service) RunAndSendMail(c context.Context, date time.Time) (err error) {
var mailReceivers []string
var msg string
for _, v := range s.conf.Mail.Send {
if v.Type == 3 {
mailReceivers = v.Addr
}
}
startTime := time.Now().Unix()
err = s.run(c, date)
if err != nil {
msg = err.Error()
mailReceivers = []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"}
} else {
msg = fmt.Sprintf("%s 计算完成，耗时%ds", date.Format("2006-01-02"), time.Now().Unix()-startTime)
}
emailErr := s.email.SendMail(date, msg, "创作激励每日计算%d年%d月%d日", mailReceivers...)
if emailErr != nil {
log.Error("s.email.SendMail error(%v)", emailErr)
}
return
}
func (s *Service) run(c context.Context, date time.Time) (err error) {
defer func() {
task.GetTaskService().SetTaskStatus(c, task.TaskCreativeIncome, date.Format(_layout), err)
}()
err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskAvCharge, task.TaskCmCharge, task.TaskTagRatio, task.TaskBubbleMeta, task.TaskBlacklist, task.TaskBgmSync)
if err != nil {
return
}
startWeeklyDate = getStartWeeklyDate(date)
startMonthlyDate = getStartMonthlyDate(date)
/*################ Serializable Begin ################*/
// av charge ratio
ratios, err := s.ratio.ArchiveChargeRatio(c, int64(_limitSize))
if err != nil {
return
}
// up charge ratio
urs, err := s.ratio.UpChargeRatio(c, int64(_limitSize))
if err != nil {
return
}
// av income statistics
astat, err := s.income.avIncomeStatSvr.AvIncomeStat(c, int64(_limitSize))
if err != nil {
log.Error("s.income.avIncomeStatSvr.AvIncomeStat error(%v) ", err)
return
}
log.Info("get av_income_statis : %d", len(astat))
// bgm income statistics
bstat, err := s.income.bgmIncomeStatSvr.BgmIncomeStat(c, int64(_limitSize))
if err != nil {
log.Error("s.income.bgmIncomeStatSvr.BgmIncomeStat error(%v) ", err)
return
}
log.Info("get bgm_income_statis : %d", len(bstat))
// column income statistics
cstat, err := s.income.columnIncomeStatSvr.ColumnIncomeStat(c, int64(_limitSize))
if err != nil {
log.Error("s.income.columnIncomeStatSvr.ColumnIncomeStat error(%v) ", err)
return
}
log.Info("get column_income_statis : %d", len(cstat))
// up income statistics
ustat, err := s.income.upIncomeStatSvr.UpIncomeStat(c, int64(_limitSize))
if err != nil {
log.Error("s.income.upIncomeStatSvr.UpIncomeStat error(%v) ", err)
return
}
log.Info("get up_income_statis : %d", len(ustat))
// up accounts
accs, err := s.income.upAccountSvr.UpAccount(c, int64(_limitSize))
if err != nil {
log.Error("s.income.upAccountSvr.UpAccount error(%v)", err)
return
}
log.Info("get up_account : %d", len(accs))
// bubble meta
bubbleMeta, err := s.GetBubbleMeta(c)
if err != nil {
log.Error("s.GetBubbleMeta error(%v)", err)
return
}
log.Info("get lottery_av_info avids: %d", len(bubbleMeta))
bubbleRatio := s.avToBubbleRatio(bubbleMeta)
// av signed ups
var (
avFilters []AvFilter
columnFilters []ColumnFilter
bgmFilter BgmFilter
)
//black list
blacks, err := s.Blacklist(c, 2000)
if err != nil {
return
}
// black ctype 0: av
avBlackFilter := avFilter(blacks[0])
// black ctype 2: column
columnBlackFilter := columnFilter(blacks[2])
// business orders
bos, err := s.GetBusinessOrders(c, 2000)
if err != nil {
return
}
bosFilter := avFilter(bos)
// signed up
signed := make(map[int64]bool)
signedAv, err := s.Signed(c, "video", 2000)
if err != nil {
return
}
savf := signedAvFilter(signedAv, date)
for mid := range signedAv {
signed[mid] = true
}
signedColumn, err := s.Signed(c, "column", 2000)
if err != nil {
return
}
for mid := range signedColumn {
signed[mid] = true
}
signedBgm, err := s.Signed(c, "bgm", 2000)
if err != nil {
return
}
for mid := range signedBgm {
signed[mid] = true
}
bgms, err := s.BGMs(c, 2000)
if err != nil {
return
}
{
avFilters = append(avFilters, avBlackFilter)
avFilters = append(avFilters, bosFilter)
avFilters = append(avFilters, savf)
bgmFilter = signedBgmFilter(signedBgm, date)
columnFilters = append(columnFilters, signedColumnFilter(signedColumn, date))
columnFilters = append(columnFilters, columnBlackFilter)
}
/*################ Serializable End ##################*/
var (
readGroup errgroup.Group
sourceCh = make(chan []*model.AvCharge, 1000)
// av
incomeCh = make(chan []*model.AvCharge, 1000)
// bgm
bgmCh = make(chan []*model.AvCharge, 1000)
// column
columnSourceCh = make(chan []*model.ColumnCharge, 1000)
// business income
businessCh = make(chan map[int64]*model.UpBusinessIncome, 10)
)
// get av daily charge and repost to other channels
readGroup.Go(func() (err error) {
err = s.avCharge.AvCharges(c, date, sourceCh, bubbleRatio)
if err != nil {
log.Error("s.avCharge.AvCharges error(%v)", err)
return
}
log.Info("av_daily_charge finished")
return
})
// get column daily charge
readGroup.Go(func() (err error) {
err = s.columnCharges(c, date, columnSourceCh)
if err != nil {
log.Error("s.columnCharges error(%v)", err)
return
}
log.Info("column_daily_charge finished")
return
})
readGroup.Go(func() (err error) {
defer func() {
close(incomeCh)
close(bgmCh)
}()
for charges := range sourceCh {
incomeCh <- charges
bgmCh <- charges
}
return
})
// up and av income compute
var (
um map[int64]*model.UpIncome
am map[int64][]*model.AvIncome
bm map[int64]map[int64]map[int64]*model.BgmIncome
cm map[int64][]*model.ColumnIncome
)
readGroup.Go(func() (err error) {
//um, am = s.income.Compute(c, date, incomeCh, urs, ars, ustat, astat, accs, filters, signed)
var business map[int64]*model.UpBusinessIncome
am, business = s.income.CalAvIncome(incomeCh, urs[1], ratios[1], avFilters, signed)
businessCh <- business
return
})
readGroup.Go(func() (err error) {
var business map[int64]*model.UpBusinessIncome
bm, business = s.income.CalBgmIncome(bgmCh, bgms, urs[3], ratios[3], avFilters, bgmFilter, blacks[3], signed)
businessCh <- business
return
})
readGroup.Go(func() (err error) {
var business map[int64]*model.UpBusinessIncome
cm, business = s.income.CalColumnIncome(columnSourceCh, urs[2], ratios[2], columnFilters, signed)
businessCh <- business
return
})
readGroup.Go(func() (err error) {
um = s.income.CalUpIncome(businessCh, date)
s.income.IncomeStat(um, am, bm, cm, ustat, astat, bstat, cstat)
s.income.PurgeUpAccount(date, accs, um)
return
})
if err = readGroup.Wait(); err != nil {
log.Error("run readGroup.Wait error(%v)", err)
return
}
// security verification
{
if len(am) == 0 {
err = fmt.Errorf("Error: insert 0 av_income")
return
}
if len(bm) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_income")
return
}
if len(cm) == 0 {
err = fmt.Errorf("Error: insert 0 column_income")
return
}
if len(um) == 0 {
err = fmt.Errorf("Error: insert 0 up_income")
return
}
if len(astat) == 0 {
err = fmt.Errorf("Error: insert 0 av_income_statis")
return
}
if len(bstat) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_income_statis")
return
}
if len(cstat) == 0 {
err = fmt.Errorf("Error: insert 0 column_income_statis")
return
}
if len(ustat) == 0 {
err = fmt.Errorf("Error: insert 0 up_income_statis")
return
}
if len(accs) == 0 {
err = fmt.Errorf("Error: insert 0 up_account")
return
}
}
// persistent
var writeGroup errgroup.Group
// av_income
writeGroup.Go(func() (err error) {
err = s.income.avIncomeSvr.BatchInsertAvIncome(c, am)
if err != nil {
log.Error("s.income.BatchInsertAvIncome error(%v)", err)
return
}
log.Info("insert av_income : %d", len(am))
return
})
// column_income
writeGroup.Go(func() (err error) {
err = s.income.columnIncomeSvr.BatchInsertColumnIncome(c, cm)
if err != nil {
log.Error("s.income.BatchInsertColumnIncome error(%v)", err)
return
}
log.Info("insert column_income : %d", len(cm))
return
})
// bgm_income
writeGroup.Go(func() (err error) {
err = s.income.bgmIncomeSvr.BatchInsertBgmIncome(c, bm)
if err != nil {
log.Error("s.income.BatchInsertBgmIncome error(%v)", err)
return
}
log.Info("insert bgm_income : %d", len(bm))
return
})
// up income
writeGroup.Go(func() (err error) {
err = s.income.upIncomeSvr.BatchInsertUpIncome(c, um)
if err != nil {
log.Error("s.income.BatchInsertUpIncome error(%v)", err)
return
}
log.Info("insert up_income : %d", len(um))
return
})
// av_income_statis
writeGroup.Go(func() (err error) {
err = s.income.avIncomeStatSvr.BatchInsertAvIncomeStat(c, astat)
if err != nil {
log.Error("s.income.BatchInsertAvIncomeStat error(%v)", err)
return
}
log.Info("insert av_income_statis : %d", len(astat))
return
})
// column_income_statis
writeGroup.Go(func() (err error) {
err = s.income.columnIncomeStatSvr.BatchInsertColumnIncomeStat(c, cstat)
if err != nil {
log.Error("s.income.BatchInsertColumnIncomeStat error(%v)", err)
return
}
log.Info("insert column_income_statis : %d", len(cstat))
return
})
// bgm_income_statis
writeGroup.Go(func() (err error) {
err = s.income.bgmIncomeStatSvr.BatchInsertBgmIncomeStat(c, bstat)
if err != nil {
log.Error("s.income.BatchInsertBgmIncomeStat error(%v)", err)
return
}
log.Info("insert bgm_income_statis : %d", len(bstat))
return
})
// up_income_statis
writeGroup.Go(func() (err error) {
err = s.income.upIncomeStatSvr.BatchInsertUpIncomeStat(c, ustat)
if err != nil {
log.Error("s.income.BatchInsertUpIncomeStat error(%v)", err)
return
}
log.Info("insert up_income_statis : %d", len(ustat))
return
})
// up_account batch insert
writeGroup.Go(func() (err error) {
err = s.income.upAccountSvr.BatchInsertUpAccount(c, accs)
if err != nil {
log.Error("s.income.BatchInsertUpAccount error(%v)", err)
return
}
log.Info("insert up_account : %d", len(accs))
return
})
// up account single update
writeGroup.Go(func() (err error) {
err = s.income.upAccountSvr.UpdateUpAccount(c, accs)
if err != nil {
log.Error("s.income.UpdateUpAccount error(%v)", err)
return
}
log.Info("update up_account : %d", len(accs))
return
})
if err = writeGroup.Wait(); err != nil {
log.Error("run writeGroup.Wait error(%v)", err)
}
return
}
func signedBgmFilter(m map[int64]*model.Signed, date time.Time) BgmFilter {
return func(charge *model.AvCharge, bgm *model.BGM) bool {
if up, ok := m[bgm.MID]; ok {
if charge.Date.Time().Before(up.SignedAt.Time()) {
return true
}
if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
return true
}
} else {
return true
}
return false
}
}
func signedAvFilter(m map[int64]*model.Signed, date time.Time) AvFilter {
return func(charge *model.AvCharge) bool {
if up, ok := m[charge.MID]; ok {
if charge.UploadTime.Time().Before(up.SignedAt.Time()) {
return true
}
if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
return true
}
} else {
return true
}
return false
}
}
func signedColumnFilter(m map[int64]*model.Signed, date time.Time) ColumnFilter {
return func(charge *model.ColumnCharge) bool {
if up, ok := m[charge.MID]; ok {
if charge.UploadTime.Time().Before(up.SignedAt.Time()) {
return true
}
if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
return true
}
} else {
return true
}
return false
}
}
func avFilter(m map[int64]bool) AvFilter {
return func(charge *model.AvCharge) bool {
return m[charge.AvID]
}
}
func columnFilter(m map[int64]bool) ColumnFilter {
return func(charge *model.ColumnCharge) bool {
return m[charge.ArticleID]
}
}
func (s *Service) avToBubbleRatio(bubbleMeta map[int64][]int) map[int64]float64 {
var (
res = make(map[int64]float64)
typeToRatio = make(map[int]float64)
chooseBType = func(bTypes []int) (bType int) {
if len(bTypes) == 1 {
return bTypes[0]
}
sort.Slice(bTypes, func(i, j int) bool {
bti, btj := bTypes[i], bTypes[j]
if typeToRatio[bti] == typeToRatio[btj] {
return bti < btj
}
return typeToRatio[bti] < typeToRatio[btj]
})
return bTypes[0]
}
)
for _, v := range s.conf.Bubble.BRatio {
typeToRatio[v.BType] = v.Ratio
}
for avID, bTypes := range bubbleMeta {
bType := chooseBType(bTypes)
res[avID] = typeToRatio[bType]
}
return res
}
// BgmFilter av charge filter
type BgmFilter func(*model.AvCharge, *model.BGM) bool
// AvFilter av charge filter
type AvFilter func(*model.AvCharge) bool
// ColumnFilter column charge filter
type ColumnFilter func(*model.ColumnCharge) bool
// ChargeRegulator regulates av charge
type ChargeRegulator func(*model.AvCharge)
// UpdateBusinessIncome ..
func (s *Service) UpdateBusinessIncome(c context.Context, date string) (err error) {
return s.income.UpdateBusinessIncomeByDate(c, date)
}

548
app/job/main/growup/service/income/run_statis.go Normal file
View File

@@ -0,0 +1,548 @@
package income
import (
"context"
"fmt"
"time"
model "go-common/app/job/main/growup/model/income"
task "go-common/app/job/main/growup/service"
"go-common/library/log"
"golang.org/x/sync/errgroup"
)
// RunStatis run income statistics
func (s *Service) RunStatis(c context.Context, date time.Time) (err error) {
var msg string
mailReceivers := []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"}
startTime := time.Now().Unix()
err = s.runStatis(c, date)
if err != nil {
msg = err.Error()
} else {
msg = fmt.Sprintf("%s 计算完成，耗时%ds", date.Format("2006-01-02"), time.Now().Unix()-startTime)
}
err = s.email.SendMail(date, msg, "创作激励每日统计%d年%d月%d日", mailReceivers...)
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
return
}
func (s *Service) runStatis(c context.Context, date time.Time) (err error) {
defer func() {
task.GetTaskService().SetTaskStatus(c, task.TaskCreativeStatis, date.Format(_layout), err)
}()
// task status type
err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskCreativeIncome)
if err != nil {
return
}
startWeeklyDate = getStartWeeklyDate(date)
startMonthlyDate = getStartMonthlyDate(date)
var (
readGroup errgroup.Group
avSections = SectionEntries{}
cmSections = SectionEntries{}
bgmSections = SectionEntries{}
upSection = make([]*model.DateStatis, 0)
upAvSection = make([]*model.DateStatis, 0)
upCmSection = make([]*model.DateStatis, 0)
upBgmSection = make([]*model.DateStatis, 0)
upIncomeWeekly = make(map[int64]*model.UpIncome)
upIncomeMonthly = make(map[int64]*model.UpIncome)
upAvStatisCh = make(chan map[int64]*model.UpArchStatis, 1)
upCmStatisCh = make(chan map[int64]*model.UpArchStatis, 1)
upBgmStatisCh = make(chan map[int64]*model.UpArchStatis, 1)
avSourceCh = make(chan []*model.ArchiveIncome, 1000)
avDailyCh = make(chan []*model.ArchiveIncome, 1000)
avWeeklyCh = make(chan []*model.ArchiveIncome, 1000)
avMonthlyCh = make(chan []*model.ArchiveIncome, 1000)
upAvCh = make(chan []*model.ArchiveIncome, 1000)
cmSourceCh = make(chan []*model.ArchiveIncome, 1000)
cmDailyCh = make(chan []*model.ArchiveIncome, 1000)
cmWeeklyCh = make(chan []*model.ArchiveIncome, 1000)
cmMonthlyCh = make(chan []*model.ArchiveIncome, 1000)
upCmCh = make(chan []*model.ArchiveIncome, 1000)
bgmSourceCh = make(chan []*model.ArchiveIncome, 1000)
bgmDailyCh = make(chan []*model.ArchiveIncome, 1000)
bgmWeeklyCh = make(chan []*model.ArchiveIncome, 1000)
bgmMonthlyCh = make(chan []*model.ArchiveIncome, 1000)
upBgmCh = make(chan []*model.ArchiveIncome, 1000)
upSourceCh = make(chan []*model.UpIncome, 1000)
upDailyCh = make(chan []*model.UpIncome, 1000)
upStatisCh = make(chan []*model.UpIncome, 1000)
)
// get income by date to statistics
preDate := startMonthlyDate
if startWeeklyDate.Before(startMonthlyDate) {
preDate = startWeeklyDate
}
// get av_income
readGroup.Go(func() (err error) {
err = s.income.dateStatisSvr.getArchiveByDate(c, avSourceCh, preDate, date, _video, _limitSize)
if err != nil {
log.Error("s.getArchiveByDate error(%v)", err)
}
return
})
// get column_income
readGroup.Go(func() (err error) {
err = s.income.dateStatisSvr.getArchiveByDate(c, cmSourceCh, preDate, date, _column, _limitSize)
if err != nil {
log.Error("s.getArchiveByDate error(%v)", err)
}
return
})
// get bgm_income
readGroup.Go(func() (err error) {
err = s.income.dateStatisSvr.getArchiveByDate(c, bgmSourceCh, preDate, date, _bgm, _limitSize)
if err != nil {
log.Error("s.getArchiveByDate error(%v)", err)
}
return
})
// get up_income
readGroup.Go(func() (err error) {
err = s.income.upIncomeSvr.getUpIncomeByDate(c, upSourceCh, date.Format(_layout), _limitSize)
if err != nil {
log.Error("s.getUpIncomeByDate error(%v)", err)
}
return
})
readGroup.Go(func() (err error) {
defer func() {
close(avDailyCh)
close(avWeeklyCh)
close(avMonthlyCh)
close(upAvCh)
}()
for av := range avSourceCh {
avDailyCh <- av
avWeeklyCh <- av
avMonthlyCh <- av
upAvCh <- av
}
return
})
readGroup.Go(func() (err error) {
defer func() {
close(cmDailyCh)
close(cmWeeklyCh)
close(cmMonthlyCh)
close(upCmCh)
}()
for cm := range cmSourceCh {
cmDailyCh <- cm
cmWeeklyCh <- cm
cmMonthlyCh <- cm
upCmCh <- cm
}
return
})
readGroup.Go(func() (err error) {
defer func() {
close(bgmDailyCh)
close(bgmWeeklyCh)
close(bgmMonthlyCh)
close(upBgmCh)
}()
for bgm := range bgmSourceCh {
bgmDailyCh <- bgm
bgmWeeklyCh <- bgm
bgmMonthlyCh <- bgm
upBgmCh <- bgm
}
return
})
readGroup.Go(func() (err error) {
defer func() {
close(upStatisCh)
close(upDailyCh)
}()
for up := range upSourceCh {
upStatisCh <- up
upDailyCh <- up
}
return
})
// up
readGroup.Go(func() (err error) {
upSection, upAvSection, upCmSection, upBgmSection, err = s.income.dateStatisSvr.handleDateUp(c, upStatisCh, date)
if err != nil {
log.Error("s.income.dateStatisSvr.HandleUp error(%v)", err)
}
return
})
// video
readGroup.Go(func() (err error) {
avSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, avDailyCh, date, _avIncomeDailyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
readGroup.Go(func() (err error) {
avSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, avWeeklyCh, startWeeklyDate, _avIncomeWeeklyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
readGroup.Go(func() (err error) {
avSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, avMonthlyCh, startMonthlyDate, _avIncomeMonthlyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
// column
readGroup.Go(func() (err error) {
cmSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, cmDailyCh, date, _cmIncomeDailyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
readGroup.Go(func() (err error) {
cmSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, cmWeeklyCh, startWeeklyDate, _cmIncomeWeeklyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
readGroup.Go(func() (err error) {
cmSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, cmMonthlyCh, startMonthlyDate, _cmIncomeMonthlyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
// bgm
readGroup.Go(func() (err error) {
bgmSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, bgmDailyCh, date, _bgmIncomeDailyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
readGroup.Go(func() (err error) {
bgmSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, bgmWeeklyCh, startWeeklyDate, _bgmIncomeWeeklyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
readGroup.Go(func() (err error) {
bgmSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, bgmMonthlyCh, startMonthlyDate, _bgmIncomeMonthlyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
}
return
})
// up_av_statis
readGroup.Go(func() (err error) {
err = s.income.upIncomeSvr.handleUpArchStatis(c, upAvStatisCh, upAvCh)
if err != nil {
log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err)
}
return
})
// up_column_statis
readGroup.Go(func() (err error) {
err = s.income.upIncomeSvr.handleUpArchStatis(c, upCmStatisCh, upCmCh)
if err != nil {
log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err)
}
return
})
// up_bgm_statis
readGroup.Go(func() (err error) {
err = s.income.upIncomeSvr.handleUpArchStatis(c, upBgmStatisCh, upBgmCh)
if err != nil {
log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err)
}
return
})
// up_income_weekly up_income_monthly
readGroup.Go(func() (err error) {
upIncomeWeekly, upIncomeMonthly, err = s.income.upIncomeSvr.handleUpIncomeWeeklyAndMonthly(c, date, upAvStatisCh, upCmStatisCh, upBgmStatisCh, upDailyCh)
if err != nil {
log.Error("p.upIncomeSvr.handleUpIncomeWeeklyAndMonthly error(%v)", err)
}
return
})
if err = readGroup.Wait(); err != nil {
log.Error("run readGroup.Wait error(%v)", err)
return
}
{
if len(avSections.avDaily) == 0 {
err = fmt.Errorf("Error: insert 0 av_income_daily_statis")
return
}
if len(avSections.avWeekly) == 0 {
err = fmt.Errorf("Error: insert 0 av_income_weekly_statis")
return
}
if len(avSections.avMonthly) == 0 {
err = fmt.Errorf("Error: insert 0 av_income_monthly_statis")
return
}
if len(cmSections.avDaily) == 0 {
err = fmt.Errorf("Error: insert 0 cm_income_daily_statis")
return
}
if len(cmSections.avWeekly) == 0 {
err = fmt.Errorf("Error: insert 0 cm_income_weekly_statis")
return
}
if len(cmSections.avMonthly) == 0 {
err = fmt.Errorf("Error: insert 0 cm_income_monthly_statis")
return
}
if len(bgmSections.avDaily) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_income_daily_statis")
return
}
if len(bgmSections.avWeekly) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_income_weekly_statis")
return
}
if len(bgmSections.avMonthly) == 0 {
err = fmt.Errorf("Error: insert 0 bgm_income_monthly_statis")
return
}
if len(upSection) == 0 {
err = fmt.Errorf("Error: insert 0 up_income_daily_statis")
return
}
if len(upAvSection) == 0 {
err = fmt.Errorf("Error: insert 0 up_av_daily_statis")
return
}
if len(upCmSection) == 0 {
err = fmt.Errorf("Error: insert 0 up_column_daily_statis")
return
}
if len(upBgmSection) == 0 {
err = fmt.Errorf("Error: insert 0 up_bgm_daily_statis")
return
}
if len(upIncomeWeekly) == 0 {
err = fmt.Errorf("Error: insert 0 up_income_weekly")
return
}
if len(upIncomeMonthly) == 0 {
err = fmt.Errorf("Error: insert 0 up_income_monthly")
return
}
}
// persistent
var writeGroup errgroup.Group
// av_income_daily_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avDaily, _avIncomeDailyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err)
return
}
log.Info("insert av_income_daily_statis : %d", len(avSections.avDaily))
return
})
// av_income_weekly_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avWeekly, _avIncomeWeeklyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err)
return
}
log.Info("insert av_income_weekly_statis : %d", len(avSections.avWeekly))
return
})
// av_income_monthly_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avMonthly, _avIncomeMonthlyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err)
return
}
log.Info("insert av_income_monthly_statis : %d", len(avSections.avMonthly))
return
})
// column_income_daily_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avDaily, _cmIncomeDailyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err)
return
}
log.Info("insert column_income_daily_statis : %d", len(cmSections.avDaily))
return
})
// column_income_weekly_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avWeekly, _cmIncomeWeeklyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err)
return
}
log.Info("insert column_income_weekly_statis : %d", len(cmSections.avWeekly))
return
})
// column_income_monthly_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avMonthly, _cmIncomeMonthlyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err)
return
}
log.Info("insert column_income_monthly_statis : %d", len(cmSections.avMonthly))
return
})
// bgm_income_daily_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avDaily, _bgmIncomeDailyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err)
return
}
log.Info("insert bgm_income_daily_statis : %d", len(bgmSections.avDaily))
return
})
// bgm_income_weekly_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avWeekly, _bgmIncomeWeeklyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err)
return
}
log.Info("insert bgm_income_weekly_statis : %d", len(bgmSections.avWeekly))
return
})
// bgm_income_monthly_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avMonthly, _bgmIncomeMonthlyStatis)
if err != nil {
log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err)
return
}
log.Info("insert bgm_income_monthly_statis : %d", len(bgmSections.avMonthly))
return
})
// up_income_daily_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upSection, "up_income_daily_statis")
if err != nil {
log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
return
}
log.Info("insert up_income_daily_statis : %d", len(upSection))
return
})
// up_av_daily_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upAvSection, "up_av_daily_statis")
if err != nil {
log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
return
}
log.Info("insert up_av_daily_statis : %d", len(upAvSection))
return
})
// up_column_daily_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upCmSection, "up_column_daily_statis")
if err != nil {
log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
return
}
log.Info("insert up_column_daily_statis : %d", len(upCmSection))
return
})
// up_bgm_daily_statis
writeGroup.Go(func() (err error) {
_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upBgmSection, "up_bgm_daily_statis")
if err != nil {
log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
return
}
log.Info("insert up_bgm_daily_statis : %d", len(upBgmSection))
return
})
// up_income_weekly
writeGroup.Go(func() (err error) {
err = s.income.upIncomeSvr.UpIncomeDBStoreBatch(c, _upIncomeWeekly, upIncomeWeekly)
if err != nil {
log.Error("s.UpIncomeDBStoreBatch up_income_weekly error(%v)", err)
return
}
log.Info("insert up_income_weekly : %d", len(upIncomeWeekly))
return
})
// up_income_monthly
writeGroup.Go(func() (err error) {
err = s.income.upIncomeSvr.UpIncomeDBStoreBatch(c, _upIncomeMonthly, upIncomeMonthly)
if err != nil {
log.Error("s.UpIncomeDBStoreBatch up_income_monthly error(%v)", err)
return
}
log.Info("insert up_income_monthly : %d", len(upIncomeMonthly))
return
})
if err = writeGroup.Wait(); err != nil {
log.Error("run writeGroup.Wait error(%v)", err)
}
return
}

1025
app/job/main/growup/service/income/run_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

93
app/job/main/growup/service/income/service.go Normal file
View File

@@ -0,0 +1,93 @@
package income
import (
"context"
"fmt"
"time"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/dao/dataplatform"
"go-common/app/job/main/growup/dao/email"
incomeD "go-common/app/job/main/growup/dao/income"
"go-common/app/job/main/growup/service"
"go-common/app/job/main/growup/service/ctrl"
"go-common/library/log"
)
// Service struct
type Service struct {
conf *conf.Config
dao *incomeD.Dao
avCharge *AvChargeSvr
income *Income
ratio *ChargeRatioSvr
email *email.Dao
dp *dataplatform.Dao
}
// New fn
func New(c *conf.Config, executor ctrl.Executor) (s *Service) {
s = &Service{
conf: c,
dao: incomeD.New(c),
email: email.New(c),
dp: dataplatform.New(c),
}
s.avCharge = NewAvChargeSvr(s.dao)
s.income = NewIncome(batchSize, s.dao)
s.ratio = NewChargeRatioSvr(s.dao)
log.Info("income service start")
executor.Submit(
s.calDailyCreativeIncome,
s.syncBGM,
)
return s
}
func (s *Service) syncBGM(ctx context.Context) {
for {
time.Sleep(service.NextDay(1, 0, 0))
msg := ""
log.Info("sync BGM begin:%v", time.Now().Format("2006-01-02 15:04:05"))
err := s.SyncBgmInfo(context.TODO())
if err != nil {
log.Error("s.SyncBgmInfo error(%v)", err)
}
if err != nil {
msg = fmt.Sprintf("SyncBgmInfo error(%v)", err)
} else {
msg = "Success"
}
err = s.email.SendMail(time.Now(), msg, "创作激励同步bgm%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("sync BGM end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) calDailyCreativeIncome(ctx context.Context) {
for {
time.Sleep(service.NextDay(18, 0, 0))
log.Info("calDailyCreativeIncome begin:%v", time.Now().Format("2006-01-02 15:04:05"))
now := time.Now()
date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local)
err := s.RunAndSendMail(context.TODO(), date)
if err != nil {
log.Error("s.RunAndSendMail error(%v)", err)
}
if err == nil {
err = s.RunStatis(context.TODO(), date)
if err != nil {
log.Error("s.RunStatis error(%v)", err)
}
}
log.Info("calDailyCreativeIncome end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
// Ping check dao health.
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}

35
app/job/main/growup/service/income/service_test.go Normal file
View File

@@ -0,0 +1,35 @@
package income
import (
"context"
"flag"
"path/filepath"
"testing"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/service/ctrl"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *Service
charge *AvChargeSvr
)
func init() {
dir, _ := filepath.Abs("../../cmd/growup-job.toml")
flag.Set("conf", dir)
conf.Init()
if s == nil {
s = New(conf.Conf, ctrl.NewUnboundedExecutor())
}
charge = s.avCharge
}
func TestPing(t *testing.T) {
Convey("Test_Ping", t, func() {
err := s.Ping(context.Background())
So(err, ShouldBeNil)
})
}

118
app/job/main/growup/service/income/tax.go Normal file
View File

@@ -0,0 +1,118 @@
package income
const (
_range0_30 float64 = 0.0
_range30_50 float64 = 0.05
_range50_100 float64 = 0.10
_range100_150 float64 = 0.15
_range150_300 float64 = 0.20
_range300_500 float64 = 0.25
_range500_1000 float64 = 0.30
_range1000_2000 float64 = 0.40
_range2000_3000 float64 = 0.50
_range3000max float64 = 0.60
)
// TaxRate tax rate
type TaxRate struct {
start float64
end float64
rate float64
}
// Tax tax
func Tax(income float64) (tax float64) {
rs := rates(income)
for _, r := range rs {
if income >= r.end {
tax += Mul(float64(r.end-r.start), r.rate)
} else {
tax += Mul(float64(income-r.start), r.rate)
}
}
return
}
func rates(income float64) (rs []*TaxRate) {
if income > 0 {
r := &TaxRate{
start: 0,
end: 30,
rate: _range0_30,
}
rs = append(rs, r)
}
if income > 30 {
r := &TaxRate{
start: 30,
end: 50,
rate: _range30_50,
}
rs = append(rs, r)
}
if income > 50 {
r := &TaxRate{
start: 50,
end: 100,
rate: _range50_100,
}
rs = append(rs, r)
}
if income > 100 {
r := &TaxRate{
start: 100,
end: 150,
rate: _range100_150,
}
rs = append(rs, r)
}
if income > 150 {
r := &TaxRate{
start: 150,
end: 300,
rate: _range150_300,
}
rs = append(rs, r)
}
if income > 300 {
r := &TaxRate{
start: 300,
end: 500,
rate: _range300_500,
}
rs = append(rs, r)
}
if income > 500 {
r := &TaxRate{
start: 500,
end: 1000,
rate: _range500_1000,
}
rs = append(rs, r)
}
if income > 1000 {
r := &TaxRate{
start: 1000,
end: 2000,
rate: _range1000_2000,
}
rs = append(rs, r)
}
if income > 2000 {
r := &TaxRate{
start: 2000,
end: 3000,
rate: _range2000_3000,
}
rs = append(rs, r)
}
if income > 3000 {
r := &TaxRate{
start: 3000,
end: 1<<31 - 1,
rate: _range3000max,
}
rs = append(rs, r)
}
return
}

56
app/job/main/growup/service/income/tools.go Normal file
View File

@@ -0,0 +1,56 @@
package income
import (
"math"
"math/big"
)
// Div div
func Div(x, y float64) float64 {
a := big.NewFloat(x)
b := big.NewFloat(y)
c := new(big.Float).Quo(a, b)
d, _ := c.Float64()
return d
}
// Mul mul
func Mul(x, y float64) float64 {
a := big.NewFloat(x)
b := big.NewFloat(y)
c := new(big.Float).Mul(a, b)
d, _ := c.Float64()
return d
}
// DivWithRound div with round
func DivWithRound(x, y float64, places int) float64 {
a := big.NewFloat(x)
b := big.NewFloat(y)
c := new(big.Float).Quo(a, b)
d, _ := c.Float64()
return Round(d, places)
}
// MulWithRound mul with round
func MulWithRound(x, y float64, places int) float64 {
a := big.NewFloat(x)
b := big.NewFloat(y)
c := new(big.Float).Mul(a, b)
d, _ := c.Float64()
return Round(d, places)
}
// Round round
func Round(val float64, places int) float64 {
var round float64
pow := math.Pow(10, float64(places))
digit := pow * val
_, div := math.Modf(digit)
if div >= 0.5 {
round = math.Ceil(digit)
} else {
round = math.Floor(digit)
}
return round / pow
}

137
app/job/main/growup/service/income/up_account.go Normal file
View File

@@ -0,0 +1,137 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// UpAccountSvr up account service
type UpAccountSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewUpAccountSvr new up account service
func NewUpAccountSvr(dao *incomeD.Dao, batchSize int) (svr *UpAccountSvr) {
return &UpAccountSvr{
batchSize: batchSize,
dao: dao,
}
}
// UpAccount get up account
func (s *UpAccountSvr) UpAccount(c context.Context, limit int64) (m map[int64]*model.UpAccount, err error) {
var id int64
m = make(map[int64]*model.UpAccount)
for {
var um map[int64]*model.UpAccount
um, id, err = s.dao.UpAccounts(c, id, limit)
if err != nil {
return
}
if len(um) == 0 {
break
}
for mid, acc := range um {
m[mid] = acc
}
}
return
}
// BatchInsertUpAccount batch insert up account
func (s *UpAccountSvr) BatchInsertUpAccount(c context.Context, us map[int64]*model.UpAccount) (err error) {
var (
buff = make([]*model.UpAccount, s.batchSize)
buffEnd = 0
)
for _, u := range us {
if u.DataState != 1 {
continue
}
buff[buffEnd] = u
buffEnd++
if buffEnd >= s.batchSize {
values := upAccountValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.InsertUpAccount(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := upAccountValues(buff[:buffEnd])
buffEnd = 0
_, err = s.dao.InsertUpAccount(c, values)
}
return
}
// UpdateUpAccount update up account
func (s *UpAccountSvr) UpdateUpAccount(c context.Context, us map[int64]*model.UpAccount) (err error) {
for _, u := range us {
if u.DataState != 2 {
continue
}
var time int
for {
var rows int64
rows, err = s.dao.UpdateUpAccount(c, u.MID, u.Version, u.TotalIncome, u.TotalUnwithdrawIncome)
if err != nil {
return
}
time++
if rows > 0 {
break
}
if time >= 10 {
break
}
s.reload(c, u)
}
}
return
}
func (s *UpAccountSvr) reload(c context.Context, upAccount *model.UpAccount) (err error) {
result, err := s.dao.UpAccount(c, upAccount.MID)
if err != nil {
return
}
upAccount.TotalIncome = result.TotalIncome
upAccount.TotalUnwithdrawIncome = result.TotalUnwithdrawIncome
upAccount.Version = result.Version
upAccount.WithdrawDateVersion = result.WithdrawDateVersion
return
}
func upAccountValues(us []*model.UpAccount) (values string) {
var buf bytes.Buffer
for _, u := range us {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(u.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(u.HasSignContract))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.TotalUnwithdrawIncome, 10))
buf.WriteByte(',')
buf.WriteString("'" + u.WithdrawDateVersion + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.Version, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

31
app/job/main/growup/service/income/up_account_test.go Normal file
View File

@@ -0,0 +1,31 @@
package income
import (
"context"
"testing"
model "go-common/app/job/main/growup/model/income"
. "github.com/smartystreets/goconvey/convey"
)
func Test_UpAccount1(t *testing.T) {
Convey("UpAccount", t, func() {
_, err := s.income.upAccountSvr.UpAccount(context.Background(), 10)
So(err, ShouldBeNil)
})
}
func Test_BatchInsertUpAccount(t *testing.T) {
Convey("BatchInsertUpAccount", t, func() {
err := s.income.upAccountSvr.BatchInsertUpAccount(context.Background(), map[int64]*model.UpAccount{})
So(err, ShouldBeNil)
})
}
func Test_UpdateUpAccount(t *testing.T) {
Convey("UpdateUpAccount", t, func() {
err := s.income.upAccountSvr.UpdateUpAccount(context.Background(), map[int64]*model.UpAccount{})
So(err, ShouldBeNil)
})
}

68
app/job/main/growup/service/income/up_archive_statis.go Normal file
View File

@@ -0,0 +1,68 @@
package income
import (
"context"
"strconv"
"strings"
model "go-common/app/job/main/growup/model/income"
xtime "go-common/library/time"
)
func (s *UpIncomeSvr) handleUpArchStatis(c context.Context, upArchStatisCh chan map[int64]*model.UpArchStatis, archiveCh chan []*model.ArchiveIncome) (err error) {
defer close(upArchStatisCh)
upArchMap := make(map[int64]*model.UpArchStatis)
for income := range archiveCh {
s.calUpArchStatis(income, upArchMap)
}
upArchStatisCh <- upArchMap
return
}
func (s *UpIncomeSvr) calUpArchStatis(incomes []*model.ArchiveIncome, upArch map[int64]*model.UpArchStatis) {
for _, income := range incomes {
if _, ok := upArch[income.MID]; !ok {
upArch[income.MID] = addUpArchStatis(income)
}
updateUpArchStatis(income.AID, income.Date, upArch[income.MID])
}
}
func addUpArchStatis(income *model.ArchiveIncome) *model.UpArchStatis {
return &model.UpArchStatis{
MID: income.MID,
WeeklyDate: xtime.Time(startWeeklyDate.Unix()),
WeeklyAIDs: "",
MonthlyDate: xtime.Time(startMonthlyDate.Unix()),
MonthlyAIDs: "",
}
}
func updateUpArchStatis(aid int64, date xtime.Time, statis *model.UpArchStatis) {
idStr := strconv.FormatInt(aid, 10)
if date >= statis.WeeklyDate {
if statis.WeeklyAIDs == "" {
statis.WeeklyAIDs = idStr
} else if !isExist(idStr, statis.WeeklyAIDs) {
statis.WeeklyAIDs += "," + idStr
}
}
if date >= statis.MonthlyDate {
if statis.MonthlyAIDs == "" {
statis.MonthlyAIDs = idStr
} else if !isExist(idStr, statis.MonthlyAIDs) {
statis.MonthlyAIDs += "," + idStr
}
}
}
func isExist(id string, old string) bool {
oldSli := strings.Split(old, ",")
for _, str := range oldSli {
if id == str {
return true
}
}
return false
}

277
app/job/main/growup/service/income/up_income.go Normal file
View File

@@ -0,0 +1,277 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// UpIncomeSvr up income service
type UpIncomeSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewUpIncomeSvr new income service
func NewUpIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *UpIncomeSvr) {
return &UpIncomeSvr{
batchSize: batchSize,
dao: dao,
}
}
func (p *UpIncomeSvr) getUpIncomeByDate(c context.Context, upCh chan []*model.UpIncome, date string, limit int) (err error) {
defer close(upCh)
var id int64
for {
var up []*model.UpIncome
up, err = p.dao.GetUpIncomeTable(c, "up_income", date, id, limit)
if err != nil {
return
}
upCh <- up
if len(up) < limit {
break
}
id = up[len(up)-1].ID
}
return
}
// BatchInsertUpIncome insert up_income batch
func (p *UpIncomeSvr) BatchInsertUpIncome(c context.Context, us map[int64]*model.UpIncome) (err error) {
var (
buff = make([]*model.UpIncome, p.batchSize)
buffEnd = 0
)
for _, u := range us {
buff[buffEnd] = u
buffEnd++
if buffEnd >= p.batchSize {
values := upIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertUpIncome(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := upIncomeValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertUpIncome(c, values)
}
return
}
func upIncomeValues(us []*model.UpIncome) (values string) {
var buf bytes.Buffer
for _, u := range us {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(u.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.PlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AudioIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.TaxMoney, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvBaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvTax, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnBaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnTax, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmBaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmTax, 10))
buf.WriteByte(',')
buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}
///************************************************************ FOR HISTORY ****************************************************************/
//
//// UpdateBusinessIncomeByDate update business income by date
//func (p *UpIncomeSvr) UpdateBusinessIncomeByDate(c context.Context, date string, ustat map[int64]*model.UpIncomeStat) (err error) {
// us, err := p.businessTotalIncome(c, ustat, date)
// if err != nil {
// return
// }
// err = p.batchUpdateUpIncome(c, us)
// if err != nil {
// return
// }
// return p.batchUpdateUpIncomeStat(c, ustat)
//}
//
//// for history data m: map[mid]map[date]*model.UpIncome
//func (p *UpIncomeSvr) businessTotalIncome(c context.Context, ustat map[int64]*model.UpIncomeStat, date string) (m []*model.UpIncome, err error) {
// var id int64
// for {
// var ups []*model.UpIncome
// ups, err = p.dao.GetUpIncomeTable(c, "up_income", date, id, 2000)
// if err != nil {
// return
// }
// for _, up := range ups {
// ut := ustat[up.MID]
//
// ut.AvTotalIncome += up.AvIncome
// up.AvTotalIncome = ut.AvTotalIncome
//
// ut.ColumnTotalIncome += up.ColumnIncome
// up.ColumnTotalIncome = ut.ColumnTotalIncome
//
// ut.BgmTotalIncome += up.BgmIncome
// up.BgmTotalIncome = ut.BgmTotalIncome
// m = append(m, up)
// }
// if len(ups) < 2000 {
// break
// }
// id = ups[len(ups)-1].ID
// }
// return
//}
//
//// BatchUpdateUpIncome insert up_income batch
//func (p *UpIncomeSvr) batchUpdateUpIncome(c context.Context, us []*model.UpIncome) (err error) {
// var (
// buff = make([]*model.UpIncome, p.batchSize)
// buffEnd = 0
// )
//
// for _, u := range us {
// buff[buffEnd] = u
// buffEnd++
//
// if buffEnd >= p.batchSize {
// values := businessValues(buff[:buffEnd])
// buffEnd = 0
// _, err = p.dao.FixInsertUpIncome(c, values)
// if err != nil {
// return
// }
// }
// }
// if buffEnd > 0 {
// values := businessValues(buff[:buffEnd])
// buffEnd = 0
// _, err = p.dao.FixInsertUpIncome(c, values)
// }
// return
//}
//
//func businessValues(us []*model.UpIncome) (values string) {
// var buf bytes.Buffer
// for _, u := range us {
// buf.WriteString("(")
// buf.WriteString(strconv.FormatInt(u.MID, 10))
// buf.WriteByte(',')
// buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
// buf.WriteByte(',')
// buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
// buf.WriteByte(',')
// buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
// buf.WriteByte(',')
// buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
// buf.WriteString(")")
// buf.WriteByte(',')
// }
// if buf.Len() > 0 {
// buf.Truncate(buf.Len() - 1)
// }
// values = buf.String()
// buf.Reset()
// return
//}
//
//// BatchUpdateUpIncome insert up_income batch
//func (p *UpIncomeSvr) batchUpdateUpIncomeStat(c context.Context, us map[int64]*model.UpIncomeStat) (err error) {
// var (
// buff = make([]*model.UpIncomeStat, p.batchSize)
// buffEnd = 0
// )
//
// for _, u := range us {
// buff[buffEnd] = u
// buffEnd++
//
// if buffEnd >= p.batchSize {
// values := businessStatValues(buff[:buffEnd])
// buffEnd = 0
// _, err = p.dao.FixInsertUpIncomeStat(c, values)
// if err != nil {
// return
// }
// }
// }
// if buffEnd > 0 {
// values := businessStatValues(buff[:buffEnd])
// buffEnd = 0
// _, err = p.dao.FixInsertUpIncomeStat(c, values)
// }
// return
//}
//
//func businessStatValues(ustat []*model.UpIncomeStat) (values string) {
// var buf bytes.Buffer
// for _, u := range ustat {
// buf.WriteString("(")
// buf.WriteString(strconv.FormatInt(u.MID, 10))
// buf.WriteByte(',')
// buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
// buf.WriteByte(',')
// buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
// buf.WriteByte(',')
// buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
// buf.WriteString(")")
// buf.WriteByte(',')
// }
// if buf.Len() > 0 {
// buf.Truncate(buf.Len() - 1)
// }
// values = buf.String()
// buf.Reset()
// return
//}

318
app/job/main/growup/service/income/up_income_date.go Normal file
View File

@@ -0,0 +1,318 @@
package income
import (
"bytes"
"context"
"strconv"
"strings"
"time"
model "go-common/app/job/main/growup/model/income"
"go-common/library/log"
xtime "go-common/library/time"
)
var (
_upIncomeWeekly = "up_income_weekly"
_upIncomeMonthly = "up_income_monthly"
)
func (s *UpIncomeSvr) handleUpIncomeWeeklyAndMonthly(
c context.Context,
date time.Time,
upAvStatisCh chan map[int64]*model.UpArchStatis,
upCmStatisCh chan map[int64]*model.UpArchStatis,
upBgmStatisCh chan map[int64]*model.UpArchStatis,
upSliceCh chan []*model.UpIncome) (weeklyMap, monthlyMap map[int64]*model.UpIncome, err error) {
weeklyMap, monthlyMap, err = s.GetUpIncomeWeeklyAndMonthly(c, date)
if err != nil {
log.Error("s.GetUpIncomeWeeklyAndMonthly error(%v)", err)
return
}
upAvStatis := <-upAvStatisCh
upCmStatis := <-upCmStatisCh
upBgmStatis := <-upBgmStatisCh
s.calUpIncomeWeeklyAndMonthly(weeklyMap, monthlyMap, upAvStatis, upCmStatis, upBgmStatis, upSliceCh)
return
}
// GetUpIncomeWeeklyAndMonthly get up_income_weekly and up_income_monthly
func (s *UpIncomeSvr) GetUpIncomeWeeklyAndMonthly(c context.Context, date time.Time) (weeklyMap map[int64]*model.UpIncome, monthlyMap map[int64]*model.UpIncome, err error) {
upIncomeWeekly, err := s.GetUpIncomeTable(c, startWeeklyDate, _upIncomeWeekly)
if err != nil {
log.Error("s.GetUpIncomeTable error(%v)", err)
return
}
upIncomeMonthly, err := s.GetUpIncomeTable(c, startMonthlyDate, _upIncomeMonthly)
if err != nil {
log.Error("s.GetUpIncomeTable error(%v)", err)
return
}
weeklyMap = make(map[int64]*model.UpIncome)
monthlyMap = make(map[int64]*model.UpIncome)
for _, weeklyIncome := range upIncomeWeekly {
weeklyMap[weeklyIncome.MID] = weeklyIncome
}
for _, monthlyIncome := range upIncomeMonthly {
monthlyMap[monthlyIncome.MID] = monthlyIncome
}
return
}
// GetUpIncomeTable get up income table
func (s *UpIncomeSvr) GetUpIncomeTable(c context.Context, date time.Time, table string) (upIncomes []*model.UpIncome, err error) {
var id int64
for {
upIncome, err1 := s.dao.GetUpIncomeTable(c, table, date.Format(_layout), id, _limitSize)
if err1 != nil {
err = err1
return
}
upIncomes = append(upIncomes, upIncome...)
if len(upIncome) < _limitSize {
break
}
id = upIncome[len(upIncome)-1].ID
}
return
}
func (s *UpIncomeSvr) calUpIncomeWeeklyAndMonthly(weeklyMap, monthlyMap map[int64]*model.UpIncome,
upAvStatis, upCmStatis, upBgmStatis map[int64]*model.UpArchStatis, upSliceCh chan []*model.UpIncome) {
for upIncome := range upSliceCh {
s.calUpIncome(upIncome, weeklyMap, monthlyMap, upAvStatis, upCmStatis, upBgmStatis)
}
}
func (s *UpIncomeSvr) calUpIncome(upIncome []*model.UpIncome, weeklyMap, monthlyMap map[int64]*model.UpIncome,
upAvStatis, upCmStatis, upBgmStatis map[int64]*model.UpArchStatis) {
var weeklyAvCount, monthlyAvCount int
var weeklyCmCount, monthlyCmCount int
var weeklyBgmCount, monthlyBgmCount int
for _, income := range upIncome {
weeklyAvCount, monthlyAvCount = 0, 0
weeklyCmCount, monthlyCmCount = 0, 0
weeklyBgmCount, monthlyBgmCount = 0, 0
if statis, ok := upAvStatis[income.MID]; ok {
weeklyAvCount = len(strings.Split(statis.WeeklyAIDs, ","))
monthlyAvCount = len(strings.Split(statis.MonthlyAIDs, ","))
}
if statis, ok := upCmStatis[income.MID]; ok {
weeklyCmCount = len(strings.Split(statis.WeeklyAIDs, ","))
monthlyCmCount = len(strings.Split(statis.MonthlyAIDs, ","))
}
if statis, ok := upBgmStatis[income.MID]; ok {
weeklyBgmCount = len(strings.Split(statis.WeeklyAIDs, ","))
monthlyBgmCount = len(strings.Split(statis.MonthlyAIDs, ","))
}
if weeklyIncome, ok := weeklyMap[income.MID]; ok {
updateUpIncome(weeklyIncome, income, weeklyAvCount, weeklyCmCount, weeklyBgmCount)
} else {
weeklyMap[income.MID] = addUpIncome(income, startWeeklyDate, weeklyAvCount, weeklyCmCount, weeklyBgmCount)
}
if weeklyIncome, ok := monthlyMap[income.MID]; ok {
updateUpIncome(weeklyIncome, income, monthlyAvCount, monthlyCmCount, monthlyBgmCount)
} else {
monthlyMap[income.MID] = addUpIncome(income, startMonthlyDate, monthlyAvCount, monthlyCmCount, monthlyBgmCount)
}
}
}
func addUpIncome(daily *model.UpIncome, fixDate time.Time, avCount, cmCount, bgmCount int) *model.UpIncome {
return &model.UpIncome{
MID: daily.MID,
AvCount: int64(avCount),
PlayCount: daily.PlayCount,
AvIncome: daily.AvIncome,
AvBaseIncome: daily.AvBaseIncome,
AvTax: daily.AvTax,
AvTotalIncome: daily.AvTotalIncome,
ColumnCount: int64(cmCount),
ColumnIncome: daily.ColumnIncome,
ColumnBaseIncome: daily.ColumnBaseIncome,
ColumnTax: daily.ColumnTax,
ColumnTotalIncome: daily.ColumnTotalIncome,
BgmCount: int64(bgmCount),
BgmIncome: daily.BgmIncome,
BgmBaseIncome: daily.BgmBaseIncome,
BgmTax: daily.BgmTax,
BgmTotalIncome: daily.BgmTotalIncome,
AudioIncome: daily.AudioIncome,
TaxMoney: daily.TaxMoney,
Income: daily.Income,
BaseIncome: daily.BaseIncome,
TotalIncome: daily.TotalIncome,
Date: xtime.Time(fixDate.Unix()),
DBState: _dbInsert,
}
}
func updateUpIncome(origin, daily *model.UpIncome, avCount, cmCount, bgmCount int) {
origin.AvCount = int64(avCount)
origin.PlayCount += daily.PlayCount
origin.AvIncome += daily.AvIncome
origin.AvBaseIncome += daily.AvBaseIncome
origin.AvTax += daily.AvTax
origin.AvTotalIncome = daily.AvTotalIncome
origin.ColumnCount = int64(cmCount)
origin.ColumnIncome += daily.ColumnIncome
origin.ColumnBaseIncome += daily.ColumnBaseIncome
origin.ColumnTax += daily.ColumnTax
origin.ColumnTotalIncome = daily.ColumnTotalIncome
origin.BgmCount = int64(bgmCount)
origin.BgmIncome += daily.BgmIncome
origin.BgmBaseIncome += daily.BgmBaseIncome
origin.BgmTax += daily.BgmTax
origin.BgmTotalIncome = daily.BgmTotalIncome
origin.AudioIncome += daily.AudioIncome
origin.TaxMoney += daily.TaxMoney
origin.Income += daily.Income
origin.BaseIncome += daily.BaseIncome
origin.TotalIncome = daily.TotalIncome
origin.DBState = _dbUpdate
}
// UpIncomeDBStore insert up_income
func (s *UpIncomeSvr) UpIncomeDBStore(c context.Context, weeklyMap, monthlyMap map[int64]*model.UpIncome) (err error) {
err = s.UpIncomeDBStoreBatch(c, _upIncomeWeekly, weeklyMap)
if err != nil {
log.Error("s.UpIncomeDBStoreBatch up_income_weekly error(%v)", err)
return
}
err = s.UpIncomeDBStoreBatch(c, _upIncomeMonthly, monthlyMap)
if err != nil {
log.Error("s.UpIncomeDBStoreBatch up_income_monthly error(%v)", err)
return
}
return
}
// UpIncomeDBStoreBatch up income db batch store
func (s *UpIncomeSvr) UpIncomeDBStoreBatch(c context.Context, table string, upIncomeMap map[int64]*model.UpIncome) error {
insert, update := make([]*model.UpIncome, batchSize), make([]*model.UpIncome, batchSize)
insertIndex, updateIndex := 0, 0
for _, income := range upIncomeMap {
if income.DBState == _dbInsert {
insert[insertIndex] = income
insertIndex++
} else if income.DBState == _dbUpdate {
update[updateIndex] = income
updateIndex++
}
if insertIndex >= batchSize {
_, err := s.upIncomeBatchInsert(c, table, insert[:insertIndex])
if err != nil {
log.Error("s.upIncomeBatchInsert error(%v)", err)
return err
}
insertIndex = 0
}
if updateIndex >= batchSize {
_, err := s.upIncomeBatchInsert(c, table, update[:updateIndex])
if err != nil {
log.Error("s.upIncomeBatchInsert error(%v)", err)
return err
}
updateIndex = 0
}
}
if insertIndex > 0 {
_, err := s.upIncomeBatchInsert(c, table, insert[:insertIndex])
if err != nil {
log.Error("s.upIncomeBatchInsert error(%v)", err)
return err
}
}
if updateIndex > 0 {
_, err := s.upIncomeBatchInsert(c, table, update[:updateIndex])
if err != nil {
log.Error("s.upIncomeBatchInsert error(%v)", err)
return err
}
}
return nil
}
func (s *UpIncomeSvr) upIncomeBatchInsert(c context.Context, table string, us []*model.UpIncome) (rows int64, err error) {
var buf bytes.Buffer
for _, u := range us {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(u.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.PlayCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AudioIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmCount, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.TaxMoney, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvBaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvTax, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnBaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnTax, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmBaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmTax, 10))
buf.WriteByte(',')
buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values := buf.String()
buf.Reset()
rows, err = s.dao.InsertUpIncomeTable(c, table, values)
return
}

32
app/job/main/growup/service/income/up_income_date_test.go Normal file
View File

@@ -0,0 +1,32 @@
package income
import (
"context"
"testing"
"time"
model "go-common/app/job/main/growup/model/income"
. "github.com/smartystreets/goconvey/convey"
)
func Test_GetUpIncomeTable(t *testing.T) {
Convey("GetUpIncomeTable", t, func() {
_, err := s.income.upIncomeSvr.GetUpIncomeTable(context.Background(), time.Now(), _upIncomeWeekly)
So(err, ShouldBeNil)
})
}
func Test_GetUpIncomeWeeklyAndMonthly(t *testing.T) {
Convey("GetUpIncomeWeeklyAndMonthly", t, func() {
_, _, err := s.income.upIncomeSvr.GetUpIncomeWeeklyAndMonthly(context.Background(), time.Now())
So(err, ShouldBeNil)
})
}
func Test_UpIncomeDBStore(t *testing.T) {
Convey("UpIncomeDBStore", t, func() {
err := s.income.upIncomeSvr.UpIncomeDBStore(context.Background(), map[int64]*model.UpIncome{}, map[int64]*model.UpIncome{})
So(err, ShouldBeNil)
})
}

100
app/job/main/growup/service/income/up_income_stat.go Normal file
View File

@@ -0,0 +1,100 @@
package income
import (
"bytes"
"context"
"strconv"
incomeD "go-common/app/job/main/growup/dao/income"
model "go-common/app/job/main/growup/model/income"
)
// UpIncomeStatSvr up_income_stat svr
type UpIncomeStatSvr struct {
batchSize int
dao *incomeD.Dao
}
// NewUpIncomeStatSvr new server
func NewUpIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *UpIncomeStatSvr) {
return &UpIncomeStatSvr{
batchSize: batchSize,
dao: dao,
}
}
// UpIncomeStat return stats, key: mid, value: total_income
func (p *UpIncomeStatSvr) UpIncomeStat(c context.Context, limit int64) (m map[int64]*model.UpIncomeStat, err error) {
var id int64
m = make(map[int64]*model.UpIncomeStat)
for {
var um map[int64]*model.UpIncomeStat
um, id, err = p.dao.UpIncomeStat(c, id, limit)
if err != nil {
return
}
if len(um) == 0 {
break
}
for mid, u := range um {
if u.IsDeleted == 0 {
m[mid] = u
}
}
}
return
}
// BatchInsertUpIncomeStat insert up_income_statis batch
func (p *UpIncomeStatSvr) BatchInsertUpIncomeStat(c context.Context, us map[int64]*model.UpIncomeStat) (err error) {
var (
buff = make([]*model.UpIncomeStat, batchSize)
buffEnd = 0
)
for _, u := range us {
if u.DataState == 0 {
continue
}
buff[buffEnd] = u
buffEnd++
if buffEnd >= p.batchSize {
values := upIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertUpIncomeStat(c, values)
if err != nil {
return
}
}
}
if buffEnd > 0 {
values := upIncomeStatValues(buff[:buffEnd])
buffEnd = 0
_, err = p.dao.InsertUpIncomeStat(c, values)
}
return
}
func upIncomeStatValues(us []*model.UpIncomeStat) (values string) {
var buf bytes.Buffer
for _, u := range us {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(u.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

29
app/job/main/growup/service/income/up_info_video.go Normal file
View File

@@ -0,0 +1,29 @@
package income
import (
"context"
model "go-common/app/job/main/growup/model/income"
)
// Signed signed up by business
func (s *Service) Signed(c context.Context, business string, limit int64) (m map[int64]*model.Signed, err error) {
var offset int64
m = make(map[int64]*model.Signed)
for {
var us map[int64]*model.Signed
offset, us, err = s.dao.Ups(c, business, offset, limit)
if err != nil {
return
}
if len(us) == 0 {
break
}
for k, v := range us {
if v.AccountState == 3 || v.AccountState == 5 || v.AccountState == 6 {
m[k] = v
}
}
}
return
}

85
app/job/main/growup/service/income_test.go Normal file
View File

@@ -0,0 +1,85 @@
package service
import (
"context"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_InsertTagIncome(t *testing.T) {
Convey("growup-job InsertTagIncome", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
err := s.InsertTagIncome(context.Background(), t)
So(err, ShouldBeNil)
}))
}
func Test_GetAvIncomeStatis(t *testing.T) {
Convey("growup-job GetAvIncomeStatis", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
err := s.GetAvIncomeStatis(context.Background(), t.Format(_layout))
So(err, ShouldBeNil)
}))
}
func Test_GetAvIncomes(t *testing.T) {
Convey("growup-job GetAvIncome", t, WithService(func(s *Service) {
res, err := s.GetAvIncome(context.Background())
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}
func Test_GetUpIncomeStatis(t *testing.T) {
Convey("growup-job GetUpIncomeStatis", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
hasWithdraw := 1
err := s.GetUpIncomeStatis(context.Background(), t.Format(_layout), hasWithdraw)
So(err, ShouldBeNil)
}))
}
func Test_GetUpAccount(t *testing.T) {
Convey("growup-job GetUpAccount", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
res, err := s.GetUpAccount(context.Background(), t.Format(_layout), date.Format("2006-01-02 15:04:05"))
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}
func Test_GetUpIncome(t *testing.T) {
Convey("growup-job GetUpIncome", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
table := "up_account"
res, err := s.GetUpIncome(context.Background(), table, t.Format(_layout))
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}
func Test_GetUpWithdraw(t *testing.T) {
Convey("growup-job GetUpWithdraw", t, WithService(func(s *Service) {
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
res, err := s.GetUpWithdraw(context.Background(), t.Format(_layout))
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}
func Test_GetUpNickname(t *testing.T) {
Convey("growup-job GetUpNickname", t, WithService(func(s *Service) {
mids := []int64{int64(1), int64(2)}
res, err := s.GetUpNickname(context.Background(), mids)
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}

325
app/job/main/growup/service/lottery.go Normal file
View File

@@ -0,0 +1,325 @@
package service
import (
"bytes"
"context"
"sort"
"strconv"
"sync"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/ecode"
"go-common/library/log"
"golang.org/x/sync/errgroup"
)
type bubbleType int
const (
_bubbleLottery bubbleType = 1
_bubbleVote bubbleType = 2
)
type bubbleMeta struct {
avid int64
aType bubbleType
}
// use var map[] + register map ???
func (s *Service) syncBubbleMetaFuncM() map[bubbleType]func(c context.Context, start, end time.Time) ([]*bubbleMeta, error) {
return map[bubbleType]func(c context.Context, start, end time.Time) ([]*bubbleMeta, error){
_bubbleLottery: s.syncLotteryAvs,
_bubbleVote: s.syncVoteBIZAvs,
}
}
func (s *Service) syncLotteryAvs(c context.Context, start, end time.Time) (data []*bubbleMeta, err error) {
var (
offset int64
hasMore = 1
avs []int64
)
for hasMore == 1 {
var info *model.LotteryRes
info, err = s.dao.GetLotteryRIDs(c, start.Unix(), end.Unix()-1, offset)
if err != nil {
log.Error("s.dao.GetLotteryRIDs error(%v)", err)
return
}
if len(info.RIDs) > 0 {
avs = append(avs, info.RIDs...)
}
hasMore = info.HasMore
offset = info.Offset
}
data = make([]*bubbleMeta, 0, len(avs))
for _, av := range avs {
data = append(data, &bubbleMeta{
avid: av,
aType: _bubbleLottery,
})
}
return
}
func (s *Service) syncVoteBIZAvs(c context.Context, start, end time.Time) (data []*bubbleMeta, err error) {
if end.After(start.AddDate(0, 6, 0)) {
err = ecode.Error(ecode.RequestErr, "同步投票视频时间区间不能超过半年")
return
}
data = make([]*bubbleMeta, 0)
var voteAVs []*model.VoteBIZArchive
voteAVs, err = s.dao.VoteBIZArchive(c, start.Unix(), end.Unix())
if err != nil {
return
}
if len(voteAVs) == 0 {
return
}
for _, av := range voteAVs {
data = append(data, &bubbleMeta{
avid: av.Aid,
aType: _bubbleVote,
})
}
return
}
func (s *Service) saveBubbleMeta(c context.Context, metas []*bubbleMeta, date time.Time) (rows int64, err error) {
return s.dao.InsertBubbleMeta(c, assembleBubbleMeta(metas, date))
}
func assembleBubbleMeta(data []*bubbleMeta, syncDate time.Time) (values string) {
var buf bytes.Buffer
for _, v := range data {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(v.avid, 10))
buf.WriteByte(',')
buf.WriteString("'" + syncDate.Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(int(v.aType)))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}
// SyncIncomeBubbleMeta .
func (s *Service) SyncIncomeBubbleMeta(c context.Context, start, end time.Time, tp int) (rows int64, err error) {
syncFuncM := s.syncBubbleMetaFuncM()
syncFunc, ok := syncFuncM[bubbleType(tp)]
if !ok {
err = ecode.Errorf(ecode.ReqParamErr, "illegal bubbleType(%d)", tp)
return
}
metas, err := syncFunc(c, start, end)
if err != nil {
log.Error("s.syncVoteBIZAvs err(%v)", err)
return
}
rows, err = s.saveBubbleMeta(c, metas, time.Now())
if err != nil {
log.Error("s.saveBubbleMeta err(%v)", err)
}
return
}
// SyncIncomeBubbleMetaTask sync meta data for income bubble to growup
func (s *Service) SyncIncomeBubbleMetaTask(c context.Context, date time.Time) (err error) {
defer func() {
GetTaskService().SetTaskStatus(c, TaskBubbleMeta, date.Format(_layout), err)
}()
var (
lock sync.Mutex
group errgroup.Group
syncMetaFuncM = s.syncBubbleMetaFuncM()
start, end = date, date.AddDate(0, 0, 1)
metas []*bubbleMeta
)
for _, syncMetaFunc := range syncMetaFuncM {
group.Go(func() error {
var data []*bubbleMeta
data, err = syncMetaFunc(c, start, end)
if err != nil {
return err
}
lock.Lock()
metas = append(metas, data...)
lock.Unlock()
return nil
})
}
err = group.Wait()
if err != nil {
return
}
_, err = s.saveBubbleMeta(c, metas, date)
if err != nil {
log.Error("s.saveBubbleMeta err(%v)", err)
}
return
}
func (s *Service) avToBType(bubbleMeta map[int64][]int) map[int64]int {
var (
res = make(map[int64]int)
typeToRatio = make(map[int]float64)
chooseBType = func(bTypes []int) (bType int) {
if len(bTypes) == 1 {
return bTypes[0]
}
sort.Slice(bTypes, func(i, j int) bool {
bti, btj := bTypes[i], bTypes[j]
if typeToRatio[bti] == typeToRatio[btj] {
return bti < btj
}
return typeToRatio[bti] < typeToRatio[btj]
})
return bTypes[0]
}
)
for _, v := range s.conf.Bubble.BRatio {
typeToRatio[v.BType] = v.Ratio
}
for avID, bTypes := range bubbleMeta {
res[avID] = chooseBType(bTypes)
}
return res
}
// SnapshotBubbleIncomeTask get income lottery
func (s *Service) SnapshotBubbleIncomeTask(c context.Context, date time.Time) (err error) {
defer func() {
GetTaskService().SetTaskStatus(c, TaskSnapshotBubbleIncome, date.Format("2006-01-02"), err)
}()
err = GetTaskService().TaskReady(c, date.Format("2006-01-02"), TaskCreativeIncome)
if err != nil {
return
}
bubbleMeta, err := s.getBubbleMeta(c)
if err != nil {
log.Error("s.getBubbleAVs error(%v)", err)
return
}
avToBType := s.avToBType(bubbleMeta)
var (
eg errgroup.Group
incomeCh = make(chan []*model.IncomeInfo, 2000)
snapshotCh = make(chan []*model.IncomeInfo, 2000)
)
// get av income by date
eg.Go(func() (err error) {
defer close(incomeCh)
var from int64
for {
var infos []*model.IncomeInfo
infos, err = s.dao.GetAvIncome(c, date, from, int64(_dbLimit))
if err != nil {
log.Error("dao.GetAvIncome error(%v)", err)
return
}
if len(infos) == 0 {
break
}
incomeCh <- infos
from = infos[len(infos)-1].ID
}
return
})
eg.Go(func() (err error) {
defer close(snapshotCh)
for income := range incomeCh {
lotteryIncome := make([]*model.IncomeInfo, 0)
for _, av := range income {
if bType, ok := avToBType[av.AVID]; ok {
av.BType = bType
lotteryIncome = append(lotteryIncome, av)
}
}
if len(lotteryIncome) > 0 {
snapshotCh <- lotteryIncome
}
}
return
})
eg.Go(func() (err error) {
for income := range snapshotCh {
_, err = s.dao.InsertBubbleIncome(c, assembleLotteryIncome(income))
if err != nil {
return
}
}
return
})
if err = eg.Wait(); err != nil {
log.Error("eg.Wait error(%v)", err)
}
return
}
func (s *Service) getBubbleMeta(c context.Context) (data map[int64][]int, err error) {
var id int64
data = make(map[int64][]int)
for {
var meta map[int64][]int
meta, id, err = s.income.GetBubbleMeta(c, id, int64(_dbLimit))
if err != nil {
return
}
if len(meta) == 0 {
break
}
for avID, bTypes := range meta {
data[avID] = append(data[avID], bTypes...)
}
}
return
}
func assembleLotteryIncome(as []*model.IncomeInfo) (values string) {
var buf bytes.Buffer
for _, a := range as {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(a.AVID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TagID, 10))
buf.WriteByte(',')
buf.WriteString("'" + a.UploadTime.Format("2006-01-02 15:04:05") + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TaxMoney, 10))
buf.WriteByte(',')
buf.WriteString("'" + a.Date.Format(_layout) + "'")
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.BaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(a.BType))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values = buf.String()
buf.Reset()
return
}

53
app/job/main/growup/service/recompute.go Normal file
View File

@@ -0,0 +1,53 @@
package service
import (
"context"
"go-common/app/job/main/growup/model"
)
// AvIncomes av_income
func (s *Service) AvIncomes(c context.Context, mid int64, date string) (result map[int64]*model.Patch, err error) {
avs, err := s.dao.GetAvs(c, date, mid)
if err != nil {
return
}
var avIds []int64
for avID := range avs {
avIds = append(avIds, avID)
}
charges, err := s.dao.GetAvCharges(c, avIds, date)
if err != nil {
return
}
result = avIncomes(charges, avs)
return
}
// result key: av_id, value: income
func avIncomes(avCharges map[int64]int64, avs map[int64]*model.Av) (result map[int64]*model.Patch) {
var totalCharge int64
for _, charge := range avCharges {
totalCharge += charge
}
tax := int64(Round(Tax(Div(float64(totalCharge), 100))*100, 0))
netIncome := totalCharge - tax
percent := Div(float64(netIncome), float64(totalCharge))
result = make(map[int64]*model.Patch)
for avID, charge := range avCharges {
avIncome := int64(float64(charge) * percent)
avTax := int64(Round(Mul(float64(tax), Div(float64(avIncome), float64(netIncome))), 0))
result[avID] = &model.Patch{
Tax: avTax,
Income: avIncome,
OldTax: avs[avID].TaxMoney,
OldIncome: avs[avID].Income,
MID: avs[avID].MID,
TagID: avs[avID].TagID,
}
}
return
}

20
app/job/main/growup/service/recompute_test.go Normal file
View File

@@ -0,0 +1,20 @@
package service
import (
"context"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_AvIncomes(t *testing.T) {
Convey("growup-job AvIncomes", t, WithService(func(s *Service) {
mid := int64(1)
date := time.Now().Add(-24 * time.Hour)
t := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.Local)
res, err := s.AvIncomes(context.Background(), mid, t.Format("2006-01-02 15:04:05"))
So(err, ShouldBeNil)
So(res, ShouldNotBeEmpty)
}))
}

203
app/job/main/growup/service/sendincome.go Normal file
View File

@@ -0,0 +1,203 @@
package service
import (
"context"
"encoding/csv"
"fmt"
"io"
"os"
"sort"
"strconv"
"strings"
"time"
"go-common/app/job/main/growup/model"
"go-common/library/log"
)
const (
_income = "%d年%d月%d日收入统计"
)
func (s *Service) execIncome(c context.Context, date time.Time) (body string, err error) {
upInfo, err := s.getUpIncome(c, date)
if err != nil {
log.Error("s.execIncome s.getUpIncome error(%v)", err)
return
}
avInfo, err := s.getAvIncome(c, date)
if err != nil {
log.Error("s.execIncome s.getAvIncome error(%v)", err)
return
}
totalIncome, err := s.getTotalIncome(c)
if err != nil {
log.Error("s.execIncome s.getTotalIncome error(%v)", err)
return
}
var income, avgUpIncome, avgAVIncome, upCnt int64
for _, up := range upInfo {
income += up.Income
upCnt++
}
if upCnt > 0 {
avgUpIncome = income / upCnt // avg up income
}
if len(avInfo) > 0 {
avgAVIncome = income / int64(len(avInfo)) // avg av income
}
midAVIncome := getMIDAVIncome(avInfo)
midUpIncome := getMIDUpIncome(upInfo)
data := [][]string{
{fmt.Sprintf(_income, date.Year(), date.Month(), date.Day())},
{"日期", "新增收入(元)", "累计收入(元)", "获得收入UP数", "UP平均收入(元)", "UP收入中位数(元)", "获得收入稿件数", "稿件平均收入(元)", "稿件收入中位数(元)"},
{fmt.Sprintf("%d-%d-%d", date.Year(), date.Month(), date.Day()), strconv.FormatFloat(float64(income)/100, 'f', 2, 32), strconv.FormatFloat(float64(totalIncome)/100, 'f', 2, 32), strconv.FormatInt(upCnt, 10), strconv.FormatFloat(float64(avgUpIncome)/100, 'f', 2, 32), strconv.FormatFloat(float64(midUpIncome)/100, 'f', 2, 32), strconv.Itoa(len(avInfo)), strconv.FormatFloat(float64(avgAVIncome)/100, 'f', 2, 32), strconv.FormatFloat(float64(midAVIncome)/100, 'f', 2, 32)},
}
f, err := os.Create("upincome.csv")
if err != nil {
log.Error("growup-job create upincome.csv error(%v)", err)
return
}
defer f.Close()
w := csv.NewWriter(f)
w.WriteAll(data)
w.Flush()
rfile, err := os.Open("upincome.csv")
if err != nil {
log.Error("growup-job open upincome.csv error(%v)", err)
return
}
defer rfile.Close()
r := csv.NewReader(rfile)
for {
var strs []string
strs, err = r.Read()
if err == io.EOF {
break
}
body += fmt.Sprintf("<tr><td>%s</td></tr>", strings.Join(strs, "</td><td>"))
}
err = os.Remove("upincome.csv")
if err != nil {
log.Error("growup-job s.execIncome remove upincome.csv error(%v)", err)
}
return
}
func (s *Service) getTotalIncome(c context.Context) (totalIncome int64, err error) {
var (
totalInfos, infos []*model.MIDInfo
from, limit int64
)
limit = 2000
for {
infos, err = s.dao.GetUpTotalIncome(c, from, limit)
if err != nil {
log.Error("s.getTotalIncome email.GetUpTotalIncome error(%v)", err)
return
}
totalInfos = append(totalInfos, infos...)
if int64(len(infos)) < limit {
break
}
from = infos[len(infos)-1].ID
}
for _, v := range totalInfos {
if v.IsDeleted == 0 {
totalIncome += v.TotalIncome
}
}
return
}
func (s *Service) getUpIncome(c context.Context, date time.Time) (totalInfos []*model.MIDInfo, err error) {
var (
infos []*model.MIDInfo
from, limit int64
)
limit = 2000
for {
infos, err = s.dao.GetUpIncome(c, date, from, limit)
if err != nil {
log.Error("s.getUpIncome dao.GetUpIncome error(%v)", err)
return
}
totalInfos = append(totalInfos, infos...)
if int64(len(infos)) < limit {
break
}
from = infos[len(infos)-1].ID
}
return
}
func (s *Service) getAvIncome(c context.Context, date time.Time) (totalInfos []*model.IncomeInfo, err error) {
var (
infos []*model.IncomeInfo
from, limit int64
)
limit = 3000
for {
infos, err = s.dao.GetAvIncome(c, date, from, limit)
if err != nil {
log.Error("s.getAvIncome dao.GetAvIncome error(%v)", err)
return
}
for _, i := range infos {
if i.Income > 0 {
totalInfos = append(totalInfos, i)
}
}
if int64(len(infos)) < limit {
break
}
from = infos[len(infos)-1].ID
}
return
}
func getMIDAVIncome(avs []*model.IncomeInfo) (income int64) {
var cnt = len(avs)
if cnt <= 0 {
return
}
if cnt == 1 {
income = avs[0].Income
return
}
sort.SliceStable(avs, func(i, j int) bool {
return avs[i].Income < avs[j].Income
})
if cnt%2 == 0 {
income = (avs[cnt/2].Income + avs[cnt/2-1].Income) / 2
} else {
income = avs[cnt/2].Income
}
return
}
func getMIDUpIncome(ups []*model.MIDInfo) (income int64) {
var cnt = len(ups)
if cnt == 0 {
return
}
if cnt == 1 {
income = ups[0].Income
return
}
sort.SliceStable(ups, func(i, j int) bool {
return ups[i].Income < ups[j].Income
})
if cnt%2 == 0 {
income = (ups[cnt/2].Income + ups[cnt/2-1].Income) / 2
} else {
income = ups[cnt/2].Income
}
return
}

320
app/job/main/growup/service/service.go Normal file
View File

@@ -0,0 +1,320 @@
package service
import (
"context"
"fmt"
"time"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/dao"
"go-common/app/job/main/growup/dao/charge"
"go-common/app/job/main/growup/dao/dataplatform"
"go-common/app/job/main/growup/dao/email"
"go-common/app/job/main/growup/dao/income"
"go-common/app/job/main/growup/dao/tag"
"go-common/app/job/main/growup/service/ctrl"
"go-common/library/log"
"go-common/library/queue/databus"
)
// Service struct
type Service struct {
conf *conf.Config
dao *dao.Dao
email *email.Dao
dp *dataplatform.Dao
tag *tag.Dao
income *income.Dao
charge *charge.Dao
// databus sub
arcSub *databus.Databus
}
// New fn
func New(c *conf.Config, executor ctrl.Executor) (s *Service) {
s = &Service{
conf: c,
dao: dao.New(c),
email: email.New(c),
dp: dataplatform.New(c),
tag: tag.New(c),
income: income.New(c),
charge: charge.New(c),
arcSub: databus.New(c.ArchiveSub),
}
// init task status service
taskSvr = &taskService{
dao: s.dao,
dp: s.dp,
}
log.Info("service start")
executor.Submit(
s.checkExpired,
s.updateDateBlacklist,
s.sendMail,
s.updateCheat,
s.mailTagIncome,
s.calCreativeActivity,
s.updateUpInfoVideo,
s.creativeBudget,
s.checkAvBreach,
s.checkUpPunish,
s.archiveConsume,
s.syncIncomeBubbleMetaTask,
s.snapshotIncomeBubbleTask,
)
return s
}
// check the account state of video up
func (s *Service) checkExpired(ctx context.Context) {
for {
// up_info_video
s.expiredCheck(0, 5, 0, "up_info_video") // check all expired from quit to default
s.expiredCheck(0, 7, 3, "up_info_video") // check all expired from forbidden to signed
s.expiredCheck(1, 4, 0, "up_info_video") // check ugc expired from reject to default
s.expiredCheck(2, 4, 1, "up_info_video") // check pgc expired from reject to pre-audit
// up_info_column
s.expiredCheck(0, 5, 0, "up_info_column") // check all expired from quit to default
s.expiredCheck(0, 7, 3, "up_info_column") // check all expired from forbidden to signed
s.expiredCheck(1, 4, 0, "up_info_column") // check ugc expired from reject to default
s.expiredCheck(2, 4, 1, "up_info_column") // check pgc expired from reject to pre-audit
time.Sleep(NextDay(0, 0, 0))
}
}
// updateDateBlacklist update blacklist at 17:00 every day
func (s *Service) updateDateBlacklist(ctx context.Context) {
for {
time.Sleep(NextDay(17, 0, 0))
msg := ""
log.Info("Exec growup-job updateDateBlacklist begin:%v", time.Now().Format("2006-01-02 15:04:05"))
err := s.UpdateBlacklist(context.TODO())
if err != nil {
msg = fmt.Sprintf("UpdateBlacklist error(%v)", err)
} else {
msg = "Success"
}
err = s.email.SendMail(time.Now(), msg, "创作激励同步黑名单%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("Exec growup-job updateDateBlacklist end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) updateUpInfoVideo(ctx context.Context) {
var mailReceivers []string
for _, v := range s.conf.Mail.Send {
if v.Type == 4 {
mailReceivers = v.Addr
break
}
}
for {
time.Sleep(NextDay(11, 0, 0))
log.Info("Exec growup-job updateUpInfoVideo begin:%v", time.Now().Format("2006-01-02 15:04:05"))
msg := ""
err := s.UpdateUpInfo(context.TODO())
if err != nil {
msg = fmt.Sprintf("s.UpdateUpInfo error(%v)", err)
} else {
msg, err = s.autoExamination(context.TODO())
if err != nil {
msg = fmt.Sprintf("s.autoExamination error(%v)", err)
}
}
err = s.email.SendMail(time.Now(), msg, "创作激励自动过审%d年%d月%d日", mailReceivers...)
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("Exec growup-job updateUpInfoVideo end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) sendMail(ctx context.Context) {
for {
time.Sleep(NextDay(12, 0, 0))
log.Info("Exec growup-job sendMail begin:%v", time.Now().Format("2006-01-02 15:04:05"))
s.CombineMails()
log.Info("Exec growup-job sendMail end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) updateCheat(ctx context.Context) {
for {
time.Sleep(NextDay(13, 0, 0))
log.Info("Exec growup-job updateCheat begin: %v", time.Now().Format("2006-01-02 15:04:05"))
t := time.Now().Add(-24 * time.Hour)
s.CheatStatistics(context.TODO(), t)
log.Info("Exec growup-job updateCheat end: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) mailTagIncome(ctx context.Context) {
for {
time.Sleep(NextDay(19, 30, 0))
log.Info("Exec growup-job mailTagIncome begin: %v", time.Now().Format("2006-01-02 15:04:05"))
t := time.Now().Add(-24 * time.Hour)
date := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
s.sendTagIncome(context.TODO(), date)
log.Info("End growup-job mailTagIncome: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) calCreativeActivity(ctx context.Context) {
for {
time.Sleep(NextDay(15, 30, 0))
msg := ""
log.Info("Exec growup-job calCreativeActivity begin: %v", time.Now().Format("2006-01-02 15:04:05"))
t := time.Now().Add(-24 * time.Hour)
date := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
err := s.CreativeActivity(context.TODO(), date)
if err != nil {
msg = fmt.Sprintf("calCreativeActivity error(%v)", err)
} else {
msg = "Success"
}
err = s.email.SendMail(date, msg, "创作激励活动每日计算%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("End growup-job calCreativeActivity: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) creativeBudget(ctx context.Context) {
for {
time.Sleep(NextDay(20, 0, 0))
msg := ""
log.Info("Exec growup-job creativeBudget begin: %v", time.Now().Format("2006-01-02 15:04:05"))
t := time.Now().Add(-24 * time.Hour)
date := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
err := s.CreativeBudget(context.TODO(), date)
if err != nil {
msg = fmt.Sprintf("creativeBudget error(%v)", err)
} else {
msg = "Success"
}
err = s.email.SendMail(date, msg, "创作激励预算每日计算%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("End growup-job creativeBudget: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) checkAvBreach(ctx context.Context) {
var mailReceivers []string
for _, v := range s.conf.Mail.Send {
if v.Type == 4 {
mailReceivers = v.Addr
break
}
}
for {
time.Sleep(NextDay(0, 0, 0))
log.Info("Exec growup-job check and auto breach av begin: %v", time.Now().Format("2006-01-02 15:04:05"))
t := time.Now().Add(-48 * time.Hour)
date := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
msg, err := s.autoAvBreach(context.TODO(), date.Format(_layout))
if err != nil {
msg = fmt.Sprintf("autoAvBreach error(%v)", err)
}
err = s.email.SendMail(date, msg, "创作激励自制转转载每日扣除%d年%d月%d日", mailReceivers...)
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("End growup-job autoAvBreach: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) checkUpPunish(ctx context.Context) {
var mailReceivers []string
for _, v := range s.conf.Mail.Send {
if v.Type == 4 {
mailReceivers = v.Addr
break
}
}
for {
time.Sleep(NextDay(10, 0, 0))
if time.Now().Weekday() != time.Monday {
continue
}
log.Info("Exec growup-job checkUpPunish begin: %v", time.Now().Format("2006-01-02 15:04:05"))
msg, err := s.autoUpPunish(context.TODO())
if err != nil {
msg = fmt.Sprintf("s.autoUpPunish error(%v)", err)
}
err = s.email.SendMail(time.Now(), msg, "创作激励自制转转载处罚%d年%d月%d日", mailReceivers...)
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("End growup-job checkUpPunish: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) syncIncomeBubbleMetaTask(ctx context.Context) {
for {
msg := "ok"
time.Sleep(NextDay(12, 0, 0))
log.Info("Exec growup-job syncIncomeBubbleMetaTask begin: %v", time.Now().Format("2006-01-02 15:04:05"))
t := time.Now().Add(-24 * time.Hour)
date := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
err := s.SyncIncomeBubbleMetaTask(context.TODO(), date)
if err != nil {
msg = fmt.Sprintf("s.syncIncomeBubbleMetaTask error(%v)", err)
}
err = s.email.SendMail(time.Now(), msg, "创作激励同步动态转发抽奖%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("End growup-job syncIncomeBubbleMetaTask: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) snapshotIncomeBubbleTask(ctx context.Context) {
for {
msg := "ok"
time.Sleep(NextDay(19, 0, 0))
log.Info("Exec growup-job snapshotIncomeBubbleTask begin: %v", time.Now().Format("2006-01-02 15:04:05"))
t := time.Now().Add(-24 * time.Hour)
date := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
err := s.SnapshotBubbleIncomeTask(context.TODO(), date)
if err != nil {
msg = fmt.Sprintf("s.snapshotIncomeBubbleTask error(%v)", err)
}
err = s.email.SendMail(time.Now(), msg, "创作激励动态转发抽奖收入同步%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("End growup-job snapshotIncomeBubbleTask: %v", time.Now().Format("2006-01-02 15:04:05"))
}
}
// NextDay next day x hours
func NextDay(hour, min, second int) time.Duration {
n := time.Now()
d := time.Date(n.Year(), n.Month(), n.Day(), hour, min, second, 0, n.Location())
for !d.After(n) {
d = d.AddDate(0, 0, 1)
}
return time.Until(d)
}
// Close close the service
func (s *Service) Close() {
s.dao.Close()
s.tag.Close()
s.income.Close()
s.charge.Close()
}
// Ping check server ok.
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}

36
app/job/main/growup/service/service_test.go Normal file
View File

@@ -0,0 +1,36 @@
package service
import (
"flag"
"path/filepath"
"testing"
"time"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/service/ctrl"
. "github.com/smartystreets/goconvey/convey"
)
func Test_Temp(t *testing.T) {
Convey("temp test", t, func() {})
}
var (
srv *Service
)
func init() {
dir, _ := filepath.Abs("../cmd/growup-job.toml")
flag.Set("conf", dir)
conf.Init()
srv = New(conf.Conf, ctrl.NewUnboundedExecutor())
time.Sleep(time.Second)
}
func WithService(f func(s *Service)) func() {
return func() {
// Reset(func() { CleanCache() })
f(srv)
}
}

102
app/job/main/growup/service/signedups.go Normal file
View File

@@ -0,0 +1,102 @@
package service
import (
"context"
"encoding/csv"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"go-common/library/log"
)
const (
_SignedUps = "%d年%d月%d日加入人数统计"
)
func (s *Service) execSignedUps(c context.Context, date time.Time) (body string, err error) {
tm1, err := s.dao.GetDateSignedUps(c, date, date.Add(24*time.Hour))
if err != nil {
log.Error("s.execSignedUps dao.GetDateSignedUps T-1(%v) signed ups error(%v)", date, err)
return
}
tm2, err := s.dao.GetDateSignedUps(c, date.Add(-24*time.Hour), date)
if err != nil {
log.Error("s.execSignedUps dao.GetDateSignedUps T-2(%v) signed ups error(%v)", date.Add(-24*time.Hour), err)
return
}
tm8, err := s.dao.GetDateSignedUps(c, date.Add(-24*7*time.Hour), date.Add(-24*6*time.Hour))
if err != nil {
log.Error("s.execSignedUps dao.GetDateSignedUps T-8(%v) signed ups error(%v)", date.Add(-24*6*time.Hour), err)
return
}
log.Info("t-8:%d, %d-%d-%d ", tm8, date.Add(-24*7*time.Hour).Year(), date.Add(-24*7*time.Hour).Month(), date.Add(-24*7*time.Hour).Day())
sa, err := s.dao.GetAllSignedUps(c, date)
if err != nil {
log.Error("s.execSignedUps dao.GetAllSignedUps date(%v) error(%v)", date, err)
return
}
applyDm1, err := s.dao.GetVideoApplyUpCount(c, date, date.Add(24*time.Hour))
if err != nil {
log.Error("s.execSignedUps dao.GetVideoUpCount applyDm1 error(%v)", err)
return
}
applyDm2, err := s.dao.GetVideoApplyUpCount(c, date.Add(-24*time.Hour), date)
if err != nil {
log.Error("s.execSignedUps dao.GetVideoUpCount applyDm2 error(%v)", err)
return
}
var du, wu float64
if tm2 > 0 {
du = float64(tm1-tm2) / float64(tm2) * 100
}
if tm8 > 0 {
wu = float64(tm1-tm8) / float64(tm8) * 100
}
var applyDu float64
if applyDm2 > 0 {
applyDu = float64(applyDm1-applyDm2) / float64(applyDm2) * 100
}
data := [][]string{
{fmt.Sprintf(_SignedUps, date.Year(), date.Month(), date.Day())},
{"日期", "已签约新增加人数", "已签约人数日增长率", "已签约人数同比上周增长率", "累计签约人数", "已申请新增加人数", "已申请人数日增长率"},
{fmt.Sprintf("%d-%d-%d", date.Year(), date.Month(), date.Day()), strconv.Itoa(tm1), strconv.FormatFloat(du, 'f', 2, 32), strconv.FormatFloat(wu, 'f', 2, 32), strconv.Itoa(sa), strconv.Itoa(applyDm1), strconv.FormatFloat(applyDu, 'f', 2, 32)},
}
f, err := os.Create("signedups.csv")
if err != nil {
log.Error("s.execSignedUps create signedups.csv error(%v)", err)
return
}
log.Info("tm1:%d, tm2:%d, tm8:%d, sa:%d", tm1, tm2, tm8, sa)
defer f.Close()
w := csv.NewWriter(f)
w.WriteAll(data)
w.Flush()
rfile, err := os.Open("signedups.csv")
if err != nil {
log.Error("s.execSignedUps open signedups.csv error(%v)", err)
return
}
defer rfile.Close()
r := csv.NewReader(rfile)
for {
var strs []string
strs, err = r.Read()
if err == io.EOF {
break
}
body += fmt.Sprintf("<tr><td>%s</td></tr>", strings.Join(strs, "</td><td>"))
}
err = os.Remove("signedups.csv")
if err != nil {
log.Error("growup-job s.execSignedUps remove signedups.csv error(%v)", err)
}
return
}

373
app/job/main/growup/service/tag.go Normal file
View File

@@ -0,0 +1,373 @@
package service
import (
"bytes"
"context"
"strconv"
"time"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/model"
"go-common/library/log"
)
// DeleteAvRatio clean the av_charge_ratio
func (s *Service) DeleteAvRatio(c context.Context, limit int64) (rows int64, err error) {
rows, err = s.dao.DelAvRatio(c, limit)
if err != nil {
log.Error("growup-job s.DeleteAvRatio error(%v)", err)
}
return
}
// DeleteUpIncome clean the up_tag_income
func (s *Service) DeleteUpIncome(c context.Context, limit int64) (rows int64, err error) {
rows, err = s.dao.DelIncome(c, limit)
if err != nil {
log.Error("growup-job s.DeleteUpIncome error(%v)", err)
}
return
}
// DelActivity clean activity_info table.
func (s *Service) DelActivity(c context.Context, limit int64) (rows int64, err error) {
rows, err = s.dao.DelActivity(c, limit)
if err != nil {
log.Error("growup-job s.DelActivity error(%v)", err)
}
return
}
func (s *Service) aids(c context.Context, categoryID int, date time.Time) (aids []int64, err error) {
var offset int64
m := int(date.Month())
var sm string
if m < 10 {
sm += "0"
sm += strconv.Itoa(m)
}
for {
var as []*model.AID
as, err = s.dao.AIDs(c, categoryID, offset, conf.Conf.Ratio.Limit, sm, date)
if err != nil {
return
}
if len(as) == 0 {
break
}
offset += int64(len(as))
for _, a := range as {
if a.IsDeleted == 0 && a.IncCharge > 0 {
aids = append(aids, a.AvID)
}
}
}
return
}
func (s *Service) aidsByMID(c context.Context, mid int64, categoryID int, date time.Time) (aids []int64, err error) {
var offset int64
m := int(date.Month())
var sm string
if m < 10 {
sm += "0"
sm += strconv.Itoa(m)
}
for {
var as []*model.AID
as, err = s.dao.AIDsByMID(c, mid, categoryID, offset, conf.Conf.Ratio.Limit, sm, date)
if err != nil {
return
}
if len(as) == 0 {
break
}
offset += int64(len(as))
for _, a := range as {
if a.IsDeleted == 0 && a.IncCharge > 0 {
aids = append(aids, a.AvID)
}
}
}
return
}
// ExecRatioForHTTP exec http
func (s *Service) ExecRatioForHTTP(c context.Context, year int, month int, day int) (err error) {
t := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local)
err = s.execAvRatio(c, t)
if err != nil {
log.Error("growup-job s.execAvRatio error!(%v)", err)
}
return
}
func (s *Service) execAvRatio(c context.Context, date time.Time) (err error) {
commonTags, err := s.dao.CommonTagInfo(c, date)
if err != nil {
return
}
m := make(map[int64]*model.TagUpInfo)
//m := make(map[int64]int)
for _, commonTag := range commonTags {
var as []int64
as, err = s.aids(c, commonTag.Category, date)
if err != nil {
return
}
getAvRatio(as, commonTag.TagID, commonTag.Ratio, m)
}
noCommonTags, err := s.dao.NoCommonTagInfo(c, date)
if err != nil {
return
}
for _, noCommonTag := range noCommonTags {
var as []int64
as, err = s.aidsByMID(c, noCommonTag.MID, noCommonTag.Category, date)
if err != nil {
return
}
getAvRatio(as, noCommonTag.TagID, noCommonTag.Ratio, m)
}
activityTags, err := s.dao.ActivityTagInfo(c, date)
if err != nil {
log.Error("s.dao.ActivityTagInfo error(%v)", err)
return
}
var res []*model.ActivityAVInfo
res, err = s.getActivityInfo(c, activityTags)
if err != nil {
log.Error("s.getActivityaInfo error(%v)", err)
return
}
tm, err := s.handleActivityData(c, activityTags, res)
if err != nil {
log.Error("s.handleActivityData error(%v)", err)
return
}
for key, value := range tm {
var avIDs []int64
var ratio int
for _, v := range value {
avIDs = append(avIDs, v.AVID)
ratio = v.Ratio
}
getAvRatio(avIDs, key, ratio, m)
}
err = s.insertAvRatio(c, m)
if err != nil {
log.Error("s.insertAvRatio error(%v)", err)
return
}
log.Info("s.insertAvRatio av_map len:%d", len(m))
return
}
func (s *Service) insertAvRatio(c context.Context, m map[int64]*model.TagUpInfo) (err error) {
var buf bytes.Buffer
var count int64
for k, v := range m {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(v.TagID, 10))
buf.WriteString(",")
buf.WriteString(strconv.FormatInt(k, 10))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(v.Ratio))
buf.WriteString("),")
count++
if count%2000 == 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.InsertRatio(c, buf.String())
if err != nil {
log.Error("growup-job s.insertAvRatio error(%v)", err)
return
}
buf.Reset()
}
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.InsertRatio(c, buf.String())
if err != nil {
log.Error("growup-job s.insertAvRatio error(%v)", err)
return
}
}
return
}
func (s *Service) handleActivityData(c context.Context, tag []*model.TagUpInfo, res []*model.ActivityAVInfo) (tm map[int64][]*model.ActivityAVInfo, err error) {
tm = make(map[int64][]*model.ActivityAVInfo)
types, err := s.getAllTypes(c)
if err != nil {
log.Error("s.handleActivityData getAllTypes error(%v)", err)
return
}
for _, act := range tag {
for _, r := range res {
if act.IsCommon == 0 {
_, err = s.dao.ActivityMIDExist(c, act.TagID, r.MID)
if err != nil {
err = nil
continue
}
}
if (getPIDByID(types, int16(r.Category)) == int16(act.Category)) && (r.ActivityID == act.ActivityID) && (act.Business == 1) {
_, ok := tm[act.TagID]
if !ok {
arr := make([]*model.ActivityAVInfo, 0)
v := &model.ActivityAVInfo{ActivityID: r.ActivityID, TagID: act.TagID, AVID: r.AVID, MID: r.MID, Category: r.Category, Ratio: act.Ratio}
arr = append(arr, v)
tm[act.TagID] = arr
} else {
v := &model.ActivityAVInfo{ActivityID: r.ActivityID, TagID: act.TagID, AVID: r.AVID, MID: r.MID, Category: r.Category, Ratio: act.Ratio}
tm[act.TagID] = append(tm[act.TagID], v)
}
}
}
}
return
}
func (s *Service) getActivityInfo(c context.Context, activityTags []*model.TagUpInfo) (res []*model.ActivityAVInfo, err error) {
res = make([]*model.ActivityAVInfo, 0)
am := make(map[int64]bool)
var ai []int64
for _, act := range activityTags {
if _, ok := am[act.ActivityID]; !ok {
am[act.ActivityID] = true
ai = append(ai, act.ActivityID)
}
}
var ts []*model.ActivityAVInfo
var tv []int64
var total int
for i, a := range ai {
tv = append(tv, a)
if (i+1)%20 == 0 {
pn := 1
ps := 30
ts, total, err = s.dao.GetActivityAVInfo(c, pn, ps, tv)
if err != nil {
log.Error("s.dao.ActivityTagInfo error(%v)", err)
return
}
if len(ts) == 0 {
return
}
for ps*pn <= (total + ps) {
ts, total, err = s.dao.GetActivityAVInfo(c, pn, ps, tv)
if err != nil {
log.Error("s.dao.ActivityTagInfo error(%v)", err)
return
}
pn++
res = append(res, ts...)
}
res = append(res, ts...)
ts = ts[0:0:0]
tv = tv[0:0:0]
}
}
if len(tv) > 0 {
pn := 1
ps := 30
ts, total, err = s.dao.GetActivityAVInfo(c, pn, ps, tv)
if err != nil {
log.Error("s.dao.ActivityTagInfo error(%v)", err)
return
}
if len(ts) == 0 {
return
}
for pn*ps <= (total + ps) {
ts, total, err = s.dao.GetActivityAVInfo(c, pn, ps, tv)
if err != nil {
log.Error("s.dao.ActivityTagInfo error(%v)", err)
return
}
pn++
res = append(res, ts...)
}
}
log.Info("res len(%d), total(%d)", len(res), total)
return
}
// UpdateAvRatio update av ratio everyday
func (s *Service) UpdateAvRatio() (err error) {
c := context.TODO()
yesterday := time.Now().Add(-time.Hour * 24)
t := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location())
err = s.execAvRatio(c, t)
if err != nil {
log.Error("growup-job s.execAvRatio error!")
}
return
}
func getAvRatio(aids []int64, tagID int64, ratio int, m map[int64]*model.TagUpInfo) {
for _, avid := range aids {
v, ok := m[avid]
if ok {
if v.Ratio < ratio {
m[avid].Ratio = ratio
m[avid].TagID = tagID
}
} else {
a := &model.TagUpInfo{TagID: tagID, Ratio: ratio}
m[avid] = a
}
}
}
// ExecIncomeForHTTP income
func (s *Service) ExecIncomeForHTTP(c context.Context, year int, month int, day int) (err error) {
t := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local)
for {
rows, _ := s.DeleteAvRatio(c, conf.Conf.Ratio.Num)
time.Sleep(time.Duration(conf.Conf.Ratio.Sleep) * time.Millisecond)
if rows == 0 {
break
}
}
for {
rows, _ := s.DelActivity(c, conf.Conf.Ratio.Num)
time.Sleep(time.Duration(conf.Conf.Ratio.Sleep) * time.Millisecond)
if rows == 0 {
break
}
}
err = s.ExecRatioForHTTP(c, year, month, day)
if err != nil {
log.Error("Exec avRatio from http error!(%v)", err)
return
}
err = s.InsertTagIncome(c, t)
if err != nil {
log.Error("s.InsertTagIncome error!")
}
return
}
func (s *Service) getAllTypes(c context.Context) (tm map[int16]int16, err error) {
tm, err = s.dao.GetAllTypes(c)
if err != nil {
log.Error("s.getAllTypes error(%v)", err)
}
return
}
func getPIDByID(tm map[int16]int16, id int16) (pid int16) {
for k, v := range tm {
if k == id {
return v
}
}
return
}

71
app/job/main/growup/service/tag/BUILD Normal file
View File

@@ -0,0 +1,71 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"service_test.go",
"tag_test.go",
"tag_up_info_test.go",
"up_tag_income_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"activity.go",
"archive_charge.go",
"archive_tag.go",
"category.go",
"charge_ratio.go",
"extra.go",
"income.go",
"service.go",
"tag.go",
"tag_up_info.go",
"up_tag.go",
"up_tag_income.go",
],
importpath = "go-common/app/job/main/growup/service/tag",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/growup/conf:go_default_library",
"//app/job/main/growup/dao/email:go_default_library",
"//app/job/main/growup/dao/tag:go_default_library",
"//app/job/main/growup/model/tag:go_default_library",
"//app/job/main/growup/service:go_default_library",
"//app/job/main/growup/service/ctrl:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/time:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

243
app/job/main/growup/service/tag/activity.go Normal file
View File

@@ -0,0 +1,243 @@
package tag
import (
"bytes"
"context"
"fmt"
"strconv"
"time"
model "go-common/app/job/main/growup/model/tag"
"go-common/library/log"
xtime "go-common/library/time"
)
func (s *Service) addActivityInfo(c context.Context, date string, tagAvs map[int64]*model.AvTagRatio, ctype int) (err error) {
tagAvMap := make(map[int64]bool)
for _, av := range tagAvs {
tagAvMap[av.AvID] = true
}
tagInfo, err := s.dao.GetTagInfoByDate(c, 0, ctype, date, date)
if err != nil {
log.Error("s.dao.GetTagInfoByDate error(%v)", err)
return
}
activityAvs, err := s.getActivityArchives(c, tagInfo, ctype)
if err != nil {
log.Error("s.getActivityArchives error(%v)", err)
return
}
avsExists, err := s.dao.ListActivityInfo(c)
if err != nil {
log.Error("s.ListActivityInfo error(%v)", err)
return
}
inList := make([]*model.ArchiveCharge, 0)
for _, av := range activityAvs {
if tagAvMap[av.AID] && !avsExists[av.AID] {
inList = append(inList, av)
}
}
log.Info("get %d avs, need to insert %d avs", len(activityAvs), len(inList))
err = s.insertActivityInfo(c, inList)
if err != nil {
log.Error("s.insertActivityInfo error(%v)", err)
}
return
}
func (s *Service) insertActivityInfo(c context.Context, avs []*model.ArchiveCharge) (err error) {
var buf bytes.Buffer
for _, a := range avs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(a.AID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.ActivityID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.CategoryID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TagID, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values := buf.String()
buf.Reset()
_, err = s.dao.InsertActivityInfo(c, values)
return
}
func (s *Service) getActivityArchives(c context.Context, tagInfo []*model.TagInfo, ctype int) (archives []*model.ArchiveCharge, err error) {
archives = make([]*model.ArchiveCharge, 0)
if ctype == _bgm {
return
}
activityMap := make(map[int64]struct{})
activityIDs := make([]int64, 0)
tagMap := make(map[int64]map[int64]int64) // map[activityID][categoryID]tagID
for _, tag := range tagInfo {
if tag.ActivityID == 0 {
continue
}
if _, ok := activityMap[tag.ActivityID]; !ok {
activityMap[tag.ActivityID] = struct{}{}
activityIDs = append(activityIDs, tag.ActivityID)
tagMap[tag.ActivityID] = make(map[int64]int64)
}
tagMap[tag.ActivityID][tag.CategoryID] = tag.ID
}
if len(activityIDs) == 0 {
return
}
activityInfos, err := s.getActivityInfos(c, activityIDs, ctype)
if err != nil {
log.Error("s.GetActivityInfos error(%v)", err)
return
}
// 获取一级分区和二级分区的对应关系 key:二级分区
categoryMap := make(map[int64]int64)
if ctype == _video {
categoryMap, err = s.dao.GetVideoTypes(c)
if err != nil {
log.Error("s.dao.GetAllTypes error(%v)", err)
return
}
} else if ctype == _column {
categoryMap, err = s.dao.GetColumnTypes(c)
if err != nil {
log.Error("s.dao.GetColumnTypes error(%v)", err)
return
}
}
archives, err = convertArchiveCharge(activityInfos, categoryMap, tagMap)
if err != nil {
log.Error("convertArchiveCharge error(%v)", err)
return
}
return
}
func convertArchiveCharge(activityInfo []*model.ActivityInfo, categoryMap map[int64]int64, tagMap map[int64]map[int64]int64) (archives []*model.ArchiveCharge, err error) {
archives = make([]*model.ArchiveCharge, 0)
for _, act := range activityInfo {
var uploadTime time.Time
uploadTime, err = time.Parse("2006-01-02 15:04:05", act.CDate)
if err != nil {
return
}
av := &model.ArchiveCharge{
AID: act.AvID,
MID: act.MID,
CategoryID: categoryMap[act.TypeID],
ActivityID: act.ActivityID,
UploadTime: xtime.Time(uploadTime.Unix()),
}
if t, ok := tagMap[av.ActivityID]; ok {
av.TagID = t[av.CategoryID]
}
archives = append(archives, av)
}
return
}
func (s *Service) getActivityInfos(c context.Context, activityID []int64, ctype int) ([]*model.ActivityInfo, error) {
switch ctype {
case _video:
return s.getVideoActivity(c, activityID)
case _column:
return s.getCmActivity(c, activityID)
}
return nil, fmt.Errorf("getActivityInfos ctype error")
}
func (s *Service) getVideoActivity(c context.Context, activityID []int64) (activityInfos []*model.ActivityInfo, err error) {
activityInfos = make([]*model.ActivityInfo, 0)
start, offset := 0, 20
if len(activityID) < offset {
offset = len(activityID)
}
for start+offset <= len(activityID) {
var act []*model.ActivityInfo
act, err = s.getVideoActivityInfos(c, activityID[start:start+offset])
if err != nil {
return
}
if len(act) != 0 {
activityInfos = append(activityInfos, act...)
}
start += offset
if start < len(activityID) && start+offset > len(activityID) {
offset = len(activityID) - start
}
}
return
}
func (s *Service) getVideoActivityInfos(c context.Context, activityID []int64) (acts []*model.ActivityInfo, err error) {
acts = make([]*model.ActivityInfo, 0)
page, size := 1, 30
for {
var act []*model.ActivityInfo
act, err = s.dao.GetVideoActivityInfo(c, activityID, page, size)
if err != nil {
return
}
if len(act) == 0 {
break
}
acts = append(acts, act...)
page++
// qps控制
if page%50 == 0 {
time.Sleep(1 * time.Second)
}
}
return
}
func (s *Service) getCmActivity(c context.Context, activityID []int64) (activityInfos []*model.ActivityInfo, err error) {
activityInfos = make([]*model.ActivityInfo, 0)
for _, id := range activityID {
var act []*model.ActivityInfo
act, err = s.getCmActivityInfo(c, id)
if err != nil {
return
}
if len(act) != 0 {
activityInfos = append(activityInfos, act...)
}
}
return
}
func (s *Service) getCmActivityInfo(c context.Context, id int64) (acts []*model.ActivityInfo, err error) {
acts = make([]*model.ActivityInfo, 0)
page, size := 1, 30
for {
var act []*model.ActivityInfo
act, err = s.dao.GetCmActivityInfo(c, id, page, size)
if err != nil {
return
}
if len(act) == 0 {
break
}
acts = append(acts, act...)
page++
// qps控制
if page%50 == 0 {
time.Sleep(1 * time.Second)
}
}
return
}

80
app/job/main/growup/service/tag/archive_charge.go Normal file
View File

@@ -0,0 +1,80 @@
package tag
import (
"context"
model "go-common/app/job/main/growup/model/tag"
)
// getAvDailyCharge get av_daily_charge
func (s *Service) getAvDailyCharge(c context.Context, month, query string) (avs []*model.ArchiveCharge, err error) {
if query == "" {
return
}
from, limit := 0, 2000
for {
var av []*model.ArchiveCharge
av, err = s.dao.GetAvDailyCharge(c, month, query, from, limit)
if err != nil {
return
}
for _, a := range av {
if a.IncCharge > 0 && a.IsDeleted == 0 {
avs = append(avs, a)
}
}
if len(av) < limit {
break
}
from += limit
}
return
}
// getCmDailyCharge get column_daily_charge
func (s *Service) getCmDailyCharge(c context.Context, query string) (cms []*model.ArchiveCharge, err error) {
if query == "" {
return
}
from, limit := 0, 2000
for {
var cm []*model.ArchiveCharge
cm, err = s.dao.GetCmDailyCharge(c, query, from, limit)
if err != nil {
return
}
for _, c := range cm {
if c.IncCharge > 0 && c.IsDeleted == 0 {
cms = append(cms, c)
}
}
if len(cm) < limit {
break
}
from += limit
}
return
}
func (s *Service) getBackgroundMusic(c context.Context, limit int64) (bgms []*model.ArchiveCharge, err error) {
var id int64
bm := make(map[int64]bool)
bgms = make([]*model.ArchiveCharge, 0)
for {
var bs []*model.ArchiveCharge
bs, id, err = s.dao.GetBgm(c, id, limit)
if err != nil {
return
}
if len(bs) == 0 {
break
}
for _, b := range bs {
if _, ok := bm[b.AID]; !ok {
bm[b.AID] = true
bgms = append(bgms, b)
}
}
}
return
}

271
app/job/main/growup/service/tag/archive_tag.go Normal file
View File

@@ -0,0 +1,271 @@
package tag
import (
"context"
"fmt"
"strings"
model "go-common/app/job/main/growup/model/tag"
"go-common/library/log"
)
var (
_video = 1
_column = 2
_bgm = 3
)
// IGetArchivesToTag get avs to tag
type IGetArchivesToTag func(c context.Context, date string, categoryIDs []int64) (archives []*model.ArchiveCharge, err error)
func (s *Service) tagArchiveRatio(c context.Context, date string, ctype int) (err error) {
// delete
err = s.delArchiveRatio(c, ctype)
if err != nil {
log.Error("s.delArchiveRatio error(%v)", err)
return
}
var getArchives IGetArchivesToTag
switch ctype {
case _video:
getArchives = s.getAvsBeforeCal
case _column:
getArchives = s.getCmsBeforeCal
case _bgm:
getArchives = s.getBgmBeforeCal
}
ratios, err := s.getTagArchive(c, date, ctype, getArchives)
if err != nil {
log.Error("s.getTagArchive error(%v)", err)
return
}
// insert
err = s.insertAvRatio(c, ratios, ctype)
if err != nil {
log.Error("s.InsertAvRatio error(%v)", err)
}
return
}
func (s *Service) getTagArchive(c context.Context, date string, ctype int, getArchives IGetArchivesToTag) (ratios map[int64]*model.AvTagRatio, err error) {
tagInfo, err := s.dao.GetTagInfoByDate(c, 0, ctype, date, date)
if err != nil {
log.Error("s.dao.GetTagInfoByDate error(%v)", err)
return
}
if len(tagInfo) == 0 {
return
}
// get tag_id and category_id
tagIDs := make([]int64, 0)
categoryIDs := make([]int64, 0)
categoryMap := make(map[int64]struct{})
for _, tag := range tagInfo {
tagIDs = append(tagIDs, tag.ID)
if tag.ActivityID != 0 {
continue
}
if _, ok := categoryMap[tag.CategoryID]; !ok {
categoryMap[tag.CategoryID] = struct{}{}
categoryIDs = append(categoryIDs, tag.CategoryID)
}
}
archives, err := getArchives(c, date, categoryIDs)
if err != nil {
log.Error("s.getAvs error(%v)", err)
return
}
log.Info("GET matched archives(%d)", len(archives))
// get archives by activityID
actArchives, err := s.getActivityArchives(c, tagInfo, ctype)
if err != nil {
log.Error("s.getActivityAvs error(%v)", err)
return
}
log.Info("GET matched avtivity archive(%d)", len(actArchives))
if len(actArchives) > 0 {
archives = append(archives, actArchives...)
}
if len(archives) <= 0 {
return
}
// 获取tag指定mid
tagMID, err := s.GetTagUpInfoMID(c, tagIDs)
if err != nil {
log.Error("s.GetTagUpInfoMID error(%v)", err)
return
}
ratios = getRatioByTagInfo(archives, tagInfo, tagMID)
return
}
// get avs from av_daily_charge
func (s *Service) getAvsBeforeCal(c context.Context, date string, categoryIDs []int64) (archives []*model.ArchiveCharge, err error) {
archives = make([]*model.ArchiveCharge, 0)
dateS := strings.Split(date, "-")
for _, id := range categoryIDs {
var avc []*model.ArchiveCharge
query := fmt.Sprintf("tag_id = %d AND date = '%s'", id, date)
avc, err = s.getAvDailyCharge(c, dateS[1], query)
if err != nil {
log.Error("s.getAvDailyCharge error(%v)", err)
return
}
archives = append(archives, avc...)
}
return
}
// get avs from av_income_statis
func (s *Service) getAvsAfterCal(c context.Context, date string, categoryIDs []int64) (archives []*model.ArchiveCharge, err error) {
return s.getAvIncomeStatis(c)
}
// get columns from column_daily_charge
func (s *Service) getCmsBeforeCal(c context.Context, date string, categoryIDs []int64) (archives []*model.ArchiveCharge, err error) {
archives = make([]*model.ArchiveCharge, 0)
for _, id := range categoryIDs {
var cm []*model.ArchiveCharge
query := fmt.Sprintf("tag_id = %d AND date = '%s'", id, date)
cm, err = s.getCmDailyCharge(c, query)
if err != nil {
log.Error("s.getAvDailyCharge error(%v)", err)
return
}
archives = append(archives, cm...)
}
return
}
// get cms from column_income_statis
func (s *Service) getCmsAfterCal(c context.Context, date string, categoryIDs []int64) (archives []*model.ArchiveCharge, err error) {
return s.getCmIncomeStatis(c)
}
// get bgm from backgroud_music
func (s *Service) getBgmBeforeCal(c context.Context, date string, categoryIDs []int64) (archives []*model.ArchiveCharge, err error) {
return s.getBackgroundMusic(c, 2000)
}
func getRatioByTagInfo(archives []*model.ArchiveCharge, tagInfo []*model.TagInfo, tagMID map[int64][]int64) (avRatio map[int64]*model.AvTagRatio) {
avRatio = make(map[int64]*model.AvTagRatio)
midMap := make(map[int64]int64)
categoryAvs := make(map[int64][]*model.ArchiveCharge) // 非活动 category
midCategoryAvs := make(map[string][]*model.ArchiveCharge) // 非活动 category+mid
actCategoryAvs := make(map[string][]*model.ArchiveCharge) // 活动 activity+category
actMidCategoryAvs := make(map[string][]*model.ArchiveCharge) // 活动 activity+category+mid
for _, av := range archives {
// mid和avid对应map
midMap[av.AID] = av.MID
// 活动id需要特殊处理
if av.ActivityID != 0 {
// 活动 不指定mid
key := fmt.Sprintf("act%d+cate%d", av.ActivityID, av.CategoryID)
if _, ok := actCategoryAvs[key]; ok {
actCategoryAvs[key] = append(actCategoryAvs[key], av)
} else {
actCategoryAvs[key] = []*model.ArchiveCharge{av}
}
// 活动 指定mid
mckey := fmt.Sprintf("act%d+cate%d+mid%d", av.ActivityID, av.CategoryID, av.MID)
if _, ok := actMidCategoryAvs[mckey]; ok {
actMidCategoryAvs[mckey] = append(actMidCategoryAvs[mckey], av)
} else {
actMidCategoryAvs[mckey] = []*model.ArchiveCharge{av}
}
} else {
// 非活动 不指定mid
if _, ok := categoryAvs[av.CategoryID]; ok {
categoryAvs[av.CategoryID] = append(categoryAvs[av.CategoryID], av)
} else {
categoryAvs[av.CategoryID] = []*model.ArchiveCharge{av}
}
// 非活动 指定mid
mcKey := fmt.Sprintf("mid%d+cate%d", av.MID, av.CategoryID)
if _, ok := midCategoryAvs[mcKey]; ok {
midCategoryAvs[mcKey] = append(midCategoryAvs[mcKey], av)
} else {
midCategoryAvs[mcKey] = []*model.ArchiveCharge{av}
}
}
}
for _, tag := range tagInfo {
// 非活动 不指定mid
if tag.IsCommon == 1 && tag.ActivityID == 0 {
if av, ok := categoryAvs[tag.CategoryID]; ok {
getAvTagRatio(avRatio, av, tag, midMap)
}
}
// 活动 不指定mid
if tag.IsCommon == 1 && tag.ActivityID != 0 {
key := fmt.Sprintf("act%d+cate%d", tag.ActivityID, tag.CategoryID)
if av, ok := actCategoryAvs[key]; ok {
getAvTagRatio(avRatio, av, tag, midMap)
}
}
// 指定mid
if tag.IsCommon == 0 {
if mids, ok := tagMID[tag.ID]; ok {
for _, mid := range mids {
// 非活动
if tag.ActivityID == 0 {
mcKey := fmt.Sprintf("mid%d+cate%d", mid, tag.CategoryID)
if av, ok := midCategoryAvs[mcKey]; ok {
getAvTagRatio(avRatio, av, tag, midMap)
}
} else { // 活动
mcKey := fmt.Sprintf("act%d+cate%d+mid%d", tag.ActivityID, tag.CategoryID, mid)
if av, ok := actMidCategoryAvs[mcKey]; ok {
getAvTagRatio(avRatio, av, tag, midMap)
}
}
}
}
}
}
return
}
func getAvTagRatio(avRatio map[int64]*model.AvTagRatio, avs []*model.ArchiveCharge, tag *model.TagInfo, midMap map[int64]int64) {
for _, av := range avs {
if av.UploadTime < tag.UploadStartTime || int64(av.UploadTime) >= tag.UploadEndTime.Time().AddDate(0, 0, 1).Unix() {
continue
}
if val, ok := avRatio[av.AID]; !ok {
avRatio[av.AID] = &model.AvTagRatio{
AvID: av.AID,
TagID: tag.ID,
AdjustType: tag.AdjustType,
Ratio: tag.Ratio,
MID: midMap[av.AID],
}
} else if tag.AdjustType == val.AdjustType && tag.Ratio > val.Ratio { // 调节方式相同，取大
avRatio[av.AID].Ratio = tag.Ratio
avRatio[av.AID].TagID = tag.ID
} else if tag.AdjustType != val.AdjustType && tag.AdjustType == 1 { // 调节方式不同，取固定调节
avRatio[av.AID].Ratio = tag.Ratio
avRatio[av.AID].TagID = tag.ID
avRatio[av.AID].AdjustType = tag.AdjustType
}
}
}

50
app/job/main/growup/service/tag/category.go Normal file
View File

@@ -0,0 +1,50 @@
package tag
import (
"context"
)
// 获取所有分区
func (s *Service) getArchiveCategory(c context.Context, ctype int) ([]int64, error) {
switch ctype {
case _video:
return s.getAvCategory(c)
case _column:
return s.getCmCategory(c)
case _bgm:
return []int64{0}, nil
}
return []int64{}, nil
}
func (s *Service) getAvCategory(c context.Context) (ids []int64, err error) {
ids = make([]int64, 0)
allCategory, err := s.dao.GetVideoTypes(c)
if err != nil {
return
}
categoryMap := make(map[int64]struct{})
for _, id := range allCategory {
if _, ok := categoryMap[id]; !ok {
categoryMap[id] = struct{}{}
ids = append(ids, id)
}
}
return
}
func (s *Service) getCmCategory(c context.Context) (ids []int64, err error) {
ids = make([]int64, 0)
allCategory, err := s.dao.GetColumnTypes(c)
if err != nil {
return
}
categoryMap := make(map[int64]struct{})
for _, id := range allCategory {
if _, ok := categoryMap[id]; !ok {
categoryMap[id] = struct{}{}
ids = append(ids, id)
}
}
return
}

114
app/job/main/growup/service/tag/charge_ratio.go Normal file
View File

@@ -0,0 +1,114 @@
package tag
import (
"bytes"
"context"
"strconv"
model "go-common/app/job/main/growup/model/tag"
)
func (s *Service) delArchiveRatio(c context.Context, ctype int) (err error) {
var limit int64 = 2000
for {
var rows int64
rows, err = s.dao.DelArchiveRatio(c, ctype, limit)
if err != nil {
return
}
if rows < limit {
break
}
}
return
}
// delUpChargeRatio del up_charge_ratio
func (s *Service) delUpChargeRatio(c context.Context, ctype int) (err error) {
var limit int64 = 2000
for {
var rows int64
rows, err = s.dao.DelUpRatio(c, ctype, limit)
if err != nil {
return
}
if rows < limit {
break
}
}
return
}
// insertAvRatio insert av_charge_ratio
func (s *Service) insertAvRatio(c context.Context, avs map[int64]*model.AvTagRatio, ctype int) (err error) {
var buf bytes.Buffer
var count int64
for _, av := range avs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(av.TagID, 10))
buf.WriteString(",")
buf.WriteString(strconv.FormatInt(av.AvID, 10))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(av.Ratio))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(av.AdjustType))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(ctype))
buf.WriteString("),")
count++
if count%2000 == 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.InsertAvRatio(c, buf.String())
if err != nil {
return
}
buf.Reset()
}
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.InsertAvRatio(c, buf.String())
if err != nil {
return
}
buf.Reset()
}
return
}
// insertUpRatio insert up_charge_ratio
func (s *Service) insertUpRatio(c context.Context, ups map[int64]*model.AvTagRatio, ctype int) (err error) {
var buf bytes.Buffer
var count int64
for _, up := range ups {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(up.TagID, 10))
buf.WriteString(",")
buf.WriteString(strconv.FormatInt(up.MID, 10))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(up.Ratio))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(up.AdjustType))
buf.WriteString(",")
buf.WriteString(strconv.Itoa(ctype))
buf.WriteString("),")
count++
if count%2000 == 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.InsertUpRatio(c, buf.String())
if err != nil {
return
}
buf.Reset()
}
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
_, err = s.dao.InsertUpRatio(c, buf.String())
if err != nil {
return
}
buf.Reset()
}
return
}

143
app/job/main/growup/service/tag/extra.go Normal file
View File

@@ -0,0 +1,143 @@
package tag
import (
"bytes"
"context"
"fmt"
"strconv"
"time"
model "go-common/app/job/main/growup/model/tag"
"go-common/library/log"
)
// TagExtraIncome get tag extra income
func (s *Service) TagExtraIncome(c context.Context, tagIDs []int64, tagName string, ratio int64, start, end string) (err error) {
startDate, err := time.ParseInLocation("2006-01-02", start, time.Local)
if err != nil {
return
}
endDate, err := time.ParseInLocation("2006-01-02", end, time.Local)
if err != nil {
return
}
var ups map[int64]int64
if ratio == 0 {
ups, err = s.getExtraByUpIncome(c, tagIDs, startDate, endDate)
} else {
ups, err = s.getExtraByTag(c, tagIDs, ratio)
}
if err != nil {
log.Error("getExtra income error(%v)", err)
return
}
err = s.insertUpTagYear(c, ups, tagName)
if err != nil {
log.Error("s.insertUpTagYear error(%v)", err)
}
return
}
func (s *Service) getExtraByUpIncome(c context.Context, tagIDs []int64, startDate, endDate time.Time) (ups map[int64]int64, err error) {
ups = make(map[int64]int64)
upTags, err := s.getExtraByTag(c, tagIDs, 0)
if err != nil {
return
}
upIncomes := make([]*model.UpIncome, 0)
for !startDate.After(endDate) {
query := fmt.Sprintf("date = '%s'", startDate.Format("2006-01-02"))
var upIncome []*model.UpIncome
upIncome, err = s.getUpIncome(c, query)
if err != nil {
log.Error("s.GetUpIncome error(%v)", err)
return
}
upIncomes = append(upIncomes, upIncome...)
startDate = startDate.AddDate(0, 0, 1)
}
for _, up := range upIncomes {
if _, ok := upTags[up.MID]; ok {
ups[up.MID] += up.AvIncome - up.AvBaseIncome
}
}
return
}
func (s *Service) getExtraByTag(c context.Context, tagIDs []int64, ratio int64) (ups map[int64]int64, err error) {
ups = make(map[int64]int64)
if len(tagIDs) == 0 {
return
}
for _, tagID := range tagIDs {
var upTags []*model.UpTagIncome
upTags, err = s.getTagUpById(c, tagID)
if err != nil {
return
}
for _, up := range upTags {
ups[up.MID] = ratio
}
}
return
}
func (s *Service) getTagUpById(c context.Context, tagID int64) (upTags []*model.UpTagIncome, err error) {
upTags = make([]*model.UpTagIncome, 0)
var id int64
limit := 2000
for {
var upTag []*model.UpTagIncome
upTag, err = s.dao.GetTagUpByID(c, tagID, id, limit)
if err != nil {
log.Error("s.dao.GetTagUpByID error(%v)", err)
return
}
upTags = append(upTags, upTag...)
if len(upTag) < limit {
break
}
id = upTag[len(upTag)-1].ID
}
return
}
func (s *Service) insertUpTagYear(c context.Context, ups map[int64]int64, tagName string) (err error) {
up := make(map[int64]int64)
count := 0
for mid, income := range ups {
up[mid] = income
count++
if count >= 2000 {
_, err = s.dao.InsertUpTagYear(c, assembleUpTagYear(up), tagName)
if err != nil {
return
}
up = make(map[int64]int64)
count = 0
}
}
if len(up) > 0 {
_, err = s.dao.InsertUpTagYear(c, assembleUpTagYear(up), tagName)
}
return
}
func assembleUpTagYear(ups map[int64]int64) (vals string) {
var buf bytes.Buffer
for mid, income := range ups {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(mid, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(income, 10))
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
vals = buf.String()
buf.Reset()
return
}

195
app/job/main/growup/service/tag/income.go Normal file
View File

@@ -0,0 +1,195 @@
package tag
import (
"context"
"fmt"
model "go-common/app/job/main/growup/model/tag"
"go-common/library/log"
)
// updateTagIncome update tag_income
func (s *Service) updateTagIncome(c context.Context, avTagRatio, upTagRatio map[int64]*model.AvTagRatio, date string, ctype int) (err error) {
log.Info("GET %d av tag ratio", len(avTagRatio))
log.Info("GET %d up tag ratio", len(upTagRatio))
query := fmt.Sprintf("date = '%s'", date)
archiveIncome, err := s.getArchiveIncome(c, query, ctype)
if err != nil {
log.Error("s.getArchiveIncome error(%v)", err)
return
}
log.Info("GET %d archive_income", len(archiveIncome))
upIncome, err := s.getUpIncome(c, query)
if err != nil {
log.Error("s.GetUpIncome error(%v)", err)
return
}
log.Info("GET %d up_income", len(upIncome))
// 获取之前有标签收入的稿件, 计算稿件活动期间总收入
// preTagIncome, err := s.GetUpTagIncomeMap(c)
// if err != nil {
// log.Error("s.GetUpTagIncome error(%v)", err)
// return
// }
// 获取有标签收入的稿件，每个标签的当日总收入
tagAvs := make([]*model.AvTagRatio, 0)
tagIncome := make(map[int64]int64)
// 稿件维度
for _, av := range archiveIncome {
if val, ok := avTagRatio[av.AID]; ok {
val.Income = av.Income
val.BaseIncome = av.BaseIncome
val.TotalIncome = av.TotalIncome
val.TaxMoney = av.TaxMoney
val.Date = date
tagAvs = append(tagAvs, val)
tagIncome[val.TagID] += av.Income
}
}
// up主维度
for _, up := range upIncome {
if val, ok := upTagRatio[up.MID]; ok {
switch ctype {
case _video:
up.Income, up.BaseIncome, up.TotalIncome, up.TaxMoney = up.AvIncome, up.AvBaseIncome, up.AvTotalIncome, up.AvTax
case _column:
up.Income, up.BaseIncome, up.TotalIncome, up.TaxMoney = up.ColumnIncome, up.ColumnBaseIncome, up.ColumnTotalIncome, up.ColumnTax
case _bgm:
up.Income, up.BaseIncome, up.TotalIncome, up.TaxMoney = up.BgmIncome, up.BgmBaseIncome, up.BgmTotalIncome, up.BgmTax
}
t := &model.AvTagRatio{
Income: up.Income,
BaseIncome: up.BaseIncome,
TotalIncome: up.TotalIncome,
TaxMoney: up.TaxMoney,
Date: date,
AvID: 0,
MID: up.MID,
TagID: val.TagID,
}
tagAvs = append(tagAvs, t)
tagIncome[val.TagID] += up.Income
}
}
tx, err := s.dao.BeginTran(c)
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
log.Info("Insert(update) %d avs to up_tag_income", len(tagAvs))
err = s.TxInsertUpTagIncome(tx, tagAvs)
if err != nil {
log.Error("s.InsertUpTagIncome error(%v)", err)
return
}
err = s.TxUpdateTagInfoIncome(tx, tagIncome)
if err != nil {
log.Error("s.UpdateTagInfo error(%v)", err)
return
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error")
return
}
return
}
func (s *Service) getArchiveIncome(c context.Context, query string, ctype int) (archives []*model.ArchiveIncome, err error) {
var id int64
limit := 2000
for {
var archive []*model.ArchiveIncome
archive, err = s.dao.GetArchiveIncome(c, id, query, limit, ctype)
if err != nil {
return
}
archives = append(archives, archive...)
if len(archive) < limit {
break
}
id = archive[len(archive)-1].ID
}
if ctype == _bgm {
bgms := make(map[int64]*model.ArchiveIncome)
for _, archive := range archives {
if b, ok := bgms[archive.AID]; ok {
b.Income += archive.Income
b.BaseIncome += archive.BaseIncome
b.TaxMoney += archive.TaxMoney
} else {
bgms[archive.AID] = archive
}
}
archives = make([]*model.ArchiveIncome, 0)
for _, b := range bgms {
archives = append(archives, b)
}
}
return
}
// getUpIncome get up_income by query
func (s *Service) getUpIncome(c context.Context, query string) (ups []*model.UpIncome, err error) {
var id int64
limit := 2000
for {
var up []*model.UpIncome
up, err = s.dao.GetUpIncome(c, id, query, limit)
if err != nil {
return
}
ups = append(ups, up...)
if len(up) < limit {
break
}
id = up[len(up)-1].ID
}
return
}
func (s *Service) getAvIncomeStatis(c context.Context) (avs []*model.ArchiveCharge, err error) {
var id int64
limit := 2000
for {
var av []*model.ArchiveCharge
av, err = s.dao.GetAvIncomeStatis(c, id, limit)
if err != nil {
return
}
avs = append(avs, av...)
if len(av) < limit {
break
}
id = av[len(av)-1].ID
}
return
}
func (s *Service) getCmIncomeStatis(c context.Context) (cms []*model.ArchiveCharge, err error) {
var id int64
limit := 2000
for {
var cm []*model.ArchiveCharge
cm, err = s.dao.GetCmIncomeStatis(c, id, limit)
if err != nil {
return
}
cms = append(cms, cm...)
if len(cm) < limit {
break
}
id = cm[len(cm)-1].ID
}
return
}

91
app/job/main/growup/service/tag/service.go Normal file
View File

@@ -0,0 +1,91 @@
package tag
import (
"context"
"fmt"
"time"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/dao/email"
dao "go-common/app/job/main/growup/dao/tag"
"go-common/app/job/main/growup/service"
"go-common/app/job/main/growup/service/ctrl"
"go-common/library/log"
)
// Service struct
type Service struct {
conf *conf.Config
dao *dao.Dao
email *email.Dao
}
// New fn
func New(c *conf.Config, executor ctrl.Executor) (s *Service) {
s = &Service{
conf: c,
dao: dao.New(c),
email: email.New(c),
}
log.Info("tag service start")
executor.Submit(
s.calDailyTagRatio,
s.calDailyTagIncome,
)
return s
}
// Ping check dao health.
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close dao
func (s *Service) Close() {
s.dao.Close()
}
func (s *Service) calDailyTagRatio(ctx context.Context) {
for {
time.Sleep(service.NextDay(14, 0, 0))
msg := ""
log.Info("calDailyTagRatio begin:%v", time.Now().Format("2006-01-02 15:04:05"))
date := time.Now().AddDate(0, 0, -1)
err := s.TagRatioAll(context.TODO(), date.Format("2006-01-02"))
if err != nil {
msg = fmt.Sprintf("calDailyTagRatio error(%v)", err)
} else {
msg = "Success"
}
err = s.email.SendMail(date, msg, "标签每日计算%d年%d月%d日", "shaozhenyu@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("calDailyTagRatio end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}
func (s *Service) calDailyTagIncome(ctx context.Context) {
for {
time.Sleep(service.NextDay(19, 0, 0))
msg := ""
log.Info("calDailyTagIncome begin:%v", time.Now().Format("2006-01-02 15:04:05"))
date := time.Now().AddDate(0, 0, -1)
err := s.TagIncomeAll(context.TODO(), date.Format("2006-01-02"))
if err != nil {
msg = fmt.Sprintf("calDailyTagIncome error(%v)", err)
} else {
msg = "Success"
}
err = s.TagUps(context.TODO(), date)
if err != nil {
msg = fmt.Sprintf("s.TagUps error(%v)", err)
}
err = s.email.SendMail(date, msg, "标签每日收入计算%d年%d月%d日", "shaozhenyu@bilibili.com")
if err != nil {
log.Error("s.email.SendMail error(%v)", err)
}
log.Info("calDailyTagIncome end:%v", time.Now().Format("2006-01-02 15:04:05"))
}
}

29
app/job/main/growup/service/tag/service_test.go Normal file
View File

@@ -0,0 +1,29 @@
package tag
import (
"flag"
"path/filepath"
"time"
"go-common/app/job/main/growup/conf"
"go-common/app/job/main/growup/service/ctrl"
)
var (
s *Service
)
func init() {
dir, _ := filepath.Abs("../../cmd/growup-job.toml")
flag.Set("conf", dir)
conf.Init()
s = New(conf.Conf, ctrl.NewUnboundedExecutor())
time.Sleep(time.Second)
}
func WithService(f func(s *Service)) func() {
return func() {
// Reset(func() { CleanCache() })
f(s)
}
}

142
app/job/main/growup/service/tag/tag.go Normal file
View File

@@ -0,0 +1,142 @@
package tag
import (
"context"
"fmt"
task "go-common/app/job/main/growup/service"
"go-common/library/database/sql"
"go-common/library/log"
)
// TagRatioAll cal tag ratio all
func (s *Service) TagRatioAll(c context.Context, date string) (err error) {
defer func() {
task.GetTaskService().SetTaskStatus(c, task.TaskTagRatio, date, err)
}()
// ready
err = task.GetTaskService().TaskReady(c, date, task.TaskAvCharge, task.TaskCmCharge, task.TaskBgmSync)
if err != nil {
return
}
err = s.TagRatio(c, date, _video)
if err != nil {
log.Error("s.TagRatio video error(%v)", err)
return
}
err = s.TagRatio(c, date, _column)
if err != nil {
log.Error("s.TagRatio column error(%v)", err)
return
}
err = s.TagRatio(c, date, _bgm)
if err != nil {
log.Error("s.TagRatio bgm error(%v)", err)
}
return
}
// TagRatio handle tag ratio
func (s *Service) TagRatio(c context.Context, date string, ctype int) (err error) {
err = s.tagArchiveRatio(c, date, ctype)
if err != nil {
log.Error("s.tagArchiveRatio error(%v)", err)
return
}
err = s.tagUpRatio(c, date, ctype)
if err != nil {
log.Error("s.tagUpRatio error(%v)", err)
}
return
}
// TagIncomeAll tag income all
func (s *Service) TagIncomeAll(c context.Context, date string) (err error) {
defer func() {
task.GetTaskService().SetTaskStatus(c, task.TaskTagIncome, date, err)
}()
// ready
err = task.GetTaskService().TaskReady(c, date, task.TaskCreativeIncome)
if err != nil {
return
}
err = s.TagIncome(c, date, _video)
if err != nil {
log.Error("s.TagIncome video error(%v)", err)
return
}
err = s.TagIncome(c, date, _column)
if err != nil {
log.Error("s.TagIncome column error(%v)", err)
return
}
err = s.TagIncome(c, date, _bgm)
if err != nil {
log.Error("s.TagIncome bgm error(%v)", err)
}
return
}
// TagIncome handle tag
func (s *Service) TagIncome(c context.Context, date string, ctype int) (err error) {
var getArchives IGetArchivesToTag
switch ctype {
case _video:
getArchives = s.getAvsAfterCal
case _column:
getArchives = s.getCmsAfterCal
case _bgm:
getArchives = s.getBgmBeforeCal
}
tagAvs, err := s.getTagArchive(c, date, ctype, getArchives)
if err != nil {
log.Error("s.getTagArchive error(%v)", err)
return
}
tagUps, err := s.getTagUps(c, date, ctype, getArchives)
if err != nil {
log.Error("s.getTagAvs error(%v)", err)
return
}
err = s.updateTagIncome(c, tagAvs, tagUps, date, ctype)
if err != nil {
log.Error("s.getActivityAvs error(%v)", err)
return
}
err = s.addActivityInfo(c, date, tagAvs, ctype)
if err != nil {
log.Error("s.addActivityInfo error(%v)", err)
}
return
}
// TxUpdateTagInfoIncome update tag_info's income
func (s *Service) TxUpdateTagInfoIncome(tx *sql.Tx, tagIncome map[int64]int64) (err error) {
for id, income := range tagIncome {
if income <= 0 {
continue
}
var rows int64
rows, err = s.dao.TxUpdateTagInfoIncome(tx, id, income)
if err != nil {
tx.Rollback()
log.Error("s.dao.UpdateTagInfoIncome error(%v)", err)
return
}
if rows == 0 {
tx.Rollback()
return fmt.Errorf("TxUpdateTagInfoIncome error rows = 0")
}
}
return
}

15
app/job/main/growup/service/tag/tag_test.go Normal file
View File

@@ -0,0 +1,15 @@
package tag
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_TagIncome(t *testing.T) {
Convey("TagIncome", t, WithService(func(s *Service) {
err := s.TagIncome(context.Background(), "2018-01-01", _video)
So(err, ShouldBeNil)
}))
}

23
app/job/main/growup/service/tag/tag_up_info.go Normal file
View File

@@ -0,0 +1,23 @@
package tag
import (
"context"
)
// GetTagUpInfoMID get tag_up_info mid map
func (s *Service) GetTagUpInfoMID(c context.Context, tags []int64) (tagMID map[int64][]int64, err error) {
tagMID = make(map[int64][]int64)
from, limit := 0, 2000
for {
count := 0
count, err = s.dao.GetTagUpInfoByTag(c, tags, from, limit, tagMID)
if err != nil {
return
}
if count < limit {
break
}
from += limit
}
return
}

20
app/job/main/growup/service/tag/tag_up_info_test.go Normal file
View File

@@ -0,0 +1,20 @@
package tag
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_GetTagUpInfoMID(t *testing.T) {
Convey("GetTagUpInfoMID", t, WithService(func(s *Service) {
_, err := s.GetTagUpInfoMID(context.Background(), []int64{})
So(err, ShouldNotBeNil)
}))
Convey("GetTagUpInfoMID", t, WithService(func(s *Service) {
_, err := s.GetTagUpInfoMID(context.Background(), []int64{1})
So(err, ShouldBeNil)
}))
}

127
app/job/main/growup/service/tag/up_tag.go Normal file
View File

@@ -0,0 +1,127 @@
package tag
import (
"context"
model "go-common/app/job/main/growup/model/tag"
"go-common/library/log"
)
// tagUpRatio update tag up_charge_ratio
func (s *Service) tagUpRatio(c context.Context, date string, ctype int) (err error) {
// delete
err = s.delUpChargeRatio(c, ctype)
if err != nil {
log.Error("s.delUpChargeRatio error(%v)", err)
return
}
var getArchives IGetArchivesToTag
switch ctype {
case _video:
getArchives = s.getAvsBeforeCal
case _column:
getArchives = s.getCmsBeforeCal
case _bgm:
getArchives = s.getBgmBeforeCal
}
ups, err := s.getTagUps(c, date, ctype, getArchives)
if err != nil {
log.Error("s.getTagUps error(%v)", err)
return
}
log.Info("s.getTagUps insert ups(%d)", len(ups))
// insert
err = s.insertUpRatio(c, ups, ctype)
if err != nil {
log.Error("s.insertUpRatio error(%v)", err)
}
return
}
func (s *Service) getTagUps(c context.Context, date string, ctype int, getArchives IGetArchivesToTag) (ups map[int64]*model.AvTagRatio, err error) {
ratioForAll := false
tagInfo, err := s.dao.GetTagInfoByDate(c, 1, ctype, date, date)
if err != nil {
log.Error("s.dao.GetTagInfoByDate error(%v)", err)
return
}
if len(tagInfo) == 0 {
return
}
tagIDs := make([]int64, 0)
for _, tag := range tagInfo {
if tag.IsCommon == 1 {
ratioForAll = true
}
tagIDs = append(tagIDs, tag.ID)
}
categoryIDs, err := s.getArchiveCategory(c, ctype)
if err != nil {
log.Error("s.getArchiveCategory error(%v)", err)
return
}
archives := make([]*model.ArchiveCharge, 0)
if ratioForAll {
archives, err = getArchives(c, date, categoryIDs)
if err != nil {
log.Error("s.getAvserror(%v)", err)
return
}
}
// 获取tag指定mid
tagMID, err := s.GetTagUpInfoMID(c, tagIDs)
if err != nil {
log.Error("s.GetTagUpInfoMID error(%v)", err)
return
}
ups = getUpsByTagInfo(archives, tagInfo, tagMID)
return
}
func getUpsByTagInfo(archives []*model.ArchiveCharge, tagInfo []*model.TagInfo, tagMID map[int64][]int64) (upRatio map[int64]*model.AvTagRatio) {
upRatio = make(map[int64]*model.AvTagRatio)
chargeMIDs := make([]int64, 0)
for _, archive := range archives {
chargeMIDs = append(chargeMIDs, archive.MID)
}
for _, tag := range tagInfo {
if tag.IsCommon == 0 {
if mids, ok := tagMID[tag.ID]; ok {
getUpTagRatio(upRatio, mids, tag)
}
} else {
getUpTagRatio(upRatio, chargeMIDs, tag)
}
}
return
}
func getUpTagRatio(upRatio map[int64]*model.AvTagRatio, mids []int64, tag *model.TagInfo) {
for _, mid := range mids {
if val, ok := upRatio[mid]; !ok {
upRatio[mid] = &model.AvTagRatio{
TagID: tag.ID,
AdjustType: tag.AdjustType,
Ratio: tag.Ratio,
MID: mid,
}
} else if tag.AdjustType == val.AdjustType && tag.Ratio > val.Ratio { // 调节方式相同，取大
upRatio[mid].Ratio = tag.Ratio
upRatio[mid].TagID = tag.ID
} else if tag.AdjustType != val.AdjustType && tag.AdjustType == 1 { // 调节方式不同，取固定调节
upRatio[mid].Ratio = tag.Ratio
upRatio[mid].TagID = tag.ID
upRatio[mid].AdjustType = tag.AdjustType
}
}
}

146
app/job/main/growup/service/tag/up_tag_income.go Normal file
View File

@@ -0,0 +1,146 @@
package tag
import (
"bytes"
"context"
"strconv"
"time"
model "go-common/app/job/main/growup/model/tag"
"go-common/library/database/sql"
"go-common/library/log"
xtime "go-common/library/time"
)
// TagUps cal tag effect up count
func (s *Service) TagUps(c context.Context, date time.Time) (err error) {
tags, err := s.dao.AllTagInfo(c)
if err != nil {
log.Error("s.dao.AllTagInfo error(%v)", err)
return
}
if len(tags) == 0 {
return
}
d := xtime.Time(date.Unix())
startTime := d
for _, t := range tags {
if t.EndAt > d && t.StartAt < startTime {
startTime = t.StartAt
}
}
uptags, err := s.upTagIncomeByDate(c, startTime.Time().Format("2006-01-02"))
if err != nil {
log.Error("s.upTagIncomeByDate error(%v)", err)
return
}
tagMap := make(map[int64]map[int64]struct{})
for _, u := range uptags {
if _, ok := tagMap[u.TagID]; !ok {
tagMap[u.TagID] = make(map[int64]struct{})
}
tagMap[u.TagID][u.MID] = struct{}{}
}
// update
for id, tag := range tagMap {
_, err = s.dao.UpdateTagUps(c, id, len(tag))
if err != nil {
log.Error("s.dao.UpdateTagUps error(%v)", err)
return
}
}
return
}
func (s *Service) upTagIncomeByDate(c context.Context, date string) (upTags []*model.UpTagIncome, err error) {
upTags = make([]*model.UpTagIncome, 0)
var id int64
limit := 2000
for {
var upTag []*model.UpTagIncome
upTag, err = s.dao.UpTagIncomeByDate(c, date, id, limit)
if err != nil {
log.Error("s.dao.UpTagIncomeByDate error(%v)", err)
return
}
upTags = append(upTags, upTag...)
if len(upTag) < limit {
break
}
id = upTag[len(upTag)-1].ID
}
return
}
// GetUpTagIncomeMap get up_tag_income map[mid]income
func (s *Service) GetUpTagIncomeMap(c context.Context) (avs map[int64]int64, err error) {
avs = make(map[int64]int64)
var id int64
count, limit := 0, 2000
for {
id, count, err = s.dao.GetUpTagIncomeMap(c, id, limit, avs)
if err != nil {
return
}
if count < limit {
break
}
}
return
}
// TxInsertUpTagIncome insert up_tag_income
func (s *Service) TxInsertUpTagIncome(tx *sql.Tx, avs []*model.AvTagRatio) (err error) {
if len(avs) == 0 {
return
}
start, offset := 0, 2000
if len(avs) < offset {
offset = len(avs)
}
for start+offset <= len(avs) {
_, err = s.InsertUpTagIncomeBatch(tx, avs[start:start+offset])
if err != nil {
tx.Rollback()
return
}
start += offset
if start < len(avs) && start+offset > len(avs) {
offset = len(avs) - start
}
}
return
}
// InsertUpTagIncomeBatch insert up_tag_income batch
func (s *Service) InsertUpTagIncomeBatch(tx *sql.Tx, avs []*model.AvTagRatio) (rows int64, err error) {
var buf bytes.Buffer
for _, a := range avs {
buf.WriteString("(")
buf.WriteString(strconv.FormatInt(a.TagID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.MID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.AvID, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.Income, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.BaseIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TotalIncome, 10))
buf.WriteByte(',')
buf.WriteString(strconv.FormatInt(a.TaxMoney, 10))
buf.WriteByte(',')
buf.WriteString("\"" + a.Date + "\"")
buf.WriteString(")")
buf.WriteByte(',')
}
if buf.Len() > 0 {
buf.Truncate(buf.Len() - 1)
}
values := buf.String()
buf.Reset()
_, err = s.dao.TxInsertUpTagIncome(tx, values)
return
}

15
app/job/main/growup/service/tag/up_tag_income_test.go Normal file
View File

@@ -0,0 +1,15 @@
package tag
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_GetUpTagIncomeMap(t *testing.T) {
Convey("GetUpTagIncomeMap", t, WithService(func(s *Service) {
_, err := s.GetUpTagIncomeMap(context.Background())
So(err, ShouldBeNil)
}))
}

Some files were not shown because too many files have changed in this diff Show More