Create & Init Project...

app/job/main/growup/service/income/BUILD

@@ -0,0 +1,96 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "av_charge_statis_test.go",
+        "av_charge_test.go",
+        "blacklist_test.go",
+        "charge_ratio_test.go",
+        "run_test.go",
+        "service_test.go",
+        "up_account_test.go",
+        "up_income_date_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/growup/conf:go_default_library",
+        "//app/job/main/growup/model/income:go_default_library",
+        "//app/job/main/growup/service/ctrl:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "archive.go",
+        "av_charge.go",
+        "av_charge_statis.go",
+        "av_income.go",
+        "av_income_stat.go",
+        "bgm.go",
+        "bgm_income.go",
+        "bgm_income_stat.go",
+        "blacklist.go",
+        "business_order.go",
+        "charge.go",
+        "charge_ratio.go",
+        "column_charge.go",
+        "column_income.go",
+        "column_income_stat.go",
+        "common.go",
+        "income.go",
+        "income_date_statis.go",
+        "lottery.go",
+        "run.go",
+        "run_statis.go",
+        "service.go",
+        "tax.go",
+        "tools.go",
+        "up_account.go",
+        "up_archive_statis.go",
+        "up_income.go",
+        "up_income_date.go",
+        "up_income_stat.go",
+        "up_info_video.go",
+    ],
+    importpath = "go-common/app/job/main/growup/service/income",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/growup/conf:go_default_library",
+        "//app/job/main/growup/dao/dataplatform:go_default_library",
+        "//app/job/main/growup/dao/email:go_default_library",
+        "//app/job/main/growup/dao/income:go_default_library",
+        "//app/job/main/growup/model/income:go_default_library",
+        "//app/job/main/growup/service:go_default_library",
+        "//app/job/main/growup/service/ctrl:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/golang.org/x/sync/errgroup:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/growup/service/income/archive.go

@@ -0,0 +1,53 @@
+package income
+
+import (
+	"context"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+)
+
+func (s *DateStatis) getArchiveByDate(c context.Context, archiveCh chan []*model.ArchiveIncome, startDate, endDate time.Time, typ, limit int) (err error) {
+	defer close(archiveCh)
+	var aid, table string
+	switch typ {
+	case _video:
+		aid, table = "av_id", "av_income"
+	case _column:
+		aid, table = "aid", "column_income"
+	case _bgm:
+		aid, table = "sid", "bgm_income"
+	default:
+		return
+	}
+	endDate = endDate.AddDate(0, 0, 1)
+	for startDate.Before(endDate) {
+		err = s.getArchiveIncome(c, archiveCh, aid, table, startDate.Format(_layout), limit)
+		if err != nil {
+			return
+		}
+		startDate = startDate.AddDate(0, 0, 1)
+	}
+	return
+}
+
+func (s *DateStatis) getArchiveIncome(c context.Context, archiveCh chan []*model.ArchiveIncome, aid, table, date string, limit int) (err error) {
+	var id int64
+	for {
+		var archive []*model.ArchiveIncome
+		if aid == "sid" {
+			archive, err = s.dao.GetBgmIncomeByDate(c, date, id, limit)
+		} else {
+			archive, err = s.dao.GetArchiveByDate(c, aid, table, date, id, limit)
+		}
+		if err != nil {
+			return
+		}
+		archiveCh <- archive
+		if len(archive) < limit {
+			break
+		}
+		id = archive[len(archive)-1].ID
+	}
+	return
+}

app/job/main/growup/service/income/av_charge.go

@@ -0,0 +1,375 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+	"strings"
+	"time"
+
+	dao "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	_avWeeklyCharge  = "av_weekly_charge"
+	_avMonthlyCharge = "av_monthly_charge"
+)
+
+// HandleAvCharge handle av daily charge
+func (s *AvChargeSvr) HandleAvCharge(c context.Context, date time.Time,
+	dailyChannel chan []*model.AvCharge) (weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
+	var eg errgroup.Group
+	weeklyChargeMap = make(map[int64]*model.AvCharge)
+	monthlyChargeMap = make(map[int64]*model.AvCharge)
+
+	eg.Go(func() (err error) {
+		avWeeklyCharge, err := s.GetAvCharge(c, getStartWeeklyDate(date), s.dao.AvWeeklyCharge)
+		if err != nil {
+			log.Error("s.GetAvCharge(av_weekly_charge) error(%v)", err)
+			return
+		}
+		for _, weeklyCharge := range avWeeklyCharge {
+			weeklyChargeMap[weeklyCharge.AvID] = weeklyCharge
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		avMonthlyCharge, err := s.GetAvCharge(c, getStartMonthlyDate(date), s.dao.AvMonthlyCharge)
+		if err != nil {
+			log.Error("s.GetAvCharge(av_monthly_charge) error(%v)", err)
+			return
+		}
+		for _, monthlyCharge := range avMonthlyCharge {
+			monthlyChargeMap[monthlyCharge.AvID] = monthlyCharge
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		chargeStatisMap, err = s.GetAvChargeStatisMap(c)
+		if err != nil {
+			log.Error("s.GetAvChargeStatisMap error(%v)", err)
+		}
+		return
+	})
+
+	if err = eg.Wait(); err != nil {
+		log.Error("HandleAvCharge eg.Wait error(%v)", err)
+		return
+	}
+
+	s.CalAvCharge(date, weeklyChargeMap, monthlyChargeMap, chargeStatisMap, dailyChannel)
+	return
+}
+
+// AvCharges get av charges
+func (s *AvChargeSvr) AvCharges(c context.Context, date time.Time, ch chan []*model.AvCharge, bubbleRatio map[int64]float64) (err error) {
+	defer func() {
+		close(ch)
+	}()
+	var id int64
+	for {
+		var charges []*model.AvCharge
+		charges, err = s.dao.AvDailyCharge(c, date, id, _limitSize)
+		if err != nil {
+			return
+		}
+		for _, charge := range charges {
+			if ratio, ok := bubbleRatio[charge.AvID]; ok {
+				charge.IncCharge = int64(Round(Mul(float64(charge.IncCharge), ratio), 0))
+			}
+		}
+		ch <- charges
+		if len(charges) < _limitSize {
+			break
+		}
+		id = charges[len(charges)-1].ID
+	}
+	return
+}
+
+// GetAvCharge get av charge
+func (s *AvChargeSvr) GetAvCharge(c context.Context, date time.Time, fn dao.IAvCharge) (avCharges []*model.AvCharge, err error) {
+	var id int64
+	for {
+		avCharge, err1 := fn(c, date, id, _limitSize)
+		if err1 != nil {
+			err = err1
+			return
+		}
+		avCharges = append(avCharges, avCharge...)
+		if len(avCharge) < _limitSize {
+			break
+		}
+		id = avCharge[len(avCharge)-1].ID
+	}
+	return
+}
+
+// CalAvCharge cal av charge
+func (s *AvChargeSvr) CalAvCharge(date time.Time, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, dailyChannel chan []*model.AvCharge) {
+	for avDailyCharge := range dailyChannel {
+		s.calAvCharge(date, avDailyCharge, weeklyChargeMap, monthlyChargeMap, chargeStatisMap)
+	}
+}
+
+func (s *AvChargeSvr) calAvCharge(date time.Time, avDailyCharge []*model.AvCharge, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
+	startWeeklyDate = getStartWeeklyDate(date)
+	startMonthlyDate = getStartMonthlyDate(date)
+
+	for _, dailyCharge := range avDailyCharge {
+		if weeklyCharge, ok := weeklyChargeMap[dailyCharge.AvID]; ok {
+			updateAvCharge(weeklyCharge, dailyCharge)
+		} else {
+			weeklyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startWeeklyDate)
+		}
+
+		if monthlyCharge, ok := monthlyChargeMap[dailyCharge.AvID]; ok {
+			updateAvCharge(monthlyCharge, dailyCharge)
+		} else {
+			monthlyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startMonthlyDate)
+		}
+		s.CalAvChargeStatis(dailyCharge, chargeStatisMap)
+	}
+}
+
+func addAvCharge(daily *model.AvCharge, fixDate time.Time) *model.AvCharge {
+	return &model.AvCharge{
+		AvID:           daily.AvID,
+		MID:            daily.MID,
+		TagID:          daily.TagID,
+		IsOriginal:     daily.IsOriginal,
+		DanmakuCount:   daily.DanmakuCount,
+		CommentCount:   daily.CommentCount,
+		CollectCount:   daily.CollectCount,
+		CoinCount:      daily.CoinCount,
+		ShareCount:     daily.ShareCount,
+		ElecPayCount:   daily.ElecPayCount,
+		TotalPlayCount: daily.TotalPlayCount,
+		WebPlayCount:   daily.WebPlayCount,
+		AppPlayCount:   daily.AppPlayCount,
+		H5PlayCount:    daily.H5PlayCount,
+		LvUnknown:      daily.LvUnknown,
+		Lv0:            daily.Lv0,
+		Lv1:            daily.Lv1,
+		Lv2:            daily.Lv2,
+		Lv3:            daily.Lv3,
+		Lv4:            daily.Lv4,
+		Lv5:            daily.Lv5,
+		Lv6:            daily.Lv6,
+		VScore:         daily.VScore,
+		IncCharge:      daily.IncCharge,
+		TotalCharge:    daily.IncCharge,
+		Date:           xtime.Time(fixDate.Unix()),
+		UploadTime:     daily.UploadTime,
+		DBState:        _dbInsert,
+	}
+}
+
+func updateAvCharge(origin, daily *model.AvCharge) {
+	origin.DanmakuCount += daily.DanmakuCount
+	origin.CommentCount += daily.CommentCount
+	origin.CollectCount += daily.CollectCount
+	origin.CoinCount += daily.CoinCount
+	origin.ShareCount += daily.ShareCount
+	origin.ElecPayCount += daily.ElecPayCount
+	origin.TotalPlayCount += daily.TotalPlayCount
+	origin.WebPlayCount += daily.WebPlayCount
+	origin.AppPlayCount += daily.AppPlayCount
+	origin.H5PlayCount += daily.H5PlayCount
+	origin.LvUnknown += daily.LvUnknown
+	origin.Lv0 += daily.Lv0
+	origin.Lv1 += daily.Lv1
+	origin.Lv2 += daily.Lv2
+	origin.Lv3 += daily.Lv3
+	origin.Lv4 += daily.Lv4
+	origin.Lv5 += daily.Lv5
+	origin.Lv6 += daily.Lv6
+	origin.VScore += daily.VScore
+	origin.IncCharge += daily.IncCharge
+	origin.TotalCharge += daily.IncCharge
+	origin.DBState = _dbUpdate
+}
+
+// AvChargeDBStore av charge db store
+func (s *AvChargeSvr) AvChargeDBStore(c context.Context, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge) (err error) {
+	err = s.avChargeDBStoreBatch(c, _avWeeklyCharge, weeklyChargeMap)
+	if err != nil {
+		log.Error("s.avChargeDBStoreBatch av_weekly_charge error(%v)", err)
+		return
+	}
+
+	err = s.avChargeDBStoreBatch(c, _avMonthlyCharge, monthlyChargeMap)
+	if err != nil {
+		log.Error("s.avChargeDBStoreBatch av_monthly_charge error(%v)", err)
+		return
+	}
+	return
+}
+
+func (s *AvChargeSvr) avChargeDBStoreBatch(c context.Context, table string, avChargeMap map[int64]*model.AvCharge) error {
+	insert, update := make([]*model.AvCharge, batchSize), make([]*model.AvCharge, batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range avChargeMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+
+		if insertIndex >= batchSize {
+			_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
+			if err != nil {
+				log.Error("s.avChargeBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= batchSize {
+			_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
+			if err != nil {
+				log.Error("s.avChargeBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
+		if err != nil {
+			log.Error("s.avChargeBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
+		if err != nil {
+			log.Error("s.avChargeBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func assembleAvCharge(avCharge []*model.AvCharge) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range avCharge {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.AvID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(row.IsOriginal))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.DanmakuCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CommentCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CollectCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CoinCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.ShareCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.ElecPayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalPlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.WebPlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.AppPlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.H5PlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.LvUnknown, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv0, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv1, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv2, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv3, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv4, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv5, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv6, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.VScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *AvChargeSvr) avChargeBatchInsert(c context.Context, avCharge []*model.AvCharge, table string) (rows int64, err error) {
+	vals := assembleAvCharge(avCharge)
+	rows, err = s.dao.InsertAvChargeTable(c, vals, table)
+	return
+}
+
+// for test
+func assembleColumnCharge(cols []*model.ColumnCharge) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range cols {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.ArticleID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("\"" + strings.Replace(row.Title, "\"", "\\\"", -1) + "\"")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.UploadTime.Time().Unix(), 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *AvChargeSvr) columnChargeBatchInsert(c context.Context, colCharge []*model.ColumnCharge, table string) (rows int64, err error) {
+	vals := assembleColumnCharge(colCharge)
+	rows, err = s.dao.InsertColumnChargeTable(c, vals, table)
+	return
+}

app/job/main/growup/service/income/av_charge_statis.go

@@ -0,0 +1,155 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	model "go-common/app/job/main/growup/model/income"
+
+	"go-common/library/log"
+)
+
+// GetAvChargeStatisMap get av charge statis map
+func (s *AvChargeSvr) GetAvChargeStatisMap(c context.Context) (chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
+	avChargeStatis, err := s.GetAvChargeStatis(c)
+	if err != nil {
+		log.Error("s.GetAvChargeStatis error(%v)", err)
+		return
+	}
+
+	chargeStatisMap = make(map[int64]*model.AvChargeStatis)
+	for _, chargeStatis := range avChargeStatis {
+		chargeStatisMap[chargeStatis.AvID] = chargeStatis
+	}
+	return
+}
+
+// GetAvChargeStatis get av charge statis
+func (s *AvChargeSvr) GetAvChargeStatis(c context.Context) (avChargeStatis []*model.AvChargeStatis, err error) {
+	var id int64
+	for {
+		statis, err1 := s.dao.AvChargeStatis(c, id, _limitSize)
+		if err1 != nil {
+			err = err1
+			return
+		}
+		avChargeStatis = append(avChargeStatis, statis...)
+		if len(statis) < _limitSize {
+			break
+		}
+		id = statis[len(statis)-1].ID
+	}
+	return
+}
+
+// CalAvChargeStatis cal av charge statis
+func (s *AvChargeSvr) CalAvChargeStatis(dailyCharge *model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
+	if statisCharge, ok := chargeStatisMap[dailyCharge.AvID]; ok {
+		updateAvChargeStatis(statisCharge, dailyCharge)
+	} else {
+		chargeStatisMap[dailyCharge.AvID] = addAvChargeStatis(dailyCharge)
+	}
+}
+
+func addAvChargeStatis(daily *model.AvCharge) *model.AvChargeStatis {
+	return &model.AvChargeStatis{
+		AvID:        daily.AvID,
+		MID:         daily.MID,
+		TagID:       daily.TagID,
+		IsOriginal:  daily.IsOriginal,
+		UploadTime:  daily.UploadTime,
+		TotalCharge: daily.IncCharge,
+		DBState:     _dbInsert,
+	}
+}
+
+func updateAvChargeStatis(avChargeStatis *model.AvChargeStatis, daily *model.AvCharge) {
+	avChargeStatis.TotalCharge += daily.IncCharge
+	avChargeStatis.DBState = _dbUpdate
+}
+
+// AvChargeStatisDBStore store charge statis
+func (s *AvChargeSvr) AvChargeStatisDBStore(c context.Context, chargeStatisMap map[int64]*model.AvChargeStatis) error {
+	insert, update := make([]*model.AvChargeStatis, batchSize), make([]*model.AvChargeStatis, batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range chargeStatisMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+
+		if insertIndex >= batchSize {
+			_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
+			if err != nil {
+				log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= batchSize {
+			_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
+			if err != nil {
+				log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
+		if err != nil {
+			log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
+		if err != nil {
+			log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func assembleAvChargeStatis(avChargeStatis []*model.AvChargeStatis) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range avChargeStatis {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.AvID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(row.IsOriginal))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *AvChargeSvr) avChargeStatisBatchInsert(c context.Context, avChargeStatis []*model.AvChargeStatis) (rows int64, err error) {
+	vals := assembleAvChargeStatis(avChargeStatis)
+	rows, err = s.dao.InsertAvChargeStatisBatch(c, vals)
+	return
+}

app/job/main/growup/service/income/av_charge_statis_test.go

@@ -0,0 +1,75 @@
+package income
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+	xtime "go-common/library/time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_GetAvChargeStatisMap(t *testing.T) {
+	Convey("GetAvChargeStatisMap", t, func() {
+		_, err := charge.GetAvChargeStatisMap(context.Background())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_GetAvChargeStatis(t *testing.T) {
+	Convey("GetAvChargeStatis", t, func() {
+		_, err := charge.GetAvChargeStatis(context.Background())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_AvChargeStatisDBStore(t *testing.T) {
+	Convey("AvChargeStatisDBStore", t, func() {
+		chargeStatisMap := make(map[int64]*model.AvChargeStatis)
+		value := &model.AvChargeStatis{
+			AvID:    11,
+			MID:     11,
+			TagID:   11,
+			DBState: 1,
+		}
+		chargeStatisMap[11] = value
+		err := charge.AvChargeStatisDBStore(context.Background(), chargeStatisMap)
+		So(err, ShouldBeNil)
+	})
+}
+
+func benchmarkAvChargeStatisDBStore(bsize int, size int64, b *testing.B) {
+	batchSize = bsize
+	var i int64
+	chargeStatisMap := make(map[int64]*model.AvChargeStatis)
+	for i = 0; i < size; i++ {
+		chargeStatisMap[i] = &model.AvChargeStatis{
+			AvID:        i,
+			MID:         i,
+			TagID:       i,
+			IsOriginal:  int(i),
+			UploadTime:  xtime.Time(time.Now().Unix()),
+			TotalCharge: i,
+			DBState:     int(i % 2),
+		}
+	}
+
+	for n := 0; n < b.N; n++ {
+		charge.AvChargeStatisDBStore(context.Background(), chargeStatisMap)
+	}
+}
+
+func BenchmarkAvChargeStatisDBStore100(b *testing.B) {
+	benchmarkAvChargeStatisDBStore(100, 100000, b)
+}
+func BenchmarkAvChargeStatisDBStore1000(b *testing.B) {
+	benchmarkAvChargeStatisDBStore(1000, 100000, b)
+}
+func BenchmarkAvChargeStatisDBStore2000(b *testing.B) {
+	benchmarkAvChargeStatisDBStore(2000, 100000, b)
+}
+func BenchmarkAvChargeStatisDBStore10000(b *testing.B) {
+	benchmarkAvChargeStatisDBStore(10000, 100000, b)
+}

app/job/main/growup/service/income/av_charge_test.go

@@ -0,0 +1,93 @@
+package income
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+	xtime "go-common/library/time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_GetAvCharge(t *testing.T) {
+	Convey("GetAvCharge", t, func() {
+		_, err := charge.GetAvCharge(context.Background(), time.Now(), charge.dao.AvMonthlyCharge)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_AvChargeDBStore(t *testing.T) {
+	Convey("AvChargeDBStore", t, func() {
+		monthlyChargeMap := make(map[int64]*model.AvCharge)
+		value := &model.AvCharge{
+			AvID:    11,
+			MID:     11,
+			TagID:   11,
+			DBState: 1,
+		}
+		monthlyChargeMap[11] = value
+		err := charge.AvChargeDBStore(context.Background(), monthlyChargeMap, monthlyChargeMap)
+		So(err, ShouldBeNil)
+	})
+}
+
+func benchmarkAvChargeDBStore(bsize int, size int64, b *testing.B) {
+	batchSize = bsize
+	for n := 0; n < b.N; n++ {
+		var i int64
+		weeklyChargeMap := make(map[int64]*model.AvCharge, size)
+		for i = 0; i < size; i++ {
+			v := int64(n+1) * i
+			weeklyChargeMap[v] = &model.AvCharge{
+				AvID:           v,
+				MID:            v,
+				TagID:          v,
+				IsOriginal:     int(v),
+				DanmakuCount:   v,
+				CommentCount:   v,
+				CollectCount:   v,
+				CoinCount:      v,
+				ShareCount:     v,
+				ElecPayCount:   v,
+				TotalPlayCount: v,
+				WebPlayCount:   v,
+				AppPlayCount:   v,
+				H5PlayCount:    v,
+				LvUnknown:      v,
+				Lv0:            v,
+				Lv1:            v,
+				Lv2:            v,
+				Lv3:            v,
+				Lv4:            v,
+				Lv5:            v,
+				Lv6:            v,
+				VScore:         v,
+				IncCharge:      v,
+				TotalCharge:    v,
+				UploadTime:     xtime.Time(time.Now().Unix()),
+				Date:           xtime.Time(time.Now().Unix()),
+				DBState:        1,
+			}
+		}
+
+		charge.AvChargeDBStore(context.Background(), weeklyChargeMap, weeklyChargeMap)
+	}
+}
+
+func BenchmarkAvChargeDBStore100(b *testing.B) {
+	benchmarkAvChargeDBStore(100, 100000, b)
+}
+
+func BenchmarkAvChargeDBStore1000(b *testing.B) {
+	benchmarkAvChargeDBStore(1000, 100000, b)
+}
+
+func BenchmarkAvChargeDBStore2000(b *testing.B) {
+	benchmarkAvChargeDBStore(2000, 100000, b)
+}
+
+func BenchmarkAvChargeDBStore10000(b *testing.B) {
+	benchmarkAvChargeDBStore(10000, 100000, b)
+}

app/job/main/growup/service/income/av_income.go

@@ -0,0 +1,90 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// AvIncomeSvr Av income service
+type AvIncomeSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewAvIncomeSvr new av income service
+func NewAvIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *AvIncomeSvr) {
+	return &AvIncomeSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// BatchInsertAvIncome batch insert av income
+func (p *AvIncomeSvr) BatchInsertAvIncome(c context.Context, am map[int64][]*model.AvIncome) (err error) {
+	var (
+		buff    = make([]*model.AvIncome, p.batchSize)
+		buffEnd = 0
+	)
+
+	for _, as := range am {
+		for _, a := range as {
+			buff[buffEnd] = a
+			buffEnd++
+
+			if buffEnd >= p.batchSize {
+				values := avIncomeValues(buff[:buffEnd])
+				buffEnd = 0
+				_, err = p.dao.InsertAvIncome(c, values)
+				if err != nil {
+					return
+				}
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := avIncomeValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertAvIncome(c, values)
+	}
+	return
+}
+
+func avIncomeValues(as []*model.AvIncome) (values string) {
+	var buf bytes.Buffer
+	for _, a := range as {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(a.AvID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(a.IsOriginal))
+		buf.WriteByte(',')
+		buf.WriteString("'" + a.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.PlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.TotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.Income, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.TaxMoney, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + a.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.BaseIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/av_income_stat.go

@@ -0,0 +1,100 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// AvIncomeStatSvr av income stat service
+type AvIncomeStatSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewAvIncomeStatSvr new av income service
+func NewAvIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *AvIncomeStatSvr) {
+	return &AvIncomeStatSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// AvIncomeStat get av income stat
+func (p *AvIncomeStatSvr) AvIncomeStat(c context.Context, limit int64) (m map[int64]*model.AvIncomeStat, err error) {
+	m = make(map[int64]*model.AvIncomeStat)
+	var id int64
+	for {
+		var am map[int64]*model.AvIncomeStat
+		am, id, err = p.dao.AvIncomeStat(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(am) == 0 {
+			break
+		}
+		for avID, stat := range am {
+			m[avID] = stat
+		}
+	}
+	return
+}
+
+// BatchInsertAvIncomeStat batch insert av income statistics
+func (p *AvIncomeStatSvr) BatchInsertAvIncomeStat(c context.Context, as map[int64]*model.AvIncomeStat) (err error) {
+	var (
+		buff    = make([]*model.AvIncomeStat, p.batchSize)
+		buffEnd = 0
+	)
+	for _, a := range as {
+		if a.DataState == 0 {
+			continue
+		}
+		buff[buffEnd] = a
+		buffEnd++
+
+		if buffEnd >= p.batchSize {
+			values := avIncomeStatValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = p.dao.InsertAvIncomeStat(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := avIncomeStatValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertAvIncomeStat(c, values)
+	}
+	return
+}
+
+func avIncomeStatValues(as []*model.AvIncomeStat) (values string) {
+	var buf bytes.Buffer
+	for _, a := range as {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(a.AvID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(a.IsOriginal))
+		buf.WriteByte(',')
+		buf.WriteString("'" + a.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(a.TotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/bgm.go

@@ -0,0 +1,143 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+	task "go-common/app/job/main/growup/service"
+
+	"go-common/library/log"
+)
+
+// BGMs map[avid][]*model.BGM,
+func (s *Service) BGMs(c context.Context, limit int64) (bm map[int64][]*model.BGM, err error) {
+	var id int64
+	bm = make(map[int64][]*model.BGM)
+	for {
+		var bs []*model.BGM
+		bs, id, err = s.dao.GetBGM(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(bs) == 0 {
+			break
+		}
+		for _, b := range bs {
+			if _, ok := bm[b.AID]; ok {
+				bm[b.AID] = append(bm[b.AID], b)
+			} else {
+				bm[b.AID] = []*model.BGM{b}
+			}
+		}
+	}
+	return
+}
+
+func (s *Service) batchInsertBGMs(c context.Context, bs []*model.BGM) (err error) {
+	var (
+		buff    = make([]*model.BGM, 2000)
+		buffEnd = 0
+	)
+	for _, b := range bs {
+		buff[buffEnd] = b
+		buffEnd++
+		if buffEnd >= 2000 {
+			values := bgmValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = s.dao.InsertBGM(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := bgmValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = s.dao.InsertBGM(c, values)
+	}
+	return
+}
+
+func bgmValues(bs []*model.BGM) (values string) {
+	var buf bytes.Buffer
+	for _, b := range bs {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(b.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.SID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.AID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.CID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + b.JoinAt + "'")
+		buf.WriteByte(',')
+		buf.WriteString("\"" + strings.Replace(b.Title, "\"", "\\\"", -1) + "\"")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}
+
+const query = "{\"select\": [],\"page\": {\"skip\": %d,\"limit\": %d}}"
+
+func (s *Service) getBGM(c context.Context) (bgms []*model.BGM, err error) {
+	from, limit := 0, 1000
+	var info []*model.BGM
+	for {
+		info, err = s.dp.SendBGMRequest(c, fmt.Sprintf(query, from, limit))
+		if err != nil {
+			log.Error("s.getBGM error(%v)", err)
+			return
+		}
+		if len(info) == 0 {
+			break
+		}
+		bgms = append(bgms, info...)
+		from += len(info)
+		time.Sleep(15 * time.Second) // 数据平台限流，api 10s内只能请求一次
+	}
+	log.Info("get bgm data total (%d) rows", from)
+	return
+}
+
+// SyncBgmInfo sync bgm info from data platform
+func (s *Service) SyncBgmInfo(c context.Context) (err error) {
+	defer func() {
+		task.GetTaskService().SetTaskStatus(c, task.TaskBgmSync, time.Now().AddDate(0, 0, -1).Format(_layout), err)
+	}()
+	err = s.delBGM(c)
+	if err != nil {
+		return
+	}
+	bgms, err := s.getBGM(c)
+	if err != nil {
+		return
+	}
+	return s.batchInsertBGMs(c, bgms)
+}
+
+func (s *Service) delBGM(c context.Context) (err error) {
+	var limit int64 = 2000
+	for {
+		var rows int64
+		rows, err = s.dao.DelBGM(c, limit)
+		if err != nil {
+			return
+		}
+		if rows < limit {
+			break
+		}
+	}
+	return
+}

app/job/main/growup/service/income/bgm_income.go

@@ -0,0 +1,90 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// BgmIncomeSvr bgm income service
+type BgmIncomeSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewBgmIncomeSvr new bgm income service
+func NewBgmIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *BgmIncomeSvr) {
+	return &BgmIncomeSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// BatchInsertBgmIncome batch insert bgm income
+func (p *BgmIncomeSvr) BatchInsertBgmIncome(ctx context.Context, bm map[int64]map[int64]map[int64]*model.BgmIncome) (err error) {
+	var (
+		buff    = make([]*model.BgmIncome, p.batchSize)
+		buffEnd = 0
+	)
+
+	for _, sm := range bm {
+		for _, bs := range sm {
+			for _, b := range bs {
+				buff[buffEnd] = b
+				buffEnd++
+
+				if buffEnd >= p.batchSize {
+					values := bgmIncomeValues(buff[:buffEnd])
+					buffEnd = 0
+					_, err = p.dao.InsertBgmIncome(ctx, values)
+					if err != nil {
+						return
+					}
+				}
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := bgmIncomeValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertBgmIncome(ctx, values)
+	}
+	return
+}
+
+func bgmIncomeValues(bs []*model.BgmIncome) (values string) {
+	var buf bytes.Buffer
+	for _, b := range bs {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(b.AID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.SID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.CID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.Income, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.TotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.TaxMoney, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + b.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.BaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.DailyTotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/bgm_income_stat.go

@@ -0,0 +1,92 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// BgmIncomeStatSvr bgm income stat service
+type BgmIncomeStatSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewBgmIncomeStatSvr new bgm income stat svr
+func NewBgmIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *BgmIncomeStatSvr) {
+	return &BgmIncomeStatSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// BgmIncomeStat get bgm income stat
+func (p *BgmIncomeStatSvr) BgmIncomeStat(c context.Context, limit int64) (m map[int64]*model.BgmIncomeStat, err error) {
+	m = make(map[int64]*model.BgmIncomeStat)
+	var id int64
+	for {
+		var bm map[int64]*model.BgmIncomeStat
+		bm, id, err = p.dao.BgmIncomeStat(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(bm) == 0 {
+			break
+		}
+		for sid, stat := range bm {
+			m[sid] = stat
+		}
+	}
+	return
+}
+
+// BatchInsertBgmIncomeStat batch insert bgm income stat
+func (p *BgmIncomeStatSvr) BatchInsertBgmIncomeStat(c context.Context, bs map[int64]*model.BgmIncomeStat) (err error) {
+	var (
+		buff    = make([]*model.BgmIncomeStat, p.batchSize)
+		buffEnd = 0
+	)
+	for _, b := range bs {
+		if b.DataState == 0 {
+			continue
+		}
+		buff[buffEnd] = b
+		buffEnd++
+
+		if buffEnd >= p.batchSize {
+			values := bgmIncomeStatValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = p.dao.InsertBgmIncomeStat(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := bgmIncomeStatValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertBgmIncomeStat(c, values)
+	}
+	return
+}
+
+func bgmIncomeStatValues(bs []*model.BgmIncomeStat) (values string) {
+	var buf bytes.Buffer
+	for _, b := range bs {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(b.SID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(b.TotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/blacklist.go

@@ -0,0 +1,33 @@
+package income
+
+import (
+	"context"
+)
+
+// Blacklist map[ctype]map[int64]bool
+func (s *Service) Blacklist(c context.Context, limit int64) (m map[int]map[int64]bool, err error) {
+	var id int64
+	m = make(map[int]map[int64]bool)
+	for {
+		var ab map[int][]int64
+		ab, id, err = s.dao.Blacklist(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(ab) == 0 {
+			break
+		}
+		for ctype, avIDs := range ab {
+			for _, avID := range avIDs {
+				if _, ok := m[ctype]; ok {
+					m[ctype][avID] = true
+				} else {
+					m[ctype] = map[int64]bool{
+						avID: true,
+					}
+				}
+			}
+		}
+	}
+	return
+}

app/job/main/growup/service/income/blacklist_test.go

@@ -0,0 +1,15 @@
+package income
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Blacklist(t *testing.T) {
+	Convey("Blacklist", t, func() {
+		_, err := s.Blacklist(context.Background(), 10)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/growup/service/income/business_order.go

@@ -0,0 +1,23 @@
+package income
+
+import "context"
+
+// GetBusinessOrders get business orders
+func (s *Service) GetBusinessOrders(c context.Context, limit int64) (result map[int64]bool, err error) {
+	var id int64
+	result = make(map[int64]bool)
+	for {
+		var m map[int64]bool
+		id, m, err = s.dao.BusinessOrders(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(m) == 0 {
+			break
+		}
+		for k, v := range m {
+			result[k] = v
+		}
+	}
+	return
+}

app/job/main/growup/service/income/charge.go

@@ -0,0 +1,17 @@
+package income
+
+import (
+	incomeD "go-common/app/job/main/growup/dao/income"
+)
+
+// AvChargeSvr av charge service
+type AvChargeSvr struct {
+	dao *incomeD.Dao
+}
+
+// NewAvChargeSvr new av charge service
+func NewAvChargeSvr(dao *incomeD.Dao) (svr *AvChargeSvr) {
+	return &AvChargeSvr{
+		dao: dao,
+	}
+}

app/job/main/growup/service/income/charge_ratio.go

@@ -0,0 +1,70 @@
+package income
+
+import (
+	"context"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// ChargeRatioSvr charge ratio service
+type ChargeRatioSvr struct {
+	dao *incomeD.Dao
+}
+
+// NewChargeRatioSvr new charge ratio service
+func NewChargeRatioSvr(dao *incomeD.Dao) *ChargeRatioSvr {
+	return &ChargeRatioSvr{dao: dao}
+}
+
+// ArchiveChargeRatio get av charge ratio
+func (p *ChargeRatioSvr) ArchiveChargeRatio(c context.Context, limit int64) (rs map[int]map[int64]*model.ArchiveChargeRatio, err error) {
+	rs = make(map[int]map[int64]*model.ArchiveChargeRatio)
+	var id int64
+	for {
+		var ros map[int]map[int64]*model.ArchiveChargeRatio
+		ros, id, err = p.dao.ArchiveChargeRatio(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(ros) == 0 {
+			break
+		}
+		for ctype, m := range ros {
+			if _, ok := rs[ctype]; ok {
+				for aid, ratio := range m {
+					rs[ctype][aid] = ratio
+				}
+			} else {
+				rs[ctype] = m
+			}
+		}
+	}
+	return
+}
+
+// UpChargeRatio get up charge ratio
+func (p *ChargeRatioSvr) UpChargeRatio(c context.Context, limit int64) (rs map[int]map[int64]*model.UpChargeRatio, err error) {
+	rs = make(map[int]map[int64]*model.UpChargeRatio)
+	var id int64
+	for {
+		var ros map[int]map[int64]*model.UpChargeRatio
+		ros, id, err = p.dao.UpChargeRatio(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(ros) == 0 {
+			break
+		}
+		for ctype, m := range ros {
+			if _, ok := rs[ctype]; ok {
+				for mid, ratio := range m {
+					rs[ctype][mid] = ratio
+				}
+			} else {
+				rs[ctype] = m
+			}
+		}
+	}
+	return
+}

app/job/main/growup/service/income/charge_ratio_test.go

@@ -0,0 +1,22 @@
+package income
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_AvChargeRatio(t *testing.T) {
+	Convey("AvChargeRatio", t, func() {
+		_, err := s.ratio.ArchiveChargeRatio(context.Background(), 10)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_UpChargeRatio(t *testing.T) {
+	Convey("UpChargeRatio", t, func() {
+		_, err := s.ratio.UpChargeRatio(context.Background(), 10)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/growup/service/income/column_charge.go

@@ -0,0 +1,28 @@
+package income
+
+import (
+	"context"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+)
+
+func (s *Service) columnCharges(c context.Context, date time.Time, ch chan []*model.ColumnCharge) (err error) {
+	defer func() {
+		close(ch)
+	}()
+	var id int64
+	for {
+		var charges []*model.ColumnCharge
+		charges, err = s.dao.ColumnDailyCharge(c, date, id, _limitSize)
+		if err != nil {
+			return
+		}
+		ch <- charges
+		if len(charges) < _limitSize {
+			break
+		}
+		id = charges[len(charges)-1].ID
+	}
+	return
+}

app/job/main/growup/service/income/column_income.go

@@ -0,0 +1,88 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// ColumnIncomeSvr column income service
+type ColumnIncomeSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewColumnIncomeSvr new income service
+func NewColumnIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *ColumnIncomeSvr) {
+	return &ColumnIncomeSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// BatchInsertColumnIncome batch insert column income
+func (p *ColumnIncomeSvr) BatchInsertColumnIncome(ctx context.Context, cm map[int64][]*model.ColumnIncome) (err error) {
+	var (
+		buff    = make([]*model.ColumnIncome, p.batchSize)
+		buffEnd = 0
+	)
+
+	for _, cs := range cm {
+		for _, c := range cs {
+			buff[buffEnd] = c
+			buffEnd++
+
+			if buffEnd >= p.batchSize {
+				values := columnIncomeValues(buff[:buffEnd])
+				buffEnd = 0
+				_, err = p.dao.InsertColumnIncome(ctx, values)
+				if err != nil {
+					return
+				}
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := columnIncomeValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertColumnIncome(ctx, values)
+	}
+	return
+}
+
+func columnIncomeValues(cs []*model.ColumnIncome) (values string) {
+	var buf bytes.Buffer
+	for _, c := range cs {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(c.ArticleID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + c.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.ViewCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.Income, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.TotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.TaxMoney, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + c.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.BaseIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/column_income_stat.go

@@ -0,0 +1,101 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+	"strings"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// ColumnIncomeStatSvr column income statistics service
+type ColumnIncomeStatSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewColumnIncomeStatSvr new column income stat service
+func NewColumnIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *ColumnIncomeStatSvr) {
+	return &ColumnIncomeStatSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// ColumnIncomeStat column income statistisc
+func (p *ColumnIncomeStatSvr) ColumnIncomeStat(c context.Context, limit int64) (m map[int64]*model.ColumnIncomeStat, err error) {
+	m = make(map[int64]*model.ColumnIncomeStat)
+	var id int64
+	for {
+		var cm map[int64]*model.ColumnIncomeStat
+		cm, id, err = p.dao.ColumnIncomeStat(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(cm) == 0 {
+			break
+		}
+		for sid, stat := range cm {
+			m[sid] = stat
+		}
+	}
+	return
+}
+
+// BatchInsertColumnIncomeStat batch insert column income stat
+func (p *ColumnIncomeStatSvr) BatchInsertColumnIncomeStat(ctx context.Context, cs map[int64]*model.ColumnIncomeStat) (err error) {
+	var (
+		buff    = make([]*model.ColumnIncomeStat, p.batchSize)
+		buffEnd = 0
+	)
+	for _, c := range cs {
+		if c.DataState == 0 {
+			continue
+		}
+		buff[buffEnd] = c
+		buffEnd++
+
+		if buffEnd >= p.batchSize {
+			values := columnIncomeStatValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = p.dao.InsertColumnIncomeStat(ctx, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := columnIncomeStatValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertColumnIncomeStat(ctx, values)
+	}
+	return
+}
+
+func columnIncomeStatValues(cs []*model.ColumnIncomeStat) (values string) {
+	var buf bytes.Buffer
+	for _, c := range cs {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(c.ArticleID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("\"" + strings.Replace(c.Title, "\"", "\\\"", -1) + "\"")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + c.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(c.TotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/common.go

@@ -0,0 +1,38 @@
+package income
+
+import (
+	"time"
+)
+
+const (
+	_layout    = "2006-01-02"
+	_layoutSec = "2006-01-02 15:04:05"
+)
+
+var (
+	startWeeklyDate  time.Time
+	startMonthlyDate time.Time
+)
+
+var (
+	batchSize = 2000
+	_dbInsert = 1
+	_dbUpdate = 2
+
+	_video  = 0
+	_column = 2
+	_bgm    = 3
+
+	_limitSize = 2000
+)
+
+func getStartWeeklyDate(date time.Time) time.Time {
+	for date.Weekday() != time.Monday {
+		date = date.AddDate(0, 0, -1)
+	}
+	return date
+}
+
+func getStartMonthlyDate(date time.Time) time.Time {
+	return time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+}

app/job/main/growup/service/income/income.go

@@ -0,0 +1,963 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+	"time"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+	xtime "go-common/library/time"
+)
+
+// Income income service
+type Income struct {
+	avIncomeSvr         *AvIncomeSvr
+	upIncomeSvr         *UpIncomeSvr
+	avIncomeStatSvr     *AvIncomeStatSvr
+	bgmIncomeStatSvr    *BgmIncomeStatSvr
+	columnIncomeStatSvr *ColumnIncomeStatSvr
+	upIncomeStatSvr     *UpIncomeStatSvr
+	upAccountSvr        *UpAccountSvr
+	dateStatisSvr       *DateStatis
+	bgmIncomeSvr        *BgmIncomeSvr
+	columnIncomeSvr     *ColumnIncomeSvr
+}
+
+// NewIncome new income service
+func NewIncome(batchSize int, dao *incomeD.Dao) *Income {
+	return &Income{
+		avIncomeSvr:         NewAvIncomeSvr(dao, batchSize),
+		bgmIncomeSvr:        NewBgmIncomeSvr(dao, batchSize),
+		columnIncomeSvr:     NewColumnIncomeSvr(dao, batchSize),
+		upIncomeSvr:         NewUpIncomeSvr(dao, batchSize),
+		avIncomeStatSvr:     NewAvIncomeStatSvr(dao, batchSize),
+		bgmIncomeStatSvr:    NewBgmIncomeStatSvr(dao, batchSize),
+		columnIncomeStatSvr: NewColumnIncomeStatSvr(dao, batchSize),
+		upIncomeStatSvr:     NewUpIncomeStatSvr(dao, batchSize),
+		upAccountSvr:        NewUpAccountSvr(dao, batchSize),
+		dateStatisSvr:       NewDateStatis(dao),
+	}
+}
+
+// CalAvIncome cal av income
+func (p *Income) CalAvIncome(ch chan []*model.AvCharge, urs map[int64]*model.UpChargeRatio, ars map[int64]*model.ArchiveChargeRatio, filters []AvFilter, signed map[int64]bool) (am map[int64][]*model.AvIncome, total map[int64]*model.UpBusinessIncome) {
+	am, total = getClassifyAvIncome(ch, ars, filters)
+	classifyAvIncome(am, urs, ars, total, signed)
+	return
+}
+
+// CalColumnIncome cal column income
+func (p *Income) CalColumnIncome(ch chan []*model.ColumnCharge, urs map[int64]*model.UpChargeRatio, ars map[int64]*model.ArchiveChargeRatio, filters []ColumnFilter, signed map[int64]bool) (cm map[int64][]*model.ColumnIncome, total map[int64]*model.UpBusinessIncome) {
+	cm, total = getClassifyColumnIncome(ch, ars, filters)
+	classifyColumnIncome(cm, urs, ars, total, signed)
+	return
+}
+
+// CalBgmIncome cal bgm income
+func (p *Income) CalBgmIncome(ch chan []*model.AvCharge, bgms map[int64][]*model.BGM, urs map[int64]*model.UpChargeRatio, ars map[int64]*model.ArchiveChargeRatio, avf []AvFilter, exclude BgmFilter, black map[int64]bool, signed map[int64]bool) (bm map[int64]map[int64]map[int64]*model.BgmIncome, total map[int64]*model.UpBusinessIncome) {
+	bm, total = getClassifyBgmIncome(ch, bgms, avf, exclude, black, ars)
+	classifyBgmIncome(bm, urs, ars, total, signed)
+	return
+}
+
+func trans2AvIncome(charge *model.AvCharge, incr int64) *model.AvIncome {
+	return &model.AvIncome{
+		AvID:       charge.AvID,
+		MID:        charge.MID,
+		TagID:      charge.TagID,
+		IsOriginal: charge.IsOriginal,
+		UploadTime: charge.UploadTime,
+		PlayCount:  charge.TotalPlayCount,
+		Date:       charge.Date,
+		Income:     incr,
+		BaseIncome: charge.IncCharge,
+	}
+}
+
+func av2BusinessIncome(a *model.AvIncome) *model.UpBusinessIncome {
+	return &model.UpBusinessIncome{
+		MID:       a.MID,
+		Income:    a.Income,
+		PlayCount: a.PlayCount,
+		AvCount:   1,
+		Business:  1,
+	}
+}
+
+// av income before tax
+// total key:mid, value:total_income
+func getClassifyAvIncome(ch chan []*model.AvCharge, ratio map[int64]*model.ArchiveChargeRatio, filters []AvFilter) (am map[int64][]*model.AvIncome, total map[int64]*model.UpBusinessIncome) {
+	am = make(map[int64][]*model.AvIncome)
+	total = make(map[int64]*model.UpBusinessIncome)
+	for charges := range ch {
+	CHARGE:
+		for _, charge := range charges {
+			for _, filter := range filters {
+				if filter(charge) {
+					continue CHARGE
+				}
+			}
+			var incr int64
+			if r, ok := ratio[charge.AvID]; ok {
+				if r.AdjustType == 0 {
+					incr = int64(Round(float64(charge.IncCharge)*Div(float64(r.Ratio), float64(100)), 0))
+				} else if r.AdjustType == 1 {
+					incr = charge.IncCharge
+				}
+			} else {
+				incr = charge.IncCharge
+			}
+			avIncome := trans2AvIncome(charge, incr)
+			if _, ok := am[charge.MID]; ok {
+				am[charge.MID] = append(am[charge.MID], avIncome)
+			} else {
+				am[charge.MID] = []*model.AvIncome{avIncome}
+			}
+
+			if business, ok := total[charge.MID]; ok {
+				business.PlayCount += avIncome.PlayCount
+				business.AvCount++
+				business.Income += avIncome.Income
+			} else {
+				business := av2BusinessIncome(avIncome)
+				total[charge.MID] = business
+			}
+		}
+	}
+	return
+}
+
+func classifyAvIncome(am map[int64][]*model.AvIncome,
+	urs map[int64]*model.UpChargeRatio,
+	ars map[int64]*model.ArchiveChargeRatio,
+	total map[int64]*model.UpBusinessIncome,
+	signed map[int64]bool) {
+
+	for mid, business := range total {
+		realIncome := business.Income
+		if r, ok := urs[mid]; ok {
+			if r.AdjustType == 0 {
+				// up主浮动调节,分配到视频业务
+				realIncome = int64(Round(float64(business.Income)*Div(float64(r.Ratio), float64(100)), 0))
+			}
+		}
+		// up主浮动调节后视频业务的税
+		tax := int64(Round(Tax(Div(float64(realIncome), 100))*100, 0))
+		// 税后收入
+		netIncome := realIncome - tax
+		// update up income
+		business.Percent = Div(float64(netIncome), float64(business.Income))
+		business.Income = netIncome
+		business.Tax = tax
+	}
+
+	// 计算每个视频浮动调节后收入
+	for mid, as := range am {
+		business := total[mid]
+		// up主的固定调节收入
+		var patchIncome int64
+		var c bool
+		for _, a := range as {
+			avIncome := int64(Round(Mul(float64(a.Income), business.Percent), 0))
+			a.Income = avIncome
+			a.TaxMoney = int64(Round(Mul(float64(business.Tax), Div(float64(avIncome), float64(business.Income))), 0))
+
+			// 以下是计算up主基础收入
+			// original raito: 100
+			var o int64 = 100
+			if upRatio, ok := urs[mid]; ok {
+				if upRatio.AdjustType == 0 {
+					o = upRatio.Ratio
+					c = true
+				}
+			}
+			avBaseIncome := int64(Round(Div(float64(avIncome), Div(float64(o), 100)), 0))
+			a.BaseIncome = avBaseIncome
+			if r, ok := ars[a.AvID]; ok {
+				if r.AdjustType == 0 {
+					// 如果视频收入被加倍过,还原
+					originIncome := int64(Round(Div(float64(avBaseIncome), Div(float64(r.Ratio), 100)), 0))
+					a.BaseIncome = originIncome
+					business.BaseIncome += originIncome
+					c = true
+				}
+				if r.AdjustType == 1 {
+					// av 固定调节, 更新av的收入和up主的基础收入
+					a.Income += r.Ratio
+					// 更新up主的av收入, Income在计算BaseIncome后更新
+					// upIncome.AvIncome += r.Ratio
+					// business.Income += r.Ratio
+					patchIncome += r.Ratio
+					business.BaseIncome += avBaseIncome
+				}
+			} else {
+				business.BaseIncome += avBaseIncome
+			}
+		}
+
+		// 如果没有被加倍过, 那么避免误差将BaseIncome直接置为Income
+		if !c {
+			business.BaseIncome = business.Income
+		}
+		// 最后加上该up主av的总固定调节收入
+		business.Income += patchIncome
+	}
+
+	// + up主的固定调节
+	for mid, ratio := range urs {
+		if _, ok := signed[mid]; !ok {
+			continue
+		}
+		if ratio.AdjustType == 0 {
+			continue
+		}
+
+		if business, ok := total[mid]; ok {
+			business.Income += ratio.Ratio
+		} else {
+			total[mid] = &model.UpBusinessIncome{
+				MID:      mid,
+				Income:   ratio.Ratio,
+				Business: 1,
+			}
+		}
+	}
+}
+
+/*################################################################## BGM ########################################################################*/
+
+func trans2BgmIncome(b *model.BGM, charge int64, date xtime.Time) *model.BgmIncome {
+	return &model.BgmIncome{
+		AID:    b.AID,
+		SID:    b.SID,
+		MID:    b.MID,
+		CID:    b.CID,
+		Income: charge,
+		Date:   date,
+	}
+}
+
+func bgm2BusinessIncome(b *model.BgmIncome) *model.UpBusinessIncome {
+	return &model.UpBusinessIncome{
+		MID:      b.MID,
+		Income:   b.Income,
+		Business: 3,
+		BgmCount: map[int64]bool{
+			b.SID: true,
+		},
+	}
+}
+
+// bgms map[avid][sid]*model.BGM | bm map[mid]map[sid]map[avid]*model.BgmIncome
+func getClassifyBgmIncome(ch chan []*model.AvCharge, bgms map[int64][]*model.BGM, filters []AvFilter, exclude BgmFilter, black map[int64]bool, ratio map[int64]*model.ArchiveChargeRatio) (bm map[int64]map[int64]map[int64]*model.BgmIncome, total map[int64]*model.UpBusinessIncome) {
+	bm = make(map[int64]map[int64]map[int64]*model.BgmIncome)
+	total = make(map[int64]*model.UpBusinessIncome)
+	for charges := range ch {
+	CHARGE:
+		for _, charge := range charges {
+			for _, filter := range filters {
+				if filter(charge) {
+					continue CHARGE
+				}
+			}
+
+			if bs, ok := bgms[charge.AvID]; ok {
+				bgmCharge := int64(Round(Div(Mul(float64(charge.IncCharge), float64(0.3)), float64(len(bs))), 0))
+
+				for _, b := range bs {
+					// if av's bgm is own, continue
+					if b.MID == charge.MID {
+						continue
+					}
+
+					if _, ok := black[b.SID]; ok {
+						continue
+					}
+
+					if exclude(charge, b) {
+						continue
+					}
+
+					var incr int64
+					if r, ok := ratio[b.SID]; ok {
+						if r.AdjustType == 0 {
+							incr = int64(Round(float64(bgmCharge)*Div(float64(r.Ratio), float64(100)), 0))
+						} else if r.AdjustType == 1 {
+							incr = bgmCharge
+						}
+					} else {
+						incr = bgmCharge
+					}
+
+					// bm map[mid]map[sid]map[avid]*model.BgmIncome
+					var bgmIncome *model.BgmIncome
+					if sm, ok := bm[b.MID]; ok {
+						if am, ok := sm[b.SID]; ok {
+							if bgmIncome, ok = am[b.AID]; ok {
+								bgmIncome.Income += incr
+							} else {
+								bgmIncome = trans2BgmIncome(b, incr, charge.Date)
+								am[b.AID] = bgmIncome
+							}
+						} else {
+							bgmIncome = trans2BgmIncome(b, incr, charge.Date)
+							sm[b.SID] = map[int64]*model.BgmIncome{
+								b.AID: bgmIncome,
+							}
+						}
+					} else {
+						bgmIncome = trans2BgmIncome(b, incr, charge.Date)
+						// am map[avid]*model.BgmIncome
+						am := map[int64]*model.BgmIncome{
+							b.AID: bgmIncome,
+						}
+						// sm map[sid]map[avid]*model.BgmIncome
+						sm := map[int64]map[int64]*model.BgmIncome{
+							b.SID: am,
+						}
+						// bm map[mid]map[sid]map[avid]*model.BgmIncome
+						bm[b.MID] = sm
+					}
+
+					if business, ok := total[b.MID]; ok {
+						business.Income += incr
+						business.BgmCount[b.SID] = true
+					} else {
+						business := bgm2BusinessIncome(bgmIncome)
+						total[b.MID] = business
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+// 算税并分配收入
+func classifyBgmIncome(bm map[int64]map[int64]map[int64]*model.BgmIncome,
+	urs map[int64]*model.UpChargeRatio,
+	ars map[int64]*model.ArchiveChargeRatio,
+	total map[int64]*model.UpBusinessIncome,
+	signed map[int64]bool) {
+	for mid, business := range total {
+		if business.Income == 0 {
+			delete(total, mid)
+			delete(bm, mid)
+			continue
+		}
+		realIncome := business.Income
+		if r, ok := urs[mid]; ok {
+			if r.AdjustType == 0 {
+				// up主浮动调节,分配到视频业务
+				realIncome = int64(Round(float64(business.Income)*Div(float64(r.Ratio), float64(100)), 0))
+			}
+		}
+		// up主浮动调节后视频业务的税
+		tax := int64(Round(Tax(Div(float64(realIncome), 100))*100, 0))
+		// 税后收入
+		netIncome := realIncome - tax
+		// update up income
+		business.Percent = Div(float64(netIncome), float64(business.Income))
+		business.Income = netIncome
+		business.Tax = tax
+	}
+	// bm map[mid]map[sid]map[avid]*model.BgmIncome
+	for mid, sm := range bm {
+		business := total[mid]
+		var c bool
+		var patchIncome int64
+		for sid, bs := range sm {
+			var dailyTotalIncome int64
+			for _, b := range bs {
+				income := int64(Round(Mul(float64(b.Income), business.Percent), 0))
+				b.Income = income
+				b.TaxMoney = int64(Round(Mul(float64(business.Tax), Div(float64(income), float64(business.Income))), 0))
+				dailyTotalIncome += b.Income
+
+				// 以下是计算up主基础收入
+				// original raito: 100
+				var o int64 = 100
+				if upRatio, ok := urs[mid]; ok {
+					if upRatio.AdjustType == 0 {
+						o = upRatio.Ratio
+						c = true
+					}
+				}
+
+				bgmBaseIncome := int64(Round(Div(float64(income), Div(float64(o), 100)), 0))
+				b.BaseIncome = bgmBaseIncome
+				if r, ok := ars[sid]; ok {
+					if r.AdjustType == 0 {
+						// 如果bgm收入被加倍过,还原
+						originIncome := int64(Round(Div(float64(bgmBaseIncome), Div(float64(r.Ratio), 100)), 0))
+						b.BaseIncome = originIncome
+						business.BaseIncome += originIncome
+						c = true
+					} else {
+						business.BaseIncome += bgmBaseIncome
+					}
+				} else {
+					business.BaseIncome += bgmBaseIncome
+				}
+			}
+			if r, ok := ars[sid]; ok {
+				if r.AdjustType == 1 {
+					patchIncome += r.Ratio
+					dailyTotalIncome += r.Ratio
+				}
+			}
+			// update bgm daily total income
+			for _, b := range bs {
+				b.DailyTotalIncome = dailyTotalIncome
+			}
+		}
+
+		if !c {
+			business.BaseIncome = business.Income
+		}
+		// 最后加上该up主bgm的总固定调节收入
+		business.Income += patchIncome
+	}
+
+	// + up主的固定调节
+	for mid, ratio := range urs {
+		if _, ok := signed[mid]; !ok {
+			continue
+		}
+		if ratio.AdjustType == 0 {
+			continue
+		}
+
+		if business, ok := total[mid]; ok {
+			business.Income += ratio.Ratio
+		} else {
+			total[mid] = &model.UpBusinessIncome{
+				MID:      mid,
+				Income:   ratio.Ratio,
+				Business: 3,
+			}
+		}
+	}
+}
+
+/*###################################################### Column #######################################################*/
+
+func trans2ColumnIncome(charge *model.ColumnCharge, incr int64) (c *model.ColumnIncome) {
+	return &model.ColumnIncome{
+		ArticleID:  charge.ArticleID,
+		Title:      charge.Title,
+		MID:        charge.MID,
+		TagID:      charge.TagID,
+		UploadTime: charge.UploadTime,
+		ViewCount:  charge.IncViewCount,
+		Date:       charge.Date,
+		Income:     incr,
+		BaseIncome: charge.IncCharge,
+	}
+}
+
+// business type 2: column
+func column2BusinessIncome(c *model.ColumnIncome) *model.UpBusinessIncome {
+	return &model.UpBusinessIncome{
+		MID:         c.MID,
+		Income:      c.Income,
+		ViewCount:   c.ViewCount,
+		ColumnCount: 1,
+		Business:    2,
+	}
+}
+
+func getClassifyColumnIncome(ch chan []*model.ColumnCharge, ratio map[int64]*model.ArchiveChargeRatio, filters []ColumnFilter) (cm map[int64][]*model.ColumnIncome, total map[int64]*model.UpBusinessIncome) {
+	cm = make(map[int64][]*model.ColumnIncome)
+	total = make(map[int64]*model.UpBusinessIncome)
+
+	for charges := range ch {
+	CHARGE:
+		for _, charge := range charges {
+			for _, filter := range filters {
+				if filter(charge) {
+					continue CHARGE
+				}
+			}
+
+			var incr int64
+			if r, ok := ratio[charge.ArticleID]; ok {
+				if r.AdjustType == 0 {
+					incr = int64(Round(float64(charge.IncCharge)*Div(float64(r.Ratio), float64(100)), 0))
+				} else if r.AdjustType == 1 {
+					incr = charge.IncCharge
+				}
+			} else {
+				incr = charge.IncCharge
+			}
+
+			columnIncome := trans2ColumnIncome(charge, incr)
+			if _, ok := cm[charge.MID]; ok {
+				cm[charge.MID] = append(cm[charge.MID], columnIncome)
+			} else {
+				cm[charge.MID] = []*model.ColumnIncome{columnIncome}
+			}
+
+			if business, ok := total[charge.MID]; ok {
+				business.ViewCount += columnIncome.ViewCount
+				business.ColumnCount++
+				business.Income += columnIncome.Income
+			} else {
+				business := column2BusinessIncome(columnIncome)
+				total[charge.MID] = business
+			}
+		}
+	}
+	return
+}
+
+// calculate column income and archive income
+func classifyColumnIncome(cm map[int64][]*model.ColumnIncome,
+	urs map[int64]*model.UpChargeRatio,
+	ars map[int64]*model.ArchiveChargeRatio,
+	total map[int64]*model.UpBusinessIncome,
+	signed map[int64]bool) {
+	for mid, business := range total {
+		realIncome := business.Income
+		if r, ok := urs[mid]; ok {
+			if r.AdjustType == 0 {
+				// up主浮动调节,分配到视频业务
+				realIncome = int64(Round(float64(business.Income)*Div(float64(r.Ratio), float64(100)), 0))
+			}
+		}
+		// up主浮动调节后视频业务的税
+		tax := int64(Round(Tax(Div(float64(realIncome), 100))*100, 0))
+		// 税后收入
+		netIncome := realIncome - tax
+		// update up income
+		business.Percent = Div(float64(netIncome), float64(business.Income))
+		business.Income = netIncome
+		business.Tax = tax
+	}
+	for mid, cs := range cm {
+		business := total[mid]
+		var x bool
+		var patchIncome int64
+		for _, c := range cs {
+			columnIncome := int64(Round(Mul(float64(c.Income), business.Percent), 0))
+			c.Income = columnIncome
+			c.TaxMoney = int64(Round(Mul(float64(business.Tax), Div(float64(columnIncome), float64(business.Income))), 0))
+
+			// 以下是计算up主基础收入
+			// original raito: 100
+			var o int64 = 100
+			if upRatio, ok := urs[mid]; ok {
+				if upRatio.AdjustType == 0 {
+					o = upRatio.Ratio
+					x = true
+				}
+			}
+			columnBaseIncome := int64(Round(Div(float64(columnIncome), Div(float64(o), 100)), 0))
+			c.BaseIncome = columnBaseIncome
+			if r, ok := ars[c.ArticleID]; ok {
+				if r.AdjustType == 0 {
+					// 如果视频收入被加倍过,还原
+					originIncome := int64(Round(Div(float64(columnBaseIncome), Div(float64(r.Ratio), 100)), 0))
+					c.BaseIncome = originIncome
+					business.BaseIncome += originIncome
+					x = true
+				}
+				if r.AdjustType == 1 {
+					// column 固定调节, 更新column 的收入和up主的基础收入
+					c.Income += r.Ratio
+					// 更新up主的column 收入, Income在计算BaseIncome后更新
+					patchIncome += r.Ratio
+					business.BaseIncome += columnBaseIncome
+				}
+			} else {
+				business.BaseIncome += columnBaseIncome
+			}
+		}
+		if !x {
+			business.BaseIncome = business.Income
+		}
+		// 最后加上该up主column的总固定调节收入
+		business.Income += patchIncome
+	}
+
+	// + up主的固定调节
+	for mid, ratio := range urs {
+		if _, ok := signed[mid]; !ok {
+			continue
+		}
+		if ratio.AdjustType == 0 {
+			continue
+		}
+
+		if business, ok := total[mid]; ok {
+			business.Income += ratio.Ratio
+		} else {
+			total[mid] = &model.UpBusinessIncome{
+				MID:      mid,
+				Income:   ratio.Ratio,
+				Business: 2,
+			}
+		}
+	}
+}
+
+/**######################################################## UpIncome ###################################################################**/
+
+// CalUpIncome calculate upIncome
+func (p *Income) CalUpIncome(ch chan map[int64]*model.UpBusinessIncome, date time.Time) (m map[int64]*model.UpIncome) {
+	defer close(ch)
+	m = make(map[int64]*model.UpIncome)
+	var finished int
+	for bm := range ch {
+		for mid, business := range bm {
+			if upIncome, ok := m[mid]; ok {
+				if business.Business == 1 {
+					upIncome.AvCount = business.AvCount
+					upIncome.PlayCount = business.PlayCount
+					upIncome.AvIncome = business.Income
+					upIncome.AvBaseIncome = business.BaseIncome
+					upIncome.AvTax = business.Tax
+				}
+				if business.Business == 2 {
+					upIncome.ColumnCount = business.ColumnCount
+					upIncome.ColumnIncome = business.Income
+					upIncome.ColumnBaseIncome = business.BaseIncome
+					upIncome.ColumnTax = business.Tax
+				}
+				if business.Business == 3 {
+					upIncome.BgmIncome = business.Income
+					upIncome.BgmBaseIncome = business.BaseIncome
+					upIncome.BgmTax = business.Tax
+					upIncome.BgmCount = int64(len(business.BgmCount))
+				}
+				upIncome.TaxMoney += business.Tax
+				upIncome.BaseIncome += business.BaseIncome
+				upIncome.Income += business.Income
+			} else {
+				var upIncome *model.UpIncome
+				if business.Business == 1 {
+					upIncome = &model.UpIncome{
+						AvCount:      business.AvCount,
+						PlayCount:    business.PlayCount,
+						AvIncome:     business.Income,
+						AvBaseIncome: business.BaseIncome,
+						AvTax:        business.Tax,
+					}
+				}
+				if business.Business == 2 {
+					upIncome = &model.UpIncome{
+						ColumnCount:      business.ColumnCount,
+						ColumnIncome:     business.Income,
+						ColumnBaseIncome: business.BaseIncome,
+						ColumnTax:        business.Tax,
+					}
+				}
+				if business.Business == 3 {
+					upIncome = &model.UpIncome{
+						BgmIncome:     business.Income,
+						BgmBaseIncome: business.BaseIncome,
+						BgmTax:        business.Tax,
+						BgmCount:      int64(len(business.BgmCount)),
+					}
+				}
+				upIncome.MID = mid
+				upIncome.TaxMoney = business.Tax
+				upIncome.BaseIncome = business.BaseIncome
+				upIncome.Income = business.Income
+				upIncome.Date = xtime.Time(date.Unix())
+				m[mid] = upIncome
+			}
+		}
+		finished++
+		if finished == 3 {
+			break
+		}
+	}
+	return
+}
+
+// IncomeStat income statistics
+func (p *Income) IncomeStat(
+	um map[int64]*model.UpIncome,
+	am map[int64][]*model.AvIncome,
+	bm map[int64]map[int64]map[int64]*model.BgmIncome,
+	cm map[int64][]*model.ColumnIncome,
+	ustat map[int64]*model.UpIncomeStat,
+	astat map[int64]*model.AvIncomeStat,
+	bstat map[int64]*model.BgmIncomeStat,
+	cstat map[int64]*model.ColumnIncomeStat) {
+	for mid, upIncome := range um {
+		// update up income total income
+		ut, ok := ustat[mid]
+		if !ok {
+			ut = &model.UpIncomeStat{
+				MID:       upIncome.MID,
+				DataState: 1, // insert
+			}
+			ustat[mid] = ut
+		} else {
+			ut.DataState = 2 // update
+		}
+
+		// update up income statis up total income
+		// up av total income
+		// up column total income
+		// up bgm total income
+		ut.TotalIncome += upIncome.Income
+		upIncome.TotalIncome = ut.TotalIncome
+
+		ut.AvTotalIncome += upIncome.AvIncome
+		upIncome.AvTotalIncome = ut.AvTotalIncome
+
+		ut.ColumnTotalIncome += upIncome.ColumnIncome
+		upIncome.ColumnTotalIncome = ut.ColumnTotalIncome
+
+		ut.BgmTotalIncome += upIncome.BgmIncome
+		upIncome.BgmTotalIncome = ut.BgmTotalIncome
+
+		// update av income total income
+		for _, a := range am[mid] {
+			at, ok := astat[a.AvID]
+			if !ok {
+				at = &model.AvIncomeStat{
+					AvID:       a.AvID,
+					MID:        a.MID,
+					TagID:      a.TagID,
+					IsOriginal: a.IsOriginal,
+					UploadTime: a.UploadTime,
+					DataState:  1, // insert
+				}
+				astat[a.AvID] = at
+			} else {
+				at.DataState = 2 // update
+			}
+			at.TotalIncome += a.Income
+			a.TotalIncome = at.TotalIncome
+		}
+
+		// update bgmIncome total income
+		for sid, am := range bm[mid] {
+			bt, ok := bstat[sid]
+			if !ok {
+				bt = &model.BgmIncomeStat{
+					SID:       sid,
+					DataState: 1, // insert
+				}
+				bstat[sid] = bt
+			} else {
+				bt.DataState = 2 // update
+			}
+			for _, b := range am {
+				bt.TotalIncome += b.DailyTotalIncome
+				break
+			}
+			for _, b := range am {
+				b.TotalIncome = bt.TotalIncome
+			}
+		}
+
+		// update columnIncome total income
+		for _, c := range cm[mid] {
+			ct, ok := cstat[c.ArticleID]
+			if !ok {
+				ct = &model.ColumnIncomeStat{
+					ArticleID:  c.ArticleID,
+					Title:      c.Title,
+					TagID:      c.TagID,
+					MID:        c.MID,
+					UploadTime: c.UploadTime,
+					DataState:  1, // insert
+				}
+				cstat[c.ArticleID] = ct
+			} else {
+				ct.DataState = 2 // update
+			}
+			ct.TotalIncome += c.Income
+			c.TotalIncome = ct.TotalIncome
+		}
+	}
+}
+
+// PurgeUpAccount purge up account
+func (p *Income) PurgeUpAccount(date time.Time, accs map[int64]*model.UpAccount, um map[int64]*model.UpIncome) {
+	fd := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+	last := fd.AddDate(0, 0, -1).Format("2006-01")
+
+	// TODO check
+	for mid, upIncome := range um {
+		// update old up account
+		if upAccount, ok := accs[mid]; ok {
+			upAccount.TotalIncome += upIncome.Income
+			if upAccount.WithdrawDateVersion == last {
+				upAccount.TotalUnwithdrawIncome += upIncome.Income
+			}
+			upAccount.DataState = 2
+		} else { // append new up account
+			accs[mid] = &model.UpAccount{
+				MID:                   mid,
+				HasSignContract:       1,
+				TotalIncome:           upIncome.Income,
+				TotalUnwithdrawIncome: upIncome.Income,
+				WithdrawDateVersion:   last,
+				DataState:             1,
+			}
+		}
+	}
+}
+
+/************************************************************ FOR HISTORY ****************************************************************/
+
+// UpdateBusinessIncomeByDate update business income by date
+func (p *Income) UpdateBusinessIncomeByDate(c context.Context, date string) (err error) {
+	ustat, err := p.upIncomeStatSvr.UpIncomeStat(c, int64(_limitSize))
+	if err != nil {
+		return
+	}
+	return p.updateBusinessIncomeByDate(c, date, ustat)
+}
+
+func (p *Income) updateBusinessIncomeByDate(c context.Context, date string, ustat map[int64]*model.UpIncomeStat) (err error) {
+	us, err := p.businessTotalIncome(c, ustat, date)
+	if err != nil {
+		return
+	}
+	err = p.batchUpdateUpIncome(c, us)
+	if err != nil {
+		return
+	}
+	return p.batchUpdateUpIncomeStat(c, ustat)
+}
+
+// for history data m: map[mid]map[date]*model.UpIncome
+func (p *Income) businessTotalIncome(c context.Context, ustat map[int64]*model.UpIncomeStat, date string) (m []*model.UpIncome, err error) {
+	var id int64
+	for {
+		var ups []*model.UpIncome
+		ups, err = p.upIncomeSvr.dao.GetUpIncomeTable(c, "up_income", date, id, 2000)
+		if err != nil {
+			return
+		}
+		for _, up := range ups {
+			if ut, ok := ustat[up.MID]; ok {
+				ut.AvTotalIncome += up.AvIncome
+				up.AvTotalIncome = ut.AvTotalIncome
+
+				ut.ColumnTotalIncome += up.ColumnIncome
+				up.ColumnTotalIncome = ut.ColumnTotalIncome
+
+				ut.BgmTotalIncome += up.BgmIncome
+				up.BgmTotalIncome = ut.BgmTotalIncome
+				m = append(m, up)
+			}
+		}
+		if len(ups) < 2000 {
+			break
+		}
+		id = ups[len(ups)-1].ID
+	}
+	return
+}
+
+// BatchUpdateUpIncome insert up_income batch
+func (p *Income) batchUpdateUpIncome(c context.Context, us []*model.UpIncome) (err error) {
+	var (
+		buff    = make([]*model.UpIncome, p.upIncomeSvr.batchSize)
+		buffEnd = 0
+	)
+
+	for _, u := range us {
+		buff[buffEnd] = u
+		buffEnd++
+
+		if buffEnd >= p.upIncomeSvr.batchSize {
+			values := businessValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = p.upIncomeSvr.dao.FixInsertUpIncome(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := businessValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.upIncomeSvr.dao.FixInsertUpIncome(c, values)
+	}
+	return
+}
+
+func businessValues(us []*model.UpIncome) (values string) {
+	var buf bytes.Buffer
+	for _, u := range us {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(u.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}
+
+// BatchUpdateUpIncome insert up_income batch
+func (p *Income) batchUpdateUpIncomeStat(c context.Context, us map[int64]*model.UpIncomeStat) (err error) {
+	var (
+		buff    = make([]*model.UpIncomeStat, p.upIncomeStatSvr.batchSize)
+		buffEnd = 0
+	)
+
+	for _, u := range us {
+		buff[buffEnd] = u
+		buffEnd++
+
+		if buffEnd >= p.upIncomeStatSvr.batchSize {
+			values := businessStatValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = p.upIncomeStatSvr.dao.FixInsertUpIncomeStat(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := businessStatValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.upIncomeStatSvr.dao.FixInsertUpIncomeStat(c, values)
+	}
+	return
+}
+
+func businessStatValues(ustat []*model.UpIncomeStat) (values string) {
+	var buf bytes.Buffer
+	for _, u := range ustat {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(u.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/income_date_statis.go

@@ -0,0 +1,272 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math"
+	"strconv"
+	"time"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+)
+
+var (
+	_avIncomeDailyStatis   = "av_income_daily_statis"
+	_avIncomeWeeklyStatis  = "av_income_weekly_statis"
+	_avIncomeMonthlyStatis = "av_income_monthly_statis"
+
+	_cmIncomeDailyStatis   = "column_income_daily_statis"
+	_cmIncomeWeeklyStatis  = "column_income_weekly_statis"
+	_cmIncomeMonthlyStatis = "column_income_monthly_statis"
+
+	_bgmIncomeDailyStatis   = "bgm_income_daily_statis"
+	_bgmIncomeWeeklyStatis  = "bgm_income_weekly_statis"
+	_bgmIncomeMonthlyStatis = "bgm_income_monthly_statis"
+)
+
+// SectionEntries section entries
+type SectionEntries struct {
+	avDaily   []*model.DateStatis
+	avWeekly  []*model.DateStatis
+	avMonthly []*model.DateStatis
+}
+
+// DateStatis income date statistics
+type DateStatis struct {
+	dao *incomeD.Dao
+}
+
+// NewDateStatis new income date statistics service
+func NewDateStatis(dao *incomeD.Dao) *DateStatis {
+	return &DateStatis{dao: dao}
+}
+
+func initIncomeSections(income, tagID int64, date xtime.Time) []*model.DateStatis {
+	incomeSections := make([]*model.DateStatis, 12)
+	incomeSections[0] = initIncomeSection(0, 1, 0, income, tagID, date)
+	incomeSections[1] = initIncomeSection(1, 5, 1, income, tagID, date)
+	incomeSections[2] = initIncomeSection(5, 10, 2, income, tagID, date)
+	incomeSections[3] = initIncomeSection(10, 30, 3, income, tagID, date)
+	incomeSections[4] = initIncomeSection(30, 50, 4, income, tagID, date)
+	incomeSections[5] = initIncomeSection(50, 100, 5, income, tagID, date)
+	incomeSections[6] = initIncomeSection(100, 200, 6, income, tagID, date)
+	incomeSections[7] = initIncomeSection(200, 500, 7, income, tagID, date)
+	incomeSections[8] = initIncomeSection(500, 1000, 8, income, tagID, date)
+	incomeSections[9] = initIncomeSection(1000, 3000, 9, income, tagID, date)
+	incomeSections[10] = initIncomeSection(3000, 5000, 10, income, tagID, date)
+	incomeSections[11] = initIncomeSection(5000, math.MaxInt32, 11, income, tagID, date)
+	return incomeSections
+}
+
+func initIncomeSection(min, max, section, income, tagID int64, date xtime.Time) *model.DateStatis {
+	var tips string
+	if max == math.MaxInt32 {
+		tips = fmt.Sprintf("\"%d+\"", min)
+	} else {
+		tips = fmt.Sprintf("\"%d~%d\"", min, max)
+	}
+	return &model.DateStatis{
+		MinIncome:    min,
+		MaxIncome:    max,
+		MoneySection: section,
+		MoneyTips:    tips,
+		Income:       income,
+		CategoryID:   tagID,
+		CDate:        date,
+	}
+}
+
+func (s *DateStatis) handleDateStatis(c context.Context, archiveCh chan []*model.ArchiveIncome, date time.Time, table string) (incomeSections []*model.DateStatis, err error) {
+	// delete
+	if table != "" {
+		_, err = s.dao.DelIncomeStatisTable(c, table, date.Format(_layout))
+		if err != nil {
+			log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
+			return
+		}
+	}
+	// add
+	incomeSections = s.handleArchives(c, archiveCh, date)
+	return
+}
+
+// handleArchives handle archive_income_daily_statis, archive_income_weekly_statis, archive_income_monthly_statis
+func (s *DateStatis) handleArchives(c context.Context, archiveCh chan []*model.ArchiveIncome, date time.Time) (incomeSections []*model.DateStatis) {
+	archTagMap := make(map[int64]map[int64]int64) // key TagID, value map[int64]int64 -> key aid, value income
+	tagIncomeMap := make(map[int64]int64)         // key TagID, value TagID total income
+	for archive := range archiveCh {
+		handleArchive(archive, archTagMap, tagIncomeMap, date)
+	}
+	incomeSections = make([]*model.DateStatis, 0)
+	for tagID, avMap := range archTagMap {
+		incomeSection := countIncomeDailyStatis(avMap, tagIncomeMap[tagID], tagID, date)
+		incomeSections = append(incomeSections, incomeSection...)
+	}
+	return
+}
+
+func handleArchive(archives []*model.ArchiveIncome, archTagMap map[int64]map[int64]int64, tagIncomeMap map[int64]int64, startDate time.Time) {
+	if archives == nil {
+		return
+	}
+	if archTagMap == nil {
+		archTagMap = make(map[int64]map[int64]int64)
+	}
+	if tagIncomeMap == nil {
+		tagIncomeMap = make(map[int64]int64)
+	}
+
+	for _, archive := range archives {
+		if !startDate.After(archive.Date.Time()) {
+			tagIncomeMap[archive.TagID] += archive.Income
+			if _, ok := archTagMap[archive.TagID]; !ok {
+				archTagMap[archive.TagID] = make(map[int64]int64)
+			}
+			archTagMap[archive.TagID][archive.AID] += archive.Income
+		}
+	}
+}
+
+func (s *DateStatis) handleDateUp(c context.Context, upStatisCh chan []*model.UpIncome, date time.Time) (upSections, upAvSections, upCmSections, upBgmSections []*model.DateStatis, err error) {
+	_, err = s.dao.DelIncomeStatisTable(c, "up_income_daily_statis", date.Format(_layout))
+	if err != nil {
+		log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
+		return
+	}
+
+	_, err = s.dao.DelIncomeStatisTable(c, "up_av_daily_statis", date.Format(_layout))
+	if err != nil {
+		log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
+		return
+	}
+
+	_, err = s.dao.DelIncomeStatisTable(c, "up_column_daily_statis", date.Format(_layout))
+	if err != nil {
+		log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
+		return
+	}
+
+	_, err = s.dao.DelIncomeStatisTable(c, "up_bgm_daily_statis", date.Format(_layout))
+	if err != nil {
+		log.Error("s.dao.DelIncomeStatisTable error(%v)", err)
+		return
+	}
+
+	upMap := make(map[int64]int64)
+	upAvMap := make(map[int64]int64)
+	upCmMap := make(map[int64]int64)
+	upBgmMap := make(map[int64]int64)
+	var upTotal, avTotal, cmTotal, bgmTotal int64
+	for up := range upStatisCh {
+		up, av, cm, bgm := handleUp(up, upMap, upAvMap, upCmMap, upBgmMap, date)
+		upTotal += up
+		avTotal += av
+		cmTotal += cm
+		bgmTotal += bgm
+	}
+	upSections = countIncomeDailyStatis(upMap, upTotal, 0, date)
+	upAvSections = countIncomeDailyStatis(upAvMap, avTotal, 0, date)
+	upCmSections = countIncomeDailyStatis(upCmMap, cmTotal, 0, date)
+	upBgmSections = countIncomeDailyStatis(upBgmMap, bgmTotal, 0, date)
+	return
+}
+
+func handleUp(upIncomes []*model.UpIncome, upMap, upAvMap, upCmMap, upBgmMap map[int64]int64, startDate time.Time) (income, avIncome, cmIncome, bgmIncome int64) {
+	if len(upIncomes) == 0 {
+		return
+	}
+	for _, upIncome := range upIncomes {
+		if startDate.Equal(upIncome.Date.Time()) {
+			income += upIncome.Income
+			avIncome += upIncome.AvIncome
+			cmIncome += upIncome.ColumnIncome
+			bgmIncome += upIncome.BgmIncome
+			upMap[upIncome.MID] += upIncome.Income
+			if upIncome.AvIncome > 0 {
+				upAvMap[upIncome.MID] += upIncome.AvIncome
+			}
+			if upIncome.ColumnIncome > 0 {
+				upCmMap[upIncome.MID] += upIncome.ColumnIncome
+			}
+			if upIncome.BgmIncome > 0 {
+				upBgmMap[upIncome.MID] += upIncome.BgmIncome
+			}
+		}
+	}
+	return
+}
+
+func (s *DateStatis) incomeDateStatisInsert(c context.Context, incomeSection []*model.DateStatis, table string) (rows int64, err error) {
+	var buf bytes.Buffer
+	for _, row := range incomeSection {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.Count, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MoneySection, 10))
+		buf.WriteByte(',')
+		buf.WriteString(row.MoneyTips)
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Income, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CategoryID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.CDate.Time().Format(_layout) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals := buf.String()
+	buf.Reset()
+	rows, err = s.dao.InsertIncomeStatisTable(c, table, vals)
+	return
+}
+
+func countIncomeDailyStatis(incomes map[int64]int64, totalIncome, tagID int64, date time.Time) (incomeSections []*model.DateStatis) {
+	if len(incomes) == 0 {
+		return
+	}
+	incomeSections = initIncomeSections(totalIncome, tagID, xtime.Time(date.Unix()))
+	for _, income := range incomes {
+		for _, section := range incomeSections {
+			min, max := section.MinIncome*100, section.MaxIncome*100
+			if income >= min && income < max {
+				section.Count++
+			}
+		}
+	}
+	return
+}
+
+func (s *DateStatis) upIncomeDailyStatisInsert(c context.Context, upIncomeSection []*model.DateStatis, table string) (rows int64, err error) {
+	var buf bytes.Buffer
+	for _, row := range upIncomeSection {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.Count, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MoneySection, 10))
+		buf.WriteByte(',')
+		buf.WriteString(row.MoneyTips)
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Income, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.CDate.Time().Format(_layout) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals := buf.String()
+	buf.Reset()
+	rows, err = s.dao.InsertUpIncomeDailyStatis(c, table, vals)
+	return
+}

app/job/main/growup/service/income/lottery.go

@@ -0,0 +1,25 @@
+package income
+
+import (
+	"context"
+)
+
+// GetBubbleMeta .
+func (s *Service) GetBubbleMeta(c context.Context) (data map[int64][]int, err error) {
+	var id int64
+	data = make(map[int64][]int)
+	for {
+		var meta map[int64][]int
+		meta, id, err = s.dao.GetBubbleMeta(c, id, int64(_limitSize))
+		if err != nil {
+			return
+		}
+		if len(meta) == 0 {
+			break
+		}
+		for avID, bTypes := range meta {
+			data[avID] = append(data[avID], bTypes...)
+		}
+	}
+	return
+}

app/job/main/growup/service/income/run.go

@@ -0,0 +1,538 @@
+package income
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+	task "go-common/app/job/main/growup/service"
+	"go-common/library/log"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// RunAndSendMail run and send email
+func (s *Service) RunAndSendMail(c context.Context, date time.Time) (err error) {
+	var mailReceivers []string
+	var msg string
+	for _, v := range s.conf.Mail.Send {
+		if v.Type == 3 {
+			mailReceivers = v.Addr
+		}
+	}
+	startTime := time.Now().Unix()
+	err = s.run(c, date)
+	if err != nil {
+		msg = err.Error()
+		mailReceivers = []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"}
+	} else {
+		msg = fmt.Sprintf("%s 计算完成，耗时%ds", date.Format("2006-01-02"), time.Now().Unix()-startTime)
+	}
+	emailErr := s.email.SendMail(date, msg, "创作激励每日计算%d年%d月%d日", mailReceivers...)
+	if emailErr != nil {
+		log.Error("s.email.SendMail error(%v)", emailErr)
+	}
+	return
+}
+
+func (s *Service) run(c context.Context, date time.Time) (err error) {
+	defer func() {
+		task.GetTaskService().SetTaskStatus(c, task.TaskCreativeIncome, date.Format(_layout), err)
+	}()
+
+	err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskAvCharge, task.TaskCmCharge, task.TaskTagRatio, task.TaskBubbleMeta, task.TaskBlacklist, task.TaskBgmSync)
+	if err != nil {
+		return
+	}
+
+	startWeeklyDate = getStartWeeklyDate(date)
+	startMonthlyDate = getStartMonthlyDate(date)
+
+	/*################ Serializable Begin  ################*/
+
+	// av charge ratio
+	ratios, err := s.ratio.ArchiveChargeRatio(c, int64(_limitSize))
+	if err != nil {
+		return
+	}
+	// up charge ratio
+	urs, err := s.ratio.UpChargeRatio(c, int64(_limitSize))
+	if err != nil {
+		return
+	}
+	// av income statistics
+	astat, err := s.income.avIncomeStatSvr.AvIncomeStat(c, int64(_limitSize))
+	if err != nil {
+		log.Error("s.income.avIncomeStatSvr.AvIncomeStat error(%v) ", err)
+		return
+	}
+	log.Info("get av_income_statis : %d", len(astat))
+
+	// bgm income statistics
+	bstat, err := s.income.bgmIncomeStatSvr.BgmIncomeStat(c, int64(_limitSize))
+	if err != nil {
+		log.Error("s.income.bgmIncomeStatSvr.BgmIncomeStat error(%v) ", err)
+		return
+	}
+	log.Info("get bgm_income_statis : %d", len(bstat))
+
+	// column income statistics
+	cstat, err := s.income.columnIncomeStatSvr.ColumnIncomeStat(c, int64(_limitSize))
+	if err != nil {
+		log.Error("s.income.columnIncomeStatSvr.ColumnIncomeStat error(%v) ", err)
+		return
+	}
+	log.Info("get column_income_statis : %d", len(cstat))
+
+	// up income statistics
+	ustat, err := s.income.upIncomeStatSvr.UpIncomeStat(c, int64(_limitSize))
+	if err != nil {
+		log.Error("s.income.upIncomeStatSvr.UpIncomeStat error(%v) ", err)
+		return
+	}
+	log.Info("get up_income_statis : %d", len(ustat))
+
+	// up accounts
+	accs, err := s.income.upAccountSvr.UpAccount(c, int64(_limitSize))
+	if err != nil {
+		log.Error("s.income.upAccountSvr.UpAccount error(%v)", err)
+		return
+	}
+	log.Info("get up_account : %d", len(accs))
+
+	// bubble meta
+	bubbleMeta, err := s.GetBubbleMeta(c)
+	if err != nil {
+		log.Error("s.GetBubbleMeta error(%v)", err)
+		return
+	}
+	log.Info("get lottery_av_info avids: %d", len(bubbleMeta))
+	bubbleRatio := s.avToBubbleRatio(bubbleMeta)
+
+	// av signed ups
+	var (
+		avFilters     []AvFilter
+		columnFilters []ColumnFilter
+		bgmFilter     BgmFilter
+	)
+
+	//black list
+	blacks, err := s.Blacklist(c, 2000)
+	if err != nil {
+		return
+	}
+	// black ctype 0: av
+	avBlackFilter := avFilter(blacks[0])
+
+	// black ctype 2: column
+	columnBlackFilter := columnFilter(blacks[2])
+
+	// business orders
+	bos, err := s.GetBusinessOrders(c, 2000)
+	if err != nil {
+		return
+	}
+	bosFilter := avFilter(bos)
+
+	// signed up
+	signed := make(map[int64]bool)
+
+	signedAv, err := s.Signed(c, "video", 2000)
+	if err != nil {
+		return
+	}
+	savf := signedAvFilter(signedAv, date)
+	for mid := range signedAv {
+		signed[mid] = true
+	}
+
+	signedColumn, err := s.Signed(c, "column", 2000)
+	if err != nil {
+		return
+	}
+	for mid := range signedColumn {
+		signed[mid] = true
+	}
+
+	signedBgm, err := s.Signed(c, "bgm", 2000)
+	if err != nil {
+		return
+	}
+	for mid := range signedBgm {
+		signed[mid] = true
+	}
+
+	bgms, err := s.BGMs(c, 2000)
+	if err != nil {
+		return
+	}
+
+	{
+		avFilters = append(avFilters, avBlackFilter)
+		avFilters = append(avFilters, bosFilter)
+		avFilters = append(avFilters, savf)
+
+		bgmFilter = signedBgmFilter(signedBgm, date)
+
+		columnFilters = append(columnFilters, signedColumnFilter(signedColumn, date))
+		columnFilters = append(columnFilters, columnBlackFilter)
+	}
+
+	/*################ Serializable End  ##################*/
+
+	var (
+		readGroup errgroup.Group
+		sourceCh  = make(chan []*model.AvCharge, 1000)
+		// av
+		incomeCh = make(chan []*model.AvCharge, 1000)
+		// bgm
+		bgmCh = make(chan []*model.AvCharge, 1000)
+		// column
+		columnSourceCh = make(chan []*model.ColumnCharge, 1000)
+		// business income
+		businessCh = make(chan map[int64]*model.UpBusinessIncome, 10)
+	)
+
+	// get av daily charge and repost to other channels
+	readGroup.Go(func() (err error) {
+		err = s.avCharge.AvCharges(c, date, sourceCh, bubbleRatio)
+		if err != nil {
+			log.Error("s.avCharge.AvCharges error(%v)", err)
+			return
+		}
+		log.Info("av_daily_charge finished")
+		return
+	})
+
+	// get column daily charge
+	readGroup.Go(func() (err error) {
+		err = s.columnCharges(c, date, columnSourceCh)
+		if err != nil {
+			log.Error("s.columnCharges error(%v)", err)
+			return
+		}
+		log.Info("column_daily_charge finished")
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(incomeCh)
+			close(bgmCh)
+		}()
+
+		for charges := range sourceCh {
+			incomeCh <- charges
+			bgmCh <- charges
+		}
+		return
+	})
+
+	// up and av income compute
+	var (
+		um map[int64]*model.UpIncome
+		am map[int64][]*model.AvIncome
+		bm map[int64]map[int64]map[int64]*model.BgmIncome
+		cm map[int64][]*model.ColumnIncome
+	)
+
+	readGroup.Go(func() (err error) {
+		//um, am = s.income.Compute(c, date, incomeCh, urs, ars, ustat, astat, accs, filters, signed)
+		var business map[int64]*model.UpBusinessIncome
+		am, business = s.income.CalAvIncome(incomeCh, urs[1], ratios[1], avFilters, signed)
+		businessCh <- business
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		var business map[int64]*model.UpBusinessIncome
+		bm, business = s.income.CalBgmIncome(bgmCh, bgms, urs[3], ratios[3], avFilters, bgmFilter, blacks[3], signed)
+		businessCh <- business
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		var business map[int64]*model.UpBusinessIncome
+		cm, business = s.income.CalColumnIncome(columnSourceCh, urs[2], ratios[2], columnFilters, signed)
+		businessCh <- business
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		um = s.income.CalUpIncome(businessCh, date)
+		s.income.IncomeStat(um, am, bm, cm, ustat, astat, bstat, cstat)
+		s.income.PurgeUpAccount(date, accs, um)
+		return
+	})
+
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	// security verification
+	{
+		if len(am) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_income")
+			return
+		}
+		if len(bm) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_income")
+			return
+		}
+		if len(cm) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_income")
+			return
+		}
+		if len(um) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_income")
+			return
+		}
+		if len(astat) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_income_statis")
+			return
+		}
+		if len(bstat) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_income_statis")
+			return
+		}
+		if len(cstat) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_income_statis")
+			return
+		}
+		if len(ustat) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_income_statis")
+			return
+		}
+		if len(accs) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_account")
+			return
+		}
+	}
+
+	// persistent
+	var writeGroup errgroup.Group
+
+	// av_income
+	writeGroup.Go(func() (err error) {
+		err = s.income.avIncomeSvr.BatchInsertAvIncome(c, am)
+		if err != nil {
+			log.Error("s.income.BatchInsertAvIncome error(%v)", err)
+			return
+		}
+		log.Info("insert av_income : %d", len(am))
+		return
+	})
+
+	// column_income
+	writeGroup.Go(func() (err error) {
+		err = s.income.columnIncomeSvr.BatchInsertColumnIncome(c, cm)
+		if err != nil {
+			log.Error("s.income.BatchInsertColumnIncome error(%v)", err)
+			return
+		}
+		log.Info("insert column_income : %d", len(cm))
+		return
+	})
+
+	// bgm_income
+	writeGroup.Go(func() (err error) {
+		err = s.income.bgmIncomeSvr.BatchInsertBgmIncome(c, bm)
+		if err != nil {
+			log.Error("s.income.BatchInsertBgmIncome error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_income : %d", len(bm))
+		return
+	})
+
+	// up income
+	writeGroup.Go(func() (err error) {
+		err = s.income.upIncomeSvr.BatchInsertUpIncome(c, um)
+		if err != nil {
+			log.Error("s.income.BatchInsertUpIncome error(%v)", err)
+			return
+		}
+		log.Info("insert up_income : %d", len(um))
+		return
+	})
+
+	// av_income_statis
+	writeGroup.Go(func() (err error) {
+		err = s.income.avIncomeStatSvr.BatchInsertAvIncomeStat(c, astat)
+		if err != nil {
+			log.Error("s.income.BatchInsertAvIncomeStat error(%v)", err)
+			return
+		}
+		log.Info("insert av_income_statis : %d", len(astat))
+		return
+	})
+
+	// column_income_statis
+	writeGroup.Go(func() (err error) {
+		err = s.income.columnIncomeStatSvr.BatchInsertColumnIncomeStat(c, cstat)
+		if err != nil {
+			log.Error("s.income.BatchInsertColumnIncomeStat error(%v)", err)
+			return
+		}
+		log.Info("insert column_income_statis : %d", len(cstat))
+		return
+	})
+
+	// bgm_income_statis
+	writeGroup.Go(func() (err error) {
+		err = s.income.bgmIncomeStatSvr.BatchInsertBgmIncomeStat(c, bstat)
+		if err != nil {
+			log.Error("s.income.BatchInsertBgmIncomeStat error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_income_statis : %d", len(bstat))
+		return
+	})
+
+	// up_income_statis
+	writeGroup.Go(func() (err error) {
+		err = s.income.upIncomeStatSvr.BatchInsertUpIncomeStat(c, ustat)
+		if err != nil {
+			log.Error("s.income.BatchInsertUpIncomeStat error(%v)", err)
+			return
+		}
+		log.Info("insert up_income_statis : %d", len(ustat))
+		return
+	})
+
+	// up_account batch insert
+	writeGroup.Go(func() (err error) {
+		err = s.income.upAccountSvr.BatchInsertUpAccount(c, accs)
+		if err != nil {
+			log.Error("s.income.BatchInsertUpAccount error(%v)", err)
+			return
+		}
+		log.Info("insert up_account : %d", len(accs))
+		return
+	})
+
+	// up account single update
+	writeGroup.Go(func() (err error) {
+		err = s.income.upAccountSvr.UpdateUpAccount(c, accs)
+		if err != nil {
+			log.Error("s.income.UpdateUpAccount error(%v)", err)
+			return
+		}
+		log.Info("update up_account : %d", len(accs))
+		return
+	})
+
+	if err = writeGroup.Wait(); err != nil {
+		log.Error("run writeGroup.Wait error(%v)", err)
+	}
+	return
+}
+
+func signedBgmFilter(m map[int64]*model.Signed, date time.Time) BgmFilter {
+	return func(charge *model.AvCharge, bgm *model.BGM) bool {
+		if up, ok := m[bgm.MID]; ok {
+			if charge.Date.Time().Before(up.SignedAt.Time()) {
+				return true
+			}
+			if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
+				return true
+			}
+		} else {
+			return true
+		}
+		return false
+	}
+}
+
+func signedAvFilter(m map[int64]*model.Signed, date time.Time) AvFilter {
+	return func(charge *model.AvCharge) bool {
+		if up, ok := m[charge.MID]; ok {
+			if charge.UploadTime.Time().Before(up.SignedAt.Time()) {
+				return true
+			}
+			if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
+				return true
+			}
+		} else {
+			return true
+		}
+		return false
+	}
+}
+
+func signedColumnFilter(m map[int64]*model.Signed, date time.Time) ColumnFilter {
+	return func(charge *model.ColumnCharge) bool {
+		if up, ok := m[charge.MID]; ok {
+			if charge.UploadTime.Time().Before(up.SignedAt.Time()) {
+				return true
+			}
+			if (up.AccountState == 5 || up.AccountState == 6) && up.QuitAt.Time().Before(date.AddDate(0, 0, 1)) {
+				return true
+			}
+		} else {
+			return true
+		}
+		return false
+	}
+}
+
+func avFilter(m map[int64]bool) AvFilter {
+	return func(charge *model.AvCharge) bool {
+		return m[charge.AvID]
+	}
+}
+
+func columnFilter(m map[int64]bool) ColumnFilter {
+	return func(charge *model.ColumnCharge) bool {
+		return m[charge.ArticleID]
+	}
+}
+
+func (s *Service) avToBubbleRatio(bubbleMeta map[int64][]int) map[int64]float64 {
+	var (
+		res         = make(map[int64]float64)
+		typeToRatio = make(map[int]float64)
+		chooseBType = func(bTypes []int) (bType int) {
+			if len(bTypes) == 1 {
+				return bTypes[0]
+			}
+			sort.Slice(bTypes, func(i, j int) bool {
+				bti, btj := bTypes[i], bTypes[j]
+				if typeToRatio[bti] == typeToRatio[btj] {
+					return bti < btj
+				}
+				return typeToRatio[bti] < typeToRatio[btj]
+			})
+			return bTypes[0]
+		}
+	)
+
+	for _, v := range s.conf.Bubble.BRatio {
+		typeToRatio[v.BType] = v.Ratio
+	}
+	for avID, bTypes := range bubbleMeta {
+		bType := chooseBType(bTypes)
+		res[avID] = typeToRatio[bType]
+	}
+	return res
+}
+
+// BgmFilter av charge filter
+type BgmFilter func(*model.AvCharge, *model.BGM) bool
+
+// AvFilter av charge filter
+type AvFilter func(*model.AvCharge) bool
+
+// ColumnFilter column charge filter
+type ColumnFilter func(*model.ColumnCharge) bool
+
+// ChargeRegulator regulates av charge
+type ChargeRegulator func(*model.AvCharge)
+
+// UpdateBusinessIncome ..
+func (s *Service) UpdateBusinessIncome(c context.Context, date string) (err error) {
+	return s.income.UpdateBusinessIncomeByDate(c, date)
+}

app/job/main/growup/service/income/run_statis.go

@@ -0,0 +1,548 @@
+package income
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+	task "go-common/app/job/main/growup/service"
+
+	"go-common/library/log"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// RunStatis run income statistics
+func (s *Service) RunStatis(c context.Context, date time.Time) (err error) {
+	var msg string
+	mailReceivers := []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"}
+	startTime := time.Now().Unix()
+	err = s.runStatis(c, date)
+	if err != nil {
+		msg = err.Error()
+	} else {
+		msg = fmt.Sprintf("%s 计算完成，耗时%ds", date.Format("2006-01-02"), time.Now().Unix()-startTime)
+	}
+	err = s.email.SendMail(date, msg, "创作激励每日统计%d年%d月%d日", mailReceivers...)
+	if err != nil {
+		log.Error("s.email.SendMail error(%v)", err)
+	}
+	return
+}
+
+func (s *Service) runStatis(c context.Context, date time.Time) (err error) {
+	defer func() {
+		task.GetTaskService().SetTaskStatus(c, task.TaskCreativeStatis, date.Format(_layout), err)
+	}()
+	// task status type
+	err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskCreativeIncome)
+	if err != nil {
+		return
+	}
+
+	startWeeklyDate = getStartWeeklyDate(date)
+	startMonthlyDate = getStartMonthlyDate(date)
+
+	var (
+		readGroup   errgroup.Group
+		avSections  = SectionEntries{}
+		cmSections  = SectionEntries{}
+		bgmSections = SectionEntries{}
+
+		upSection    = make([]*model.DateStatis, 0)
+		upAvSection  = make([]*model.DateStatis, 0)
+		upCmSection  = make([]*model.DateStatis, 0)
+		upBgmSection = make([]*model.DateStatis, 0)
+
+		upIncomeWeekly  = make(map[int64]*model.UpIncome)
+		upIncomeMonthly = make(map[int64]*model.UpIncome)
+
+		upAvStatisCh  = make(chan map[int64]*model.UpArchStatis, 1)
+		upCmStatisCh  = make(chan map[int64]*model.UpArchStatis, 1)
+		upBgmStatisCh = make(chan map[int64]*model.UpArchStatis, 1)
+
+		avSourceCh  = make(chan []*model.ArchiveIncome, 1000)
+		avDailyCh   = make(chan []*model.ArchiveIncome, 1000)
+		avWeeklyCh  = make(chan []*model.ArchiveIncome, 1000)
+		avMonthlyCh = make(chan []*model.ArchiveIncome, 1000)
+		upAvCh      = make(chan []*model.ArchiveIncome, 1000)
+
+		cmSourceCh  = make(chan []*model.ArchiveIncome, 1000)
+		cmDailyCh   = make(chan []*model.ArchiveIncome, 1000)
+		cmWeeklyCh  = make(chan []*model.ArchiveIncome, 1000)
+		cmMonthlyCh = make(chan []*model.ArchiveIncome, 1000)
+		upCmCh      = make(chan []*model.ArchiveIncome, 1000)
+
+		bgmSourceCh  = make(chan []*model.ArchiveIncome, 1000)
+		bgmDailyCh   = make(chan []*model.ArchiveIncome, 1000)
+		bgmWeeklyCh  = make(chan []*model.ArchiveIncome, 1000)
+		bgmMonthlyCh = make(chan []*model.ArchiveIncome, 1000)
+		upBgmCh      = make(chan []*model.ArchiveIncome, 1000)
+
+		upSourceCh = make(chan []*model.UpIncome, 1000)
+		upDailyCh  = make(chan []*model.UpIncome, 1000)
+		upStatisCh = make(chan []*model.UpIncome, 1000)
+	)
+
+	// get income by date to statistics
+	preDate := startMonthlyDate
+	if startWeeklyDate.Before(startMonthlyDate) {
+		preDate = startWeeklyDate
+	}
+	// get av_income
+	readGroup.Go(func() (err error) {
+		err = s.income.dateStatisSvr.getArchiveByDate(c, avSourceCh, preDate, date, _video, _limitSize)
+		if err != nil {
+			log.Error("s.getArchiveByDate error(%v)", err)
+		}
+		return
+	})
+
+	// get column_income
+	readGroup.Go(func() (err error) {
+		err = s.income.dateStatisSvr.getArchiveByDate(c, cmSourceCh, preDate, date, _column, _limitSize)
+		if err != nil {
+			log.Error("s.getArchiveByDate error(%v)", err)
+		}
+		return
+	})
+
+	// get bgm_income
+	readGroup.Go(func() (err error) {
+		err = s.income.dateStatisSvr.getArchiveByDate(c, bgmSourceCh, preDate, date, _bgm, _limitSize)
+		if err != nil {
+			log.Error("s.getArchiveByDate error(%v)", err)
+		}
+		return
+	})
+
+	// get up_income
+	readGroup.Go(func() (err error) {
+		err = s.income.upIncomeSvr.getUpIncomeByDate(c, upSourceCh, date.Format(_layout), _limitSize)
+		if err != nil {
+			log.Error("s.getUpIncomeByDate error(%v)", err)
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(avDailyCh)
+			close(avWeeklyCh)
+			close(avMonthlyCh)
+			close(upAvCh)
+		}()
+		for av := range avSourceCh {
+			avDailyCh <- av
+			avWeeklyCh <- av
+			avMonthlyCh <- av
+			upAvCh <- av
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(cmDailyCh)
+			close(cmWeeklyCh)
+			close(cmMonthlyCh)
+			close(upCmCh)
+		}()
+		for cm := range cmSourceCh {
+			cmDailyCh <- cm
+			cmWeeklyCh <- cm
+			cmMonthlyCh <- cm
+			upCmCh <- cm
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(bgmDailyCh)
+			close(bgmWeeklyCh)
+			close(bgmMonthlyCh)
+			close(upBgmCh)
+		}()
+		for bgm := range bgmSourceCh {
+			bgmDailyCh <- bgm
+			bgmWeeklyCh <- bgm
+			bgmMonthlyCh <- bgm
+			upBgmCh <- bgm
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(upStatisCh)
+			close(upDailyCh)
+		}()
+		for up := range upSourceCh {
+			upStatisCh <- up
+			upDailyCh <- up
+		}
+		return
+	})
+
+	// up
+	readGroup.Go(func() (err error) {
+		upSection, upAvSection, upCmSection, upBgmSection, err = s.income.dateStatisSvr.handleDateUp(c, upStatisCh, date)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.HandleUp error(%v)", err)
+		}
+		return
+	})
+
+	// video
+	readGroup.Go(func() (err error) {
+		avSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, avDailyCh, date, _avIncomeDailyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		avSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, avWeeklyCh, startWeeklyDate, _avIncomeWeeklyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		avSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, avMonthlyCh, startMonthlyDate, _avIncomeMonthlyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	// column
+	readGroup.Go(func() (err error) {
+		cmSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, cmDailyCh, date, _cmIncomeDailyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		cmSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, cmWeeklyCh, startWeeklyDate, _cmIncomeWeeklyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		cmSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, cmMonthlyCh, startMonthlyDate, _cmIncomeMonthlyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	// bgm
+	readGroup.Go(func() (err error) {
+		bgmSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, bgmDailyCh, date, _bgmIncomeDailyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		bgmSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, bgmWeeklyCh, startWeeklyDate, _bgmIncomeWeeklyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		bgmSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, bgmMonthlyCh, startMonthlyDate, _bgmIncomeMonthlyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err)
+		}
+		return
+	})
+
+	// up_av_statis
+	readGroup.Go(func() (err error) {
+		err = s.income.upIncomeSvr.handleUpArchStatis(c, upAvStatisCh, upAvCh)
+		if err != nil {
+			log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err)
+		}
+		return
+	})
+
+	// up_column_statis
+	readGroup.Go(func() (err error) {
+		err = s.income.upIncomeSvr.handleUpArchStatis(c, upCmStatisCh, upCmCh)
+		if err != nil {
+			log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err)
+		}
+		return
+	})
+
+	// up_bgm_statis
+	readGroup.Go(func() (err error) {
+		err = s.income.upIncomeSvr.handleUpArchStatis(c, upBgmStatisCh, upBgmCh)
+		if err != nil {
+			log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err)
+		}
+		return
+	})
+
+	// up_income_weekly up_income_monthly
+	readGroup.Go(func() (err error) {
+		upIncomeWeekly, upIncomeMonthly, err = s.income.upIncomeSvr.handleUpIncomeWeeklyAndMonthly(c, date, upAvStatisCh, upCmStatisCh, upBgmStatisCh, upDailyCh)
+		if err != nil {
+			log.Error("p.upIncomeSvr.handleUpIncomeWeeklyAndMonthly error(%v)", err)
+		}
+		return
+	})
+
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	{
+		if len(avSections.avDaily) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_income_daily_statis")
+			return
+		}
+		if len(avSections.avWeekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_income_weekly_statis")
+			return
+		}
+		if len(avSections.avMonthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_income_monthly_statis")
+			return
+		}
+		if len(cmSections.avDaily) == 0 {
+			err = fmt.Errorf("Error: insert 0 cm_income_daily_statis")
+			return
+		}
+		if len(cmSections.avWeekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 cm_income_weekly_statis")
+			return
+		}
+		if len(cmSections.avMonthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 cm_income_monthly_statis")
+			return
+		}
+		if len(bgmSections.avDaily) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_income_daily_statis")
+			return
+		}
+		if len(bgmSections.avWeekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_income_weekly_statis")
+			return
+		}
+		if len(bgmSections.avMonthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_income_monthly_statis")
+			return
+		}
+		if len(upSection) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_income_daily_statis")
+			return
+		}
+		if len(upAvSection) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_av_daily_statis")
+			return
+		}
+		if len(upCmSection) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_column_daily_statis")
+			return
+		}
+		if len(upBgmSection) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_bgm_daily_statis")
+			return
+		}
+		if len(upIncomeWeekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_income_weekly")
+			return
+		}
+		if len(upIncomeMonthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_income_monthly")
+			return
+		}
+	}
+
+	// persistent
+	var writeGroup errgroup.Group
+	// av_income_daily_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avDaily, _avIncomeDailyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err)
+			return
+		}
+		log.Info("insert av_income_daily_statis : %d", len(avSections.avDaily))
+		return
+	})
+
+	// av_income_weekly_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avWeekly, _avIncomeWeeklyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err)
+			return
+		}
+		log.Info("insert av_income_weekly_statis : %d", len(avSections.avWeekly))
+		return
+	})
+
+	// av_income_monthly_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avMonthly, _avIncomeMonthlyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err)
+			return
+		}
+		log.Info("insert av_income_monthly_statis : %d", len(avSections.avMonthly))
+		return
+	})
+
+	// column_income_daily_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avDaily, _cmIncomeDailyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err)
+			return
+		}
+		log.Info("insert column_income_daily_statis : %d", len(cmSections.avDaily))
+		return
+	})
+
+	// column_income_weekly_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avWeekly, _cmIncomeWeeklyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err)
+			return
+		}
+		log.Info("insert column_income_weekly_statis : %d", len(cmSections.avWeekly))
+		return
+	})
+
+	// column_income_monthly_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avMonthly, _cmIncomeMonthlyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err)
+			return
+		}
+		log.Info("insert column_income_monthly_statis : %d", len(cmSections.avMonthly))
+		return
+	})
+
+	// bgm_income_daily_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avDaily, _bgmIncomeDailyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_income_daily_statis : %d", len(bgmSections.avDaily))
+		return
+	})
+
+	// bgm_income_weekly_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avWeekly, _bgmIncomeWeeklyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_income_weekly_statis : %d", len(bgmSections.avWeekly))
+		return
+	})
+
+	// bgm_income_monthly_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avMonthly, _bgmIncomeMonthlyStatis)
+		if err != nil {
+			log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_income_monthly_statis : %d", len(bgmSections.avMonthly))
+		return
+	})
+
+	// up_income_daily_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upSection, "up_income_daily_statis")
+		if err != nil {
+			log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert up_income_daily_statis : %d", len(upSection))
+		return
+	})
+
+	// up_av_daily_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upAvSection, "up_av_daily_statis")
+		if err != nil {
+			log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert up_av_daily_statis : %d", len(upAvSection))
+		return
+	})
+
+	// up_column_daily_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upCmSection, "up_column_daily_statis")
+		if err != nil {
+			log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert up_column_daily_statis : %d", len(upCmSection))
+		return
+	})
+
+	// up_bgm_daily_statis
+	writeGroup.Go(func() (err error) {
+		_, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upBgmSection, "up_bgm_daily_statis")
+		if err != nil {
+			log.Error("s.upIncomeDailyStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert up_bgm_daily_statis : %d", len(upBgmSection))
+		return
+	})
+
+	// up_income_weekly
+	writeGroup.Go(func() (err error) {
+		err = s.income.upIncomeSvr.UpIncomeDBStoreBatch(c, _upIncomeWeekly, upIncomeWeekly)
+		if err != nil {
+			log.Error("s.UpIncomeDBStoreBatch up_income_weekly error(%v)", err)
+			return
+		}
+		log.Info("insert up_income_weekly : %d", len(upIncomeWeekly))
+		return
+	})
+
+	// up_income_monthly
+	writeGroup.Go(func() (err error) {
+		err = s.income.upIncomeSvr.UpIncomeDBStoreBatch(c, _upIncomeMonthly, upIncomeMonthly)
+		if err != nil {
+			log.Error("s.UpIncomeDBStoreBatch up_income_monthly error(%v)", err)
+			return
+		}
+		log.Info("insert up_income_monthly : %d", len(upIncomeMonthly))
+		return
+	})
+
+	if err = writeGroup.Wait(); err != nil {
+		log.Error("run writeGroup.Wait error(%v)", err)
+	}
+	return
+}

app/job/main/growup/service/income/service.go

@@ -0,0 +1,93 @@
+package income
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/growup/conf"
+	"go-common/app/job/main/growup/dao/dataplatform"
+	"go-common/app/job/main/growup/dao/email"
+	incomeD "go-common/app/job/main/growup/dao/income"
+	"go-common/app/job/main/growup/service"
+	"go-common/app/job/main/growup/service/ctrl"
+	"go-common/library/log"
+)
+
+// Service struct
+type Service struct {
+	conf     *conf.Config
+	dao      *incomeD.Dao
+	avCharge *AvChargeSvr
+	income   *Income
+	ratio    *ChargeRatioSvr
+	email    *email.Dao
+	dp       *dataplatform.Dao
+}
+
+// New fn
+func New(c *conf.Config, executor ctrl.Executor) (s *Service) {
+	s = &Service{
+		conf:  c,
+		dao:   incomeD.New(c),
+		email: email.New(c),
+		dp:    dataplatform.New(c),
+	}
+	s.avCharge = NewAvChargeSvr(s.dao)
+	s.income = NewIncome(batchSize, s.dao)
+	s.ratio = NewChargeRatioSvr(s.dao)
+	log.Info("income service start")
+
+	executor.Submit(
+		s.calDailyCreativeIncome,
+		s.syncBGM,
+	)
+	return s
+}
+
+func (s *Service) syncBGM(ctx context.Context) {
+	for {
+		time.Sleep(service.NextDay(1, 0, 0))
+		msg := ""
+		log.Info("sync BGM begin:%v", time.Now().Format("2006-01-02 15:04:05"))
+		err := s.SyncBgmInfo(context.TODO())
+		if err != nil {
+			log.Error("s.SyncBgmInfo error(%v)", err)
+		}
+		if err != nil {
+			msg = fmt.Sprintf("SyncBgmInfo error(%v)", err)
+		} else {
+			msg = "Success"
+		}
+		err = s.email.SendMail(time.Now(), msg, "创作激励同步bgm%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
+		if err != nil {
+			log.Error("s.email.SendMail error(%v)", err)
+		}
+		log.Info("sync BGM end:%v", time.Now().Format("2006-01-02 15:04:05"))
+	}
+}
+
+func (s *Service) calDailyCreativeIncome(ctx context.Context) {
+	for {
+		time.Sleep(service.NextDay(18, 0, 0))
+		log.Info("calDailyCreativeIncome begin:%v", time.Now().Format("2006-01-02 15:04:05"))
+		now := time.Now()
+		date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local)
+		err := s.RunAndSendMail(context.TODO(), date)
+		if err != nil {
+			log.Error("s.RunAndSendMail error(%v)", err)
+		}
+		if err == nil {
+			err = s.RunStatis(context.TODO(), date)
+			if err != nil {
+				log.Error("s.RunStatis error(%v)", err)
+			}
+		}
+		log.Info("calDailyCreativeIncome end:%v", time.Now().Format("2006-01-02 15:04:05"))
+	}
+}
+
+// Ping check dao health.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}

app/job/main/growup/service/income/service_test.go

@@ -0,0 +1,35 @@
+package income
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/growup/conf"
+	"go-common/app/job/main/growup/service/ctrl"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s      *Service
+	charge *AvChargeSvr
+)
+
+func init() {
+	dir, _ := filepath.Abs("../../cmd/growup-job.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	if s == nil {
+		s = New(conf.Conf, ctrl.NewUnboundedExecutor())
+	}
+	charge = s.avCharge
+}
+
+func TestPing(t *testing.T) {
+	Convey("Test_Ping", t, func() {
+		err := s.Ping(context.Background())
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/growup/service/income/tax.go

@@ -0,0 +1,118 @@
+package income
+
+const (
+	_range0_30      float64 = 0.0
+	_range30_50     float64 = 0.05
+	_range50_100    float64 = 0.10
+	_range100_150   float64 = 0.15
+	_range150_300   float64 = 0.20
+	_range300_500   float64 = 0.25
+	_range500_1000  float64 = 0.30
+	_range1000_2000 float64 = 0.40
+	_range2000_3000 float64 = 0.50
+	_range3000max   float64 = 0.60
+)
+
+// TaxRate tax rate
+type TaxRate struct {
+	start float64
+	end   float64
+	rate  float64
+}
+
+// Tax tax
+func Tax(income float64) (tax float64) {
+	rs := rates(income)
+	for _, r := range rs {
+		if income >= r.end {
+			tax += Mul(float64(r.end-r.start), r.rate)
+		} else {
+			tax += Mul(float64(income-r.start), r.rate)
+		}
+	}
+	return
+}
+
+func rates(income float64) (rs []*TaxRate) {
+	if income > 0 {
+		r := &TaxRate{
+			start: 0,
+			end:   30,
+			rate:  _range0_30,
+		}
+		rs = append(rs, r)
+	}
+	if income > 30 {
+		r := &TaxRate{
+			start: 30,
+			end:   50,
+			rate:  _range30_50,
+		}
+		rs = append(rs, r)
+	}
+	if income > 50 {
+		r := &TaxRate{
+			start: 50,
+			end:   100,
+			rate:  _range50_100,
+		}
+		rs = append(rs, r)
+	}
+	if income > 100 {
+		r := &TaxRate{
+			start: 100,
+			end:   150,
+			rate:  _range100_150,
+		}
+		rs = append(rs, r)
+	}
+	if income > 150 {
+		r := &TaxRate{
+			start: 150,
+			end:   300,
+			rate:  _range150_300,
+		}
+		rs = append(rs, r)
+	}
+	if income > 300 {
+		r := &TaxRate{
+			start: 300,
+			end:   500,
+			rate:  _range300_500,
+		}
+		rs = append(rs, r)
+	}
+	if income > 500 {
+		r := &TaxRate{
+			start: 500,
+			end:   1000,
+			rate:  _range500_1000,
+		}
+		rs = append(rs, r)
+	}
+	if income > 1000 {
+		r := &TaxRate{
+			start: 1000,
+			end:   2000,
+			rate:  _range1000_2000,
+		}
+		rs = append(rs, r)
+	}
+	if income > 2000 {
+		r := &TaxRate{
+			start: 2000,
+			end:   3000,
+			rate:  _range2000_3000,
+		}
+		rs = append(rs, r)
+	}
+	if income > 3000 {
+		r := &TaxRate{
+			start: 3000,
+			end:   1<<31 - 1,
+			rate:  _range3000max,
+		}
+		rs = append(rs, r)
+	}
+	return
+}

app/job/main/growup/service/income/tools.go

@@ -0,0 +1,56 @@
+package income
+
+import (
+	"math"
+	"math/big"
+)
+
+// Div div
+func Div(x, y float64) float64 {
+	a := big.NewFloat(x)
+	b := big.NewFloat(y)
+	c := new(big.Float).Quo(a, b)
+	d, _ := c.Float64()
+	return d
+}
+
+// Mul mul
+func Mul(x, y float64) float64 {
+	a := big.NewFloat(x)
+	b := big.NewFloat(y)
+	c := new(big.Float).Mul(a, b)
+	d, _ := c.Float64()
+	return d
+}
+
+// DivWithRound div with round
+func DivWithRound(x, y float64, places int) float64 {
+	a := big.NewFloat(x)
+	b := big.NewFloat(y)
+	c := new(big.Float).Quo(a, b)
+	d, _ := c.Float64()
+	return Round(d, places)
+}
+
+// MulWithRound mul with round
+func MulWithRound(x, y float64, places int) float64 {
+	a := big.NewFloat(x)
+	b := big.NewFloat(y)
+	c := new(big.Float).Mul(a, b)
+	d, _ := c.Float64()
+	return Round(d, places)
+}
+
+// Round round
+func Round(val float64, places int) float64 {
+	var round float64
+	pow := math.Pow(10, float64(places))
+	digit := pow * val
+	_, div := math.Modf(digit)
+	if div >= 0.5 {
+		round = math.Ceil(digit)
+	} else {
+		round = math.Floor(digit)
+	}
+	return round / pow
+}

app/job/main/growup/service/income/up_account.go

@@ -0,0 +1,137 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// UpAccountSvr up account service
+type UpAccountSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewUpAccountSvr new up account service
+func NewUpAccountSvr(dao *incomeD.Dao, batchSize int) (svr *UpAccountSvr) {
+	return &UpAccountSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// UpAccount get up account
+func (s *UpAccountSvr) UpAccount(c context.Context, limit int64) (m map[int64]*model.UpAccount, err error) {
+	var id int64
+	m = make(map[int64]*model.UpAccount)
+	for {
+		var um map[int64]*model.UpAccount
+		um, id, err = s.dao.UpAccounts(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(um) == 0 {
+			break
+		}
+		for mid, acc := range um {
+			m[mid] = acc
+		}
+	}
+	return
+}
+
+// BatchInsertUpAccount batch insert up account
+func (s *UpAccountSvr) BatchInsertUpAccount(c context.Context, us map[int64]*model.UpAccount) (err error) {
+	var (
+		buff    = make([]*model.UpAccount, s.batchSize)
+		buffEnd = 0
+	)
+	for _, u := range us {
+		if u.DataState != 1 {
+			continue
+		}
+		buff[buffEnd] = u
+		buffEnd++
+		if buffEnd >= s.batchSize {
+			values := upAccountValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = s.dao.InsertUpAccount(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := upAccountValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = s.dao.InsertUpAccount(c, values)
+	}
+	return
+}
+
+// UpdateUpAccount update up account
+func (s *UpAccountSvr) UpdateUpAccount(c context.Context, us map[int64]*model.UpAccount) (err error) {
+	for _, u := range us {
+		if u.DataState != 2 {
+			continue
+		}
+		var time int
+		for {
+			var rows int64
+			rows, err = s.dao.UpdateUpAccount(c, u.MID, u.Version, u.TotalIncome, u.TotalUnwithdrawIncome)
+			if err != nil {
+				return
+			}
+			time++
+			if rows > 0 {
+				break
+			}
+			if time >= 10 {
+				break
+			}
+			s.reload(c, u)
+		}
+	}
+	return
+}
+
+func (s *UpAccountSvr) reload(c context.Context, upAccount *model.UpAccount) (err error) {
+	result, err := s.dao.UpAccount(c, upAccount.MID)
+	if err != nil {
+		return
+	}
+	upAccount.TotalIncome = result.TotalIncome
+	upAccount.TotalUnwithdrawIncome = result.TotalUnwithdrawIncome
+	upAccount.Version = result.Version
+	upAccount.WithdrawDateVersion = result.WithdrawDateVersion
+	return
+}
+
+func upAccountValues(us []*model.UpAccount) (values string) {
+	var buf bytes.Buffer
+	for _, u := range us {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(u.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(u.HasSignContract))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.TotalUnwithdrawIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + u.WithdrawDateVersion + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.Version, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/up_account_test.go

@@ -0,0 +1,31 @@
+package income
+
+import (
+	"context"
+	"testing"
+
+	model "go-common/app/job/main/growup/model/income"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_UpAccount1(t *testing.T) {
+	Convey("UpAccount", t, func() {
+		_, err := s.income.upAccountSvr.UpAccount(context.Background(), 10)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_BatchInsertUpAccount(t *testing.T) {
+	Convey("BatchInsertUpAccount", t, func() {
+		err := s.income.upAccountSvr.BatchInsertUpAccount(context.Background(), map[int64]*model.UpAccount{})
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_UpdateUpAccount(t *testing.T) {
+	Convey("UpdateUpAccount", t, func() {
+		err := s.income.upAccountSvr.UpdateUpAccount(context.Background(), map[int64]*model.UpAccount{})
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/growup/service/income/up_archive_statis.go

@@ -0,0 +1,68 @@
+package income
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	model "go-common/app/job/main/growup/model/income"
+
+	xtime "go-common/library/time"
+)
+
+func (s *UpIncomeSvr) handleUpArchStatis(c context.Context, upArchStatisCh chan map[int64]*model.UpArchStatis, archiveCh chan []*model.ArchiveIncome) (err error) {
+	defer close(upArchStatisCh)
+	upArchMap := make(map[int64]*model.UpArchStatis)
+	for income := range archiveCh {
+		s.calUpArchStatis(income, upArchMap)
+	}
+	upArchStatisCh <- upArchMap
+	return
+}
+
+func (s *UpIncomeSvr) calUpArchStatis(incomes []*model.ArchiveIncome, upArch map[int64]*model.UpArchStatis) {
+	for _, income := range incomes {
+		if _, ok := upArch[income.MID]; !ok {
+			upArch[income.MID] = addUpArchStatis(income)
+		}
+		updateUpArchStatis(income.AID, income.Date, upArch[income.MID])
+	}
+}
+
+func addUpArchStatis(income *model.ArchiveIncome) *model.UpArchStatis {
+	return &model.UpArchStatis{
+		MID:         income.MID,
+		WeeklyDate:  xtime.Time(startWeeklyDate.Unix()),
+		WeeklyAIDs:  "",
+		MonthlyDate: xtime.Time(startMonthlyDate.Unix()),
+		MonthlyAIDs: "",
+	}
+}
+
+func updateUpArchStatis(aid int64, date xtime.Time, statis *model.UpArchStatis) {
+	idStr := strconv.FormatInt(aid, 10)
+	if date >= statis.WeeklyDate {
+		if statis.WeeklyAIDs == "" {
+			statis.WeeklyAIDs = idStr
+		} else if !isExist(idStr, statis.WeeklyAIDs) {
+			statis.WeeklyAIDs += "," + idStr
+		}
+	}
+	if date >= statis.MonthlyDate {
+		if statis.MonthlyAIDs == "" {
+			statis.MonthlyAIDs = idStr
+		} else if !isExist(idStr, statis.MonthlyAIDs) {
+			statis.MonthlyAIDs += "," + idStr
+		}
+	}
+}
+
+func isExist(id string, old string) bool {
+	oldSli := strings.Split(old, ",")
+	for _, str := range oldSli {
+		if id == str {
+			return true
+		}
+	}
+	return false
+}

app/job/main/growup/service/income/up_income.go

@@ -0,0 +1,277 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// UpIncomeSvr up income service
+type UpIncomeSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewUpIncomeSvr new income service
+func NewUpIncomeSvr(dao *incomeD.Dao, batchSize int) (svr *UpIncomeSvr) {
+	return &UpIncomeSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+func (p *UpIncomeSvr) getUpIncomeByDate(c context.Context, upCh chan []*model.UpIncome, date string, limit int) (err error) {
+	defer close(upCh)
+	var id int64
+	for {
+		var up []*model.UpIncome
+		up, err = p.dao.GetUpIncomeTable(c, "up_income", date, id, limit)
+		if err != nil {
+			return
+		}
+		upCh <- up
+		if len(up) < limit {
+			break
+		}
+		id = up[len(up)-1].ID
+	}
+	return
+}
+
+// BatchInsertUpIncome insert up_income batch
+func (p *UpIncomeSvr) BatchInsertUpIncome(c context.Context, us map[int64]*model.UpIncome) (err error) {
+	var (
+		buff    = make([]*model.UpIncome, p.batchSize)
+		buffEnd = 0
+	)
+
+	for _, u := range us {
+		buff[buffEnd] = u
+		buffEnd++
+
+		if buffEnd >= p.batchSize {
+			values := upIncomeValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = p.dao.InsertUpIncome(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := upIncomeValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertUpIncome(c, values)
+	}
+	return
+}
+
+func upIncomeValues(us []*model.UpIncome) (values string) {
+	var buf bytes.Buffer
+	for _, u := range us {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(u.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.PlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AudioIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.TaxMoney, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.Income, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvBaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvTax, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnBaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnTax, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmBaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmTax, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}
+
+///************************************************************ FOR HISTORY ****************************************************************/
+//
+//// UpdateBusinessIncomeByDate update business income by date
+//func (p *UpIncomeSvr) UpdateBusinessIncomeByDate(c context.Context, date string, ustat map[int64]*model.UpIncomeStat) (err error) {
+//	us, err := p.businessTotalIncome(c, ustat, date)
+//	if err != nil {
+//		return
+//	}
+//	err = p.batchUpdateUpIncome(c, us)
+//	if err != nil {
+//		return
+//	}
+//	return p.batchUpdateUpIncomeStat(c, ustat)
+//}
+//
+//// for history data m: map[mid]map[date]*model.UpIncome
+//func (p *UpIncomeSvr) businessTotalIncome(c context.Context, ustat map[int64]*model.UpIncomeStat, date string) (m []*model.UpIncome, err error) {
+//	var id int64
+//	for {
+//		var ups []*model.UpIncome
+//		ups, err = p.dao.GetUpIncomeTable(c, "up_income", date, id, 2000)
+//		if err != nil {
+//			return
+//		}
+//		for _, up := range ups {
+//			ut := ustat[up.MID]
+//
+//			ut.AvTotalIncome += up.AvIncome
+//			up.AvTotalIncome = ut.AvTotalIncome
+//
+//			ut.ColumnTotalIncome += up.ColumnIncome
+//			up.ColumnTotalIncome = ut.ColumnTotalIncome
+//
+//			ut.BgmTotalIncome += up.BgmIncome
+//			up.BgmTotalIncome = ut.BgmTotalIncome
+//			m = append(m, up)
+//		}
+//		if len(ups) < 2000 {
+//			break
+//		}
+//		id = ups[len(ups)-1].ID
+//	}
+//	return
+//}
+//
+//// BatchUpdateUpIncome insert up_income batch
+//func (p *UpIncomeSvr) batchUpdateUpIncome(c context.Context, us []*model.UpIncome) (err error) {
+//	var (
+//		buff    = make([]*model.UpIncome, p.batchSize)
+//		buffEnd = 0
+//	)
+//
+//	for _, u := range us {
+//		buff[buffEnd] = u
+//		buffEnd++
+//
+//		if buffEnd >= p.batchSize {
+//			values := businessValues(buff[:buffEnd])
+//			buffEnd = 0
+//			_, err = p.dao.FixInsertUpIncome(c, values)
+//			if err != nil {
+//				return
+//			}
+//		}
+//	}
+//	if buffEnd > 0 {
+//		values := businessValues(buff[:buffEnd])
+//		buffEnd = 0
+//		_, err = p.dao.FixInsertUpIncome(c, values)
+//	}
+//	return
+//}
+//
+//func businessValues(us []*model.UpIncome) (values string) {
+//	var buf bytes.Buffer
+//	for _, u := range us {
+//		buf.WriteString("(")
+//		buf.WriteString(strconv.FormatInt(u.MID, 10))
+//		buf.WriteByte(',')
+//		buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
+//		buf.WriteByte(',')
+//		buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
+//		buf.WriteByte(',')
+//		buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
+//		buf.WriteByte(',')
+//		buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
+//		buf.WriteString(")")
+//		buf.WriteByte(',')
+//	}
+//	if buf.Len() > 0 {
+//		buf.Truncate(buf.Len() - 1)
+//	}
+//	values = buf.String()
+//	buf.Reset()
+//	return
+//}
+//
+//// BatchUpdateUpIncome insert up_income batch
+//func (p *UpIncomeSvr) batchUpdateUpIncomeStat(c context.Context, us map[int64]*model.UpIncomeStat) (err error) {
+//	var (
+//		buff    = make([]*model.UpIncomeStat, p.batchSize)
+//		buffEnd = 0
+//	)
+//
+//	for _, u := range us {
+//		buff[buffEnd] = u
+//		buffEnd++
+//
+//		if buffEnd >= p.batchSize {
+//			values := businessStatValues(buff[:buffEnd])
+//			buffEnd = 0
+//			_, err = p.dao.FixInsertUpIncomeStat(c, values)
+//			if err != nil {
+//				return
+//			}
+//		}
+//	}
+//	if buffEnd > 0 {
+//		values := businessStatValues(buff[:buffEnd])
+//		buffEnd = 0
+//		_, err = p.dao.FixInsertUpIncomeStat(c, values)
+//	}
+//	return
+//}
+//
+//func businessStatValues(ustat []*model.UpIncomeStat) (values string) {
+//	var buf bytes.Buffer
+//	for _, u := range ustat {
+//		buf.WriteString("(")
+//		buf.WriteString(strconv.FormatInt(u.MID, 10))
+//		buf.WriteByte(',')
+//		buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
+//		buf.WriteByte(',')
+//		buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
+//		buf.WriteByte(',')
+//		buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
+//		buf.WriteString(")")
+//		buf.WriteByte(',')
+//	}
+//	if buf.Len() > 0 {
+//		buf.Truncate(buf.Len() - 1)
+//	}
+//	values = buf.String()
+//	buf.Reset()
+//	return
+//}

app/job/main/growup/service/income/up_income_date.go

@@ -0,0 +1,318 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+	"strings"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+)
+
+var (
+	_upIncomeWeekly  = "up_income_weekly"
+	_upIncomeMonthly = "up_income_monthly"
+)
+
+func (s *UpIncomeSvr) handleUpIncomeWeeklyAndMonthly(
+	c context.Context,
+	date time.Time,
+	upAvStatisCh chan map[int64]*model.UpArchStatis,
+	upCmStatisCh chan map[int64]*model.UpArchStatis,
+	upBgmStatisCh chan map[int64]*model.UpArchStatis,
+	upSliceCh chan []*model.UpIncome) (weeklyMap, monthlyMap map[int64]*model.UpIncome, err error) {
+	weeklyMap, monthlyMap, err = s.GetUpIncomeWeeklyAndMonthly(c, date)
+	if err != nil {
+		log.Error("s.GetUpIncomeWeeklyAndMonthly error(%v)", err)
+		return
+	}
+	upAvStatis := <-upAvStatisCh
+	upCmStatis := <-upCmStatisCh
+	upBgmStatis := <-upBgmStatisCh
+	s.calUpIncomeWeeklyAndMonthly(weeklyMap, monthlyMap, upAvStatis, upCmStatis, upBgmStatis, upSliceCh)
+	return
+}
+
+// GetUpIncomeWeeklyAndMonthly get up_income_weekly and up_income_monthly
+func (s *UpIncomeSvr) GetUpIncomeWeeklyAndMonthly(c context.Context, date time.Time) (weeklyMap map[int64]*model.UpIncome, monthlyMap map[int64]*model.UpIncome, err error) {
+	upIncomeWeekly, err := s.GetUpIncomeTable(c, startWeeklyDate, _upIncomeWeekly)
+	if err != nil {
+		log.Error("s.GetUpIncomeTable error(%v)", err)
+		return
+	}
+
+	upIncomeMonthly, err := s.GetUpIncomeTable(c, startMonthlyDate, _upIncomeMonthly)
+	if err != nil {
+		log.Error("s.GetUpIncomeTable error(%v)", err)
+		return
+	}
+
+	weeklyMap = make(map[int64]*model.UpIncome)
+	monthlyMap = make(map[int64]*model.UpIncome)
+	for _, weeklyIncome := range upIncomeWeekly {
+		weeklyMap[weeklyIncome.MID] = weeklyIncome
+	}
+
+	for _, monthlyIncome := range upIncomeMonthly {
+		monthlyMap[monthlyIncome.MID] = monthlyIncome
+	}
+
+	return
+}
+
+// GetUpIncomeTable get up income table
+func (s *UpIncomeSvr) GetUpIncomeTable(c context.Context, date time.Time, table string) (upIncomes []*model.UpIncome, err error) {
+	var id int64
+	for {
+		upIncome, err1 := s.dao.GetUpIncomeTable(c, table, date.Format(_layout), id, _limitSize)
+		if err1 != nil {
+			err = err1
+			return
+		}
+		upIncomes = append(upIncomes, upIncome...)
+		if len(upIncome) < _limitSize {
+			break
+		}
+		id = upIncome[len(upIncome)-1].ID
+	}
+	return
+}
+
+func (s *UpIncomeSvr) calUpIncomeWeeklyAndMonthly(weeklyMap, monthlyMap map[int64]*model.UpIncome,
+	upAvStatis, upCmStatis, upBgmStatis map[int64]*model.UpArchStatis, upSliceCh chan []*model.UpIncome) {
+	for upIncome := range upSliceCh {
+		s.calUpIncome(upIncome, weeklyMap, monthlyMap, upAvStatis, upCmStatis, upBgmStatis)
+	}
+}
+
+func (s *UpIncomeSvr) calUpIncome(upIncome []*model.UpIncome, weeklyMap, monthlyMap map[int64]*model.UpIncome,
+	upAvStatis, upCmStatis, upBgmStatis map[int64]*model.UpArchStatis) {
+	var weeklyAvCount, monthlyAvCount int
+	var weeklyCmCount, monthlyCmCount int
+	var weeklyBgmCount, monthlyBgmCount int
+	for _, income := range upIncome {
+		weeklyAvCount, monthlyAvCount = 0, 0
+		weeklyCmCount, monthlyCmCount = 0, 0
+		weeklyBgmCount, monthlyBgmCount = 0, 0
+		if statis, ok := upAvStatis[income.MID]; ok {
+			weeklyAvCount = len(strings.Split(statis.WeeklyAIDs, ","))
+			monthlyAvCount = len(strings.Split(statis.MonthlyAIDs, ","))
+		}
+
+		if statis, ok := upCmStatis[income.MID]; ok {
+			weeklyCmCount = len(strings.Split(statis.WeeklyAIDs, ","))
+			monthlyCmCount = len(strings.Split(statis.MonthlyAIDs, ","))
+		}
+
+		if statis, ok := upBgmStatis[income.MID]; ok {
+			weeklyBgmCount = len(strings.Split(statis.WeeklyAIDs, ","))
+			monthlyBgmCount = len(strings.Split(statis.MonthlyAIDs, ","))
+		}
+
+		if weeklyIncome, ok := weeklyMap[income.MID]; ok {
+			updateUpIncome(weeklyIncome, income, weeklyAvCount, weeklyCmCount, weeklyBgmCount)
+		} else {
+			weeklyMap[income.MID] = addUpIncome(income, startWeeklyDate, weeklyAvCount, weeklyCmCount, weeklyBgmCount)
+		}
+
+		if weeklyIncome, ok := monthlyMap[income.MID]; ok {
+			updateUpIncome(weeklyIncome, income, monthlyAvCount, monthlyCmCount, monthlyBgmCount)
+		} else {
+			monthlyMap[income.MID] = addUpIncome(income, startMonthlyDate, monthlyAvCount, monthlyCmCount, monthlyBgmCount)
+		}
+	}
+}
+
+func addUpIncome(daily *model.UpIncome, fixDate time.Time, avCount, cmCount, bgmCount int) *model.UpIncome {
+	return &model.UpIncome{
+		MID:           daily.MID,
+		AvCount:       int64(avCount),
+		PlayCount:     daily.PlayCount,
+		AvIncome:      daily.AvIncome,
+		AvBaseIncome:  daily.AvBaseIncome,
+		AvTax:         daily.AvTax,
+		AvTotalIncome: daily.AvTotalIncome,
+
+		ColumnCount:       int64(cmCount),
+		ColumnIncome:      daily.ColumnIncome,
+		ColumnBaseIncome:  daily.ColumnBaseIncome,
+		ColumnTax:         daily.ColumnTax,
+		ColumnTotalIncome: daily.ColumnTotalIncome,
+
+		BgmCount:       int64(bgmCount),
+		BgmIncome:      daily.BgmIncome,
+		BgmBaseIncome:  daily.BgmBaseIncome,
+		BgmTax:         daily.BgmTax,
+		BgmTotalIncome: daily.BgmTotalIncome,
+
+		AudioIncome: daily.AudioIncome,
+
+		TaxMoney:    daily.TaxMoney,
+		Income:      daily.Income,
+		BaseIncome:  daily.BaseIncome,
+		TotalIncome: daily.TotalIncome,
+		Date:        xtime.Time(fixDate.Unix()),
+		DBState:     _dbInsert,
+	}
+}
+
+func updateUpIncome(origin, daily *model.UpIncome, avCount, cmCount, bgmCount int) {
+	origin.AvCount = int64(avCount)
+	origin.PlayCount += daily.PlayCount
+	origin.AvIncome += daily.AvIncome
+	origin.AvBaseIncome += daily.AvBaseIncome
+	origin.AvTax += daily.AvTax
+	origin.AvTotalIncome = daily.AvTotalIncome
+
+	origin.ColumnCount = int64(cmCount)
+	origin.ColumnIncome += daily.ColumnIncome
+	origin.ColumnBaseIncome += daily.ColumnBaseIncome
+	origin.ColumnTax += daily.ColumnTax
+	origin.ColumnTotalIncome = daily.ColumnTotalIncome
+
+	origin.BgmCount = int64(bgmCount)
+	origin.BgmIncome += daily.BgmIncome
+	origin.BgmBaseIncome += daily.BgmBaseIncome
+	origin.BgmTax += daily.BgmTax
+	origin.BgmTotalIncome = daily.BgmTotalIncome
+
+	origin.AudioIncome += daily.AudioIncome
+
+	origin.TaxMoney += daily.TaxMoney
+	origin.Income += daily.Income
+	origin.BaseIncome += daily.BaseIncome
+	origin.TotalIncome = daily.TotalIncome
+	origin.DBState = _dbUpdate
+}
+
+// UpIncomeDBStore insert up_income
+func (s *UpIncomeSvr) UpIncomeDBStore(c context.Context, weeklyMap, monthlyMap map[int64]*model.UpIncome) (err error) {
+	err = s.UpIncomeDBStoreBatch(c, _upIncomeWeekly, weeklyMap)
+	if err != nil {
+		log.Error("s.UpIncomeDBStoreBatch up_income_weekly error(%v)", err)
+		return
+	}
+
+	err = s.UpIncomeDBStoreBatch(c, _upIncomeMonthly, monthlyMap)
+	if err != nil {
+		log.Error("s.UpIncomeDBStoreBatch up_income_monthly error(%v)", err)
+		return
+	}
+	return
+}
+
+// UpIncomeDBStoreBatch up income db batch store
+func (s *UpIncomeSvr) UpIncomeDBStoreBatch(c context.Context, table string, upIncomeMap map[int64]*model.UpIncome) error {
+	insert, update := make([]*model.UpIncome, batchSize), make([]*model.UpIncome, batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, income := range upIncomeMap {
+		if income.DBState == _dbInsert {
+			insert[insertIndex] = income
+			insertIndex++
+		} else if income.DBState == _dbUpdate {
+			update[updateIndex] = income
+			updateIndex++
+		}
+
+		if insertIndex >= batchSize {
+			_, err := s.upIncomeBatchInsert(c, table, insert[:insertIndex])
+			if err != nil {
+				log.Error("s.upIncomeBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= batchSize {
+			_, err := s.upIncomeBatchInsert(c, table, update[:updateIndex])
+			if err != nil {
+				log.Error("s.upIncomeBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.upIncomeBatchInsert(c, table, insert[:insertIndex])
+		if err != nil {
+			log.Error("s.upIncomeBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.upIncomeBatchInsert(c, table, update[:updateIndex])
+		if err != nil {
+			log.Error("s.upIncomeBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *UpIncomeSvr) upIncomeBatchInsert(c context.Context, table string, us []*model.UpIncome) (rows int64, err error) {
+	var buf bytes.Buffer
+	for _, u := range us {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(u.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.PlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AudioIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.TaxMoney, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.Income, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvBaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvTax, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnBaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnTax, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmBaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmTax, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + u.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BaseIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values := buf.String()
+	buf.Reset()
+	rows, err = s.dao.InsertUpIncomeTable(c, table, values)
+	return
+}

app/job/main/growup/service/income/up_income_date_test.go

@@ -0,0 +1,32 @@
+package income
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	model "go-common/app/job/main/growup/model/income"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_GetUpIncomeTable(t *testing.T) {
+	Convey("GetUpIncomeTable", t, func() {
+		_, err := s.income.upIncomeSvr.GetUpIncomeTable(context.Background(), time.Now(), _upIncomeWeekly)
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_GetUpIncomeWeeklyAndMonthly(t *testing.T) {
+	Convey("GetUpIncomeWeeklyAndMonthly", t, func() {
+		_, _, err := s.income.upIncomeSvr.GetUpIncomeWeeklyAndMonthly(context.Background(), time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_UpIncomeDBStore(t *testing.T) {
+	Convey("UpIncomeDBStore", t, func() {
+		err := s.income.upIncomeSvr.UpIncomeDBStore(context.Background(), map[int64]*model.UpIncome{}, map[int64]*model.UpIncome{})
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/growup/service/income/up_income_stat.go

@@ -0,0 +1,100 @@
+package income
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	incomeD "go-common/app/job/main/growup/dao/income"
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// UpIncomeStatSvr up_income_stat svr
+type UpIncomeStatSvr struct {
+	batchSize int
+	dao       *incomeD.Dao
+}
+
+// NewUpIncomeStatSvr new server
+func NewUpIncomeStatSvr(dao *incomeD.Dao, batchSize int) (svr *UpIncomeStatSvr) {
+	return &UpIncomeStatSvr{
+		batchSize: batchSize,
+		dao:       dao,
+	}
+}
+
+// UpIncomeStat return stats, key: mid, value: total_income
+func (p *UpIncomeStatSvr) UpIncomeStat(c context.Context, limit int64) (m map[int64]*model.UpIncomeStat, err error) {
+	var id int64
+	m = make(map[int64]*model.UpIncomeStat)
+	for {
+		var um map[int64]*model.UpIncomeStat
+		um, id, err = p.dao.UpIncomeStat(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(um) == 0 {
+			break
+		}
+		for mid, u := range um {
+			if u.IsDeleted == 0 {
+				m[mid] = u
+			}
+		}
+	}
+	return
+}
+
+// BatchInsertUpIncomeStat insert up_income_statis batch
+func (p *UpIncomeStatSvr) BatchInsertUpIncomeStat(c context.Context, us map[int64]*model.UpIncomeStat) (err error) {
+	var (
+		buff    = make([]*model.UpIncomeStat, batchSize)
+		buffEnd = 0
+	)
+	for _, u := range us {
+		if u.DataState == 0 {
+			continue
+		}
+		buff[buffEnd] = u
+		buffEnd++
+		if buffEnd >= p.batchSize {
+			values := upIncomeStatValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = p.dao.InsertUpIncomeStat(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	if buffEnd > 0 {
+		values := upIncomeStatValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = p.dao.InsertUpIncomeStat(c, values)
+	}
+	return
+}
+
+func upIncomeStatValues(us []*model.UpIncomeStat) (values string) {
+	var buf bytes.Buffer
+	for _, u := range us {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(u.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.TotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.AvTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.ColumnTotalIncome, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(u.BgmTotalIncome, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/growup/service/income/up_info_video.go

@@ -0,0 +1,29 @@
+package income
+
+import (
+	"context"
+
+	model "go-common/app/job/main/growup/model/income"
+)
+
+// Signed signed up by business
+func (s *Service) Signed(c context.Context, business string, limit int64) (m map[int64]*model.Signed, err error) {
+	var offset int64
+	m = make(map[int64]*model.Signed)
+	for {
+		var us map[int64]*model.Signed
+		offset, us, err = s.dao.Ups(c, business, offset, limit)
+		if err != nil {
+			return
+		}
+		if len(us) == 0 {
+			break
+		}
+		for k, v := range us {
+			if v.AccountState == 3 || v.AccountState == 5 || v.AccountState == 6 {
+				m[k] = v
+			}
+		}
+	}
+	return
+}