Create & Init Project...

app/job/main/growup/service/charge/BUILD

@@ -0,0 +1,72 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "av_charge_statis_test.go",
+        "run_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/growup/conf:go_default_library",
+        "//app/job/main/growup/model/charge:go_default_library",
+        "//app/job/main/growup/service/ctrl:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "av_charge.go",
+        "av_charge_statis.go",
+        "bgm_charge.go",
+        "column_charge.go",
+        "date_statis.go",
+        "run.go",
+        "run_bgm.go",
+        "run_column.go",
+        "run_video.go",
+        "service.go",
+        "trans.go",
+        "up_charge.go",
+    ],
+    importpath = "go-common/app/job/main/growup/service/charge",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/growup/conf:go_default_library",
+        "//app/job/main/growup/dao/charge:go_default_library",
+        "//app/job/main/growup/dao/email:go_default_library",
+        "//app/job/main/growup/model/charge:go_default_library",
+        "//app/job/main/growup/service:go_default_library",
+        "//app/job/main/growup/service/ctrl:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/golang.org/x/sync/errgroup:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/growup/service/charge/av_charge.go

@@ -0,0 +1,308 @@
+package charge
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+	"time"
+
+	dao "go-common/app/job/main/growup/dao/charge"
+	model "go-common/app/job/main/growup/model/charge"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+func (s *Service) handleAvCharge(c context.Context, date time.Time,
+	dailyChannel chan []*model.AvCharge) (weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
+	var eg errgroup.Group
+	weeklyChargeMap = make(map[int64]*model.AvCharge)
+	monthlyChargeMap = make(map[int64]*model.AvCharge)
+
+	eg.Go(func() (err error) {
+		avWeeklyCharge, err := s.GetAvCharge(c, getStartWeeklyDate(date), s.dao.AvWeeklyCharge)
+		if err != nil {
+			log.Error("s.GetAvCharge(av_weekly_charge) error(%v)", err)
+			return
+		}
+		for _, weeklyCharge := range avWeeklyCharge {
+			weeklyChargeMap[weeklyCharge.AvID] = weeklyCharge
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		avMonthlyCharge, err := s.GetAvCharge(c, getStartMonthlyDate(date), s.dao.AvMonthlyCharge)
+		if err != nil {
+			log.Error("s.GetAvCharge(av_monthly_charge) error(%v)", err)
+			return
+		}
+		for _, monthlyCharge := range avMonthlyCharge {
+			monthlyChargeMap[monthlyCharge.AvID] = monthlyCharge
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		chargeStatisMap, err = s.GetAvChargeStatisMap(c)
+		if err != nil {
+			log.Error("s.GetAvChargeStatisMap error(%v)", err)
+		}
+		return
+	})
+
+	if err = eg.Wait(); err != nil {
+		log.Error("HandleAvCharge eg.Wait error(%v)", err)
+		return
+	}
+
+	s.calAvCharges(date, weeklyChargeMap, monthlyChargeMap, chargeStatisMap, dailyChannel)
+	return
+}
+
+func (s *Service) avDailyCharges(c context.Context, date time.Time, ch chan []*model.AvCharge) (err error) {
+	defer func() {
+		close(ch)
+	}()
+	var id int64
+	for {
+		var charges []*model.AvCharge
+		charges, err = s.dao.AvDailyCharge(c, date, id, _limitSize)
+		if err != nil {
+			return
+		}
+		ch <- charges
+		if len(charges) < _limitSize {
+			break
+		}
+		id = charges[len(charges)-1].ID
+	}
+	return
+}
+
+// GetAvCharge get av charge
+func (s *Service) GetAvCharge(c context.Context, date time.Time, fn dao.IAvCharge) (avCharges []*model.AvCharge, err error) {
+	var id int64
+	for {
+		var avCharge []*model.AvCharge
+		avCharge, err = fn(c, date, id, _limitSize)
+		if err != nil {
+			return
+		}
+		avCharges = append(avCharges, avCharge...)
+		if len(avCharge) < _limitSize {
+			break
+		}
+		id = avCharge[len(avCharge)-1].ID
+	}
+	return
+}
+
+func (s *Service) calAvCharges(date time.Time, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis, dailyChannel chan []*model.AvCharge) {
+	for avDailyCharge := range dailyChannel {
+		s.calAvCharge(date, avDailyCharge, weeklyChargeMap, monthlyChargeMap, chargeStatisMap)
+	}
+}
+
+func (s *Service) calAvCharge(date time.Time, avDailyCharge []*model.AvCharge, weeklyChargeMap, monthlyChargeMap map[int64]*model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
+	for _, dailyCharge := range avDailyCharge {
+		if weeklyCharge, ok := weeklyChargeMap[dailyCharge.AvID]; ok {
+			updateAvCharge(weeklyCharge, dailyCharge)
+		} else {
+			weeklyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startWeeklyDate)
+		}
+
+		if monthlyCharge, ok := monthlyChargeMap[dailyCharge.AvID]; ok {
+			updateAvCharge(monthlyCharge, dailyCharge)
+		} else {
+			monthlyChargeMap[dailyCharge.AvID] = addAvCharge(dailyCharge, startMonthlyDate)
+		}
+		s.CalAvChargeStatis(dailyCharge, chargeStatisMap)
+	}
+}
+
+func addAvCharge(daily *model.AvCharge, fixDate time.Time) *model.AvCharge {
+	return &model.AvCharge{
+		AvID:           daily.AvID,
+		MID:            daily.MID,
+		TagID:          daily.TagID,
+		IsOriginal:     daily.IsOriginal,
+		DanmakuCount:   daily.DanmakuCount,
+		CommentCount:   daily.CommentCount,
+		CollectCount:   daily.CollectCount,
+		CoinCount:      daily.CoinCount,
+		ShareCount:     daily.ShareCount,
+		ElecPayCount:   daily.ElecPayCount,
+		TotalPlayCount: daily.TotalPlayCount,
+		WebPlayCount:   daily.WebPlayCount,
+		AppPlayCount:   daily.AppPlayCount,
+		H5PlayCount:    daily.H5PlayCount,
+		LvUnknown:      daily.LvUnknown,
+		Lv0:            daily.Lv0,
+		Lv1:            daily.Lv1,
+		Lv2:            daily.Lv2,
+		Lv3:            daily.Lv3,
+		Lv4:            daily.Lv4,
+		Lv5:            daily.Lv5,
+		Lv6:            daily.Lv6,
+		VScore:         daily.VScore,
+		IncCharge:      daily.IncCharge,
+		TotalCharge:    daily.IncCharge,
+		Date:           xtime.Time(fixDate.Unix()),
+		UploadTime:     daily.UploadTime,
+		DBState:        _dbInsert,
+	}
+}
+
+func updateAvCharge(origin, daily *model.AvCharge) {
+	origin.DanmakuCount += daily.DanmakuCount
+	origin.CommentCount += daily.CommentCount
+	origin.CollectCount += daily.CollectCount
+	origin.CoinCount += daily.CoinCount
+	origin.ShareCount += daily.ShareCount
+	origin.ElecPayCount += daily.ElecPayCount
+	origin.TotalPlayCount += daily.TotalPlayCount
+	origin.WebPlayCount += daily.WebPlayCount
+	origin.AppPlayCount += daily.AppPlayCount
+	origin.H5PlayCount += daily.H5PlayCount
+	origin.LvUnknown += daily.LvUnknown
+	origin.Lv0 += daily.Lv0
+	origin.Lv1 += daily.Lv1
+	origin.Lv2 += daily.Lv2
+	origin.Lv3 += daily.Lv3
+	origin.Lv4 += daily.Lv4
+	origin.Lv5 += daily.Lv5
+	origin.Lv6 += daily.Lv6
+	origin.VScore += daily.VScore
+	origin.IncCharge += daily.IncCharge
+	origin.TotalCharge += daily.IncCharge
+	origin.DBState = _dbUpdate
+}
+
+// AvChargeDBStore store data
+func (s *Service) AvChargeDBStore(c context.Context, table string, avChargeMap map[int64]*model.AvCharge) error {
+	insert, update := make([]*model.AvCharge, _batchSize), make([]*model.AvCharge, _batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range avChargeMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+
+		if insertIndex >= _batchSize {
+			_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
+			if err != nil {
+				log.Error("s.avChargeBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= _batchSize {
+			_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
+			if err != nil {
+				log.Error("s.avChargeBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.avChargeBatchInsert(c, insert[:insertIndex], table)
+		if err != nil {
+			log.Error("s.avChargeBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.avChargeBatchInsert(c, update[:updateIndex], table)
+		if err != nil {
+			log.Error("s.avChargeBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func assembleAvCharge(avCharge []*model.AvCharge) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range avCharge {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.AvID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(row.IsOriginal))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.DanmakuCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CommentCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CollectCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CoinCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.ShareCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.ElecPayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalPlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.WebPlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.AppPlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.H5PlayCount, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.LvUnknown, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv0, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv1, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv2, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv3, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv4, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv5, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Lv6, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.VScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) avChargeBatchInsert(c context.Context, avCharge []*model.AvCharge, table string) (rows int64, err error) {
+	vals := assembleAvCharge(avCharge)
+	rows, err = s.dao.InsertAvChargeTable(c, vals, table)
+	return
+}

app/job/main/growup/service/charge/av_charge_statis.go

@@ -0,0 +1,155 @@
+package charge
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	"go-common/library/log"
+)
+
+// GetAvChargeStatisMap get av charge statistics map
+func (s *Service) GetAvChargeStatisMap(c context.Context) (chargeStatisMap map[int64]*model.AvChargeStatis, err error) {
+	avChargeStatis, err := s.GetAvChargeStatis(c)
+	if err != nil {
+		log.Error("s.GetAvChargeStatis error(%v)", err)
+		return
+	}
+
+	chargeStatisMap = make(map[int64]*model.AvChargeStatis)
+	for _, chargeStatis := range avChargeStatis {
+		chargeStatisMap[chargeStatis.AvID] = chargeStatis
+	}
+	return
+}
+
+// GetAvChargeStatis get av charge statistics
+func (s *Service) GetAvChargeStatis(c context.Context) (avChargeStatis []*model.AvChargeStatis, err error) {
+	var id int64
+	for {
+		statis, err1 := s.dao.AvChargeStatis(c, id, _limitSize)
+		if err1 != nil {
+			err = err1
+			return
+		}
+		avChargeStatis = append(avChargeStatis, statis...)
+		if len(statis) < _limitSize {
+			break
+		}
+		id = statis[len(statis)-1].ID
+	}
+	return
+}
+
+// CalAvChargeStatis cal av charge statis
+func (s *Service) CalAvChargeStatis(dailyCharge *model.AvCharge, chargeStatisMap map[int64]*model.AvChargeStatis) {
+	if statisCharge, ok := chargeStatisMap[dailyCharge.AvID]; ok {
+		updateAvChargeStatis(statisCharge, dailyCharge)
+	} else {
+		chargeStatisMap[dailyCharge.AvID] = addAvChargeStatis(dailyCharge)
+	}
+}
+
+func addAvChargeStatis(daily *model.AvCharge) *model.AvChargeStatis {
+	return &model.AvChargeStatis{
+		AvID:        daily.AvID,
+		MID:         daily.MID,
+		TagID:       daily.TagID,
+		IsOriginal:  daily.IsOriginal,
+		UploadTime:  daily.UploadTime,
+		TotalCharge: daily.IncCharge,
+		DBState:     _dbInsert,
+	}
+}
+
+func updateAvChargeStatis(avChargeStatis *model.AvChargeStatis, daily *model.AvCharge) {
+	avChargeStatis.TotalCharge += daily.IncCharge
+	avChargeStatis.DBState = _dbUpdate
+}
+
+// AvChargeStatisDBStore store data
+func (s *Service) AvChargeStatisDBStore(c context.Context, chargeStatisMap map[int64]*model.AvChargeStatis) error {
+	insert, update := make([]*model.AvChargeStatis, _batchSize), make([]*model.AvChargeStatis, _batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range chargeStatisMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+
+		if insertIndex >= _batchSize {
+			_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
+			if err != nil {
+				log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= _batchSize {
+			_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
+			if err != nil {
+				log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.avChargeStatisBatchInsert(c, insert[:insertIndex])
+		if err != nil {
+			log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.avChargeStatisBatchInsert(c, update[:updateIndex])
+		if err != nil {
+			log.Error("s.avChargeStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func assembleAvChargeStatis(avChargeStatis []*model.AvChargeStatis) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range avChargeStatis {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.AvID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(row.IsOriginal))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.UploadTime.Time().Format(_layoutSec) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) avChargeStatisBatchInsert(c context.Context, avChargeStatis []*model.AvChargeStatis) (rows int64, err error) {
+	vals := assembleAvChargeStatis(avChargeStatis)
+	rows, err = s.dao.InsertAvChargeStatisBatch(c, vals)
+	return
+}

app/job/main/growup/service/charge/av_charge_statis_test.go

@@ -0,0 +1,39 @@
+package charge
+
+import (
+	"context"
+	"testing"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_GetAvChargeStatisMap(t *testing.T) {
+	Convey("GetAvChargeStatisMap", t, func() {
+		_, err := s.GetAvChargeStatisMap(context.Background())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_GetAvChargeStatis(t *testing.T) {
+	Convey("GetAvChargeStatis", t, func() {
+		_, err := s.GetAvChargeStatis(context.Background())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_AvChargeStatisDBStore(t *testing.T) {
+	Convey("AvChargeStatisDBStore", t, func() {
+		chargeStatisMap := make(map[int64]*model.AvChargeStatis)
+		value := &model.AvChargeStatis{
+			AvID:    11,
+			MID:     11,
+			TagID:   11,
+			DBState: 1,
+		}
+		chargeStatisMap[11] = value
+		err := s.AvChargeStatisDBStore(context.Background(), chargeStatisMap)
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/growup/service/charge/bgm_charge.go

@@ -0,0 +1,431 @@
+package charge
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math"
+	"math/big"
+	"strconv"
+	"strings"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+	"golang.org/x/sync/errgroup"
+)
+
+func (s *Service) handleBgm(c context.Context, date time.Time,
+	dailyChannel chan []*model.BgmCharge) (weekly, monthly map[string]*model.BgmCharge, statis map[string]*model.BgmStatis, err error) {
+	var eg errgroup.Group
+	weekly = make(map[string]*model.BgmCharge)
+	monthly = make(map[string]*model.BgmCharge)
+	statis = make(map[string]*model.BgmStatis)
+
+	eg.Go(func() (err error) {
+		weekly, err = s.getBgmCharge(c, getStartWeeklyDate(date), "bgm_weekly_charge")
+		if err != nil {
+			log.Error("s.GetBgmCharge(bgm_weekly_charge) error(%v)", err)
+			return
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		monthly, err = s.getBgmCharge(c, getStartMonthlyDate(date), "bgm_monthly_charge")
+		if err != nil {
+			log.Error("s.GetBgmCharge(bgm_monthly_charge) error(%v)", err)
+			return
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		statis, err = s.getBgmStatis(c)
+		if err != nil {
+			log.Error("s.getBgmStatisMap error(%v)", err)
+		}
+		return
+	})
+
+	if err = eg.Wait(); err != nil {
+		log.Error("handleBgm eg.Wait error(%v)", err)
+		return
+	}
+
+	s.calBgms(date, weekly, monthly, statis, dailyChannel)
+	return
+}
+
+func (s *Service) calBgms(date time.Time, weeklyMap, monthlyMap map[string]*model.BgmCharge, statisMap map[string]*model.BgmStatis, dailyChannel chan []*model.BgmCharge) {
+	for daily := range dailyChannel {
+		s.calBgm(date, daily, weeklyMap, monthlyMap, statisMap)
+	}
+}
+
+func (s *Service) calBgm(date time.Time, daily []*model.BgmCharge, weeklyMap, monthlyMap map[string]*model.BgmCharge, statisMap map[string]*model.BgmStatis) {
+	for _, charge := range daily {
+		key := fmt.Sprintf("%d+%d", charge.SID, charge.AID)
+		if weekly, ok := weeklyMap[key]; ok {
+			updateBgmCharge(weekly, charge)
+		} else {
+			weeklyMap[key] = addBgmCharge(charge, startWeeklyDate)
+		}
+		if monthly, ok := monthlyMap[key]; ok {
+			updateBgmCharge(monthly, charge)
+		} else {
+			monthlyMap[key] = addBgmCharge(charge, startMonthlyDate)
+		}
+		if statis, ok := statisMap[key]; ok {
+			updateBgmStatis(statis, charge)
+		} else {
+			statisMap[key] = addBgmStatis(charge)
+		}
+	}
+}
+
+func addBgmCharge(daily *model.BgmCharge, fixDate time.Time) *model.BgmCharge {
+	return &model.BgmCharge{
+		SID:       daily.SID,
+		AID:       daily.AID,
+		MID:       daily.MID,
+		CID:       daily.CID,
+		JoinAt:    daily.JoinAt,
+		Title:     daily.Title,
+		IncCharge: daily.IncCharge,
+		Date:      xtime.Time(fixDate.Unix()),
+		DBState:   _dbInsert,
+	}
+}
+
+func updateBgmCharge(origin, daily *model.BgmCharge) {
+	origin.IncCharge += daily.IncCharge
+	origin.DBState = _dbUpdate
+}
+
+func addBgmStatis(daily *model.BgmCharge) *model.BgmStatis {
+	return &model.BgmStatis{
+		SID:         daily.SID,
+		AID:         daily.AID,
+		MID:         daily.MID,
+		CID:         daily.CID,
+		JoinAt:      daily.JoinAt,
+		Title:       daily.Title,
+		TotalCharge: daily.IncCharge,
+		DBState:     _dbInsert,
+	}
+}
+
+func updateBgmStatis(statis *model.BgmStatis, daily *model.BgmCharge) {
+	statis.TotalCharge += daily.IncCharge
+	statis.DBState = _dbUpdate
+}
+
+func (s *Service) getBgmCharge(c context.Context, date time.Time, table string) (bgms map[string]*model.BgmCharge, err error) {
+	bgms = make(map[string]*model.BgmCharge)
+	var id int64
+	for {
+		var bgm []*model.BgmCharge
+		bgm, err = s.dao.BgmCharge(c, date, id, _limitSize, table)
+		if err != nil {
+			return
+		}
+		for _, b := range bgm {
+			key := fmt.Sprintf("%d+%d", b.SID, b.AID)
+			bgms[key] = b
+		}
+		if len(bgm) < _limitSize {
+			break
+		}
+		id = bgm[len(bgm)-1].ID
+	}
+	return
+}
+
+func (s *Service) getBgmStatis(c context.Context) (bgms map[string]*model.BgmStatis, err error) {
+	bgms = make(map[string]*model.BgmStatis)
+	var id int64
+	for {
+		var bgm []*model.BgmStatis
+		bgm, err = s.dao.BgmStatis(c, id, _limitSize)
+		if err != nil {
+			return
+		}
+		for _, b := range bgm {
+			key := fmt.Sprintf("%d+%d", b.SID, b.AID)
+			bgms[key] = b
+		}
+		if len(bgm) < _limitSize {
+			break
+		}
+		id = bgm[len(bgm)-1].ID
+	}
+	return
+}
+
+func (s *Service) bgmCharges(c context.Context, date time.Time, ch chan []*model.BgmCharge, avBgmCharge chan []*model.AvCharge) (dailyMap map[string]*model.BgmCharge, err error) {
+	dailyMap = make(map[string]*model.BgmCharge)
+	bgms, err := s.GetBgms(c, int64(_limitSize))
+	if err != nil {
+		log.Error("s.GetBgms error(%v)", err)
+		return
+	}
+	defer func() {
+		close(ch)
+	}()
+	for avs := range avBgmCharge {
+		for _, av := range avs {
+			bgm, ok := bgms[av.AvID]
+			if !ok {
+				continue
+			}
+			incCharge := int64(round(div(mul(float64(av.IncCharge), float64(0.3)), float64(len(bgm))), 0))
+			if incCharge == 0 {
+				continue
+			}
+			charges := make([]*model.BgmCharge, 0, len(bgm))
+			for _, b := range bgm {
+				if b.MID == av.MID {
+					continue
+				}
+				c := &model.BgmCharge{
+					SID:       b.SID,
+					MID:       b.MID,
+					AID:       b.AID,
+					CID:       b.CID,
+					IncCharge: incCharge,
+					JoinAt:    b.JoinAt,
+					Date:      av.Date,
+					Title:     b.Title,
+					DBState:   _dbInsert,
+				}
+				dailyMap[fmt.Sprintf("%d+%d", c.SID, c.AID)] = c
+				charges = append(charges, c)
+			}
+			ch <- charges
+		}
+	}
+	return
+}
+
+// GetBgms map[av_id][]*model.Bgm
+func (s *Service) GetBgms(c context.Context, limit int64) (bm map[int64][]*model.Bgm, err error) {
+	var id int64
+	bm = make(map[int64][]*model.Bgm)
+	for {
+		var bs []*model.Bgm
+		bs, id, err = s.dao.GetBgm(c, id, limit)
+		if err != nil {
+			return
+		}
+		if len(bs) == 0 {
+			break
+		}
+		for _, b := range bs {
+			if _, ok := bm[b.AID]; ok {
+				bm[b.AID] = append(bm[b.AID], b)
+			} else {
+				bm[b.AID] = []*model.Bgm{b}
+			}
+		}
+	}
+	return
+}
+
+func round(val float64, places int) float64 {
+	var round float64
+	pow := math.Pow(10, float64(places))
+	digit := pow * val
+	_, div := math.Modf(digit)
+	if div >= 0.5 {
+		round = math.Ceil(digit)
+	} else {
+		round = math.Floor(digit)
+	}
+	return round / pow
+}
+
+func div(x, y float64) float64 {
+	a := big.NewFloat(x)
+	b := big.NewFloat(y)
+	c := new(big.Float).Quo(a, b)
+	d, _ := c.Float64()
+	return d
+}
+
+func mul(x, y float64) float64 {
+	a := big.NewFloat(x)
+	b := big.NewFloat(y)
+	c := new(big.Float).Mul(a, b)
+	d, _ := c.Float64()
+	return d
+}
+
+func (s *Service) bgmDBStore(c context.Context, table string, bgmMap map[string]*model.BgmCharge) error {
+	insert, update := make([]*model.BgmCharge, _batchSize), make([]*model.BgmCharge, _batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range bgmMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+
+		if insertIndex >= _batchSize {
+			_, err := s.bgmBatchInsert(c, insert[:insertIndex], table)
+			if err != nil {
+				log.Error("s.bgmBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= _batchSize {
+			_, err := s.bgmBatchInsert(c, update[:updateIndex], table)
+			if err != nil {
+				log.Error("s.bgmBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.bgmBatchInsert(c, insert[:insertIndex], table)
+		if err != nil {
+			log.Error("s.bgmBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.bgmBatchInsert(c, update[:updateIndex], table)
+		if err != nil {
+			log.Error("s.bgmBatchInsert error(%v)", err)
+			return err
+		}
+	}
+	return nil
+}
+
+func assembleBgmCharge(charges []*model.BgmCharge) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range charges {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.SID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.AID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("\"" + strings.Replace(row.Title, "\"", "\\\"", -1) + "\"")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.JoinAt.Time().Format(_layoutSec) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) bgmBatchInsert(c context.Context, charges []*model.BgmCharge, table string) (rows int64, err error) {
+	vals := assembleBgmCharge(charges)
+	rows, err = s.dao.InsertBgmChargeTable(c, vals, table)
+	return
+}
+
+func (s *Service) bgmStatisDBStore(c context.Context, statisMap map[string]*model.BgmStatis) error {
+	insert, update := make([]*model.BgmStatis, _batchSize), make([]*model.BgmStatis, _batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range statisMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+		if insertIndex >= _batchSize {
+			_, err := s.bgmStatisBatchInsert(c, insert[:insertIndex])
+			if err != nil {
+				log.Error("s.bgmStatisBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= _batchSize {
+			_, err := s.bgmStatisBatchInsert(c, update[:updateIndex])
+			if err != nil {
+				log.Error("s.bgmStatisBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.bgmStatisBatchInsert(c, insert[:insertIndex])
+		if err != nil {
+			log.Error("s.bgmStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.bgmStatisBatchInsert(c, update[:updateIndex])
+		if err != nil {
+			log.Error("s.bgmStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+	return nil
+}
+
+func assembleBgmStatis(statis []*model.BgmStatis) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range statis {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.SID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.AID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("\"" + strings.Replace(row.Title, "\"", "\\\"", -1) + "\"")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.JoinAt.Time().Format(_layoutSec) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) bgmStatisBatchInsert(c context.Context, statis []*model.BgmStatis) (rows int64, err error) {
+	vals := assembleBgmStatis(statis)
+	rows, err = s.dao.InsertBgmStatisBatch(c, vals)
+	return
+}

app/job/main/growup/service/charge/column_charge.go

@@ -0,0 +1,357 @@
+package charge
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+	task "go-common/app/job/main/growup/service"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+	"golang.org/x/sync/errgroup"
+)
+
+func (s *Service) columnCharges(c context.Context, date time.Time, ch chan []*model.Column) (err error) {
+	defer func() {
+		close(ch)
+	}()
+	var id int64
+	for {
+		var charges []*model.Column
+		charges, err = s.dao.ColumnCharge(c, date, id, _limitSize, "column_daily_charge")
+		if err != nil {
+			return
+		}
+		ch <- charges
+		if len(charges) < _limitSize {
+			break
+		}
+		id = charges[len(charges)-1].ID
+	}
+	return
+}
+
+func (s *Service) handleColumn(c context.Context, date time.Time,
+	dailyChannel chan []*model.Column) (weeklyMap, monthlyMap map[int64]*model.Column, statisMap map[int64]*model.ColumnStatis, err error) {
+	var eg errgroup.Group
+	weeklyMap = make(map[int64]*model.Column)
+	monthlyMap = make(map[int64]*model.Column)
+	statisMap = make(map[int64]*model.ColumnStatis)
+
+	eg.Go(func() (err error) {
+		weekly, err := s.getColumnCharge(c, getStartWeeklyDate(date), "column_weekly_charge")
+		if err != nil {
+			log.Error("s.GetColumnCharge(column_weekly_charge) error(%v)", err)
+			return
+		}
+		for _, cm := range weekly {
+			weeklyMap[cm.AID] = cm
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		monthly, err := s.getColumnCharge(c, getStartMonthlyDate(date), "column_monthly_charge")
+		if err != nil {
+			log.Error("s.GetColumnCharge(column_monthly_charge) error(%v)", err)
+			return
+		}
+		for _, cm := range monthly {
+			monthlyMap[cm.AID] = cm
+		}
+		return
+	})
+
+	eg.Go(func() (err error) {
+		statis, err := s.getCmStatisMap(c)
+		if err != nil {
+			log.Error("s.getCmStatisMap error(%v)", err)
+		}
+		for _, cm := range statis {
+			statisMap[cm.AID] = cm
+		}
+		return
+	})
+
+	if err = eg.Wait(); err != nil {
+		log.Error("handleColumn eg.Wait error(%v)", err)
+		return
+	}
+
+	s.calColumns(date, weeklyMap, monthlyMap, statisMap, dailyChannel)
+	return
+}
+
+func (s *Service) calColumns(date time.Time, weeklyMap, monthlyMap map[int64]*model.Column, statisMap map[int64]*model.ColumnStatis, dailyChannel chan []*model.Column) {
+	for daily := range dailyChannel {
+		s.calColumn(date, daily, weeklyMap, monthlyMap, statisMap)
+	}
+}
+
+func (s *Service) calColumn(date time.Time, daily []*model.Column, weeklyMap, monthlyMap map[int64]*model.Column, statisMap map[int64]*model.ColumnStatis) {
+	for _, charge := range daily {
+		if weekly, ok := weeklyMap[charge.AID]; ok {
+			updateColumnCharge(weekly, charge)
+		} else {
+			weeklyMap[charge.AID] = addColumnCharge(charge, startWeeklyDate)
+		}
+		if monthly, ok := monthlyMap[charge.AID]; ok {
+			updateColumnCharge(monthly, charge)
+		} else {
+			monthlyMap[charge.AID] = addColumnCharge(charge, startMonthlyDate)
+		}
+		if statis, ok := statisMap[charge.AID]; ok {
+			updateColumnStatis(statis, charge)
+		} else {
+			statisMap[charge.AID] = addColumnStatis(charge)
+		}
+	}
+}
+
+func addColumnCharge(daily *model.Column, fixDate time.Time) *model.Column {
+	return &model.Column{
+		AID:        daily.AID,
+		MID:        daily.MID,
+		Title:      daily.Title,
+		TagID:      daily.TagID,
+		Words:      daily.Words,
+		UploadTime: daily.UploadTime,
+		IncCharge:  daily.IncCharge,
+		Date:       xtime.Time(fixDate.Unix()),
+		DBState:    _dbInsert,
+	}
+}
+
+func updateColumnCharge(origin, daily *model.Column) {
+	origin.IncCharge += daily.IncCharge
+	origin.DBState = _dbUpdate
+}
+
+func addColumnStatis(daily *model.Column) *model.ColumnStatis {
+	return &model.ColumnStatis{
+		AID:         daily.AID,
+		MID:         daily.MID,
+		Title:       daily.Title,
+		TagID:       daily.TagID,
+		UploadTime:  daily.UploadTime,
+		TotalCharge: daily.IncCharge,
+		DBState:     _dbInsert,
+	}
+}
+
+func updateColumnStatis(statis *model.ColumnStatis, daily *model.Column) {
+	statis.TotalCharge += daily.IncCharge
+	statis.DBState = _dbUpdate
+}
+
+func (s *Service) getColumnCharge(c context.Context, date time.Time, table string) (cms []*model.Column, err error) {
+	cms = make([]*model.Column, 0)
+	var id int64
+	for {
+		var cm []*model.Column
+		cm, err = s.dao.ColumnCharge(c, date, id, _limitSize, table)
+		if err != nil {
+			return
+		}
+		cms = append(cms, cm...)
+		if len(cm) < _limitSize {
+			break
+		}
+		id = cm[len(cm)-1].ID
+	}
+	return
+}
+
+func (s *Service) getCmStatisMap(c context.Context) (cms []*model.ColumnStatis, err error) {
+	cms = make([]*model.ColumnStatis, 0)
+	var id int64
+	for {
+		var cm []*model.ColumnStatis
+		cm, err = s.dao.CmStatis(c, id, _limitSize)
+		if err != nil {
+			return
+		}
+		cms = append(cms, cm...)
+		if len(cm) < _limitSize {
+			break
+		}
+		id = cm[len(cm)-1].ID
+	}
+	return
+}
+
+func (s *Service) cmDBStore(c context.Context, table string, cmMap map[int64]*model.Column) error {
+	insert, update := make([]*model.Column, _batchSize), make([]*model.Column, _batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range cmMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+
+		if insertIndex >= _batchSize {
+			_, err := s.cmBatchInsert(c, insert[:insertIndex], table)
+			if err != nil {
+				log.Error("s.cmBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= _batchSize {
+			_, err := s.cmBatchInsert(c, update[:updateIndex], table)
+			if err != nil {
+				log.Error("s.cmBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.cmBatchInsert(c, insert[:insertIndex], table)
+		if err != nil {
+			log.Error("s.cmBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.cmBatchInsert(c, update[:updateIndex], table)
+		if err != nil {
+			log.Error("s.cmBatchInsert error(%v)", err)
+			return err
+		}
+	}
+	return nil
+}
+
+func assembleCmCharge(charges []*model.Column) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range charges {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.AID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.IncCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.Date.Time().Format(_layout) + "'")
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.UploadTime, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) cmBatchInsert(c context.Context, charges []*model.Column, table string) (rows int64, err error) {
+	vals := assembleCmCharge(charges)
+	rows, err = s.dao.InsertCmChargeTable(c, vals, table)
+	return
+}
+
+func (s *Service) cmStatisDBStore(c context.Context, statisMap map[int64]*model.ColumnStatis) error {
+	insert, update := make([]*model.ColumnStatis, _batchSize), make([]*model.ColumnStatis, _batchSize)
+	insertIndex, updateIndex := 0, 0
+	for _, charge := range statisMap {
+		if charge.DBState == _dbInsert {
+			insert[insertIndex] = charge
+			insertIndex++
+		} else if charge.DBState == _dbUpdate {
+			update[updateIndex] = charge
+			updateIndex++
+		}
+		if insertIndex >= _batchSize {
+			_, err := s.cmStatisBatchInsert(c, insert[:insertIndex])
+			if err != nil {
+				log.Error("s.cmStatisBatchInsert error(%v)", err)
+				return err
+			}
+			insertIndex = 0
+		}
+
+		if updateIndex >= _batchSize {
+			_, err := s.cmStatisBatchInsert(c, update[:updateIndex])
+			if err != nil {
+				log.Error("s.cmStatisBatchInsert error(%v)", err)
+				return err
+			}
+			updateIndex = 0
+		}
+	}
+
+	if insertIndex > 0 {
+		_, err := s.cmStatisBatchInsert(c, insert[:insertIndex])
+		if err != nil {
+			log.Error("s.cmStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+
+	if updateIndex > 0 {
+		_, err := s.cmStatisBatchInsert(c, update[:updateIndex])
+		if err != nil {
+			log.Error("s.cmStatisBatchInsert error(%v)", err)
+			return err
+		}
+	}
+	return nil
+}
+
+func assembleCmStatis(statis []*model.ColumnStatis) (vals string) {
+	var buf bytes.Buffer
+	for _, row := range statis {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.AID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.TotalCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + time.Unix(row.UploadTime, 0).Format(_layoutSec) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) cmStatisBatchInsert(c context.Context, statis []*model.ColumnStatis) (rows int64, err error) {
+	vals := assembleCmStatis(statis)
+	rows, err = s.dao.InsertCmStatisBatch(c, vals)
+	return
+}
+
+// CheckTaskColumn  check column count by date
+func (s *Service) CheckTaskColumn(c context.Context, date string) (err error) {
+	defer func() {
+		task.GetTaskService().SetTaskStatus(c, task.TaskCmCharge, date, err)
+	}()
+
+	count, err := s.dao.CountCmDailyCharge(c, date)
+	if count == 0 {
+		err = fmt.Errorf("date(%s) column_daily_charge = 0", date)
+	}
+	return
+}

app/job/main/growup/service/charge/date_statis.go

@@ -0,0 +1,143 @@
+package charge
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math"
+	"strconv"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+)
+
+// SectionEntries section entries
+type SectionEntries struct {
+	daily   []*model.DateStatis
+	weekly  []*model.DateStatis
+	monthly []*model.DateStatis
+}
+
+func initChargeSections(charge, tagID int64, date xtime.Time) []*model.DateStatis {
+	chargeSections := make([]*model.DateStatis, 12)
+	chargeSections[0] = initChargeSection(0, 1, 0, charge, tagID, date)
+	chargeSections[1] = initChargeSection(1, 5, 1, charge, tagID, date)
+	chargeSections[2] = initChargeSection(5, 10, 2, charge, tagID, date)
+	chargeSections[3] = initChargeSection(10, 20, 3, charge, tagID, date)
+	chargeSections[4] = initChargeSection(20, 50, 4, charge, tagID, date)
+	chargeSections[5] = initChargeSection(50, 100, 5, charge, tagID, date)
+	chargeSections[6] = initChargeSection(100, 200, 6, charge, tagID, date)
+	chargeSections[7] = initChargeSection(200, 500, 7, charge, tagID, date)
+	chargeSections[8] = initChargeSection(500, 1000, 8, charge, tagID, date)
+	chargeSections[9] = initChargeSection(1000, 3000, 9, charge, tagID, date)
+	chargeSections[10] = initChargeSection(3000, 5000, 10, charge, tagID, date)
+	chargeSections[11] = initChargeSection(5000, math.MaxInt32, 11, charge, tagID, date)
+	return chargeSections
+}
+
+func initChargeSection(min, max, section, charge, tagID int64, date xtime.Time) *model.DateStatis {
+	var tips string
+	if max == math.MaxInt32 {
+		tips = fmt.Sprintf("\"%d+\"", min)
+	} else {
+		tips = fmt.Sprintf("\"%d~%d\"", min, max)
+	}
+	return &model.DateStatis{
+		MinCharge:    min,
+		MaxCharge:    max,
+		MoneySection: section,
+		MoneyTips:    tips,
+		Charge:       charge,
+		CategoryID:   tagID,
+		CDate:        date,
+	}
+}
+
+func (s *Service) handleDateStatis(c context.Context, sourceCh chan []*model.Archive, date time.Time, table string) (sections []*model.DateStatis, err error) {
+	// delete
+	if table != "" {
+		_, err = s.dao.DelStatisTable(c, table, date.Format(_layout))
+		if err != nil {
+			log.Error("s.dao.DelChargeStatisTable error(%v)", err)
+			return
+		}
+	}
+	// add
+	sections = s.handleArchives(c, sourceCh, date)
+	return
+}
+
+// HandleAv handle archive_charge_daily_statis, archive_charge_weekly_statis, archive_charge_monthly_statis
+func (s *Service) handleArchives(c context.Context, archiveCh chan []*model.Archive, date time.Time) (sections []*model.DateStatis) {
+	archiveTagMap := make(map[int64]map[int64]int64) // key TagID, value map[int64]int64 -> key avId, value charge
+	tagChargeMap := make(map[int64]int64)            // key TagID, value TagID total Charge
+	handleArchive(archiveCh, archiveTagMap, tagChargeMap, date)
+
+	sections = make([]*model.DateStatis, 0)
+	for tagID, archives := range archiveTagMap {
+		section := countDateStatis(archives, tagChargeMap[tagID], tagID, date)
+		sections = append(sections, section...)
+	}
+	return
+}
+
+func handleArchive(archiveCh chan []*model.Archive, archiveTagMap map[int64]map[int64]int64, tagChargeMap map[int64]int64, startDate time.Time) {
+	for archives := range archiveCh {
+		for _, ac := range archives {
+			if !startDate.After(ac.Date.Time()) {
+				tagChargeMap[ac.TagID] += ac.IncCharge
+				if _, ok := archiveTagMap[ac.TagID]; !ok {
+					archiveTagMap[ac.TagID] = make(map[int64]int64)
+				}
+				archiveTagMap[ac.TagID][ac.ID] += ac.IncCharge
+			}
+		}
+	}
+}
+
+func countDateStatis(charges map[int64]int64, totalCharge, tagID int64, date time.Time) (sections []*model.DateStatis) {
+	if len(charges) == 0 {
+		return
+	}
+	sections = initChargeSections(totalCharge, tagID, xtime.Time(date.Unix()))
+	for _, charge := range charges {
+		for _, section := range sections {
+			min, max := section.MinCharge*100, section.MaxCharge*100
+			if charge >= min && charge < max {
+				section.Count++
+			}
+		}
+	}
+	return
+}
+
+func (s *Service) dateStatisInsert(c context.Context, avChargeSection []*model.DateStatis, table string) (rows int64, err error) {
+	var buf bytes.Buffer
+	for _, row := range avChargeSection {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(row.Count, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.MoneySection, 10))
+		buf.WriteByte(',')
+		buf.WriteString(row.MoneyTips)
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.Charge, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(row.CategoryID, 10))
+		buf.WriteByte(',')
+		buf.WriteString("'" + row.CDate.Time().Format(_layout) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals := buf.String()
+	buf.Reset()
+	rows, err = s.dao.InsertStatisTable(c, table, vals)
+	return
+}

app/job/main/growup/service/charge/run.go

@@ -0,0 +1,99 @@
+package charge
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+	task "go-common/app/job/main/growup/service"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	_layout    = "2006-01-02"
+	_layoutSec = "2006-01-02 15:04:05"
+)
+
+var (
+	_dbInsert = 1
+	_dbUpdate = 2
+
+	_batchSize = 2000
+	_limitSize = 2000
+
+	startWeeklyDate  time.Time
+	startMonthlyDate time.Time
+)
+
+// RunAndSendMail run and send mail
+func (s *Service) RunAndSendMail(c context.Context, date time.Time) (err error) {
+	var msg, msgVideo, msgColumn, msgBgm string
+	mailReceivers := []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"}
+	var (
+		eg          errgroup.Group
+		avBgmCharge = make(chan []*model.AvCharge, 1000)
+	)
+	// check task
+	defer func() {
+		task.GetTaskService().SetTaskStatus(c, task.TaskCreativeCharge, date.Format(_layout), err)
+		if err != nil {
+			msg = err.Error()
+		}
+		msgErr := s.email.SendMail(date, msg, "创作激励每日补贴%d年%d月%d日", mailReceivers...)
+		if msgErr != nil {
+			log.Error("s.email.SendMail error(%v)", msgErr)
+		}
+	}()
+
+	err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskAvCharge, task.TaskCmCharge, task.TaskBgmSync)
+	if err != nil {
+		return
+	}
+
+	eg.Go(func() (err error) {
+		startTime := time.Now().Unix()
+		if err = s.runVideo(c, date, avBgmCharge); err != nil {
+			log.Error("s.runVideo error(%v)", err)
+		} else {
+			msgVideo = fmt.Sprintf("%s 视频补贴计算完成，耗时%ds\n", date.Format("2006-01-02"), time.Now().Unix()-startTime)
+		}
+		return
+	})
+	eg.Go(func() (err error) {
+		startTime := time.Now().Unix()
+		if err = s.runColumn(c, date); err != nil {
+			log.Error("s.runColumn error(%v)", err)
+		} else {
+			msgColumn = fmt.Sprintf("%s 专栏补贴计算完成，耗时%ds\n", date.Format("2006-01-02"), time.Now().Unix()-startTime)
+		}
+		return
+	})
+	eg.Go(func() (err error) {
+		startTime := time.Now().Unix()
+		if err = s.runBgm(c, date, avBgmCharge); err != nil {
+			log.Error("s.runBgm error(%v)", err)
+		} else {
+			msgBgm = fmt.Sprintf("%s 素材补贴计算完成，耗时%ds\n", date.Format("2006-01-02"), time.Now().Unix()-startTime)
+		}
+		return
+	})
+	if err = eg.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+	}
+	msg = fmt.Sprintf("%s,%s,%s", msgVideo, msgColumn, msgBgm)
+	return
+}
+
+func getStartWeeklyDate(date time.Time) time.Time {
+	for date.Weekday() != time.Monday {
+		date = date.AddDate(0, 0, -1)
+	}
+	return date
+}
+
+func getStartMonthlyDate(date time.Time) time.Time {
+	return time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+}

app/job/main/growup/service/charge/run_bgm.go

@@ -0,0 +1,251 @@
+package charge
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	_bgmDailyCharge   = "bgm_daily_charge"
+	_bgmWeeklyCharge  = "bgm_weekly_charge"
+	_bgmMonthlyCharge = "bgm_monthly_charge"
+
+	_bgmDailyStatis   = "bgm_charge_daily_statis"
+	_bgmWeeklyStatis  = "bgm_charge_weekly_statis"
+	_bgmMonthlyStatis = "bgm_charge_monthly_statis"
+)
+
+func (s *Service) runBgm(c context.Context, date time.Time, avBgmCharge chan []*model.AvCharge) (err error) {
+	startWeeklyDate = getStartWeeklyDate(date)
+	startMonthlyDate = getStartMonthlyDate(date)
+	var (
+		readGroup     errgroup.Group
+		dailyMap      = make(map[string]*model.BgmCharge)
+		sourceCh      = make(chan []*model.BgmCharge, 1000)
+		dailyStatisCh = make(chan []*model.BgmCharge, 1000)
+		bgmCh         = make(chan []*model.BgmCharge, 1000)
+	)
+
+	readGroup.Go(func() (err error) {
+		dailyMap, err = s.bgmCharges(c, date, sourceCh, avBgmCharge)
+		if err != nil {
+			log.Error("s.bgmCharges error(%v)", err)
+			return
+		}
+		log.Info("get bgm_daily_charge finished")
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(bgmCh)
+			close(dailyStatisCh)
+		}()
+		for charges := range sourceCh {
+			bgmCh <- charges
+			dailyStatisCh <- charges
+		}
+		return
+	})
+
+	var (
+		weeklyMap  map[string]*model.BgmCharge
+		monthlyMap map[string]*model.BgmCharge
+		statisMap  map[string]*model.BgmStatis
+		weeklyCh   = make(chan map[string]*model.BgmCharge, 1)
+		monthlyCh  = make(chan map[string]*model.BgmCharge, 1)
+	)
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(weeklyCh)
+			close(monthlyCh)
+		}()
+		weeklyMap, monthlyMap, statisMap, err = s.handleBgm(c, date, bgmCh)
+		if err != nil {
+			log.Error("s.handleBgm error(%v)", err)
+			return
+		}
+		weeklyCh <- weeklyMap
+		monthlyCh <- monthlyMap
+		log.Info("handleBgm finished")
+		return
+	})
+
+	var (
+		dateStatis = &SectionEntries{}
+		bgmDaily   = make(chan []*model.Archive, 2000)
+		bgmWeekly  = make(chan []*model.Archive, 1)
+		bgmMonthly = make(chan []*model.Archive, 1)
+	)
+	readGroup.Go(func() (err error) {
+		defer close(bgmDaily)
+		for bgms := range dailyStatisCh {
+			bgmDaily <- transBgm2Archive(bgms)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		defer close(bgmWeekly)
+		bgmWeekly <- transBgmMap2Archive(<-weeklyCh)
+		return
+	})
+	readGroup.Go(func() (err error) {
+		defer close(bgmMonthly)
+		bgmMonthly <- transBgmMap2Archive(<-monthlyCh)
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		dateStatis.daily, err = s.handleDateStatis(c, bgmDaily, date, _bgmDailyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _bgmDailyStatis, err)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		dateStatis.weekly, err = s.handleDateStatis(c, bgmWeekly, startWeeklyDate, _bgmWeeklyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _bgmWeeklyStatis, err)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		dateStatis.monthly, err = s.handleDateStatis(c, bgmMonthly, startMonthlyDate, _bgmMonthlyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _bgmMonthlyStatis, err)
+		}
+		return
+	})
+
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	{
+		if len(dailyMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_daily_charge")
+			return
+		}
+		if len(weeklyMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_weekly_charge")
+			return
+		}
+		if len(monthlyMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_monthly_charge")
+			return
+		}
+		if len(statisMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_charge_statis")
+			return
+		}
+		if len(dateStatis.daily) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_charge_daily_statis")
+			return
+		}
+		if len(dateStatis.weekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_charge_weekly_statis")
+			return
+		}
+		if len(dateStatis.monthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 bgm_charge_monthly_statis")
+			return
+		}
+	}
+
+	// persist
+	var writeGroup errgroup.Group
+	// bgm_daily_charge
+	writeGroup.Go(func() (err error) {
+		err = s.bgmDBStore(c, _bgmDailyCharge, dailyMap)
+		if err != nil {
+			log.Error("s.bgmDBStore bgm_daily_charge error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_daily_charge : %d", len(dailyMap))
+
+		// bgm_weekly_charge
+		err = s.bgmDBStore(c, _bgmWeeklyCharge, weeklyMap)
+		if err != nil {
+			log.Error("s.bgmDBStore bgm_weekly_charge error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_weekly_charge : %d", len(weeklyMap))
+
+		// bgm_monthly_charge
+		err = s.bgmDBStore(c, _bgmMonthlyCharge, monthlyMap)
+		if err != nil {
+			log.Error("s.bgmDBStore bgm_monthly_charge error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_monthly_charge : %d", len(monthlyMap))
+
+		// bgm_charge_statis
+		err = s.bgmStatisDBStore(c, statisMap)
+		if err != nil {
+			log.Error("s.bgmStatisDBStore error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_charge_statis : %d", len(statisMap))
+
+		// bgm_charge_daily_statis
+		_, err = s.dateStatisInsert(c, dateStatis.daily, _bgmDailyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_charge_daily_statis : %d", len(dateStatis.daily))
+
+		// bgm_charge_weekly_statis
+		_, err = s.dateStatisInsert(c, dateStatis.weekly, _bgmWeeklyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_charge_weekly_statis : %d", len(dateStatis.weekly))
+
+		// bgm_charge_monthly_statis
+		_, err = s.dateStatisInsert(c, dateStatis.monthly, _bgmMonthlyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert bgm_charge_monthly_statis : %d", len(dateStatis.monthly))
+		return
+	})
+
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+
+	if err = writeGroup.Wait(); err != nil {
+		log.Error("run writeGroup.Wait error(%v)", err)
+	}
+	return
+}

app/job/main/growup/service/charge/run_column.go

@@ -0,0 +1,232 @@
+package charge
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	_cmWeeklyCharge  = "column_weekly_charge"
+	_cmMonthlyCharge = "column_monthly_charge"
+
+	_cmDailyStatis   = "column_charge_daily_statis"
+	_cmWeeklyStatis  = "column_charge_weekly_statis"
+	_cmMonthlyStatis = "column_charge_monthly_statis"
+)
+
+func (s *Service) runColumn(c context.Context, date time.Time) (err error) {
+	startWeeklyDate = getStartWeeklyDate(date)
+	startMonthlyDate = getStartMonthlyDate(date)
+	var (
+		readGroup     errgroup.Group
+		sourceCh      = make(chan []*model.Column, 1000)
+		cmCh          = make(chan []*model.Column, 1000)
+		dailyStatisCh = make(chan []*model.Column, 1000)
+	)
+	readGroup.Go(func() (err error) {
+		err = s.columnCharges(c, date, sourceCh)
+		if err != nil {
+			log.Error("s.columnCharges error(%v)", err)
+			return
+		}
+		log.Info("read column_daily_charge finished")
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(cmCh)
+			close(dailyStatisCh)
+		}()
+		for charges := range sourceCh {
+			cmCh <- charges
+			dailyStatisCh <- charges
+		}
+		return
+	})
+
+	var (
+		weeklyMap  map[int64]*model.Column
+		monthlyMap map[int64]*model.Column
+		statisMap  map[int64]*model.ColumnStatis
+		weeklyCh   = make(chan map[int64]*model.Column, 1)
+		monthlyCh  = make(chan map[int64]*model.Column, 1)
+	)
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(weeklyCh)
+			close(monthlyCh)
+		}()
+		weeklyMap, monthlyMap, statisMap, err = s.handleColumn(c, date, cmCh)
+		if err != nil {
+			log.Error("s.handleColumn error(%v)", err)
+			return
+		}
+		weeklyCh <- weeklyMap
+		monthlyCh <- monthlyMap
+		log.Info("handleColumn finished")
+		return
+	})
+
+	var (
+		dateStatis = &SectionEntries{}
+		cmDaily    = make(chan []*model.Archive, 2000)
+		cmWeekly   = make(chan []*model.Archive, 1)
+		cmMonthly  = make(chan []*model.Archive, 1)
+	)
+	readGroup.Go(func() (err error) {
+		defer close(cmDaily)
+		for cms := range dailyStatisCh {
+			cmDaily <- transCm2Archive(cms)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		defer close(cmWeekly)
+		cmWeekly <- transCmMap2Archive(<-weeklyCh)
+		return
+	})
+	readGroup.Go(func() (err error) {
+		defer close(cmMonthly)
+		cmMonthly <- transCmMap2Archive(<-monthlyCh)
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		dateStatis.daily, err = s.handleDateStatis(c, cmDaily, date, _cmDailyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _cmDailyStatis, err)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		dateStatis.weekly, err = s.handleDateStatis(c, cmWeekly, startWeeklyDate, _cmWeeklyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _cmWeeklyStatis, err)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		dateStatis.monthly, err = s.handleDateStatis(c, cmMonthly, startMonthlyDate, _cmMonthlyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _cmMonthlyStatis, err)
+		}
+		return
+	})
+
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	{
+		if len(weeklyMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_weekly_charge")
+			return
+		}
+		if len(monthlyMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_monthly_charge")
+			return
+		}
+		if len(statisMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_charge_statis")
+			return
+		}
+		if len(dateStatis.daily) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_charge_daily_statis")
+			return
+		}
+		if len(dateStatis.weekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_charge_weekly_statis")
+			return
+		}
+		if len(dateStatis.monthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 column_charge_monthly_statis")
+			return
+		}
+	}
+
+	// persist
+	var writeGroup errgroup.Group
+	// column_weekly_charge
+	writeGroup.Go(func() (err error) {
+		err = s.cmDBStore(c, _cmWeeklyCharge, weeklyMap)
+		if err != nil {
+			log.Error("s.cmDBStore column_weekly_charge error(%v)", err)
+			return
+		}
+		log.Info("insert column_weekly_charge : %d", len(weeklyMap))
+
+		// column_monthly_charge
+		err = s.cmDBStore(c, _cmMonthlyCharge, monthlyMap)
+		if err != nil {
+			log.Error("s.cmDBStore column_monthly_charge error(%v)", err)
+			return
+		}
+		log.Info("insert column_monthly_charge : %d", len(monthlyMap))
+
+		// column_charge_statis
+		err = s.cmStatisDBStore(c, statisMap)
+		if err != nil {
+			log.Error("s.cmStatisDBStore error(%v)", err)
+			return
+		}
+		log.Info("insert column_charge_statis : %d", len(statisMap))
+
+		// column_charge_daily_statis
+		_, err = s.dateStatisInsert(c, dateStatis.daily, _cmDailyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert column_charge_daily_statis : %d", len(dateStatis.daily))
+
+		// column_charge_weekly_statis
+		_, err = s.dateStatisInsert(c, dateStatis.weekly, _cmWeeklyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert column_charge_weekly_statis : %d", len(dateStatis.weekly))
+
+		// column_charge_monthly_statis
+		_, err = s.dateStatisInsert(c, dateStatis.monthly, _cmMonthlyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert column_charge_monthly_statis : %d", len(dateStatis.monthly))
+		return
+	})
+
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+
+	if err = writeGroup.Wait(); err != nil {
+		log.Error("run writeGroup.Wait error(%v)", err)
+	}
+	return
+}

app/job/main/growup/service/charge/run_test.go

@@ -0,0 +1,369 @@
+package charge
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+	xtime "go-common/library/time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_AvCharge(t *testing.T) {
+	Convey("Test av weekly and monthly charge\n", t, func() {
+		c := context.Background()
+		deleteAll(c)
+		ac1 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1101, 110, 10)
+		ac2 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1102, 110, 10)
+		d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
+		err := s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvWeeklyCharge(c, ac1, 30, 10, 10)
+		checkAvWeeklyCharge(c, ac2, 30, 10, 10)
+		checkAvMonthlyCharge(c, ac1, 30, 10, 10)
+		checkAvMonthlyCharge(c, ac2, 30, 10, 10)
+
+		ac1 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1101, 110, 10)
+		ac2 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1102, 110, 20)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvWeeklyCharge(c, ac1, 60, 20, 20)
+		checkAvWeeklyCharge(c, ac2, 90, 30, 30)
+		checkAvMonthlyCharge(c, ac1, 60, 20, 20)
+		checkAvMonthlyCharge(c, ac2, 90, 30, 30)
+
+		ac1 = insertAvDailyCharge(c, "2018-06-05", "2018-06-01 15:02:03", 1101, 110, 15)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-06-05", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvWeeklyCharge(c, ac1, 45, 15, 15)
+		checkAvMonthlyCharge(c, ac1, 105, 35, 35)
+	})
+}
+
+func Test_AvChargeDateStatis(t *testing.T) {
+	Convey("Test av charge date statis\n", t, func() {
+		c := context.Background()
+		deleteAll(c)
+		insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1101, 110, 10)
+		insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1102, 110, 10)
+		insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1103, 110, 3500)
+		d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
+		err := s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvChargeDateStatis(c, 0, 2, 1, 3520, 12, d, "av_charge_daily_statis")
+		checkAvChargeDateStatis(c, 1, 1, 1, 3520, 12, d, "av_charge_daily_statis")
+		weekD := getStartWeeklyDate(d)
+		checkAvChargeDateStatis(c, 0, 2, 1, 3520, 12, weekD, "av_charge_weekly_statis")
+		checkAvChargeDateStatis(c, 1, 1, 1, 3520, 12, weekD, "av_charge_weekly_statis")
+		monthD := getStartMonthlyDate(d)
+		checkAvChargeDateStatis(c, 0, 2, 1, 3520, 12, monthD, "av_charge_monthly_statis")
+		checkAvChargeDateStatis(c, 1, 1, 1, 3520, 12, monthD, "av_charge_monthly_statis")
+
+		insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1101, 110, 1000)
+		insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1102, 110, 1000)
+		insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1103, 110, 4000)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvChargeDateStatis(c, 0, 2, 1, 6000, 12, d, "av_charge_daily_statis")
+		checkAvChargeDateStatis(c, 1, 1, 1, 6000, 12, d, "av_charge_daily_statis")
+		weekD = getStartWeeklyDate(d)
+		charge := 6000 + 3520
+		checkAvChargeDateStatis(c, 0, 2, 1, charge, 12, weekD, "av_charge_weekly_statis")
+		checkAvChargeDateStatis(c, 1, 0, 1, charge, 12, weekD, "av_charge_weekly_statis")
+		checkAvChargeDateStatis(c, 2, 1, 1, charge, 12, weekD, "av_charge_weekly_statis")
+		monthD = getStartMonthlyDate(d)
+		checkAvChargeDateStatis(c, 0, 2, 1, charge, 12, monthD, "av_charge_monthly_statis")
+		checkAvChargeDateStatis(c, 1, 0, 1, charge, 12, monthD, "av_charge_monthly_statis")
+		checkAvChargeDateStatis(c, 2, 1, 1, charge, 12, monthD, "av_charge_monthly_statis")
+
+		insertAvDailyCharge(c, "2018-06-10", "2018-06-01 15:02:03", 1101, 110, 15000)
+		insertAvDailyCharge(c, "2018-06-10", "2018-06-01 15:02:03", 1102, 110, 15000)
+		insertAvDailyCharge(c, "2018-06-10", "2018-06-01 15:02:03", 1103, 110, 40000)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-06-10", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvChargeDateStatis(c, 0, 0, 1, 70000, 12, d, "av_charge_daily_statis")
+		checkAvChargeDateStatis(c, 1, 0, 1, 70000, 12, d, "av_charge_daily_statis")
+		checkAvChargeDateStatis(c, 3, 2, 1, 70000, 12, d, "av_charge_daily_statis")
+		checkAvChargeDateStatis(c, 4, 1, 1, 70000, 12, d, "av_charge_daily_statis")
+		weekD = getStartWeeklyDate(d)
+		checkAvChargeDateStatis(c, 0, 0, 1, 70000, 12, weekD, "av_charge_weekly_statis")
+		checkAvChargeDateStatis(c, 1, 0, 1, 70000, 12, weekD, "av_charge_weekly_statis")
+		checkAvChargeDateStatis(c, 3, 2, 1, 70000, 12, weekD, "av_charge_weekly_statis")
+		checkAvChargeDateStatis(c, 4, 1, 1, 70000, 12, weekD, "av_charge_weekly_statis")
+		monthD = getStartMonthlyDate(d)
+		checkAvChargeDateStatis(c, 0, 0, 1, 79520, 12, monthD, "av_charge_monthly_statis")
+		checkAvChargeDateStatis(c, 1, 0, 1, 79520, 12, monthD, "av_charge_monthly_statis")
+		checkAvChargeDateStatis(c, 3, 2, 1, 79520, 12, monthD, "av_charge_monthly_statis")
+		checkAvChargeDateStatis(c, 4, 1, 1, 79520, 12, monthD, "av_charge_monthly_statis")
+
+		insertAvDailyCharge(c, "2018-07-10", "2018-06-01 15:02:03", 1101, 110, 100)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-07-10", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		monthD = getStartMonthlyDate(d)
+		checkAvChargeDateStatis(c, 0, 1, 1, 100, 12, monthD, "av_charge_monthly_statis")
+		checkAvChargeDateStatis(c, 1, 0, 1, 100, 12, monthD, "av_charge_monthly_statis")
+	})
+}
+
+func Test_AvChargeStatis(t *testing.T) {
+	Convey("Test av charge statis\n", t, func() {
+		c := context.Background()
+		deleteAll(c)
+		ac1 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1101, 110, 1000)
+		ac2 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 15:02:03", 1102, 110, 1000)
+		d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
+		err := s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvChargeStatis(c, ac1, 1000)
+		checkAvChargeStatis(c, ac2, 1000)
+
+		ac1 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1101, 110, 1000)
+		ac2 = insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1102, 110, 1000)
+		ac3 := insertAvDailyCharge(c, "2018-06-02", "2018-06-01 15:02:03", 1103, 110, 1000)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvChargeStatis(c, ac1, 2000)
+		checkAvChargeStatis(c, ac2, 2000)
+		checkAvChargeStatis(c, ac3, 1000)
+
+		ac2 = insertAvDailyCharge(c, "2018-07-03", "2018-06-01 15:02:03", 1102, 110, 1000)
+		ac3 = insertAvDailyCharge(c, "2018-07-03", "2018-06-01 15:02:03", 1103, 110, 1000)
+		ac4 := insertAvDailyCharge(c, "2018-07-03", "2018-06-01 15:02:03", 1104, 110, 1000)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-07-03", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkAvChargeStatis(c, ac1, 2000)
+		checkAvChargeStatis(c, ac2, 3000)
+		checkAvChargeStatis(c, ac3, 2000)
+		checkAvChargeStatis(c, ac4, 1000)
+	})
+}
+
+// up_charge
+func Test_UpCharge(t *testing.T) {
+	Convey("Test up daily weekly monthly charge\n", t, func() {
+		c := context.Background()
+		deleteAll(c)
+		ac1 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 20:02:03", 1101, 110, 10)
+		ac2 := insertAvDailyCharge(c, "2018-06-01", "2018-06-01 20:02:03", 1102, 110, 10)
+		d, _ := time.ParseInLocation("2006-01-02", "2018-06-01", time.Local)
+		err := s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		acs := []*model.AvCharge{ac1, ac2}
+		checkUpDailyCharge(c, acs, 2, 20, "up_daily_charge", "2018-06-01")
+		checkUpWeeklyCharge(c, acs, 2, 20)
+		checkUpMonthlyCharge(c, acs, 2, 20)
+
+		ac11 := insertAvDailyCharge(c, "2018-06-02", "2018-06-01 20:02:03", 1101, 110, 10)
+		ac21 := insertAvDailyCharge(c, "2018-06-02", "2018-06-01 20:02:03", 1102, 110, 10)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-06-02", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		acs = append(acs, []*model.AvCharge{ac11, ac21}...)
+		checkUpDailyCharge(c, []*model.AvCharge{ac11, ac21}, 2, 20, "up_daily_charge", "2018-06-01")
+		checkUpWeeklyCharge(c, acs, 2, 40)
+		checkUpMonthlyCharge(c, acs, 2, 40)
+
+		ac12 := insertAvDailyCharge(c, "2018-06-05", "2018-06-01 20:02:03", 1101, 110, 10)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-06-05", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		acs = append(acs, []*model.AvCharge{ac12}...)
+		checkUpDailyCharge(c, []*model.AvCharge{ac12}, 1, 10, "up_daily_charge", "2018-06-05")
+		checkUpWeeklyCharge(c, []*model.AvCharge{ac12}, 1, 10)
+		checkUpMonthlyCharge(c, acs, 2, 50)
+
+		ac13 := insertAvDailyCharge(c, "2018-07-05", "2018-06-01 20:02:03", 1101, 110, 10)
+		d, _ = time.ParseInLocation("2006-01-02", "2018-07-05", time.Local)
+		err = s.runVideo(c, d, nil)
+		So(err, ShouldBeNil)
+		checkUpDailyCharge(c, []*model.AvCharge{ac13}, 1, 10, "up_daily_charge", "2018-07-05")
+		checkUpWeeklyCharge(c, []*model.AvCharge{ac13}, 1, 10)
+		checkUpMonthlyCharge(c, []*model.AvCharge{ac13}, 1, 10)
+	})
+}
+
+func checkAvWeeklyCharge(c context.Context, ac *model.AvCharge, totalPlayCount, incCharge, totalCharge int64) {
+	avWeeklyCharge, err := s.GetAvCharge(c, startWeeklyDate, s.dao.AvWeeklyCharge)
+	So(err, ShouldBeNil)
+	var aw *model.AvCharge
+	for _, av := range avWeeklyCharge {
+		if av.AvID == ac.AvID {
+			aw = av
+			break
+		}
+	}
+	So(ac.AvID, ShouldEqual, aw.AvID)
+	So(ac.MID, ShouldEqual, aw.MID)
+	So(ac.TagID, ShouldEqual, aw.TagID)
+	So(ac.IsOriginal, ShouldEqual, aw.IsOriginal)
+	So(ac.UploadTime, ShouldEqual, aw.UploadTime)
+	So(totalPlayCount, ShouldEqual, aw.TotalPlayCount)
+	So(totalCharge, ShouldEqual, aw.TotalCharge)
+	So(incCharge, ShouldEqual, aw.IncCharge)
+	So(xtime.Time(getStartWeeklyDate(ac.Date.Time()).Unix()), ShouldEqual, aw.Date)
+}
+
+func checkAvMonthlyCharge(c context.Context, ac *model.AvCharge, totalPlayCount, incCharge, totalCharge int64) {
+	avCharge, err := s.GetAvCharge(c, startMonthlyDate, s.dao.AvMonthlyCharge)
+	So(err, ShouldBeNil)
+	var aw *model.AvCharge
+	for _, av := range avCharge {
+		if av.AvID == ac.AvID {
+			aw = av
+			break
+		}
+	}
+	So(ac.AvID, ShouldEqual, aw.AvID)
+	So(ac.MID, ShouldEqual, aw.MID)
+	So(ac.TagID, ShouldEqual, aw.TagID)
+	So(ac.IsOriginal, ShouldEqual, aw.IsOriginal)
+	So(ac.UploadTime, ShouldEqual, aw.UploadTime)
+	So(totalPlayCount, ShouldEqual, aw.TotalPlayCount)
+	So(totalCharge, ShouldEqual, aw.TotalCharge)
+	So(incCharge, ShouldEqual, aw.IncCharge)
+	So(xtime.Time(getStartMonthlyDate(ac.Date.Time()).Unix()), ShouldEqual, aw.Date)
+}
+
+func checkAvChargeStatis(c context.Context, ac *model.AvCharge, totalCharge int64) {
+	ai := &model.AvChargeStatis{}
+	err := s.dao.QueryRow(c, fmt.Sprintf("select av_id,mid,tag_id,is_original,upload_time,total_charge from av_charge_statis where av_id = %d", ac.AvID)).Scan(
+		&ai.AvID, &ai.MID, &ai.TagID, &ai.IsOriginal, &ai.UploadTime, &ai.TotalCharge)
+	So(err, ShouldBeNil)
+	So(ac.AvID, ShouldEqual, ai.AvID)
+	So(ac.MID, ShouldEqual, ai.MID)
+	So(ac.TagID, ShouldEqual, ai.TagID)
+	So(ac.IsOriginal, ShouldEqual, ai.IsOriginal)
+	So(ac.UploadTime, ShouldEqual, ai.UploadTime)
+	So(totalCharge, ShouldEqual, ai.TotalCharge)
+}
+
+func checkUpDailyCharge(c context.Context, acs []*model.AvCharge, count, total int64, table, date string) {
+	if len(acs) == 0 {
+		return
+	}
+	if date == "" {
+		date = acs[0].Date.Time().Format(_layout)
+	}
+	var incCharge int64
+	for _, ac := range acs {
+		incCharge += ac.IncCharge
+	}
+	up := &model.UpCharge{}
+	err := s.dao.QueryRow(c, fmt.Sprintf("select av_count,inc_charge, total_charge from %s where mid = %d and date = '%s'", table, acs[0].MID, date)).Scan(
+		&up.AvCount, &up.IncCharge, &up.TotalCharge)
+	So(err, ShouldBeNil)
+	So(total, ShouldEqual, up.TotalCharge)
+	So(incCharge, ShouldEqual, up.IncCharge)
+}
+
+func checkUpWeeklyCharge(c context.Context, avs []*model.AvCharge, count, total int64) {
+	m := make(map[string][]*model.AvCharge)
+	for _, av := range avs {
+		d := getStartWeeklyDate(av.Date.Time()).Format(_layout)
+		if _, ok := m[d]; !ok {
+			m[d] = make([]*model.AvCharge, 0)
+		}
+		m[d] = append(m[d], av)
+	}
+	for date, avs := range m {
+		checkUpDailyCharge(c, avs, count, total, "up_weekly_charge", date)
+	}
+}
+
+func checkUpMonthlyCharge(c context.Context, avs []*model.AvCharge, count, total int64) {
+	m := make(map[string][]*model.AvCharge)
+	for _, av := range avs {
+		d := getStartMonthlyDate(av.Date.Time()).Format(_layout)
+		if _, ok := m[d]; !ok {
+			m[d] = make([]*model.AvCharge, 0)
+		}
+		m[d] = append(m[d], av)
+	}
+	for date, avs := range m {
+		checkUpDailyCharge(c, avs, count, total, "up_monthly_charge", date)
+	}
+}
+
+func checkAvChargeDateStatis(c context.Context, section, avs, categoryID, charge, count int, d time.Time, table string) {
+	xd := xtime.Time(d.Unix())
+	ads := &model.DateStatis{}
+	err := s.dao.QueryRow(c, fmt.Sprintf("select avs,charge,cdate from %s where money_section = %d and category_id = %d and cdate = '%s'", table, section, categoryID, d.Format(_layout))).Scan(
+		&ads.Count, &ads.Charge, &ads.CDate)
+	So(err, ShouldBeNil)
+	So(charge, ShouldEqual, ads.Charge)
+	So(xd, ShouldEqual, ads.CDate)
+	So(avs, ShouldEqual, ads.Count)
+	var ccount int64
+	err = s.dao.QueryRow(c, fmt.Sprintf("select count(*) from %s where category_id = %d and cdate = '%s'", table, categoryID, d.Format(_layout))).Scan(&ccount)
+	So(err, ShouldBeNil)
+	So(count, ShouldEqual, ccount)
+}
+
+func deleteAll(c context.Context) {
+	s.dao.Exec(c, "truncate av_daily_charge_06")
+	s.dao.Exec(c, "truncate av_daily_charge_07")
+	s.dao.Exec(c, "truncate av_weekly_charge")
+	s.dao.Exec(c, "truncate av_monthly_charge")
+	s.dao.Exec(c, "truncate av_charge_statis")
+	s.dao.Exec(c, "truncate up_daily_charge")
+	s.dao.Exec(c, "truncate up_weekly_charge")
+	s.dao.Exec(c, "truncate up_monthly_charge")
+	s.dao.Exec(c, "truncate av_charge_daily_statis")
+	s.dao.Exec(c, "truncate av_charge_weekly_statis")
+	s.dao.Exec(c, "truncate av_charge_monthly_statis")
+}
+
+func getAvDailyChargeStruct(date string, uploadDate string, avID, mid, charge int64) *model.AvCharge {
+	d, _ := time.ParseInLocation(_layout, date, time.Local)
+	upD, _ := time.ParseInLocation(_layoutSec, uploadDate, time.Local)
+	ac := &model.AvCharge{
+		AvID:           avID,
+		MID:            mid,
+		TagID:          1,
+		IsOriginal:     1,
+		DanmakuCount:   charge,
+		CommentCount:   charge,
+		CollectCount:   charge,
+		CoinCount:      charge,
+		ShareCount:     charge,
+		ElecPayCount:   charge,
+		TotalPlayCount: charge * int64(3),
+		WebPlayCount:   charge,
+		AppPlayCount:   charge,
+		H5PlayCount:    charge,
+		LvUnknown:      charge,
+		Lv0:            charge,
+		Lv1:            charge,
+		Lv2:            charge,
+		Lv3:            charge,
+		Lv4:            charge,
+		Lv5:            charge,
+		Lv6:            charge,
+		VScore:         charge,
+		IncCharge:      charge,
+		TotalCharge:    0,
+		Date:           xtime.Time(d.Unix()),
+		UploadTime:     xtime.Time(upD.Unix()),
+		DBState:        _dbInsert,
+	}
+	return ac
+}
+
+func insertAvDailyCharge(c context.Context, date string, uploadDate string, avID, mid, charge int64) *model.AvCharge {
+	s.dao.Exec(c, fmt.Sprintf("insert into task_status(type,date,status) values(1,'%s',1) on duplicate key update date=values(date)", date))
+	ac := getAvDailyChargeStruct(date, uploadDate, avID, mid, charge)
+	_, err := s.avChargeBatchInsert(c, []*model.AvCharge{ac}, fmt.Sprintf("av_daily_charge_%s", strings.Split(date, "-")[1]))
+	So(err, ShouldBeNil)
+	return ac
+}

app/job/main/growup/service/charge/run_video.go

@@ -0,0 +1,299 @@
+package charge
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	_avChargeDailyStatis   = "av_charge_daily_statis"
+	_avChargeWeeklyStatis  = "av_charge_weekly_statis"
+	_avChargeMonthlyStatis = "av_charge_monthly_statis"
+
+	_avWeeklyCharge  = "av_weekly_charge"
+	_avMonthlyCharge = "av_monthly_charge"
+)
+
+func (s *Service) runVideo(c context.Context, date time.Time, avBgmCharge chan []*model.AvCharge) (err error) {
+	startWeeklyDate = getStartWeeklyDate(date)
+	startMonthlyDate = getStartMonthlyDate(date)
+	var (
+		readGroup     errgroup.Group
+		sourceCh      = make(chan []*model.AvCharge, 1000)
+		avChargeCh    = make(chan []*model.AvCharge, 1000)
+		upChargeCh    = make(chan []*model.AvCharge, 1000)
+		dailyStatisCh = make(chan []*model.AvCharge, 1000)
+	)
+	readGroup.Go(func() (err error) {
+		err = s.avDailyCharges(c, date, sourceCh)
+		if err != nil {
+			log.Error("s.avCharge.AvCharges error(%v)", err)
+			return
+		}
+		log.Info("read av_daily_charge finished")
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(avChargeCh)
+			close(upChargeCh)
+			close(dailyStatisCh)
+			close(avBgmCharge)
+		}()
+		for charges := range sourceCh {
+			avChargeCh <- charges
+			upChargeCh <- charges
+			dailyStatisCh <- charges
+			avBgmCharge <- charges
+		}
+		return
+	})
+
+	var (
+		weeklyChargeMap  map[int64]*model.AvCharge
+		monthlyChargeMap map[int64]*model.AvCharge
+		chargeStatisMap  map[int64]*model.AvChargeStatis
+		weeklyCh         = make(chan map[int64]*model.AvCharge, 1)
+		monthlyCh        = make(chan map[int64]*model.AvCharge, 1)
+	)
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(weeklyCh)
+			close(monthlyCh)
+		}()
+		weeklyChargeMap, monthlyChargeMap, chargeStatisMap, err = s.handleAvCharge(c, date, avChargeCh)
+		if err != nil {
+			log.Error("s.handleAvCharge error(%v)", err)
+			return
+		}
+		weeklyCh <- weeklyChargeMap
+		monthlyCh <- monthlyChargeMap
+		log.Info("handleAvCharge finished")
+		return
+	})
+
+	var (
+		dateStatis = &SectionEntries{}
+		avDaily    = make(chan []*model.Archive, 2000)
+		avWeekly   = make(chan []*model.Archive, 1)
+		avMonthly  = make(chan []*model.Archive, 1)
+	)
+	readGroup.Go(func() (err error) {
+		defer close(avDaily)
+		for avs := range dailyStatisCh {
+			avDaily <- transAv2Archive(avs)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		defer close(avWeekly)
+		avWeekly <- transAvMap2Archive(<-weeklyCh)
+		return
+	})
+	readGroup.Go(func() (err error) {
+		defer close(avMonthly)
+		avMonthly <- transAvMap2Archive(<-monthlyCh)
+		return
+	})
+
+	readGroup.Go(func() (err error) {
+		dateStatis.daily, err = s.handleDateStatis(c, avDaily, date, _avChargeDailyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _avChargeDailyStatis, err)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		dateStatis.weekly, err = s.handleDateStatis(c, avWeekly, startWeeklyDate, _avChargeWeeklyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _avChargeWeeklyStatis, err)
+		}
+		return
+	})
+	readGroup.Go(func() (err error) {
+		dateStatis.monthly, err = s.handleDateStatis(c, avMonthly, startMonthlyDate, _avChargeMonthlyStatis)
+		if err != nil {
+			log.Error("s.handleDateStatis(%s) error(%v)", _avChargeMonthlyStatis, err)
+		}
+		return
+	})
+
+	var (
+		daily   map[int64]*model.UpCharge
+		weekly  map[int64]*model.UpCharge
+		monthly map[int64]*model.UpCharge
+	)
+	readGroup.Go(func() (err error) {
+		daily, weekly, monthly, err = s.calUpCharge(c, date, upChargeCh)
+		if err != nil {
+			log.Error("s.calUpCharge error(%v)", err)
+			return
+		}
+		log.Info("s.calUpCharge finished")
+		return
+	})
+
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	{
+		if len(weeklyChargeMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_weekly_charge")
+			return
+		}
+		if len(monthlyChargeMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_monthly_charge")
+			return
+		}
+		if len(chargeStatisMap) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_charge_statis")
+			return
+		}
+		if len(daily) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_daily_charge")
+			return
+		}
+		if len(weekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_weekly_charge")
+			return
+		}
+		if len(monthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 up_monthly_charge")
+			return
+		}
+		if len(dateStatis.daily) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_charge_daily_statis")
+			return
+		}
+		if len(dateStatis.weekly) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_charge_weekly_statis")
+			return
+		}
+		if len(dateStatis.monthly) == 0 {
+			err = fmt.Errorf("Error: insert 0 av_charge_monthly_statis")
+			return
+		}
+	}
+
+	// persist
+	var writeGroup errgroup.Group
+	// av_weekly_charge
+	writeGroup.Go(func() (err error) {
+		err = s.AvChargeDBStore(c, _avWeeklyCharge, weeklyChargeMap)
+		if err != nil {
+			log.Error("s.AvChargeDBStore av_weekly_charge error(%v)", err)
+			return
+		}
+		log.Info("insert av_weekly_charge : %d", len(weeklyChargeMap))
+		return
+	})
+
+	// av_monthly_charge
+	writeGroup.Go(func() (err error) {
+		err = s.AvChargeDBStore(c, _avMonthlyCharge, monthlyChargeMap)
+		if err != nil {
+			log.Error("s.AvChargeDBStore av_monthly_charge error(%v)", err)
+			return
+		}
+		log.Info("insert av_monthly_charge : %d", len(monthlyChargeMap))
+
+		// av_charge_statis
+		err = s.AvChargeStatisDBStore(c, chargeStatisMap)
+		if err != nil {
+			log.Error("s.AvChargeStatisDBStore error(%v)", err)
+			return
+		}
+		log.Info("insert av_charge_statis : %d", len(chargeStatisMap))
+
+		// up_daily_charge
+		err = s.BatchInsertUpCharge(c, "up_daily_charge", daily)
+		if err != nil {
+			log.Error("s.BatchInsertUpDailyCharge error(%v)", err)
+			return
+		}
+		log.Info("insert up_daily_charge : %d", len(daily))
+
+		// up_weekly_charge
+		err = s.BatchInsertUpCharge(c, "up_weekly_charge", weekly)
+		if err != nil {
+			log.Error("s.BatchInsertUpWeeklyCharge error(%v)", err)
+			return
+		}
+		log.Info("insert up_weekly_charge : %d", len(weekly))
+
+		// up_monthly_charge
+		err = s.BatchInsertUpCharge(c, "up_monthly_charge", monthly)
+		if err != nil {
+			log.Error("s.BatchInsertUpMonthlyCharge error(%v)", err)
+			return
+		}
+		log.Info("insert up_monthly_charge : %d", len(monthly))
+
+		// av_charge_daily_statis
+		_, err = s.dateStatisInsert(c, dateStatis.daily, _avChargeDailyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert av_charge_daily_statis : %d", len(dateStatis.daily))
+
+		// av_charge_weekly_statis
+		_, err = s.dateStatisInsert(c, dateStatis.weekly, _avChargeWeeklyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert av_charge_weekly_statis : %d", len(dateStatis.weekly))
+
+		// av_charge_monthly_statis
+		_, err = s.dateStatisInsert(c, dateStatis.monthly, _avChargeMonthlyStatis)
+		if err != nil {
+			log.Error("s.dateStatisInsert error(%v)", err)
+			return
+		}
+		log.Info("insert av_charge_monthly_statis : %d", len(dateStatis.monthly))
+		return
+	})
+
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+	//
+	//	writeGroup.Go(func() (err error) {
+	//		return
+	//	})
+
+	if err = writeGroup.Wait(); err != nil {
+		log.Error("run writeGroup.Wait error(%v)", err)
+	}
+	return
+}

app/job/main/growup/service/charge/service.go

@@ -0,0 +1,79 @@
+package charge
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/growup/conf"
+	chargeD "go-common/app/job/main/growup/dao/charge"
+	"go-common/app/job/main/growup/dao/email"
+	"go-common/app/job/main/growup/service"
+	"go-common/app/job/main/growup/service/ctrl"
+	"go-common/library/log"
+)
+
+// Service struct
+type Service struct {
+	conf  *conf.Config
+	dao   *chargeD.Dao
+	email *email.Dao
+}
+
+// New fn
+func New(c *conf.Config, executor ctrl.Executor) (s *Service) {
+	s = &Service{
+		conf:  c,
+		dao:   chargeD.New(c),
+		email: email.New(c),
+	}
+	log.Info("charge service start")
+	executor.Submit(
+		s.calDailyCreativeCharge,
+		s.checkColumnDailyCharge,
+	)
+	return s
+}
+
+func (s *Service) calDailyCreativeCharge(ctx context.Context) {
+	for {
+		time.Sleep(service.NextDay(17, 0, 0))
+		log.Info("calDailyCreativeCharge begin:%v", time.Now().Format("2006-01-02 15:04:05"))
+		now := time.Now()
+		date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local)
+		err := s.RunAndSendMail(context.TODO(), date)
+		if err != nil {
+			log.Error("s.RunAndSendMail error(%v)", err)
+		}
+		log.Info("calDailyCreativeCharge end:%v", time.Now().Format("2006-01-02 15:04:05"))
+	}
+}
+
+func (s *Service) checkColumnDailyCharge(ctx context.Context) {
+	for {
+		time.Sleep(service.NextDay(13, 0, 0))
+		msg := ""
+		log.Info("checkColumnDailyCharge begin:%v", time.Now().Format("2006-01-02 15:04:05"))
+		now := time.Now()
+		date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local)
+		err := s.CheckTaskColumn(context.TODO(), date.Format(_layout))
+		if err != nil {
+			log.Error("s.CheckTaskColumn error(%v)", err)
+		}
+		if err != nil {
+			msg = fmt.Sprintf("CheckTaskColumn error(%v)", err)
+		} else {
+			msg = "Success"
+		}
+		err = s.email.SendMail(date, msg, "创作激励专栏补贴数据%d年%d月%d日", "shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com")
+		if err != nil {
+			log.Error("s.email.SendMail error(%v)", err)
+		}
+		log.Info("checkColumnDailyCharge end:%v", time.Now().Format("2006-01-02 15:04:05"))
+	}
+}
+
+// Ping check dao health.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}

app/job/main/growup/service/charge/service_test.go

@@ -0,0 +1,33 @@
+package charge
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/growup/conf"
+	"go-common/app/job/main/growup/service/ctrl"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../../cmd/growup-job.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	if s == nil {
+		s = New(conf.Conf, ctrl.NewUnboundedExecutor())
+	}
+}
+
+func TestPing(t *testing.T) {
+	Convey("Test_Ping", t, func() {
+		err := s.Ping(context.Background())
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/growup/service/charge/trans.go

@@ -0,0 +1,83 @@
+package charge
+
+import (
+	model "go-common/app/job/main/growup/model/charge"
+)
+
+func transAv2Archive(avs []*model.AvCharge) (archs []*model.Archive) {
+	archs = make([]*model.Archive, 0, len(avs))
+	for _, av := range avs {
+		archs = append(archs, &model.Archive{
+			ID:        av.AvID,
+			IncCharge: av.IncCharge,
+			TagID:     av.TagID,
+			Date:      av.Date,
+		})
+	}
+	return
+}
+
+func transAvMap2Archive(avs map[int64]*model.AvCharge) (archs []*model.Archive) {
+	archs = make([]*model.Archive, 0, len(avs))
+	for _, av := range avs {
+		archs = append(archs, &model.Archive{
+			ID:        av.AvID,
+			IncCharge: av.IncCharge,
+			TagID:     av.TagID,
+			Date:      av.Date,
+		})
+	}
+	return
+}
+
+func transCm2Archive(cms []*model.Column) (archs []*model.Archive) {
+	archs = make([]*model.Archive, 0, len(cms))
+	for _, cm := range cms {
+		archs = append(archs, &model.Archive{
+			ID:        cm.AID,
+			IncCharge: cm.IncCharge,
+			TagID:     cm.TagID,
+			Date:      cm.Date,
+		})
+	}
+	return
+}
+
+func transCmMap2Archive(cms map[int64]*model.Column) (archs []*model.Archive) {
+	archs = make([]*model.Archive, 0, len(cms))
+	for _, cm := range cms {
+		archs = append(archs, &model.Archive{
+			ID:        cm.AID,
+			IncCharge: cm.IncCharge,
+			TagID:     cm.TagID,
+			Date:      cm.Date,
+		})
+	}
+	return
+}
+
+func transBgm2Archive(bgms []*model.BgmCharge) (archs []*model.Archive) {
+	archs = make([]*model.Archive, 0, len(bgms))
+	for _, bgm := range bgms {
+		archs = append(archs, &model.Archive{
+			ID:        bgm.SID,
+			IncCharge: bgm.IncCharge,
+			TagID:     0,
+			Date:      bgm.Date,
+		})
+	}
+	return
+}
+
+func transBgmMap2Archive(bgms map[string]*model.BgmCharge) (archs []*model.Archive) {
+	archs = make([]*model.Archive, 0, len(bgms))
+	for _, bgm := range bgms {
+		archs = append(archs, &model.Archive{
+			ID:        bgm.SID,
+			IncCharge: bgm.IncCharge,
+			TagID:     0,
+			Date:      bgm.Date,
+		})
+	}
+	return
+}

app/job/main/growup/service/charge/up_charge.go

@@ -0,0 +1,170 @@
+package charge
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	model "go-common/app/job/main/growup/model/charge"
+
+	xtime "go-common/library/time"
+)
+
+func (s *Service) calUpCharge(c context.Context, t time.Time, avc chan []*model.AvCharge) (daily, weekly, monthly map[int64]*model.UpCharge, err error) {
+	od, err := s.getUpDailyCharge(c, t)
+	if err != nil {
+		return
+	}
+	daily = calUpDailyCharge(od, avc)
+
+	weekly, err = s.getUpWeeklyCharge(c)
+	if err != nil {
+		return
+	}
+	// group by weekly
+	upChargeStat(startWeeklyDate, weekly, daily)
+
+	monthly, err = s.getUpMonthlyCharge(c)
+	if err != nil {
+		return
+	}
+	// group by monthly
+	upChargeStat(startMonthlyDate, monthly, daily)
+
+	return
+}
+
+// 1 od: old daily up charges
+func (s *Service) getUpDailyCharge(c context.Context, t time.Time) (od map[int64]*model.UpCharge, err error) {
+	// old: yesterday
+	old := t.AddDate(0, 0, -1)
+	return s.getUpCharges(c, "up_daily_charge", old.Format("2006-01-02"))
+}
+
+// 2 ow: old weekly up charges
+func (s *Service) getUpWeeklyCharge(c context.Context) (ow map[int64]*model.UpCharge, err error) {
+	return s.getUpCharges(c, "up_weekly_charge", startWeeklyDate.Format("2006-01-02"))
+}
+
+// 3 om: old monthly up charges
+func (s *Service) getUpMonthlyCharge(c context.Context) (om map[int64]*model.UpCharge, err error) {
+	return s.getUpCharges(c, "up_monthly_charge", startMonthlyDate.Format("2006-01-02"))
+}
+
+func calUpDailyCharge(od map[int64]*model.UpCharge, avc chan []*model.AvCharge) (mu map[int64]*model.UpCharge) {
+	mu = make(map[int64]*model.UpCharge)
+	for charges := range avc {
+		for _, charge := range charges {
+			if charge.IncCharge <= 0 {
+				continue
+			}
+			// udc: up daily charge
+			if udc, ok := mu[charge.MID]; ok {
+				udc.IncCharge += charge.IncCharge
+				udc.TotalCharge += charge.IncCharge
+			} else {
+				var total int64
+				if o, ok := od[charge.MID]; ok {
+					total = o.TotalCharge
+				}
+				mu[charge.MID] = &model.UpCharge{
+					MID:         charge.MID,
+					IncCharge:   charge.IncCharge,
+					TotalCharge: total + charge.IncCharge,
+					Date:        charge.Date,
+				}
+			}
+		}
+	}
+	return
+}
+
+// os: old weekly/month charge chan, empty maybe
+func upChargeStat(t time.Time, os, daily map[int64]*model.UpCharge) {
+	for mid, ucd := range daily {
+		if charge, ok := os[mid]; ok {
+			// update
+			charge.IncCharge += ucd.IncCharge
+			charge.TotalCharge += ucd.IncCharge
+			charge.Date = xtime.Time(t.Unix())
+		} else {
+			// new
+			os[mid] = &model.UpCharge{
+				MID:         mid,
+				IncCharge:   ucd.IncCharge,
+				TotalCharge: ucd.TotalCharge,
+				Date:        xtime.Time(t.Unix()),
+			}
+		}
+	}
+}
+
+// get up charges by date
+func (s *Service) getUpCharges(c context.Context, table, date string) (m map[int64]*model.UpCharge, err error) {
+	var id int64
+	m = make(map[int64]*model.UpCharge)
+	for {
+		var charges map[int64]*model.UpCharge
+		id, charges, err = s.dao.GetUpCharges(c, table, date, id, 2000)
+		if err != nil {
+			return
+		}
+		if len(charges) == 0 {
+			break
+		}
+		for k, v := range charges {
+			m[k] = v
+		}
+	}
+	return
+}
+
+// BatchInsertUpCharge batch insert up charge
+func (s *Service) BatchInsertUpCharge(c context.Context, table string, us map[int64]*model.UpCharge) (err error) {
+	var (
+		buff    = make([]*model.UpCharge, _batchSize)
+		buffEnd = 0
+	)
+	for _, u := range us {
+		buff[buffEnd] = u
+		buffEnd++
+		if buffEnd >= _batchSize {
+			values := upChargeValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = s.dao.InsertUpCharge(c, table, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := upChargeValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = s.dao.InsertUpCharge(c, table, values)
+	}
+	return
+}
+
+func upChargeValues(us []*model.UpCharge) (values string) {
+	var buf bytes.Buffer
+	for _, charge := range us {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(charge.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(charge.IncCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(charge.TotalCharge, 10))
+		buf.WriteByte(',')
+		buf.WriteString(fmt.Sprintf("'%s'", charge.Date.Time().Format("2006-01-02")))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}