package income import ( "context" "fmt" "time" model "go-common/app/job/main/growup/model/income" task "go-common/app/job/main/growup/service" "go-common/library/log" "golang.org/x/sync/errgroup" ) // RunStatis run income statistics func (s *Service) RunStatis(c context.Context, date time.Time) (err error) { var msg string mailReceivers := []string{"shaozhenyu@bilibili.com", "gaopeng@bilibili.com", "limengqing@bilibili.com"} startTime := time.Now().Unix() err = s.runStatis(c, date) if err != nil { msg = err.Error() } else { msg = fmt.Sprintf("%s 计算完成,耗时%ds", date.Format("2006-01-02"), time.Now().Unix()-startTime) } err = s.email.SendMail(date, msg, "创作激励每日统计%d年%d月%d日", mailReceivers...) if err != nil { log.Error("s.email.SendMail error(%v)", err) } return } func (s *Service) runStatis(c context.Context, date time.Time) (err error) { defer func() { task.GetTaskService().SetTaskStatus(c, task.TaskCreativeStatis, date.Format(_layout), err) }() // task status type err = task.GetTaskService().TaskReady(c, date.Format("2006-01-02"), task.TaskCreativeIncome) if err != nil { return } startWeeklyDate = getStartWeeklyDate(date) startMonthlyDate = getStartMonthlyDate(date) var ( readGroup errgroup.Group avSections = SectionEntries{} cmSections = SectionEntries{} bgmSections = SectionEntries{} upSection = make([]*model.DateStatis, 0) upAvSection = make([]*model.DateStatis, 0) upCmSection = make([]*model.DateStatis, 0) upBgmSection = make([]*model.DateStatis, 0) upIncomeWeekly = make(map[int64]*model.UpIncome) upIncomeMonthly = make(map[int64]*model.UpIncome) upAvStatisCh = make(chan map[int64]*model.UpArchStatis, 1) upCmStatisCh = make(chan map[int64]*model.UpArchStatis, 1) upBgmStatisCh = make(chan map[int64]*model.UpArchStatis, 1) avSourceCh = make(chan []*model.ArchiveIncome, 1000) avDailyCh = make(chan []*model.ArchiveIncome, 1000) avWeeklyCh = make(chan []*model.ArchiveIncome, 1000) avMonthlyCh = make(chan []*model.ArchiveIncome, 1000) upAvCh = make(chan []*model.ArchiveIncome, 1000) cmSourceCh = make(chan []*model.ArchiveIncome, 1000) cmDailyCh = make(chan []*model.ArchiveIncome, 1000) cmWeeklyCh = make(chan []*model.ArchiveIncome, 1000) cmMonthlyCh = make(chan []*model.ArchiveIncome, 1000) upCmCh = make(chan []*model.ArchiveIncome, 1000) bgmSourceCh = make(chan []*model.ArchiveIncome, 1000) bgmDailyCh = make(chan []*model.ArchiveIncome, 1000) bgmWeeklyCh = make(chan []*model.ArchiveIncome, 1000) bgmMonthlyCh = make(chan []*model.ArchiveIncome, 1000) upBgmCh = make(chan []*model.ArchiveIncome, 1000) upSourceCh = make(chan []*model.UpIncome, 1000) upDailyCh = make(chan []*model.UpIncome, 1000) upStatisCh = make(chan []*model.UpIncome, 1000) ) // get income by date to statistics preDate := startMonthlyDate if startWeeklyDate.Before(startMonthlyDate) { preDate = startWeeklyDate } // get av_income readGroup.Go(func() (err error) { err = s.income.dateStatisSvr.getArchiveByDate(c, avSourceCh, preDate, date, _video, _limitSize) if err != nil { log.Error("s.getArchiveByDate error(%v)", err) } return }) // get column_income readGroup.Go(func() (err error) { err = s.income.dateStatisSvr.getArchiveByDate(c, cmSourceCh, preDate, date, _column, _limitSize) if err != nil { log.Error("s.getArchiveByDate error(%v)", err) } return }) // get bgm_income readGroup.Go(func() (err error) { err = s.income.dateStatisSvr.getArchiveByDate(c, bgmSourceCh, preDate, date, _bgm, _limitSize) if err != nil { log.Error("s.getArchiveByDate error(%v)", err) } return }) // get up_income readGroup.Go(func() (err error) { err = s.income.upIncomeSvr.getUpIncomeByDate(c, upSourceCh, date.Format(_layout), _limitSize) if err != nil { log.Error("s.getUpIncomeByDate error(%v)", err) } return }) readGroup.Go(func() (err error) { defer func() { close(avDailyCh) close(avWeeklyCh) close(avMonthlyCh) close(upAvCh) }() for av := range avSourceCh { avDailyCh <- av avWeeklyCh <- av avMonthlyCh <- av upAvCh <- av } return }) readGroup.Go(func() (err error) { defer func() { close(cmDailyCh) close(cmWeeklyCh) close(cmMonthlyCh) close(upCmCh) }() for cm := range cmSourceCh { cmDailyCh <- cm cmWeeklyCh <- cm cmMonthlyCh <- cm upCmCh <- cm } return }) readGroup.Go(func() (err error) { defer func() { close(bgmDailyCh) close(bgmWeeklyCh) close(bgmMonthlyCh) close(upBgmCh) }() for bgm := range bgmSourceCh { bgmDailyCh <- bgm bgmWeeklyCh <- bgm bgmMonthlyCh <- bgm upBgmCh <- bgm } return }) readGroup.Go(func() (err error) { defer func() { close(upStatisCh) close(upDailyCh) }() for up := range upSourceCh { upStatisCh <- up upDailyCh <- up } return }) // up readGroup.Go(func() (err error) { upSection, upAvSection, upCmSection, upBgmSection, err = s.income.dateStatisSvr.handleDateUp(c, upStatisCh, date) if err != nil { log.Error("s.income.dateStatisSvr.HandleUp error(%v)", err) } return }) // video readGroup.Go(func() (err error) { avSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, avDailyCh, date, _avIncomeDailyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) readGroup.Go(func() (err error) { avSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, avWeeklyCh, startWeeklyDate, _avIncomeWeeklyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) readGroup.Go(func() (err error) { avSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, avMonthlyCh, startMonthlyDate, _avIncomeMonthlyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) // column readGroup.Go(func() (err error) { cmSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, cmDailyCh, date, _cmIncomeDailyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) readGroup.Go(func() (err error) { cmSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, cmWeeklyCh, startWeeklyDate, _cmIncomeWeeklyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) readGroup.Go(func() (err error) { cmSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, cmMonthlyCh, startMonthlyDate, _cmIncomeMonthlyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) // bgm readGroup.Go(func() (err error) { bgmSections.avDaily, err = s.income.dateStatisSvr.handleDateStatis(c, bgmDailyCh, date, _bgmIncomeDailyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) readGroup.Go(func() (err error) { bgmSections.avWeekly, err = s.income.dateStatisSvr.handleDateStatis(c, bgmWeeklyCh, startWeeklyDate, _bgmIncomeWeeklyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) readGroup.Go(func() (err error) { bgmSections.avMonthly, err = s.income.dateStatisSvr.handleDateStatis(c, bgmMonthlyCh, startMonthlyDate, _bgmIncomeMonthlyStatis) if err != nil { log.Error("s.income.dateStatisSvr.handleDateStatis error(%v)", err) } return }) // up_av_statis readGroup.Go(func() (err error) { err = s.income.upIncomeSvr.handleUpArchStatis(c, upAvStatisCh, upAvCh) if err != nil { log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err) } return }) // up_column_statis readGroup.Go(func() (err error) { err = s.income.upIncomeSvr.handleUpArchStatis(c, upCmStatisCh, upCmCh) if err != nil { log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err) } return }) // up_bgm_statis readGroup.Go(func() (err error) { err = s.income.upIncomeSvr.handleUpArchStatis(c, upBgmStatisCh, upBgmCh) if err != nil { log.Error("p.upIncomeSvr.handleUpArchStatis error(%v)", err) } return }) // up_income_weekly up_income_monthly readGroup.Go(func() (err error) { upIncomeWeekly, upIncomeMonthly, err = s.income.upIncomeSvr.handleUpIncomeWeeklyAndMonthly(c, date, upAvStatisCh, upCmStatisCh, upBgmStatisCh, upDailyCh) if err != nil { log.Error("p.upIncomeSvr.handleUpIncomeWeeklyAndMonthly error(%v)", err) } return }) if err = readGroup.Wait(); err != nil { log.Error("run readGroup.Wait error(%v)", err) return } { if len(avSections.avDaily) == 0 { err = fmt.Errorf("Error: insert 0 av_income_daily_statis") return } if len(avSections.avWeekly) == 0 { err = fmt.Errorf("Error: insert 0 av_income_weekly_statis") return } if len(avSections.avMonthly) == 0 { err = fmt.Errorf("Error: insert 0 av_income_monthly_statis") return } if len(cmSections.avDaily) == 0 { err = fmt.Errorf("Error: insert 0 cm_income_daily_statis") return } if len(cmSections.avWeekly) == 0 { err = fmt.Errorf("Error: insert 0 cm_income_weekly_statis") return } if len(cmSections.avMonthly) == 0 { err = fmt.Errorf("Error: insert 0 cm_income_monthly_statis") return } if len(bgmSections.avDaily) == 0 { err = fmt.Errorf("Error: insert 0 bgm_income_daily_statis") return } if len(bgmSections.avWeekly) == 0 { err = fmt.Errorf("Error: insert 0 bgm_income_weekly_statis") return } if len(bgmSections.avMonthly) == 0 { err = fmt.Errorf("Error: insert 0 bgm_income_monthly_statis") return } if len(upSection) == 0 { err = fmt.Errorf("Error: insert 0 up_income_daily_statis") return } if len(upAvSection) == 0 { err = fmt.Errorf("Error: insert 0 up_av_daily_statis") return } if len(upCmSection) == 0 { err = fmt.Errorf("Error: insert 0 up_column_daily_statis") return } if len(upBgmSection) == 0 { err = fmt.Errorf("Error: insert 0 up_bgm_daily_statis") return } if len(upIncomeWeekly) == 0 { err = fmt.Errorf("Error: insert 0 up_income_weekly") return } if len(upIncomeMonthly) == 0 { err = fmt.Errorf("Error: insert 0 up_income_monthly") return } } // persistent var writeGroup errgroup.Group // av_income_daily_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avDaily, _avIncomeDailyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err) return } log.Info("insert av_income_daily_statis : %d", len(avSections.avDaily)) return }) // av_income_weekly_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avWeekly, _avIncomeWeeklyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err) return } log.Info("insert av_income_weekly_statis : %d", len(avSections.avWeekly)) return }) // av_income_monthly_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, avSections.avMonthly, _avIncomeMonthlyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err) return } log.Info("insert av_income_monthly_statis : %d", len(avSections.avMonthly)) return }) // column_income_daily_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avDaily, _cmIncomeDailyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err) return } log.Info("insert column_income_daily_statis : %d", len(cmSections.avDaily)) return }) // column_income_weekly_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avWeekly, _cmIncomeWeeklyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err) return } log.Info("insert column_income_weekly_statis : %d", len(cmSections.avWeekly)) return }) // column_income_monthly_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, cmSections.avMonthly, _cmIncomeMonthlyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err) return } log.Info("insert column_income_monthly_statis : %d", len(cmSections.avMonthly)) return }) // bgm_income_daily_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avDaily, _bgmIncomeDailyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(daily) error(%v)", err) return } log.Info("insert bgm_income_daily_statis : %d", len(bgmSections.avDaily)) return }) // bgm_income_weekly_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avWeekly, _bgmIncomeWeeklyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(weekly) error(%v)", err) return } log.Info("insert bgm_income_weekly_statis : %d", len(bgmSections.avWeekly)) return }) // bgm_income_monthly_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.incomeDateStatisInsert(c, bgmSections.avMonthly, _bgmIncomeMonthlyStatis) if err != nil { log.Error("s.income.dateStatisSvr.incomeDateStatisInsert(monthly) error(%v)", err) return } log.Info("insert bgm_income_monthly_statis : %d", len(bgmSections.avMonthly)) return }) // up_income_daily_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upSection, "up_income_daily_statis") if err != nil { log.Error("s.upIncomeDailyStatisInsert error(%v)", err) return } log.Info("insert up_income_daily_statis : %d", len(upSection)) return }) // up_av_daily_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upAvSection, "up_av_daily_statis") if err != nil { log.Error("s.upIncomeDailyStatisInsert error(%v)", err) return } log.Info("insert up_av_daily_statis : %d", len(upAvSection)) return }) // up_column_daily_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upCmSection, "up_column_daily_statis") if err != nil { log.Error("s.upIncomeDailyStatisInsert error(%v)", err) return } log.Info("insert up_column_daily_statis : %d", len(upCmSection)) return }) // up_bgm_daily_statis writeGroup.Go(func() (err error) { _, err = s.income.dateStatisSvr.upIncomeDailyStatisInsert(c, upBgmSection, "up_bgm_daily_statis") if err != nil { log.Error("s.upIncomeDailyStatisInsert error(%v)", err) return } log.Info("insert up_bgm_daily_statis : %d", len(upBgmSection)) return }) // up_income_weekly writeGroup.Go(func() (err error) { err = s.income.upIncomeSvr.UpIncomeDBStoreBatch(c, _upIncomeWeekly, upIncomeWeekly) if err != nil { log.Error("s.UpIncomeDBStoreBatch up_income_weekly error(%v)", err) return } log.Info("insert up_income_weekly : %d", len(upIncomeWeekly)) return }) // up_income_monthly writeGroup.Go(func() (err error) { err = s.income.upIncomeSvr.UpIncomeDBStoreBatch(c, _upIncomeMonthly, upIncomeMonthly) if err != nil { log.Error("s.UpIncomeDBStoreBatch up_income_monthly error(%v)", err) return } log.Info("insert up_income_monthly : %d", len(upIncomeMonthly)) return }) if err = writeGroup.Wait(); err != nil { log.Error("run writeGroup.Wait error(%v)", err) } return }