Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,66 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"email.go",
"mcn.go",
"recommend.go",
"service.go",
"summary.go",
"up.go",
],
importpath = "go-common/app/job/main/mcn/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/mcn/conf:go_default_library",
"//app/job/main/mcn/dao:go_default_library",
"//app/job/main/mcn/model:go_default_library",
"//app/service/main/account/api:go_default_library",
"//library/log:go_default_library",
"//library/sync/pipeline/fanout:go_default_library",
"//library/time:go_default_library",
"//library/xstr:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/github.com/robfig/cron:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"mcn_test.go",
"recommend_test.go",
"service_test.go",
"summary_test.go",
"up_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/mcn/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

View File

@@ -0,0 +1,290 @@
package service
import (
"bytes"
"context"
"fmt"
"html/template"
"time"
"go-common/app/job/main/mcn/conf"
"go-common/app/job/main/mcn/model"
accgrpc "go-common/app/service/main/account/api"
"go-common/library/log"
"go-common/library/xstr"
"github.com/pkg/errors"
)
// var .
var (
// ErrNoAdminName no admin name
ErrNoAdminName = errors.New("no admin name")
tmplSignDueTitle *template.Template
tmplSignDueContent *template.Template
tmplPayDueTitle *template.Template
tmplPayDueContent *template.Template
)
// use for template function call
var funcHelper = template.FuncMap{
"Now": time.Now,
}
func (s *Service) initEmailTemplate() (err error) {
if conf.Conf.MailTemplateConf.SignTmplTitle == "" ||
conf.Conf.MailTemplateConf.SignTmplContent == "" ||
conf.Conf.MailTemplateConf.PayTmplTitle == "" ||
conf.Conf.MailTemplateConf.PayTmplContent == "" {
err = fmt.Errorf(`mail template conf is invalid, check mail-template.toml file, make sure all the following has value:
TaskTmplContent
TaskTmplTitle
PayTmplContent
PayTmplTitle`)
return
}
tmplSignDueTitle, err = template.New("signTitle").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.SignTmplTitle)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplSignDueContent, err = template.New("signContent").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.SignTmplContent)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplPayDueTitle, err = template.New("payTitle").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.PayTmplTitle)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplPayDueContent, err = template.New("payContent").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.PayTmplContent)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
return
}
// CheckDateDueCron .
func (s *Service) CheckDateDueCron() {
log.Info("start run CheckDateDueJob, date=%s", time.Now().Format(model.TimeFormatSec))
s.checkSignUpDue()
log.Info("finish run CheckDateDueJob, date=%s", time.Now().Format(model.TimeFormatSec))
}
type stateFunc func(context.Context, []int64) (int64, error)
type emailData struct {
IDs []int64
AdminName []string
Data interface{}
UpStateFunc stateFunc
Title, Content *template.Template
}
func (e *emailData) addEmailDatas(es *[]*emailData) {
*es = append(*es, e)
}
// buildEmail .
func buildEmail(ids []int64, data interface{}, title, content *template.Template, upStateFunc stateFunc, adminName ...string) *emailData {
return &emailData{
IDs: ids,
Data: data,
Title: title,
Content: content,
UpStateFunc: upStateFunc,
AdminName: adminName,
}
}
type dueData struct {
Signs []*model.MCNSignInfo
Pays []*model.SignPayInfo
Sids, Pids []int64
}
func (d *dueData) addSign(sign *model.MCNSignInfo) {
d.Signs = append(d.Signs, sign)
d.Sids = append(d.Sids, sign.SignID)
}
func (d *dueData) addPay(pay *model.SignPayInfo) {
d.Pays = append(d.Pays, pay)
d.Pids = append(d.Pids, pay.SignPayID)
}
func (d *dueData) addName(infoMap map[int64]*accgrpc.Info) {
for _, v := range d.Signs {
v.McnName = getName(infoMap, v.McnMid)
}
for _, v := range d.Pays {
v.McnName = getName(infoMap, v.McnMid)
}
}
// func getOrCreate(dataMap map[string]*dueData, key string) *dueData {
// var data, ok = dataMap[key]
// if !ok {
// data = &dueData{}
// dataMap[key] = data
// }
// return data
// }
func getName(infoMap map[int64]*accgrpc.Info, mid int64) string {
if info, ok := infoMap[mid]; ok {
return info.Name
}
return ""
}
func (s *Service) checkSignUpDue() {
var (
mids []int64
emailDatas []*emailData
data = &dueData{}
c = context.Background()
infoMap map[int64]*accgrpc.Info
)
// 30天内到期 sign
listDue, err := s.dao.McnSignDues(c)
if err != nil {
log.Error("s.dao.McnSignDues error(%+v)", err)
return
}
for _, v := range listDue {
mids = append(mids, v.McnMid)
data.addSign(v)
}
// 7天内到期的pay
listPayDue, err := s.dao.McnSignPayDues(c)
if err != nil {
log.Error("s.dao.McnSignPayDues error(%+v)", err)
return
}
for _, v := range listPayDue {
mids = append(mids, v.McnMid)
data.addPay(v)
}
mids = uniqNoEmpty(mids)
infosReply, err := s.accGRPC.Infos3(c, &accgrpc.MidsReq{Mids: mids})
if err != nil {
log.Error("s.accGRPC.Infos3(%s) error(%+v)", xstr.JoinInts(mids), err)
err = nil
} else {
infoMap = infosReply.Infos
}
emailDatas = make([]*emailData, 0)
data.addName(infoMap)
buildEmail(data.Sids, data.Signs, tmplSignDueTitle, tmplSignDueContent, s.dao.UpMcnSignEmailState, conf.Conf.MailConf.DueAuthorityGroups...).addEmailDatas(&emailDatas)
buildEmail(data.Pids, data.Pays, tmplPayDueTitle, tmplPayDueContent, s.dao.UpMcnSignPayEmailState, conf.Conf.MailConf.DueAuthorityGroups...).addEmailDatas(&emailDatas)
for _, e := range emailDatas {
s.doSendEmailFunc(c, e)
}
}
func (s *Service) doSendEmailFunc(c context.Context, e *emailData) {
s.worker.Do(c, func(c context.Context) {
if len(e.IDs) == 0 {
log.Warn("not need to update")
return
}
var err error
if err = s.sendMailWithTemplate(e.Data, e.Title, e.Content, e.AdminName...); err != nil {
log.Error("s.sendMailWithTemplate(%+v,%+v,%+v,%+v) error(%+v)", e.Data, e.Title, e.Content, e.AdminName, err)
return
}
if _, err = e.UpStateFunc(c, e.IDs); err != nil {
log.Error("upfunc(%+v,%s) error(%+v)", e.UpStateFunc, xstr.JoinInts(e.IDs), err)
return
}
log.Info("func(%s) update succ", xstr.JoinInts(e.IDs))
})
}
// data, data to generate email content
// contentTmpl, template to generate email content
// adminname, slice for all admin name
//
func (s *Service) sendMailWithTemplate(data interface{}, subjectTmpl, contentTmpl *template.Template, adminName ...string) (err error) {
if contentTmpl == nil {
err = fmt.Errorf("template for email is nil, data=%+v", data)
log.Error("%s", err)
return
}
var contentBuf = bytes.NewBuffer(nil)
err = contentTmpl.Execute(contentBuf, data)
if err != nil {
log.Error("template fail to execute, err=%v", err)
return
}
var subjectBuf = bytes.NewBuffer(nil)
err = subjectTmpl.Execute(subjectBuf, data)
if err != nil {
log.Error("template fail to execute, err=%v", err)
return
}
var addrs []string
for _, v := range adminName {
if v == "" {
log.Warn("admin name is empty")
continue
}
addrs = append(addrs, v)
}
if len(addrs) == 0 {
log.Error("admin name is empty, cannot send email, data=%+v", data)
err = ErrNoAdminName
return
}
if err = s.dao.SendMail(contentBuf.String(), subjectBuf.String(), addrs); err != nil {
log.Error("s.dao.SendMail(%s,%s,%+v) error(%+v)", contentBuf.String(), subjectBuf.String(), addrs, err)
return
}
log.Info("email send succ, sub=%s, admin=%s", subjectBuf.String(), adminName)
return
}
func chain(ids ...[]int64) []int64 {
res := make([]int64, 0, len(ids))
for _, l := range ids {
res = append(res, l...)
}
return res
}
// func uniq(ids ...[]int64) []int64 {
// hm := make(map[int64]struct{})
// for _, i := range chain(ids...) {
// hm[i] = struct{}{}
// }
// res := make([]int64, 0, len(ids))
// for i := range hm {
// res = append(res, i)
// }
// return res
// }
func uniqNoEmpty(ids ...[]int64) []int64 {
hm := make(map[int64]struct{})
for _, i := range chain(ids...) {
hm[i] = struct{}{}
}
res := make([]int64, 0, len(ids))
for i := range hm {
if i > 0 {
res = append(res, i)
}
}
return res
}

View File

@@ -0,0 +1,96 @@
package service
import (
"context"
"time"
"go-common/app/job/main/mcn/model"
"go-common/library/log"
"github.com/pkg/errors"
)
// UpMcnSignStateCron .
func (s *Service) UpMcnSignStateCron() {
defer func() {
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("recover panic error(%+v)", r)
}
}()
var (
err error
c = context.TODO()
now = time.Now()
nowDate = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
mss []*model.MCNSignInfo
)
if mss, err = s.dao.McnSigns(c); err != nil {
log.Error("s.dao.McnSigns error(%+v)", err)
return
}
if len(mss) == 0 {
log.Warn("mcn sign data is empty!")
return
}
for _, v := range mss {
var state int8
switch {
case v.State.NotDealState():
continue
case nowDate > v.EndDate.Time().Unix() && nowDate-v.EndDate.Time().Unix() <= model.ThirtyDayUnixTime && v.State != model.MCNSignStateOnCooling:
state = int8(model.MCNSignStateOnCooling)
case nowDate > v.EndDate.Time().Unix() && nowDate-v.EndDate.Time().Unix() > model.ThirtyDayUnixTime:
state = int8(model.MCNSignStateOnExpire)
case nowDate < v.BeginDate.Time().Unix() && v.State != model.MCNSignStateOnPreOpen:
state = int8(model.MCNSignStateOnPreOpen)
case v.BeginDate.Time().Unix() <= nowDate && nowDate <= v.EndDate.Time().Unix() && v.State != model.MCNSignStateOnSign && v.State == model.MCNSignStateOnPreOpen:
state = int8(model.MCNSignStateOnSign)
default:
continue
}
if _, err = s.dao.UpMcnSignStateOP(c, v.SignID, state); err != nil {
log.Error("s.dao.UpMcnSignStateOP(%d,%d) error(%+v)", v.SignID, state, err)
continue
}
if err = s.dao.DelMcnSignCache(c, v.McnMid); err != nil {
log.Error("s.dao.DelMcnSignCache(%d) error(%+v)", v.McnMid, err)
continue
}
log.Info("signID(%d) change old state(%d) to new state(%d)", v.SignID, v.State, state)
}
}
// UpExpirePayCron .
func (s *Service) UpExpirePayCron() {
defer func() {
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("recover panic error(%+v)", r)
}
}()
var (
err error
c = context.TODO()
sps []*model.SignPayInfo
)
if sps, err = s.dao.McnSignPayWarns(c); err != nil {
log.Error("s.dao.McnSignPayWarns error(%+v)", err)
return
}
if len(sps) == 0 {
log.Warn("mcn sign pay date is empty!")
return
}
ms := make(map[int64]struct{})
for _, v := range sps {
ms[v.SignID] = struct{}{}
}
for signID := range ms {
if _, err = s.dao.UpMcnSignPayExpOP(c, signID); err != nil {
log.Error("s.dao.UpMcnSignPayExpOP(%d) error(%+v)", signID, err)
continue
}
log.Info("sign_id(%d) change pay data warn state to 2", signID)
}
}

View File

@@ -0,0 +1,27 @@
package service
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceUpMcnSignStateCron(t *testing.T) {
convey.Convey("UpMcnSignStateCron", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
s.UpMcnSignStateCron()
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}
func TestServiceUpExpirePayCron(t *testing.T) {
convey.Convey("UpExpirePayCron", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
s.UpExpirePayCron()
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}

View File

@@ -0,0 +1,67 @@
package service
import (
"context"
"go-common/app/job/main/mcn/model"
"go-common/library/log"
"github.com/pkg/errors"
)
// McnRecommendCron .
func (s *Service) McnRecommendCron() {
defer func() {
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("recover panic error(%+v)", r)
}
}()
var (
err error
limit = 100
c = context.TODO()
rps []*model.McnUpRecommendPool
)
for {
if rps, err = s.dao.McnUpRecommendSources(c, limit); err != nil {
log.Error("s.dao.McnUpRecommendSources(%d) error(%+v)", limit, err)
return
}
if len(rps) == 0 {
log.Warn("big data recommend up data is empty!")
return
}
for _, v := range rps {
rp := new(model.McnUpRecommendPool)
*rp = *v
rp.GenerateTime = v.Mtime
if _, err = s.dao.AddMcnUpRecommend(c, rp); err != nil {
log.Error("s.dao.AddMcnUpRecommend(%+v) error(%+v)", rp, err)
continue
}
if _, err = s.dao.DelMcnUpRecommendSource(c, v.ID); err != nil {
log.Error("s.dao.DelMcnUpRecommendSource(%d) error(%+v)", v.ID, err)
continue
}
log.Info("source id(%d) sync to recommend poll(%+v)", v.ID, rp)
}
}
}
// DealFailRecommendCron .
func (s *Service) DealFailRecommendCron() {
defer func() {
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("recover panic error(%+v)", r)
}
}()
var (
err error
c = context.TODO()
)
if _, err = s.dao.DelMcnUpRecommendPool(c); err != nil {
log.Error("s.dao.DelMcnUpRecommendPool error(%+v)", err)
}
}

View File

@@ -0,0 +1,27 @@
package service
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceMcnRecommendCron(t *testing.T) {
convey.Convey("McnRecommendCron", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
s.McnRecommendCron()
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}
func TestServiceDealFailRecommendCron(t *testing.T) {
convey.Convey("DealFailRecommendCron", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
s.DealFailRecommendCron()
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}

View File

@@ -0,0 +1,60 @@
package service
import (
"context"
"runtime"
"go-common/app/job/main/mcn/conf"
"go-common/app/job/main/mcn/dao"
accgrpc "go-common/app/service/main/account/api"
"go-common/library/sync/pipeline/fanout"
"github.com/pkg/errors"
"github.com/robfig/cron"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
// rpc
accGRPC accgrpc.AccountClient
worker *fanout.Fanout
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
worker: fanout.New("cache", fanout.Worker(runtime.NumCPU()), fanout.Buffer(1024)),
}
var err error
if s.accGRPC, err = accgrpc.NewClient(c.GRPCClient.Account); err != nil {
panic(errors.WithMessage(err, "Failed to dial account service"))
}
if err := s.initEmailTemplate(); err != nil {
panic(err)
}
t := cron.New()
t.AddFunc(c.Property.UpMcnSignStateCron, s.UpMcnSignStateCron)
t.AddFunc(c.Property.UpMcnUpStateCron, s.UpMcnUpStateCron)
t.AddFunc(c.Property.UpExpirePayCron, s.UpExpirePayCron)
//t.AddFunc(c.Property.UpMcnDataSummaryCron, s.UpMcnDataSummaryCron)
t.AddFunc(c.Property.McnRecommendCron, s.McnRecommendCron)
t.AddFunc(c.Property.DealFailRecommendCron, s.DealFailRecommendCron)
t.AddFunc(c.Property.CheckMcnSignUpDueCron, s.CheckDateDueCron)
t.Start()
return s
}
// Ping Service
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close Service
func (s *Service) Close() {
s.dao.Close()
s.worker.Close()
}

View File

@@ -0,0 +1,24 @@
package service
import (
"flag"
"os"
"testing"
"go-common/app/job/main/mcn/conf"
)
var (
s *Service
)
func TestMain(m *testing.M) {
flag.Set("conf", "../cmd/mcn-job-test.toml")
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
s = New(conf.Conf)
m.Run()
os.Exit(m.Run())
}

View File

@@ -0,0 +1,77 @@
package service
import (
"context"
"time"
"go-common/library/log"
xtime "go-common/library/time"
"go-common/library/xstr"
"github.com/pkg/errors"
)
// UpMcnDataSummaryCron .
func (s *Service) UpMcnDataSummaryCron() {
defer func() {
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("recover panic error(%+v)", r)
}
}()
var (
err error
sids []int64
msid map[int64]int64
mmc map[int64]int64
mup map[int64][]int64
c = context.TODO()
)
if msid, sids, err = s.dao.McnSignMids(c); err != nil {
log.Error("s.dao.McnSignMids error(%+v)", err)
return
}
if len(sids) == 0 {
log.Warn("mcn sign data summary empty!")
return
}
if mmc, err = s.dao.McnUPCount(c, sids); err != nil {
log.Error("s.dao.McnUPCount(%s) error(%+v)", xstr.JoinInts(sids), err)
return
}
if mup, err = s.dao.McnUPMids(c, sids); err != nil {
log.Error("s.dao.McnUPMids(%s) error(%+v)", xstr.JoinInts(sids), err)
return
}
for sid, smid := range msid {
var (
upOK, upMidOK bool
upNums int64
upMids []int64
totalFans int64
now = time.Now()
gDate = time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local)
)
if upNums, upOK = mmc[sid]; !upOK {
upNums = 0
}
if upMids, upMidOK = mup[sid]; upMidOK {
if len(upMids) == 0 {
totalFans = 0
} else {
if totalFans, err = s.dao.CrmUpMidsSum(c, upMids); err != nil {
log.Error("s.dao.CrmUpMidsSum(%s) error(%+v)", xstr.JoinInts(upMids), err)
err = nil
totalFans = 0
}
}
} else {
totalFans = 0
}
if err = s.dao.AddMcnDataSummary(c, smid, sid, upNums, totalFans, xtime.Time(gDate.Unix())); err != nil {
log.Error("s.dao.UpMcnUpStateOP(%d,%d,%d,%d,%+v) error(%+v)", smid, sid, upNums, totalFans, xtime.Time(gDate.Unix()), err)
continue
}
log.Info("mcnMid(%d) signID(%d) upNum(%d) totalFans(%d) date(%+v) add data summary table", smid, sid, upNums, totalFans, xtime.Time(gDate.Unix()))
}
}

View File

@@ -0,0 +1,17 @@
package service
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceUpMcnDataSummaryCron(t *testing.T) {
convey.Convey("UpMcnDataSummaryCron", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
s.UpMcnDataSummaryCron()
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}

View File

@@ -0,0 +1,60 @@
package service
import (
"context"
"time"
"go-common/app/job/main/mcn/model"
"go-common/library/log"
"github.com/pkg/errors"
)
// UpMcnUpStateCron .
func (s *Service) UpMcnUpStateCron() {
defer func() {
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("recover panic error(%+v)", r)
}
}()
var (
err error
page = 1
limit = 100
c = context.TODO()
now = time.Now()
nowDate = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
mus []*model.MCNUPInfo
)
for {
offset := int64((page - 1) * limit)
if mus, err = s.dao.McnUps(c, offset, int64(limit)); err != nil {
log.Error("s.dao.McnUps(%d,%d) error(%+v)", offset, limit, err)
return
}
if len(mus) == 0 {
log.Warn("mcn up data is empty!")
return
}
for _, v := range mus {
var state int8
switch {
case v.State.NotDealState():
continue
case v.BeginDate.Time().Unix() <= nowDate && nowDate <= v.EndDate.Time().Unix() && v.State != model.MCNUPStateOnSign && v.State == model.MCNUPStateOnPreOpen:
state = int8(model.MCNUPStateOnSign)
case nowDate > v.EndDate.Time().Unix() && v.State != model.MCNUPStateOnExpire:
state = int8(model.MCNUPStateOnExpire)
default:
continue
}
if _, err = s.dao.UpMcnUpStateOP(c, v.SignUpID, state); err != nil {
log.Error("s.dao.UpMcnUpStateOP(%d,%d) error(%+v)", v.SignUpID, state, err)
continue
}
log.Info("signUpID(%d) change old state(%d) to new state(%d)", v.SignUpID, v.State, state)
}
page++
}
}

View File

@@ -0,0 +1,17 @@
package service
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceUpMcnUpStateCron(t *testing.T) {
convey.Convey("UpMcnUpStateCron", t, func(ctx convey.C) {
ctx.Convey("When everything goes positive", func(ctx convey.C) {
s.UpMcnUpStateCron()
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}