openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/job/main/up/BUILD Normal file
View File

@@ -0,0 +1,23 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/up/cmd:all-srcs",
"//app/job/main/up/conf:all-srcs",
"//app/job/main/up/dao/account:all-srcs",
"//app/job/main/up/dao/email:all-srcs",
"//app/job/main/up/dao/upcrm:all-srcs",
"//app/job/main/up/model:all-srcs",
"//app/job/main/up/server/http:all-srcs",
"//app/job/main/up/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

26
app/job/main/up/CHANGELOG.md Normal file
View File

@@ -0,0 +1,26 @@
# v1.4.3
> 1.增加grpc config
# v1.4.2
> 1.修复databus bug
# v1.4.1
> 1.修复databus bug
# v1.4.0
> 1.up主列表相关 + 联合投稿staff
# v1.3.1
1. 调整时间间隔
# v1.3.0
1. up主列表相关
# v1.2.1
1. 优化邮件发送逻辑
# v1.2.0
1. crm签约管理二期需求
# v1.0.0
1. 初版

11
app/job/main/up/CONTRIBUTORS.md Normal file
View File

@@ -0,0 +1,11 @@
# Owner
wangzhe01
shencen
# Author
wangzhe01
hejianbing
# Reviewer
wangzhe01
shencen

16
app/job/main/up/OWNERS Normal file
View File

@@ -0,0 +1,16 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- hejianbing
- shencen
- wangzhe01
labels:
- job
- job/main/up
- main
options:
no_parent_owners: true
reviewers:
- hejianbing
- shencen
- wangzhe01

12
app/job/main/up/README.md Normal file
View File

@@ -0,0 +1,12 @@
# up-job
# 项目简介
1.
# 编译环境
# 依赖包
# 编译执行

1
app/job/main/up/api/http/api.md Normal file
View File

@@ -0,0 +1 @@
# HTTP API文档

43
app/job/main/up/cmd/BUILD Normal file
View File

@@ -0,0 +1,43 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["up-job.toml"],
importpath = "go-common/app/job/main/up/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/up/conf:go_default_library",
"//app/job/main/up/server/http:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

75
app/job/main/up/cmd/mail-template.toml Normal file
View File

@@ -0,0 +1,75 @@
[MailTemplateConf]
SignTmplTitle = """[签约Up即将到期]{{Now.Format "2006-01-02"}}"""
SignTmplContent = """
<html>
<body>
<h4>您有以下签约即将到期，请尽快去<a href="http://up-profit.bilibili.co/allowance/#/contract-manage/all-contracts">【up主CRM-签约管理】查看</a></h4>
<table border="1">
<tr>
<td>签约id</td>
<td>昵称</td>
<td>uid</td>
<td>到期时间</td>
</tr>
{{range .}}
<tr>
<td>{{.ID}}</td>
<td>{{.Name}}</td>
<td>{{.Mid}}</td>
<td>{{.EndDate.Time.Format "2006-01-02"}}</td>
</tr>
{{end}}
</table>
</body>
</html>
"""
PayTmplTitle = """"[签约Up付款即将到期]{{Now.Format "2006-01-02"}}"""
PayTmplContent = """
<html>
<body>
<h4>您有以下付款即将到期,请尽快去<a href="http://up-profit.bilibili.co/allowance/#/contract-manage/all-contracts">【up主CRM-签约管理】查看</a></h4>
<table border=1>
<tr>
<td>签约id</td>
<td>昵称</td>
<td>uid</td>
<td>到期时间</td>
<td>金额</td>
</tr>
{{range .}}
<tr>
<td>{{.SignID}}</td>
<td>{{.Name}}</td>
<td>{{.Mid}}</td>
<td>{{.DueDate.Time.Format "2006-01-02"}}</td>
<td>{{printf "%.2f" .GetPayValue}}</td>
</tr>
{{end}}
</table>
</body>
</html>"""
TaskTmplTitle = """[签约Up任务还未完成]{{Now.Format "2006-01-02"}}"""
TaskTmplContent = """
<html>
<body>
您录入的签约up主任务还未完成，请尽快去<a href="http://up-profit.bilibili.co/allowance/#/contract-manage/all-contracts">【up主CRM-签约管理】查看</a>
<table border=1>
<tr>
<td>签约id</td>
<td>昵称</td>
<td>uid</td>
<td>到期时间</td>
<td>状态</td>
</tr>
{{range .}}
<tr>
<td>{{.SignID}}</td>
<td>{{.Name}}</td>
<td>{{.Mid}}</td>
<td>{{(.EndDate.Time.AddDate 0 0 -1).Format "2006-01-02"}}</td>
<td>{{(printf "%s投稿 %d/%d" .TypeDesc .TaskCounter .TaskCondition)}}</td>
</tr>
{{end}}
</table>
</body>
</html>"""

43
app/job/main/up/cmd/main.go Normal file
View File

@@ -0,0 +1,43 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"go-common/app/job/main/up/conf"
"go-common/app/job/main/up/server/http"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
log.SetFormat("[%D %T] [%L] [%S] %M")
defer log.Close()
log.Info("server start")
trace.Init(conf.Conf.Tracer)
defer trace.Close()
ecode.Init(conf.Conf.Ecode)
http.Init(conf.Conf)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
log.Info("[server] exit")
return
case syscall.SIGHUP:
default:
return
}
}
}

69
app/job/main/up/cmd/up-job.toml Normal file
View File

@@ -0,0 +1,69 @@
IsTest = true
[log]
stdout = true
# dir = "/data/log/up/"
[Upcrm]
dsn = "upcrm:DdL6c5JaWCYKMAQ10PURbfeImow9HXlx@tcp(172.16.33.205:3306)/bilibili_upcrm?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 1
idleTimeout = "4h"
[ArchiveOrm]
dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_archive?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 1
idleTimeout = "4h"
[job]
UpCheckDateDueTaskTime = "0 0 7 * * ?"
TaskScheduleTime = "0 0 1 * * ?"
CheckStateJobTime = "0 30 /1 * * ?"
UpdateUpTidJobTime = "0 15 0 * * ?"
[MailConf]
host = "smtp.exmail.qq.com"
port = 465
username = "manager@bilibili.com"
password = "123"
DueMailReceivers = [""] # 需要发送的额外管理员名称
[DatabusConf]
[DatabusConf.ArchiveNotify]
key = "4c76cbb7a985ac90"
secret= "43bb22ce34a6b13e7814f09cb8116522"
group= "ArchiveNotify-MainArchive-UpArchives-S"
topic= "ArchiveNotify-T"
action = "sub"
name = "up-job"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 5
active = 20
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[DatabusConf.Archive]
key = "4c76cbb7a985ac90"
secret= "43bb22ce34a6b13e7814f09cb8116522"
group= "Archive-MainArchive-UpService-S"
topic= "Archive-T"
action = "sub"
name = "up-job"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 5
active = 20
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[grpcclient]
[grpcclient.up]
timeout = "5s"
[grpcclient.archive]
timeout = "2s"

41
app/job/main/up/conf/BUILD Normal file
View File

@@ -0,0 +1,41 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/main/up/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/conf:go_default_library",
"//library/database/orm:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/rpc:go_default_library",
"//library/net/rpc/warden:go_default_library",
"//library/net/trace:go_default_library",
"//library/queue/databus:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

142
app/job/main/up/conf/conf.go Normal file
View File

@@ -0,0 +1,142 @@
package conf
import (
"errors"
"flag"
"go-common/library/conf"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/trace"
"github.com/BurntSushi/toml"
"go-common/library/database/orm"
"go-common/library/net/rpc"
"go-common/library/net/rpc/warden"
"go-common/library/queue/databus"
"path"
)
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config .
type Config struct {
Log *log.Config
BM *bm.ServerConfig
Verify *verify.Config
Tracer *trace.Config
Ecode *ecode.Config
// rpc client
AccountRPC *rpc.ClientConfig
// gorm
Upcrm *orm.Config
ArchiveOrm *orm.Config
MailConf *Mail
MailTemplateConf *MailTemplateConfig
DatabusConf *DataBusConfig
GRPCClient *GRPCClient
// cron job
Job *JobCron
IsTest bool
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
if err != nil {
return
}
var templateConfPath = path.Join(path.Dir(confPath), "mail-template.toml")
_, err = toml.DecodeFile(templateConfPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}
//Mail 邮件配置
type Mail struct {
Host string
Port int
Username, Password string
DueMailReceivers []string // []adminname, send to adminname@bilibili.com
}
//JobCron 任务时间配置
type JobCron struct {
UpCheckDateDueTaskTime string
TaskScheduleTime string
CheckStateJobTime string
UpdateUpTidJobTime string
}
//MailTemplateConfig mail template conf
type MailTemplateConfig struct {
SignTmplTitle string
SignTmplContent string
PayTmplTitle string
PayTmplContent string
TaskTmplTitle string
TaskTmplContent string
}
//DataBusConfig databus config
type DataBusConfig struct {
ArchiveNotify *databus.Config
Archive *databus.Config
}
// GRPCClient .
type GRPCClient struct {
Up *warden.ClientConfig
Archive *warden.ClientConfig
}

52
app/job/main/up/dao/account/BUILD Normal file
View File

@@ -0,0 +1,52 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["dao_test.go"],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/job/main/up/conf:go_default_library",
"//library/ecode:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"account_cache.go",
"dao.go",
"rpc.go",
],
importpath = "go-common/app/job/main/up/dao/account",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/up/conf:go_default_library",
"//app/service/main/account/model:go_default_library",
"//app/service/main/account/rpc/client:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

76
app/job/main/up/dao/account/account_cache.go Normal file
View File

@@ -0,0 +1,76 @@
package account
import (
"context"
"sync"
"time"
account "go-common/app/service/main/account/model"
"go-common/library/log"
)
var (
infoCache = make(map[int64]*account.Info)
nextClearCacheTime time.Time
lock = sync.Mutex{}
)
const (
cacheClearInterval = 60 * time.Minute
)
//GetCachedInfos get cache info
func (d *Dao) GetCachedInfos(c context.Context, mids []int64, ip string) (infos map[int64]*account.Info, err error) {
d.checkClearCache()
var needFindMids []int64
infos = make(map[int64]*account.Info)
lock.Lock()
for _, v := range mids {
var info, ok = infoCache[v]
if !ok {
needFindMids = append(needFindMids, v)
continue
}
log.Info("hit cache! mid=%d", v)
infos[v] = info
}
lock.Unlock()
if len(needFindMids) == 0 {
return
}
var findinfo, e = d.Infos(c, needFindMids, ip)
err = e
if e != nil {
log.Error("try get uid info fail, err=%v", e)
return
}
lock.Lock()
for k, v := range findinfo {
infos[k] = v
infoCache[k] = v
}
lock.Unlock()
return
}
func (d *Dao) checkClearCache() {
var now = time.Now()
if now.Before(nextClearCacheTime) {
return
}
d.clearCache()
}
func (d *Dao) clearCache() {
nextClearCacheTime = time.Now().Add(cacheClearInterval)
if len(infoCache) == 0 {
return
}
lock.Lock()
infoCache = make(map[int64]*account.Info)
lock.Unlock()
}

70
app/job/main/up/dao/account/dao.go Normal file
View File

@@ -0,0 +1,70 @@
package account
import (
"context"
"go-common/app/job/main/up/conf"
"go-common/app/service/main/account/model"
account "go-common/app/service/main/account/rpc/client"
"go-common/library/ecode"
"go-common/library/log"
)
// Dao dao is account dao.
type Dao struct {
// config
c *conf.Config
// rpc
acc *account.Service3
}
// New new a account dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
// config
c: c,
// rpc
acc: account.New3(c.AccountRPC),
}
return
}
// Close close resource.
func (d *Dao) Close() {
}
// Ping ping success.
func (d *Dao) Ping(c context.Context) (err error) {
return
}
// IdentifyInfo 获取用户实名认证状态
func (d *Dao) IdentifyInfo(c context.Context, ak, ck, ip string, mid int64) (err error) {
var mf *model.Profile
if mf, err = d.Profile(c, mid, ip); err != nil {
log.Error("d.Profile mid(%d),ip(%s),error(%v)", mid, ip, err)
err = ecode.CreativeAccServiceErr
return
}
if mf.Identification == 1 {
return
}
//switch for FrontEnd return json format, return OldPhone, and newError
if err = d.switchIDInfoRet(mf.TelStatus); err != nil {
log.Error("switchIDInfoRet url(%s) res(%v)", mf.TelStatus)
return
}
return
}
func (d *Dao) switchIDInfoRet(phoneRet int32) (err error) {
switch phoneRet {
case 0:
err = ecode.UserCheckNoPhone
case 1:
err = nil
case 2:
err = ecode.UserCheckInvalidPhone
}
return
}

62
app/job/main/up/dao/account/dao_test.go Normal file
View File

@@ -0,0 +1,62 @@
package account
import (
"context"
"flag"
"path/filepath"
"testing"
"go-common/library/ecode"
. "github.com/smartystreets/goconvey/convey"
"go-common/app/job/main/up/conf"
)
var (
d *Dao
)
func init() {
dir, _ := filepath.Abs("../../cmd/up-admin.toml")
flag.Set("conf", dir)
conf.Init()
d = New(conf.Conf)
}
func WithDao(f func(d *Dao)) func() {
return func() {
Reset(func() {})
f(d)
}
}
func Test_IdentifyInfo(t *testing.T) {
var (
c = context.Background()
err error
mid = int64(27515256)
ak = "efbf2e093c3c04008acbd9906ba970db"
ck = ""
ip = "127.0.0.1"
)
Convey("IdentifyInfo", t, WithDao(func(d *Dao) {
err = d.IdentifyInfo(c, ak, ck, ip, mid)
So(err, ShouldNotBeNil)
So(err, ShouldEqual, ecode.CreativeAccServiceErr)
}))
}
func Test_GetCachedInfos(t *testing.T) {
var (
c = context.Background()
err error
mids = []int64{1234, 12345}
ip = "127.0.0.1"
)
Convey("GetCachedInfos", t, WithDao(func(d *Dao) {
_, err = d.GetCachedInfos(c, mids, ip)
So(err, ShouldNotBeNil)
}))
}

33
app/job/main/up/dao/account/rpc.go Normal file
View File

@@ -0,0 +1,33 @@
package account
import (
"context"
account "go-common/app/service/main/account/model"
"go-common/library/ecode"
"go-common/library/log"
)
// Profile get profile from rpc
func (d *Dao) Profile(c context.Context, mid int64, ip string) (p *account.Profile, err error) {
arg := &account.ArgMid{
Mid: mid,
}
if p, err = d.acc.Profile3(c, arg); err != nil {
log.Error("d.acc.Profile3 error(%v) | mid(%d) ip(%s) arg(%v)", err, mid, ip, arg)
err = ecode.CreativeAccServiceErr
}
return
}
//Infos get up infos
func (d *Dao) Infos(c context.Context, mids []int64, ip string) (infos map[int64]*account.Info, err error) {
arg := &account.ArgMids{
Mids: mids,
}
if infos, err = d.acc.Infos3(c, arg); err != nil {
log.Error("d.acc.info3 error(%v) arg(%v)", err, arg)
err = ecode.CreativeAccServiceErr
}
return
}

45
app/job/main/up/dao/email/BUILD Normal file
View File

@@ -0,0 +1,45 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["dao_test.go"],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = ["//vendor/github.com/smartystreets/goconvey/convey:go_default_library"],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"email.go",
],
importpath = "go-common/app/job/main/up/dao/email",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/up/conf:go_default_library",
"//library/log:go_default_library",
"//vendor/gopkg.in/gomail.v2:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

26
app/job/main/up/dao/email/dao.go Normal file
View File

@@ -0,0 +1,26 @@
package email
import (
"crypto/tls"
"go-common/app/job/main/up/conf"
gomail "gopkg.in/gomail.v2"
)
// Dao is redis dao.
type Dao struct {
email *gomail.Dialer
}
// New is new redis dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
// mail
email: gomail.NewDialer(c.MailConf.Host, c.MailConf.Port, c.MailConf.Username, c.MailConf.Password),
}
d.email.TLSConfig = &tls.Config{
InsecureSkipVerify: true,
}
return
}

11
app/job/main/up/dao/email/dao_test.go Normal file
View File

@@ -0,0 +1,11 @@
package email
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func Test_Temp(t *testing.T) {
Convey("temp test", t, func() {})
}

39
app/job/main/up/dao/email/email.go Normal file
View File

@@ -0,0 +1,39 @@
package email
import (
"go-common/library/log"
"os"
"go-common/app/job/main/up/conf"
"gopkg.in/gomail.v2"
)
//SendMail send the email.
func (d *Dao) SendMail(body string, subject string, send []string) (err error) {
log.Info("send mail send:%v", send)
msg := gomail.NewMessage()
msg.SetHeader("From", conf.Conf.MailConf.Username)
msg.SetHeader("To", send...)
msg.SetHeader("Subject", subject)
msg.SetBody("text/html", body, gomail.SetPartEncoding(gomail.Base64))
if err = d.email.DialAndSend(msg); err != nil {
log.Error("s.email.DialAndSend error(%v)", err)
return
}
return
}
//SendMailAttach send the email.
func (d *Dao) SendMailAttach(filename string, subject string, send []string) (err error) {
msg := gomail.NewMessage()
msg.SetHeader("From", conf.Conf.MailConf.Username)
msg.SetHeader("To", send...)
msg.SetHeader("Subject", subject)
msg.Attach(filename)
if err = d.email.DialAndSend(msg); err != nil {
log.Error("s.email.DialAndSend error(%v)", err)
return
}
err = os.Remove(filename)
return
}

52
app/job/main/up/dao/upcrm/BUILD Normal file
View File

@@ -0,0 +1,52 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["dao_test.go"],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = ["//app/job/main/up/conf:go_default_library"],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"sign_up.go",
"task_info.go",
"up_base_info.go",
],
importpath = "go-common/app/job/main/up/dao/upcrm",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/admin/main/up/util/now:go_default_library",
"//app/job/main/up/conf:go_default_library",
"//app/job/main/up/model/signmodel:go_default_library",
"//app/job/main/up/model/upcrmmodel:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/jinzhu/gorm:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

55
app/job/main/up/dao/upcrm/dao.go Normal file
View File

@@ -0,0 +1,55 @@
package upcrm
import (
"context"
"go-common/app/job/main/up/conf"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"github.com/jinzhu/gorm"
)
//Dao upcrm dao
type Dao struct {
// config
conf *conf.Config
// db
crmdb *gorm.DB
httpClient *bm.Client
}
//New new dao
func New(c *conf.Config) *Dao {
var d = &Dao{
conf: c,
}
crmdb, err := gorm.Open("mysql", c.Upcrm.DSN)
if crmdb == nil {
log.Error("connect to db fail, err=%v", err)
return nil
}
crmdb.SingularTable(true)
d.crmdb = crmdb
d.crmdb.LogMode(c.IsTest)
return d
}
//SetHTTPClient set http client
func (d *Dao) SetHTTPClient(client *bm.Client) {
d.httpClient = client
}
//GetDb get current gorm db
func (d *Dao) GetDb() *gorm.DB {
return d.crmdb
}
// Close close resource.
func (d *Dao) Close() {
d.crmdb.Close()
}
// Ping ping success.
func (d *Dao) Ping(c context.Context) (err error) {
return
}

50
app/job/main/up/dao/upcrm/dao_test.go Normal file
View File

@@ -0,0 +1,50 @@
package upcrm
import (
"flag"
"os"
"testing"
"time"
"go-common/app/job/main/up/conf"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.archive.mcn-admin")
flag.Set("conf_token", "220af473858ad67f75586b66bece0e6b")
flag.Set("tree_id", "58930")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Set("conf", "../../cmd/up-job.toml")
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
m.Run()
os.Exit(0)
}
func TestGetDueTask(t *testing.T) {
var date, _ = time.Parse("2006-01-02", "2018-11-02")
var res, err = d.GetDueTask(date)
if err != nil {
t.Errorf("err get tasks, err=%v", err)
t.FailNow()
}
for i, v := range res {
t.Logf("[%d]res=%+v", i, v)
}
}

314
app/job/main/up/dao/upcrm/sign_up.go Normal file
View File

@@ -0,0 +1,314 @@
package upcrm
import (
"time"
"go-common/app/admin/main/up/util/now"
"go-common/app/job/main/up/model/signmodel"
"go-common/app/job/main/up/model/upcrmmodel"
"go-common/library/log"
xtime "go-common/library/time"
"github.com/jinzhu/gorm"
)
const (
//PayStateUnpay not pay
PayStateUnpay = 0
//PayStatePayed payed
PayStatePayed = 1
)
// var days
var (
Day = time.Hour * 24
Day3 = Day * 3
Week = Day * 7
Week3 = Week * 3
)
// GetTaskDuration this will return task duration, [startDate, endDate)
func GetTaskDuration(date time.Time, taskType int8) (startDate, endDate time.Time) {
var ndate = now.New(date)
now.WeekStartDay = time.Monday
switch taskType {
case signmodel.TaskTypeDay:
var begin = ndate.BeginningOfDay()
return begin, begin.AddDate(0, 0, 1)
case signmodel.TaskTypeWeek:
var begin = ndate.BeginningOfWeek()
return begin, begin.AddDate(0, 0, 7)
case signmodel.TaskTypeMonth:
var begin = ndate.BeginningOfMonth()
return begin, begin.AddDate(0, 1, 0)
case signmodel.TaskTypeQuarter:
var begin = ndate.BeginningOfQuarter()
return begin, begin.AddDate(0, 3, 0)
}
return
}
// GetTaskExpireLimit this will return generate date range that needs to send email
func GetTaskExpireLimit(today time.Time, taskType int8) (minDate, maxDate time.Time) {
// 0<=（endDate-today)<= limit
// 0<=(generateDate + duration - today) <= limit
// today-duration <= generateDate <= today - duration + limit
// duration = endDate - startDate
switch taskType {
case signmodel.TaskTypeDay:
var tmp = today.AddDate(0, 0, -1)
return tmp, tmp.Add(Day)
case signmodel.TaskTypeWeek:
var tmp = today.AddDate(0, 0, -7)
return tmp, tmp.Add(Day3)
case signmodel.TaskTypeMonth:
var tmp = today.AddDate(0, -1, 0)
return tmp, tmp.Add(Week)
case signmodel.TaskTypeQuarter:
var tmp = today.AddDate(0, -3, 0)
return tmp, tmp.Add(Week3)
}
return
}
//InsertSignUp insert sign up
// up : sign up
func (d *Dao) InsertSignUp(up *signmodel.SignUp) (affectedRow int64, err error) {
var db = d.crmdb.Save(up)
err = db.Error
affectedRow = db.RowsAffected
return
}
//InsertPayInfo inert pay
func (d *Dao) InsertPayInfo(info *signmodel.SignPay) (affectedRow int64, err error) {
var db = d.crmdb.Save(info)
err = db.Error
affectedRow = db.RowsAffected
return
}
//InsertTaskInfo insert task
func (d *Dao) InsertTaskInfo(info *signmodel.SignTask) (affectedRow int64, err error) {
var db = d.crmdb.Save(info)
err = db.Error
affectedRow = db.RowsAffected
return
}
//InsertContractInfo insert contract
func (d *Dao) InsertContractInfo(info interface{}) (affectedRow int64, err error) {
var db = d.crmdb.Save(info)
err = db.Error
affectedRow = db.RowsAffected
return
}
//GetSignUpByID signid 可以是nil，如果是nil，则会取所有的信息
// query, args, 额外的查询条件
func (d *Dao) GetSignUpByID(signID []uint32, order string, offset int, limit int, query interface{}, args ...interface{}) (result []signmodel.SignUp, err error) {
var db = d.crmdb.Table("sign_up")
if signID != nil {
db = db.Where("id in (?)", signID)
}
if query != nil {
db = db.Where(query, args)
}
if order != "" {
db = db.Order(order)
}
err = db.
Offset(offset).
Limit(limit).
Find(&result).Error
if err == gorm.ErrRecordNotFound {
err = nil
}
return
}
//GetSignUpCount get sign up's count
func (d *Dao) GetSignUpCount(query string, args ...interface{}) (count int) {
d.crmdb.Table(signmodel.TableNameSignUp).Where(query, args).Count(&count)
return
}
//GetTask get task by sign id and state
func (d *Dao) GetTask(signID []uint32, state ...int) (result []signmodel.SignTask, err error) {
err = d.crmdb.Table(signmodel.TableNameSignTask).Where("sign_id in (?) and state = ?", signID, state).Find(&result).Error
if err == gorm.ErrRecordNotFound {
err = nil
}
return
}
//GetPay get get sign id
func (d *Dao) GetPay(signID []uint32) (result []signmodel.SignPay, err error) {
err = d.crmdb.Table(signmodel.TableNameSignPay).Where("sign_id in (?)", signID).Find(&result).Error
if err == gorm.ErrRecordNotFound {
err = nil
}
return
}
//GetContract get get sign id
func (d *Dao) GetContract(signID []uint32) (result []signmodel.SignContract, err error) {
err = d.crmdb.Table(signmodel.TableNameSignContract).Where("sign_id in (?)", signID).Find(&result).Error
if err == gorm.ErrRecordNotFound {
err = nil
}
return
}
//PayComplete finish pay by pay id
func (d *Dao) PayComplete(ids []uint32) (affectedRow int64, err error) {
var db = d.crmdb.Table(signmodel.TableNameSignPay).Where("id in (?)", ids).Updates(map[string]interface{}{"state": PayStatePayed})
err = db.Error
if err == nil {
affectedRow = db.RowsAffected
}
return
}
//SignWithName sign with name, used to send mail
type SignWithName struct {
signmodel.SignUp
Name string
}
//GetEndDate used for template
func (s *SignWithName) GetEndDate() string {
return s.EndDate.Time().Format(upcrmmodel.TimeFmtDate)
}
//GetDueSignUp check due
// expireAfterDays : how many days to expire
func (d *Dao) GetDueSignUp(now time.Time, expireAfterDays int) (result []*SignWithName, err error) {
var dueDate = now.AddDate(0, 0, expireAfterDays)
var db = d.crmdb.Table(signmodel.TableNameSignUp).Select("id, mid, end_date, admin_id, admin_name")
// email_state = 0 是未发过邮件的意思
db = db.Where("end_date <= ? and end_date >= ? and email_state = 0", dueDate.Format(upcrmmodel.TimeFmtDate), now.Format(upcrmmodel.TimeFmtDate))
err = db.Find(&result).Error
return
}
//PayWithAdmin pay with name, used to send mail
type PayWithAdmin struct {
signmodel.SignPay
Name string
AdminID int
AdminName string
}
//GetEndDate used for template
func (s *PayWithAdmin) GetEndDate() string {
return s.DueDate.Time().Format(upcrmmodel.TimeFmtDate)
}
//GetPayValue for template
func (s *PayWithAdmin) GetPayValue() float64 {
return float64(s.PayValue) / 1000.0
}
//GetDuePay check due
func (d *Dao) GetDuePay(now time.Time, expireAfterDays int) (result []*PayWithAdmin, err error) {
var dueDate = now.AddDate(0, 0, expireAfterDays)
err = d.crmdb.Raw("select p.id, "+
"p.sign_id, "+
"p.mid, "+
"p.due_date, "+
"p.pay_value,"+
"s.admin_id, "+
"s.admin_name"+
" from sign_pay as p left join sign_up as s on p.sign_id = s.id "+
" where p.due_date <= ? and p.email_state = 0 and p.state = 0", dueDate.Format(upcrmmodel.TimeFmtDate)).
Scan(&result).Error
return
}
//TaskWithAdmin task with admin
type TaskWithAdmin struct {
signmodel.SignTaskHistory
Name string
AdminID int
AdminName string
}
//EndDate used for template
func (s *TaskWithAdmin) EndDate() xtime.Time {
var _, end = GetTaskDuration(s.GenerateDate.Time(), s.TaskType)
return xtime.Time(end.Unix())
}
//TypeDesc descrption
func (s *TaskWithAdmin) TypeDesc() string {
return signmodel.TaskTypeStr(int(s.TaskType))
}
var (
//NeedCheckTaskType task type
NeedCheckTaskType = []int8{
signmodel.TaskTypeWeek,
signmodel.TaskTypeMonth,
signmodel.TaskTypeQuarter,
}
)
//GetDueTask get due tasks
func (d *Dao) GetDueTask(now time.Time) (result []*TaskWithAdmin, err error) {
// 到期的任务，
// 从sign_task_history表中查询任务
for _, t := range NeedCheckTaskType {
var min, max = GetTaskExpireLimit(now, t)
// 从sign_task_history中查询min<=generate_date <=max and task_type = t and state != 2
var tasks []*TaskWithAdmin
err = d.crmdb.Raw(`
select t.*, u.admin_id, u.admin_name
from sign_task_history as t
left join sign_up as u
on t.sign_id=u.id
where t.generate_date=? and t.task_type=? and t.state!=2`, // t.generate_date=? 现在只在时间刚好达到时发送邮件，这样可以只发送一次，但是有可能会漏掉那些早已经过期的。
max.Format(upcrmmodel.TimeFmtDate), t).
Find(&tasks).Error
if err != nil {
log.Error("fail to get from db, task type=%d, min=%s, max=%s", t, min, max)
return
}
log.Info("get from db, task type=%d, min=%s, max=%s, len=%d", t, min, max, len(tasks))
result = append(result, tasks...)
}
return
}
//UpdateEmailState update email send state
// state : @
func (d *Dao) UpdateEmailState(table string, ids []uint32, state int8) (affectedRow int64, err error) {
if len(ids) == 0 {
log.Warn("no ids to update email state, state=%d", state)
return
}
var db = d.crmdb.Table(table).Where("id in (?)", ids).Update("email_state", state)
err = db.Error
if err == nil {
affectedRow = db.RowsAffected
} else {
log.Error("update email state fail, err=%+v", err)
}
return
}
//CheckUpHasValidContract check if has valid contract
func (d *Dao) CheckUpHasValidContract(mid int64, date time.Time) (exist bool, err error) {
var ids []struct {
ID int
}
err = d.crmdb.Table(signmodel.TableNameSignUp).Select("id").Where("mid=? and end_date>=?", mid, date.Format(upcrmmodel.TimeFmtDate)).Limit(1).Find(&ids).Error
if err != nil && err != gorm.ErrRecordNotFound {
log.Error("check exist from db fail, err=%+v", err)
return
}
exist = len(ids) > 0
return
}

44
app/job/main/up/dao/upcrm/task_info.go Normal file
View File

@@ -0,0 +1,44 @@
package upcrm
import (
"go-common/app/job/main/up/model/upcrmmodel"
"go-common/library/log"
xtime "go-common/library/time"
"time"
)
//StartTask start task
func (d *Dao) StartTask(taskType int, now time.Time) (affectedRow int64, err error) {
var task = &upcrmmodel.TaskInfo{}
task.TaskType = int8(taskType)
task.GenerateDate = now.Format(upcrmmodel.TimeFmtDate)
task.StartTime = xtime.Time(now.Unix())
task.TaskState = upcrmmodel.TaskStateStart
var db = d.crmdb.Model(task).Save(task)
err = db.Error
if err != nil {
log.Error("error start task info, err=%+v", err)
return
}
affectedRow = db.RowsAffected
return
}
//FinishTask finish task
func (d *Dao) FinishTask(taskType int, now time.Time, state int) (affectedRow int64, err error) {
var task = &upcrmmodel.TaskInfo{}
task.TaskType = int8(taskType)
task.GenerateDate = now.Format(upcrmmodel.TimeFmtDate)
task.EndTime = xtime.Time(now.Unix())
task.TaskState = int16(state)
var db = d.crmdb.Model(task).Where("generate_date=? and task_type=?", task.GenerateDate, taskType).Update(task)
err = db.Error
if err != nil {
log.Error("error end task info, err=%+v", err)
return
}
affectedRow = db.RowsAffected
return
}

21
app/job/main/up/dao/upcrm/up_base_info.go Normal file
View File

@@ -0,0 +1,21 @@
package upcrm
import "go-common/app/job/main/up/model/upcrmmodel"
//QueryUpBaseInfo query db
func (d *Dao) QueryUpBaseInfo(mid int64, fields string) (result upcrmmodel.UpBaseInfo, err error) {
err = d.crmdb.Select(fields).Where("mid=?", mid).Find(&result).Error
return
}
//QueryUpBaseInfoBatchByMid query db
func (d *Dao) QueryUpBaseInfoBatchByMid(fields string, mid ...int64) (result []upcrmmodel.UpBaseInfo, err error) {
err = d.crmdb.Select(fields).Where("mid in(?)", mid).Find(&result).Error
return
}
//QueryUpBaseInfoBatchByID query db
func (d *Dao) QueryUpBaseInfoBatchByID(fields string, id ...int64) (result []upcrmmodel.UpBaseInfo, err error) {
err = d.crmdb.Select(fields).Where("id in(?)", id).Find(&result).Error
return
}

37
app/job/main/up/model/BUILD Normal file
View File

@@ -0,0 +1,37 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"model.go",
"reply.go",
"request.go",
],
importpath = "go-common/app/job/main/up/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/up/model/archivemodel:all-srcs",
"//app/job/main/up/model/signmodel:all-srcs",
"//app/job/main/up/model/upcrmmodel:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

28
app/job/main/up/model/archivemodel/BUILD Normal file
View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["tables.go"],
importpath = "go-common/app/job/main/up/model/archivemodel",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

40
app/job/main/up/model/archivemodel/tables.go Normal file
View File

@@ -0,0 +1,40 @@
package archivemodel
//ArchiveCanal struct from cannal
type ArchiveCanal struct {
ID int64 `json:"id"`
AID int64 `json:"aid"`
Mid int64 `json:"mid"`
TypeID int16 `json:"typeid"`
Videos int `json:"videos"`
Title string `json:"title"`
Cover string `json:"cover"`
Content string `json:"content"`
Duration int `json:"duration"`
Attribute int32 `json:"attribute"`
Copyright int8 `json:"copyright"`
Access int `json:"access"`
State int `json:"state"`
MissionID int64 `json:"mission_id"`
OrderID int64 `json:"order_id"`
RedirectURL string `json:"redirect_url"`
Forward int64 `json:"forward"`
Dynamic string `json:"dynamic"`
}
// ArchiveStaff state值
const (
StaffStateNormal = 1 // 正常
StaffStateDismissed = 2 // 解除
)
//ArchiveStaff .
type ArchiveStaff struct {
ID int64 `json:"id"`
Aid int64 `json:"aid"`
Mid int64 `json:"mid"`
StaffMid int64 `json:"staff_mid"`
StaffTitle string `json:"staff_title"`
StaffTitleId int64 `json:"staff_title_id"`
State int64 `json:"state"`
}

1
app/job/main/up/model/model.go Normal file
View File

@@ -0,0 +1 @@
package model

10
app/job/main/up/model/reply.go Normal file
View File

@@ -0,0 +1,10 @@
package model
//EmptyReply .
type EmptyReply struct {
}
//WarmUpReply .
type WarmUpReply struct {
LastID int `json:"last_id"`
}

18
app/job/main/up/model/request.go Normal file
View File

@@ -0,0 +1,18 @@
package model
//EmptyReq nothing
type EmptyReq struct {
}
//WarmUpReq warm up
type WarmUpReq struct {
Mid int64 `form:"mid"`
LastID int `form:"last_id"`
Size int `form:"size"`
}
//AddStaffReq .
type AddStaffReq struct {
StaffMid int64 `form:"staff_mid"`
Aid int64 `form:"aid"`
}

32
app/job/main/up/model/signmodel/BUILD Normal file
View File

@@ -0,0 +1,32 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["sign_up.go"],
importpath = "go-common/app/job/main/up/model/signmodel",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/admin/main/up/util:go_default_library",
"//library/time:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

247
app/job/main/up/model/signmodel/sign_up.go Normal file
View File

@@ -0,0 +1,247 @@
package signmodel
import (
"go-common/app/admin/main/up/util"
"go-common/library/time"
)
const (
//TableNameSignTask .
TableNameSignTask = "sign_task"
//TableNameSignTaskHistory .
TableNameSignTaskHistory = "sign_task_history"
//TableNameSignUp .
TableNameSignUp = "sign_up"
//TableNameSignContract .
TableNameSignContract = "sign_contract"
//TableNameSignPay .
TableNameSignPay = "sign_pay"
//TableNameSignTaskAbsence .
TableNameSignTaskAbsence = "sign_task_absence"
)
const (
//TaskTypeAccumulate 0
TaskTypeAccumulate = 0
//TaskTypeDay 1
TaskTypeDay = 1
//TaskTypeWeek 2
TaskTypeWeek = 2
//TaskTypeMonth 3
TaskTypeMonth = 3
//TaskTypeQuarter 4 季度
TaskTypeQuarter = 4
)
//TaskTypeStr get task type str
func TaskTypeStr(taskType int) string {
switch taskType {
case TaskTypeAccumulate:
return "累计"
case TaskTypeDay:
return "每日"
case TaskTypeWeek:
return "每周"
case TaskTypeMonth:
return "每月"
case TaskTypeQuarter:
return "每季度"
}
return "未知"
}
const (
//EmailStateNotSend 0
EmailStateNotSend = 0
//EmailStateSendSucc 1
EmailStateSendSucc = 1
)
//SignUpOnlyID struct
type SignUpOnlyID struct {
ID uint32
}
//SignUpOnlySignID struct
type SignUpOnlySignID struct {
SignID uint32
}
// const sign_up中的state定义
const (
SignStateOnSign = 0
SignStateExpire = 1
)
// const sign_up中的due_warn定义
const (
DueWarnNoWarn = 1
DueWarnWarn = 2
)
// const sign_up中的pay_expire_state定义
const (
// PayExpireStateNormal 未到期
PayExpireStateNormal = 1
// PayExpireStateDue 即将到期
PayExpireStateDue = 2
)
//SignUp struct
type SignUp struct {
ID uint32 `gorm:"column:id"`
Sex int8 `gorm:"column:sex"`
Mid int64 `gorm:"column:mid"`
BeginDate time.Time `gorm:"column:begin_date"`
EndDate time.Time `gorm:"column:end_date"`
State int8 `gorm:"column:state"`
DueWarn int8 `gorm:"column:due_warn"`
PayExpireState int8 `gorm:"column:pay_expire_state"`
Country string `gorm:"column:country"`
Province string `gorm:"column:province"`
City string `gorm:"column:city"`
Note string `gorm:"column:note"`
AdminID int `gorm:"column:admin_id"`
AdminName string `gorm:"column:admin_name"`
EmailState int8 `gorm:"column:email_state"`
Ctime time.Time `gorm:"column:ctime"`
Mtime time.Time `gorm:"column:mtime"`
}
//TableName .
func (s *SignUp) TableName() string {
return TableNameSignUp
}
//SignPay struct
type SignPay struct {
ID uint32 `gorm:"column:id"`
Mid int64 `gorm:"column:mid"`
SignID uint32 `gorm:"column:sign_id"`
DueDate time.Time `gorm:"column:due_date"`
PayValue int64 `gorm:"column:pay_value"`
State int8 `gorm:"column:state"`
Note string `gorm:"column:note"`
EmailState int8 `gorm:"column:email_state"`
Ctime time.Time `gorm:"column:ctime"`
Mtime time.Time `gorm:"column:mtime"`
}
//TableName .
func (s *SignPay) TableName() string {
return TableNameSignPay
}
//SignTaskState sign task's state
type SignTaskState int8
const (
//SignTaskStateInit init
SignTaskStateInit SignTaskState = 0
//SignTaskStateRunning task running
SignTaskStateRunning SignTaskState = 1
//SignTaskStateFinish finish
SignTaskStateFinish SignTaskState = 2
//SignTaskStateDelete delete
SignTaskStateDelete SignTaskState = 100
)
const (
// SignTaskAttrBitBusiness 商单标记
SignTaskAttrBitBusiness = 0
)
//SignTask struct
type SignTask struct {
ID uint32 `gorm:"column:id"`
Mid int64 `gorm:"column:mid"`
SignID uint32 `gorm:"column:sign_id"`
TaskType int8 `gorm:"column:task_type"`
TaskCounter int32 `gorm:"column:task_counter"`
TaskCondition int32 `gorm:"column:task_condition"`
TaskData string `gorm:"column:task_data"`
Attribute int64 `gorm:"column:attribute"`
GenerateDate time.Time `gorm:"column:generate_date"`
State SignTaskState `gorm:"column:state"`
Ctime time.Time `gorm:"column:ctime"`
Mtime time.Time `gorm:"column:mtime"`
}
//IsAttrSet is attribute set, see SignTaskAttrBitXXX above
func (s *SignTask) IsAttrSet(bit int) bool {
return util.IsBitSet64(s.Attribute, uint(bit))
}
//TableName .
func (s *SignTask) TableName() string {
return TableNameSignTask
}
//SignContract struct
type SignContract struct {
ID uint32 `gorm:"column:id"`
Mid int64 `gorm:"column:mid"`
SignID uint32 `gorm:"column:sign_id"`
Filename string `gorm:"column:filename"`
Filelink string `gorm:"column:filelink"`
State int8 `gorm:"column:state"`
Ctime time.Time `gorm:"column:ctime"`
Mtime time.Time `gorm:"column:mtime"`
}
//TableName .
func (s *SignContract) TableName() string {
return TableNameSignContract
}
//SignTaskHistory struct
type SignTaskHistory struct {
ID uint32 `gorm:"column:id"`
Mid int64 `gorm:"column:mid"`
SignID uint32 `gorm:"column:sign_id"`
TaskTemplateID uint32 `gorm:"column:task_template_id"`
TaskType int8 `gorm:"column:task_type"`
TaskCounter int32 `gorm:"column:task_counter"`
TaskCondition int32 `gorm:"column:task_condition"`
TaskData string `gorm:"column:task_data"`
Attribute int64 `gorm:"column:attribute"`
GenerateDate time.Time `gorm:"column:generate_date"`
State SignTaskState `gorm:"column:state"`
Ctime time.Time `gorm:"column:ctime"`
Mtime time.Time `gorm:"column:mtime"`
}
//TableName .
func (s *SignTaskHistory) TableName() string {
return TableNameSignTaskHistory
}
//SignTaskAbsenceState .
type SignTaskAbsenceState int8
const (
//SignTaskAbsenceStateInit initial
SignTaskAbsenceStateInit SignTaskAbsenceState = 0
//SignTaskAbsenceStateDelete deleted
SignTaskAbsenceStateDelete SignTaskAbsenceState = 100
)
//SignTaskAbsence table
type SignTaskAbsence struct {
ID uint32 `gorm:"column:id" json:"id"`
SignId uint32 `gorm:"column:sign_id" json:"sign_id"`
Mid int64 `gorm:"column:mid" json:"mid"`
TaskHistoryId uint32 `gorm:"column:task_history_id" json:"task_history_id"`
AbsenceCount uint32 `gorm:"column:absence_count" json:"absence_count"`
Reason string `gorm:"column:reason" json:"reason"`
State int8 `gorm:"column:state" json:"state"`
AdminId int64 `gorm:"column:admin_id" json:"admin_id"`
AdminName string `gorm:"column:admin_name" json:"admin_name"`
Ctime time.Time `gorm:"column:ctime" json:"ctime"`
Mtime time.Time `gorm:"column:mtime" json:"mtime"`
}
//TableName .
func (s *SignTaskAbsence) TableName() string {
return TableNameSignTaskAbsence
}

35
app/job/main/up/model/upcrmmodel/BUILD Normal file
View File

@@ -0,0 +1,35 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"task_info.go",
"up_base_info.go",
],
importpath = "go-common/app/job/main/up/model/upcrmmodel",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/time:go_default_library",
"//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

45
app/job/main/up/model/upcrmmodel/task_info.go Normal file
View File

@@ -0,0 +1,45 @@
package upcrmmodel
import (
"github.com/siddontang/go-mysql/mysql"
"go-common/library/time"
)
const (
//TaskStateStart 0
TaskStateStart = 0
//TaskStateFinish 1
TaskStateFinish = 1
//TaskStateError 2
TaskStateError = 2
)
const (
//TaskTypeCreditDaily 1
TaskTypeCreditDaily = 1
//TaskTypeScoreSectionDaily 2
TaskTypeScoreSectionDaily = 2
//TaskTypeSignTaskCalculate 3
TaskTypeSignTaskCalculate = 3
//TaskTypeSignCheckDue 4
TaskTypeSignCheckDue = 4
)
var (
//TimeFmtMysql mysql time format
TimeFmtMysql = mysql.TimeFormat
//TimeFmtDate with only date
TimeFmtDate = "2006-01-02"
)
//TaskInfo struct
type TaskInfo struct {
ID uint32 `gorm:"column:id"`
GenerateDate string
TaskType int8
StartTime time.Time
EndTime time.Time
TaskState int16
CTime time.Time `gorm:"column:ctime"`
MTime time.Time `gorm:"column:mtime"`
}

42
app/job/main/up/model/upcrmmodel/up_base_info.go Normal file
View File

@@ -0,0 +1,42 @@
package upcrmmodel
import "go-common/library/time"
import xtime "time"
// const table name
const (
TableNameUpBaseInfo = "up_base_info"
)
//UpBaseInfo struct
type UpBaseInfo struct {
ID uint32 `gorm:"column:id"`
Mid int64 `gorm:"column:mid"`
Name string `gorm:"column:name"`
Sex int8 `gorm:"column:sex"`
JoinTime time.Time `gorm:"column:join_time"`
FirstUpTime time.Time `gorm:"column:first_up_time"`
Level int16 `gorm:"column:level"`
FansCount int `gorm:"column:fans_count"`
AccountState int8 `gorm:"column:account_state"`
Activity int `gorm:"column:activity"`
ArticleCount30day int `gorm:"column:article_count_30day"`
ArticleCountAccumulate int `gorm:"column:article_count_accumulate"`
Birthday xtime.Time `gorm:"column:birthday"`
ActiveCity string `gorm:"column:active_city"` // 市，存的是城市的名字
ActiveProvince string `gorm:"column:active_province"` // 省，省的名字
VerifyType int8 `gorm:"column:verify_type"`
BusinessType int8 `gorm:"column:business_type"`
CreditScore int `gorm:"column:credit_score"`
PrScore int `gorm:"column:pr_score"`
QualityScore int `gorm:"column:quality_score"`
ActiveTid int64 `gorm:"column:active_tid"`
Attr int `gorm:"column:attr"`
CTime time.Time `gorm:"column:ctime"`
MTime time.Time `gorm:"column:mtime"`
}
//TableName get table name
func (s *UpBaseInfo) TableName() string {
return TableNameUpBaseInfo
}

39
app/job/main/up/server/http/BUILD Normal file
View File

@@ -0,0 +1,39 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"http.go",
"http_job.go",
],
importpath = "go-common/app/job/main/up/server/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/up/conf:go_default_library",
"//app/job/main/up/model:go_default_library",
"//app/job/main/up/model/upcrmmodel:go_default_library",
"//app/job/main/up/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

50
app/job/main/up/server/http/http.go Normal file
View File

@@ -0,0 +1,50 @@
package http
import (
"net/http"
"go-common/app/job/main/up/conf"
"go-common/app/job/main/up/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
var (
svc *service.Service
)
// Init init
func Init(c *conf.Config) {
svc = service.New(c)
engine := bm.DefaultServer(c.BM)
route(engine)
if err := engine.Start(); err != nil {
log.Error("bm Start error(%v)", err)
panic(err)
}
}
func route(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
g := e.Group("/x/job/up")
{
g.GET("/job/run", runJob)
g.GET("/start")
g.GET("/job/warm-up", warmUp)
g.GET("/job/warm-up-mid", warmUpMid)
g.GET("/job/add-staff", addStaff)
g.GET("/job/delete-staff", deleteStaff)
}
}
func ping(c *bm.Context) {
if err := svc.Ping(c); err != nil {
log.Error("ping error(%v)", err)
c.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(map[string]interface{}{}, nil)
}

146
app/job/main/up/server/http/http_job.go Normal file
View File

@@ -0,0 +1,146 @@
package http
import (
"context"
"go-common/app/job/main/up/model"
"go-common/app/job/main/up/model/upcrmmodel"
"go-common/library/log"
"go-common/library/net/http/blademaster"
"strings"
"time"
)
func runJob(c *blademaster.Context) {
var err error
var res interface{}
switch {
default:
var arg struct {
Date string `form:"date"`
Job string `form:"job"`
}
var err = c.Bind(&arg)
if err != nil {
break
}
var date time.Time
if arg.Date == "" {
date = time.Now()
} else {
date, err = time.Parse(upcrmmodel.TimeFmtDate, arg.Date)
if err != nil {
log.Error("parse date err")
break
}
}
switch strings.ToLower(arg.Job) {
case "task":
svc.CheckTaskFinish(date)
case "due":
svc.CheckDateDueJob(date)
case "state":
svc.CheckStateJob(date)
case "tid":
svc.UpdateUpTidJob(date)
case "":
svc.UpdateUpTidJob(date)
svc.CheckStateJob(date)
svc.CheckDateDueJob(date)
svc.CheckTaskFinish(date)
}
}
if err != nil {
c.JSON(nil, err)
} else {
c.JSON(res, err)
}
}
func warmUp(c *blademaster.Context) {
var err error
var res interface{}
switch {
default:
var arg = &model.WarmUpReq{}
var err = c.Bind(arg)
if err != nil {
break
}
go func() {
res, err = svc.WarmUp(context.Background(), arg)
}()
}
if err != nil {
c.JSON(nil, err)
} else {
c.JSON(res, err)
}
}
func warmUpMid(c *blademaster.Context) {
var err error
var res interface{}
switch {
default:
var arg = &model.WarmUpReq{}
var err = c.Bind(arg)
if err != nil {
break
}
go func() {
res, err = svc.WarmUpMid(context.Background(), arg)
}()
}
if err != nil {
c.JSON(nil, err)
} else {
c.JSON(res, err)
}
}
func addStaff(c *blademaster.Context) {
var err error
var res interface{}
switch {
default:
var arg = &model.AddStaffReq{}
var err = c.Bind(arg)
if err != nil {
break
}
res, _ = svc.AddStaff(c, arg)
}
if err != nil {
c.JSON(nil, err)
} else {
c.JSON(res, err)
}
}
func deleteStaff(c *blademaster.Context) {
var err error
var res interface{}
switch {
default:
var arg = &model.AddStaffReq{}
var err = c.Bind(arg)
if err != nil {
break
}
res, _ = svc.DeleteStaff(c, arg)
}
if err != nil {
c.JSON(nil, err)
} else {
c.JSON(res, err)
}
}

75
app/job/main/up/service/BUILD Normal file
View File

@@ -0,0 +1,75 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"archive_test.go",
"service_test.go",
],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/job/main/up/conf:go_default_library",
"//app/job/main/up/dao/upcrm:go_default_library",
"//app/job/main/up/model/archivemodel:go_default_library",
"//app/job/main/up/model/signmodel:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"archive.go",
"service.go",
"sign_due_job.go",
"sign_state_job.go",
"sign_task_job.go",
"sign_update_tid_job.go",
],
importpath = "go-common/app/job/main/up/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/admin/main/up/util:go_default_library",
"//app/admin/main/up/util/databusutil:go_default_library",
"//app/admin/main/up/util/mathutil:go_default_library",
"//app/interface/main/mcn/tool/worker:go_default_library",
"//app/job/main/up/conf:go_default_library",
"//app/job/main/up/dao/account:go_default_library",
"//app/job/main/up/dao/email:go_default_library",
"//app/job/main/up/dao/upcrm:go_default_library",
"//app/job/main/up/model:go_default_library",
"//app/job/main/up/model/archivemodel:go_default_library",
"//app/job/main/up/model/signmodel:go_default_library",
"//app/job/main/up/model/upcrmmodel:go_default_library",
"//app/service/main/account/model:go_default_library",
"//app/service/main/archive/api:go_default_library",
"//app/service/main/up/api/v1:go_default_library",
"//app/service/main/upcredit/mathutil:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/jinzhu/gorm:go_default_library",
"//vendor/github.com/robfig/cron:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

269
app/job/main/up/service/archive.go Normal file
View File

@@ -0,0 +1,269 @@
package service
import (
"context"
"encoding/json"
"math"
"time"
"go-common/app/admin/main/up/util/mathutil"
"go-common/app/job/main/up/model"
"go-common/app/job/main/up/model/upcrmmodel"
upGRPCv1 "go-common/app/service/main/up/api/v1"
"go-common/library/log"
"go-common/library/queue/databus"
"go-common/app/job/main/up/model/archivemodel"
)
// action
const (
ActionUpdate = "update"
ActionInsert = "insert"
)
// table name
const (
TableArchiveStaff = "archive_staff"
)
//ArchiveUpInfo .
type ArchiveUpInfo struct {
Table string `json:"table"`
Action string `json:"action"`
New *archivemodel.ArchiveCanal `json:"new"`
Old *archivemodel.ArchiveCanal `json:"old"`
}
// CanalMsg canal message struct
type CanalMsg struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
}
// handle notify t message
func (s *Service) handleArchiveNotifyT(msg *databus.Message) (err error) {
m := &ArchiveUpInfo{}
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
return
}
switch m.Table {
case "archive":
err = s.checkArchiveNotify(m)
}
if err != nil {
log.Error("handle msg fail, err=%s", err)
}
return
}
func (s *Service) checkArchiveNotify(msg *ArchiveUpInfo) (err error) {
var arch *archivemodel.ArchiveCanal
switch {
default:
if msg.Action == ActionInsert {
if msg.New != nil && msg.New.State >= 0 {
arch = msg.New
}
break
}
if msg.Action == ActionUpdate {
if archiveStateChange(msg.New, msg.Old) {
arch = msg.New
}
break
}
}
if arch == nil {
log.Warn("no need to update up cache, msg value=%v", msg)
return
}
s.worker.Add(func() {
s.upRPC.UpCount(context.Background(), &upGRPCv1.UpCountReq{
Mid: arch.Mid,
})
var upCacheReq = &upGRPCv1.UpCacheReq{
Mid: arch.Mid,
Aid: arch.AID,
}
if arch.State >= 0 {
s.upRPC.AddUpPassedCache(context.Background(), upCacheReq)
log.Info("rpc add up cache, mid=%d, aid=%d", upCacheReq.Mid, upCacheReq.Aid)
} else {
s.upRPC.DelUpPassedCache(context.Background(), upCacheReq)
log.Info("rpc delete up cache, mid=%d, aid=%d", upCacheReq.Mid, upCacheReq.Aid)
}
})
return
}
func (s *Service) handleArchiveT(msg *databus.Message) (err error) {
var m = &CanalMsg{}
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
return
}
switch m.Table {
case TableArchiveStaff:
var new, old archivemodel.ArchiveStaff
switch m.Action {
case ActionInsert:
if err = json.Unmarshal(m.New, &new); err != nil {
log.Error("m.New -> json.Unmarshal(%v) error(%v)", m.New, err)
return
}
case ActionUpdate:
if err = json.Unmarshal(m.New, &new); err != nil {
log.Error("m.New -> json.Unmarshal(%v) error(%v)", m.New, err)
return
}
if err = json.Unmarshal(m.Old, &old); err != nil {
log.Error("m.Old -> json.Unmarshal(%v) error(%v)", m.New, err)
return
}
if new.State == old.State {
log.Warn("new staff state(%d) eq old staff state(%d)", new.State, old.State)
return
}
}
// state是正常说明是新增，否则是删除
needInsert := new.State == archivemodel.StaffStateNormal
var req = &upGRPCv1.UpCacheReq{Mid: new.StaffMid, Aid: new.Aid}
if needInsert {
_, err = s.upRPC.AddUpPassedCacheByStaff(context.Background(), req)
if err != nil {
log.Error("rpc call add up staff, new=%v, err=%v", new, err)
} else {
log.Info("rpc call add up staff, new=%v", new)
}
} else {
_, err = s.upRPC.DelUpPassedCacheByStaff(context.Background(), req)
if err != nil {
log.Error("rpc call del up staff, new=%v, err=%v", new, err)
} else {
log.Info("rpc call del up staff, new=%v", new)
}
}
}
return
}
func archiveStateChange(a, b *archivemodel.ArchiveCanal) bool {
if a == b {
return false
} else if a == nil || b == nil {
return true
}
if a.State == b.State {
return false
}
var min, max int
if a.State > b.State {
min, max = b.State, a.State
} else {
min, max = a.State, b.State
}
if min < 0 && max >= 0 {
return true
}
return false
}
//WarmUp warm up
func (s *Service) WarmUp(c context.Context, req *model.WarmUpReq) (res *model.WarmUpReply, err error) {
var (
d = s.crmdb.GetDb()
lastID = req.LastID
limit, thisCount = 100, 100
)
var count = 0
if req.Size == -1 {
req.Size = math.MaxInt32
}
for ; count < req.Size && thisCount == limit; count += thisCount {
time.Sleep(time.Millisecond * 1000)
var end = count + limit
if end > req.Size {
limit = req.Size - count
}
var upList []*upcrmmodel.UpBaseInfo
err = d.Select("mid, id").Where("id>?", lastID).Limit(limit).Find(&upList).Error
if err != nil {
log.Error("fail to query db, err=%v", err)
return
}
thisCount = len(upList)
var mids []int64
for _, v := range upList {
lastID = mathutil.Max(lastID, int(v.ID))
mids = append(mids, v.Mid)
}
var _, e = s.upRPC.UpsArcs(context.Background(), &upGRPCv1.UpsArcsReq{
Mids: mids,
Pn: 1,
Ps: 1,
})
if e != nil {
log.Warn("up rpc UpsArcs return err=%v", e)
}
_, e = s.upRPC.UpsCount(context.Background(), &upGRPCv1.UpsCountReq{
Mids: mids,
})
if e != nil {
log.Warn("up rpc UpsCount return err=%v", e)
}
log.Info("warm ups, handled last id=%d, count=%d", lastID, count)
}
log.Info("warm ups, begin id=%d, expect up size=%d, end id=%d, real count=%d", req.LastID, req.Size, lastID, count)
res = &model.WarmUpReply{
LastID: lastID,
}
return
}
//WarmUpMid warm up by mid
func (s *Service) WarmUpMid(c context.Context, req *model.WarmUpReq) (res *model.WarmUpReply, err error) {
if _, err = s.upRPC.UpArcs(context.Background(), &upGRPCv1.UpArcsReq{
Mid: req.Mid,
Pn: 1,
Ps: 1,
}); err != nil {
log.Error("up rpc UpsArc(%d) return err=%v", req.Mid, err)
return
}
var (
count int64
cntReply *upGRPCv1.UpCountReply
)
if cntReply, err = s.upRPC.UpCount(context.Background(), &upGRPCv1.UpCountReq{
Mid: req.Mid,
}); err != nil {
log.Error("up rpc UpsCount(%d) return err=%v", req.Mid, err)
return
}
if cntReply != nil {
count = cntReply.Count
}
log.Info("warm up(%d) real count=%d", req.Mid, count)
return
}
//AddStaff .
func (s *Service) AddStaff(c context.Context, req *model.AddStaffReq) (res *upGRPCv1.NoReply, err error) {
return s.upRPC.AddUpPassedCacheByStaff(c, &upGRPCv1.UpCacheReq{Aid: req.Aid, Mid: req.StaffMid})
}
//DeleteStaff .
func (s *Service) DeleteStaff(c context.Context, req *model.AddStaffReq) (res *upGRPCv1.NoReply, err error) {
return s.upRPC.DelUpPassedCacheByStaff(c, &upGRPCv1.UpCacheReq{Aid: req.Aid, Mid: req.StaffMid})
}

33
app/job/main/up/service/archive_test.go Normal file
View File

@@ -0,0 +1,33 @@
package service
import (
"testing"
"go-common/app/job/main/up/model/archivemodel"
)
func Test_archiveStateChange(t *testing.T) {
var (
testcase = [][]*archivemodel.ArchiveCanal{
{{State: 0}, {State: -1}},
{{State: -5}, {State: 0}},
{{State: 10}, {State: 10}},
{{State: -5}, {State: -5}},
}
testresult = []bool{
true,
true,
false,
false,
}
)
for i := range testcase {
var cas = testcase[i]
if archiveStateChange(cas[0], cas[1]) != testresult[i] {
t.Errorf("test fail, testcase[%d]=%v, expect=%t", i, cas, testresult[i])
t.Fail()
}
}
}

100
app/job/main/up/service/service.go Normal file
View File

@@ -0,0 +1,100 @@
package service
import (
"context"
"sync"
"time"
"go-common/app/interface/main/mcn/tool/worker"
"go-common/app/job/main/up/conf"
"go-common/app/job/main/up/dao/account"
"go-common/app/job/main/up/dao/email"
"go-common/app/job/main/up/dao/upcrm"
archive "go-common/app/service/main/archive/api"
upGRPCv1 "go-common/app/service/main/up/api/v1"
"go-common/library/queue/databus"
"github.com/robfig/cron"
"go-common/app/admin/main/up/util/databusutil"
)
// Service struct
type Service struct {
c *conf.Config
maildao *email.Dao
crmdb *upcrm.Dao
acc *account.Dao
arcRPC archive.ArchiveClient
cron *cron.Cron
worker *worker.Pool
wg sync.WaitGroup
archiveNotifyT *databus.Databus
archiveT *databus.Databus
closeCh chan struct{}
upRPC upGRPCv1.UpClient
databusHandler *databusutil.DatabusHandler
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
cron: cron.New(),
crmdb: upcrm.New(c),
acc: account.New(c),
maildao: email.New(c),
worker: worker.New(&worker.Conf{
WorkerProcMax: 10,
QueueSize: 1024,
WorkerNumber: 4}),
archiveNotifyT: databus.New(c.DatabusConf.ArchiveNotify),
archiveT: databus.New(c.DatabusConf.Archive),
closeCh: make(chan struct{}),
databusHandler: databusutil.NewDatabusHandler(),
}
var err error
s.arcRPC, err = archive.NewClient(c.GRPCClient.Archive)
if err != nil {
panic(err)
}
if err = s.initEmailTemplate(); err != nil {
panic(err)
}
if s.upRPC, err = upGRPCv1.NewClient(c.GRPCClient.Up); err != nil {
panic(err)
}
s.createJobs()
s.databusHandler.GoWatch(s.archiveNotifyT, s.handleArchiveNotifyT)
s.databusHandler.GoWatch(s.archiveT, s.handleArchiveT)
return s
}
func (s *Service) createJobs() {
s.cron.AddFunc(conf.Conf.Job.UpCheckDateDueTaskTime, cronWrap(s.CheckDateDueJob))
s.cron.AddFunc(conf.Conf.Job.TaskScheduleTime, cronWrap(s.CheckTaskJob))
s.cron.AddFunc(conf.Conf.Job.CheckStateJobTime, cronWrap(s.CheckStateJob))
s.cron.AddFunc(conf.Conf.Job.UpdateUpTidJobTime, cronWrap(s.UpdateUpTidJob))
s.cron.Start()
}
func cronWrap(f func(tm time.Time)) func() {
return func() {
f(time.Now())
}
}
// Ping Service
func (s *Service) Ping(c context.Context) (err error) {
return s.crmdb.Ping(c)
}
// Close Service
func (s *Service) Close() {
s.databusHandler.Close()
s.wg.Wait()
s.crmdb.Close()
}

114
app/job/main/up/service/service_test.go Normal file
View File

@@ -0,0 +1,114 @@
package service
import (
"flag"
"go-common/app/job/main/up/conf"
"go-common/app/job/main/up/dao/upcrm"
"go-common/app/job/main/up/model/signmodel"
"html/template"
"os"
"testing"
)
func TestMain(m *testing.M) {
flag.Set("conf", "../cmd/up-job.toml")
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
m.Run()
os.Exit(0)
}
func TestTemplateSign(t *testing.T) {
var data = &dueData{
Signs: []*upcrm.SignWithName{
{Name: "test", SignUp: signmodel.SignUp{Mid: 123, EndDate: 1540901779}},
{Name: "tes2t", SignUp: signmodel.SignUp{Mid: 1234, EndDate: 1540902000}},
},
}
tmpl, err := template.New("signTitle").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.SignTmplTitle)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
err = tmpl.Execute(os.Stdout, data.Signs)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
tmpl, err = template.New("sign").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.SignTmplContent)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
err = tmpl.Execute(os.Stdout, data.Signs)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
}
func TestTemplatePay(t *testing.T) {
var data = &dueData{
Pays: []*upcrm.PayWithAdmin{
{Name: "test", SignPay: signmodel.SignPay{Mid: 123, DueDate: 1540901779, PayValue: 10000}},
{Name: "test", SignPay: signmodel.SignPay{Mid: 123, DueDate: 1540901779, PayValue: 10000}},
{Name: "test", SignPay: signmodel.SignPay{Mid: 123, DueDate: 1540901779, PayValue: 10000}},
},
}
tmpl, err := template.New("payTitle").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.PayTmplTitle)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
err = tmpl.Execute(os.Stdout, data.Pays)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
tmpl, err = template.New("pay").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.PayTmplContent)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
err = tmpl.Execute(os.Stdout, data.Pays)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
}
func TestTemplateTask(t *testing.T) {
var data = &dueData{
Tasks: []*upcrm.TaskWithAdmin{
{Name: "test", SignTaskHistory: signmodel.SignTaskHistory{Mid: 123, GenerateDate: 1540901779, TaskType: 2, TaskCounter: 1, TaskCondition: 10}},
{Name: "test", SignTaskHistory: signmodel.SignTaskHistory{Mid: 123, GenerateDate: 1540901779, TaskType: 3, TaskCounter: 1, TaskCondition: 10}},
{Name: "test", SignTaskHistory: signmodel.SignTaskHistory{Mid: 123, GenerateDate: 1540901779, TaskType: 0, TaskCounter: 1, TaskCondition: 10}},
{Name: "test", SignTaskHistory: signmodel.SignTaskHistory{Mid: 123, GenerateDate: 1540901779, TaskType: 1, TaskCounter: 1, TaskCondition: 10}},
}}
tmpl, err := template.New("task").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.TaskTmplTitle)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
err = tmpl.Execute(os.Stdout, data.Tasks)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
tmpl, err = template.New("task").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.TaskTmplContent)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
err = tmpl.Execute(os.Stdout, data.Tasks)
if err != nil {
t.Errorf("err=%v", err)
t.FailNow()
}
}

295
app/job/main/up/service/sign_due_job.go Normal file
View File

@@ -0,0 +1,295 @@
package service
import (
"bytes"
"context"
"errors"
"fmt"
"html/template"
"time"
//"go-common/app/job/main/up/conf"
"go-common/app/admin/main/up/util"
"go-common/app/job/main/up/dao/upcrm"
"go-common/app/job/main/up/model/signmodel"
"go-common/app/job/main/up/model/upcrmmodel"
account "go-common/app/service/main/account/model"
"go-common/library/log"
"go-common/app/job/main/up/conf"
)
var (
//ErrNoAdminName no admin name
ErrNoAdminName = errors.New("no admin name")
tmplSignDueTitle *template.Template
tmplSignDueContent *template.Template
tmplPayDueTitle *template.Template
tmplPayDueContent *template.Template
tmplTaskDueTitle *template.Template
tmplTaskDueContent *template.Template
)
// use for template function call
var funcHelper = template.FuncMap{
"Now": time.Now,
}
func (s *Service) initEmailTemplate() (err error) {
if conf.Conf.MailTemplateConf.SignTmplTitle == "" ||
conf.Conf.MailTemplateConf.SignTmplContent == "" ||
conf.Conf.MailTemplateConf.PayTmplTitle == "" ||
conf.Conf.MailTemplateConf.PayTmplContent == "" ||
conf.Conf.MailTemplateConf.TaskTmplTitle == "" ||
conf.Conf.MailTemplateConf.TaskTmplContent == "" {
err = fmt.Errorf(`mail template conf is invalid, check mail-template.toml file, make sure all the following has value:
TaskTmplContent
TaskTmplTitle
PayTmplContent
PayTmplTitle
SignTmplContent
SignTmplTitle`)
return
}
tmplSignDueTitle, err = template.New("signTitle").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.SignTmplTitle)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplSignDueContent, err = template.New("signContent").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.SignTmplContent)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplPayDueTitle, err = template.New("payTitle").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.PayTmplTitle)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplPayDueContent, err = template.New("payContent").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.PayTmplContent)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplTaskDueTitle, err = template.New("taskTitle").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.TaskTmplTitle)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
tmplTaskDueContent, err = template.New("taskContent").Funcs(funcHelper).Parse(conf.Conf.MailTemplateConf.TaskTmplContent)
if err != nil {
log.Error("parse template fail, err=%v", err)
return
}
return
}
//CheckDateDueJob check task due
/*
快到期的job提醒
*/
func (s *Service) CheckDateDueJob(date time.Time) {
log.Info("start run CheckDateDueJob, date=%s", date)
s.checkSignUpDue(date)
log.Info("finish run CheckDateDueJob, date=%s", date)
}
type dueData struct {
Signs []*upcrm.SignWithName
Pays []*upcrm.PayWithAdmin
Tasks []*upcrm.TaskWithAdmin
}
func (d *dueData) addSign(sign *upcrm.SignWithName) {
d.Signs = append(d.Signs, sign)
}
func (d *dueData) addPay(pay *upcrm.PayWithAdmin) {
d.Pays = append(d.Pays, pay)
}
func (d *dueData) addTask(task *upcrm.TaskWithAdmin) {
d.Tasks = append(d.Tasks, task)
}
func getOrCreate(dataMap map[string]*dueData, key string) *dueData {
var data, ok = dataMap[key]
if !ok {
data = &dueData{}
dataMap[key] = data
}
return data
}
func getName(infoMap map[int64]*account.Info, mid int64) string {
if info, ok := infoMap[mid]; ok {
return info.Name
}
return ""
}
func (s *Service) checkSignUpDue(date time.Time) {
// 30天内到期 sign
list, err := s.crmdb.GetDueSignUp(date, 30)
s.crmdb.StartTask(upcrmmodel.TaskTypeSignCheckDue, date)
defer func() {
if err == nil {
s.crmdb.FinishTask(upcrmmodel.TaskTypeSignCheckDue, date, upcrmmodel.TaskStateFinish)
} else {
s.crmdb.FinishTask(upcrmmodel.TaskTypeSignCheckDue, date, upcrmmodel.TaskStateError)
}
}()
if err != nil {
log.Error("fail to get due sign, date=%+v, err=%+v", date, err)
return
}
var adminDueDataMap = make(map[string]*dueData)
var ids []int64
for _, v := range list {
ids = append(ids, v.Mid)
var data = getOrCreate(adminDueDataMap, v.AdminName)
data.addSign(v)
}
// 7天内到期的pay
listPayDue, err := s.crmdb.GetDuePay(date, 7)
if err != nil {
log.Error("fail to get due pay, date=%+v, err=%+v", date, err)
return
}
for _, v := range listPayDue {
ids = append(ids, v.Mid)
var data = getOrCreate(adminDueDataMap, v.AdminName)
data.addPay(v)
}
// 到期的任务，
listTaskDue, err := s.crmdb.GetDueTask(date)
if err != nil {
log.Error("fail to get due task, date=%+v, err=%+v", date, err)
return
}
for _, v := range listTaskDue {
ids = append(ids, v.Mid)
var data = getOrCreate(adminDueDataMap, v.AdminName)
data.addTask(v)
}
ids = util.Unique(ids)
infoMap, e := s.acc.GetCachedInfos(context.Background(), ids, "")
if e == nil {
for _, v := range list {
v.Name = getName(infoMap, v.Mid)
}
for _, v := range listPayDue {
v.Name = getName(infoMap, v.Mid)
}
for _, v := range listTaskDue {
v.Name = getName(infoMap, v.Mid)
}
}
for admin, v := range adminDueDataMap {
var adminAll = append(conf.Conf.MailConf.DueMailReceivers, admin)
// 发送sign到期邮件
var due = v
s.worker.Add(func() {
var succIds []uint32
if len(due.Signs) > 0 {
var e = s.sendMailWithTemplate(due.Signs, tmplSignDueTitle, tmplSignDueContent, adminAll...)
if e == nil {
for _, data := range due.Signs {
succIds = append(succIds, data.ID)
}
} else {
log.Warn("fail to send email, err=%v", e)
}
// 更新邮件发送标记
s.crmdb.UpdateEmailState(signmodel.TableNameSignUp, succIds, signmodel.EmailStateSendSucc)
}
if len(due.Pays) > 0 {
// 发送pay到期邮件
succIds = nil
e = s.sendMailWithTemplate(due.Pays, tmplPayDueTitle, tmplPayDueContent, adminAll...)
if e == nil {
for _, data := range due.Pays {
succIds = append(succIds, data.ID)
}
} else {
log.Warn("fail to send email, err=%v", e)
}
// 更新邮件发送标记
s.crmdb.UpdateEmailState(signmodel.TableNameSignPay, succIds, signmodel.EmailStateSendSucc)
}
if len(due.Tasks) > 0 {
// 发送task到期邮件
e = s.sendMailWithTemplate(due.Tasks, tmplTaskDueTitle, tmplTaskDueContent, adminAll...)
if e != nil {
log.Warn("fail to send email, err=%v", e)
}
// 这个没有邮件发送的标记
//s.crmdb.UpdateEmailState(signmodel.TableNameSignPay, succIds, signmodel.EmailStateSendSucc)
}
})
}
}
// data, data to generate email content
// contentTmpl, template to generate email content
// adminname, slice for all admin name
//
func (s *Service) sendMailWithTemplate(data interface{}, subjectTmpl, contentTmpl *template.Template, adminName ...string) (err error) {
if contentTmpl == nil {
err = fmt.Errorf("template for email is nil, data=%+v", data)
log.Error("%s", err)
return
}
var contentBuf = bytes.NewBuffer(nil)
err = contentTmpl.Execute(contentBuf, data)
if err != nil {
log.Error("template fail to execute, err=%v", err)
return
}
var subjectBuf = bytes.NewBuffer(nil)
err = subjectTmpl.Execute(subjectBuf, data)
if err != nil {
log.Error("template fail to execute, err=%v", err)
return
}
var addrs []string
for _, v := range adminName {
if v == "" {
log.Warn("admin name is empty")
continue
}
var addr = fmt.Sprintf("%s@bilibili.com", v)
addrs = append(addrs, addr)
}
//log.Info("email sub=%s, content=%s", subjectBuf.String(), contentBuf.String())
if len(addrs) == 0 {
log.Error("admin name is empty, cannot send email, data=%+v", data)
err = ErrNoAdminName
return
}
log.Info("email send , sub=%s, admin=%s, data=%+v", subjectBuf.String(), adminName, data)
err = s.maildao.SendMail(contentBuf.String(), subjectBuf.String(), addrs)
return
}

134
app/job/main/up/service/sign_state_job.go Normal file
View File

@@ -0,0 +1,134 @@
package service
import (
"github.com/jinzhu/gorm"
"go-common/app/job/main/up/dao/upcrm"
"go-common/app/job/main/up/model/signmodel"
"go-common/app/service/main/upcredit/mathutil"
"go-common/library/log"
"time"
)
// change a lot state for sign up
// state, 签约状态，正常或过期
// due_wan, 签约状态即将过期
// pay_expire_state，付费状态
//CheckStateJob 检查sign_up中的状态
func (s *Service) CheckStateJob(date time.Time) {
log.Info("start run state job, date=%s", date)
s.checkState(date)
log.Info("finish run state job, date=%s", date)
}
func (s *Service) checkState(date time.Time) {
// 更新签约状态：
var crmdb = s.crmdb.GetDb()
// 签约30天内到期
var signExpireList []*signmodel.SignUp
var signDueList []*signmodel.SignUp
var limit, offset = 200, 0
var count = limit
var err error
var signEndDate = date.AddDate(0, 0, 30)
for count == limit {
var items []*signmodel.SignUp
// 把endDate在[-60, +30]范围内的用户全都找出来，过一遍
err = crmdb.Offset(offset).Limit(limit).Where("end_date<? and end_date>?", signEndDate, date.AddDate(0, -4, 0)).Find(&items).Error
if err != nil {
log.Error("fail to get sign ups, err=%v", err)
break
}
count = len(items)
offset += count
for _, v := range items {
if v.EndDate.Time().Before(date) && v.State == signmodel.SignStateOnSign {
// 过期
signExpireList = append(signExpireList, v)
} else if !(v.EndDate.Time().Before(date) || v.EndDate.Time().After(signEndDate)) && v.DueWarn <= signmodel.DueWarnNoWarn {
// 快到期
signDueList = append(signDueList, v)
}
}
}
var expireIDList []uint32
for _, v := range signExpireList {
expireIDList = append(expireIDList, v.ID)
}
err = updateListWithLimit(crmdb, "state", signmodel.SignStateExpire, expireIDList, limit)
if err != nil {
log.Error("fail to update sign state, err=%v", err)
return
}
var dueIDList []uint32
for _, v := range signDueList {
dueIDList = append(dueIDList, v.ID)
}
err = updateListWithLimit(crmdb, "due_warn", signmodel.DueWarnWarn, dueIDList, limit)
if err != nil {
log.Error("fail to update sign due_warn, err=%v", err)
return
}
// 付款7天内到期
offset = 0
count = limit
var payDueDate = date.AddDate(0, 0, 7)
var payExpireMap = map[uint32]int8{}
for count == limit {
var items []*signmodel.SignPay
// 把endDate在[-max, +7]范围内的用户全都找出来，过一遍
err = crmdb.Offset(offset).Limit(limit).Where("due_date<?", payDueDate).Find(&items).Error
if err != nil {
log.Error("fail to get sign ups, err=%v", err)
break
}
count = len(items)
offset += count
for _, v := range items {
// 如果有即将到期，则立即标记为即将到期
if v.State == upcrm.PayStateUnpay {
payExpireMap[v.SignID] = signmodel.PayExpireStateDue
} else {
// 如果没有即将到期的状态，则把其标记为未到期
// 即：只有所有的付款都已标记完成，才会认为是未到期状态
if payExpireMap[v.SignID] != signmodel.PayExpireStateDue {
payExpireMap[v.SignID] = signmodel.PayExpireStateNormal
}
}
}
}
// 更新到期状态
var stateSignIDListMap = map[int8][]uint32{}
for k, v := range payExpireMap {
stateSignIDListMap[v] = append(stateSignIDListMap[v], k)
}
// 分区更新
limit = 200
for state, list := range stateSignIDListMap {
err = updateListWithLimit(crmdb, "pay_expire_state", state, list, limit)
if err != nil {
log.Error("err update pay_expire_state, err=%v", err)
return
}
}
}
func updateListWithLimit(crmdb *gorm.DB, field string, state int8, list []uint32, limit int) (err error) {
for begin := 0; begin < len(list); begin += limit {
var end = mathutil.Min(begin+limit, len(list))
var needUpdate = list[begin:end]
err = updateSignTable(crmdb, needUpdate, field, state)
if err != nil {
log.Error("fail to update state [%s], err=%v", field, err)
return
}
}
return
}
func updateSignTable(crmdb *gorm.DB, ids interface{}, field string, state int8) (err error) {
err = crmdb.Table(signmodel.TableNameSignUp).Where("id in (?)", ids).Update(field, state).Error
return
}

300
app/job/main/up/service/sign_task_job.go Normal file
View File

@@ -0,0 +1,300 @@
package service
import (
"context"
"time"
"go-common/app/admin/main/up/util"
"go-common/app/job/main/up/conf"
"go-common/app/job/main/up/dao/upcrm"
"go-common/app/job/main/up/model/signmodel"
"go-common/app/job/main/up/model/upcrmmodel"
v1 "go-common/app/service/main/archive/api"
"go-common/library/log"
xtime "go-common/library/time"
"github.com/jinzhu/gorm"
)
//CheckTaskJob check task job
func (s *Service) CheckTaskJob(tm time.Time) {
// 今天计算昨天的数据
var yesterday = tm.AddDate(0, 0, -1)
log.Info("start to run CheckTaskJob, date=%s, yesterday=%s", tm, yesterday)
s.CheckTaskFinish(yesterday)
log.Info("finish run CheckTaskJob, date=%s", tm)
}
//Archive data
type Archive struct {
ID int64 `gorm:"column:id"`
}
//CheckTaskFinish check task finish, calculate datas in (-,date]
func (s *Service) CheckTaskFinish(date time.Time) {
// 1.查找所有有效的合同id, begin_date <= date && end _date >= date
// 2.找到所有合同id对应的任务id,
// 3.根据任务类型，日、周、月、累计，计算任务周期[a,b)
// 4.计算完成数量
var crmdb = s.crmdb.GetDb()
var dateStr = date.Format(upcrmmodel.TimeFmtDate)
var offset = 0
var limit = 200
var actualSize = limit
log.Info("start to check task state")
archiveDb, err := gorm.Open("mysql", conf.Conf.ArchiveOrm.DSN)
s.crmdb.StartTask(upcrmmodel.TaskTypeSignTaskCalculate, date)
archiveDb.LogMode(true)
if err != nil {
log.Error("connect archive db fail")
return
}
defer archiveDb.Close()
defer func() {
if err == nil {
s.crmdb.FinishTask(upcrmmodel.TaskTypeSignTaskCalculate, date, upcrmmodel.TaskStateFinish)
} else {
s.crmdb.FinishTask(upcrmmodel.TaskTypeSignTaskCalculate, date, upcrmmodel.TaskStateError)
}
}()
var taskTotalCount = 0
for actualSize == limit {
var signUps []*signmodel.SignUp
var signUpMap = make(map[uint32]*signmodel.SignUp)
// 1
err = crmdb.Table(signmodel.TableNameSignUp).
Select("id, begin_date, end_date").
Where("begin_date <= ? and end_date >= ?", dateStr, dateStr).
Offset(offset).
Limit(limit).
Find(&signUps).Error
offset += limit
if err != nil && err != gorm.ErrRecordNotFound {
log.Error("err get signs, err=%+v", err)
return
}
actualSize = len(signUps)
var signIDs []uint32
for _, v := range signUps {
signUpMap[v.ID] = v
signIDs = append(signIDs, v.ID)
}
// 2
var taskList []*signmodel.SignTask
err = crmdb.Where("sign_id in (?) and state != ? and generate_date<?", signIDs, signmodel.SignTaskStateDelete, dateStr).
Find(&taskList).Error
if err != nil {
log.Error("err get tasks, err=%+v", err)
return
}
// 3
for _, task := range taskList {
taskTotalCount++
if task.Mid == 0 {
log.Error("task's mid is zero, please check! task id=%d", task.ID)
continue
}
var signInfo, ok = signUpMap[task.SignID]
if !ok {
log.Error("sign not found, err=%v", err)
continue
}
// 4 计算数量
err = s.checkSingleTask(task, signInfo, date, archiveDb)
if err != nil {
log.Error("check task err, task_id=%d, err=%v", task.ID, err)
continue
}
}
log.Info("finish to check task state, task total num=%d", taskTotalCount)
}
}
// get task history, if not exist, then will create it
func (s *Service) getOrCreateTaskHistory(task *signmodel.SignTask, generateDate time.Time) (res *signmodel.SignTaskHistory, err error) {
var crmdb = s.crmdb.GetDb()
res = new(signmodel.SignTaskHistory)
err = crmdb.Select("*").Where("task_template_id=? and generate_date=?", task.ID, generateDate).
Find(&res).Error
// 创建一条，如果没找到的话
if err == gorm.ErrRecordNotFound {
res = &signmodel.SignTaskHistory{
Mid: task.Mid,
SignID: task.SignID,
TaskTemplateID: task.ID,
TaskType: task.TaskType,
TaskCondition: task.TaskCondition,
Attribute: task.Attribute,
GenerateDate: xtime.Time(generateDate.Unix()),
State: signmodel.SignTaskStateRunning,
}
err = crmdb.Save(&res).Error
if err != nil {
log.Error("create task history fail, err=%v, task=%v", err, task)
return
}
}
return
}
// check task state
func (s *Service) checkSingleTask(task *signmodel.SignTask, signInfo *signmodel.SignUp, date time.Time, archiveDb *gorm.DB) (err error) {
var taskBegin, taskEnd time.Time
if task.TaskType == signmodel.TaskTypeAccumulate {
taskBegin = signInfo.BeginDate.Time()
taskEnd = signInfo.EndDate.Time()
} else {
taskBegin, taskEnd = upcrm.GetTaskDuration(date, task.TaskType)
}
if task.Mid == 0 {
log.Error("task's mid is zero, please check! task id=%d", task.ID)
return
}
// get task history
// 如果是累计任务，这里的taskBegin要设置为0
var tBegin = taskBegin
if task.TaskType == signmodel.TaskTypeAccumulate {
tBegin = time.Time{}
}
taskHistory, err := s.getOrCreateTaskHistory(task, tBegin)
if err != nil {
log.Error("get task history fail, task=%+v, err=%+v", task, err)
return
}
var dateStr = date.Format(upcrmmodel.TimeFmtDate)
switch {
default:
var crmdb = s.crmdb.GetDb()
// 4.去稿件库中查找对应的稿件数量
var archiveCount = 0
var archiveList []*Archive
err = archiveDb.Table("archive").
Where("mid = ? and ctime>= ? and ctime <? and (state >= 0 or state = -6)",
task.Mid, taskBegin, taskEnd).
Select("id").
Find(&archiveList).
Error
if err != nil && err != gorm.ErrRecordNotFound {
log.Error("check archive count fail, taskid=%+v err=%+v", task, err)
break
}
var finalResult []int64
// 5.任务完成度统计时，
// 若录入为不包含商单，
// 则任务完成数=新增稿件数-减去（绿洲/商单报备）稿件数+请假任务数；
// 若录入为包含商单，
// 则任务完成数=新增稿件数+请假任务数；
// 如果没有archive，就直接返回
if len(archiveList) != 0 {
// 需要判断商单
if task.IsAttrSet(signmodel.SignTaskAttrBitBusiness) {
var ids []int64
for _, v := range archiveList {
ids = append(ids, v.ID)
}
ids = util.Unique(ids)
// 查询archive服务
archiveResult, e := s.arcRPC.Arcs(context.Background(), &v1.ArcsRequest{Aids: ids})
if e != nil {
err = e
log.Error("get archive result err, err=%+v", err)
break
}
for _, v := range archiveList {
a, ok := archiveResult.Arcs[v.ID]
// 是商单的要排除
if !ok || a.OrderID > 0 {
continue
}
finalResult = append(finalResult, v.ID)
}
} else {
for _, v := range archiveList {
finalResult = append(finalResult, v.ID)
}
}
archiveCount = len(finalResult)
}
// 请假任务数
var absence signmodel.SignTaskAbsence
err = crmdb.Select("sum(absence_count) as absence_count").
Where("task_history_id=? and state!=?", taskHistory.ID, signmodel.SignTaskAbsenceStateDelete).
Find(&absence).Error
if err != nil {
log.Error("get task absence fail, task history=%+v", taskHistory)
return
}
archiveCount += int(absence.AbsenceCount)
log.Info("task count=%d, archive=%d, absence=%d, task=%+v", archiveCount, len(finalResult), absence.AbsenceCount, taskHistory)
// 更新task history的数量
task.TaskCounter = int32(archiveCount)
var tx = crmdb.Begin()
defer func() {
if r := recover(); r != nil || err != nil {
log.Error("roll back task update, task=%+v, r=%+v | err=%+v", task, r, err)
tx.Rollback()
}
}()
err = tx.Table(signmodel.TableNameSignTask).Where("id=?", task.ID).
Updates(map[string]interface{}{
"generate_date": dateStr,
}).Error
if err != nil {
log.Error("update sign task fail, task=%+v, err=%+v", task, err)
return
}
// update history
var state = signmodel.SignTaskStateRunning
if archiveCount >= int(task.TaskCondition) {
state = signmodel.SignTaskStateFinish
}
err = tx.Table(signmodel.TableNameSignTaskHistory).Where("id=?", taskHistory.ID).
Updates(map[string]interface{}{
"task_counter": task.TaskCounter,
"task_condition": task.TaskCondition,
"state": state,
"attribute": task.Attribute,
"task_type": task.TaskType,
}).Error
if err != nil {
log.Error("update sign task history fail, task=%+v, err=%+v", task, err)
return
}
// update sign
err = tx.Table(signmodel.TableNameSignUp).Where("id=?", task.SignID).
Updates(map[string]interface{}{
"task_state": state,
}).Error
if err != nil {
log.Error("update sign up fail, task=%+v, err=%+v", task, err)
return
}
err = tx.Commit().Error
if err != nil {
log.Error("commit err, err=%+v", err)
}
}
return
}

66
app/job/main/up/service/sign_update_tid_job.go Normal file
View File

@@ -0,0 +1,66 @@
package service
import (
"go-common/app/admin/main/up/util"
"go-common/app/admin/main/up/util/mathutil"
"go-common/app/job/main/up/model/signmodel"
"go-common/app/job/main/up/model/upcrmmodel"
"go-common/library/log"
"time"
)
// UpdateUpTidJob 检查sign_up中的状态
func (s *Service) UpdateUpTidJob(date time.Time) {
log.Info("start run UpdateUpTidJob, date=%s", date)
s.updateUpTidJob(date)
log.Info("finish run UpdateUpTidJob, date=%s", date)
}
func (s *Service) updateUpTidJob(date time.Time) {
// 获取所有mid信息
var limit = 200
var count = limit
var err error
var mids []int64
var offset = 0
for limit == count {
var signList []*signmodel.SignUp
if err = s.crmdb.GetDb().Offset(offset).Limit(limit).Find(&signList).Error; err != nil {
log.Error("fail to get signs from sign ups, err=%v", err)
return
}
count = len(signList)
offset += count
for _, v := range signList {
mids = append(mids, v.Mid)
}
}
mids = util.Unique(mids)
// 从up_base_info中读取tid并更新
// <tid, mid list>
var tidMidMap = make(map[int64][]int64)
for begin := 0; begin < len(mids); begin += limit {
var end = mathutil.Min(begin+limit, len(mids))
var baseInfoList []*upcrmmodel.UpBaseInfo
if err = s.crmdb.GetDb().Select("mid, active_tid").Where("mid in (?)", mids[begin:end]).Limit(limit).Find(&baseInfoList).Error; err != nil {
log.Error("fail to get signs from sign ups, err=%v", err)
return
}
// 更新到sign表中
for _, v := range baseInfoList {
tidMidMap[v.ActiveTid] = append(tidMidMap[v.ActiveTid], v.Mid)
}
}
// 更新到sign_up表中
for k, v := range tidMidMap {
if err = s.crmdb.GetDb().Table(signmodel.TableNameSignUp).Where("mid in (?)", v).Update("active_tid", k).Error; err != nil {
log.Error("update sign up's active tid fail, err=%v", err)
return
}
}
}