Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@ -0,0 +1,28 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/upcredit/cmd:all-srcs",
"//app/service/main/upcredit/common/election:all-srcs",
"//app/service/main/upcredit/common/fsm:all-srcs",
"//app/service/main/upcredit/conf:all-srcs",
"//app/service/main/upcredit/dao/account:all-srcs",
"//app/service/main/upcredit/dao/monitor:all-srcs",
"//app/service/main/upcredit/dao/upcrmdao:all-srcs",
"//app/service/main/upcredit/http:all-srcs",
"//app/service/main/upcredit/mathutil:all-srcs",
"//app/service/main/upcredit/model:all-srcs",
"//app/service/main/upcredit/rpc/client:all-srcs",
"//app/service/main/upcredit/rpc/server:all-srcs",
"//app/service/main/upcredit/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,33 @@
#### up主信用分服务
##### Version 0.1.10
> 1.修复election的问题
##### Version 0.1.9
> 1.补充UT
##### Version 0.1.8
> 1.修改identify服务为verify【基础服务更新】
##### Version 0.1.7
> 1.增加master选举功能
##### Version 0.1.6
> 1.去掉parser
##### Version 0.1.5
> 1.批量写入以解决DB QPS过高的问题
##### Version 0.1.4
> 1.修复DB QPS过高的问题
##### Version 0.1.3
> 1.修复慢查询的问题
##### Version 0.1.2
> 1.修复分数没有写入历史记录库
##### Version 0.1.1
> 1.修复数据库字段错误问题
##### Version 0.1.0
> 1.基本框架

View File

@ -0,0 +1,11 @@
# Owner
shencen
wangzhe01
# Author
wangzhe01
yanjinbin
# Reviewer
shencen
wangzhe01

View File

@ -0,0 +1,16 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- shencen
- wangzhe01
- yanjinbin
labels:
- main
- service
- service/main/upcredit
options:
no_parent_owners: true
reviewers:
- shencen
- wangzhe01
- yanjinbin

View File

@ -0,0 +1,10 @@
#### up主信用分服务
##### 项目简介
> 1.up主信用分服务
##### 编译环境
> 请只用golang v1.8.x以上版本编译执行。
##### 依赖包
> 1.公共包go-common

View File

@ -0,0 +1,46 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = [
"credit_score_conf.toml",
"upcredit-service.toml",
],
importmap = "go-common/app/service/main/upcredit/cmd",
importpath = "go-common/app/service/main/upcredit/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/http:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,26 @@
[CalculateConf]
[CalculateConf.TimeWeight]
# map
# year_from_now = weight
# 0 = 100 , means this year's score weight is 100
0 = 100
1 = 60
2 = 30
3 = 10
[ArticleRule]
AcceptOptypeData = [0, 1]
RejectOpTypeData = [-2, -3, -4, -5]
ArticleMaxOpenCount = 5 #一个最大开放次数超过之后将不再记录由于一个稿件可以被up主不停的编辑并提交状态都会变成close然后再open
[ArticleRule.OptypeScoreData]
# score = [state]
# state will be mapped to a credit score
5 = [0] # 正常通过
2 = [1] # 橙色通过
-10 = [ -2, -3, -4, -5]
[ArticleRule.InitState]
state = -1
round = 0
reason = 0

View File

@ -0,0 +1,50 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/http"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
log.Init(conf.Conf.Xlog)
trace.Init(conf.Conf.Tracer)
defer trace.Close()
defer log.Close()
log.SetFormat("[%D %T] [%L] [%S] %M")
log.Info("serverstart")
// service init
http.Init(conf.Conf)
// init signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("server get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
http.Svc.Close()
log.Info("serverexit")
time.Sleep(1 * time.Second)
return
case syscall.SIGHUP:
// TODO reload
default:
return
}
}
}

View File

@ -0,0 +1,238 @@
# This is a TOML document. Boom.
version = "1.0.0"
user = "nobody"
pid = "/tmp/upcredit-service.pid"
dir = "./"
log = "/data/log/upcredit-service/"
family = "upcredit-service"
trace = false
debug = false
istest = true
[app]
key = "b8f239ca38a53308"
secret = "5460ef72fe13c10dfb53442b9111427e"
[xlog]
dir = "/data/log/upcredit-service/"
family = "upcredit-service"
stdout = true
[statsd]
project = "upcredit-service"
addr = "172.18.20.15:8200"
chanSize = 10240
[httpClient]
[httpClient.normal]
key = "b8f239ca38a53308"
secret = "5460ef72fe13c10dfb53442b9111427e"
dial = "500ms"
timeout = "1s"
keepAlive = "60s"
timer = 10
[httpClient.normal.breaker]
window = "10s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[httpClient.slow]
key = "b8f239ca38a53308"
secret = "5460ef72fe13c10dfb53442b9111427e"
dial = "1s"
timeout = "10s"
keepAlive = "60s"
timer = 10
[httpClient.slow.breaker]
window = "10s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[bm]
[bm.inner]
addr = "0.0.0.0:7730"
maxListen = 1000
timeout = "1s"
[bm.local]
addr = "0.0.0.0:7731"
maxListen = 100
timeout = "1s"
[db]
[db.upcrm]
dsn = "upcrm:DdL6c5JaWCYKMAQ10PURbfeImow9HXlx@tcp(172.16.33.205:3306)/bilibili_upcrm?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 1
idleTimeout ="4h"
# crm读库
[db.upcrmreader]
dsn = "upcrm:DdL6c5JaWCYKMAQ10PURbfeImow9HXlx@tcp(172.16.33.205:3306)/bilibili_upcrm?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 1
idleTimeout ="4h"
[ecode]
domain = "uat-api.bilibili.co"
all = "1h"
diff = "5m"
[ecode.clientconfig]
key = "c05dd4e1638a8af0"
secret = "7daa7f8c06cd33c5c3067063c746fdcb"
dial = "2000ms"
timeout = "2s"
keepAlive = "10s"
timer = 128
[ecode.clientconfig.breaker]
window ="3s"
sleep ="100ms"
bucket = 10
ratio = 0.5
request = 100
[identify]
whiteAccessKey = ""
whiteMid = 0
csrf = false
[identify.app]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
[identify.memcache]
name = "go-business/identify"
proto = "tcp"
addr = "/tmp/uat-platform-identify-mc.sock"
active = 10
idle = 5
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
[identify.host]
auth = "http://uat-passport.bilibili.com"
secret = "http://uat-open.bilibili.com"
[identify.httpClient]
key = "53e2fa226f5ad348"
secret = "3cf6bd1b0ff671021da5f424fea4b04a"
dial = "500ms"
timeout = "800ms"
keepAlive = "60s"
[identify.httpClient.breaker]
window = "10s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[identify.httpClient.url]
"http://uat-passport.bilibili.co/intranet/auth/tokenInfo" = {timeout = "100ms"}
"http://uat-passport.bilibili.co/intranet/auth/cookieInfo" = {timeout = "100ms"}
"http://uat-open.bilibili.co/api/getsecret" = {timeout = "500ms"}
[monitor]
moni="http://moni.bilibili.co/dashboard/db/databus?orgId=1"
host = "http://bap.bilibili.co"
username="fengpengfei"
appToken = "jLgSvndTeranxGMN"
appSecret = "CcCdlEBYqxqrgAieJuFVZUsgPmweLRms"
intervalAlarm = "5m"
[auth]
managerHost = "http://uat-manager.bilibili.co"
dashboardHost = "http://dashboard-mng.bilibili.co"
dashboardCaller = "manager-go"
[auth.DsHTTPClient]
key = "manager-go"
secret = "949bbb2dd3178252638c2407578bc7ad"
dial = "1s"
timeout = "1s"
keepAlive = "60s"
[auth.DsHTTPClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[auth.MaHTTPClient]
key = "f6433799dbd88751"
secret = "36f8ddb1806207fe07013ab6a77a3935"
dial = "1s"
timeout = "1s"
keepAlive = "60s"
[auth.MaHTTPClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[auth.session]
sessionIDLength = 32
cookieLifeTime = 1800
cookieName = "mng-go"
domain = ".bilibili.co"
[auth.session.Memcache]
name = "go-business/auth"
proto = "unix"
addr = "/tmp/uat-manager-auth-mc.sock"
active = 10
idle = 5
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "80s"
[CreditLogSub]
key = "0PvKGhAqDvsK7zitmS8t"
secret = "0PvKGhAqDvsK7zitmS8u"
group = "UpCreditLog-MainArchive-S"
topic = "UpCreditLog-T"
action = "sub"
offset = "old"
buffer = 128
name = "upcredit-service/sub"
proto = "tcp"
addr = "172.16.33.158:6205"
idle = 1
active = 10
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "5s"
upChanSize = 1024
consumeLimit = 10
routineLimit = 5
[BusinessBinLogSub]
key = "0PvKGhAqDvsK7zitmS8t"
secret = "0PvKGhAqDvsK7zitmS8u"
group = "BusinessUpBinLog-MainArchive-S"
topic = "BusinessUpBinLog-T"
action = "sub"
offset = "old"
buffer = 128
name = "upcredit-service/sub"
proto = "tcp"
addr = "172.16.33.158:6205"
idle = 1
active = 10
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "5s"
upChanSize = 1024
[RunStatJobConf]
StartTime = "16:41:00"
WorkerNumber = 10
[MiscConf]
CreditLogWriteRoutineNum = 2
BusinessBinLogLimitRate = 500.0
[ElectionZooKeeper]
Root = "/microservice/upcredit-service"
Addrs = ["172.18.33.50:2199"]
Timeout = "5s"

View File

@ -0,0 +1,42 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["example_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = ["//library/log:go_default_library"],
)
go_library(
name = "go_default_library",
srcs = ["election.go"],
importpath = "go-common/app/service/main/upcredit/common/election",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/log:go_default_library",
"//vendor/github.com/samuel/go-zookeeper/zk:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,184 @@
package election
import (
"errors"
"github.com/samuel/go-zookeeper/zk"
"go-common/library/log"
"path"
"sort"
"time"
)
var (
//ErrNotInit init fail
ErrNotInit = errors.New("init first")
//ErrFailConn fail to connect
ErrFailConn = errors.New("fail to connect zk")
)
//ZkElection election by zk
type ZkElection struct {
dir string
servers []string
conn *zk.Conn
timeout time.Duration
RootPath string
NodePath string
MasterPath string
IsMaster bool
// wait for this channel, true means master, false means follower
C <-chan bool
leaderChan chan bool
running bool
}
// New create new ZkElection
func New(servers []string, dir string, timeout time.Duration) *ZkElection {
var z = &ZkElection{}
z.servers = servers
z.dir = dir
z.timeout = timeout
z.running = true
return z
}
//Init init the elections
// dir is root path for election, if dir = "/project", then, election will use "/project/election" as election path
// and node would be "/project/election/n_xxxxxxxx"
// if error happens, the election would work
func (z *ZkElection) Init() (err error) {
z.conn, _, err = zk.Connect(z.servers, z.timeout)
if err != nil {
log.Error("fail connect zk, err=%s", err)
return
}
z.RootPath = path.Join(z.dir, "election")
exist, _, err := z.conn.Exists(z.RootPath)
if err != nil {
log.Error("fail to check path, path=%s, err=%v", z.RootPath, err)
return
}
if !exist {
_, err = z.conn.Create(z.RootPath, []byte(""), 0, zk.WorldACL(zk.PermAll))
if err != nil {
log.Error("create election fail, path=%s, err=%v", z.RootPath, err)
return
}
}
var pathPrefix = path.Join(z.RootPath, "/n_")
z.NodePath, err = z.conn.Create(pathPrefix, []byte(""), zk.FlagEphemeral|zk.FlagSequence, zk.WorldACL(zk.PermAll))
if err != nil {
log.Error("fail create node path, path=%s, err=%s", pathPrefix, err)
return
}
return
}
//Elect elect for leader
// wait for the chan, if get a true, mean you are the leader
// if you already a leader, a false means you are kicked
func (z *ZkElection) Elect() (err error) {
if z.conn == nil {
return ErrNotInit
}
if z.leaderChan != nil {
return
}
z.leaderChan = make(chan bool)
z.C = z.leaderChan
go func() {
defer close(z.leaderChan)
var ch, err = z.watchChange()
if err != nil {
log.Error("fail to watch, err=%s", err)
return
}
for z.running {
event := <-ch
switch event.Type {
case zk.EventNodeDeleted:
ch, err = z.watchChange()
if err != nil {
log.Error("fail to watch, try next time in seconds, err=%s", err)
time.Sleep(time.Second * 5)
}
case zk.EventNotWatching:
log.Warn("receive not watching event, event=%+v", event)
if event.State == zk.StateDisconnected {
log.Info("reinit zk")
if err = z.Init(); err != nil {
log.Error("err init zk again, sleep 5s, err=%v", err)
time.Sleep(5*time.Second)
} else {
ch, err = z.watchChange()
if err != nil {
log.Error("fail to watch, err=%s", err)
return
}
}
}
}
log.Info("zk event, event=%+v", event)
}
log.Error("exit election proc")
}()
return
}
func (z *ZkElection) watchChange() (ch <-chan zk.Event, err error) {
children, _, e := z.conn.Children(z.RootPath)
if err != nil {
err = e
log.Error("get children error, err=%+v\n", err)
return
}
if len(children) == 0 {
log.Warn("no child get from root: %s", z.RootPath)
return
}
var min string
if len(children) > 0 {
sort.SliceStable(children, func(i, j int) bool {
return children[i] < children[j]
})
min = children[0]
z.MasterPath = min
}
var nodebase = path.Base(z.NodePath)
for i, v := range children {
if v == nodebase {
if min == nodebase {
log.Info("this is master, node=%s", z.NodePath)
z.IsMaster = true
z.leaderChan <- true
_, _, ch, err = z.conn.GetW(z.NodePath)
} else {
log.Info("master is %s", min)
prev := children[i-1]
var preNode = path.Join(z.RootPath, prev)
_, _, ch, err = z.conn.GetW(preNode)
if err != nil {
log.Error("watch node fail, node=%s, err=%s", preNode, err)
}
z.IsMaster = false
z.leaderChan <- false
}
} else {
log.Warn("v=%s, not same with base, base=%s", v, nodebase)
}
}
log.Info("watchChange, len(children)=%d", len(children))
return
}
//Close close the election
func (z *ZkElection) Close() {
z.running = false
}

View File

@ -0,0 +1,30 @@
package election
import (
"go-common/library/log"
"testing"
"time"
)
func TestElection(t *testing.T) {
//log.AddFilter("all", log.DEBUG, log.NewConsoleLogWriter())
var hosts = []string{"172.18.33.50:2199"}
var root = "/microservice/upcredit-service/nodes"
var elect = New(hosts, root, time.Second*5)
var err = elect.Init()
if err != nil {
log.Error("fail to init elect")
return
}
elect.Elect()
for {
isMaster := <-elect.C
if isMaster {
log.Info("this is master, node=%s", elect.NodePath)
} else {
log.Info("this is follower, node=%s, master=%s", elect.NodePath, elect.MasterPath)
}
}
}

View File

@ -0,0 +1,28 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.sublime-project
*.sublime-workspace
.DS_Store
.wercker

View File

@ -0,0 +1,45 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"errors_test.go",
"fsm_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = [
"errors.go",
"event.go",
"fsm.go",
"utils.go",
],
importpath = "go-common/app/service/main/upcredit/common/fsm",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,191 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct or
indirect, to cause the direction or management of such entity, whether by
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.
"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.
"Object" form shall mean any form resulting from mechanical transformation or
translation of a Source form, including but not limited to compiled object code,
generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form, made
available under the License, as indicated by a copyright notice that is included
in or attached to the work (an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object form, that
is based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative Works
shall not include works that remain separable from, or merely link (or bind by
name) to the interfaces of, the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative Works
thereof, that is intentionally submitted to Licensor for inclusion in the Work
by the copyright owner or by an individual or Legal Entity authorized to submit
on behalf of the copyright owner. For the purposes of this definition,
"submitted" means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems, and
issue tracking systems that are managed by, or on behalf of, the Licensor for
the purpose of discussing and improving the Work, but excluding communication
that is conspicuously marked or otherwise designated in writing by the copyright
owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
of whom a Contribution has been received by Licensor and subsequently
incorporated within the Work.
2. Grant of Copyright License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the Work and such
Derivative Works in Source or Object form.
3. Grant of Patent License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to make, have
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
such license applies only to those patent claims licensable by such Contributor
that are necessarily infringed by their Contribution(s) alone or by combination
of their Contribution(s) with the Work to which such Contribution(s) was
submitted. If You institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
Contribution incorporated within the Work constitutes direct or contributory
patent infringement, then any patent licenses granted to You under this License
for that Work shall terminate as of the date such litigation is filed.
4. Redistribution.
You may reproduce and distribute copies of the Work or Derivative Works thereof
in any medium, with or without modifications, and in Source or Object form,
provided that You meet the following conditions:
You must give any other recipients of the Work or Derivative Works a copy of
this License; and
You must cause any modified files to carry prominent notices stating that You
changed the files; and
You must retain, in the Source form of any Derivative Works that You distribute,
all copyright, patent, trademark, and attribution notices from the Source form
of the Work, excluding those notices that do not pertain to any part of the
Derivative Works; and
If the Work includes a "NOTICE" text file as part of its distribution, then any
Derivative Works that You distribute must include a readable copy of the
attribution notices contained within such NOTICE file, excluding those notices
that do not pertain to any part of the Derivative Works, in at least one of the
following places: within a NOTICE text file distributed as part of the
Derivative Works; within the Source form or documentation, if provided along
with the Derivative Works; or, within a display generated by the Derivative
Works, if and wherever such third-party notices normally appear. The contents of
the NOTICE file are for informational purposes only and do not modify the
License. You may add Your own attribution notices within Derivative Works that
You distribute, alongside or as an addendum to the NOTICE text from the Work,
provided that such additional attribution notices cannot be construed as
modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction, or
distribution of Your modifications, or for any such Derivative Works as a whole,
provided Your use, reproduction, and distribution of the Work otherwise complies
with the conditions stated in this License.
5. Submission of Contributions.
Unless You explicitly state otherwise, any Contribution intentionally submitted
for inclusion in the Work by You to the Licensor shall be under the terms and
conditions of this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify the terms of
any separate license agreement you may have executed with Licensor regarding
such Contributions.
6. Trademarks.
This License does not grant permission to use the trade names, trademarks,
service marks, or product names of the Licensor, except as required for
reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.
7. Disclaimer of Warranty.
Unless required by applicable law or agreed to in writing, Licensor provides the
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
including, without limitation, any warranties or conditions of TITLE,
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
solely responsible for determining the appropriateness of using or
redistributing the Work and assume any risks associated with Your exercise of
permissions under this License.
8. Limitation of Liability.
In no event and under no legal theory, whether in tort (including negligence),
contract, or otherwise, unless required by applicable law (such as deliberate
and grossly negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special, incidental,
or consequential damages of any character arising as a result of this License or
out of the use or inability to use the Work (including but not limited to
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
any and all other commercial damages or losses), even if such Contributor has
been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability.
While redistributing the Work or Derivative Works thereof, You may choose to
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
other liability obligations and/or rights consistent with this License. However,
in accepting such obligations, You may act only on Your own behalf and on Your
sole responsibility, not on behalf of any other Contributor, and only if You
agree to indemnify, defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason of your
accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work
To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own
identifying information. (Don't include the brackets!) The text should be
enclosed in the appropriate comment syntax for the file format. We also
recommend that a file or class name and description of purpose be included on
the same "printed page" as the copyright notice for easier identification within
third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -0,0 +1,121 @@
[![wercker status](https://app.wercker.com/status/517d98fe7a8da9bf9a6060e7906c0d17/s "wercker status")](https://app.wercker.com/project/bykey/517d98fe7a8da9bf9a6060e7906c0d17)
[![Coverage Status](https://img.shields.io/coveralls/looplab/fsm.svg)](https://coveralls.io/r/looplab/fsm)
[![GoDoc](https://godoc.org/github.com/looplab/fsm?status.svg)](https://godoc.org/github.com/looplab/fsm)
[![Go Report Card](https://goreportcard.com/badge/looplab/fsm)](https://goreportcard.com/report/looplab/fsm)
# FSM for Go
FSM is a finite state machine for Go.
It is heavily based on two FSM implementations:
- Javascript Finite State Machine, https://github.com/jakesgordon/javascript-state-machine
- Fysom for Python, https://github.com/oxplot/fysom (forked at https://github.com/mriehl/fysom)
For API docs and examples see http://godoc.org/github.com/looplab/fsm
# Basic Example
From examples/simple.go:
```go
package main
import (
"fmt"
"github.com/looplab/fsm"
)
func main() {
fsm := fsm.NewFSM(
"closed",
fsm.Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
fsm.Callbacks{},
)
fmt.Println(fsm.Current())
err := fsm.Event("open")
if err != nil {
fmt.Println(err)
}
fmt.Println(fsm.Current())
err = fsm.Event("close")
if err != nil {
fmt.Println(err)
}
fmt.Println(fsm.Current())
}
```
# Usage as a struct field
From examples/struct.go:
```go
package main
import (
"fmt"
"github.com/looplab/fsm"
)
type Door struct {
To string
FSM *fsm.FSM
}
func NewDoor(to string) *Door {
d := &Door{
To: to,
}
d.FSM = fsm.NewFSM(
"closed",
fsm.Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
fsm.Callbacks{
"enter_state": func(e *fsm.Event) { d.enterState(e) },
},
)
return d
}
func (d *Door) enterState(e *fsm.Event) {
fmt.Printf("The door to %s is %s\n", d.To, e.Dst)
}
func main() {
door := NewDoor("heaven")
err := door.FSM.Event("open")
if err != nil {
fmt.Println(err)
}
err = door.FSM.Event("close")
if err != nil {
fmt.Println(err)
}
}
```
# License
FSM is licensed under Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0

View File

@ -0,0 +1,100 @@
// Copyright (c) 2013 - Max Persson <max@looplab.se>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
// InvalidEventError is returned by FSM.Event() when the event cannot be called
// in the current state.
type InvalidEventError struct {
Event string
State string
}
func (e InvalidEventError) Error() string {
return "event " + e.Event + " inappropriate in current state " + e.State
}
// UnknownEventError is returned by FSM.Event() when the event is not defined.
type UnknownEventError struct {
Event string
}
func (e UnknownEventError) Error() string {
return "event " + e.Event + " does not exist"
}
// InTransitionError is returned by FSM.Event() when an asynchronous transition
// is already in progress.
type InTransitionError struct {
Event string
}
func (e InTransitionError) Error() string {
return "event " + e.Event + " inappropriate because previous transition did not complete"
}
// NotInTransitionError is returned by FSM.Transition() when an asynchronous
// transition is not in progress.
type NotInTransitionError struct{}
func (e NotInTransitionError) Error() string {
return "transition inappropriate because no state change in progress"
}
// NoTransitionError is returned by FSM.Event() when no transition have happened,
// for example if the source and destination states are the same.
type NoTransitionError struct {
Err error
}
func (e NoTransitionError) Error() string {
if e.Err != nil {
return "no transition with error: " + e.Err.Error()
}
return "no transition"
}
// CanceledError is returned by FSM.Event() when a callback have canceled a
// transition.
type CanceledError struct {
Err error
}
func (e CanceledError) Error() string {
if e.Err != nil {
return "transition canceled with error: " + e.Err.Error()
}
return "transition canceled"
}
// AsyncError is returned by FSM.Event() when a callback have initiated an
// asynchronous state transition.
type AsyncError struct {
Err error
}
func (e AsyncError) Error() string {
if e.Err != nil {
return "async started with error: " + e.Err.Error()
}
return "async started"
}
// InternalError is returned by FSM.Event() and should never occur. It is a
// probably because of a bug.
type InternalError struct{}
func (e InternalError) Error() string {
return "internal error on state transition"
}

View File

@ -0,0 +1,92 @@
// Copyright (c) 2013 - Max Persson <max@looplab.se>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
import (
"errors"
"testing"
)
func TestInvalidEventError(t *testing.T) {
event := "invalid event"
state := "state"
e := InvalidEventError{Event: event, State: state}
if e.Error() != "event "+e.Event+" inappropriate in current state "+e.State {
t.Error("InvalidEventError string mismatch")
}
}
func TestUnknownEventError(t *testing.T) {
event := "invalid event"
e := UnknownEventError{Event: event}
if e.Error() != "event "+e.Event+" does not exist" {
t.Error("UnknownEventError string mismatch")
}
}
func TestInTransitionError(t *testing.T) {
event := "in transition"
e := InTransitionError{Event: event}
if e.Error() != "event "+e.Event+" inappropriate because previous transition did not complete" {
t.Error("InTransitionError string mismatch")
}
}
func TestNotInTransitionError(t *testing.T) {
e := NotInTransitionError{}
if e.Error() != "transition inappropriate because no state change in progress" {
t.Error("NotInTransitionError string mismatch")
}
}
func TestNoTransitionError(t *testing.T) {
e := NoTransitionError{}
if e.Error() != "no transition" {
t.Error("NoTransitionError string mismatch")
}
e.Err = errors.New("no transition")
if e.Error() != "no transition with error: "+e.Err.Error() {
t.Error("NoTransitionError string mismatch")
}
}
func TestCanceledError(t *testing.T) {
e := CanceledError{}
if e.Error() != "transition canceled" {
t.Error("CanceledError string mismatch")
}
e.Err = errors.New("canceled")
if e.Error() != "transition canceled with error: "+e.Err.Error() {
t.Error("CanceledError string mismatch")
}
}
func TestAsyncError(t *testing.T) {
e := AsyncError{}
if e.Error() != "async started" {
t.Error("AsyncError string mismatch")
}
e.Err = errors.New("async")
if e.Error() != "async started with error: "+e.Err.Error() {
t.Error("AsyncError string mismatch")
}
}
func TestInternalError(t *testing.T) {
e := InternalError{}
if e.Error() != "internal error on state transition" {
t.Error("InternalError string mismatch")
}
}

View File

@ -0,0 +1,62 @@
// Copyright (c) 2013 - Max Persson <max@looplab.se>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
// Event is the info that get passed as a reference in the callbacks.
type Event struct {
// FSM is a reference to the current FSM.
FSM *FSM
// Event is the event name.
Event string
// Src is the state before the transition.
Src string
// Dst is the state after the transition.
Dst string
// Err is an optional error that can be returned from a callback.
Err error
// Args is a optinal list of arguments passed to the callback.
Args []interface{}
// canceled is an internal flag set if the transition is canceled.
canceled bool
// async is an internal flag set if the transition should be asynchronous
async bool
}
// Cancel can be called in before_<EVENT> or leave_<STATE> to cancel the
// current transition before it happens. It takes an opitonal error, which will
// overwrite e.Err if set before.
func (e *Event) Cancel(err ...error) {
e.canceled = true
if len(err) > 0 {
e.Err = err[0]
}
}
// Async can be called in leave_<STATE> to do an asynchronous state transition.
//
// The current state transition will be on hold in the old state until a final
// call to Transition is made. This will comlete the transition and possibly
// call the other callbacks.
func (e *Event) Async() {
e.async = true
}

View File

@ -0,0 +1,447 @@
// Copyright (c) 2013 - Max Persson <max@looplab.se>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package fsm implements a finite state machine.
//
// It is heavily based on two FSM implementations:
//
// Javascript Finite State Machine
// https://github.com/jakesgordon/javascript-state-machine
//
// Fysom for Python
// https://github.com/oxplot/fysom (forked at https://github.com/mriehl/fysom)
//
package fsm
import (
"strings"
"sync"
)
// transitioner is an interface for the FSM's transition function.
type transitioner interface {
transition(*FSM) error
}
// FSM is the state machine that holds the current state.
//
// It has to be created with NewFSM to function properly.
type FSM struct {
// current is the state that the FSM is currently in.
current string
// transitions maps events and source states to destination states.
transitions map[eKey]string
// callbacks maps events and targers to callback functions.
callbacks map[cKey]Callback
// transition is the internal transition functions used either directly
// or when Transition is called in an asynchronous state transition.
transition func()
// transitionerObj calls the FSM's transition() function.
transitionerObj transitioner
// stateMu guards access to the current state.
stateMu sync.RWMutex
// eventMu guards access to Event() and Transition().
eventMu sync.Mutex
}
// EventDesc represents an event when initializing the FSM.
//
// The event can have one or more source states that is valid for performing
// the transition. If the FSM is in one of the source states it will end up in
// the specified destination state, calling all defined callbacks as it goes.
type EventDesc struct {
// Name is the event name used when calling for a transition.
Name string
// Src is a slice of source states that the FSM must be in to perform a
// state transition.
Src []string
// Dst is the destination state that the FSM will be in if the transition
// succeds.
Dst string
}
// Callback is a function type that callbacks should use. Event is the current
// event info as the callback happens.
type Callback func(*Event)
// Events is a shorthand for defining the transition map in NewFSM.
type Events []EventDesc
// Callbacks is a shorthand for defining the callbacks in NewFSM.a
type Callbacks map[string]Callback
// NewFSM constructs a FSM from events and callbacks.
//
// The events and transitions are specified as a slice of Event structs
// specified as Events. Each Event is mapped to one or more internal
// transitions from Event.Src to Event.Dst.
//
// Callbacks are added as a map specified as Callbacks where the key is parsed
// as the callback event as follows, and called in the same order:
//
// 1. before_<EVENT> - called before event named <EVENT>
//
// 2. before_event - called before all events
//
// 3. leave_<OLD_STATE> - called before leaving <OLD_STATE>
//
// 4. leave_state - called before leaving all states
//
// 5. enter_<NEW_STATE> - called after entering <NEW_STATE>
//
// 6. enter_state - called after entering all states
//
// 7. after_<EVENT> - called after event named <EVENT>
//
// 8. after_event - called after all events
//
// There are also two short form versions for the most commonly used callbacks.
// They are simply the name of the event or state:
//
// 1. <NEW_STATE> - called after entering <NEW_STATE>
//
// 2. <EVENT> - called after event named <EVENT>
//
// If both a shorthand version and a full version is specified it is undefined
// which version of the callback will end up in the internal map. This is due
// to the psuedo random nature of Go maps. No checking for multiple keys is
// currently performed.
func NewFSM(initial string, events []EventDesc, callbacks map[string]Callback) *FSM {
f := &FSM{
transitionerObj: &transitionerStruct{},
current: initial,
transitions: make(map[eKey]string),
callbacks: make(map[cKey]Callback),
}
// Build transition map and store sets of all events and states.
allEvents := make(map[string]bool)
allStates := make(map[string]bool)
for _, e := range events {
for _, src := range e.Src {
f.transitions[eKey{e.Name, src}] = e.Dst
allStates[src] = true
allStates[e.Dst] = true
}
allEvents[e.Name] = true
}
// Map all callbacks to events/states.
for name, fn := range callbacks {
var target string
var callbackType int
switch {
case strings.HasPrefix(name, "before_"):
target = strings.TrimPrefix(name, "before_")
if target == "event" {
target = ""
callbackType = callbackBeforeEvent
} else if _, ok := allEvents[target]; ok {
callbackType = callbackBeforeEvent
}
case strings.HasPrefix(name, "leave_"):
target = strings.TrimPrefix(name, "leave_")
if target == "state" {
target = ""
callbackType = callbackLeaveState
} else if _, ok := allStates[target]; ok {
callbackType = callbackLeaveState
}
case strings.HasPrefix(name, "enter_"):
target = strings.TrimPrefix(name, "enter_")
if target == "state" {
target = ""
callbackType = callbackEnterState
} else if _, ok := allStates[target]; ok {
callbackType = callbackEnterState
}
case strings.HasPrefix(name, "after_"):
target = strings.TrimPrefix(name, "after_")
if target == "event" {
target = ""
callbackType = callbackAfterEvent
} else if _, ok := allEvents[target]; ok {
callbackType = callbackAfterEvent
}
default:
target = name
if _, ok := allStates[target]; ok {
callbackType = callbackEnterState
} else if _, ok := allEvents[target]; ok {
callbackType = callbackAfterEvent
}
}
if callbackType != callbackNone {
f.callbacks[cKey{target, callbackType}] = fn
}
}
return f
}
// Current returns the current state of the FSM.
func (f *FSM) Current() string {
f.stateMu.RLock()
defer f.stateMu.RUnlock()
return f.current
}
// Is returns true if state is the current state.
func (f *FSM) Is(state string) bool {
f.stateMu.RLock()
defer f.stateMu.RUnlock()
return state == f.current
}
// SetState allows the user to move to the given state from current state.
// The call does not trigger any callbacks, if defined.
func (f *FSM) SetState(state string) {
f.stateMu.Lock()
defer f.stateMu.Unlock()
f.current = state
}
// Can returns true if event can occur in the current state.
func (f *FSM) Can(event string) bool {
f.stateMu.RLock()
defer f.stateMu.RUnlock()
_, ok := f.transitions[eKey{event, f.current}]
return ok && (f.transition == nil)
}
// AvailableTransitions returns a list of transitions avilable in the
// current state.
func (f *FSM) AvailableTransitions() []string {
f.stateMu.RLock()
defer f.stateMu.RUnlock()
var transitions []string
for key := range f.transitions {
if key.src == f.current {
transitions = append(transitions, key.event)
}
}
return transitions
}
// Cannot returns true if event can not occure in the current state.
// It is a convenience method to help code read nicely.
func (f *FSM) Cannot(event string) bool {
return !f.Can(event)
}
// Event initiates a state transition with the named event.
//
// The call takes a variable number of arguments that will be passed to the
// callback, if defined.
//
// It will return nil if the state change is ok or one of these errors:
//
// - event X inappropriate because previous transition did not complete
//
// - event X inappropriate in current state Y
//
// - event X does not exist
//
// - internal error on state transition
//
// The last error should never occur in this situation and is a sign of an
// internal bug.
func (f *FSM) Event(event string, args ...interface{}) error {
f.eventMu.Lock()
defer f.eventMu.Unlock()
f.stateMu.RLock()
defer f.stateMu.RUnlock()
if f.transition != nil {
return InTransitionError{event}
}
dst, ok := f.transitions[eKey{event, f.current}]
if !ok {
for ekey := range f.transitions {
if ekey.event == event {
return InvalidEventError{event, f.current}
}
}
return UnknownEventError{event}
}
e := &Event{f, event, f.current, dst, nil, args, false, false}
err := f.beforeEventCallbacks(e)
if err != nil {
return err
}
// need reenter the same state
//if f.current == dst {
// f.afterEventCallbacks(e)
// return NoTransitionError{e.Err}
//}
// Setup the transition, call it later.
f.transition = func() {
f.stateMu.Lock()
f.current = dst
f.stateMu.Unlock()
f.enterStateCallbacks(e)
f.afterEventCallbacks(e)
}
if err = f.leaveStateCallbacks(e); err != nil {
if _, ok := err.(CanceledError); ok {
f.transition = nil
}
return err
}
// Perform the rest of the transition, if not asynchronous.
f.stateMu.RUnlock()
err = f.doTransition()
f.stateMu.RLock()
if err != nil {
return InternalError{}
}
return e.Err
}
// Transition wraps transitioner.transition.
func (f *FSM) Transition() error {
f.eventMu.Lock()
defer f.eventMu.Unlock()
return f.doTransition()
}
// doTransition wraps transitioner.transition.
func (f *FSM) doTransition() error {
return f.transitionerObj.transition(f)
}
// transitionerStruct is the default implementation of the transitioner
// interface. Other implementations can be swapped in for testing.
type transitionerStruct struct{}
// Transition completes an asynchrounous state change.
//
// The callback for leave_<STATE> must prviously have called Async on its
// event to have initiated an asynchronous state transition.
func (t transitionerStruct) transition(f *FSM) error {
if f.transition == nil {
return NotInTransitionError{}
}
f.transition()
f.transition = nil
return nil
}
// beforeEventCallbacks calls the before_ callbacks, first the named then the
// general version.
func (f *FSM) beforeEventCallbacks(e *Event) error {
if fn, ok := f.callbacks[cKey{e.Event, callbackBeforeEvent}]; ok {
fn(e)
if e.canceled {
return CanceledError{e.Err}
}
}
if fn, ok := f.callbacks[cKey{"", callbackBeforeEvent}]; ok {
fn(e)
if e.canceled {
return CanceledError{e.Err}
}
}
return nil
}
// leaveStateCallbacks calls the leave_ callbacks, first the named then the
// general version.
func (f *FSM) leaveStateCallbacks(e *Event) error {
if fn, ok := f.callbacks[cKey{f.current, callbackLeaveState}]; ok {
fn(e)
if e.canceled {
return CanceledError{e.Err}
} else if e.async {
return AsyncError{e.Err}
}
}
if fn, ok := f.callbacks[cKey{"", callbackLeaveState}]; ok {
fn(e)
if e.canceled {
return CanceledError{e.Err}
} else if e.async {
return AsyncError{e.Err}
}
}
return nil
}
// enterStateCallbacks calls the enter_ callbacks, first the named then the
// general version.
func (f *FSM) enterStateCallbacks(e *Event) {
if fn, ok := f.callbacks[cKey{f.current, callbackEnterState}]; ok {
fn(e)
}
if fn, ok := f.callbacks[cKey{"", callbackEnterState}]; ok {
fn(e)
}
}
// afterEventCallbacks calls the after_ callbacks, first the named then the
// general version.
func (f *FSM) afterEventCallbacks(e *Event) {
if fn, ok := f.callbacks[cKey{e.Event, callbackAfterEvent}]; ok {
fn(e)
}
if fn, ok := f.callbacks[cKey{"", callbackAfterEvent}]; ok {
fn(e)
}
}
const (
callbackNone int = iota
callbackBeforeEvent
callbackLeaveState
callbackEnterState
callbackAfterEvent
)
// cKey is a struct key used for keeping the callbacks mapped to a target.
type cKey struct {
// target is either the name of a state or an event depending on which
// callback type the key refers to. It can also be "" for a non-targeted
// callback like before_event.
target string
// callbackType is the situation when the callback will be run.
callbackType int
}
// eKey is a struct key used for storing the transition map.
type eKey struct {
// event is the name of the event that the keys refers to.
event string
// src is the source from where the event can transition.
src string
}

View File

@ -0,0 +1,810 @@
// Copyright (c) 2013 - Max Persson <max@looplab.se>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
import (
"fmt"
"sort"
"sync"
"testing"
"time"
)
type fakeTransitionerObj struct {
}
func (t fakeTransitionerObj) transition(f *FSM) error {
return &InternalError{}
}
func TestSameState(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "start"},
},
Callbacks{},
)
fsm.Event("run")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestSetState(t *testing.T) {
fsm := NewFSM(
"walking",
Events{
{Name: "walk", Src: []string{"start"}, Dst: "walking"},
},
Callbacks{},
)
fsm.SetState("start")
if fsm.Current() != "start" {
t.Error("expected state to be 'walking'")
}
err := fsm.Event("walk")
if err != nil {
t.Error("transition is expected no error")
}
}
func TestBadTransition(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "running"},
},
Callbacks{},
)
fsm.transitionerObj = new(fakeTransitionerObj)
err := fsm.Event("run")
if err == nil {
t.Error("bad transition should give an error")
}
}
func TestInappropriateEvent(t *testing.T) {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{},
)
err := fsm.Event("close")
if e, ok := err.(InvalidEventError); !ok && e.Event != "close" && e.State != "closed" {
t.Error("expected 'InvalidEventError' with correct state and event")
}
}
func TestInvalidEvent(t *testing.T) {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{},
)
err := fsm.Event("lock")
if e, ok := err.(UnknownEventError); !ok && e.Event != "close" {
t.Error("expected 'UnknownEventError' with correct event")
}
}
func TestMultipleSources(t *testing.T) {
fsm := NewFSM(
"one",
Events{
{Name: "first", Src: []string{"one"}, Dst: "two"},
{Name: "second", Src: []string{"two"}, Dst: "three"},
{Name: "reset", Src: []string{"one", "two", "three"}, Dst: "one"},
},
Callbacks{},
)
fsm.Event("first")
if fsm.Current() != "two" {
t.Error("expected state to be 'two'")
}
fsm.Event("reset")
if fsm.Current() != "one" {
t.Error("expected state to be 'one'")
}
fsm.Event("first")
fsm.Event("second")
if fsm.Current() != "three" {
t.Error("expected state to be 'three'")
}
fsm.Event("reset")
if fsm.Current() != "one" {
t.Error("expected state to be 'one'")
}
}
func TestMultipleEvents(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "first", Src: []string{"start"}, Dst: "one"},
{Name: "second", Src: []string{"start"}, Dst: "two"},
{Name: "reset", Src: []string{"one"}, Dst: "reset_one"},
{Name: "reset", Src: []string{"two"}, Dst: "reset_two"},
{Name: "reset", Src: []string{"reset_one", "reset_two"}, Dst: "start"},
},
Callbacks{},
)
fsm.Event("first")
fsm.Event("reset")
if fsm.Current() != "reset_one" {
t.Error("expected state to be 'reset_one'")
}
fsm.Event("reset")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
fsm.Event("second")
fsm.Event("reset")
if fsm.Current() != "reset_two" {
t.Error("expected state to be 'reset_two'")
}
fsm.Event("reset")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestGenericCallbacks(t *testing.T) {
beforeEvent := false
leaveState := false
enterState := false
afterEvent := false
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"before_event": func(e *Event) {
beforeEvent = true
},
"leave_state": func(e *Event) {
leaveState = true
},
"enter_state": func(e *Event) {
enterState = true
},
"after_event": func(e *Event) {
afterEvent = true
},
},
)
fsm.Event("run")
if !(beforeEvent && leaveState && enterState && afterEvent) {
t.Error("expected all callbacks to be called")
}
}
func TestSpecificCallbacks(t *testing.T) {
beforeEvent := false
leaveState := false
enterState := false
afterEvent := false
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"before_run": func(e *Event) {
beforeEvent = true
},
"leave_start": func(e *Event) {
leaveState = true
},
"enter_end": func(e *Event) {
enterState = true
},
"after_run": func(e *Event) {
afterEvent = true
},
},
)
fsm.Event("run")
if !(beforeEvent && leaveState && enterState && afterEvent) {
t.Error("expected all callbacks to be called")
}
}
func TestSpecificCallbacksShortform(t *testing.T) {
enterState := false
afterEvent := false
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"end": func(e *Event) {
enterState = true
},
"run": func(e *Event) {
afterEvent = true
},
},
)
fsm.Event("run")
if !(enterState && afterEvent) {
t.Error("expected all callbacks to be called")
}
}
func TestBeforeEventWithoutTransition(t *testing.T) {
beforeEvent := true
fsm := NewFSM(
"start",
Events{
{Name: "dontrun", Src: []string{"start"}, Dst: "start"},
},
Callbacks{
"before_event": func(e *Event) {
beforeEvent = true
},
},
)
err := fsm.Event("dontrun")
if e, ok := err.(NoTransitionError); !ok && e.Err != nil {
t.Error("expected 'NoTransitionError' without custom error")
}
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
if !beforeEvent {
t.Error("expected callback to be called")
}
}
func TestCancelBeforeGenericEvent(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"before_event": func(e *Event) {
e.Cancel()
},
},
)
fsm.Event("run")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestCancelBeforeSpecificEvent(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"before_run": func(e *Event) {
e.Cancel()
},
},
)
fsm.Event("run")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestCancelLeaveGenericState(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"leave_state": func(e *Event) {
e.Cancel()
},
},
)
fsm.Event("run")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestCancelLeaveSpecificState(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"leave_start": func(e *Event) {
e.Cancel()
},
},
)
fsm.Event("run")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestCancelWithError(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"before_event": func(e *Event) {
e.Cancel(fmt.Errorf("error"))
},
},
)
err := fsm.Event("run")
if _, ok := err.(CanceledError); !ok {
t.Error("expected only 'CanceledError'")
}
if e, ok := err.(CanceledError); ok && e.Err.Error() != "error" {
t.Error("expected 'CanceledError' with correct custom error")
}
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestAsyncTransitionGenericState(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"leave_state": func(e *Event) {
e.Async()
},
},
)
fsm.Event("run")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
fsm.Transition()
if fsm.Current() != "end" {
t.Error("expected state to be 'end'")
}
}
func TestAsyncTransitionSpecificState(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"leave_start": func(e *Event) {
e.Async()
},
},
)
fsm.Event("run")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
fsm.Transition()
if fsm.Current() != "end" {
t.Error("expected state to be 'end'")
}
}
func TestAsyncTransitionInProgress(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
{Name: "reset", Src: []string{"end"}, Dst: "start"},
},
Callbacks{
"leave_start": func(e *Event) {
e.Async()
},
},
)
fsm.Event("run")
err := fsm.Event("reset")
if e, ok := err.(InTransitionError); !ok && e.Event != "reset" {
t.Error("expected 'InTransitionError' with correct state")
}
fsm.Transition()
fsm.Event("reset")
if fsm.Current() != "start" {
t.Error("expected state to be 'start'")
}
}
func TestAsyncTransitionNotInProgress(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
{Name: "reset", Src: []string{"end"}, Dst: "start"},
},
Callbacks{},
)
err := fsm.Transition()
if _, ok := err.(NotInTransitionError); !ok {
t.Error("expected 'NotInTransitionError'")
}
}
func TestCallbackNoError(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"run": func(e *Event) {
},
},
)
e := fsm.Event("run")
if e != nil {
t.Error("expected no error")
}
}
func TestCallbackError(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"run": func(e *Event) {
e.Err = fmt.Errorf("error")
},
},
)
e := fsm.Event("run")
if e.Error() != "error" {
t.Error("expected error to be 'error'")
}
}
func TestCallbackArgs(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"run": func(e *Event) {
if len(e.Args) != 1 {
t.Error("too few arguments")
}
arg, ok := e.Args[0].(string)
if !ok {
t.Error("not a string argument")
}
if arg != "test" {
t.Error("incorrect argument")
}
},
},
)
fsm.Event("run", "test")
}
func TestNoDeadLock(t *testing.T) {
var fsm *FSM
fsm = NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"run": func(e *Event) {
fsm.Current() // Should not result in a panic / deadlock
},
},
)
fsm.Event("run")
}
func TestThreadSafetyRaceCondition(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"run": func(e *Event) {
},
},
)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
_ = fsm.Current()
}()
fsm.Event("run")
wg.Wait()
}
func TestDoubleTransition(t *testing.T) {
var fsm *FSM
var wg sync.WaitGroup
wg.Add(2)
fsm = NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "end"},
},
Callbacks{
"before_run": func(e *Event) {
wg.Done()
// Imagine a concurrent event coming in of the same type while
// the data access mutex is unlocked because the current transition
// is running its event callbacks, getting around the "active"
// transition checks
if len(e.Args) == 0 {
// Must be concurrent so the test may pass when we add a mutex that synchronizes
// calls to Event(...). It will then fail as an inappropriate transition as we
// have changed state.
go func() {
if err := fsm.Event("run", "second run"); err != nil {
fmt.Println(err)
wg.Done() // It should fail, and then we unfreeze the test.
}
}()
time.Sleep(20 * time.Millisecond)
} else {
panic("Was able to reissue an event mid-transition")
}
},
},
)
if err := fsm.Event("run"); err != nil {
fmt.Println(err)
}
wg.Wait()
}
func TestNoTransition(t *testing.T) {
fsm := NewFSM(
"start",
Events{
{Name: "run", Src: []string{"start"}, Dst: "start"},
},
Callbacks{},
)
err := fsm.Event("run")
if _, ok := err.(NoTransitionError); !ok {
t.Error("expected 'NoTransitionError'")
}
}
func ExampleNewFSM() {
fsm := NewFSM(
"green",
Events{
{Name: "warn", Src: []string{"green"}, Dst: "yellow"},
{Name: "panic", Src: []string{"yellow"}, Dst: "red"},
{Name: "panic", Src: []string{"green"}, Dst: "red"},
{Name: "calm", Src: []string{"red"}, Dst: "yellow"},
{Name: "clear", Src: []string{"yellow"}, Dst: "green"},
},
Callbacks{
"before_warn": func(e *Event) {
fmt.Println("before_warn")
},
"before_event": func(e *Event) {
fmt.Println("before_event")
},
"leave_green": func(e *Event) {
fmt.Println("leave_green")
},
"leave_state": func(e *Event) {
fmt.Println("leave_state")
},
"enter_yellow": func(e *Event) {
fmt.Println("enter_yellow")
},
"enter_state": func(e *Event) {
fmt.Println("enter_state")
},
"after_warn": func(e *Event) {
fmt.Println("after_warn")
},
"after_event": func(e *Event) {
fmt.Println("after_event")
},
},
)
fmt.Println(fsm.Current())
err := fsm.Event("warn")
if err != nil {
fmt.Println(err)
}
fmt.Println(fsm.Current())
// Output:
// green
// before_warn
// before_event
// leave_green
// leave_state
// enter_yellow
// enter_state
// after_warn
// after_event
// yellow
}
func ExampleFSM_Current() {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{},
)
fmt.Println(fsm.Current())
// Output: closed
}
func ExampleFSM_Is() {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{},
)
fmt.Println(fsm.Is("closed"))
fmt.Println(fsm.Is("open"))
// Output:
// true
// false
}
func ExampleFSM_Can() {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{},
)
fmt.Println(fsm.Can("open"))
fmt.Println(fsm.Can("close"))
// Output:
// true
// false
}
func ExampleFSM_AvailableTransitions() {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
{Name: "kick", Src: []string{"closed"}, Dst: "broken"},
},
Callbacks{},
)
// sort the results ordering is consistent for the output checker
transitions := fsm.AvailableTransitions()
sort.Strings(transitions)
fmt.Println(transitions)
// Output:
// [kick open]
}
func ExampleFSM_Cannot() {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{},
)
fmt.Println(fsm.Cannot("open"))
fmt.Println(fsm.Cannot("close"))
// Output:
// false
// true
}
func ExampleFSM_Event() {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{},
)
fmt.Println(fsm.Current())
err := fsm.Event("open")
if err != nil {
fmt.Println(err)
}
fmt.Println(fsm.Current())
err = fsm.Event("close")
if err != nil {
fmt.Println(err)
}
fmt.Println(fsm.Current())
// Output:
// closed
// open
// closed
}
func ExampleFSM_Transition() {
fsm := NewFSM(
"closed",
Events{
{Name: "open", Src: []string{"closed"}, Dst: "open"},
{Name: "close", Src: []string{"open"}, Dst: "closed"},
},
Callbacks{
"leave_closed": func(e *Event) {
e.Async()
},
},
)
err := fsm.Event("open")
if e, ok := err.(AsyncError); !ok && e.Err != nil {
fmt.Println(err)
}
fmt.Println(fsm.Current())
err = fsm.Transition()
if err != nil {
fmt.Println(err)
}
fmt.Println(fsm.Current())
// Output:
// closed
// open
}

View File

@ -0,0 +1,45 @@
package fsm
import (
"bytes"
"fmt"
)
// Visualize outputs a visualization of a FSM in Graphviz format.
func Visualize(fsm *FSM) string {
var buf bytes.Buffer
states := make(map[string]int)
buf.WriteString(fmt.Sprintf(`digraph fsm {`))
buf.WriteString("\n")
// make sure the initial state is at top
for k, v := range fsm.transitions {
if k.src == fsm.current {
states[k.src]++
states[v]++
buf.WriteString(fmt.Sprintf(` "%s" -> "%s" [ label = "%s" ];`, k.src, v, k.event))
buf.WriteString("\n")
}
}
for k, v := range fsm.transitions {
if k.src != fsm.current {
states[k.src]++
states[v]++
buf.WriteString(fmt.Sprintf(` "%s" -> "%s" [ label = "%s" ];`, k.src, v, k.event))
buf.WriteString("\n")
}
}
buf.WriteString("\n")
for k := range states {
buf.WriteString(fmt.Sprintf(` "%s";`, k))
buf.WriteString("\n")
}
buf.WriteString(fmt.Sprintln("}"))
return buf.String()
}

View File

@ -0,0 +1,45 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"conf.go",
"credit_conf.go",
],
importpath = "go-common/app/service/main/upcredit/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/conf:go_default_library",
"//library/database/orm:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/permit:go_default_library",
"//library/net/rpc:go_default_library",
"//library/net/trace:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,237 @@
package conf
import (
"errors"
"flag"
"fmt"
"os"
"path"
"strings"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/conf"
"go-common/library/database/orm"
"go-common/library/log"
"go-common/library/net/http/blademaster"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/permit"
"go-common/library/net/rpc"
"go-common/library/net/trace"
"go-common/library/queue/databus"
"go-common/library/time"
"github.com/BurntSushi/toml"
)
// Conf info.
var (
ConfPath string
Conf = &Config{}
client *conf.Client
CreditConfig = &CreditConf{}
IsMaster = true
)
const (
//ServiceName service name
ServiceName = "upcredit-service"
)
// Config struct.
type Config struct {
// bm
BM *HTTPServers
// db
DB *DB
// base
// elk
Xlog *log.Config
// report log
LogCli *log.AgentConfig
// httpClinet
HTTPClient *HTTPClient
// tracer
Tracer *trace.Config
// Redis
Redis *Redis
// rpc server
RPCServer *rpc.ServerConfig
// auth
Auth *permit.Config
IsTest bool
CreditLogSub *databus.Config
BusinessBinLogSub *databus.Config
RunStatJobConf *RunStatJob
MiscConf *MiscConfig
ElectionZooKeeper *Zookeeper
}
//UpSub upsub config
//type upSub struct {
// *databus.Config
// UpChanSize int
// ConsumeLimit int
// RoutineLimit int
//}
//MiscConfig other config set
type MiscConfig struct {
CreditLogWriteRoutineNum int
BusinessBinLogLimitRate float64 // 每秒多少个business bin log 消费速度
}
//HTTPServers for http server.
type HTTPServers struct {
Inner *blademaster.ServerConfig
}
// DB conf.
type DB struct {
Upcrm *orm.Config
UpcrmReader *orm.Config
}
// Redis conf.
type Redis struct {
Databus *struct {
*redis.Config
Expire time.Duration
}
}
// HTTPClient conf.
type HTTPClient struct {
Normal *bm.ClientConfig
Slow *bm.ClientConfig
}
// Host conf.
type Host struct {
API string
Live string
Search string
Manager string
}
// Monitor conf.
type Monitor struct {
Host string
Moni string
UserName string
AppSecret string
AppToken string
IntervalAlarm time.Duration
}
//RunStatJob 定时任务时间
type RunStatJob struct {
// 启动时间,比如 12:00:00每天定时运行
StartTime string
// 起的计算线程数
WorkerNumber int
}
//App for key secret.
type App struct {
Key string
Secret string
}
//MC memcache
type MC struct {
UpExpire time.Duration
Up *memcache.Config
}
//CreditLog 需要记录日志的那些稿件状态,在配置文件中配置。只有这些状态,才会记录信用日志
type CreditLog struct {
NeedLogState map[int]CreditLogStateInfo
}
//CreditLogStateInfo nothing
type CreditLogStateInfo struct {
}
// Zookeeper Server&Client settings.
type Zookeeper struct {
Root string
Addrs []string
Timeout time.Duration
}
func init() {
flag.StringVar(&ConfPath, "conf", "", "default config path")
}
// Init conf.
func Init() (err error) {
if ConfPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(ConfPath, &Conf)
if err != nil {
return
}
ConfPath = strings.Replace(ConfPath, string(os.PathSeparator), "/", -1)
var dir = path.Dir(ConfPath)
var articleConfPath = path.Join(dir, "credit_score_conf.toml")
_, err = toml.DecodeFile(articleConfPath, &CreditConfig)
CreditConfig.AfterLoad()
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
tomlStr string
ok bool
tmpConf *Config
)
if tomlStr, ok = client.Value("upcredit-service.toml"); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(tomlStr, &tmpConf); err != nil {
return errors.New("could not decode toml config")
}
*Conf = *tmpConf
fmt.Printf("loading credit_score_conf.toml from remoate...")
if tomlStr, ok = client.Value("credit_score_conf.toml"); !ok {
return errors.New("load config center error for credit_score_conf.toml")
}
var tmpConf2 *CreditConf
if _, err = toml.Decode(tomlStr, &tmpConf2); err != nil {
return errors.New("could not decode toml config for credit_score_conf.toml")
}
*CreditConfig = *tmpConf2
CreditConfig.AfterLoad()
return
}

View File

@ -0,0 +1,106 @@
package conf
import (
"go-common/library/log"
"strconv"
)
//CreditConf 信用分数配置
type CreditConf struct {
CalculateConf *CreditScoreCalculateConfig
ArticleRule *ArticleRuleConf
}
//AfterLoad load之后进行一些计算和整理
func (c *CreditConf) AfterLoad() {
c.CalculateConf.AfterLoad()
c.ArticleRule.AfterLoad()
}
//CreditScoreCalculateConfig 分数计算配置
type CreditScoreCalculateConfig struct {
// 时间衰减因子
// [离今年的差值] = 权重值
TimeWeight map[string]int
TimeWeight2 map[int]int
}
//AfterLoad after load
func (c *CreditScoreCalculateConfig) AfterLoad() {
c.TimeWeight2 = make(map[int]int)
for k, v := range c.TimeWeight {
key, _ := strconv.Atoi(k)
c.TimeWeight2[key] = v
}
}
//StateMachineState 状态机数据
type StateMachineState struct {
State int
Round int
Reason int
}
//ArticleRuleConf 稿件记分规则
type ArticleRuleConf struct {
AcceptOptypeData []int
RejectOpTypeData []int
// [score] ->[ optype list ]
OptypeScoreData map[string][]int
AcceptOptypeMap map[int]struct{}
RejectOptypeMap map[int]struct{}
// [optype] -> score
OptypeScoreMap map[int]int
InitState StateMachineState
ArticleMaxOpenCount int
}
//AfterLoad after load
func (a *ArticleRuleConf) AfterLoad() {
a.AcceptOptypeMap = make(map[int]struct{})
a.RejectOptypeMap = make(map[int]struct{})
a.OptypeScoreMap = make(map[int]int)
for _, v := range a.AcceptOptypeData {
a.AcceptOptypeMap[v] = struct{}{}
}
for _, v := range a.RejectOpTypeData {
a.RejectOptypeMap[v] = struct{}{}
}
for k, varr := range a.OptypeScoreData {
var score, err = strconv.ParseInt(k, 10, 64)
if err != nil {
log.Error("score config wrong, k=%s", k)
return
}
for _, v := range varr {
a.OptypeScoreMap[v] = int(score)
}
}
}
//GetScore get score by type, opType, reason
func (a *ArticleRuleConf) GetScore(typ int, opType int, reason int) (score int) {
var s, ok = a.OptypeScoreMap[opType]
if !ok {
s = 0
}
score = s
return
}
//IsRejected is article reject
func (a *ArticleRuleConf) IsRejected(typ int, opType int, reason int) (res bool) {
_, res = a.RejectOptypeMap[opType]
return
}
//IsAccepted is article accepted
func (a *ArticleRuleConf) IsAccepted(typ int, opType int, reason int) (res bool) {
_, res = a.AcceptOptypeMap[opType]
return
}

View File

@ -0,0 +1,21 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,96 @@
#初始table
CREATE TABLE `up_rank` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增ID',
`mid` int(11) unsigned NOT NULL DEFAULT '0' COMMENT 'up主id',
`type` smallint(6) unsigned NOT NULL DEFAULT '0' COMMENT '排行榜类型',
`value` int(11) unsigned NOT NULL DEFAULT '0' COMMENT '排行榜数值,根据type不同代表的含义不同',
`generate_date` date NOT NULL DEFAULT '0000-00-00' COMMENT '排行榜日',
`ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后修改时间',
`value2` int(11) NOT NULL DEFAULT '0' COMMENT '分数2',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_type_date` (`generate_date`,`type`),
KEY `ix_mtime` (`mtime`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Up蹿升榜'
# fat uat表增加字段
# 增加默认值
alter table up_base_info alter column active_tid set default 0;
alter table up_base_info alter column attr set default 0;
alter table up_base_info alter column mid set default 0;
alter table up_base_info alter column mid set default 0;
alter table up_play_info alter column mid set default 0;
alter table up_play_info alter column business_type set default 0;
alter table up_play_info alter column article_count set default 0;
alter table up_play_info alter column play_count_90day set default 0;
alter table up_play_info alter column play_count_7day set default 0;
alter table up_play_info alter column play_count_30day set default 0;
alter table up_play_info alter column play_count_accumulate set default 0;
alter table up_stats_history alter column type set default 0;
alter table up_stats_history alter column sub_type set default 0;
alter table up_stats_history alter column generate_date set default '0000-00-00';
alter table up_rank alter column mid set default 0;
alter table up_rank alter column type set default 0;
alter table up_rank alter column value set default 0;
alter table up_rank alter column generate_date set default '0000-00-00';
alter table task_info alter column generate_date set default '0000-00-00';
alter table task_info alter column task_type set default 0;
alter table up_base_info add column active_tid smallint(6) unsigned NOT NULL DEFAULT '0' COMMENT '最多稿件分区';
alter table up_base_info add column attr int(11) NOT NULL COMMENT '属性,以位区分';
alter table up_rank add column value2 int(11) NOT NULL DEFAULT 0 COMMENT '分数2';
ALTER TABLE up_base_info MODIFY active_tid smallint(6) unsigned NOT NULL DEFAULT 0 COMMENT '最多稿件分区';
#FAT
alter table up_play_info drop column play_count_avg;
alter table up_play_info drop column play_count_avg_90day;
alter table up_play_info add column `article_count` int(11) NOT NULL DEFAULT '0' COMMENT '总稿件数';
alter table up_play_info add column `play_count_90day` int(11) NOT NULL DEFAULT '0' COMMENT '90天内稿件总播放次数';
alter table up_play_info add column `play_count_30day` int(11) NOT NULL DEFAULT '0' COMMENT '30天内稿件总播放次数';
alter table up_play_info add column `play_count_7day` int(11) NOT NULL DEFAULT '0' COMMENT '7天内稿件总播放次数';
#FAT UAT
DROP INDEX ix_mid ON up_base_info;
alter table up_base_info add unique key uk_mid_type (`mid`,`business_type`);
#每天一条记录 增加分数段表, (uat 1, fat 1, prod 1)
create table score_section_history(
id int(11) unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT COMMENT '自增ID',
generate_date date NOT NULL COMMENT '生成日期',
score_type SMALLINT(6) NOT NULL DEFAULT 0 COMMENT '类型, 1质量分2影响分3信用分',
section_0 int(11) NOT NULL DEFAULT 0 COMMENT '0~100的人数',
section_1 int(11) NOT NULL DEFAULT 0 COMMENT '101~200',
section_2 int(11) NOT NULL DEFAULT 0 COMMENT '201~300',
section_3 int(11) NOT NULL DEFAULT 0 COMMENT '301~400',
section_4 int(11) NOT NULL DEFAULT 0 COMMENT '401~500',
section_5 int(11) NOT NULL DEFAULT 0 COMMENT '501~600',
section_6 int(11) NOT NULL DEFAULT 0 COMMENT '601~700',
section_7 int(11) NOT NULL DEFAULT 0 COMMENT '701~800',
section_8 int(11) NOT NULL DEFAULT 0 COMMENT '801~900',
section_9 int(11) NOT NULL DEFAULT 0 COMMENT '901~1000',
ctime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
mtime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后修改时间',
UNIQUE uk_date_type(generate_date, score_type),
KEY ix_mtime (mtime)
) engine=innodb DEFAULT charset=utf8 comment='up分数段人数分布表';
#增加信用分、影响分、质量分字段 (uat 1, fat 1, prod 1)
alter table up_base_info add COLUMN credit_score INT NOT NULL DEFAULT 500 COMMENT '信用分';
alter table up_base_info add COLUMN pr_score INT NOT NULL DEFAULT 0 COMMENT '影响分';
alter table up_base_info add COLUMN quality_score INT NOT NULL DEFAULT 0 COMMENT '质量分';
#修复key错误 (uat 1, fat 1, prod 1)
DROP INDEX uk_type_date ON up_rank;
alter table up_rank add unique key uk_date_type_mid (`generate_date`,`type`, `mid`);
#增加生日、地域等字段 (uat 1, fat 1, prod 1)
alter table up_base_info add COLUMN birthday DATE NOT NULL DEFAULT '0000-00-00' COMMENT '生日';
alter table up_base_info add COLUMN active_province varchar(32) NOT NULL DEFAULT '' COMMENT '省份';
alter table up_base_info add COLUMN active_city varchar(32) NOT NULL DEFAULT '' COMMENT '城市';
#增加task info的unique key(uat 1, fat 1, prod 1)
alter table task_info add unique key uk_date_type (`generate_date`,`task_type`);

View File

@ -0,0 +1,21 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,61 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"scoresection_test.go",
"up_base_info_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/model/calculator:go_default_library",
"//vendor/github.com/go-sql-driver/mysql:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"scoresection.go",
"up_base_info.go",
],
importmap = "go-common/app/service/main/upcredit/dao/upcrmdao",
importpath = "go-common/app/service/main/upcredit/dao/upcrmdao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/model/calculator:go_default_library",
"//app/service/main/upcredit/model/upcrmmodel:go_default_library",
"//library/log:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/jinzhu/gorm:go_default_library",
"//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,98 @@
package upcrmdao
import (
"github.com/jinzhu/gorm"
"go-common/app/service/main/upcredit/conf"
"context"
"fmt"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
)
//Dao upcrm dao
type Dao struct {
conf *conf.Config
crmdb *gorm.DB
}
//New create
func New(c *conf.Config) *Dao {
var d = &Dao{
conf: c,
}
crmdb, err := gorm.Open("mysql", c.DB.Upcrm.DSN)
if crmdb == nil {
log.Error("connect to db fail, err=%v", err)
return nil
}
d.crmdb = crmdb
crmdb.SingularTable(true)
d.crmdb.LogMode(c.IsTest)
return d
}
//Close close
func (d *Dao) Close() {
if d.crmdb != nil {
d.crmdb.Close()
}
}
//AddLog add log
func (d *Dao) AddLog(arg *upcrmmodel.ArgCreditLogAdd) error {
var creditLog = &upcrmmodel.CreditLog{}
creditLog.CopyFrom(arg)
return d.crmdb.Create(creditLog).Error
}
//AddCreditScore add score
func (d *Dao) AddCreditScore(creditScore *upcrmmodel.UpScoreHistory) error {
return d.crmdb.Create(creditScore).Error
}
//AddOrUpdateCreditScore update score
func (d *Dao) AddOrUpdateCreditScore(creditScore *upcrmmodel.UpScoreHistory) (err error) {
var tablename = creditScore.TableName()
var insertSQL = fmt.Sprintf("insert into %s (mid, score_type, score, generate_date, ctime) values (?,?,?,?,?) "+
"on duplicate key update score=?", tablename)
err = d.crmdb.Exec(
insertSQL,
creditScore.Mid, creditScore.ScoreType, creditScore.Score, creditScore.GenerateDate, creditScore.CTime,
creditScore.Score).Error
if err != nil {
log.Error("add credit score fail, mid=%d, err=%+v", creditScore.Mid, err)
}
return
}
//GetCreditScore get score
func (d *Dao) GetCreditScore(c context.Context, arg *upcrmmodel.GetScoreParam) (results []*upcrmmodel.UpScoreHistory, err error) {
var mod = upcrmmodel.UpScoreHistory{
Mid: arg.Mid,
}
err = d.crmdb.Table(mod.TableName()).Select("score, generate_date").
Where("mid=? AND generate_date>=? AND generate_date<=? AND score_type=?", arg.Mid, arg.FromDate, arg.ToDate, arg.ScoreType).
Find(&results).Error
if err != nil {
log.Error("get score history fail, arg=%+v, err=%+v", arg, err)
}
return
}
//GetCreditLog get log
func (d *Dao) GetCreditLog(c context.Context, arg *upcrmmodel.ArgGetLogHistory) (results []*upcrmmodel.SimpleCreditLogWithContent, err error) {
var mod = upcrmmodel.SimpleCreditLogWithContent{}
mod.Mid = arg.Mid
err = d.crmdb.Table(mod.TableName()).Select("type, op_type, reason, business_type, ctime, content, oid").
//Limit(arg.Limit).
//Offset(arg.Offset).
Where("mid=? AND ctime>=? AND ctime<=?", arg.Mid, arg.FromDate, arg.ToDate).
Order("ctime").
Find(&results).Error
if err != nil {
log.Error("get log history fail, arg=%+v, err=%+v", arg, err)
}
return
}

View File

@ -0,0 +1,50 @@
package upcrmdao
import (
"flag"
"github.com/go-sql-driver/mysql"
"go-common/app/service/main/upcredit/conf"
"os"
"testing"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.archive.upcredit-service")
flag.Set("conf_token", "e85677843358d6c5dcd7246e6e3fc2de")
flag.Set("tree_id", "33287")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../../cmd/upcredit-service.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
m.Run()
os.Exit(0)
}
func IgnoreErr(err error) error {
if err == nil {
return err
}
var e, _ = err.(*mysql.MySQLError)
if e != nil {
switch e.Number {
case 1062:
return nil
}
}
return err
}

View File

@ -0,0 +1,30 @@
package upcrmdao
import (
"go-common/app/service/main/upcredit/model/calculator"
"go-common/app/service/main/upcredit/model/upcrmmodel"
xtime "go-common/library/time"
"time"
)
//InsertScoreSection insert score section
func (d *Dao) InsertScoreSection(statis calculator.OverAllStatistic, scoreType int, date time.Time) error {
var history upcrmmodel.ScoreSectionHistory
history.Section0 = statis.GetScore(0)
history.Section1 = statis.GetScore(1)
history.Section2 = statis.GetScore(2)
history.Section3 = statis.GetScore(3)
history.Section4 = statis.GetScore(4)
history.Section5 = statis.GetScore(5)
history.Section6 = statis.GetScore(6)
history.Section7 = statis.GetScore(7)
history.Section8 = statis.GetScore(8)
history.Section9 = statis.GetScore(9)
history.ScoreType = scoreType
history.GenerateDate = xtime.Time(date.Unix())
var now = time.Now().Unix()
history.CTime = xtime.Time(now)
history.MTime = xtime.Time(now)
// insert or update
return d.crmdb.Create(&history).Error
}

View File

@ -0,0 +1,24 @@
package upcrmdao
import (
"go-common/app/service/main/upcredit/model/calculator"
"testing"
"time"
"github.com/smartystreets/goconvey/convey"
)
func TestUpcrmdaoInsertScoreSection(t *testing.T) {
var (
statis calculator.OverAllStatistic
scoreType = int(0)
date = time.Now()
)
convey.Convey("InsertScoreSection", t, func(ctx convey.C) {
err := d.InsertScoreSection(statis, scoreType, date)
err = IgnoreErr(err)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

View File

@ -0,0 +1,99 @@
package upcrmdao
import (
"fmt"
"github.com/siddontang/go-mysql/mysql"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
xtime "go-common/library/time"
"strings"
"time"
)
const (
//TimeFmtMysql mysql time format
TimeFmtMysql = mysql.TimeFormat
//TimeFmtDate with only date
TimeFmtDate = "2006-01-02"
)
//UpQualityInfo struct
type UpQualityInfo struct {
Mid int64 `json:"mid"`
QualityValue int `json:"quality_value"`
PrValue int `json:"pr_value"`
Cdate string `json:"cdate"` // 产生时间 "2018-01-01"
}
// AsPrScore copy to db struct
func (u *UpQualityInfo) AsPrScore() (history *upcrmmodel.UpScoreHistory) {
if u == nil {
return &upcrmmodel.UpScoreHistory{}
}
history = &upcrmmodel.UpScoreHistory{
Mid: u.Mid,
ScoreType: upcrmmodel.ScoreTypePr,
Score: u.PrValue,
}
var date, _ = time.Parse(TimeFmtDate, u.Cdate)
history.GenerateDate = xtime.Time(date.Unix())
return
}
// AsQualityScore copy to db struct
func (u *UpQualityInfo) AsQualityScore() (history *upcrmmodel.UpScoreHistory) {
if u == nil {
return &upcrmmodel.UpScoreHistory{}
}
history = &upcrmmodel.UpScoreHistory{
Mid: u.Mid,
ScoreType: upcrmmodel.ScoreTypeQuality,
Score: u.QualityValue,
}
var date, _ = time.Parse(TimeFmtDate, u.Cdate)
history.GenerateDate = xtime.Time(date.Unix())
return
}
//UpdateCreditScore update score
func (d *Dao) UpdateCreditScore(score int, mid int64) (affectRow int64, err error) {
var db = d.crmdb.Model(upcrmmodel.UpBaseInfo{}).Where("mid = ? and business_type = 1", mid).Update("credit_score", score)
return db.RowsAffected, db.Error
}
//UpdateQualityAndPrScore update score
func (d *Dao) UpdateQualityAndPrScore(prScore int, qualityScore int, mid int64) (affectRow int64, err error) {
var db = d.crmdb.Model(upcrmmodel.UpBaseInfo{}).Where("mid = ? and business_type = 1", mid).Update(map[string]int{"pr_score": prScore, "quality_score": qualityScore})
return db.RowsAffected, db.Error
}
//InsertScoreHistory insert into score history
func (d *Dao) InsertScoreHistory(info *UpQualityInfo) (affectRow int64, err error) {
var qualityScoreSt = info.AsQualityScore()
err = d.crmdb.Save(qualityScoreSt).Error
if err != nil {
log.Error("insert quality score error, err=%+v", err)
}
var prScore = info.AsPrScore()
err = d.crmdb.Save(prScore).Error
if err != nil {
log.Error("insert pr score error, err=%+v", err)
}
return
}
//InsertBatchScoreHistory insert batch sql
func (d *Dao) InsertBatchScoreHistory(infoList []*UpQualityInfo, tablenum int) (affectRow int64, err error) {
var batchSQL = fmt.Sprintf("insert into up_scores_history_%02d (mid, score_type, score, generate_date) values ", tablenum)
var valueString []string
var valueArgs []interface{}
for _, info := range infoList {
valueString = append(valueString, "(?,?,?,?),(?,?,?,?)")
valueArgs = append(valueArgs, info.Mid, upcrmmodel.ScoreTypePr, info.PrValue, info.Cdate)
valueArgs = append(valueArgs, info.Mid, upcrmmodel.ScoreTypeQuality, info.QualityValue, info.Cdate)
}
var db = d.crmdb.Exec(batchSQL+strings.Join(valueString, ","), valueArgs...)
affectRow = db.RowsAffected
err = db.Error
return
}

View File

@ -0,0 +1,88 @@
package upcrmdao
import (
"testing"
"github.com/smartystreets/goconvey/convey"
"time"
)
func TestUpcrmdaoAsPrScore(t *testing.T) {
convey.Convey("AsPrScore", t, func(ctx convey.C) {
var info *UpQualityInfo
history := info.AsPrScore()
ctx.Convey("Then history should not be nil.", func(ctx convey.C) {
ctx.So(history, convey.ShouldNotBeNil)
})
})
}
func TestUpcrmdaoAsQualityScore(t *testing.T) {
convey.Convey("AsQualityScore", t, func(ctx convey.C) {
var info *UpQualityInfo
history := info.AsQualityScore()
ctx.Convey("Then history should not be nil.", func(ctx convey.C) {
ctx.So(history, convey.ShouldNotBeNil)
})
})
}
func TestUpcrmdaoUpdateCreditScore(t *testing.T) {
var (
score = int(0)
mid = int64(0)
)
convey.Convey("UpdateCreditScore", t, func(ctx convey.C) {
affectRow, err := d.UpdateCreditScore(score, mid)
err = IgnoreErr(err)
ctx.Convey("Then err should be nil.affectRow should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(affectRow, convey.ShouldNotBeNil)
})
})
}
func TestUpcrmdaoUpdateQualityAndPrScore(t *testing.T) {
var (
prScore = int(0)
qualityScore = int(0)
mid = int64(0)
)
convey.Convey("UpdateQualityAndPrScore", t, func(ctx convey.C) {
affectRow, err := d.UpdateQualityAndPrScore(prScore, qualityScore, mid)
err = IgnoreErr(err)
ctx.Convey("Then err should be nil.affectRow should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(affectRow, convey.ShouldNotBeNil)
})
})
}
func TestUpcrmdaoInsertScoreHistory(t *testing.T) {
var (
info = &UpQualityInfo{}
)
convey.Convey("InsertScoreHistory", t, func(ctx convey.C) {
affectRow, err := d.InsertScoreHistory(info)
err = IgnoreErr(err)
ctx.Convey("Then err should be nil.affectRow should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(affectRow, convey.ShouldNotBeNil)
})
})
}
func TestUpcrmdaoInsertBatchScoreHistory(t *testing.T) {
var (
infoList = []*UpQualityInfo{{Mid: 100, Cdate: time.Now().Format(TimeFmtDate)}}
tablenum = int(0)
)
convey.Convey("InsertBatchScoreHistory", t, func(ctx convey.C) {
affectRow, err := d.InsertBatchScoreHistory(infoList, tablenum)
err = IgnoreErr(err)
ctx.Convey("Then err should be nil.affectRow should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(affectRow, convey.ShouldNotBeNil)
})
})
}

View File

@ -0,0 +1,53 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["http_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = ["//app/service/main/up/conf:go_default_library"],
)
go_library(
name = "go_default_library",
srcs = [
"http.go",
"logcredit.go",
],
importmap = "go-common/app/service/main/upcredit/http",
importpath = "go-common/app/service/main/upcredit/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/model/calculator:go_default_library",
"//app/service/main/upcredit/model/upcrmmodel:go_default_library",
"//app/service/main/upcredit/service:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/permit:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,67 @@
package http
import (
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/permit"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
idfSvc *verify.Verify
//Svc service.
Svc *service.Service
authSrc *permit.Permit
)
// Init init account service.
func Init(c *conf.Config) {
// service
initService(c)
// init internal router
innerEngine := bm.DefaultServer(c.BM.Inner)
setupInnerEngine(innerEngine)
// init internal server
if err := innerEngine.Start(); err != nil {
log.Error("httpx.Serve2 error(%v)", err)
panic(err)
}
}
func initService(c *conf.Config) {
idfSvc = verify.New(nil)
Svc = service.New(c)
authSrc = permit.New(c.Auth)
}
// innerRouter
func setupInnerEngine(e *bm.Engine) {
// monitor ping
e.GET("/monitor/ping", ping)
// base
var base *bm.RouterGroup
if conf.Conf.IsTest {
base = e.Group("/x/internal/upcredit")
} else {
base = e.Group("/x/internal/upcredit", idfSvc.Verify)
}
{
base.GET("/test", test)
base.POST("/log/add", logCredit)
base.GET("/log/get", logGet)
base.GET("/score/get", scoreGet)
base.POST("/score/recalc", recalc)
base.POST("/score/calc_section", calcSection)
}
}
// ping check server ok.
func ping(ctx *bm.Context) {
}

View File

@ -0,0 +1,17 @@
package http
import (
"flag"
"path/filepath"
"time"
"go-common/app/service/main/up/conf"
)
func init() {
dir, _ := filepath.Abs("../cmd/upcredit-service.toml")
flag.Set("conf", dir)
conf.Init()
// Init(conf.Conf)
time.Sleep(time.Second)
}

View File

@ -0,0 +1,210 @@
package http
import (
"encoding/json"
"fmt"
"go-common/app/service/main/upcredit/model/calculator"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/app/service/main/upcredit/service"
"go-common/library/ecode"
"go-common/library/log"
"go-common/library/net/http/blademaster"
"io/ioutil"
"time"
)
func logCredit(c *blademaster.Context) {
var data interface{}
var err error
var errMsg string
var r = new(upcrmmodel.ArgCreditLogAdd)
switch {
default:
var body, _ = ioutil.ReadAll(c.Request.Body)
if err = json.Unmarshal(body, r); err != nil {
log.Error("request argument json decode fail, err=%v", err)
errMsg = fmt.Sprintf("wrong argument, %s, body={%s}", err.Error(), string(body))
err = ecode.RequestErr
break
}
err = Svc.LogCredit(c, r)
}
if err != nil {
service.BmHTTPErrorWithMsg(c, err, errMsg)
} else {
c.JSON(data, err)
}
}
func recalc(c *blademaster.Context) {
var data interface{}
var err error
var errMsg string
var r = new(struct {
TableNum int `form:"tablenum"`
CalcDate string `form:"date"`
AllTable bool `form:"all_table" default:"false"`
})
switch {
default:
if err = c.Bind(r); err != nil {
log.Error("param error")
err = ecode.RequestErr
break
}
var calc = calculator.New(Svc.CreditScoreInputChan)
var date = time.Time{}
if r.CalcDate != "" {
date, _ = time.Parse("2006-01-02", r.CalcDate)
} else {
date = time.Now()
}
if r.AllTable {
Svc.CalcSvc.AddCalcJob(date)
} else {
go calc.CalcLogTable(r.TableNum, date, nil)
}
log.Info("start calculate process, req=%+v", r)
}
if err != nil {
service.BmHTTPErrorWithMsg(c, err, errMsg)
} else {
c.JSON(data, err)
}
}
func scoreGet(c *blademaster.Context) {
var data interface{}
var err error
var errMsg string
var r = new(upcrmmodel.ArgMidDate)
switch {
default:
if err = c.Bind(r); err != nil {
log.Error("param error")
err = ecode.RequestErr
break
}
var arg = upcrmmodel.GetScoreParam{
Mid: r.Mid,
ScoreType: r.ScoreType,
}
var now = time.Now()
if r.Days > 60 {
r.Days = 60
}
if r.FromDate != "" {
arg.FromDate, err = time.Parse(upcrmmodel.DateStr, r.FromDate)
if err != nil {
errMsg = err.Error()
err = ecode.RequestErr
break
}
}
if r.ToDate != "" {
arg.ToDate, err = time.Parse(upcrmmodel.DateStr, r.FromDate)
if err != nil {
errMsg = err.Error()
err = ecode.RequestErr
break
}
}
if r.Days != 0 {
var y, m, d = now.Date()
arg.ToDate = time.Date(y, m, d, 0, 0, 0, 0, now.Location())
arg.FromDate = arg.ToDate.AddDate(0, 0, -r.Days)
}
var result, e = Svc.GetCreditScore(c, &arg)
err = e
if err != nil {
log.Error("fail to get credit score, req=%+v, err=%+v", arg, err)
break
}
data = map[string]interface{}{
"score_list": result,
}
log.Info("get credit score, req=%+v, datalen=%d", r, len(result))
}
if err != nil {
service.BmHTTPErrorWithMsg(c, err, errMsg)
} else {
c.JSON(data, err)
}
}
func logGet(c *blademaster.Context) {
var data interface{}
var err error
var errMsg string
var r = new(upcrmmodel.ArgGetLogHistory)
switch {
default:
if err = c.Bind(r); err != nil {
log.Error("param error")
errMsg = "param error"
err = ecode.RequestErr
break
}
var result, e = Svc.GetCreditLog(c, r)
err = e
if err != nil {
errMsg = err.Error()
log.Error("fail to get credit log, req=%+v, err=%+v", r, err)
break
}
data = map[string]interface{}{
"log_list": result,
}
log.Info("get credit log ok, req=%+v, datalen=%d", r, len(result))
}
if err != nil {
service.BmHTTPErrorWithMsg(c, err, errMsg)
} else {
c.JSON(data, err)
}
}
func calcSection(c *blademaster.Context) {
var data interface{}
var err error
var errMsg string
var r = new(struct {
CalcDate string `form:"date"`
})
switch {
default:
if err = c.Bind(r); err != nil {
log.Error("param error")
err = ecode.RequestErr
break
}
var date = time.Time{}
if r.CalcDate != "" {
date, _ = time.Parse("2006-01-02", r.CalcDate)
} else {
date = time.Now().AddDate(0, 0, -1)
}
var job = &service.CalcStatisticJob{
ID: 1,
Date: date,
Svc: Svc.CalcSvc,
}
Svc.CalcSvc.JobChannel <- job
log.Info("start calculate process, req=%+v, job=%+v", r, job)
}
if err != nil {
service.BmHTTPErrorWithMsg(c, err, errMsg)
} else {
c.JSON(data, err)
}
}
func test(c *blademaster.Context) {
Svc.Test()
}

View File

@ -0,0 +1,42 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"base_utils.go",
"limiter.go",
],
importmap = "go-common/app/service/main/upcredit/mathutil",
importpath = "go-common/app/service/main/upcredit/mathutil",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["limiter_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = ["//vendor/github.com/smartystreets/goconvey/convey:go_default_library"],
)

View File

@ -0,0 +1,45 @@
package mathutil
//EPSILON very small
var EPSILON float32 = 0.00000001
//FloatEquals float equal
func FloatEquals(a, b float32) bool {
if (a-b) < EPSILON && (b-a) < EPSILON {
return true
}
return false
}
//EPSILON64 very small
var EPSILON64 = 0.00000001
//Float64Equals float equal
func Float64Equals(a, b float64) bool {
if (a-b) < EPSILON64 && (b-a) < EPSILON64 {
return true
}
return false
}
//Min min
func Min(a, b int) int {
if a < b {
return a
}
if b < a {
return b
}
return a
}
//Max max
func Max(a, b int) int {
if a > b {
return a
}
if b > a {
return b
}
return a
}

View File

@ -0,0 +1,32 @@
package mathutil
import "time"
//Limiter speed limiter
type Limiter struct {
Rate float64 // 每秒多少个
token chan time.Time
timer *time.Ticker
}
//Token get token
func (l *Limiter) Token() (c <-chan time.Time) {
return l.token
}
func (l *Limiter) putToken() {
for t := range l.timer.C {
l.token <- t
}
}
//NewLimiter create new limiter
func NewLimiter(rate float64) *Limiter {
var l = &Limiter{
Rate: rate,
token: make(chan time.Time, 1),
timer: time.NewTicker(time.Duration(1.0 / rate * float64(time.Second))),
}
go l.putToken()
return l
}

View File

@ -0,0 +1,46 @@
package mathutil
import (
. "github.com/smartystreets/goconvey/convey"
"math"
"testing"
"time"
)
func Test_limiter(t *testing.T) {
Convey("limit interval", t, func() {
var rate = 100.0
var limit = NewLimiter(rate)
var interval = 1.0 / rate
var last time.Time
for i := 0; i < 100; i++ {
var t = <-limit.Token()
if !last.IsZero() {
var diff = t.Sub(last)
So(math.Abs(diff.Seconds()-interval), ShouldBeLessThanOrEqualTo, 0.002)
}
last = t
}
})
Convey("limit count", t, func() {
var rate = 100.0
var seconds = 10.0
var limit = NewLimiter(rate)
var expect = rate * seconds
var timer = time.NewTimer(time.Duration(float64(time.Second) * seconds))
var total = 0
var run = true
for run {
select {
case <-timer.C:
run = false
default:
<-limit.Token()
total++
}
}
So(math.Abs(float64(total)-expect), ShouldBeLessThanOrEqualTo, rate*0.01)
})
}

View File

@ -0,0 +1,32 @@
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/service/main/upcredit/model/calculator:all-srcs",
"//app/service/main/upcredit/model/canal:all-srcs",
"//app/service/main/upcredit/model/databus:all-srcs",
"//app/service/main/upcredit/model/upcrmmodel:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,54 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["arcfsm_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/model/upcrmmodel:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"creditscore.go",
"score_rules.go",
],
importmap = "go-common/app/service/main/upcredit/model/calculator",
importpath = "go-common/app/service/main/upcredit/model/calculator",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/upcredit/common/fsm:go_default_library",
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/mathutil:go_default_library",
"//app/service/main/upcredit/model/upcrmmodel:go_default_library",
"//library/log:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/jinzhu/gorm:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,51 @@
package calculator
import (
"flag"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"path/filepath"
"testing"
)
func init() {
dir, _ := filepath.Abs("../../cmd/upcredit-service.toml")
flag.Set("conf", dir)
conf.Init()
}
var (
logs = []upcrmmodel.SimpleCreditLog{
{Type: 1, OpType: -10, Reason: 0},
{Type: 1, OpType: -1, Reason: 0},
{Type: 1, OpType: -3, Reason: 0},
{Type: 1, OpType: -1, Reason: 0},
{Type: 1, OpType: -1, Reason: 0},
{Type: 1, OpType: 0, Reason: 0},
{Type: 1, OpType: 0, Reason: 0},
{Type: 1, OpType: -9, Reason: 0},
{Type: 1, OpType: 0, Reason: 0},
{Type: 1, OpType: -30, Reason: 0},
}
)
func TestArcFSM(t *testing.T) {
var stat = creditStat{}
var article = CreateArticleStateMachine(logs[0].OpType, logs[0].Type, logs[0].Reason)
for i := 1; i < len(logs); i++ {
article.OnLog(&logs[i], stat.onLogResult)
}
stat.CalcRelativeScore()
stat.CalcTotalScore()
t.Logf("stat: %+v", stat)
}
func TestArcFsmInitState(t *testing.T) {
var fsm = CreateArticleStateMachineWithInitState()
var init = conf.CreditConfig.ArticleRule.InitState
if fsm.Round != init.Round ||
fsm.Reason != init.Reason ||
fsm.State != init.State {
t.Errorf("fail to pass init state!")
}
}

View File

@ -0,0 +1,339 @@
package calculator
import (
"errors"
"fmt"
"github.com/jinzhu/gorm"
"go-common/app/service/main/upcredit/common/fsm"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/mathutil"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
xtime "go-common/library/time"
"math"
"sort"
"time"
)
/*
计算过程
1.读取up的信用日志记录
2.按照时间维度分为1年内、2年内...的记录并计算每年的分数并不是自然年是以当前时间为起点的365天内算1年,calcUpCreditScoreByYear
3.总分为每年分数的加权平均没有记录的分数为0的年份不记入加权计算,calcWeightedCreditScore, -> 标准化 calcNormalizedCreditScore
4.写入对应的分数db
5.全部完成后写入task info db
其他:
1.每个分数计算分为绝对分与相对分的加权平均
*/
const (
//ScoreRange score range, max - min
ScoreRange = float32(2000)
//MaxScore max score
MaxScore = float32(1000)
//WeightRelative relative weight
WeightRelative = float32(0.5)
)
//CreditScoreCalculator score calculator
type CreditScoreCalculator struct {
CreditScoreOutputChan chan<- *upcrmmodel.UpScoreHistory
}
//New create
func New(channel chan<- *upcrmmodel.UpScoreHistory) *CreditScoreCalculator {
var c = &CreditScoreCalculator{
CreditScoreOutputChan: channel,
}
return c
}
type upLog struct {
Mid int64
CreditLogs []upcrmmodel.SimpleCreditLog
}
type creditStat struct {
RejectArcNum int64
AcceptArcNum int64
AbsoluteScore float32
RelativeScore float32
TotalScore float32
}
func (c *creditStat) onLogResult(e *fsm.Event, l *upcrmmodel.SimpleCreditLog, a *ArticleStateMachine) {
var score = 0
var ruleConfig = conf.CreditConfig.ArticleRule
switch e.Dst {
case StateClose:
if ruleConfig.IsRejected(l.Type, l.OpType, l.Reason) {
c.RejectArcNum++
fmt.Printf("article rejected, state=%d, round=%d\n", a.State, a.Round)
} else {
fmt.Printf("article close, but not rejected, state=%d, round=%d\n", a.State, a.Round)
}
score = ruleConfig.GetScore(l.Type, l.OpType, l.Reason)
c.AbsoluteScore += float32(score)
fmt.Printf("reject, score=%d\n", score)
case StateOpen:
c.AcceptArcNum++
score = conf.CreditConfig.ArticleRule.GetScore(l.Type, l.OpType, l.Reason)
c.AbsoluteScore += float32(score)
fmt.Printf("accept, score=%d\n", score)
}
}
//CalcRelativeScore relative score
func (c *creditStat) CalcRelativeScore() {
c.RelativeScore = relativeScore(c.RejectArcNum, c.AcceptArcNum)
}
//CalcTotalScore total score
func (c *creditStat) CalcTotalScore() {
// 绝对分数限制在 (-SCORE_RANGE/2, SCORE_RANGE/2)
c.AbsoluteScore = ScoreRange * float32(1/(1+math.Exp(-float64(c.AbsoluteScore/300)))-0.5)
c.TotalScore = WeightRelative*c.RelativeScore + (1-WeightRelative)*c.AbsoluteScore
}
//AppendLog append log
func (u *upLog) AppendLog(log upcrmmodel.SimpleCreditLog) {
u.CreditLogs = append(u.CreditLogs, log)
}
//SortLog sort log by ctime asc
func (u *upLog) SortLog() {
sort.SliceStable(u.CreditLogs, func(i, j int) bool {
return u.CreditLogs[i].CTime < u.CreditLogs[j].CTime
})
}
func relativeScore(rejectArcNum int64, acceptedArcnum int64) float32 {
var total = rejectArcNum + acceptedArcnum
if total == 0 {
return 0
}
var factor = float32(1.0)
if total <= 5 {
factor = 0.2
} else if total <= 10 {
factor = 0.5
}
const middle = 0.6
var accRatio = float32(acceptedArcnum)/float32(total) - middle
if accRatio < -0.5 {
accRatio = -0.5
}
return factor * accRatio * ScoreRange
}
//
const (
//Day day
Day = time.Hour * 24 / time.Second
//Month month
Month = 30 * Day
//Year year
Year = 365 * Day
)
func calcUpCreditScoreByYear(uplog *upLog, currentTime time.Time) (yearStat map[int]*creditStat) {
// 以时间为keycreditStat为value的分类
yearStat = make(map[int]*creditStat)
var articleMachine = make(map[int]*ArticleStateMachine)
var now = currentTime.Unix()
for _, l := range uplog.CreditLogs {
var difftime = now - int64(l.CTime)
// 不计算来自未来的数据
if difftime < 0 {
continue
}
var index = int(difftime / int64(Year))
v := getOrCreateCreditScore(index, yearStat)
for _, r := range RuleList {
r(l, v, articleMachine)
}
}
// 计算每年的分数
for k, v := range yearStat {
v.CalcRelativeScore()
v.CalcTotalScore()
log.Info("score for mid:%d, [%d]s=%+v", uplog.Mid, k, v)
}
return
}
func calcWeightedCreditScore(yearStat map[int]*creditStat) (score float32) {
// 每年的分数加权平均如果当年分数为0那不进行加权平均
var totalWeight = float32(0)
var totalScore = float32(0)
for diff, weight := range conf.CreditConfig.CalculateConf.TimeWeight2 {
s, o := yearStat[diff]
if !o {
continue
}
totalWeight += float32(weight)
totalScore += float32(s.TotalScore) * float32(weight)
}
log.Info("total score=%f, total weight=%f", totalScore, totalWeight)
if !mathutil.FloatEquals(totalWeight, 0) {
score = totalScore / totalWeight
}
return
}
func calcNormalizedCreditScore(weightedScore float32) float32 {
return (weightedScore/ScoreRange + 0.5) * MaxScore
}
func getOrCreateArticleFSM(aid int, m map[int]*ArticleStateMachine) (artFsm *ArticleStateMachine) {
artFsm, ok := m[aid]
if !ok {
artFsm = CreateArticleStateMachineWithInitState()
m[aid] = artFsm
}
return
}
func getOrCreateCreditScore(index int, m map[int]*creditStat) (c *creditStat) {
c, ok := m[index]
if !ok {
c = new(creditStat)
m[index] = c
}
return
}
//OverAllStatistic score statistics
type OverAllStatistic struct {
MaxScore int
SectionCount int
// 分数段人数统计, map[分数段:0~9]int人数
ScoreSection map[int]int
sectionScore int
}
//NewOverAllStatistic create empty
func NewOverAllStatistic(maxScore int, sectionCount int) *OverAllStatistic {
if maxScore <= 0 || sectionCount <= 0 {
panic(errors.New("max score or sction count must > 0"))
}
return &OverAllStatistic{
MaxScore: maxScore,
SectionCount: sectionCount,
ScoreSection: map[int]int{},
sectionScore: maxScore / sectionCount,
}
}
//AddScore add score
/*params:
score, 分数
exceptScore不统计的分数主要是默认分数不进行统计
*/
func (s *OverAllStatistic) AddScore(score int, exceptScore int) {
if score == exceptScore {
return
}
var section = score / s.sectionScore
if section > s.SectionCount-1 {
section = s.SectionCount - 1
}
s.ScoreSection[section]++
}
// GetScore no record will return default int(0)
func (s *OverAllStatistic) GetScore(section int) int {
return s.ScoreSection[section]
}
//CalcLogTable calculate all table
func (c *CreditScoreCalculator) CalcLogTable(tableNum int, currentDate time.Time, overall *OverAllStatistic) (err error) {
var crmdb, e = gorm.Open("mysql", conf.Conf.DB.UpcrmReader.DSN)
err = e
if e != nil {
log.Error("fail to open crm db, for table=%d", tableNum)
return
}
crmdb.LogMode(true)
defer crmdb.Close()
var startTime = time.Now()
var upLogMap = make(map[int64]*upLog)
var lastID uint
var limit = 1000
var tableName = fmt.Sprintf("credit_log_%02d", tableNum)
log.Info("table[%s] start load player's data", tableName)
var total = 0
for {
var users []upcrmmodel.SimpleCreditLog
e = crmdb.Table(tableName).Where("id > ?", lastID).Limit(limit).Find(&users).Error
if e != nil {
err = e
log.Error("fail to get users from db, err=%v", e)
break
}
// 加入到列表中
for _, l := range users {
up, ok := upLogMap[l.Mid]
if !ok {
up = &upLog{}
up.Mid = l.Mid
upLogMap[l.Mid] = up
}
lastID = l.ID
up.AppendLog(l)
}
var thisCount = len(users)
total += thisCount
if thisCount < limit {
log.Info("table[%s] total read record, num=%d", tableName, thisCount)
break
}
}
//crmdb.Close()
if err != nil {
log.Error("table[%s] error happen, exit calc, err=%v", tableName, err)
return
}
log.Info("table[%s] start calculate player's data, total mid=%d", tableName, len(upLogMap))
var date = currentDate
for _, v := range upLogMap {
v.SortLog()
var yearStat = calcUpCreditScoreByYear(v, date)
var weightedScore = calcWeightedCreditScore(yearStat)
var finalScore = calcNormalizedCreditScore(weightedScore)
log.Info("mid=%d, weightscore=%f, finalscore=%f", v.Mid, weightedScore, finalScore)
c.writeUpCreditScore(v.Mid, finalScore, date, crmdb, overall)
}
var elapsed = time.Since(startTime)
log.Info("table[%s] finish calculate player's data, total mid=%d, total logs=%d, duration=%s, avg=%0.2f/s", tableName, len(upLogMap), total, elapsed, float64(total)/elapsed.Seconds())
return
}
func (c *CreditScoreCalculator) writeUpCreditScore(mid int64, score float32, date time.Time, crmdb *gorm.DB, overall *OverAllStatistic) {
if c.CreditScoreOutputChan == nil {
log.Error("output chan is nil, fail to output credit score")
return
}
if overall != nil {
overall.AddScore(int(score), 500)
}
var creditScore = &upcrmmodel.UpScoreHistory{
Mid: mid,
Score: int(score),
ScoreType: upcrmmodel.ScoreTypeCredit,
GenerateDate: xtime.Time(date.Unix()),
CTime: xtime.Time(time.Now().Unix()),
}
c.CreditScoreOutputChan <- creditScore
log.Info("output credit score, mid=%d", mid)
}

View File

@ -0,0 +1,152 @@
package calculator
import (
"fmt"
"go-common/app/service/main/upcredit/common/fsm"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
)
//Rule rule function
type Rule func(log upcrmmodel.SimpleCreditLog, stat *creditStat, args ...interface{})
var (
//RuleList rule list
RuleList = []Rule{rule1}
)
const (
//StateOpen open
StateOpen = "st_open"
//StateClose close
StateClose = "st_close"
//EventOpen ev open
EventOpen = "open"
//EventClose ev close
EventClose = "close"
)
//ArticleStateMachine state machine
type ArticleStateMachine struct {
State int
Round int
Reason int
openCount int // 记录一共open了多少次
Fsm *fsm.FSM
}
// ----------------state machine
func (a *ArticleStateMachine) onEnterOpen(e *fsm.Event) {
var l = e.Args[0].(*upcrmmodel.SimpleCreditLog)
if l == nil {
log.Error("event arg is not upcrmmodel.SimpleCreditLog, please check code")
return
}
if a.openCount >= conf.CreditConfig.ArticleRule.ArticleMaxOpenCount {
fmt.Printf("article max open count exceeded")
return
}
a.openCount++
a.setState(l)
var callback, ok = e.Args[1].(OnLogResult)
if ok && callback != nil {
(callback)(e, l, a)
}
fmt.Printf("article opened\n")
}
func (a *ArticleStateMachine) onEnterClose(e *fsm.Event) {
var l = e.Args[0].(*upcrmmodel.SimpleCreditLog)
if l == nil {
log.Error("event arg is not upcrmmodel.SimpleCreditLog, please check code")
return
}
if a.State == l.OpType && a.Round == l.Type {
fmt.Printf("nothing happend here! state=%d, round=%d\n", a.State, a.Round)
return
}
a.setState(l)
var callback, ok = e.Args[1].(OnLogResult)
if ok && callback != nil {
callback(e, l, a)
}
}
//OnLogResult on log function
type OnLogResult func(e *fsm.Event, l *upcrmmodel.SimpleCreditLog, a *ArticleStateMachine)
// OnLog -----------------state machine
func (a *ArticleStateMachine) OnLog(l *upcrmmodel.SimpleCreditLog, callback OnLogResult) {
var event = ""
if l.OpType == 0 {
event = EventOpen
} else if l.OpType < 0 {
event = EventClose
}
if event != "" {
a.Fsm.Event(event, l, callback)
}
}
func (a *ArticleStateMachine) setState(l *upcrmmodel.SimpleCreditLog) {
a.State = l.OpType
a.Round = l.Type
a.Reason = l.Reason
}
//CreateArticleStateMachineWithInitState create init machine
func CreateArticleStateMachineWithInitState() (asm *ArticleStateMachine) {
var initState = &conf.CreditConfig.ArticleRule.InitState
return CreateArticleStateMachine(initState.State, initState.Round, initState.Reason)
}
// CreateArticleStateMachine 这里的state, round, reason是稿件的state, round, reason
func CreateArticleStateMachine(state int, round int, reason int) (asm *ArticleStateMachine) {
asm = &ArticleStateMachine{
State: state,
Round: round,
Reason: reason,
}
var initstate = StateOpen
if state < 0 {
initstate = StateClose
}
asm.Fsm = fsm.NewFSM(
initstate,
fsm.Events{
{Name: EventOpen, Src: []string{StateClose}, Dst: StateOpen},
{Name: EventClose, Src: []string{StateOpen}, Dst: StateClose},
{Name: EventClose, Src: []string{StateClose}, Dst: StateClose},
},
fsm.Callbacks{
"enter_" + StateOpen: func(e *fsm.Event) { asm.onEnterOpen(e) },
"enter_" + StateClose: func(e *fsm.Event) { asm.onEnterClose(e) },
},
)
return asm
}
func rule1(l upcrmmodel.SimpleCreditLog, stat *creditStat, args ...interface{}) {
switch l.BusinessType {
case upcrmmodel.BusinessTypeArticleAudit:
if len(args) >= 1 {
machineMap, ok := args[0].(map[int]*ArticleStateMachine)
if !ok {
log.Error("fail to get machineMap")
}
articleFsm := getOrCreateArticleFSM(int(l.Oid), machineMap)
if articleFsm != nil {
articleFsm.OnLog(&l, stat.onLogResult)
}
}
}
}

View File

@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["message.go"],
importpath = "go-common/app/service/main/upcredit/model/canal",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,11 @@
package canal
import "encoding/json"
// Msg canal databus msg.
type Msg struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
}

View File

@ -0,0 +1,29 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["upcredit_log.go"],
importmap = "go-common/app/service/main/upcredit/model/databus",
importpath = "go-common/app/service/main/upcredit/model/databus",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,5 @@
package databus
//CreditLogMsg databus
type CreditLogMsg struct {
}

View File

@ -0,0 +1,5 @@
gopath="$GOPATH/src"
protobuf_path="$gopath/go-common/vendor/github.com/gogo/protobuf"
echo $protobuf_path
ls "$gopath/go-common/vendor/github.com/gogo/protobuf/gogoproto/"
protoc --gofast_out=".." -I"../" -I"$gopath/go-common/vendor/" ../*.proto

View File

@ -0,0 +1,48 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"creditlog.go",
"score_section_history.go",
"up_base_info.go",
"up_score_history.go",
],
importmap = "go-common/app/service/main/upcredit/model/upcrmmodel",
importpath = "go-common/app/service/main/upcredit/model/upcrmmodel",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["//library/time:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["creditlog_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//library/time:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

View File

@ -0,0 +1,122 @@
package upcrmmodel
import (
"encoding/json"
"fmt"
"go-common/library/time"
systime "time"
)
const (
//BusinessTypeArticleAudit 1
BusinessTypeArticleAudit = 1
)
const (
//DateStr date format
DateStr = "2006-01-02"
//CreditLogTableCount all credit log table count
CreditLogTableCount = 100
)
//ArgCreditLogAdd arg
type ArgCreditLogAdd struct {
Type int `form:"type" json:"type"` // 日志类型,具体与业务方确定
OpType int `form:"op_type" json:"optype"` // 操作类型,具体与业务方确定
Reason int `form:"reason" json:"reason"` // 原因类型,具体与业务方确定
BusinessType int `form:"bussiness_type" json:"business_type"` // 业务类型
Mid int64 `form:"mid" validate:"required" json:"mid"` // 用户id
Oid int64 `form:"oid" json:"oid"` // 对象类型如aid
UID int `form:"uid" json:"uid"` // 管理员id
Content string `form:"content" json:"content"` // 日志内容描述
CTime time.Time `form:"ctime" json:"ctime"` // 创建时间
Extra json.RawMessage `form:"extra" json:"extra,omitempty"` // 额外字段,与业务方确定
}
//ArgMidDate arg
type ArgMidDate struct {
Mid int64 `form:"mid" validate:"required"`
Days int `form:"days"` // 最近n天内的数据1表示最近1天今天2表示最近2天默认为0
FromDate string `form:"from_date"` // 2006-01-02
ToDate string `form:"to_date"` // 2006-01-02
ScoreType int `form:"score_type" default:"3"` // 分数类型, 1,2,3参见ScoreTypeCredit
}
//GetScoreParam arg
type GetScoreParam struct {
Mid int64
FromDate systime.Time
ToDate systime.Time
ScoreType int
}
//ArgGetLogHistory arg
type ArgGetLogHistory struct {
Mid int64 `form:"mid" validate:"required"`
FromDate systime.Time `form:"from_date"`
ToDate systime.Time `form:"to_date"`
Limit int `form:"limit" default:"20"`
}
//CreditLog db struct
type CreditLog struct {
ID uint `gorm:"primary_key" json:"-"`
Type int
OpType int
Reason int
BusinessType int
Mid int64
Oid int64
UID int `gorm:"column:uid"`
Content string
CTime time.Time `gorm:"column:ctime"`
MTime time.Time `gorm:"column:mtime"`
Extra string `sql:"type:text;" json:"-"`
}
//TableName table name
func (c *CreditLog) TableName() string {
return getTableName(c.Mid)
}
func getTableName(mid int64) string {
return fmt.Sprintf("credit_log_%02d", mid%CreditLogTableCount)
}
//CopyFrom copy
func (c *CreditLog) CopyFrom(arg *ArgCreditLogAdd) *CreditLog {
c.Type = arg.Type
c.OpType = arg.OpType
c.BusinessType = arg.BusinessType
c.Reason = arg.Reason
c.Mid = arg.Mid
c.Oid = arg.Oid
c.UID = arg.UID
c.Content = arg.Content
c.CTime = arg.CTime
c.MTime = arg.CTime
c.Extra = string(arg.Extra)
return c
}
//SimpleCreditLog db struct
type SimpleCreditLog struct {
ID uint `gorm:"primary_key" json:"-"`
Type int `json:"type"`
OpType int `json:"op_type"`
Reason int `json:"reason"`
BusinessType int `json:"business_type"`
Mid int64 `json:"mid"`
Oid int64 `json:"oid"`
CTime time.Time `gorm:"column:ctime" json:"ctime"`
}
//TableName table name
func (c *SimpleCreditLog) TableName() string {
return getTableName(c.Mid)
}
//SimpleCreditLogWithContent log with content
type SimpleCreditLogWithContent struct {
SimpleCreditLog
Content string `form:"content" json:"content"` // 日志内容描述
}

View File

@ -0,0 +1,32 @@
package upcrmmodel
import (
"encoding/json"
. "github.com/smartystreets/goconvey/convey"
"go-common/library/time"
"testing"
xtime "time"
)
func Test_Orm(t *testing.T) {
var (
arg = ArgCreditLogAdd{
BusinessType: 1,
Type: 1,
OpType: 2,
Reason: 101,
Mid: 12345,
Oid: 13021,
UID: 1,
Content: "稿件打回",
CTime: time.Time(xtime.Now().Unix()),
Extra: []byte("{ \"key\" : \"value\"}"),
}
)
Convey("orm", t, func() {
Convey("connect", func() {
var js, _ = json.Marshal(arg)
t.Log(string(js))
})
})
}

View File

@ -0,0 +1,22 @@
package upcrmmodel
import "go-common/library/time"
//ScoreSectionHistory score section db
type ScoreSectionHistory struct {
ID uint32
GenerateDate time.Time `gorm:"column:generate_date"`
ScoreType int `gorm:"column:score_type"`
Section0 int `gorm:"column:section_0"`
Section1 int `gorm:"column:section_1"`
Section2 int `gorm:"column:section_2"`
Section3 int `gorm:"column:section_3"`
Section4 int `gorm:"column:section_4"`
Section5 int `gorm:"column:section_5"`
Section6 int `gorm:"column:section_6"`
Section7 int `gorm:"column:section_7"`
Section8 int `gorm:"column:section_8"`
Section9 int `gorm:"column:section_9"`
CTime time.Time `gorm:"column:ctime"`
MTime time.Time `gorm:"column:mtime"`
}

View File

@ -0,0 +1,28 @@
package upcrmmodel
import "go-common/library/time"
//UpBaseInfo db struct
type UpBaseInfo struct {
ID int32
Mid int32
Name string
Sex int8
JoinTime time.Time
FirstUpTime time.Time
Level int16
FansCount int
AccountState int
Activity int
ArticleCount30day int `gorm:"article_count_30day"`
ArticleCountAccumulate int
VerifyType int8
CTime time.Time `gorm:"ctime"`
MTime time.Time `gorm:"mtim"`
BusinessType int8
CreditScore int
PrScore int
QualityScore int
ActiveTid int16
Attr int
}

View File

@ -0,0 +1,38 @@
package upcrmmodel
import (
"fmt"
"go-common/library/time"
)
const (
//ScoreTypeQuality 1
ScoreTypeQuality = 1 // 质量分
//ScoreTypePr 2
ScoreTypePr = 2 // 影响力分
//ScoreTypeCredit 3
ScoreTypeCredit = 3 // 信用分
//UpScoreHistoryTableCount table count
UpScoreHistoryTableCount = 100
)
//UpScoreHistory db struct
type UpScoreHistory struct {
ID uint `gorm:"primary_key" json:"-"`
Mid int64 `json:"mid"`
ScoreType int `json:"-"`
Score int ` json:"score"`
GenerateDate time.Time `json:"date"`
CTime time.Time `gorm:"column:ctime" json:"-"`
MTime time.Time `gorm:"column:mtime" json:"-"`
}
func getTableNameUpScoreHistory(mid int64) string {
return fmt.Sprintf("up_scores_history_%02d", mid%UpScoreHistoryTableCount)
}
//TableName table name
func (u *UpScoreHistory) TableName() string {
return getTableNameUpScoreHistory(u.Mid)
}

View File

@ -0,0 +1,39 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["up_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["up.go"],
importmap = "go-common/app/service/main/upcredit/rpc/client",
importpath = "go-common/app/service/main/upcredit/rpc/client",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["//library/net/rpc:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,26 @@
package client
import (
"go-common/library/net/rpc"
)
const (
_appid = "archive.service.upcredit"
)
//Service rpc service
type Service struct {
client *rpc.Client2
}
//RPC interface
type RPC interface {
}
//New create
func New(c *rpc.ClientConfig) (s *Service) {
s = &Service{
client: rpc.NewDiscoveryCli(_appid, c),
}
return
}

View File

@ -0,0 +1,14 @@
package client
import (
"testing"
"time"
)
func TestRpcClient(t *testing.T) {
var _ = New(nil)
time.Sleep(1 * time.Second)
}

View File

@ -0,0 +1,50 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["rpcserver_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/service:go_default_library",
"//library/net/rpc:go_default_library",
"//library/time:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = ["rpcserver.go"],
importmap = "go-common/app/service/main/upcredit/rpc/server",
importpath = "go-common/app/service/main/upcredit/rpc/server",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/service:go_default_library",
"//library/net/rpc:go_default_library",
"//library/net/rpc/context:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,28 @@
package server
import (
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/service"
"go-common/library/net/rpc"
"go-common/library/net/rpc/context"
)
//RPC rpc server
type RPC struct {
s *service.Service
}
//New create
func New(c *conf.Config, s *service.Service) (svr *rpc.Server) {
r := &RPC{s: s}
svr = rpc.NewServer(c.RPCServer)
if err := svr.Register(r); err != nil {
panic(err)
}
return
}
// Ping check connection success.
func (r *RPC) Ping(c context.Context, arg *struct{}, res *struct{}) (err error) {
return
}

View File

@ -0,0 +1,41 @@
package server
import (
"flag"
"path/filepath"
"testing"
"time"
_ "time"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/service"
"go-common/library/net/rpc"
xtime "go-common/library/time"
)
func init() {
dir, _ := filepath.Abs("../../cmd/upcredit-service.toml")
flag.Set("conf", dir)
}
func initSvrAndClient(t *testing.T) (client *rpc.Client, err error) {
if err = conf.Init(); err != nil {
t.Errorf("conf.Init() error(%v)", err)
t.FailNow()
}
svr := service.New(conf.Conf)
New(conf.Conf, svr)
client = rpc.Dial("127.0.0.1:6079", xtime.Duration(time.Second), nil)
return
}
func TestInfo(t *testing.T) {
client, err := initSvrAndClient(t)
defer client.Close()
if err != nil {
t.Errorf("rpc.Dial error(%v)", err)
t.FailNow()
}
//time.Sleep(1 * time.Second)
}

View File

@ -0,0 +1,68 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["service_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/upcredit/conf:go_default_library",
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"business_score.go",
"calculate_service.go",
"content_ext.go",
"creditscore.go",
"databus.go",
"logcredit.go",
"service.go",
],
importmap = "go-common/app/service/main/upcredit/service",
importpath = "go-common/app/service/main/upcredit/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/upcredit/common/election:go_default_library",
"//app/service/main/upcredit/common/fsm:go_default_library",
"//app/service/main/upcredit/conf:go_default_library",
"//app/service/main/upcredit/dao/upcrmdao:go_default_library",
"//app/service/main/upcredit/mathutil:go_default_library",
"//app/service/main/upcredit/model/calculator:go_default_library",
"//app/service/main/upcredit/model/canal:go_default_library",
"//app/service/main/upcredit/model/upcrmmodel:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/render:go_default_library",
"//library/queue/databus:go_default_library",
"//vendor/github.com/jinzhu/gorm:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,138 @@
package service
import (
"encoding/json"
"github.com/pkg/errors"
"go-common/app/service/main/upcredit/dao/upcrmdao"
"go-common/app/service/main/upcredit/model/canal"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
"math/rand"
"sync"
"time"
)
// 必须是upcrmmodel.UpScoreHistoryTableCount
var batchDataList = make([][]*upcrmdao.UpQualityInfo, upcrmmodel.UpScoreHistoryTableCount)
var batchMutex = sync.Mutex{}
func (s *Service) onBusinessDatabus(data *canal.Msg) {
if data == nil {
return
}
var err error
var upInfo = &upcrmdao.UpQualityInfo{}
if err = json.Unmarshal(data.New, upInfo); err != nil {
log.Error("unmarshal business canal message fail, err=%+v, value=%s", err, string(data.New))
return
}
var index = upInfo.Mid % 100
batchMutex.Lock()
batchDataList[index] = append(batchDataList[index], upInfo)
batchMutex.Unlock()
s.businessScoreChan <- upInfo
}
func (s *Service) updateScoreProc() {
defer func() {
s.wg.Done()
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("updateScoreProc Runtime error caught, try recover: %+v", r)
s.wg.Add(1)
go s.updateScoreProc()
}
}()
var stop = false
for s.running && !stop {
<-s.limit.Token()
select {
case upInfo, ok := <-s.businessScoreChan:
if !ok {
log.Error("business score chan closed")
stop = true
break
}
if upInfo == nil {
continue
}
// update up player's baseinfo
affectedRow, err := s.upcrmdb.UpdateQualityAndPrScore(upInfo.PrValue, upInfo.QualityValue, upInfo.Mid)
if err != nil {
log.Error("error when update quality score, err=%s", err)
break
}
log.Info("update up's score, info=%+v, affected=%d", upInfo, affectedRow)
case <-s.closeChan:
log.Info("server closing")
stop = true
}
}
// drain the channel
for upInfo := range s.businessScoreChan {
// update up player's baseinfo
affectedRow, err := s.upcrmdb.UpdateQualityAndPrScore(upInfo.PrValue, upInfo.QualityValue, upInfo.Mid)
if err != nil {
log.Error("error when update quality score, err=%s", err)
break
}
log.Info("update up's score, info=%+v, affected=%d", upInfo, affectedRow)
}
}
func (s *Service) batchWriteProc() {
defer func() {
s.wg.Done()
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("batchWriteProc Runtime error caught, try recover: %+v", r)
s.wg.Add(1)
go s.batchWriteProc()
}
}()
var ticker = time.NewTicker(10 * time.Second)
for s.running {
// 每秒钟查一次
select {
case <-ticker.C:
case <-s.closeChan:
log.Info("server closing")
}
batchMutex.Lock()
for i, list := range batchDataList {
s.batchUpdate(list, i)
batchDataList[i] = nil
}
batchMutex.Unlock()
}
// drain the list
for i, list := range batchDataList {
// batch update
s.batchUpdate(list, i)
}
}
// leastCount to flush
func (s *Service) batchUpdate(list []*upcrmdao.UpQualityInfo, tablenum int) {
var leftCount = len(list)
if leftCount == 0 {
return
}
var affected, err = s.upcrmdb.InsertBatchScoreHistory(list, tablenum)
if err != nil {
log.Error("error insert score history, err=%s", err)
return
}
log.Info("batch insert into score history, count=%d, affected=%d", leftCount, affected)
}
//Test test func
func (s *Service) Test() {
var list = []*upcrmdao.UpQualityInfo{
{Mid: 123, QualityValue: rand.Int() % 100, PrValue: rand.Int() % 100, Cdate: "2018-07-13"},
}
s.upcrmdb.InsertBatchScoreHistory(list, 23)
}

View File

@ -0,0 +1,279 @@
package service
import (
"go-common/app/service/main/upcredit/model/upcrmmodel"
"time"
"fmt"
"github.com/jinzhu/gorm"
"github.com/pkg/errors"
"github.com/siddontang/go-mysql/mysql"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/dao/upcrmdao"
"go-common/app/service/main/upcredit/model/calculator"
"go-common/library/log"
"sync"
)
//CalcService calc service
type CalcService struct {
c *conf.Config
JobChannel chan BaseJobInterface
OutputChan chan *upcrmmodel.UpScoreHistory
calc *calculator.CreditScoreCalculator
wg sync.WaitGroup
isRunning bool
mapLock sync.Mutex
jobsMap map[int]*CalcJob
jobID int
finishJobChan chan *CalcJob
crmdao *upcrmdao.Dao
}
//NewCalc create calc service
func NewCalc(c *conf.Config, outChan chan *upcrmmodel.UpScoreHistory, crmdao *upcrmdao.Dao) *CalcService {
var s = &CalcService{
c: c,
JobChannel: make(chan BaseJobInterface, 100),
calc: calculator.New(outChan),
isRunning: true,
jobsMap: make(map[int]*CalcJob),
jobID: 1,
finishJobChan: make(chan *CalcJob),
crmdao: crmdao,
}
return s
}
//BaseJobInterface job interface
type BaseJobInterface interface {
Run() (err error)
Description() string
}
//CalcJob calculate job
type CalcJob struct {
ID int
// 数据日
Date time.Time
// 对应的数据
TableNum int
Overall *calculator.OverAllStatistic
IsDone bool
Svc *CalcService
}
//Run run job
func (job *CalcJob) Run() (err error) {
err = job.Svc.calc.CalcLogTable(job.TableNum, job.Date, job.Overall)
if err != nil {
log.Error("calculate job error, job=%+v, err=%+v", job, err)
} else {
log.Info("calculate job finish, job=%+v", job)
}
job.IsDone = true
job.Svc.finishJobChan <- job
return
}
//Description descrpit job
func (job *CalcJob) Description() (dest string) {
return fmt.Sprintf("calc table=%d, date=%s", job.TableNum, job.Date.Format(mysql.TimeFormat))
}
//CalcStatisticJob statis job
type CalcStatisticJob struct {
ID int
// 数据日
Date time.Time
Svc *CalcService
}
//Run run
func (j *CalcStatisticJob) Run() (err error) {
return j.Svc.CalcScoreSectionData()
}
//Description desc
func (j *CalcStatisticJob) Description() (dest string) {
return fmt.Sprintf("calc statis, date=%s", j.Date.Format(mysql.TimeFormat))
}
//AddCalcJob add job
func (c *CalcService) AddCalcJob(date time.Time) {
var count = upcrmmodel.CreditLogTableCount
for i := 0; i < count; i++ {
var job = &CalcJob{
ID: c.getJobID(),
Date: date,
TableNum: i,
Overall: calculator.NewOverAllStatistic(1000, 10),
}
job.Svc = c
log.Info("send calculate job, job=%+v", job)
c.JobChannel <- job
c.mapLock.Lock()
c.jobsMap[job.ID] = job
c.mapLock.Unlock()
}
}
func (c *CalcService) getJobID() int {
c.jobID++
return c.jobID
}
//Run fun service
func (c *CalcService) Run() {
go c.ScheduleJob()
go c.jobFinishCheck()
// run workers
for i := 0; i < c.c.RunStatJobConf.WorkerNumber; i++ {
c.wg.Add(1)
go c.worker()
}
log.Info("run worker count=%d", c.c.RunStatJobConf.WorkerNumber)
}
//Close close service
func (c *CalcService) Close() {
c.isRunning = false
//c.wg.Wait()
}
//ScheduleJob schedule job
func (c *CalcService) ScheduleJob() {
var now = time.Now()
var startTime, err = time.Parse("15:04:05", c.c.RunStatJobConf.StartTime)
if err != nil {
log.Error("schedule job start time error, config=%s, should like '12:00:00'")
startTime = time.Date(2000, 1, 1, 3, 0, 0, 0, now.Location())
}
n := time.Date(now.Year(), now.Month(), now.Day(), startTime.Hour(), startTime.Minute(), startTime.Second(), 0, now.Location())
d := n.Sub(now)
if d < 0 {
n = n.Add(24 * time.Hour)
d = n.Sub(now)
}
for c.isRunning {
time.Sleep(d)
d = 24 * time.Hour
// 只有master来计算任务做的比较简单正常应该是master发任务follower来做任务然而简单起见先由master自己来完成任务
if !conf.IsMaster {
continue
}
c.AddCalcJob(time.Now())
}
}
func (c *CalcService) worker() {
defer func() {
c.wg.Done()
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("worker Runtime error caught, try recover: %+v", r)
c.wg.Add(1)
go c.worker()
}
}()
for job := range c.JobChannel {
var err = job.Run()
if err != nil {
log.Error("job run, err=%+v", err)
}
}
}
func (c *CalcService) jobFinishCheck() {
for range c.finishJobChan {
if c.isAllJobFinish() {
c.onAllJobFinish()
}
}
}
func (c *CalcService) onAllJobFinish() {
log.Info("all job is done, add calc statistic job")
var job = &CalcStatisticJob{
ID: c.getJobID(),
Date: time.Now().AddDate(0, 0, -1),
Svc: c,
}
c.JobChannel <- job
}
func (c *CalcService) isAllJobFinish() bool {
c.mapLock.Lock()
defer c.mapLock.Unlock()
if len(c.jobsMap) == 0 {
return true
}
for _, j := range c.jobsMap {
if !j.IsDone {
return false
}
}
return true
}
//CalcScoreSectionData calc score section
func (c *CalcService) CalcScoreSectionData() (err error) {
var crmdb, e = gorm.Open("mysql", conf.Conf.DB.Upcrm.DSN)
err = e
if e != nil {
log.Error("fail to open crm db, for score section")
return
}
log.Info("start calculate crm score section")
defer crmdb.Close()
var limit = 1000
var total = 0
var prSection = calculator.NewOverAllStatistic(1000, 10)
var qualitySection = calculator.NewOverAllStatistic(1000, 10)
var creditSection = calculator.NewOverAllStatistic(1000, 10)
var lastID = int32(0)
for {
var upInfos []upcrmmodel.UpBaseInfo
// 1. 从数据库中取limit条数据
err = crmdb.Table("up_base_info").Select("id, credit_score, pr_score, quality_score").Where("id>?", lastID).Limit(limit).Find(&upInfos).Error
if err != nil {
log.Error("get data err, e=%+v", e)
break
}
// 2. 计算各分数的分数段
for _, u := range upInfos {
prSection.AddScore(u.PrScore, -1)
qualitySection.AddScore(u.QualityScore, -1)
creditSection.AddScore(u.CreditScore, 500)
lastID = u.ID
}
var thisCount = len(upInfos)
total += thisCount
if thisCount < limit {
log.Info("table[up_base_info] total read record, num=%d", total)
break
}
}
// 输出到score表中
var now = time.Now()
err = c.crmdao.InsertScoreSection(*prSection, upcrmmodel.ScoreTypePr, now)
if err != nil {
log.Error("fail update pr score section, err=%+v", err)
}
err = c.crmdao.InsertScoreSection(*qualitySection, upcrmmodel.ScoreTypeQuality, now)
if err != nil {
log.Error("fail update quality score section, err=%+v", err)
}
err = c.crmdao.InsertScoreSection(*creditSection, upcrmmodel.ScoreTypeCredit, now)
if err != nil {
log.Error("fail update credit score section, err=%+v", err)
}
log.Info("finish calculate crm score section, totalcount=%d", total)
return
}

View File

@ -0,0 +1,57 @@
package service
import (
"go-common/library/ecode"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/render"
"net/http"
)
// BmHTTPErrorWithMsg custom error msg
func BmHTTPErrorWithMsg(c *bm.Context, err error, msg string) {
if c.IsAborted() {
return
}
c.Error = err
bcode := ecode.Cause(err)
if msg == "" {
msg = err.Error()
}
c.Render(http.StatusOK, render.JSON{
Code: bcode.Code(),
Message: msg,
Data: nil,
})
}
//BmGetStringOrDefault get string
func BmGetStringOrDefault(c *bm.Context, key string, defaul string) (value string, exist bool) {
i, exist := c.Get(key)
if !exist {
value = defaul
return
}
value, exist = i.(string)
if !exist {
value = defaul
}
return
}
//BmGetInt64OrDefault get int64
func BmGetInt64OrDefault(c *bm.Context, key string, defaul int64) (value int64, exist bool) {
i, exist := c.Get(key)
if !exist {
value = defaul
return
}
value, exist = i.(int64)
if !exist {
value = defaul
}
return
}

View File

@ -0,0 +1,99 @@
package service
import (
"context"
"go-common/app/service/main/upcredit/common/fsm"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/mathutil"
"go-common/app/service/main/upcredit/model/calculator"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
"time"
)
//GetCreditScore get credit score
func (c *Service) GetCreditScore(con context.Context, arg *upcrmmodel.GetScoreParam) (results []*upcrmmodel.UpScoreHistory, err error) {
results, err = c.upcrmdb.GetCreditScore(con, arg)
if err != nil {
log.Error("error when get credit score, arg=%+v, err=%+v", arg, err)
} else {
log.Info("get credit score, arg=%+v, lens=%d", arg, len(results))
}
return
}
func getOrCreateArticleFSM(aid int, m map[int]*calculator.ArticleStateMachine) (artFsm *calculator.ArticleStateMachine) {
artFsm, ok := m[aid]
if !ok {
artFsm = calculator.CreateArticleStateMachineWithInitState()
m[aid] = artFsm
}
return
}
//GetCreditLog get log
func (c *Service) GetCreditLog(con context.Context, arg *upcrmmodel.ArgGetLogHistory) (results []*upcrmmodel.SimpleCreditLogWithContent, err error) {
var now = time.Now()
if arg.FromDate.IsZero() {
arg.FromDate = now.AddDate(-1, 0, 0)
}
if arg.ToDate.IsZero() {
arg.ToDate = now
}
if arg.Limit <= 0 {
arg.Limit = 20
}
results, err = c.upcrmdb.GetCreditLog(con, arg)
if err != nil {
log.Error("error when get log, arg=%+v, err=%+v", arg, err)
return
}
log.Info("get original log, arg=%+v, lens=%d", arg, len(results))
var logListTimeAsc []*upcrmmodel.SimpleCreditLogWithContent
var machineMap = make(map[int]*calculator.ArticleStateMachine)
for _, l := range results {
switch l.BusinessType {
case upcrmmodel.BusinessTypeArticleAudit:
afsm := getOrCreateArticleFSM(int(l.Oid), machineMap)
if afsm == nil {
log.Error("fail to get article's state machine, check reason, id=%d", l.Oid)
continue
}
afsm.OnLog(&l.SimpleCreditLog, func(e *fsm.Event, l1 *upcrmmodel.SimpleCreditLog, a *calculator.ArticleStateMachine) {
var needLog = false
switch e.Dst {
case calculator.StateClose:
if conf.CreditConfig.ArticleRule.IsRejected(l.Type, l.OpType, l.Reason) {
needLog = true
}
case calculator.StateOpen:
needLog = true
if l.Content == "" {
l.Content = "稿件开放浏览"
}
}
if needLog {
logListTimeAsc = append(logListTimeAsc, l)
}
})
default:
logListTimeAsc = append(logListTimeAsc, l)
}
}
log.Info("get refined log, arg=%+v, lens=%d", arg, len(logListTimeAsc))
// 转化成按时间降序的
for i, j := 0, len(logListTimeAsc)-1; i < j; i, j = i+1, j-1 {
logListTimeAsc[i], logListTimeAsc[j] = logListTimeAsc[j], logListTimeAsc[i]
}
var max = mathutil.Min(len(logListTimeAsc), arg.Limit)
results = logListTimeAsc[0:max]
return
}

View File

@ -0,0 +1,73 @@
package service
import (
"context"
"encoding/json"
"go-common/app/service/main/upcredit/model/canal"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
"strings"
)
const (
tableUpQualityInfo = "up_quality_info_"
)
func (s *Service) arcCreditLogConsume() {
defer func() {
s.wg.Done()
log.Error("arcCreditLogConsume stop!")
}()
var (
msgs = s.creditLogSub.Messages()
err error
)
log.Info("arcCreditLogConsume start")
for {
msg, ok := <-msgs
if !ok {
log.Error("s.arcSub.Messages closed")
return
}
msg.Commit()
//s.arcMo++
m := &upcrmmodel.ArgCreditLogAdd{}
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
continue
}
var c = context.TODO()
s.LogCredit(c, m)
log.Info("arcCreditLogConsume key(%s) value(%s) partition(%d) offset(%d) commit, mid=%d, bustype=%d, optype=%d", msg.Key, msg.Value, msg.Partition, msg.Offset, m.Mid, m.BusinessType, m.OpType)
}
}
func (s *Service) arcBusinessBinLogCanalConsume() {
defer func() {
s.wg.Done()
log.Error("business CanalConsume stop!")
}()
var (
msgs = s.businessBinLogSub.Messages()
err error
)
log.Info("business CanalConsume start")
for {
msg, ok := <-msgs
if !ok {
log.Error("s.businessBinLogSub.Messages closed")
return
}
m := &canal.Msg{}
if err = json.Unmarshal(msg.Value, m); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
msg.Commit()
continue
}
if strings.HasPrefix(m.Table, tableUpQualityInfo) {
s.onBusinessDatabus(m)
}
msg.Commit()
log.Info("businessBinLog consume key(%s) value(%s) partition(%d) offset(%d) commit", msg.Key, string(msg.Value), msg.Partition, msg.Offset)
}
}

View File

@ -0,0 +1,47 @@
package service
import (
"context"
"github.com/pkg/errors"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
)
//LogCredit log credit log
func (s *Service) LogCredit(c context.Context, arg *upcrmmodel.ArgCreditLogAdd) (err error) {
err = s.upcrmdb.AddLog(arg)
if err != nil {
log.Error("fail to log credit log, log=%+v, err=%v", arg, err)
return
}
log.Info("log credit log to db, {%+v}", arg)
return
}
//WriteStatData write log to db
func (s *Service) WriteStatData() {
defer func() {
s.wg.Done()
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("write stat data Runtime error caught, try recover: %+v", r)
s.wg.Add(1)
go s.WriteStatData()
}
}()
for s.running {
select {
case creditScore := <-s.CreditScoreInputChan:
var err = s.upcrmdb.AddOrUpdateCreditScore(creditScore)
if err != nil {
log.Error("fail to insert credit score, mid=%d", creditScore.Mid)
continue
}
var affectedRow int64
affectedRow, err = s.upcrmdb.UpdateCreditScore(creditScore.Score, creditScore.Mid)
log.Info("insert credit score, mid=%d, update base info, affected=%d, err=%+v", creditScore.Mid, affectedRow, err)
case <-s.closeChan:
log.Info("server closing, close routine")
}
}
}

View File

@ -0,0 +1,113 @@
package service
import (
"go-common/app/service/main/upcredit/common/election"
"go-common/app/service/main/upcredit/conf"
"go-common/app/service/main/upcredit/dao/upcrmdao"
"go-common/app/service/main/upcredit/mathutil"
"go-common/app/service/main/upcredit/model/upcrmmodel"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/queue/databus"
"sync"
"time"
)
// Service is service.
type Service struct {
c *conf.Config
httpClient *bm.Client
upcrmdb *upcrmdao.Dao
creditLogSub *databus.Databus
businessBinLogSub *databus.Databus
wg sync.WaitGroup
CreditScoreInputChan chan *upcrmmodel.UpScoreHistory
CalcSvc *CalcService
limit *mathutil.Limiter
running bool
closeChan chan struct{}
businessScoreChan chan *upcrmdao.UpQualityInfo
zkElection *election.ZkElection
}
// New is go-common/app/service/videoup service implementation.
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
httpClient: bm.NewClient(c.HTTPClient.Normal),
upcrmdb: upcrmdao.New(c),
creditLogSub: databus.New(c.CreditLogSub),
businessBinLogSub: databus.New(c.BusinessBinLogSub),
CreditScoreInputChan: make(chan *upcrmmodel.UpScoreHistory, 10240),
running: true,
closeChan: make(chan struct{}),
businessScoreChan: make(chan *upcrmdao.UpQualityInfo, 200),
}
if conf.Conf.ElectionZooKeeper != nil {
var zc = conf.Conf.ElectionZooKeeper
s.zkElection = election.New(zc.Addrs, zc.Root, time.Duration(zc.Timeout))
var err = s.zkElection.Init()
if err != nil {
log.Error("zk elect init fail, err=%s", err)
} else {
err = s.zkElection.Elect()
if err != nil {
log.Error("zk elect fail, err=%s", err)
} else {
go func() {
for {
conf.IsMaster = <-s.zkElection.C
if conf.IsMaster {
log.Info("this is master, node=%s", s.zkElection.NodePath)
} else {
log.Info("this is follower, node=%s, master=%s", s.zkElection.NodePath, s.zkElection.MasterPath)
}
}
}()
}
}
}
s.CalcSvc = NewCalc(c, s.CreditScoreInputChan, s.upcrmdb)
if c.MiscConf.BusinessBinLogLimitRate <= 0 {
c.MiscConf.BusinessBinLogLimitRate = 300
}
s.limit = mathutil.NewLimiter(c.MiscConf.BusinessBinLogLimitRate)
s.CalcSvc.Run()
// credit log databus
{
s.wg.Add(2)
go s.arcCreditLogConsume()
go s.arcBusinessBinLogCanalConsume()
}
for i := 0; i < c.MiscConf.CreditLogWriteRoutineNum; i++ {
s.wg.Add(1)
go s.WriteStatData()
}
s.wg.Add(1)
go s.updateScoreProc()
s.wg.Add(1)
go s.batchWriteProc()
return s
}
//Close service close
func (s *Service) Close() {
s.creditLogSub.Close()
s.businessBinLogSub.Close()
s.CalcSvc.Close()
s.running = false
close(s.closeChan)
s.wg.Wait()
s.upcrmdb.Close()
}
//
//func (s *Service) ArchiveAuditResult(c *context.Context, ap *archive.ArcParam) {
// // bussiness type = 1 是稿件
// // 检查ap的state是不是需要记录的state 对应到 type 中
// // 检查对应的reason id 对应到具体的optype 分类中
// // 检查对应的
//}

View File

@ -0,0 +1,26 @@
package service
import (
"flag"
_ "github.com/davecgh/go-spew/spew"
"go-common/app/service/main/upcredit/conf"
"path/filepath"
"testing"
"time"
)
var (
s *Service
)
func init() {
dir, _ := filepath.Abs("../cmd/upcredit-service.toml")
flag.Set("conf", dir)
conf.Init()
s = New(conf.Conf)
time.Sleep(time.Second)
}
func Test_service(t *testing.T) {
}