openbilibili/go-common
1
0
Fork 0
Code Releases Activity

Create & Init Project...

Browse Source
This commit is contained in:
MiaoWoo
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
app/job/main/member/BUILD Normal file
View File

@@ -0,0 +1,21 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/member/cmd:all-srcs",
"//app/job/main/member/conf:all-srcs",
"//app/job/main/member/dao:all-srcs",
"//app/job/main/member/http:all-srcs",
"//app/job/main/member/model:all-srcs",
"//app/job/main/member/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

408
app/job/main/member/CHANGELOG.md Normal file
View File

@@ -0,0 +1,408 @@
member项目的Job
# Version 2.21.3
> 1. 删除部分无用代码
# Version 2.21.0
> 1. 删除所有 user_detail_ 依赖
# Version 2.20.1
> 1. 删除 bfsDatabus配置
# Version 2.20.0
> 1. 下线 user_verify 表
> 2. 下线 member_verify 表
# Version 2.19.4
> 1. 实名认证超时自动驳回
> 2. UpdateRealnameFromMSG方法改为一个事务内操作realnameInfo和realname_apply表
> 3. 使用 Infov 上报解析身份证后的部分公开数据，修正时间格式
# Version 2.19.1
> 1. 修正经验缓存通知时序
# Version 2.19.0
> 1. 修正视频分享经验
# Version 2.18.4
> 1. 限制单节点的经验数据库写入量
# Version 2.18.3
> 1. 使用字符串来传递登录日志中的 IP 字段
# Version 2.18.2
> 1. 补发芝麻实名认证经验
# Version 2.17.0
> 1. 增加芝麻实名认证
# Version 2.16.2
> 1. fix conf redefined
# Version 2.16.1
> 1. fix remove block source check
# Version 2.16.0
> 1. realname 新表realname_info写入
> 2. realname card_md5 更新算法
# Version 2.15.0
> 1. 所有 base 修改延迟 5 秒再删一次缓存
# Version 2.14.0
> 1. 合并block
# Version 2.13.6
> 1. 经验insert binlog 删除缓存
# Version 2.13.5
> 1. 在实名主库写入完成后增加clean cache 通知
> 2. 优化account notify
# Version 2.13.4
> 1. 删除 hbase
> 2. 删除 hbase client
# Version 2.13.2
> 1. 修复实名删除缓存策略，防止主从延迟导致脏缓存
# Version 2.13.1
> 1.add notify sender mark.
# Version 2.13.0
> 1. 增加实名删缓存兜底策略，保证最终一致
# Version 2.12.3
> 1. 增加节操变化通知业务方清理缓存
# Version 2.12.2
> 1. 修复经验等级获取
# Version 2.12.1
> 1. 增加等级变化清缓存通知
# Version 2.10.0
> 1. 观看视频补偿登陆加节操
# Version 2.9.5
> 1. 观看视频补偿登陆加节操
# Version 2.9.4
> 1. 更改realname cache key
# Version 2.9.3
> 1. 头像检查增加动态开关
# Version 2.9.2
> 1. 节操日志带 log_id
# Version 2.9.1
> 1. AI 审核头像带备注
# Version 2.9.0
> 1. 经验日志写 report
# Version 2.8.2
> 1. realname 优化
# Version 2.8.1
> 1. 修复名字全量同步bug
# Version 2.8.0
> 1. 删除 头像，签名从老库同步的逻辑
> 2. 新库头像回写老库
# Version 2.7.1
> 1. 头像自动审核添加operator
# Version 2.7.0
> 1. 去除 reload 配置文件
# Version 2.6.0
> 1. 增加头像审核相关逻辑
# Version 2.5.4
> 1. 节操恢复逻辑：由每天登录恢复一点改为恢复从上次恢复到当前时间累加
# Version 2.5.3
> 1. change recoverMoral status.
# Version 2.5.2
> 1. fix moral recover
# Version 2.5.1
> 1. 去掉节操和生日同步等代码
> 2. 增加每日登录恢复节操逻辑
# Version 2.5.0
> 1. migrate to bm
# Version 2.4.3
> 1. reissue login exp
# Version 2.4.2
> 1. del rank sync
# Version 2.4.1
> 1. fix exp init
# Version 2.4.0
> 1. fix exp init
# Version 2.3.19
> 1. fix exp init
# Version 2.3.18
> 1. del sex sync
# Version 2.3.17
> 1. 老 verify 同步到新 official 表
# Version 2.3.16
> 1. job 里不再写 base 缓存
# Version 2.3.15
> 1. 去掉老mc清理
# Version 2.3.14
> 1. 修复经验日志被不同 operator 覆盖的问题
# Version 2.3.13
> 1. 修复实名认证card_data base64解码
# Version 2.3.12
> 1. 修复实名认证card_type & country
# Version 2.3.11
> 1. 修复实名认证dao
# Version 2.3.10
> 1. 修复实名认证同步时间parse
# Version 2.3.9
> 1. 实名认证订阅老表+同步老库数据
# Version 2.3.8
> 1. 将数据库查不到，返回结果改为nil
# Version 2.3.7
> 1. 严格节操一致检查
# Version 2.3.6
> 1. 节操全量同步，如果老的里面有值新的没有就把值初始化为7000
# Version 2.3.5
> 1. 删除缓存集群由mc改为mcTmp
# Version 2.3.4
> 1. 增加删除节操缓存
> 2. 去掉detail 变化通知
# Version 2.3.3
> 1. 增加临时缓存集群清理
# Version 2.3.2
> 1. add binlog&fixer birthday sync to base.
# Version 2.3.1
> 1. super clean code.
> 2. remove add&set exp check&init logic.
> 3. configureable exp proc count.
# Version 2.3.0
>1. fix moral log.
# Version 2.2.9
>1. purge cache on base update.
# Version 2.2.8
> 1. add incr sync moral and log.
> 2. add exp databus log.
# Version 2.2.7
> 1. sync moral.
> 2. enlarge accproc goroutine.
# Version 2.2.6
> 1. support disposable exp.
> 2. change unmarshal error continue.
# Version 2.2.5
> 1. fix passportSubproc.
# Version 2.2.4
> 1. add exp init logic.
> 2. chenge log format.
> 3. perfect code.
# Version 2.2.3
> 1. fix face url
> 2. nil base protect
> 3. skip on zero mid
> 4. login log message parse with two struct
> 5. base info with cert
> 6. notify purge cache and logging
> 7. restrict exp change and readable message
> 8. unify exp log field
# Version 2.2.2
> 1. add goroutine to consume log
# Version 2.2.1
> 1. fix sync name
> 2. add sync range
> 3. add debug log
> 4. fix sql in
# Version 2.2.0
> 1. exp reconstruct
> 2. acc proc count
> 3. skip exp log sync
# Version 2.1.2
> 1. clarify spacesta and silence
# Version 2.1.1
> 1. fix user_addit info
# Version 2.1.0
> 1. fix sync aso source
# Version 2.0.9
> 1. add sync aso source
# Version 2.0.8
> 1. add feature gates
# Version 2.0.7
> 1. check data in job leader
> 2. remove statsd
# Version 2.0.6
> 1. add check exp code.
> 2. change member info sync.
# Version 2.0.5
> 1. add change exp log
# Version 2.0.4
> 1. rm change sleep
# Version 2.0.3
> 1. change sleep time
# Version 2.0.2
> 1. add time sleep
# Version 2.0.1
> 1. add log
# Version 2.0.0
> 1. check exp db to accdb
# Version 1.9.5
> 1.unify job model
> 2.add ctime/mtime field for base and detail
# Version 1.9.4
> 1.simplify code
# Version 1.9.3
> 1.fix len(Name)>0
# Version 1.9.2
> 1.fix map init
# Version 1.9.1
> 1.fix map key int64
# Version 1.9.0
> 1.fix map key type
# Version 1.8.9
> 1.fix un-init map
# Version 1.8.8
> 1.simplify code
# Version 1.8.7
> 1.fix infinity loop
# Version 1.8.6
> 1.fix atomic load int64
# Version 1.8.5
> 1.fix slice init
# Version 1.8.4
> 1.add batch get from db
# Version 1.8.3
> 1.add wocao juran recover le
# Version 1.8.2
> 1.adjust fixer strategy
# Version 1.8.1
> 1.add fixer detail field check
# Version 1.8.0
> 1.randomize checking mids
# Version 1.7.9
> 1.if cannot load baseinfo or detailinfo , rewrite it
# Version 1.7.8
> 1.ignore db timeout
# Version 1.7.7
> 1.added dat fixer
# Version 1.7.6
> 1.added missing mid
# Version 1.7.5
> 1.passthrough sign checking
# Version 1.7.3
> 1.check all data
# Version 1.7.2
> 1.passthrough sex face rank checking
# Version 1.7.1
> check all data
# Version 1.7.0
> 1.old data detail init
# Version 1.6.0
> 1.add detail init
# Version 1.5.0
> 1.修改exp 为mc
# Version 1.4.0
> 1.添加award databus
# Version 1.3.0
> 1.增加业务方缓存清理
# Version 1.2.1
> 1.添加登陆经验开关
# Version 1.2.0
> 1.经验重构
# Version 1.1.0
> 1.经验数据迁移
# Version 1.0.3
> 1.批量数据校验
# Version 1.0.0
> 1.版本初始化
2.增量数据同步
3.缓存清理

11
app/job/main/member/CONTRIBUTORS.md Normal file
View File

@@ -0,0 +1,11 @@
# Owner
chenjianrong
zhoujiahui
linmiao
zhoujixiang
# Author
chenjianrong
# Reviewer
chenjianrong

15
app/job/main/member/OWNERS Normal file
View File

@@ -0,0 +1,15 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- chenjianrong
- linmiao
- zhoujiahui
- zhoujixiang
labels:
- job
- job/main/member
- main
options:
no_parent_owners: true
reviewers:
- chenjianrong

11
app/job/main/member/README.md Normal file
View File

@@ -0,0 +1,11 @@
#### member job
##### 项目简介
> 1.订阅databus消息,异步操作
##### 编译环境
> 请只用golang v1.8.x以上版本编译执行。
##### 依赖包
> 1.公共包go-common

42
app/job/main/member/cmd/BUILD Normal file
View File

@@ -0,0 +1,42 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["member-job-dev.toml"],
importpath = "go-common/app/job/main/member/cmd",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/http:go_default_library",
"//app/job/main/member/service:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus/report:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

146
app/job/main/member/cmd/databus.py Normal file
View File

@@ -0,0 +1,146 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date : 2017-12-25 下午2:43
# @Author : Hedan (hedan@bilibili.com)
# @file : databusTest
import json
import redis
# [databus]
# key = "4ba46ba31f9a44ef"
# secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
# group = "AccountLog-MainAccount-P"
# topic = "AccountLog-T"
# action = "pub"
# name = "member-service/databus"
# proto = "tcp"
# addr = "172.16.33.158:6205"
# idle = 100
# active = 100
# dialTimeout = "1s"
# readTimeout = "60s"
# writeTimeout = "1s"
# idleTimeout = "10s"
"""参考文档 http://info.bilibili.co/pages/viewpage.action?pageId=3670491
key 服务配置中databus的key
value 服务配置中databus的value
host和port 配置服务中databus的host和port
group-topic 一个group对应一个topic(开发申请)
"""
#每日登录 passport 获得5经验值
# data_passport ={
# 'mid':4780461,
# 'loginip':1726481463,
# "timestamp":1516517576,
# }
# auth_passport = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@PassportLog-MainAccount-P/topic=PassportLog-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_passport)
# msg = json.dumps(data_passport, ensure_ascii=False)
# rc.set("1023", msg)
# 每日登录 account 获得5经验值
# data_passport ={
# 'mid':110000092,
# 'loginip':1726481463,
# "timestamp":1516517576,
# }
# auth_passport = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@AccountLoginAward-MainAccount-P/topic=AccountLoginAward-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_passport)
# msg = json.dumps(data_passport, ensure_ascii=False)
# rc.set("1023", msg)
# 每日首次分享视频 archive 获得5经验值
# data_archive ={
# 'event':"share",
# 'mid':110000092,
# 'ip':"127.0.0.1",
# "ts":111,
# }
# auth_archive = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@AccountExp-MainAccount-P/topic=AccountExp-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_archive)
# msg = json.dumps(data_archive, ensure_ascii=False)
# rc.set("1023", msg)
#
#
# 每日首次分享视频 archive 获得5经验值
data_archive ={
'event':"view",
# 'mid':110000092,
'mid':4780461,
'ip':"127.0.0.1",
"ts":1521745000,
}
auth_archive = '4ba46ba31f9a44ef:99985eb4451cfb1b899ca0fbe3c4bdc8@AccountExp-MainAccount-P/topic=AccountExp-T&role=pub&offset=new'
rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
print rc.execute_command('auth',auth_archive)
msg = json.dumps(data_archive, ensure_ascii=False)
rc.set("1023", msg)
# 每日首次观看 history 获得5经验值
# data_history ={
# "action":"insert",
# "table":"aso_account",
# "data":{"mid":110000193,
# "userid":"test_dan06",
# "uname":"test_dan08",
# "pwd":"6cfbf96b8f0eb2e0a82b46a4236e8883",
# "salt":"D8fd30Kj",
# "email":"169d9106a74d5e95de71be6cf373af04",
# "tel":"218cb4bf8762354eae473b3b612f707e",
# "country_id":1,
# "mobile_verified":0,
# "isleak":0,
# "mtime":"2018-01-02 17:18:58"},
# "flag":0
# }
# auth_history = '0QEO9F8JuuIxZzNDvklH:0QEO9F8JuuIxZzNDvklI@PassportGameTrans-ENCRYPT-P/topic=PassportGameTrans-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_history)
# msg = json.dumps(data_history, ensure_ascii=False)
# rc.set("110000193", msg)
# data_history ={
# "action":"insert",
# "table":"aso_account",
# "data":{"mid":110000194,
# "userid":"test_dan07",
# "uname":"test_dan07",
# "pwd":"d2c9d4acdfe9942979d7b4d3e711d499",
# "salt":"5Brw3JuP",
# "email":"169d9106a74d5e95de71be6cf373af04",
# "tel":"b65919197178b5db6ed5f85a229dfaf9",
# "country_id":1,
# "mobile_verified":0,
# "isleak":0,
# "mtime":"2018-01-02 17:18:58"},
# "flag":0
# }
# auth_history = '0QEO9F8JuuIxZzNDvklH:0QEO9F8JuuIxZzNDvklI@PassportGameTrans-ENCRYPT-P/topic=PassportGameTrans-T&role=pub&offset=new'
# rc = redis.Redis(host='172.16.33.158', port=6205, socket_keepalive=True)
# print rc.execute_command('auth',auth_history)
# msg = json.dumps(data_history, ensure_ascii=False)
# rc.set("110000193", msg)

56
app/job/main/member/cmd/main.go Normal file
View File

@@ -0,0 +1,56 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/http"
"go-common/app/job/main/member/service"
"go-common/library/log"
"go-common/library/queue/databus/report"
)
var (
svr *service.Service
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
log.Error("conf.Init() error(%v)", err)
panic(err)
}
log.Init(conf.Conf.Xlog)
defer log.Close()
report.InitUser(conf.Conf.UserReport)
report.InitManager(conf.Conf.ManagerReport)
log.Info("member-job start")
svr = service.New(conf.Conf)
http.Init(conf.Conf, svr)
signalHandler()
}
func signalHandler() {
var (
err error
ch = make(chan os.Signal, 1)
)
signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
si := <-ch
switch si {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
log.Info("get a signal %s, stop the consume process", si.String())
if err = svr.Close(); err != nil {
log.Error("srv close consumer error(%v)", err)
}
return
case syscall.SIGHUP:
default:
return
}
}
}

430
app/job/main/member/cmd/member-job-dev.toml Normal file
View File

@@ -0,0 +1,430 @@
#This is a TOML document. Boom.
version = "1.0.0"
user = "nobody"
pid = "/tmp/member-job.pid"
dir = "./"
perf = "0.0.0.0:6330"
family = "member-job"
address = "0.0.0.0"
[log]
dir = "/data/log/member-job"
stdout=true
[tracer]
proto = "udp"
addr = "172.16.33.46:5140"
tag = "platform/member-job"
[multiHTTP]
[multiHTTP.inner]
addrs = ["0.0.0.0:6688"]
maxListen = 10
[app]
key = "4699a07e59d7149e"
secret = "test"
[path]
accountInfo="http://uat-account.bilibili.co/api/internal/data/info"
[httpClient]
dial = "1s"
timeout = "2s"
keepAlive = "60s"
timer = 128
key = "4699a07e59d7149e"
secret = "test"
[httpClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[memcache]
name = "member"
proto = "tcp"
# addr = "172.16.33.54:11211"
addr = "172.18.33.60:11233"
idle = 5
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "24h"
[mysql]
addr = "172.16.33.205"
dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_member?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 5
idle = 2
IdleTimeout ="4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[mysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[accCheckMysql]
addr = "172.16.33.205:3306"
dsn = "account:wx2U1MwXRyWEuURw@tcp(172.16.33.205:3306)/account?timeout=50ms&readTimeout=300ms&writeTimeout=300ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 50
idle = 5
idleTimeout = "4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[accCheckMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[accMysql]
addr = "172.16.33.205:3306"
dsn = "account:wx2U1MwXRyWEuURw@tcp(172.16.33.205:3306)/account?timeout=50ms&readTimeout=300ms&writeTimeout=300ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 50
idle = 5
idleTimeout = "4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[accMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[asoMysql]
addr = "172.16.33.205:3306"
dsn = "aso:hA0DAnENNFz78kYB@tcp(172.16.33.205:3306)/aso?timeout=50ms&readTimeout=300ms&writeTimeout=300ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 50
idle = 5
idleTimeout = "4h"
queryTimeout = "100ms"
execTimeout = "100ms"
tranTimeout = "200ms"
[asoMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[PasslogMysql]
addr = "172.16.33.205"
dsn = "account:wx2U1MwXRyWEuURw@tcp(172.16.33.205:3306)/account?timeout=500ms&readTimeout=500ms&writeTimeout=500ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 20
idle = 10
idleTimeout ="4h"
queryTimeout = "1000ms"
execTimeout = "500ms"
tranTimeout = "2000ms"
[PasslogMysql.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[PLogDatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountLog-MainAccount-S"
topic = "AccountLog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[logdatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountLog-MainAccount-S"
topic = "AccountLog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[expdatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountExp-MainAccount-S"
topic = "AccountExp-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[logindatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "PassportLog-MainAccount-S"
topic = "PassportLog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[awarddatabus]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountLoginAward-MainAccount-S"
topic = "AccountLoginAward-T"
action = "sub"
offset = "new"
buffer = 2048
name = "coin-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[accountnotify]
key = "4ba46ba31f9a44ef"
secret = "99985eb4451cfb1b899ca0fbe3c4bdc8"
group = "AccountNotify-Notify-P"
topic = "AccountNotify-T"
action = "pub"
buffer = 2048
name = "account-notify-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
# addr = "172.18.33.61:6889"
# addr = "172.16.33.56:6205"
idle = 1
active = 1
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[sharemiddatabus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "ShareMid-MainAccount-S"
topic = "ShareMid-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[databus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "MemberBinlog-MainAccount-S"
topic = "MemberBinlog-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
# addr = "172.16.33.158:6205"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[passortDataBus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "PassportPurgeCache-MainAccount-S"
topic = "PassportPurgeCache-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[accDataBus]
key = "4ba46ba31f9a44ef"
secret = "e4c5a7fce28695209e6b4f0af8cf91c5"
group = "UserActive-MainAccount-S"
topic = "UserActive-T"
action = "sub"
offset = "old"
buffer = 2048
name = "member-job/databus"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
[databusutil]
size = 100
chan = 1024
num = 4
ticker="30s"
[redis]
name = "member-service"
proto = "tcp"
# addr = "172.16.33.54:6379"
addr = "172.18.33.60:6893"
idle = 100
active = 100
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "720h"
ansCountExpire = "12h"
ansAddFlagCountExpire = "1m"
[report]
taskID = "000293"
proto = "unixgram"
addr = "/var/run/lancer/collector.sock"
chan = 1024
timeout = "200ms"
[blockdb]
# addr = "172.22.34.101:3306"
# dsn = "block:fGCs8LbV6c0nSXoyA9kdKwOpIiHa5hBq@tcp(172.22.34.101:3306)/bilibili_block?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
addr = "127.0.0.1:3306"
dsn = "test:test@tcp(127.0.0.1:3306)/test?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 20
idle = 10
idleTimeout ="4h"
queryTimeout = "1s"
execTimeout = "1s"
tranTimeout = "2s"
[db.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[blockMemcache]
name = "block-service"
proto = "unix"
addr = "/tmp/uat-block-service-mc.sock"
idle = 5
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
[blockCreditDatabus]
key = "875d05ffa843e297"
secret= "1b58f18382f0e3662f64f8457c05c049"
group= "Credit-MainAccountLaw-S"
topic= "Credit-T"
action="sub"
name = "block-job/databus"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 2
active = 10
dialTimeout = "1s"
readTimeout = "60s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1h"
[blockProperty]
limitExpireCheckLimit = 100
limitExpireCheckTick = "5m"
creditExpireCheckLimit = 100
creditExpireCheckTick = "5m"
msgURL = "http://message.bilibili.co/api/notify/send.user.notify.do"
[property.flag]
# 自动解禁检查
expireCheck = true
# 小黑屋答题状态订阅
creditSub = true
# 同步老库db
[property.msg]
[property.msg.blockRemove]
code = "2_3_6"
title = "账号封禁解除通知"
content = "你的账号已经解除封禁，封禁期间禁止使用的各项社区功能已经恢复。请遵守社区规范，共同维护良好的社区氛围。"
[biz]
realnameAlipayCheckTick = "5m"
realnameAlipayCheckLimit = 100
realnameAlipayAppID = "https://openapi.alipay.com/gateway.do"
realnameAlipayGateway = "2018062060353784"
[ParsedRealnameAgent]
TaskID="001826"
Proto="tcp"
Addr="dataflow.bilibili.co:5401"

41
app/job/main/member/conf/BUILD Normal file
View File

@@ -0,0 +1,41 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/main/member/conf",
tags = ["automanaged"],
deps = [
"//app/job/main/member/model/block:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/conf:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/log/infoc:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/queue/databus:go_default_library",
"//library/queue/databus/databusutil:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

220
app/job/main/member/conf/conf.go Normal file
View File

@@ -0,0 +1,220 @@
package conf
import (
"errors"
"flag"
"go-common/app/job/main/member/model/block"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/conf"
"go-common/library/database/sql"
"go-common/library/log"
"go-common/library/log/infoc"
bm "go-common/library/net/http/blademaster"
"go-common/library/queue/databus"
"go-common/library/queue/databus/databusutil"
"go-common/library/time"
"github.com/BurntSushi/toml"
)
// Conf global variable.
var (
Conf = &Config{}
client *conf.Client
confPath string
)
// Config struct of conf.
type Config struct {
// base
// app
App *bm.App
Biz *BizConfig
// Env
Env string
// goroutine sleep
Tick time.Duration
// log
Xlog *log.Config
// databus
DataBus *databus.Config
// databus
AccDataBus *databus.Config
// passport
PassortDataBus *databus.Config
// operate log.
LogDatabus *databus.Config
// operate log for publish
PLogDatabus *databus.Config
// add exp,pub by archive,history.
ExpDatabus *databus.Config
// login award ,pub by passport.
LoginDatabus *databus.Config
AwardDatabus *databus.Config
RealnameDatabus *databus.Config
// account notify to purge cache
AccountNotify *databus.Config
ShareMidDatabus *databus.Config
// mc
Memcache *Memcache
// httpClinet
HTTPClient *bm.ClientConfig
Mysql *sql.Config
AccCheckMysql *sql.Config
AccMysql *sql.Config
AsoMysql *sql.Config
PasslogMysql *sql.Config
// hbase
// HBase *conf.HBase
// redis
Redis *redis.Config
Databusutil *databusutil.Config
On bool
FeatureGates *FeatureGates
SyncRange *SyncRange
// bm
BM *bm.ServerConfig
// Report
UserReport *databus.Config
ManagerReport *databus.Config
//block config.
BlockMemcache *memcache.Config
BlockDB *sql.Config
BlockCreditDatabus *databus.Config
BlockProperty *Property
//realname
RealnameRsaPriv []byte
RealnameAlipayPub []byte
RealnameAlipayBiliPriv []byte
// Parsed Realname Infoc
ParsedRealnameInfoc *infoc.Config
}
// Property .
type Property struct {
LimitExpireCheckLimit int
LimitExpireCheckTick time.Duration
CreditExpireCheckLimit int
CreditExpireCheckTick time.Duration
MSGURL string
MSG *MSG
Flag *struct {
ExpireCheck bool
CreditSub bool
}
}
// MSG .
type MSG struct {
BlockRemove block.MSG
}
// Memcache memcache.
type Memcache struct {
*memcache.Config
Expire time.Duration
}
// SyncRange syncRange.
type SyncRange struct {
Start int64
End int64
}
// BizConfig biz common config
type BizConfig struct {
ExpiredBase int32
ExpiredDetail int32
IsFree bool
AccprocCount int32
ExpprocCount int32
// realname alipay
RealnameAlipayCheckTick time.Duration
RealnameAlipayCheckLimit int
RealnameAlipayAppID string
RealnameAlipayGateway string
}
// FeatureGates is.
type FeatureGates struct {
DataFixer bool
FaceCheck bool
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init create config instance.
func Init() (err error) {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
// go func() {
// for range client.Event() {
// log.Info("config reload")
// if load() != nil {
// log.Error("config reload error (%v)", err)
// }
// }
// }()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
tmpConf.RealnameRsaPriv, tmpConf.RealnameAlipayPub, tmpConf.RealnameAlipayBiliPriv = loadRealnameKey()
*Conf = *tmpConf
return
}
func loadRealnameKey() (rasPriv, alipayPub, alipayBiliPriv []byte) {
var (
emptyBytes = []byte("")
)
rasPriv, alipayPub, alipayBiliPriv = emptyBytes, emptyBytes, emptyBytes
if client == nil {
return
}
if str, ok := client.Value("realname.rsa.priv"); ok {
rasPriv = []byte(str)
}
if str, ok := client.Value("realname.alipay.pub"); ok {
alipayPub = []byte(str)
}
if str, ok := client.Value("realname.alipay.bili.priv"); ok {
alipayBiliPriv = []byte(str)
}
return
}

91
app/job/main/member/dao/BUILD Normal file
View File

@@ -0,0 +1,91 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"account_notify_test.go",
"dao_test.go",
"databus_test.go",
"exp_test.go",
"http_test.go",
"member_log_test.go",
"member_old_test.go",
"member_test.go",
"memcache_test.go",
"realname_test.go",
"redis_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/model:go_default_library",
"//library/database/sql:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/bouk/monkey:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
"//vendor/gopkg.in/h2non/gock.v1:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"account_notify.go",
"dao.go",
"databus.go",
"exp.go",
"hbase.go",
"http.go",
"member.go",
"member_log.go",
"member_old.go",
"memcache.go",
"realname.go",
"redis.go",
],
importpath = "go-common/app/job/main/member/dao",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/dao/block:go_default_library",
"//app/job/main/member/model:go_default_library",
"//app/service/main/member/model:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/queue/databus:go_default_library",
"//library/queue/databus/report:go_default_library",
"//library/time:go_default_library",
"//library/xstr:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/member/dao/block:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

22
app/job/main/member/dao/account_notify.go Normal file
View File

@@ -0,0 +1,22 @@
package dao
import (
"context"
"fmt"
"go-common/app/job/main/member/model"
)
func notifyKey(mid int64) string {
return fmt.Sprintf("MemberJob-AccountNotify-T%d", mid)
}
// NotifyPurgeCache is
func (d *Dao) NotifyPurgeCache(c context.Context, mid int64, action string) error {
msg := &model.NotifyInfo{
Mid: mid,
Action: action,
}
key := notifyKey(mid)
return d.accNotify.Send(c, key, msg)
}

36
app/job/main/member/dao/account_notify_test.go Normal file
View File

@@ -0,0 +1,36 @@
package dao
import (
"github.com/smartystreets/goconvey/convey"
"testing"
)
func TestDaonotifyKey(t *testing.T) {
convey.Convey("notifyKey", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
p1 := notifyKey(mid)
convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
convCtx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
//func TestDaoNotifyPurgeCache(t *testing.T) {
// convey.Convey("NotifyPurgeCache", t, func(convCtx convey.C) {
// var (
// c = context.Background()
// mid = int64(0)
// action = ""
// )
// convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
// err := d.NotifyPurgeCache(c, mid, action)
// convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
// convCtx.So(err, convey.ShouldBeNil)
// })
// })
// })
//}

67
app/job/main/member/dao/block/BUILD Normal file
View File

@@ -0,0 +1,67 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"http_test.go",
"mc_test.go",
"mysql_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/model/block:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/database/sql:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//vendor/github.com/bouk/monkey:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
"//vendor/gopkg.in/h2non/gock.v1:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"http.go",
"mc.go",
"mysql.go",
"notify.go",
],
importpath = "go-common/app/job/main/member/dao/block",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/model/block:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

67
app/job/main/member/dao/block/dao.go Normal file
View File

@@ -0,0 +1,67 @@
package block
import (
"context"
"go-common/app/job/main/member/conf"
"go-common/library/cache/memcache"
xsql "go-common/library/database/sql"
bm "go-common/library/net/http/blademaster"
"github.com/pkg/errors"
)
type notifyFunc func(context.Context, int64, string) error
// Dao dao
type Dao struct {
conf *conf.Config
mc *memcache.Pool
db *xsql.DB
httpClient *bm.Client
notifyFunc notifyFunc
}
// New init mysql db
func New(conf *conf.Config, mc *memcache.Pool, db *xsql.DB, client *bm.Client, notifyFunc notifyFunc) (dao *Dao) {
dao = &Dao{
conf: conf,
mc: mc,
db: db,
httpClient: client,
notifyFunc: notifyFunc,
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.mc.Close()
d.db.Close()
}
// Ping dao ping
func (d *Dao) Ping(c context.Context) (err error) {
if err = d.db.Ping(c); err != nil {
return
}
if err = d.pingMC(c); err != nil {
return
}
return
}
// pingMc ping
func (d *Dao) pingMC(c context.Context) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
return
}
// BeginTX is.
func (d *Dao) BeginTX(c context.Context) (tx *xsql.Tx, err error) {
if tx, err = d.db.Begin(c); err != nil {
err = errors.WithStack(err)
}
return
}

61
app/job/main/member/dao/block/dao_test.go Normal file
View File

@@ -0,0 +1,61 @@
package block
import (
"flag"
"os"
"strings"
"testing"
"go-common/app/job/main/member/conf"
"go-common/library/cache/memcache"
xsql "go-common/library/database/sql"
bm "go-common/library/net/http/blademaster"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/h2non/gock.v1"
)
func TestTool(t *testing.T) {
Convey("tool", t, func() {
var (
mids = []int64{1, 2, 3, 46333, 35858}
)
str := midsToParam(mids)
So(str, ShouldEqual, "1,2,3,46333,35858")
})
}
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.account.member-job")
flag.Set("conf_token", "VEc5eqZNZHGQi6fsx7J6lJTqOGR9SnEO")
flag.Set("tree_id", "2134")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../cmd/member-job-dev.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
config := conf.Conf
//d.New().BlockImpl()
d = New(conf.Conf, memcache.NewPool(config.Memcache.Config), xsql.NewMySQL(config.BlockDB), bm.NewClient(config.HTTPClient), nil)
d.httpClient.SetTransport(gock.DefaultTransport)
os.Exit(m.Run())
}
func httpMock(method, url string) *gock.Request {
r := gock.New(url)
r.Method = strings.ToUpper(method)
return r
}

46
app/job/main/member/dao/block/http.go Normal file
View File

@@ -0,0 +1,46 @@
package block
import (
"context"
"fmt"
"net/url"
"strings"
"go-common/library/ecode"
"github.com/pkg/errors"
)
// SendSysMsg send sys msg.
func (d *Dao) SendSysMsg(c context.Context, code string, mids []int64, title string, content string, remoteIP string) (err error) {
params := url.Values{}
params.Set("mc", code)
params.Set("title", title)
params.Set("data_type", "4")
params.Set("context", content)
params.Set("mid_list", midsToParam(mids))
var res struct {
Code int `json:"code"`
Data *struct {
Status int8 `json:"status"`
Remark string `json:"remark"`
} `json:"data"`
}
if err = d.httpClient.Post(c, d.conf.BlockProperty.MSGURL, remoteIP, params, &res); err != nil {
err = errors.WithStack(err)
return
}
if res.Code != 0 {
err = errors.WithStack(ecode.Int(res.Code))
return
}
return
}
func midsToParam(mids []int64) (str string) {
strs := make([]string, 0, len(mids))
for _, mid := range mids {
strs = append(strs, fmt.Sprintf("%d", mid))
}
return strings.Join(strs, ",")
}

55
app/job/main/member/dao/block/http_test.go Normal file
View File

@@ -0,0 +1,55 @@
package block
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
"gopkg.in/h2non/gock.v1"
)
func TestBlockSendSysMsg(t *testing.T) {
convey.Convey("SendSysMsg", t, func(convCtx convey.C) {
var (
c = context.Background()
code = ""
mids = []int64{}
title = ""
content = ""
remoteIP = ""
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
defer gock.OffAll()
httpMock("POST", d.conf.BlockProperty.MSGURL).Reply(200).JSON(`{"code":0,"data":{"status":1,"remark":"test"}}`)
err := d.SendSysMsg(c, code, mids, title, content, remoteIP)
//println("err==",err)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
convCtx.Convey("When everything goes negative", func(convCtx convey.C) {
defer gock.OffAll()
httpMock("POST", d.conf.BlockProperty.MSGURL).Reply(200).JSON(`{"code":500,"data":{"status":1,"remark":"test"}}`)
//d.httpClient.SetTransport(gock.DefaultTransport)
err := d.SendSysMsg(c, code, mids, title, content, remoteIP)
convCtx.Convey("Then err should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
})
})
})
}
func TestBlockmidsToParam(t *testing.T) {
convey.Convey("midsToParam", t, func(convCtx convey.C) {
var (
mids = []int64{}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
str := midsToParam(mids)
convCtx.Convey("Then str should not be nil.", func(convCtx convey.C) {
convCtx.So(str, convey.ShouldNotBeNil)
})
})
})
}

70
app/job/main/member/dao/block/mc.go Normal file
View File

@@ -0,0 +1,70 @@
package block
import (
"context"
"fmt"
"go-common/library/cache/memcache"
"github.com/pkg/errors"
)
func userKey(mid int64) (key string) {
key = fmt.Sprintf("u_%d", mid)
return
}
func syncBlockTypeID() (key string) {
return "sync_bt_cur_id"
}
//SetSyncBlockTypeID is.
func (d *Dao) SetSyncBlockTypeID(c context.Context, id int64) (err error) {
var (
key = syncBlockTypeID()
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Set(&memcache.Item{Key: key, Object: id, Expiration: 3600 * 24, Flags: memcache.FlagJSON}); err != nil {
err = errors.WithStack(err)
return
}
return
}
//SyncBlockTypeID is.
func (d *Dao) SyncBlockTypeID(c context.Context) (id int64, err error) {
var (
key = syncBlockTypeID()
conn = d.mc.Get(c)
item *memcache.Item
)
defer conn.Close()
if item, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
err = errors.WithStack(err)
return
}
if err = conn.Scan(item, &id); err != nil {
err = errors.WithStack(err)
return
}
return
}
// DeleteUserCache is.
func (d *Dao) DeleteUserCache(c context.Context, mid int64) (err error) {
var (
key = userKey(mid)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Delete(key); err != nil {
err = errors.WithStack(err)
return
}
return
}

86
app/job/main/member/dao/block/mc_test.go Normal file
View File

@@ -0,0 +1,86 @@
package block
import (
"context"
"reflect"
"testing"
"go-common/library/cache/memcache"
"github.com/bouk/monkey"
"github.com/smartystreets/goconvey/convey"
)
func TestBlockuserKey(t *testing.T) {
convey.Convey("userKey", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
key := userKey(mid)
convCtx.Convey("Then key should not be nil.", func(convCtx convey.C) {
convCtx.So(key, convey.ShouldNotBeNil)
})
})
})
}
func TestBlocksyncBlockTypeID(t *testing.T) {
convey.Convey("syncBlockTypeID", t, func(convCtx convey.C) {
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
key := syncBlockTypeID()
convCtx.Convey("Then key should not be nil.", func(convCtx convey.C) {
convCtx.So(key, convey.ShouldNotBeNil)
})
})
})
}
func TestBlockSetSyncBlockTypeID(t *testing.T) {
convey.Convey("SetSyncBlockTypeID", t, func(convCtx convey.C) {
var (
c = context.Background()
id = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.SetSyncBlockTypeID(c, id)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestBlockSyncBlockTypeID(t *testing.T) {
convey.Convey("SyncBlockTypeID", t, func(convCtx convey.C) {
var (
c = context.Background()
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
id, err := d.SyncBlockTypeID(c)
convCtx.Convey("Then err should be nil.id should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(id, convey.ShouldNotBeNil)
})
})
})
}
func TestBlockDeleteUserCache(t *testing.T) {
convey.Convey("DeleteUserCache", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convCtx.Convey("connect memcache failed", func(convCtx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(d.mc), "Get", func(_ *memcache.Pool, _ context.Context) memcache.Conn {
return memcache.MockWith(memcache.ErrConnClosed)
})
defer guard.Unpatch()
err := d.DeleteUserCache(c, mid)
convCtx.Convey("Then err should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
})
})
})
}

143
app/job/main/member/dao/block/mysql.go Normal file
View File

@@ -0,0 +1,143 @@
package block
import (
"context"
"fmt"
model "go-common/app/job/main/member/model/block"
xsql "go-common/library/database/sql"
"github.com/pkg/errors"
)
const (
_upsertUser = `INSERT INTO block_user (mid,status) VALUES (?,?) ON DUPLICATE KEY UPDATE status=?`
_userStatusList = `SELECT id,mid FROM block_user WHERE status=? AND id>? LIMIT ?`
_userExtra = `SELECT id,mid,credit_answer_flag,action_time FROM block_extra WHERE mid=? ORDER BY id DESC LIMIT 1`
_userLastHistory = `SELECT id,mid,source,area,action,start_time,duration FROM block_history_%d WHERE mid=? ORDER BY id DESC LIMIT 1`
_upsertAddBlockCount = `INSERT INTO block_user_detail (mid,block_count) VALUES (?,1) ON DUPLICATE KEY UPDATE block_count=block_count+1`
_upsertExtra = `INSERT INTO block_extra (mid,credit_answer_flag,action_time) VALUES (?,?,?) ON DUPLICATE KEY UPDATE credit_answer_flag=? , action_time=?`
_insertHistory = `INSERT INTO block_history_%d (mid,admin_id,admin_name,source,area,reason,comment,action,start_time,duration,notify) VALUES (?,?,?,?,?,?,?,?,?,?,?)`
)
func historyIdx(mid int64) int64 {
return mid % 10
}
// UserStatusList is.
func (d *Dao) UserStatusList(c context.Context, status model.BlockStatus, startID int64, limit int) (maxID int64, mids []int64, err error) {
var (
rows *xsql.Rows
)
if rows, err = d.db.Query(c, _userStatusList, status, startID, limit); err != nil {
err = errors.WithStack(err)
return
}
defer rows.Close()
for rows.Next() {
var (
id, mid int64
)
if err = rows.Scan(&id, &mid); err != nil {
err = errors.WithStack(err)
return
}
if maxID < id {
maxID = id
}
mids = append(mids, mid)
}
if err = rows.Err(); err != nil {
err = errors.WithStack(err)
return
}
return
}
// UserLastHistory is.
func (d *Dao) UserLastHistory(c context.Context, mid int64) (his *model.DBHistory, err error) {
var (
sql = fmt.Sprintf(_userLastHistory, historyIdx(mid))
)
row := d.db.QueryRow(c, sql, mid)
his = &model.DBHistory{}
if err = row.Scan(&his.ID, &his.MID, &his.Source, &his.Area, &his.Action, &his.StartTime, &his.Duration); err != nil {
if err == xsql.ErrNoRows {
err = nil
his = nil
return
}
err = errors.WithStack(err)
return
}
return
}
// UserExtra is.
func (d *Dao) UserExtra(c context.Context, mid int64) (ex *model.DBExtra, err error) {
row := d.db.QueryRow(c, _userExtra, mid)
ex = &model.DBExtra{}
if err = row.Scan(&ex.ID, &ex.MID, &ex.CreditAnswerFlag, &ex.ActionTime); err != nil {
if err == xsql.ErrNoRows {
err = nil
ex = nil
return
}
err = errors.WithStack(err)
return
}
return
}
// TxUpsertUser is.
func (d *Dao) TxUpsertUser(c context.Context, tx *xsql.Tx, mid int64, status model.BlockStatus) (count int64, err error) {
rows, err := tx.Exec(_upsertUser, mid, status, status)
if err != nil {
err = errors.WithStack(err)
return
}
return rows.RowsAffected()
}
// InsertExtra is.
func (d *Dao) InsertExtra(c context.Context, ex *model.DBExtra) (err error) {
if _, err = d.db.Exec(c, _upsertExtra, ex.MID, ex.CreditAnswerFlag, ex.ActionTime, ex.CreditAnswerFlag, ex.ActionTime); err != nil {
err = errors.WithStack(err)
return
}
return
}
// TxUpsertExtra is.
func (d *Dao) TxUpsertExtra(c context.Context, tx *xsql.Tx, ex *model.DBExtra) (err error) {
if _, err = tx.Exec(_upsertExtra, ex.MID, ex.CreditAnswerFlag, ex.ActionTime, ex.CreditAnswerFlag, ex.ActionTime); err != nil {
err = errors.WithStack(err)
return
}
return
}
//TxInsertHistory is.
func (d *Dao) TxInsertHistory(c context.Context, tx *xsql.Tx, h *model.DBHistory) (err error) {
var (
sql = fmt.Sprintf(_insertHistory, historyIdx(h.MID))
)
if _, err = tx.Exec(sql, h.MID, h.AdminID, h.AdminName, h.Source, h.Area, h.Reason, h.Comment, h.Action, h.StartTime, h.Duration, h.Notify); err != nil {
err = errors.WithStack(err)
return
}
return
}
//UpsertAddBlockCount is.
func (d *Dao) UpsertAddBlockCount(c context.Context, mid int64) (err error) {
if _, err = d.db.Exec(c, _upsertAddBlockCount, mid); err != nil {
err = errors.WithStack(err)
return
}
return
}

277
app/job/main/member/dao/block/mysql_test.go Normal file
View File

@@ -0,0 +1,277 @@
package block
import (
"context"
"database/sql"
"fmt"
"reflect"
"testing"
model "go-common/app/job/main/member/model/block"
xsql "go-common/library/database/sql"
"github.com/bouk/monkey"
"github.com/smartystreets/goconvey/convey"
)
func TestBlockhistoryIdx(t *testing.T) {
convey.Convey("historyIdx", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
p1 := historyIdx(mid)
convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
convCtx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestBlockUserStatusList(t *testing.T) {
convey.Convey("UserStatusList", t, func(convCtx convey.C) {
var (
c = context.Background()
status model.BlockStatus
startID = int64(0)
limit = int(10)
)
rows, _ := d.db.Query(c, _userStatusList, status, startID, limit)
convCtx.Convey("UserStatusList success", func(convCtx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(rows), "Scan", func(_ *xsql.Rows, _ ...interface{}) error {
return nil
})
defer guard.Unpatch()
maxID, mids, err := d.UserStatusList(c, status, startID, limit)
convCtx.Convey("Then err should be nil.maxID,mids should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(mids, convey.ShouldNotBeNil)
convCtx.So(maxID, convey.ShouldNotBeNil)
})
})
})
}
func TestBlockUserLastHistory(t *testing.T) {
convey.Convey("UserLastHistory", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("UserLastHistory success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db.QueryRow(context.TODO(), fmt.Sprintf(_userLastHistory, historyIdx(mid)), 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return nil
})
defer monkey.UnpatchAll()
his, err := d.UserLastHistory(c, mid)
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(his, convey.ShouldNotBeNil)
})
convey.Convey("UserLastHistory err", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db.QueryRow(context.TODO(), fmt.Sprintf(_userLastHistory, historyIdx(mid)), 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return fmt.Errorf("row.Scan error")
})
defer monkey.UnpatchAll()
his, err := d.UserLastHistory(c, mid)
convCtx.So(err, convey.ShouldNotBeNil)
convCtx.So(his, convey.ShouldNotBeNil)
})
convey.Convey("UserLastHistory no record", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db.QueryRow(context.TODO(), fmt.Sprintf(_userLastHistory, historyIdx(mid)), 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return sql.ErrNoRows
})
defer monkey.UnpatchAll()
his, err := d.UserLastHistory(c, mid)
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(his, convey.ShouldBeNil)
})
})
}
func TestBlockUserExtra(t *testing.T) {
convey.Convey("UserExtra", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convey.Convey("UserExtra success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db.QueryRow(context.TODO(), _userExtra, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return nil
})
defer monkey.UnpatchAll()
ex, err := d.UserExtra(c, mid)
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(ex, convey.ShouldNotBeNil)
})
convey.Convey("UserExtra err", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db.QueryRow(context.TODO(), _userExtra, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return fmt.Errorf("row.Scan error")
})
defer monkey.UnpatchAll()
ex, err := d.UserExtra(c, mid)
convCtx.So(err, convey.ShouldNotBeNil)
convCtx.So(ex, convey.ShouldNotBeNil)
})
convey.Convey("UserExtra no record", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db.QueryRow(context.TODO(), _userExtra, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return sql.ErrNoRows
})
defer monkey.UnpatchAll()
ex, err := d.UserExtra(c, mid)
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(ex, convey.ShouldBeNil)
})
})
}
func TestBlockTxUpsertUser(t *testing.T) {
convey.Convey("TxUpsertUser", t, func(convCtx convey.C) {
var (
c = context.Background()
tx, _ = d.BeginTX(c)
mid = int64(0)
status model.BlockStatus
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
count, err := d.TxUpsertUser(c, tx, mid, status)
convCtx.Convey("Then err should be nil.count should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(count, convey.ShouldNotBeNil)
})
})
convCtx.Reset(func() {
tx.Commit()
})
})
}
func TestBlockInsertExtra(t *testing.T) {
convey.Convey("InsertExtra", t, func(convCtx convey.C) {
var (
c = context.Background()
ex = &model.DBExtra{}
)
convCtx.Convey("InsertExtra success", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.InsertExtra(c, ex)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
convCtx.Convey("InsertExtra err", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("insert err")
})
defer monkey.UnpatchAll()
err := d.InsertExtra(c, ex)
convCtx.Convey("Then err should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
})
})
})
}
func TestBlockTxUpsertExtra(t *testing.T) {
convey.Convey("TxUpsertExtra", t, func(convCtx convey.C) {
var (
c = context.Background()
tx, _ = d.BeginTX(c)
ex = &model.DBExtra{}
)
convCtx.Convey("TxUpsertExtra success", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(tx), "Exec", func(_ *xsql.Tx, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.TxUpsertExtra(c, tx, ex)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
convCtx.Convey("TxUpsertExtra err", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(tx), "Exec", func(_ *xsql.Tx, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("insert err")
})
defer monkey.UnpatchAll()
err := d.TxUpsertExtra(c, tx, ex)
convCtx.Convey("Then err should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
})
})
convCtx.Reset(func() {
tx.Commit()
})
})
}
func TestBlockTxInsertHistory(t *testing.T) {
convey.Convey("TxInsertHistory", t, func(convCtx convey.C) {
var (
c = context.Background()
tx, _ = d.BeginTX(c)
h = &model.DBHistory{}
)
convCtx.Convey("TxUpsertExtra success", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(tx), "Exec", func(_ *xsql.Tx, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.TxInsertHistory(c, tx, h)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
convCtx.Convey("TxUpsertExtra err", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(tx), "Exec", func(_ *xsql.Tx, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("insert err")
})
defer monkey.UnpatchAll()
err := d.TxInsertHistory(c, tx, h)
convCtx.Convey("Then err should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
})
})
convCtx.Reset(func() {
tx.Commit()
})
})
}
func TestBlockUpsertAddBlockCount(t *testing.T) {
convey.Convey("UpsertAddBlockCount", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convCtx.Convey("UpsertAddBlockCount success", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.UpsertAddBlockCount(c, mid)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
convCtx.Convey("InsertExtra err", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("insert err")
})
defer monkey.UnpatchAll()
err := d.UpsertAddBlockCount(c, mid)
convCtx.Convey("Then err should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
})
})
})
}

16
app/job/main/member/dao/block/notify.go Normal file
View File

@@ -0,0 +1,16 @@
package block
import (
"context"
"github.com/pkg/errors"
)
// AccountNotify is
func (d *Dao) AccountNotify(c context.Context, mid int64) (err error) {
action := "blockUser"
if err = d.notifyFunc(c, mid, action); err != nil {
err = errors.Errorf("mid(%d) s.accountNotify.Send(%+v) error(%+v)", mid, action, err)
}
return
}

79
app/job/main/member/dao/dao.go Normal file
View File

@@ -0,0 +1,79 @@
package dao
import (
"context"
"time"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/dao/block"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/database/sql"
bm "go-common/library/net/http/blademaster"
"go-common/library/queue/databus"
)
// Dao struct info of Dao.
type Dao struct {
c *conf.Config
block *block.Dao
db *sql.DB
accCheckDB *sql.DB
passLogDB *sql.DB
accdb *sql.DB
asodb *sql.DB
client *bm.Client
mc *memcache.Pool
mcExpire int32
redis *redis.Pool
plogDatabus *databus.Databus
accNotify *databus.Databus
}
// New new a Dao and return.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
c: c,
db: sql.NewMySQL(c.Mysql),
accCheckDB: sql.NewMySQL(c.AccCheckMysql),
passLogDB: sql.NewMySQL(c.PasslogMysql),
accdb: sql.NewMySQL(c.AccMysql),
asodb: sql.NewMySQL(c.AsoMysql),
client: bm.NewClient(c.HTTPClient),
mc: memcache.NewPool(c.Memcache.Config),
mcExpire: int32(time.Duration(c.Memcache.Expire) / time.Second),
redis: redis.NewPool(c.Redis),
plogDatabus: databus.New(c.PLogDatabus),
accNotify: databus.New(c.AccountNotify),
}
d.block = block.New(c,
memcache.NewPool(c.BlockMemcache),
sql.NewMySQL(c.BlockDB),
d.client,
d.NotifyPurgeCache,
)
return
}
// Close close connections of mc, redis, db.
func (d *Dao) Close() {
if d.db != nil {
d.db.Close()
}
if d.mc != nil {
d.mc.Close()
}
if d.block != nil {
d.block.Close()
}
}
// Ping ping health of db.
func (d *Dao) Ping(c context.Context) (err error) {
return d.db.Ping(c)
}
// BlockImpl is.
func (d *Dao) BlockImpl() *block.Dao {
return d.block
}

45
app/job/main/member/dao/dao_test.go Normal file
View File

@@ -0,0 +1,45 @@
package dao
import (
"flag"
"os"
"strings"
"testing"
"go-common/app/job/main/member/conf"
"gopkg.in/h2non/gock.v1"
)
var (
d *Dao
)
func httpMock(method, url string) *gock.Request {
r := gock.New(url)
r.Method = strings.ToUpper(method)
return r
}
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.account.member-job")
flag.Set("conf_token", "VEc5eqZNZHGQi6fsx7J6lJTqOGR9SnEO")
flag.Set("tree_id", "2134")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../cmd/member-job-dev.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
d.client.SetTransport(gock.DefaultTransport)
os.Exit(m.Run())
}

25
app/job/main/member/dao/databus.go Normal file
View File

@@ -0,0 +1,25 @@
package dao
import (
"context"
"strconv"
"go-common/app/job/main/member/model"
)
// DatabusAddLog add exp log messager to databus
func (d *Dao) DatabusAddLog(c context.Context, mid, exp, toExp, ts int64, oper, reason, ip string) (err error) {
log := &model.UserLog{
Mid: mid,
TS: ts,
IP: ip,
Content: map[string]string{
"from_exp": strconv.FormatInt(exp, 10),
"to_exp": strconv.FormatInt(toExp, 10),
"operater": oper,
"reason": reason,
},
}
err = d.plogDatabus.Send(c, strconv.FormatInt(mid, 10), log)
return
}

49
app/job/main/member/dao/databus_test.go Normal file
View File

@@ -0,0 +1,49 @@
package dao
import (
"context"
"fmt"
"reflect"
"testing"
"go-common/library/queue/databus"
"github.com/bouk/monkey"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoDatabusAddLog(t *testing.T) {
convey.Convey("DatabusAddLog", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
exp = int64(0)
toExp = int64(0)
ts = int64(0)
oper = ""
reason = ""
ip = ""
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(d.plogDatabus), "Send", func(_ *databus.Databus, _ context.Context, _ string, _ interface{}) error {
return nil
})
defer monkey.UnpatchAll()
err := d.DatabusAddLog(c, mid, exp, toExp, ts, oper, reason, ip)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
convCtx.Convey("When everything goes negative", func(convCtx convey.C) {
monkey.PatchInstanceMethod(reflect.TypeOf(d.plogDatabus), "Send", func(_ *databus.Databus, _ context.Context, _ string, _ interface{}) error {
return fmt.Errorf("Failed send data err")
})
defer monkey.UnpatchAll()
err := d.DatabusAddLog(c, mid, exp, toExp, ts, oper, reason, ip)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
})
})
})
}

52
app/job/main/member/dao/exp.go Normal file
View File

@@ -0,0 +1,52 @@
package dao
import (
"context"
"fmt"
"go-common/app/job/main/member/model"
"go-common/library/database/sql"
"go-common/library/time"
)
const (
_initExp = "INSERT IGNORE INTO user_exp_%02d (mid) VALUES (?)"
_updateExpAped = `UPDATE user_exp_%02d SET exp=exp+?,flag=flag|? WHERE mid=? AND flag&?=0`
_updateExpFlag = `UPDATE user_exp_%02d SET addtime=?,flag=? WHERE mid=? AND addtime<?`
_SelExp = "SELECT mid,exp,flag,addtime,mtime FROM user_exp_%02d where mid=?"
)
// InitExp init user exp
func (d *Dao) InitExp(c context.Context, mid int64) (err error) {
_, err = d.db.Exec(c, fmt.Sprintf(_initExp, hit(mid)), mid)
return
}
// SelExp get new user exp by mid
func (d *Dao) SelExp(c context.Context, mid int64) (exp *model.NewExp, err error) {
exp = &model.NewExp{}
row := d.db.QueryRow(c, fmt.Sprintf(_SelExp, hit(mid)), mid)
if err = row.Scan(&exp.Mid, &exp.Exp, &exp.Flag, &exp.Addtime, &exp.Mtime); err != nil {
if err == sql.ErrNoRows {
err = nil
}
return
}
return
}
// UpdateExpAped update user exp append mode.
func (d *Dao) UpdateExpAped(c context.Context, mid, exp int64, flag int32) (rows int64, err error) {
res, err := d.db.Exec(c, fmt.Sprintf(_updateExpAped, hit(mid)), exp, flag, mid, flag)
if err != nil {
return
}
rows, _ = res.RowsAffected()
return
}
// UpdateExpFlag update user exp.
func (d *Dao) UpdateExpFlag(c context.Context, mid int64, flag int32, addtime time.Time) (err error) {
_, err = d.db.Exec(c, fmt.Sprintf(_updateExpFlag, hit(mid)), addtime, flag, mid, addtime)
return
}

43
app/job/main/member/dao/exp_test.go Normal file
View File

@@ -0,0 +1,43 @@
package dao
import (
"context"
"testing"
"time"
xtime "go-common/library/time"
. "github.com/smartystreets/goconvey/convey"
)
func Test_SelExp(t *testing.T) {
Convey("selExp", t, func() {
exp, err := d.SelExp(context.TODO(), 2233)
So(err, ShouldBeNil)
So(exp, ShouldNotBeEmpty)
})
}
func Test_UpdateExpAped(t *testing.T) {
Convey("updateExpAped", t, func() {
if _, err := d.UpdateExpAped(context.TODO(), 2233, 666, 1); err != nil {
So(err, ShouldBeNil)
}
})
}
func Test_InitExp(t *testing.T) {
Convey("InitExp", t, func() {
if err := d.InitExp(context.Background(), 1111); err != nil {
So(err, ShouldNotBeNil)
}
})
}
func Test_UpdateExpFlag(t *testing.T) {
Convey("UpdateExpFlag", t, func() {
if err := d.UpdateExpFlag(context.Background(), 1111, 123, xtime.Time(time.Now().Unix())); err != nil {
So(err, ShouldNotBeNil)
}
})
}

34
app/job/main/member/dao/hbase.go Normal file
View File

@@ -0,0 +1,34 @@
package dao
// var (
// familyDetail = "detail"
// columnTs = "ts"
// )
// func rowKey(mid, ts int64, op string) string {
// return fmt.Sprintf("%d%d_%d_%s", mid%10, mid, math.MaxInt64-ts, op)
// }
// // AddLog coin modify log.
// func (d *Dao) AddLog(c context.Context, mid, ts int64, content map[string][]byte, table string) (err error) {
// var (
// mutate *hrpc.Mutate
// operator = string(content["operater"])
// key = rowKey(mid, ts, operator)
// tsB = make([]byte, 8)
// ctx, cancel = context.WithTimeout(c, time.Duration(d.c.HBase.WriteTimeout))
// )
// defer cancel()
// binary.BigEndian.PutUint64(tsB, uint64(ts))
// content[columnTs] = tsB
// values := map[string]map[string][]byte{familyDetail: content}
// if mutate, err = hrpc.NewPutStr(ctx, table, key, values); err != nil {
// log.Error("hrpc.NewPutStr(%s, %s, %v) error(%v)", table, key, values, err)
// return
// }
// if _, err = d.hbase.Put(c, mutate); err != nil {
// log.Error("hbase.Put mid %d,error(%v)", mid, err)
// return
// }
// return
// }

89
app/job/main/member/dao/http.go Normal file
View File

@@ -0,0 +1,89 @@
package dao
import (
"context"
"net/url"
"strconv"
"go-common/app/job/main/member/model"
"go-common/library/ecode"
"go-common/library/log"
)
// AsoSt aso user status.
type AsoSt struct {
Code int64
Data struct {
Email string `json:"email"`
Telphone string `json:"telphone"`
SafeQs int8 `json:"safe_question"`
Spacesta int8 `json:"spacesta"`
} `json:"data"`
}
const (
_asoURI = "http://passport.bilibili.co/intranet/acc/queryByMid"
_identityURL = "http://account.bilibili.co/api/identify/info"
_updateFaceURL = "http://account.bilibili.co/api/internal/member/updateFace"
)
// AsoStatus get aso bind status.
func (d *Dao) AsoStatus(c context.Context, mid int64) (aso *model.MemberAso, err error) {
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
res := new(struct {
Code int64 `json:"code"`
Data model.MemberAso `json:"data"`
})
if err = d.client.Get(c, _asoURI, "", params, res); err != nil {
log.Error("get aso err %v uri %s ", err, _asoURI+params.Encode())
return
}
if res.Code != 0 {
err = ecode.RequestErr
log.Error("asotatus err res %v", res)
return
}
aso = &res.Data
return
}
// IdentifyStaus identify status.
func (d *Dao) IdentifyStaus(c context.Context, mid int64) (b bool, err error) {
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
var res struct {
Code int64 `json:"code"`
Data struct {
Identify int8 `json:"identify"` // 0
} `json:"data"`
}
if err = d.client.Get(c, _identityURL, "", params, &res); err != nil {
log.Error("get aso err %v uri %s ", err, _asoURI+params.Encode())
return
}
if res.Code == 0 && res.Data.Identify == 0 {
b = true
}
return
}
// UpdateAccFace update acc face.
func (d *Dao) UpdateAccFace(c context.Context, mid int64, face string) (err error) {
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("face", face)
var res struct {
Code int `json:"code"`
}
if err = d.client.Post(c, _updateFaceURL, "", params, &res); err != nil {
log.Error("update account face err %v uri %s ", err, _updateFaceURL+params.Encode())
return
}
if res.Code != 0 {
err = ecode.Int(res.Code)
log.Error("UpdateAccFace mid(%d) error(%v)", mid, err)
return
}
return
}

29
app/job/main/member/dao/http_test.go Normal file
View File

@@ -0,0 +1,29 @@
package dao
import (
"context"
"testing"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/h2non/gock.v1"
)
func TestDao_UpdateAccFace(t *testing.T) {
Convey("UpdateAccFace", t, func() {
Convey("When everything goes positive", func() {
defer gock.OffAll()
httpMock("POST", _updateFaceURL).Reply(200).JSON(`{"code":0}`)
err := d.UpdateAccFace(context.TODO(), 1, "1456")
So(err, ShouldBeNil)
})
Convey("When everything goes negative", func() {
defer gock.OffAll()
httpMock("POST", _updateFaceURL).Reply(200).JSON(`{"code":500}`)
err := d.UpdateAccFace(context.TODO(), 1, "1456")
Convey("Then err should be nil.", func() {
So(err, ShouldNotBeNil)
})
})
})
}

140
app/job/main/member/dao/member.go Normal file
View File

@@ -0,0 +1,140 @@
package dao
import (
"context"
"database/sql"
"fmt"
"go-common/app/job/main/member/model"
smodel "go-common/app/service/main/member/model"
"go-common/library/log"
)
const (
_shard = 100
_SelBaseInfo = "SELECT mid,name,sex,face,sign,rank,birthday,ctime,mtime FROM user_base_%02d WHERE mid=?"
_SetOfficial = "INSERT INTO user_official (mid,role,title,description) values (?,?,?,?) ON DUPLICATE KEY UPDATE role=VALUES(role),title=VALUES(title),description=VALUES(description)"
_SetBaseInfo = "INSERT INTO user_base_%02d(mid,name,sex,face,sign,rank) VALUES (?,?,?,?,?,?) ON DUPLICATE KEY UPDATE name=?,sex=?,face=?,sign=?,rank=?"
_SetName = "INSERT INTO user_base_%02d(mid,name) VALUES (?,?) ON DUPLICATE KEY UPDATE name=?"
_SetSign = "INSERT INTO user_base_%02d(mid,sign) VALUES (?,?) ON DUPLICATE KEY UPDATE sign=?"
_InitBase = "INSERT IGNORE INTO user_base_%02d(mid) VALUES (?)"
_SetFace = "UPDATE user_base_%02d SET face=? WHERE mid=?"
_SelMoral = "SELECT mid,moral,added,deducted,last_recover_date from user_moral where mid = ?"
_recoverMoral = `update user_moral set moral=moral+?,added=added+?, last_recover_date=? where mid = ? and last_recover_date != ? and moral < 7000 `
_auditQueuingFace = `UPDATE user_property_review SET state=?, operator='system', remark=? WHERE property=1 AND state=10 AND is_monitor=0 AND new=?`
)
// hit get table suffix
func hit(id int64) int64 {
return id % _shard
}
// BaseInfo info of user.
//用于检查数据，不需要拼url
func (d *Dao) BaseInfo(c context.Context, mid int64) (r *model.BaseInfo, err error) {
r = &model.BaseInfo{}
row := d.db.QueryRow(c, fmt.Sprintf(_SelBaseInfo, hit(mid)), mid)
if err = row.Scan(&r.Mid, &r.Name, &r.Sex, &r.Face, &r.Sign, &r.Rank, &r.Birthday, &r.CTime, &r.MTime); err != nil {
r = nil
if err == sql.ErrNoRows {
err = nil
return
}
log.Error("row.Scan() error(%v) mid(%v)", err, mid)
}
return
}
// SelMoral select moral by mid from db
func (d *Dao) SelMoral(c context.Context, mid int64) (moral *smodel.Moral, err error) {
moral = &smodel.Moral{}
row := d.db.QueryRow(c, _SelMoral, mid)
if err = row.Scan(&moral.Mid, &moral.Moral, &moral.Added, &moral.Deducted, &moral.LastRecoverDate); err != nil {
if err == sql.ErrNoRows {
moral = nil
err = nil
return
}
log.Error(" SelMoral row.Scan() error(%v) mid(%v)", err, mid)
}
return
}
// RecoverMoral set moral by mid from db
func (d *Dao) RecoverMoral(c context.Context, mid, delta, added int64, lastRecoverDate string) (rowsAffected int64, err error) {
var res sql.Result
if res, err = d.db.Exec(c, _recoverMoral, delta, added, lastRecoverDate, mid, lastRecoverDate); err != nil {
log.Error(" RecoverMoral d.db.Exec() error(%v) mid(%v)", err, mid)
return
}
return res.RowsAffected()
}
// SetSign set sign.
func (d *Dao) SetSign(c context.Context, mid int64, sign string) (err error) {
if _, err = d.db.Exec(c, fmt.Sprintf(_SetSign, hit(mid)), mid, sign, sign); err != nil {
log.Error("setSign: tx.Exec(%d, %v) error(%v)", mid, sign, err)
return
}
return
}
// SetOfficial set official.
func (d *Dao) SetOfficial(c context.Context, mid int64, role int8, title string, desc string) error {
if _, err := d.db.Exec(c, _SetOfficial, mid, role, title, desc); err != nil {
log.Error("_SetOfficial: tx.Exec(%d,%d,%s,%s) error(%v)", mid, role, title, desc, err)
return err
}
return nil
}
// SetName set name.
func (d *Dao) SetName(c context.Context, mid int64, name string) (err error) {
if len(name) <= 0 {
log.Error("SetName: mid(%d) len(name)<=0 ", mid)
return
}
if _, err = d.db.Exec(c, fmt.Sprintf(_SetName, hit(mid)), mid, name, name); err != nil {
log.Error("setName: tx.Exec(%d, %v) error(%v)", mid, name, err)
return
}
return
}
// SetBaseInfo set base info of user.
func (d *Dao) SetBaseInfo(c context.Context, r *model.BaseInfo) (err error) {
if len(r.Name) <= 0 {
log.Error("SetBaseInfo: mid(%d) len(r.Name)<=0 BaseInfo(%v)", r.Mid, r)
return
}
if _, err = d.db.Exec(c, fmt.Sprintf(_SetBaseInfo, hit(r.Mid)), r.Mid, r.Name, r.Sex, r.Face, r.Sign, r.Rank, r.Name, r.Sex, r.Face, r.Sign, r.Rank); err != nil {
log.Error("SetBaseInfo: db.Exec(%d, %v) error(%v)", r.Mid, r, err)
}
return
}
// SetFace set face.
func (d *Dao) SetFace(c context.Context, mid int64, face string) (err error) {
if _, err = d.db.Exec(c, fmt.Sprintf(_SetFace, hit(mid)), face, mid); err != nil {
log.Error("SetFace: tx.Exec(%v,%v) error(%v)", face, mid, err)
}
return
}
// InitBase init base info.
func (d *Dao) InitBase(c context.Context, mid int64) (err error) {
if _, err = d.db.Exec(c, fmt.Sprintf(_InitBase, hit(mid)), mid); err != nil {
log.Error("InitBase: tx.Exec(%d) error(%v)", mid, err)
}
return
}
// AuditQueuingFace auditQueuingFace.
func (d *Dao) AuditQueuingFace(c context.Context, face string, remark string, state int8) error {
_, err := d.db.Exec(c, _auditQueuingFace, state, remark, face)
if err != nil {
log.Error("Failed to audit queuing face: %s, remark: %s, state: %d: %+v", face, remark, state, err)
return err
}
return nil
}

56
app/job/main/member/dao/member_log.go Normal file
View File

@@ -0,0 +1,56 @@
package dao
import (
"context"
"go-common/app/job/main/member/model"
"go-common/library/log"
"go-common/library/queue/databus/report"
"go-common/library/time"
)
// consts
const (
ExpLogID = 11
MoralLogID = 12
)
// AddExpLog is
func (d *Dao) AddExpLog(ctx context.Context, ul *model.UserLog) {
d.addLog(ctx, ExpLogID, "log_exp_change", ul)
}
// AddMoralLog is
func (d *Dao) AddMoralLog(ctx context.Context, ul *model.UserLog) {
d.addLog(ctx, MoralLogID, "log_moral_change", ul)
}
// AddExpLog is
func (d *Dao) addLog(ctx context.Context, business int, action string, ul *model.UserLog) {
t := time.Time(ul.TS)
content := make(map[string]interface{}, len(ul.Content))
for k, v := range ul.Content {
content[k] = v
}
if ul.LogID == "" {
ul.LogID = model.UUID4()
}
content["log_id"] = ul.LogID
ui := &report.UserInfo{
Mid: ul.Mid,
Platform: "",
Build: 0,
Buvid: "",
Business: business,
Type: 0,
Oid: 0,
Action: action,
Ctime: t.Time(),
IP: ul.IP,
// extra
Index: []interface{}{ul.LogID},
Content: content,
}
report.User(ui)
log.Info("add log to report: userlog: %+v userinfo: %+v", ul, ui)
}

53
app/job/main/member/dao/member_log_test.go Normal file
View File

@@ -0,0 +1,53 @@
package dao
import (
"context"
"go-common/app/job/main/member/model"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoAddExpLog(t *testing.T) {
convey.Convey("AddExpLog", t, func(convCtx convey.C) {
var (
ctx = context.Background()
ul = &model.UserLog{}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
d.AddExpLog(ctx, ul)
convCtx.Convey("No return values", func(convCtx convey.C) {
})
})
})
}
func TestDaoAddMoralLog(t *testing.T) {
convey.Convey("AddMoralLog", t, func(convCtx convey.C) {
var (
ctx = context.Background()
ul = &model.UserLog{}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
d.AddMoralLog(ctx, ul)
convCtx.Convey("No return values", func(convCtx convey.C) {
})
})
})
}
func TestDaoaddLog(t *testing.T) {
convey.Convey("addLog", t, func(convCtx convey.C) {
var (
ctx = context.Background()
business = int(0)
action = ""
ul = &model.UserLog{}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
d.addLog(ctx, business, action, ul)
convCtx.Convey("No return values", func(convCtx convey.C) {
})
})
})
}

54
app/job/main/member/dao/member_old.go Normal file
View File

@@ -0,0 +1,54 @@
package dao
import (
"context"
"fmt"
"go-common/app/job/main/member/model"
"go-common/library/log"
"go-common/library/xstr"
)
const (
_SelAsos = "SELECT mid,uname from aso_account where mid in(%s)"
_SelAsoName = "SELECT uname from aso_account where mid=?"
)
// Accounts gets account info.
func (d *Dao) Accounts(c context.Context, mids []int64) (accs map[int64]*model.AccountInfo, errs map[int64]map[string]bool, err error) {
//moralLogTpl := mids[0] % _shardMoralLog
midsStr := xstr.JoinInts(mids)
accs = make(map[int64]*model.AccountInfo)
errs = make(map[int64]map[string]bool)
// name
rows, err := d.asodb.Query(c, fmt.Sprintf(_SelAsos, midsStr))
if err != nil {
log.Error("d.asodb.Query() error(%v)", err)
return
}
defer rows.Close()
for rows.Next() {
r := &model.AccountInfo{}
if err = rows.Scan(&r.Mid, &r.Name); err != nil {
log.Error("row.Scan() error(%v) mid(%v)", err, r.Mid)
err = nil
continue
}
accs[r.Mid] = r
errs[r.Mid] = make(map[string]bool)
errs[r.Mid]["asoOK"] = true
}
return
}
//Name name.
func (d *Dao) Name(c context.Context, mid int64) (name string, err error) {
arow := d.asodb.QueryRow(c, _SelAsoName, mid)
if err = arow.Scan(&name); err != nil {
log.Error("row.Scan() error(%v)", err)
return
}
return
}

56
app/job/main/member/dao/member_old_test.go Normal file
View File

@@ -0,0 +1,56 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoAccounts(t *testing.T) {
convey.Convey("Accounts", t, func(convCtx convey.C) {
var (
c = context.Background()
mids = []int64{1, 2}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
accs, errs, err := d.Accounts(c, mids)
if err != nil {
convCtx.Convey("When no rows int result", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
convCtx.So(errs, convey.ShouldNotBeNil)
convCtx.So(accs, convey.ShouldNotBeNil)
})
} else {
convCtx.Convey("When have rows int result", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(errs, convey.ShouldNotBeNil)
convCtx.So(accs, convey.ShouldNotBeNil)
})
}
})
})
}
func TestDaoName(t *testing.T) {
convey.Convey("Name", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(1)
)
convCtx.Convey("Name search two situation:", func(convCtx convey.C) {
name, err := d.Name(c, mid)
if err != nil {
convCtx.Convey("Name occur an error", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
convCtx.So(name, convey.ShouldNotBeNil)
})
} else {
convCtx.Convey("Name no err search", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(name, convey.ShouldNotBeNil)
})
}
})
})
}

305
app/job/main/member/dao/member_test.go Normal file
View File

@@ -0,0 +1,305 @@
package dao
import (
"context"
"database/sql"
"fmt"
"reflect"
"testing"
"time"
"go-common/app/job/main/member/model"
xsql "go-common/library/database/sql"
"github.com/bouk/monkey"
. "github.com/smartystreets/goconvey/convey"
)
func TestDao_SetBaseInfo(t *testing.T) {
r := &model.BaseInfo{Mid: 100, Sex: 1, Name: "lgs", Face: "face", Sign: "sign", Rank: 3}
Convey("SetBaseInfo", t, func() {
Convey("SetBaseInfo success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.SetBaseInfo(context.TODO(), r)
So(err, ShouldBeNil)
})
Convey("SetBaseInfo failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.SetBaseInfo(context.TODO(), r)
So(err, ShouldNotBeNil)
})
})
}
func TestDao_UpdateSign(t *testing.T) {
Convey("SetSign", t, func() {
Convey("SetSign success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.SetSign(context.TODO(), 1, "my sign")
So(err, ShouldBeNil)
})
Convey("SetSign failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.SetSign(context.TODO(), 1, "my sign")
So(err, ShouldNotBeNil)
})
})
}
func TestDao_SetName(t *testing.T) {
Convey("SetName", t, func() {
Convey("SetName success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.SetName(context.TODO(), 1, "myname")
So(err, ShouldBeNil)
})
Convey("SetName failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.SetName(context.TODO(), 1, "myname")
So(err, ShouldNotBeNil)
})
})
}
func TestDao_UpdateFace(t *testing.T) {
Convey("SetFaceRank", t, func() {
Convey("SetFaceRank success", func() {
err := d.SetFace(context.TODO(), int64(1), "http://face")
So(err, ShouldBeNil)
})
Convey("SetFaceRank failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.SetFace(context.TODO(), int64(1), "http://face")
So(err, ShouldNotBeNil)
})
})
}
func TestDao_SelMoral(t *testing.T) {
Convey("SelMoral", t, func() {
Convey("SelMoral success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.accdb.QueryRow(context.TODO(), _SelMoral, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return nil
})
defer monkey.UnpatchAll()
moral, err := d.SelMoral(context.TODO(), int64(8))
So(err, ShouldBeNil)
So(moral, ShouldNotBeNil)
})
Convey("SelMoral error", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.accdb.QueryRow(context.TODO(), _SelMoral, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return fmt.Errorf("row.Scan error")
})
defer monkey.UnpatchAll()
moral, err := d.SelMoral(context.TODO(), int64(8))
So(err, ShouldNotBeNil)
So(moral, ShouldNotBeNil)
})
Convey("SelMoral no record", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.accdb.QueryRow(context.TODO(), _SelMoral, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return sql.ErrNoRows
})
defer monkey.UnpatchAll()
moral, err := d.SelMoral(context.TODO(), int64(8))
So(err, ShouldBeNil)
So(moral, ShouldBeNil)
})
})
}
func TestDao_IncrMoral(t *testing.T) {
Convey("IncrMoral", t, func() {
Convey("AuditQueuingFace success", func() {
_, err := d.RecoverMoral(context.TODO(), 2, 100, 1, time.Now().Format("2006-01-02"))
So(err, ShouldBeNil)
})
Convey("AuditQueuingFace failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
_, err := d.RecoverMoral(context.TODO(), 2, 100, 1, time.Now().Format("2006-01-02"))
So(err, ShouldNotBeNil)
})
})
}
func TestDao_AuditQueuingFace(t *testing.T) {
Convey("AuditQueuingFace", t, func() {
Convey("AuditQueuingFace success", func() {
err := d.AuditQueuingFace(context.TODO(), "/bfs/face/7e68723b9d3664ac3773c1f3c26d5e2bfabc0f22.jpg", "", 0)
So(err, ShouldBeNil)
})
Convey("AuditQueuingFace failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.AuditQueuingFace(context.TODO(), "/bfs/face/7e68723b9d3664ac3773c1f3c26d5e2bfabc0f22.jpg", "", 0)
So(err, ShouldNotBeNil)
})
})
}
func TestDao_BaseInfo(t *testing.T) {
Convey("BaseInfo", t, func() {
Convey("BaseInfo success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.accdb.QueryRow(context.TODO(), _SelBaseInfo, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return nil
})
defer monkey.UnpatchAll()
r, err := d.BaseInfo(context.TODO(), 100)
So(err, ShouldBeNil)
So(r, ShouldNotBeNil)
})
Convey("BaseInfo error", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.accdb.QueryRow(context.TODO(), _SelBaseInfo, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return fmt.Errorf("row.Scan error")
})
defer monkey.UnpatchAll()
r, err := d.BaseInfo(context.TODO(), 11)
So(err, ShouldNotBeNil)
So(r, ShouldBeNil)
})
Convey("BaseInfo no record", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.accdb.QueryRow(context.TODO(), _SelBaseInfo, 1)), "Scan", func(_ *xsql.Row, _ ...interface{}) error {
return sql.ErrNoRows
})
defer monkey.UnpatchAll()
r, err := d.BaseInfo(context.TODO(), 11)
So(err, ShouldBeNil)
So(r, ShouldBeNil)
})
})
}
// TestDao_RecoverMoral update
func TestDao_RecoverMoral(t *testing.T) {
Convey("RecoverMoral", t, func() {
Convey("RecoverMoral success", func() {
_, err := d.RecoverMoral(context.TODO(), 121, 1, 1, time.Now().Format("2006-01-02"))
So(err, ShouldBeNil)
})
Convey("RecoverMoral failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
r, err := d.RecoverMoral(context.TODO(), 121, 1, 1, time.Now().Format("2006-01-02"))
So(err, ShouldNotBeNil)
So(r, ShouldEqual, 0)
})
})
}
func TestDao_SetSign(t *testing.T) {
Convey("SetSign", t, func() {
Convey("SetSign success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.SetSign(context.TODO(), 100, fmt.Sprintf("sign%s", time.Minute.String()))
So(err, ShouldBeNil)
})
Convey("SetSign failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.SetSign(context.TODO(), 100, fmt.Sprintf("sign%s", time.Minute.String()))
So(err, ShouldNotBeNil)
})
})
}
func TestDao_SetOfficial(t *testing.T) {
Convey("SetOfficial", t, func() {
Convey("SetOfficial success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.SetOfficial(context.TODO(), 100, 1, fmt.Sprintf("title%s", time.Minute.String()), "it's a test")
So(err, ShouldBeNil)
})
Convey("SetOfficial failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.SetOfficial(context.TODO(), 100, 1, fmt.Sprintf("title%s", time.Minute.String()), "it's a test")
So(err, ShouldNotBeNil)
})
})
}
func TestDao_SetFace(t *testing.T) {
Convey("SetFace", t, func() {
err := d.SetFace(context.TODO(), 100, fmt.Sprintf("face%s", time.Minute.String()))
So(err, ShouldBeNil)
})
Convey("SetFace", t, func() {
Convey("SetFace success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.SetFace(context.TODO(), 100, fmt.Sprintf("face%s", time.Minute.String()))
So(err, ShouldBeNil)
})
Convey("SetFace failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.SetFace(context.TODO(), 100, fmt.Sprintf("face%s", time.Minute.String()))
So(err, ShouldNotBeNil)
})
})
}
func TestDao_InitBase(t *testing.T) {
Convey("InitBase", t, func() {
Convey("InitBase success", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, nil
})
defer monkey.UnpatchAll()
err := d.InitBase(context.TODO(), 1)
So(err, ShouldBeNil)
})
Convey("InitBase failed", func() {
monkey.PatchInstanceMethod(reflect.TypeOf(d.db), "Exec", func(_ *xsql.DB, _ context.Context, _ string, _ ...interface{}) (sql.Result, error) {
return nil, fmt.Errorf("cannot connect database error")
})
defer monkey.UnpatchAll()
err := d.InitBase(context.TODO(), 1)
So(err, ShouldNotBeNil)
})
})
}

123
app/job/main/member/dao/memcache.go Normal file
View File

@@ -0,0 +1,123 @@
package dao
import (
"context"
"fmt"
"go-common/app/job/main/member/model"
"go-common/library/cache/memcache"
"go-common/library/log"
"strconv"
errors "github.com/pkg/errors"
)
const (
_expPrefix = "exp_%d"
_expExpire = 86400
)
func expKey(mid int64) string {
return fmt.Sprintf(_expPrefix, mid)
}
func (d *Dao) mcBaseKey(mid int64) (key string) {
return fmt.Sprintf(model.CacheKeyBase, mid)
}
func (d *Dao) moralKey(mid int64) (key string) {
return fmt.Sprintf(model.CacheKeyMoral, mid)
}
// SetStartCache set data import start
func (d *Dao) SetStartCache(c context.Context, mid int64, key string) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
if conn.Set(&memcache.Item{
Key: key,
Object: mid,
Flags: memcache.FlagJSON,
Expiration: 0,
}); err != nil {
log.Error("conn.Set(%s, %d) error(%v)", key, mid, err)
}
return
}
// DelMoralCache delete moral cache.
func (d *Dao) DelMoralCache(c context.Context, mid int64) (err error) {
key := d.moralKey(mid)
conn := d.mc.Get(c)
defer conn.Close()
if err = conn.Delete(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
log.Error("conn.Delete(%s) error(%v)", key, err)
}
return
}
// DelBaseInfoCache delete baseInfo cache.
func (d *Dao) DelBaseInfoCache(c context.Context, mid int64) (err error) {
key := d.mcBaseKey(mid)
conn := d.mc.Get(c)
defer conn.Close()
if err = conn.Delete(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
log.Error("conn.Delete(%s) error(%v)", key, err)
}
return
}
// SetExpCache set user exp cache.
func (d *Dao) SetExpCache(c context.Context, mid, exp int64) (err error) {
conn := d.mc.Get(c)
defer conn.Close()
if err = conn.Set(&memcache.Item{
Key: expKey(mid),
Value: []byte(strconv.FormatInt(exp, 10)),
Expiration: _expExpire,
}); err != nil {
log.Error("setexpcache mid %d err %v ", mid, err)
}
return
}
func realnameInfoKey(mid int64) string {
return fmt.Sprintf("realname_info_%d", mid)
}
func realnameApplyStatusKey(mid int64) string {
return fmt.Sprintf("realname_apply_%d", mid)
}
// DeleteRealnameCache delete all realname cache
func (d *Dao) DeleteRealnameCache(c context.Context, mid int64) (err error) {
var (
key1 = realnameInfoKey(mid)
key2 = realnameApplyStatusKey(mid)
conn = d.mc.Get(c)
)
defer conn.Close()
if err = conn.Delete(key1); err != nil {
if err != memcache.ErrNotFound {
err = errors.Wrapf(err, "conn.Delete(%s)", key1)
return
}
err = nil
}
if err = conn.Delete(key2); err != nil {
if err == memcache.ErrNotFound {
err = nil
} else {
err = errors.Wrapf(err, "conn.Delete(%s)", key2)
}
return
}
return
}

155
app/job/main/member/dao/memcache_test.go Normal file
View File

@@ -0,0 +1,155 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoexpKey(t *testing.T) {
convey.Convey("expKey", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
p1 := expKey(mid)
convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
convCtx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestDaomcBaseKey(t *testing.T) {
convey.Convey("mcBaseKey", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
key := d.mcBaseKey(mid)
convCtx.Convey("Then key should not be nil.", func(convCtx convey.C) {
convCtx.So(key, convey.ShouldNotBeNil)
})
})
})
}
func TestDaomoralKey(t *testing.T) {
convey.Convey("moralKey", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
key := d.moralKey(mid)
convCtx.Convey("Then key should not be nil.", func(convCtx convey.C) {
convCtx.So(key, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoSetStartCache(t *testing.T) {
convey.Convey("SetStartCache", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
key = ""
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.SetStartCache(c, mid, key)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoDelMoralCache(t *testing.T) {
convey.Convey("DelMoralCache", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.DelMoralCache(c, mid)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoDelBaseInfoCache(t *testing.T) {
convey.Convey("DelBaseInfoCache", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.DelBaseInfoCache(c, mid)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoSetExpCache(t *testing.T) {
convey.Convey("SetExpCache", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
exp = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.SetExpCache(c, mid, exp)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaorealnameInfoKey(t *testing.T) {
convey.Convey("realnameInfoKey", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
p1 := realnameInfoKey(mid)
convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
convCtx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestDaorealnameApplyStatusKey(t *testing.T) {
convey.Convey("realnameApplyStatusKey", t, func(convCtx convey.C) {
var (
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
p1 := realnameApplyStatusKey(mid)
convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
convCtx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoDeleteRealnameCache(t *testing.T) {
convey.Convey("DeleteRealnameCache", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.DeleteRealnameCache(c, mid)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}

166
app/job/main/member/dao/realname.go Normal file
View File

@@ -0,0 +1,166 @@
package dao
import (
"context"
"net/http"
"net/url"
"time"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/model"
xsql "go-common/library/database/sql"
"go-common/library/log"
"github.com/pkg/errors"
)
var (
_selRealnameAlipayApplyList = "SELECT id,mid,realname,card,img,status,reason,bizno,ctime,mtime FROM realname_alipay_apply WHERE id>? AND status=? AND mtime>=? AND mtime<=? ORDER BY id ASC LIMIT ?"
_selRealnameInfo = `SELECT id,mid,channel,realname,country,card_type,card,card_md5,status,reason,ctime,mtime FROM realname_info WHERE mid = ? LIMIT 1`
_upsertRealnameInfo = `INSERT INTO realname_info (mid,channel,realname,country,card_type,card,card_md5,status,reason) VALUES (?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE channel=?,realname=?,country=?,card_type=?,card=?,card_md5=?,status=?,reason=?`
_upsertRealnameApply = `INSERT INTO realname_apply (id,mid,realname,country,card_type,card_num,card_md5,hand_img,front_img,back_img,status,operator,operator_id,operator_time,remark,remark_status,ctime,mtime) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE realname=?,country=?,card_type=?,card_num=?,card_md5=?,hand_img=?,front_img=?,back_img=?,status=?,operator=?,operator_time=?,remark=?,remark_status=?,mtime=?`
_upsertRealnameApplyIMG = `INSERT INTO realname_apply_img (id,img_data,ctime,mtime) VALUES (?,?,?,?) ON DUPLICATE KEY UPDATE img_data=?,mtime=?`
)
// UpdateRealnameFromMSG is
func (d *Dao) UpdateRealnameFromMSG(c context.Context, ms *model.RealnameApplyMessage) (err error) {
var (
tx *xsql.Tx
cardEncrpted = ms.CardData()
cardMD5 = ms.CardMD5()
)
if tx, err = d.db.Begin(c); err != nil {
return
}
if _, err = tx.Exec(_upsertRealnameApply, ms.ID, ms.MID, ms.Realname, ms.Country(), ms.CardType(), cardEncrpted, cardMD5, ms.FrontIMG, ms.FrontIMG2, ms.BackIMG, ms.Status, ms.Operater, 0, ms.OperaterTime(), ms.Remark, ms.RemarkStatus, ms.ApplyTime(), ms.ApplyTime(), ms.Realname, ms.Country(), ms.CardType(), cardEncrpted, cardMD5, ms.FrontIMG, ms.FrontIMG2, ms.BackIMG, ms.Status, ms.Operater, ms.OperaterTime(), ms.Remark, ms.RemarkStatus, time.Now()); err != nil {
err = errors.WithStack(err)
tx.Rollback()
return
}
if _, err = tx.Exec(_upsertRealnameInfo, ms.MID, 0, ms.Realname, ms.Country(), ms.CardType(), cardEncrpted, cardMD5, ms.Status, ms.Remark, 0, ms.Realname, ms.Country(), ms.CardType(), cardEncrpted, cardMD5, ms.Status, ms.Remark); err != nil {
err = errors.WithStack(err)
tx.Rollback()
return
}
if err = tx.Commit(); err != nil {
return
}
return
}
// RealnameInfo is.
func (d *Dao) RealnameInfo(c context.Context, mid int64) (info *model.RealnameInfo, err error) {
row := d.db.QueryRow(c, _selRealnameInfo, mid)
info = &model.RealnameInfo{}
if err = row.Scan(&info.ID, &info.MID, &info.Channel, &info.Realname, &info.Country, &info.CardType, &info.Card, &info.CardMD5, &info.Status, &info.Reason, &info.CTime, &info.MTime); err != nil {
if err == xsql.ErrNoRows {
err = nil
info = nil
return
}
err = errors.Wrapf(err, "dao RealnameInfo mid(%d)", mid)
return
}
return
}
// UpsertRealnameInfo is
func (d *Dao) UpsertRealnameInfo(c context.Context, ms *model.RealnameInfo) (err error) {
if _, err = d.db.Exec(c, _upsertRealnameInfo, ms.MID, 0, ms.Realname, ms.Country, ms.CardType, ms.Card, ms.CardMD5, ms.Status, ms.Reason, 0, ms.Realname, ms.Country, ms.CardType, ms.Card, ms.CardMD5, ms.Status, ms.Reason); err != nil {
err = errors.WithStack(err)
return
}
return
}
// UpsertRealnameApplyImg is
func (d *Dao) UpsertRealnameApplyImg(c context.Context, ms *model.RealnameApplyImgMessage) (err error) {
if _, err = d.db.Exec(c, _upsertRealnameApplyIMG, ms.ID, ms.IMGData, ms.AddTime(), ms.AddTime(), ms.IMGData, time.Now()); err != nil {
err = errors.WithStack(err)
return
}
return
}
// RealnameAlipayApplyList is
func (d *Dao) RealnameAlipayApplyList(c context.Context, startID int64, status model.RealnameApplyStatus, fromTime, toTime time.Time, limit int) (maxID int64, list []*model.RealnameAlipayApply, err error) {
var (
rows *xsql.Rows
)
if rows, err = d.db.Query(c, _selRealnameAlipayApplyList, startID, status, fromTime, toTime, limit); err != nil {
err = errors.WithStack(err)
return
}
defer rows.Close()
for rows.Next() {
var (
apply = &model.RealnameAlipayApply{}
)
if err = rows.Scan(&apply.ID, &apply.MID, &apply.Realname, &apply.Card, &apply.IMG, &apply.Status, &apply.Reason, &apply.Bizno, &apply.CTime, &apply.MTime); err != nil {
err = errors.WithStack(err)
return
}
if maxID < apply.ID {
maxID = apply.ID
}
list = append(list, apply)
}
if err = rows.Err(); err != nil {
err = errors.WithStack(err)
return
}
return
}
// AlipayQuery .
func (d *Dao) AlipayQuery(c context.Context, param url.Values) (pass bool, reason string, err error) {
var (
req *http.Request
)
url := conf.Conf.Biz.RealnameAlipayGateway + "?" + param.Encode()
if req, err = http.NewRequest("GET", url, nil); err != nil {
err = errors.Wrapf(err, "http.NewRequest(GET,%s)", url)
return
}
var resp struct {
Resp struct {
respAlipay
Passed string `json:"passed"`
FailedReason string `json:"failed_reason"`
IdentityInfo string `json:"identity_info"`
AttributeInfo string `json:"attribute_info"`
ChannelStatuses string `json:"channel_statuses"`
} `json:"zhima_customer_certification_query_response"`
Sign string `json:"sign"`
}
if err = d.client.Do(c, req, &resp); err != nil {
return
}
log.Info("Realname alipay query \n\tparam : %+v \n\tresp : %+v", param, resp)
if err = resp.Resp.Error(); err != nil {
return
}
if resp.Resp.Passed == "true" {
pass = true
} else {
pass = false
}
reason = resp.Resp.FailedReason
return
}
type respAlipay struct {
Code string `json:"code"`
Msg string `json:"msg"`
SubCode string `json:"sub_code"`
SubMsg string `json:"sub_msg"`
}
func (r *respAlipay) Error() error {
if r.Code == "10000" {
return nil
}
return errors.Errorf("alipay response failed , code : %s, msg : %s, sub_code : %s, sub_msg : %s", r.Code, r.Msg, r.SubCode, r.SubMsg)
}

111
app/job/main/member/dao/realname_test.go Normal file
View File

@@ -0,0 +1,111 @@
package dao
import (
"context"
"go-common/app/job/main/member/model"
"net/url"
"testing"
"time"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoUpdateRealnameFromMSG(t *testing.T) {
convey.Convey("UpdateRealnameFromMSG", t, func(convCtx convey.C) {
var (
c = context.Background()
ms = &model.RealnameApplyMessage{}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.UpdateRealnameFromMSG(c, ms)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoRealnameInfo(t *testing.T) {
convey.Convey("RealnameInfo", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
info, err := d.RealnameInfo(c, mid)
convCtx.Convey("Then err should be nil.info should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(info, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoUpsertRealnameInfo(t *testing.T) {
convey.Convey("UpsertRealnameInfo", t, func(convCtx convey.C) {
var (
c = context.Background()
ms = &model.RealnameInfo{}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.UpsertRealnameInfo(c, ms)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoUpsertRealnameApplyImg(t *testing.T) {
convey.Convey("UpsertRealnameApplyImg", t, func(convCtx convey.C) {
var (
c = context.Background()
ms = &model.RealnameApplyImgMessage{}
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
err := d.UpsertRealnameApplyImg(c, ms)
convCtx.Convey("Then err should be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
})
})
})
}
func TestDaoRealnameAlipayApplyList(t *testing.T) {
convey.Convey("RealnameAlipayApplyList", t, func(convCtx convey.C) {
var (
c = context.Background()
startID = int64(0)
status model.RealnameApplyStatus
fromTime = time.Now().AddDate(-1, 0, 0)
toTime = time.Now()
limit = int(10)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
status = 2
maxID, list, err := d.RealnameAlipayApplyList(c, startID, status, fromTime, toTime, limit)
convCtx.Convey("Then err should be nil.maxID,list should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(list, convey.ShouldNotBeNil)
convCtx.So(maxID, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoAlipayQuery(t *testing.T) {
convey.Convey("AlipayQuery", t, func(convCtx convey.C) {
var (
c = context.Background()
param url.Values
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
pass, _, err := d.AlipayQuery(c, param)
convCtx.Convey("Then err should be nil.pass,reason should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldNotBeNil)
//convCtx.So(reason, convey.ShouldBeNil)
convCtx.So(pass, convey.ShouldNotBeNil)
})
})
})
}

66
app/job/main/member/dao/redis.go Normal file
View File

@@ -0,0 +1,66 @@
package dao
import (
"context"
"fmt"
"os"
"time"
"go-common/library/cache/redis"
)
const (
_expShard = 10000
_expAddedPrefix = "ea_%s_%d_%d"
_expire = 86400
)
func expAddedKey(tp string, mid int64, day int) string {
return fmt.Sprintf(_expAddedPrefix, tp, day, mid/_expShard)
}
func leader() (key string, value string) {
value, _ = os.Hostname()
key = fmt.Sprintf("member-job:leader:%d", time.Now().Day())
return key, value
}
// SetExpAdded set user exp added of tp,
func (d *Dao) SetExpAdded(c context.Context, mid int64, day int, tp string) (b bool, err error) {
conn := d.redis.Get(c)
defer conn.Close()
if err = conn.Send("SETBIT", expAddedKey(tp, mid, day), mid%_expShard, 1); err != nil {
return
}
if err = conn.Send("EXPIRE", expAddedKey(tp, mid, day), _expire); err != nil {
return
}
if err = conn.Flush(); err != nil {
return
}
if b, err = redis.Bool(conn.Receive()); err != nil {
return
}
conn.Receive()
return
}
// ExpAdded check if user had add exp.
func (d *Dao) ExpAdded(c context.Context, mid int64, day int, tp string) (b bool, err error) {
conn := d.redis.Get(c)
b, err = redis.Bool(conn.Do("GETBIT", expAddedKey(tp, mid, day), mid%_expShard))
conn.Close()
return
}
// LeaderEleciton eleciton job leader.
func (d *Dao) LeaderEleciton(c context.Context) (elected bool) {
key, value := leader()
conn := d.redis.Get(c)
elected, _ = redis.Bool(conn.Do("SETNX", key, value))
if elected {
conn.Do("EXPIRE", key, 600)
}
conn.Close()
return
}

86
app/job/main/member/dao/redis_test.go Normal file
View File

@@ -0,0 +1,86 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoexpAddedKey(t *testing.T) {
convey.Convey("expAddedKey", t, func(convCtx convey.C) {
var (
tp = ""
mid = int64(0)
day = int(0)
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
p1 := expAddedKey(tp, mid, day)
convCtx.Convey("Then p1 should not be nil.", func(convCtx convey.C) {
convCtx.So(p1, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoleader(t *testing.T) {
convey.Convey("leader", t, func(convCtx convey.C) {
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
key, value := leader()
convCtx.Convey("Then key,value should not be nil.", func(convCtx convey.C) {
convCtx.So(value, convey.ShouldNotBeNil)
convCtx.So(key, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoSetExpAdded(t *testing.T) {
convey.Convey("SetExpAdded", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
day = int(0)
tp = ""
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
b, err := d.SetExpAdded(c, mid, day, tp)
convCtx.Convey("Then err should be nil.b should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(b, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoExpAdded(t *testing.T) {
convey.Convey("ExpAdded", t, func(convCtx convey.C) {
var (
c = context.Background()
mid = int64(0)
day = int(0)
tp = ""
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
b, err := d.ExpAdded(c, mid, day, tp)
convCtx.Convey("Then err should be nil.b should not be nil.", func(convCtx convey.C) {
convCtx.So(err, convey.ShouldBeNil)
convCtx.So(b, convey.ShouldNotBeNil)
})
})
})
}
func TestDaoLeaderEleciton(t *testing.T) {
convey.Convey("LeaderEleciton", t, func(convCtx convey.C) {
var (
c = context.Background()
)
convCtx.Convey("When everything goes positive", func(convCtx convey.C) {
elected := d.LeaderEleciton(c)
convCtx.Convey("Then elected should not be nil.", func(convCtx convey.C) {
convCtx.So(elected, convey.ShouldNotBeNil)
})
})
})
}

33
app/job/main/member/http/BUILD Normal file
View File

@@ -0,0 +1,33 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["http.go"],
importpath = "go-common/app/job/main/member/http",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

54
app/job/main/member/http/http.go Normal file
View File

@@ -0,0 +1,54 @@
package http
import (
"net/http"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
var srv *service.Service
// Init init http service
func Init(c *conf.Config, s *service.Service) {
srv = s
// init inner router
// intM := http.NewServeMux()
// intR := router.New(intM)
// innerRouter(intR)
engine := bm.DefaultServer(c.BM)
setupInnerEngine(engine)
if err := engine.Start(); err != nil {
panic(err)
}
// init inner server
// if err := xhttp.Serve(intM, c.MultiHTTP.Inner); err != nil {
// log.Error("xhttp.Serve error(%v)", err)
// panic(err)
// }
}
// innerRouter init local router api path.
// func innerRouter(r *router.Router) {
// r.MonitorPing(ping)
// }
func setupInnerEngine(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
}
func ping(c *bm.Context) {
if err := srv.Ping(c); err != nil {
log.Error("ping error(%v)", err)
c.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(ctx *bm.Context) {
ctx.JSON(nil, nil)
}

89
app/job/main/member/model/BUILD Normal file
View File

@@ -0,0 +1,89 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
proto_library(
name = "model_proto",
srcs = ["member.proto"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
)
go_proto_library(
name = "model_go_proto",
compilers = ["@io_bazel_rules_go//proto:gogofast_proto"],
importpath = "go-common/app/job/main/member/model",
proto = ":model_proto",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/time:go_default_library",
"@com_github_gogo_protobuf//gogoproto:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["model_test.go"],
embed = [
":go_default_library",
],
rundir = ".",
tags = ["automanaged"],
deps = [
"//library/time:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"base.go",
"model.go",
"moral.go",
"notify.go",
"official.go",
"old_account.go",
"realname.go",
],
embed = [":model_go_proto"],
importpath = "go-common/app/job/main/member/model",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//library/log:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/github.com/satori/go.uuid:go_default_library",
"@com_github_gogo_protobuf//gogoproto:go_default_library",
"@com_github_gogo_protobuf//proto:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/member/model/block:all-srcs",
"//app/job/main/member/model/queue:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

30
app/job/main/member/model/base.go Normal file
View File

@@ -0,0 +1,30 @@
package model
import (
"fmt"
"math/rand"
)
const (
//CacheKeyBase is.
CacheKeyBase = "bs_%d" // key of baseInfo
//CacheKeyMoral is.
CacheKeyMoral = "moral_%d" // key of detail
//CacheKeyInfo is.
CacheKeyInfo = "i_"
//URLNoFace is.
URLNoFace = "http://static.hdslb.com/images/member/noface.gif"
//TableExpLog is.
TableExpLog = "ugc:ExpLog"
//TableMoralLog is.
TableMoralLog = "ugc:MoralLog"
)
// RandFaceURL get face URL
func (b *BaseInfo) RandFaceURL() {
if b.Face == "" {
b.Face = URLNoFace
return
}
b.Face = fmt.Sprintf("http://i%d.hdslb.com%s", rand.Int63n(3), b.Face)
}

34
app/job/main/member/model/block/BUILD Normal file
View File

@@ -0,0 +1,34 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"blackhouse.go",
"dao.go",
"model.go",
"msg.go",
"notify.go",
],
importpath = "go-common/app/job/main/member/model/block",
tags = ["automanaged"],
deps = ["//library/time:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

11
app/job/main/member/model/block/blackhouse.go Normal file
View File

@@ -0,0 +1,11 @@
package block
import (
xtime "go-common/library/time"
)
// CreditAnswerMSG param struct
type CreditAnswerMSG struct {
MID int64 `json:"mid"`
MTime xtime.Time `json:"mtime"`
}

57
app/job/main/member/model/block/dao.go Normal file
View File

@@ -0,0 +1,57 @@
package block
import (
"time"
)
// DBUser .
type DBUser struct {
ID int64
MID int64
Status BlockStatus
CTime time.Time
MTime time.Time
}
// DBUserDetail .
type DBUserDetail struct {
ID int64
MID int64
BlockCount int
CTime time.Time
MTime time.Time
}
// DBHistory .
type DBHistory struct {
ID int64
MID int64
AdminID int64
AdminName string
Source BlockSource
Area BlockArea
Reason string
Comment string
Action BlockAction
StartTime time.Time
Duration int64
Notify bool
CTime time.Time
MTime time.Time
}
// DBExtra .
type DBExtra struct {
ID int64
MID int64
CreditAnswerFlag bool
ActionTime time.Time
CTime time.Time
MTime time.Time
}
// MCBlockInfo .
type MCBlockInfo struct {
// BlockStatus is.
BlockStatus BlockStatus `json:"bs"`
}

184
app/job/main/member/model/block/model.go Normal file
View File

@@ -0,0 +1,184 @@
package block
import (
"time"
)
// BlockStatus 封禁状态 0. 未封禁 1. 永久封禁 2. 限时封禁
type BlockStatus uint8
const (
// BlockStatusFalse 未封禁
BlockStatusFalse BlockStatus = iota
// BlockStatusForever 永久封禁
BlockStatusForever
// BlockStatusLimit 限时封禁
BlockStatusLimit
// BlockStatusCredit 小黑屋封禁
BlockStatusCredit
)
// BlockSource 封禁来源 1. 小黑屋(小黑屋和manager后台封禁) 2. 系统封禁(反作弊及监控系统上报) 3.解封 (所有后台，用户前台自助的解封)
type BlockSource uint8
const (
// BlockSourceCredit 小黑屋封禁
BlockSourceCredit BlockSource = iota + 1
// BlockSourceSys 系统封禁
BlockSourceSys
// BlockSourceRemove 解封
BlockSourceRemove
)
// String .
func (b BlockSource) String() string {
switch b {
case BlockSourceCredit:
return "小黑屋封禁"
case BlockSourceSys:
return "系统封禁"
case BlockSourceRemove:
return "解封"
default:
return ""
}
}
const (
// BlockLogBizID 用户审核日志
BlockLogBizID = 122
// BlockJOBManagerID blockjob manager uid
BlockJOBManagerID = 1493
// BlockJOBManagerName blockjob manager name
BlockJOBManagerName = "blockjob"
)
// BlockArea 封禁业务
type BlockArea uint8
// const .
const (
BlockAreaNone BlockArea = iota
BlockAreaReply
BlockAreaDanmaku
BlockAreaMessage
BlockAreaTag
BlockAreaProfile // 个人资料
BlockAreaArchive
BlockAreaMusic
BlockAreaArticle
BlockAreaSpaceBanner // 空间头图
BlockAreaDynamic // 动态
BlockAreaAlbum // 相册
BlockAreaQuickVideo //小视频
)
// String .
func (b BlockArea) String() string {
switch b {
case BlockAreaReply:
return "评论"
case BlockAreaDanmaku:
return "弹幕"
case BlockAreaMessage:
return "私信"
case BlockAreaTag:
return "标签"
case BlockAreaProfile:
return "个人资料"
case BlockAreaArchive:
return "投稿"
case BlockAreaMusic:
return "音频"
case BlockAreaArticle:
return "专栏"
case BlockAreaSpaceBanner:
return "空间头图"
case BlockAreaDynamic:
return "动态"
case BlockAreaAlbum:
return "相册"
case BlockAreaQuickVideo:
return "小视频"
default:
return ""
}
}
// BlockAction .
type BlockAction uint8
const (
// BlockActionLimit 限时封禁
BlockActionLimit BlockAction = iota + 1
// BlockActionForever 永久封禁
BlockActionForever
// BlockActionAdminRemove 后台解封
BlockActionAdminRemove
// BlockActionSelfRemove 自助解封
BlockActionSelfRemove
)
// String .
func (b BlockAction) String() string {
switch b {
case BlockActionLimit:
return "限时封禁"
case BlockActionForever:
return "永久封禁"
case BlockActionAdminRemove:
return "后台解封"
case BlockActionSelfRemove:
return "自动解封"
default:
return ""
}
}
// BlockInfo 封禁信息
type BlockInfo struct {
MID int64 `json:"mid"`
BlockStatus BlockStatus `json:"block_status"` // blockStatus 封禁状态 0. 未封禁 1. 永久封禁 2. 限时封禁
// EndTime int64 `json:"end_time"` // 结束封禁时间 unix time 永久封禁为0
}
// ParseDB .
func (b *BlockInfo) ParseDB(data *DBUser) {
b.MID = data.MID
b.BlockStatus = data.Status
}
// ParseMC .
func (b *BlockInfo) ParseMC(data *MCBlockInfo, mid int64) {
b.MID = mid
b.BlockStatus = data.BlockStatus
}
// BlockHistory 封禁历史
type BlockHistory struct {
Area BlockArea `json:"type"`
Operator string `json:"operator"` // 操作人
Reason string `json:"reason"` // 封禁原因
Action BlockAction `json:"action"` // 操作类型
ActionTime int64 `json:"action_time"` // 操作时间
RemoveTime int64 `json:"remove_time"` // 解封时间
Comment string `json:"comment"`
}
// ParseDB .
func (b *BlockHistory) ParseDB(data *DBHistory) {
b.Area = data.Area
b.Operator = data.AdminName
b.Reason = data.Reason
b.Action = data.Action
b.ActionTime = data.StartTime.Unix()
b.RemoveTime = data.StartTime.Add(time.Second * time.Duration(data.Duration)).Unix()
b.Comment = data.Comment
}
// BlockMessage 通知消息体
type BlockMessage struct {
MID int64 `json:"mid"` // 用户mid
Area BlockArea `json:"area"` // BlockArea 封禁类型 1. 小黑屋(小黑屋和manager后台封禁) 2. 系统封禁(反作弊及监控系统上报) 3.解封 (所有后台，用户前台自助的解封)
Status BlockStatus `json:"status"` // blockStatus 封禁状态 0. 未封禁 1. 永久封禁 2. 限时封禁
}

8
app/job/main/member/model/block/msg.go Normal file
View File

@@ -0,0 +1,8 @@
package block
// MSG .
type MSG struct {
Code string
Title string
Content string
}

8
app/job/main/member/model/block/notify.go Normal file
View File

@@ -0,0 +1,8 @@
package block
// AccountNotify .
type AccountNotify struct {
UID int64 `json:"mid"`
Type string `json:"type"`
Action string `json:"action"`
}

2249
app/job/main/member/model/member.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

50
app/job/main/member/model/member.proto Normal file
View File

@@ -0,0 +1,50 @@
syntax = "proto3";
package model;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
message LevelInfo {
int32 Cur = 1 [(gogoproto.jsontag) = "current_level"];
int32 Min = 2 [(gogoproto.jsontag) = "current_min"];
int32 NowExp = 3 [(gogoproto.jsontag) = "current_exp,omitempty"];
int32 NextExp = 4 [(gogoproto.jsontag) = "next_exp"];
}
message PendantInfo {
int32 Pid = 1 [(gogoproto.jsontag) = "pid",(gogoproto.casttype) = "int"];
string Name = 2 [(gogoproto.jsontag) = "name"];
string Image = 3 [(gogoproto.jsontag) = "image"];
int64 Expire = 4 [(gogoproto.jsontag) = "expire",(gogoproto.casttype) = "int"];
}
message NameplateInfo {
int32 Nid = 1 [(gogoproto.jsontag) = "nid",(gogoproto.casttype) = "int"];
string Name = 2 [(gogoproto.jsontag) = "name"];
string Image = 3 [(gogoproto.jsontag) = "image"];
string ImageSmall = 4 [(gogoproto.jsontag) = "image_small"];
string Level = 5 [(gogoproto.jsontag) = "level"];
string Condition = 6 [(gogoproto.jsontag) = "condition"];
}
message OfficialInfo {
int32 Type = 7 [(gogoproto.jsontag) = "type",(gogoproto.casttype) = "int8"];
string Desc = 8 [(gogoproto.jsontag) = "desc"];
}
message BaseInfo {
int64 Mid = 1 [(gogoproto.jsontag) = "mid"];
string Name = 2 [(gogoproto.jsontag) = "name"];
int64 Sex = 3 [(gogoproto.jsontag) = "sex"];
string Face = 4 [(gogoproto.jsontag) = "face"];
string Sign = 5 [(gogoproto.jsontag) = "sign"];
int64 Rank = 8 [(gogoproto.jsontag) = "rank"];
int64 Birthday = 9 [(gogoproto.jsontag) = "birthday", (gogoproto.casttype) = "go-common/library/time.Time"];
int64 CTime = 10 [(gogoproto.jsontag) = "ctime", (gogoproto.casttype) = "go-common/library/time.Time"];
int64 MTime = 11 [(gogoproto.jsontag) = "mtime", (gogoproto.casttype) = "go-common/library/time.Time"];
}
message Detail {
int64 Mid = 1 [(gogoproto.jsontag) = "mid"];
int64 Birthday = 2 [(gogoproto.jsontag) = "birthday", (gogoproto.casttype) = "go-common/library/time.Time"];
int64 Place = 3 [(gogoproto.jsontag) = "place"];
int64 Marital = 4 [(gogoproto.jsontag) = "marital"];
int64 Dating = 5 [(gogoproto.jsontag) = "dating"];
string Tags = 6 [(gogoproto.jsontag) = "tags"];
int64 CTime = 7 [(gogoproto.jsontag) = "ctime", (gogoproto.casttype) = "go-common/library/time.Time"];
int64 MTime = 8 [(gogoproto.jsontag) = "mtime", (gogoproto.casttype) = "go-common/library/time.Time"];
}

200
app/job/main/member/model/model.go Normal file
View File

@@ -0,0 +1,200 @@
package model
import (
"encoding/json"
"fmt"
"time"
xtime "go-common/library/time"
"github.com/satori/go.uuid"
)
const (
// FlagNo 否
FlagNo = int32(0)
// FlagYes 是
FlagYes = int32(1)
// ExpFlagOnLogin 每日登录经验
ExpFlagOnLogin = int32(1)
// ExpFlagOnShare 每日分享经验
ExpFlagOnShare = int32(2)
// ExpFlagOnView 每日播放经验
ExpFlagOnView = int32(4)
// ExpFlagOnEmail 一次性绑定邮箱
ExpFlagOnEmail = int32(8)
// ExpFlagOnPhone 一次性绑定手机
ExpFlagOnPhone = int32(16)
// ExpFlagOnSafe 一次性绑定密保
ExpFlagOnSafe = int32(32)
// ExpFlagOnIdentify 一次性实名认证
ExpFlagOnIdentify = int32(64)
// ExpActOnCoin 投币奖励动作
ExpActOnCoin = int64(1)
// ExpActOnLogin 登录奖励动作
ExpActOnLogin = int64(2)
// ExpActOnView 播放奖励动作
ExpActOnView = int64(3)
// ExpActOnShare 分享奖励动作
ExpActOnShare = int64(4)
// ExpActOnEmail 绑定邮箱动作
ExpActOnEmail = int64(5)
// ExpActOnPhone 绑定手机动作
ExpActOnPhone = int64(6)
// ExpActOnSafe 绑定密保动作
ExpActOnSafe = int64(7)
// ExpActOnIdentify 实名认证动作
ExpActOnIdentify = int64(8)
)
var (
login = &ExpOper{ExpFlagOnLogin, 5, "login", "登录奖励"}
share = &ExpOper{ExpFlagOnShare, 5, "shareClick", "分享视频奖励"}
view = &ExpOper{ExpFlagOnView, 5, "watch", "观看视频奖励"}
email = &ExpOper{ExpFlagOnEmail, 20, "bindEmail", "绑定邮箱奖励"}
safe = &ExpOper{ExpFlagOnSafe, 30, "pwdPro", "设置密保奖励"}
phone = &ExpOper{ExpFlagOnPhone, 100, "bindPhone", "绑定手机奖励"}
identify = &ExpOper{ExpFlagOnIdentify, 50, "realIdentity", "实名认证奖励"}
// ExpFlagOper exp flag map for oper
ExpFlagOper = map[string]*ExpOper{"login": login, "share": share, "view": view, "email": email, "phone": phone, "safe": safe, "identify": identify}
)
// Exp userexp for mysql scan.
type Exp struct {
Mid int64 `json:"mid"`
Exp float32 `json:"exp"`
Mtime xtime.Time `json:"modify_time"`
}
// ExpLog user exp log for mysql
type ExpLog struct {
Mid int64 `json:"mid"`
FromExp float32 `json:"from_exp"`
ToExp float32 `json:"to_exp"`
Operater string `json:"operater"`
Reason string `json:"reason"`
Action int64 `json:"actin_id"`
Mtime xtime.Time `json:"modify_time"`
}
// NewExp userexp for mysql scan.
type NewExp struct {
Mid int64 `json:"mid"`
Exp int64 `json:"exp"`
Flag int32 `json:"flag"`
Addtime xtime.Time `json:"addtime"`
Mtime xtime.Time `json:"mtime"`
}
// ExpOper exp operation
type ExpOper struct {
Flag int32
Count int64
Oper string
Reason string
}
// Message binlog databus msg.
type Message struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
}
//Flag is.
type Flag struct {
Mid int64 `json:"mid,omitempty"`
Flag int64 `json:"finish_action"`
Exp int64 `json:"modify_exp"`
}
// UserLog user log.
type UserLog struct {
Mid int64 `json:"mid"`
IP string `json:"ip"`
TS int64 `json:"ts"`
LogID string `json:"log_id"`
Content map[string]string `json:"content"`
}
// AddExp databus add exp arg.
type AddExp struct {
Event string `json:"event,omitempty"`
Mid int64 `json:"mid,omitempty"`
IP string `json:"ip,omitempty"`
Ts int64 `json:"ts,omitempty"`
}
// LoginLog user login log.
type LoginLog struct {
Mid int64 `json:"mid,omitempty"`
Loginip int64 `json:"loginip,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
}
// LoginLogIPString user login log message with string ip.
type LoginLogIPString struct {
Mid int64 `json:"mid,omitempty"`
Loginip string `json:"loginip,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
}
// FaceCheckRes face check result.
type FaceCheckRes struct {
FileName string `json:"file_name,omitempty"`
Bucket string `json:"bucket,omitempty"`
Sex float64 `json:"sex,omitempty"`
Politics float64 `json:"politics,omitempty"`
}
// FacePath is
func (fcr *FaceCheckRes) FacePath() string {
return fmt.Sprintf("/bfs/%s/%s", fcr.Bucket, fcr.FileName)
}
//String is.
func (fcr *FaceCheckRes) String() string {
return fmt.Sprintf("Sex: %.4f, Politics: %.4f", fcr.Sex, fcr.Politics)
}
// MemberMid member mid
type MemberMid struct {
Mid int64 `json:"mid"`
}
// MemberAso member aso
type MemberAso struct {
Email string `json:"email"`
Telphone string `json:"telphone"`
SafeQs int8 `json:"safe_question"`
Spacesta int8 `json:"spacesta"`
}
// ExpMessage exp msg
type ExpMessage struct {
Mid int64 `json:"mid"`
Exp int64 `json:"exp"`
}
// FlagDailyReset reset daily flag with ts.
func (e *NewExp) FlagDailyReset(now time.Time) {
e.Flag = e.Flag & ^0x7
e.Addtime = RestrictDate(xtime.Time(now.Unix()))
}
// RestrictDate restric user brithday
func RestrictDate(xt xtime.Time) xtime.Time {
t := xt.Time()
year := t.Year()
if year < 0 {
year = 0
}
return xtime.Time(time.Date(year, t.Month(), t.Day(), 0, 0, 0, 0, t.Location()).Unix())
}
// UUID4 is generate uuid
func UUID4() string {
return uuid.NewV4().String()
}

42
app/job/main/member/model/model_test.go Normal file
View File

@@ -0,0 +1,42 @@
package model
import (
xtime "go-common/library/time"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestRestrictDate(t *testing.T) {
xt1 := xtime.Time(-62169580800)
t1 := xt1.Time()
assert.True(t, t1.Year() < 0)
xt2 := RestrictDate(xt1)
t2 := xt2.Time()
assert.Equal(t, 0, t2.Year())
assert.Equal(t, t1.Month(), t2.Month())
assert.Equal(t, t1.Day(), t2.Day())
}
func TestRealname(t *testing.T) {
realnameIMG := RealnameApplyImgMessage{
AddTimeStr: "2018-05-11 15:07:00",
AddTimeDB: time.Now(),
}
t.Log("addtime", realnameIMG.AddTime().Local())
realnameApply := RealnameApplyMessage{
CardDataCanal: "dGMyNUhTcXN1NS9MYkF1bGdoKzNkTGs0eHI5UlM4SS9NY2VQa25xV3czU2grY1Q3R0JiSjlaQWhPZ294TTlEQVV0ZFhuYzIrUnlXTVN3NWk5THFWdkpmTEJaYXJhSlFHWUQ3bVlWZ3liNU1IS1hQTXZ0RE9pR1d6UnpYcUtRUEFYTHZCcXIzZVFoVURwT3VieXJzV0c1Z0dJS2dQNEdYbUV0T1B3MXV6bEE4PQ==",
}
t.Log("card data", realnameApply.CardData())
var (
card = "340702199110120012"
)
md5 := cardMD5(card, 0, 0)
t.Log("card md5", md5)
}

19
app/job/main/member/model/moral.go Normal file
View File

@@ -0,0 +1,19 @@
package model
import (
"go-common/library/time"
)
//MoralLog is.
type MoralLog struct {
Mid int64 `json:"mid"`
IP int64 `json:"ip"`
Operater string `json:"operater"`
Origin int8 `json:"origin"`
Reason string `json:"reason"`
Remark string `json:"remark"`
Status int8 `json:"status"`
FromMoral int64 `json:"from_moral"`
ToMoral int64 `json:"to_moral"`
ModifyTime time.Time `json:"modify_time"`
}

19
app/job/main/member/model/notify.go Normal file
View File

@@ -0,0 +1,19 @@
package model
// consts
const (
ActUpdateExp = "updateExp"
ActUpdateLevel = "updateLevel"
ActUpdateFace = "updateFace"
ActUpdateMoral = "updateMoral"
ActUpdateUname = "updateUname"
)
// NotifyInfo notify info.
type NotifyInfo struct {
Uname string `json:"uname"`
Mid int64 `json:"mid"`
Type string `json:"type"`
NewName string `json:"newName"`
Action string `json:"action"`
}

37
app/job/main/member/model/official.go Normal file
View File

@@ -0,0 +1,37 @@
package model
// official role const.
const (
OfficialRoleUnauth = iota
OfficialRoleUp
OfficialRoleIdentify
OfficialRoleBusiness
OfficialRoleGov
OfficialRoleMedia
OfficialRoleOther
)
// Official is.
type Official struct {
Role int8 `json:"role"`
Title string `json:"title"`
Desc string `json:"desc"`
}
// FromCert is.
func FromCert(v *MemberVerify) Official {
of := Official{}
switch v.Type {
case -1:
of.Role = 0
of.Title = ""
case 0:
of.Role = 2
of.Title = v.Desc
case 1:
of.Role = 3
of.Title = v.Desc
}
of.Desc = ""
return of
}

21
app/job/main/member/model/old_account.go Normal file
View File

@@ -0,0 +1,21 @@
package model
import (
"go-common/library/time"
)
//MemberVerify is.
type MemberVerify struct {
Mid int64 `json:"mid"`
Type int64 `json:"type"`
Desc string `json:"desc"`
}
//AccountInfo is.
type AccountInfo struct {
Mid int64 `json:"mid"`
Name string `json:"name"`
Cert int64 `json:"cert"`
CertDesc string `json:"certdesc"`
Ts time.Time `json:"ts"`
}

7
app/job/main/member/model/pro.sh Normal file
View File

@@ -0,0 +1,7 @@
#! /bin/sh
# proto.sh
gopath=$GOPATH/src
gogopath=$GOPATH/src/go-common/vendor/github.com/gogo/protobuf
protoc --gofast_out=. --proto_path=$gopath:$gogopath:. *.proto

33
app/job/main/member/model/queue/BUILD Normal file
View File

@@ -0,0 +1,33 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"error.go",
"priority_queue.go",
"queue.go",
"ring.go",
],
importpath = "go-common/app/job/main/member/model/queue",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

32
app/job/main/member/model/queue/error.go Normal file
View File

@@ -0,0 +1,32 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package queue
import "errors"
var (
// ErrDisposed is returned when an operation is performed on a disposed
// queue.
ErrDisposed = errors.New(`queue: disposed`)
// ErrTimeout is returned when an applicable queue operation times out.
ErrTimeout = errors.New(`queue: poll timed out`)
// ErrEmptyQueue is returned when an non-applicable queue operation was called
// due to the queue's empty item state
ErrEmptyQueue = errors.New(`queue: empty queue`)
)

267
app/job/main/member/model/queue/priority_queue.go Normal file
View File

@@ -0,0 +1,267 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
The priority queue is almost a spitting image of the logic
used for a regular queue. In order to keep the logic fast,
this code is repeated instead of using casts to cast to interface{}
back and forth. If Go had inheritance and generics, this problem
would be easier to solve.
*/
package queue
import "sync"
// Item is an item that can be added to the priority queue.
type Item interface {
// Compare returns a bool that can be used to determine
// ordering in the priority queue. Assuming the queue
// is in ascending order, this should return > logic.
// Return 1 to indicate this object is greater than the
// the other logic, 0 to indicate equality, and -1 to indicate
// less than other.
Compare(other Item) int
HashCode() int64
}
type priorityItems []Item
func (items *priorityItems) swap(i, j int) {
(*items)[i], (*items)[j] = (*items)[j], (*items)[i]
}
func (items *priorityItems) pop() Item {
size := len(*items)
// Move last leaf to root, and 'pop' the last item.
items.swap(size-1, 0)
item := (*items)[size-1] // Item to return.
(*items)[size-1], *items = nil, (*items)[:size-1]
// 'Bubble down' to restore heap property.
index := 0
childL, childR := 2*index+1, 2*index+2
for len(*items) > childL {
child := childL
if len(*items) > childR && (*items)[childR].Compare((*items)[childL]) < 0 {
child = childR
}
if (*items)[child].Compare((*items)[index]) < 0 {
items.swap(index, child)
index = child
childL, childR = 2*index+1, 2*index+2
} else {
break
}
}
return item
}
func (items *priorityItems) get(number int) []Item {
returnItems := make([]Item, 0, number)
for i := 0; i < number; i++ {
if len(*items) == 0 {
break
}
returnItems = append(returnItems, items.pop())
}
return returnItems
}
func (items *priorityItems) push(item Item) {
// Stick the item as the end of the last level.
*items = append(*items, item)
// 'Bubble up' to restore heap property.
index := len(*items) - 1
parent := int((index - 1) / 2)
for parent >= 0 && (*items)[parent].Compare(item) > 0 {
items.swap(index, parent)
index = parent
parent = int((index - 1) / 2)
}
}
// PriorityQueue is similar to queue except that it takes
// items that implement the Item interface and adds them
// to the queue in priority order.
type PriorityQueue struct {
waiters waiters
items priorityItems
itemMap map[int64]struct{}
lock sync.Mutex
disposeLock sync.Mutex
disposed bool
allowDuplicates bool
}
// Put adds items to the queue.
func (pq *PriorityQueue) Put(items ...Item) error {
if len(items) == 0 {
return nil
}
pq.lock.Lock()
defer pq.lock.Unlock()
if pq.disposed {
return ErrDisposed
}
for _, item := range items {
if pq.allowDuplicates {
pq.items.push(item)
} else if _, ok := pq.itemMap[item.HashCode()]; !ok {
pq.itemMap[item.HashCode()] = struct{}{}
pq.items.push(item)
}
}
for {
sema := pq.waiters.get()
if sema == nil {
break
}
sema.response.Add(1)
sema.ready <- true
sema.response.Wait()
if len(pq.items) == 0 {
break
}
}
return nil
}
// Get retrieves items from the queue. If the queue is empty,
// this call blocks until the next item is added to the queue. This
// will attempt to retrieve number of items.
func (pq *PriorityQueue) Get(number int) ([]Item, error) {
if number < 1 {
return nil, nil
}
pq.lock.Lock()
if pq.disposed {
pq.lock.Unlock()
return nil, ErrDisposed
}
var items []Item
// Remove references to popped items.
deleteItems := func(items []Item) {
for _, item := range items {
delete(pq.itemMap, item.HashCode())
}
}
if len(pq.items) == 0 {
sema := newSema()
pq.waiters.put(sema)
pq.lock.Unlock()
<-sema.ready
if pq.Disposed() {
return nil, ErrDisposed
}
items = pq.items.get(number)
if !pq.allowDuplicates {
deleteItems(items)
}
sema.response.Done()
return items, nil
}
items = pq.items.get(number)
deleteItems(items)
pq.lock.Unlock()
return items, nil
}
// Peek will look at the next item without removing it from the queue.
func (pq *PriorityQueue) Peek() Item {
pq.lock.Lock()
defer pq.lock.Unlock()
if len(pq.items) > 0 {
return pq.items[0]
}
return nil
}
// Empty returns a bool indicating if there are any items left
// in the queue.
func (pq *PriorityQueue) Empty() bool {
pq.lock.Lock()
defer pq.lock.Unlock()
return len(pq.items) == 0
}
// Len returns a number indicating how many items are in the queue.
func (pq *PriorityQueue) Len() int {
pq.lock.Lock()
defer pq.lock.Unlock()
return len(pq.items)
}
// Disposed returns a bool indicating if this queue has been disposed.
func (pq *PriorityQueue) Disposed() bool {
pq.disposeLock.Lock()
defer pq.disposeLock.Unlock()
return pq.disposed
}
// Dispose will prevent any further reads/writes to this queue
// and frees available resources.
func (pq *PriorityQueue) Dispose() {
pq.lock.Lock()
defer pq.lock.Unlock()
pq.disposeLock.Lock()
defer pq.disposeLock.Unlock()
pq.disposed = true
for _, waiter := range pq.waiters {
waiter.response.Add(1)
waiter.ready <- true
}
pq.items = nil
pq.waiters = nil
}
// NewPriorityQueue is the constructor for a priority queue.
func NewPriorityQueue(hint int, allowDuplicates bool) *PriorityQueue {
return &PriorityQueue{
items: make(priorityItems, 0, hint),
itemMap: make(map[int64]struct{}, hint),
allowDuplicates: allowDuplicates,
}
}

411
app/job/main/member/model/queue/queue.go Normal file
View File

@@ -0,0 +1,411 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Package queue includes a regular queue and a priority queue.
These queues rely on waitgroups to pause listening threads
on empty queues until a message is received. If any thread
calls Dispose on the queue, any listeners are immediately returned
with an error. Any subsequent put to the queue will return an error
as opposed to panicking as with channels. Queues will grow with unbounded
behavior as opposed to channels which can be buffered but will pause
while a thread attempts to put to a full channel.
Recently added is a lockless ring buffer using the same basic C design as
found here:
http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
Modified for use with Go with the addition of some dispose semantics providing
the capability to release blocked threads. This works for both puts
and gets, either will return an error if they are blocked and the buffer
is disposed. This could serve as a signal to kill a goroutine. All threadsafety
is acheived using CAS operations, making this buffer pretty quick.
Benchmarks:
BenchmarkPriorityQueue-8 2000000 782 ns/op
BenchmarkQueue-8 2000000 671 ns/op
BenchmarkChannel-8 1000000 2083 ns/op
BenchmarkQueuePut-8 20000 84299 ns/op
BenchmarkQueueGet-8 20000 80753 ns/op
BenchmarkExecuteInParallel-8 20000 68891 ns/op
BenchmarkRBLifeCycle-8 10000000 177 ns/op
BenchmarkRBPut-8 30000000 58.1 ns/op
BenchmarkRBGet-8 50000000 26.8 ns/op
TODO: We really need a Fibonacci heap for the priority queue.
TODO: Unify the types of queue to the same interface.
*/
package queue
import (
"runtime"
"sync"
"sync/atomic"
"time"
)
type waiters []*sema
func (w *waiters) get() *sema {
if len(*w) == 0 {
return nil
}
sema := (*w)[0]
copy((*w)[0:], (*w)[1:])
(*w)[len(*w)-1] = nil // or the zero value of T
*w = (*w)[:len(*w)-1]
return sema
}
func (w *waiters) put(sema *sema) {
*w = append(*w, sema)
}
func (w *waiters) remove(sema *sema) {
if len(*w) == 0 {
return
}
// build new slice, copy all except sema
ws := *w
newWs := make(waiters, 0, len(*w))
for i := range ws {
if ws[i] != sema {
newWs = append(newWs, ws[i])
}
}
*w = newWs
}
type items []interface{}
func (items *items) get(number int64) []interface{} {
returnItems := make([]interface{}, 0, number)
index := int64(0)
for i := int64(0); i < number; i++ {
if i >= int64(len(*items)) {
break
}
returnItems = append(returnItems, (*items)[i])
(*items)[i] = nil
index++
}
*items = (*items)[index:]
return returnItems
}
func (items *items) peek() (interface{}, bool) {
length := len(*items)
if length == 0 {
return nil, false
}
return (*items)[0], true
}
func (items *items) getUntil(checker func(item interface{}) bool) []interface{} {
length := len(*items)
if len(*items) == 0 {
// returning nil here actually wraps that nil in a list
// of interfaces... thanks go
return []interface{}{}
}
returnItems := make([]interface{}, 0, length)
index := -1
for i, item := range *items {
if !checker(item) {
break
}
returnItems = append(returnItems, item)
index = i
(*items)[i] = nil // prevent memory leak
}
*items = (*items)[index+1:]
return returnItems
}
type sema struct {
ready chan bool
response *sync.WaitGroup
}
func newSema() *sema {
return &sema{
ready: make(chan bool, 1),
response: &sync.WaitGroup{},
}
}
// Queue is the struct responsible for tracking the state
// of the queue.
type Queue struct {
waiters waiters
items items
lock sync.Mutex
disposed bool
}
// Put will add the specified items to the queue.
func (q *Queue) Put(items ...interface{}) error {
if len(items) == 0 {
return nil
}
q.lock.Lock()
if q.disposed {
q.lock.Unlock()
return ErrDisposed
}
q.items = append(q.items, items...)
for {
sema := q.waiters.get()
if sema == nil {
break
}
sema.response.Add(1)
select {
case sema.ready <- true:
sema.response.Wait()
default:
// This semaphore timed out.
}
if len(q.items) == 0 {
break
}
}
q.lock.Unlock()
return nil
}
// Get retrieves items from the queue. If there are some items in the
// queue, get will return a number UP TO the number passed in as a
// parameter. If no items are in the queue, this method will pause
// until items are added to the queue.
func (q *Queue) Get(number int64) ([]interface{}, error) {
return q.Poll(number, 0)
}
// Poll retrieves items from the queue. If there are some items in the queue,
// Poll will return a number UP TO the number passed in as a parameter. If no
// items are in the queue, this method will pause until items are added to the
// queue or the provided timeout is reached. A non-positive timeout will block
// until items are added. If a timeout occurs, ErrTimeout is returned.
func (q *Queue) Poll(number int64, timeout time.Duration) ([]interface{}, error) {
if number < 1 {
// thanks again go
return []interface{}{}, nil
}
q.lock.Lock()
if q.disposed {
q.lock.Unlock()
return nil, ErrDisposed
}
var items []interface{}
if len(q.items) == 0 {
sema := newSema()
q.waiters.put(sema)
q.lock.Unlock()
var timeoutC <-chan time.Time
if timeout > 0 {
timeoutC = time.After(timeout)
}
select {
case <-sema.ready:
// we are now inside the put's lock
if q.disposed {
return nil, ErrDisposed
}
items = q.items.get(number)
sema.response.Done()
return items, nil
case <-timeoutC:
// cleanup the sema that was added to waiters
select {
case sema.ready <- true:
// we called this before Put() could
// Remove sema from waiters.
q.lock.Lock()
q.waiters.remove(sema)
q.lock.Unlock()
default:
// Put() got it already, we need to call Done() so Put() can move on
sema.response.Done()
}
return nil, ErrTimeout
}
}
items = q.items.get(number)
q.lock.Unlock()
return items, nil
}
// Peek returns a the first item in the queue by value
// without modifying the queue.
func (q *Queue) Peek() (interface{}, error) {
q.lock.Lock()
defer q.lock.Unlock()
if q.disposed {
return nil, ErrDisposed
}
peekItem, ok := q.items.peek()
if !ok {
return nil, ErrEmptyQueue
}
return peekItem, nil
}
// TakeUntil takes a function and returns a list of items that
// match the checker until the checker returns false. This does not
// wait if there are no items in the queue.
func (q *Queue) TakeUntil(checker func(item interface{}) bool) ([]interface{}, error) {
if checker == nil {
return nil, nil
}
q.lock.Lock()
if q.disposed {
q.lock.Unlock()
return nil, ErrDisposed
}
result := q.items.getUntil(checker)
q.lock.Unlock()
return result, nil
}
// Empty returns a bool indicating if this bool is empty.
func (q *Queue) Empty() bool {
q.lock.Lock()
defer q.lock.Unlock()
return len(q.items) == 0
}
// Len returns the number of items in this queue.
func (q *Queue) Len() int64 {
q.lock.Lock()
defer q.lock.Unlock()
return int64(len(q.items))
}
// Disposed returns a bool indicating if this queue
// has had disposed called on it.
func (q *Queue) Disposed() bool {
q.lock.Lock()
defer q.lock.Unlock()
return q.disposed
}
// Dispose will dispose of this queue and returns
// the items disposed. Any subsequent calls to Get
// or Put will return an error.
func (q *Queue) Dispose() []interface{} {
q.lock.Lock()
defer q.lock.Unlock()
q.disposed = true
for _, waiter := range q.waiters {
waiter.response.Add(1)
select {
case waiter.ready <- true:
// release Poll immediately
default:
// ignore if it's a timeout or in the get
}
}
disposedItems := q.items
q.items = nil
q.waiters = nil
return disposedItems
}
// New is a constructor for a new threadsafe queue.
func New(hint int64) *Queue {
return &Queue{
items: make([]interface{}, 0, hint),
}
}
// ExecuteInParallel will (in parallel) call the provided function
// with each item in the queue until the queue is exhausted. When the queue
// is exhausted execution is complete and all goroutines will be killed.
// This means that the queue will be disposed so cannot be used again.
func ExecuteInParallel(q *Queue, fn func(interface{})) {
if q == nil {
return
}
q.lock.Lock() // so no one touches anything in the middle
// of this process
todo, done := uint64(len(q.items)), int64(-1)
// this is important or we might face an infinite loop
if todo == 0 {
return
}
numCPU := 1
if runtime.NumCPU() > 1 {
numCPU = runtime.NumCPU() - 1
}
var wg sync.WaitGroup
wg.Add(numCPU)
items := q.items
for i := 0; i < numCPU; i++ {
go func() {
for {
index := atomic.AddInt64(&done, 1)
if index >= int64(todo) {
wg.Done()
break
}
fn(items[index])
items[index] = 0
}
}()
}
wg.Wait()
q.lock.Unlock()
q.Dispose()
}

203
app/job/main/member/model/queue/ring.go Normal file
View File

@@ -0,0 +1,203 @@
/*
Copyright 2014 Workiva, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package queue
import (
"runtime"
"sync/atomic"
"time"
)
// roundUp takes a uint64 greater than 0 and rounds it up to the next
// power of 2.
func roundUp(v uint64) uint64 {
v--
v |= v >> 1
v |= v >> 2
v |= v >> 4
v |= v >> 8
v |= v >> 16
v |= v >> 32
v++
return v
}
type node struct {
position uint64
data interface{}
}
type nodes []*node
// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations
// only. A put on full or get on empty call will block until an item
// is put or retrieved. Calling Dispose on the RingBuffer will unblock
// any blocked threads with an error. This buffer is similar to the buffer
// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue
// with some minor additions.
type RingBuffer struct {
_padding0 [8]uint64
queue uint64
_padding1 [8]uint64
dequeue uint64
_padding2 [8]uint64
mask, disposed uint64
_padding3 [8]uint64
nodes nodes
}
func (rb *RingBuffer) init(size uint64) {
size = roundUp(size)
rb.nodes = make(nodes, size)
for i := uint64(0); i < size; i++ {
rb.nodes[i] = &node{position: i}
}
rb.mask = size - 1 // so we don't have to do this with every put/get operation
}
// Put adds the provided item to the queue. If the queue is full, this
// call will block until an item is added to the queue or Dispose is called
// on the queue. An error will be returned if the queue is disposed.
func (rb *RingBuffer) Put(item interface{}) error {
_, err := rb.put(item, false)
return err
}
// Offer adds the provided item to the queue if there is space. If the queue
// is full, this call will return false. An error will be returned if the
// queue is disposed.
func (rb *RingBuffer) Offer(item interface{}) (bool, error) {
return rb.put(item, true)
}
func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {
var n *node
pos := atomic.LoadUint64(&rb.queue)
L:
for {
if atomic.LoadUint64(&rb.disposed) == 1 {
return false, ErrDisposed
}
n = rb.nodes[pos&rb.mask]
seq := atomic.LoadUint64(&n.position)
switch dif := seq - pos; {
case dif == 0:
if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {
break L
}
// case dif < 0:
// panic(`Ring buffer in a compromised state during a put operation.`)
default:
pos = atomic.LoadUint64(&rb.queue)
}
if offer {
return false, nil
}
runtime.Gosched() // free up the cpu before the next iteration
}
n.data = item
atomic.StoreUint64(&n.position, pos+1)
return true, nil
}
// Get will return the next item in the queue. This call will block
// if the queue is empty. This call will unblock when an item is added
// to the queue or Dispose is called on the queue. An error will be returned
// if the queue is disposed.
func (rb *RingBuffer) Get() (interface{}, error) {
return rb.Poll(0)
}
// Poll will return the next item in the queue. This call will block
// if the queue is empty. This call will unblock when an item is added
// to the queue, Dispose is called on the queue, or the timeout is reached. An
// error will be returned if the queue is disposed or a timeout occurs. A
// non-positive timeout will block indefinitely.
func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {
var (
n *node
pos = atomic.LoadUint64(&rb.dequeue)
start time.Time
)
if timeout > 0 {
start = time.Now()
}
L:
for {
if atomic.LoadUint64(&rb.disposed) == 1 {
return nil, ErrDisposed
}
n = rb.nodes[pos&rb.mask]
seq := atomic.LoadUint64(&n.position)
switch dif := seq - (pos + 1); {
case dif == 0:
if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {
break L
}
// case dif < 0:
// panic(`Ring buffer in compromised state during a get operation.`)
default:
pos = atomic.LoadUint64(&rb.dequeue)
}
if timeout > 0 && time.Since(start) >= timeout {
return nil, ErrTimeout
}
runtime.Gosched() // free up the cpu before the next iteration
}
data := n.data
n.data = nil
atomic.StoreUint64(&n.position, pos+rb.mask+1)
return data, nil
}
// Len returns the number of items in the queue.
func (rb *RingBuffer) Len() uint64 {
return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)
}
// Cap returns the capacity of this ring buffer.
func (rb *RingBuffer) Cap() uint64 {
return uint64(len(rb.nodes))
}
// Dispose will dispose of this queue and free any blocked threads
// in the Put and/or Get methods. Calling those methods on a disposed
// queue will return an error.
func (rb *RingBuffer) Dispose() {
atomic.CompareAndSwapUint64(&rb.disposed, 0, 1)
}
// IsDisposed will return a bool indicating if this queue has been
// disposed.
func (rb *RingBuffer) IsDisposed() bool {
return atomic.LoadUint64(&rb.disposed) == 1
}
// NewRingBuffer will allocate, initialize, and return a ring buffer
// with the specified size.
func NewRingBuffer(size uint64) *RingBuffer {
rb := &RingBuffer{}
rb.init(size)
return rb
}

273
app/job/main/member/model/realname.go Normal file
View File

@@ -0,0 +1,273 @@
package model
import (
"crypto/md5"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"strings"
"time"
"go-common/app/job/main/member/conf"
"go-common/library/log"
"github.com/pkg/errors"
)
var (
realnameSalt = "biliidentification@#$%^&*()(*&^%$#"
)
//RealnamePersonMessage is.
type RealnamePersonMessage struct {
MID int64 `json:"mid"`
Realname string `json:"realname"`
IdentifyCard string `json:"identify_card"`
}
//RealnameApplyMessage is.
type RealnameApplyMessage struct {
ID int `json:"id"`
MID int64 `json:"mid"`
Realname string `json:"realname"`
Type int `json:"type"`
CardDataCanal string `json:"card_data"`
CardForSearch string `json:"card_for_search"`
FrontIMG int `json:"front_img"`
BackIMG int `json:"back_img"`
FrontIMG2 int `json:"front_img2"`
ApplyTimeUnix int64 `json:"apply_time"`
Operater string `json:"operater"`
OperaterTimeUnix int64 `json:"operater_time"`
Status int8 `json:"status"`
Remark string `json:"remark"`
RemarkStatus int8 `json:"remark_status"`
}
// CardMD5 is.
func (r *RealnameApplyMessage) CardMD5() (res string) {
return cardReMD5(r.CardData(), r.CardType(), r.Country())
}
// cardReMD5 is.
func cardReMD5(encrypedCard string, cardType int, country int) (res string) {
card, err := cardDecrypt([]byte(encrypedCard))
if err != nil {
log.Error("cardNewMD5 decrypt err : %+v", err)
return
}
return cardMD5(string(card), cardType, country)
}
func cardMD5(card string, cardType int, country int) (res string) {
var (
lowerCode = strings.ToLower(card)
key = fmt.Sprintf("%s_%s_%d_%d", realnameSalt, lowerCode, cardType, country)
)
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func cardDecrypt(data []byte) (text []byte, err error) {
var (
decryptedData []byte
size int
)
decryptedData = make([]byte, base64.StdEncoding.DecodedLen(len(data)))
if size, err = base64.StdEncoding.Decode(decryptedData, data); err != nil {
err = errors.Wrapf(err, "base decode %s", data)
return
}
if text, err = rsaDecrypt(decryptedData[:size]); err != nil {
err = errors.Wrapf(err, "rsa decrypt %s , data : %s", decryptedData, data)
return
}
return
}
func rsaDecrypt(text []byte) (content []byte, err error) {
block, _ := pem.Decode(conf.Conf.RealnameRsaPriv)
if block == nil {
err = errors.New("private key error")
return
}
var (
privateKey *rsa.PrivateKey
)
if privateKey, err = x509.ParsePKCS1PrivateKey(block.Bytes); err != nil {
err = errors.WithStack(err)
return
}
if content, err = rsa.DecryptPKCS1v15(rand.Reader, privateKey, text); err != nil {
err = errors.WithStack(err)
return
}
return
}
//ApplyTime is.
func (r *RealnameApplyMessage) ApplyTime() (t time.Time) {
t = time.Unix(r.ApplyTimeUnix, 0)
return
}
//OperaterTime is.
func (r *RealnameApplyMessage) OperaterTime() (t time.Time) {
t = time.Unix(r.OperaterTimeUnix, 0)
return
}
//CardType is.
func (r *RealnameApplyMessage) CardType() (t int) {
if r.Type > 6 {
return 6
}
return r.Type
}
//Country is.
func (r *RealnameApplyMessage) Country() (t int) {
if r.Type > 6 {
return r.Type
}
return 0
}
//CardData is.
func (r *RealnameApplyMessage) CardData() (data string) {
if r.CardDataCanal == "" {
log.Warn("card data empty (+v)", r)
return ""
}
bytes, err := base64.StdEncoding.DecodeString(r.CardDataCanal)
if err != nil {
err = errors.Wrapf(err, "decode (%+v) failed", r)
log.Error("+v", err)
return ""
}
return string(bytes)
}
//RealnameApplyImgMessage is.
type RealnameApplyImgMessage struct {
ID int `json:"id"`
IMGData string `json:"img_data"`
AddTimeStr string `json:"add_time"`
AddTimeDB time.Time
}
//AddTime is.
func (r *RealnameApplyImgMessage) AddTime() (t time.Time) {
if r.AddTimeDB.IsZero() {
var err error
if t, err = time.ParseInLocation("2006-01-02 15:04:05", r.AddTimeStr, time.Local); err != nil {
log.Error("%+v", err)
t = time.Now()
}
return
}
return r.AddTimeDB
}
// RealnameApplyStatus is.
type RealnameApplyStatus int8
const (
// RealnameApplyStatusPending is.
RealnameApplyStatusPending RealnameApplyStatus = iota
// RealnameApplyStatusPass is.
RealnameApplyStatusPass
// RealnameApplyStatusBack is.
RealnameApplyStatusBack
// RealnameApplyStatusNone is.
RealnameApplyStatusNone
)
// IsPass return is apply passed
func (r RealnameApplyStatus) IsPass() bool {
switch r {
case RealnameApplyStatusPass:
return true
default:
return false
}
}
// RealnameChannel is
type RealnameChannel int8
// RealnameChannel enum
const (
RealnameChannelMain RealnameChannel = iota
RealnameChannelAlipay
)
const (
// RealnameCountryChina is.
RealnameCountryChina = 0
// RealnameCardTypeIdentity is.
RealnameCardTypeIdentity = 0
)
// RealnameInfo is user realname status info
type RealnameInfo struct {
ID int64 `json:"id"`
MID int64 `json:"mid"`
Channel RealnameChannel `json:"channel"`
Realname string `json:"realname"`
Country int `json:"country"`
CardType int `json:"card_type"`
Card string `json:"card"`
CardMD5 string `json:"card_md5"`
Status RealnameApplyStatus `json:"status"`
Reason string `json:"reason"`
CTime time.Time `json:"ctime"`
MTime time.Time `json:"mtime"`
}
// DecryptedCard is
func (r *RealnameInfo) DecryptedCard() (string, error) {
raw, err := cardDecrypt([]byte(r.Card))
if err != nil {
return "", err
}
return string(raw), nil
}
// RealnameApply is user realname apply info from default channel.
type RealnameApply struct {
ID int `json:"id"`
MID int64 `json:"mid"`
Realname string `json:"realname"`
Country int16 `json:"country"`
CardType int8 `json:"card_type"`
CardNum string `json:"card_num"`
CardMD5 string `json:"card_md5"`
HandIMG int `json:"hand_img"`
FrontIMG int `json:"front_img"`
BackIMG int `json:"back_img"`
Status int `json:"status"`
Operator string `json:"operator"`
OperatorID int64 `json:"operator_id"`
OperatorTime time.Time `json:"operator_time"`
Remark string `json:"remark"`
RemarkStatus int8 `json:"remark_status"`
CTime time.Time `json:"ctime"`
MTime time.Time `json:"mtime"`
}
// RealnameAlipayApply is user alipay apply info from alipay channle.
type RealnameAlipayApply struct {
ID int64 `json:"id"`
MID int64 `json:"mid"`
Realname string `json:"realname"`
Card string `json:"card"`
IMG string `json:"img"`
Status RealnameApplyStatus `json:"status"`
Reason string `json:"reason"`
Bizno string `json:"bizno"`
CTime time.Time `json:"ctime"`
MTime time.Time `json:"mtime"`
}

74
app/job/main/member/service/BUILD Normal file
View File

@@ -0,0 +1,74 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["service_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/model:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"base.go",
"cache_delay.go",
"check.go",
"exp.go",
"fixer.go",
"member.go",
"moral.go",
"realname.go",
"service.go",
"subproc.go",
],
importpath = "go-common/app/job/main/member/service",
tags = ["automanaged"],
deps = [
"//app/interface/main/account/service/realname/crypto:go_default_library",
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/dao:go_default_library",
"//app/job/main/member/model:go_default_library",
"//app/job/main/member/model/queue:go_default_library",
"//app/job/main/member/service/block:go_default_library",
"//app/service/main/member/api/gorpc:go_default_library",
"//app/service/main/member/model:go_default_library",
"//app/service/main/share/model:go_default_library",
"//library/log:go_default_library",
"//library/log/infoc:go_default_library",
"//library/net/ip:go_default_library",
"//library/queue/databus:go_default_library",
"//library/queue/databus/databusutil:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/golang.org/x/time/rate:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/member/service/block:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

52
app/job/main/member/service/base.go Normal file
View File

@@ -0,0 +1,52 @@
package service
import (
"context"
"go-common/library/log"
)
// BaseInfo get user's base info.
// func (s *Service) BaseInfo(c context.Context, mid int64) (info *model.BaseInfo, err error) {
// if mid <= 0 {
// log.Info("s.BaseInfo(%d) mid not valid number!", mid)
// return
// }
// var mc = true
// if info, err = s.dao.BaseInfoCache(c, mid); err != nil {
// mc = false
// err = nil // ignore error
// }
// if info != nil {
// if info.Mid == 0 {
// log.Info("s.BaseInfo(%d) mid not exist!", mid)
// }
// return
// }
// if info, err = s.dao.BaseInfo(c, mid); err != nil {
// log.Error("s.dao.BaseInfo(%d) error(%v)", mid, err)
// return
// }
// if info == nil {
// info = &model.BaseInfo{}
// log.Info("s.BaseInfo(%d) mid not exist!", mid)
// return
// }
// if mc {
// s.dao.SetBaseInfoCache(context.TODO(), mid, info)
// }
// log.Info("s.BaseInfo(%d) info(%+v)", mid, info)
// return
// }
func (s *Service) updateAccFace(c context.Context, mid int64) error {
base, err := s.dao.BaseInfo(c, mid)
if err != nil {
log.Error("updateAccFace s.dao.BaseInfoWithoutDomain(%d) error(%v)", mid, err)
return err
}
if base == nil {
return nil
}
return s.dao.UpdateAccFace(c, mid, base.Face)
}

57
app/job/main/member/service/block/BUILD Normal file
View File

@@ -0,0 +1,57 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["service_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"audit_log.go",
"blackhouse.go",
"limit.go",
"msg.go",
"service.go",
],
importpath = "go-common/app/job/main/member/service/block",
tags = ["automanaged"],
deps = [
"//app/job/main/member/conf:go_default_library",
"//app/job/main/member/dao/block:go_default_library",
"//app/job/main/member/model/block:go_default_library",
"//library/cache:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
"//library/queue/databus/report:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

48
app/job/main/member/service/block/audit_log.go Normal file
View File

@@ -0,0 +1,48 @@
package block
import (
"context"
"strconv"
"time"
model "go-common/app/job/main/member/model/block"
"go-common/library/log"
manager "go-common/library/queue/databus/report"
)
// AddAuditLog .
func (s *Service) AddAuditLog(c context.Context, tp model.BlockAction, uid int64, uname string, oids []int64, duration time.Duration, source model.BlockSource, area model.BlockArea, reason, comment string, notify bool, stime time.Time) error {
var (
err error
dur = int64(duration / time.Second)
notifyStr = strconv.FormatBool(notify)
)
for _, oid := range oids {
managerInfo := &manager.ManagerInfo{
UID: uid,
Uname: uname,
Business: model.BlockLogBizID,
Type: int(tp),
Action: tp.String(),
Oid: oid,
Ctime: time.Now(),
Index: []interface{}{dur, uint8(source), uint8(area), reason, comment, notifyStr},
Content: map[string]interface{}{
"duration": dur,
"source": source,
"area": area,
"reason": reason,
"comment": comment,
"notify": notifyStr,
"action_time": stime.Unix(),
"remove_time": stime.Add(time.Second * time.Duration(dur)).Unix(),
},
}
if err = manager.Manager(managerInfo); err != nil {
log.Error("manager.Manager(%+v) error(%+v)", managerInfo, err)
continue
}
log.Info("s.managerSendLog(%+v)", managerInfo)
}
return err
}

201
app/job/main/member/service/block/blackhouse.go Normal file
View File

@@ -0,0 +1,201 @@
package block
import (
"context"
"encoding/json"
"fmt"
"time"
model "go-common/app/job/main/member/model/block"
xsql "go-common/library/database/sql"
"go-common/library/log"
"go-common/library/queue/databus"
"github.com/pkg/errors"
)
func (s *Service) creditExpireHandler(c context.Context) {
if s.conf.BlockProperty.CreditExpireCheckLimit <= 0 {
log.Error("conf.Conf.Property.creditExpireCheckLimit [%d] <= 0", s.conf.BlockProperty.CreditExpireCheckLimit)
return
}
var (
mids = make([]int64, s.conf.BlockProperty.CreditExpireCheckLimit)
startID int64
err error
)
for len(mids) >= s.conf.BlockProperty.CreditExpireCheckLimit {
log.Info("black house expire handle startID (%d)", startID)
if startID, mids, err = s.dao.UserStatusList(c, model.BlockStatusCredit, startID, s.conf.BlockProperty.CreditExpireCheckLimit); err != nil {
log.Error("%+v", err)
return
}
for _, mid := range mids {
log.Info("Start handle black house mid (%d)", mid)
var ok bool
if ok, err = s.creditExpireCheck(c, mid); err != nil {
log.Error("%+v", err)
continue
}
if ok {
log.Info("Start remove black house mid (%d)", mid)
if err = s.creditExpireRemove(c, mid); err != nil {
log.Error("error: %+v, mid: %d", err, mid)
}
}
}
}
}
func (s *Service) creditExpireCheck(c context.Context, mid int64) (ok bool, err error) {
var (
his *model.DBHistory
ex *model.DBExtra
)
if his, err = s.dao.UserLastHistory(c, mid); err != nil {
return
}
if his == nil {
return
}
log.Info("Credit check his (%+v)", his)
if his.Action != model.BlockActionLimit {
return
}
if ex, err = s.dao.UserExtra(c, mid); err != nil {
return
}
if ex == nil {
return
}
log.Info("Credit check extra (%+v)", his)
if ex.ActionTime.Before(his.StartTime) {
return
}
if his.StartTime.Add(time.Duration(his.Duration) * time.Second).After(time.Now()) {
return
}
ok = true
return
}
func (s *Service) creditExpireRemove(c context.Context, mid int64) (err error) {
var (
db = &model.DBHistory{
MID: mid,
AdminID: -1,
AdminName: "sys",
Source: model.BlockSourceRemove,
Area: model.BlockAreaNone,
Reason: "小黑屋自动解封",
Comment: "小黑屋自动解封",
Action: model.BlockActionSelfRemove,
StartTime: time.Now(),
Duration: 0,
Notify: false,
}
tx *xsql.Tx
)
if tx, err = s.dao.BeginTX(c); err != nil {
return
}
if err = s.dao.TxInsertHistory(c, tx, db); err != nil {
tx.Rollback()
return
}
count, err := s.dao.TxUpsertUser(c, tx, mid, model.BlockStatusFalse)
if err != nil || count == 0 {
tx.Rollback()
return
}
if err = tx.Commit(); err != nil {
err = errors.WithStack(err)
}
s.mission(func() {
if err := s.notifyRemoveMSG(context.TODO(), []int64{mid}); err != nil {
log.Error("%+v", err)
}
})
s.cache.Save(func() {
if err := s.dao.DeleteUserCache(context.TODO(), mid); err != nil {
log.Error("%+v", err)
}
if databusErr := s.dao.AccountNotify(context.TODO(), mid); databusErr != nil {
log.Error("%+v", databusErr)
}
})
return
}
func (s *Service) notifyRemoveMSG(c context.Context, mids []int64) (err error) {
code, title, content := s.MSGRemoveInfo()
if err = s.dao.SendSysMsg(c, code, mids, title, content, ""); err != nil {
return
}
return
}
// databus
func (s *Service) creditsubproc() {
defer func() {
if x := recover(); x != nil {
log.Error("%+v", errors.WithStack(fmt.Errorf("s.creditsubproc panic(%v)", x)))
go s.creditsubproc()
log.Info("s.creditsubproc recover")
}
}()
var (
msg *databus.Message
eventMSG *model.CreditAnswerMSG
err error
msgChan = s.creditSub.Messages()
c = context.TODO()
)
for msg = range msgChan {
if err = msg.Commit(); err != nil {
log.Error("msg.Commit error(%v)", err)
}
eventMSG = &model.CreditAnswerMSG{}
if err = json.Unmarshal([]byte(msg.Value), eventMSG); err != nil {
log.Error("%+v", errors.WithStack(err))
continue
}
if err = s.handleCreditAnswerMSG(c, eventMSG); err != nil {
log.Error("%+v", err)
continue
}
log.Info("s.handleCreditAnswerMSG(%v) msg", eventMSG)
}
log.Info("creditsubproc end")
}
func (s *Service) handleCreditAnswerMSG(c context.Context, msg *model.CreditAnswerMSG) (err error) {
if msg.MID <= 0 {
return
}
var (
extra = &model.DBExtra{
MID: msg.MID,
CreditAnswerFlag: true,
ActionTime: msg.MTime.Time(),
}
checkFlag bool
)
if err = s.dao.InsertExtra(c, extra); err != nil {
return
}
// 及时检查解封
log.Info("Start check black house mid (%d) from answer", extra.MID)
if checkFlag, err = s.creditExpireCheck(c, extra.MID); err != nil {
return
}
if checkFlag {
log.Info("Start remove black house mid (%d)", extra.MID)
if err = s.creditExpireRemove(c, extra.MID); err != nil {
log.Error("error: %+v, mid: %d", err, extra.MID)
return
}
}
return
}

118
app/job/main/member/service/block/limit.go Normal file
View File

@@ -0,0 +1,118 @@
package block
import (
"context"
"time"
model "go-common/app/job/main/member/model/block"
xsql "go-common/library/database/sql"
"go-common/library/log"
"github.com/pkg/errors"
)
func (s *Service) limitExpireHandler(c context.Context) {
if s.conf.BlockProperty.LimitExpireCheckLimit <= 0 {
log.Error("conf.Conf.Property.LimitExpireCheckLimit [%d] <= 0", s.conf.BlockProperty.LimitExpireCheckLimit)
return
}
var (
mids = make([]int64, s.conf.BlockProperty.LimitExpireCheckLimit)
startID int64
err error
)
for len(mids) >= s.conf.BlockProperty.LimitExpireCheckLimit {
log.Info("limit expire handle startID (%d)", startID)
if startID, mids, err = s.dao.UserStatusList(c, model.BlockStatusLimit, startID, s.conf.BlockProperty.LimitExpireCheckLimit); err != nil {
log.Error("%+v", err)
return
}
for _, mid := range mids {
var ok bool
if ok, err = s.limitExpireCheck(c, mid); err != nil {
log.Error("%+v", err)
continue
}
if ok {
if err = s.limitExpireRemove(c, mid); err != nil {
log.Error("error: %+v, mid: %d", err, mid)
continue
}
}
time.Sleep(time.Millisecond * 100)
}
}
}
func (s *Service) limitExpireCheck(c context.Context, mid int64) (ok bool, err error) {
var (
his *model.DBHistory
)
if his, err = s.dao.UserLastHistory(c, mid); err != nil {
return
}
if his == nil {
return
}
if his.Action != model.BlockActionLimit {
return
}
if his.StartTime.Add(time.Duration(his.Duration) * time.Second).After(time.Now()) {
return
}
ok = true
return
}
func (s *Service) limitExpireRemove(c context.Context, mid int64) (err error) {
var (
_reason, _comment = "系统自动解封", "系统自动解封"
stime = time.Now()
db = &model.DBHistory{
MID: mid,
AdminID: model.BlockJOBManagerID,
AdminName: model.BlockJOBManagerName,
Source: model.BlockSourceRemove,
Area: model.BlockAreaNone,
Reason: _reason,
Comment: _comment,
Action: model.BlockActionSelfRemove,
StartTime: stime,
Duration: 0,
Notify: false,
}
tx *xsql.Tx
)
if tx, err = s.dao.BeginTX(c); err != nil {
return
}
if err = s.dao.TxInsertHistory(c, tx, db); err != nil {
tx.Rollback()
return
}
count, err := s.dao.TxUpsertUser(c, tx, mid, model.BlockStatusFalse)
if err != nil || count == 0 {
tx.Rollback()
return
}
if err = tx.Commit(); err != nil {
err = errors.WithStack(err)
}
s.mission(func() {
if err := s.notifyRemoveMSG(context.TODO(), []int64{mid}); err != nil {
log.Error("%+v", err)
}
})
s.mission(func() {
s.AddAuditLog(context.TODO(), model.BlockActionSelfRemove, model.BlockJOBManagerID, model.BlockJOBManagerName, []int64{mid}, 0, model.BlockSourceRemove, model.BlockAreaNone, _reason, _comment, false, stime)
})
s.cache.Save(func() {
if err := s.dao.DeleteUserCache(context.TODO(), mid); err != nil {
log.Error("%+v", err)
}
if databusErr := s.dao.AccountNotify(context.TODO(), mid); databusErr != nil {
log.Error("%+v", databusErr)
}
})
return
}

9
app/job/main/member/service/block/msg.go Normal file
View File

@@ -0,0 +1,9 @@
package block
// MSGRemoveInfo get msg info
func (s *Service) MSGRemoveInfo() (code string, title, content string) {
code = s.conf.BlockProperty.MSG.BlockRemove.Code
title = s.conf.BlockProperty.MSG.BlockRemove.Title
content = s.conf.BlockProperty.MSG.BlockRemove.Content
return
}

114
app/job/main/member/service/block/service.go Normal file
View File

@@ -0,0 +1,114 @@
package block
import (
"context"
"fmt"
"time"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/dao/block"
"go-common/library/cache"
"go-common/library/log"
"go-common/library/queue/databus"
"github.com/pkg/errors"
)
// Service struct
type Service struct {
dao *block.Dao
conf *conf.Config
cache *cache.Cache
missch chan func()
creditSub *databus.Databus
}
// New init
func New(conf *conf.Config, dao *block.Dao, creditSub *databus.Databus) (s *Service) {
s = &Service{
dao: dao,
conf: conf,
cache: cache.New(1, 10240),
missch: make(chan func(), 10240),
creditSub: creditSub,
}
// 自动解禁检查
if s.conf.BlockProperty.Flag.ExpireCheck {
go s.limitcheckproc()
go s.creditcheckproc()
}
// 小黑屋答题状态订阅
if s.conf.BlockProperty.Flag.CreditSub {
go s.creditsubproc()
}
go s.missproc()
return s
}
// Ping Service
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close Service
func (s *Service) Close() {
s.dao.Close()
}
func (s *Service) limitcheckproc() {
defer func() {
if x := recover(); x != nil {
log.Error("service.limitcheckproc panic(%v)", x)
go s.limitcheckproc()
log.Info("service.limitcheckproc recover")
}
}()
for {
log.Info("limit check start")
s.limitExpireHandler(context.TODO())
log.Info("limit check end")
time.Sleep(time.Duration(s.conf.BlockProperty.LimitExpireCheckTick))
}
}
func (s *Service) creditcheckproc() {
defer func() {
if x := recover(); x != nil {
log.Error("%+v", errors.WithStack(fmt.Errorf("service.creditcheckproc panic(%v)", x)))
go s.creditcheckproc()
log.Info("service.creditcheckproc recover")
}
}()
for {
log.Info("black house check start")
s.creditExpireHandler(context.TODO())
log.Info("black house check end")
time.Sleep(time.Duration(s.conf.BlockProperty.CreditExpireCheckTick))
}
}
func (s *Service) missproc() {
defer func() {
if x := recover(); x != nil {
log.Error("service.missproc panic(%v)", x)
go s.missproc()
log.Info("service.missproc recover")
}
}()
for {
f := <-s.missch
f()
}
}
func (s *Service) mission(f func()) {
select {
case s.missch <- f:
default:
log.Error("s.missch full")
}
}

34
app/job/main/member/service/block/service_test.go Normal file
View File

@@ -0,0 +1,34 @@
package block
import (
"context"
"flag"
"os"
"testing"
"go-common/app/job/main/member/conf"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *Service
c context.Context
)
func TestMain(m *testing.M) {
defer os.Exit(0)
flag.Set("conf", "../cmd/member-job-dev.toml")
var err error
if err = conf.Init(); err != nil {
panic(err)
}
m.Run()
}
func TestService(t *testing.T) {
Convey("", t, func() {
s.Ping(c)
s.Close()
})
}

96
app/job/main/member/service/cache_delay.go Normal file
View File

@@ -0,0 +1,96 @@
package service
import (
"context"
"time"
"go-common/app/job/main/member/model/queue"
"go-common/library/log"
)
// Item is
type Item struct {
Mid int64
Time time.Time
Action string
}
// Compare is
func (i *Item) Compare(other queue.Item) int {
o := asItem(other)
if o == nil {
return -1
}
if i.Time.Equal(o.Time) {
return 0
}
if i.Time.After(o.Time) {
return 1
}
return -1
}
// HashCode is
func (i *Item) HashCode() int64 {
return i.Mid
}
func asItem(in queue.Item) *Item {
o, ok := in.(*Item)
if !ok {
return nil
}
return o
}
func asItems(in []queue.Item) []*Item {
out := make([]*Item, 0, len(in))
for _, i := range in {
item := asItem(i)
if item == nil {
continue
}
out = append(out, item)
}
return out
}
func (s *Service) cachedelayproc(ctx context.Context) {
fiveSeconds := time.Second * 5
t := time.NewTicker(fiveSeconds)
delayed := func(t time.Time) bool {
top := asItem(s.cachepq.Peek())
if top == nil {
log.Info("Empty cache queue top at: %v", t)
return false
}
if t.Sub(top.Time) < fiveSeconds {
log.Info("Top item is in five seconds, skip and waiting for next tick")
return false
}
return true
}
for ti := range t.C {
if !delayed(ti) {
continue
}
for {
qitems, err := s.cachepq.Get(1)
if err != nil {
log.Error("Failed to get queue items from cache queue: %+v", err)
return
}
items := asItems(qitems)
for _, it := range items {
log.Info("Notify purge cache in delay queue with mid: %d", it.Mid)
s.dao.NotifyPurgeCache(ctx, it.Mid, it.Action)
}
if s.cachepq.Empty() || !delayed(time.Now()) {
break
}
}
}
}

89
app/job/main/member/service/check.go Normal file
View File

@@ -0,0 +1,89 @@
package service
import (
"context"
"fmt"
"time"
"go-common/app/job/main/member/model"
"go-common/library/log"
)
func (s *Service) checkExpAdd(c context.Context, mid int64, event string, now time.Time) (exp *model.NewExp, eo *model.ExpOper, added, ok bool, err error) {
if eo, ok = model.ExpFlagOper[event]; !ok {
log.Info("s.checkExpAdd(%d) oper(%s) not found", mid, event)
return
}
var base *model.BaseInfo
if base, err = s.dao.BaseInfo(c, mid); err != nil {
log.Error("s.dao.BaseInfo(%d) error(%v)", mid, err)
return
}
if base == nil {
err = fmt.Errorf("No base info with mid(%v)", mid)
log.Error("Failed to checkExpAdd with mid(%d) error: %+v", mid, err)
return
}
if ok = !(base.Rank < 10000); !ok {
log.Info("s.checkExpAdd(%d) mid.Rank<10000", mid)
return
}
if added, err = s.dao.ExpAdded(c, mid, now.Day(), eo.Oper); err != nil || added {
log.Info("s.dao.ExpAdded(%d) error(%v) added(%v)", mid, err, added)
return
}
if exp, err = s.dao.SelExp(c, mid); err != nil {
log.Error("s.dao.SelExp(%d) error(%v)", mid, err)
return
}
if now.Unix()-int64(exp.Addtime) < 24*60*60 {
added = exp.Flag&eo.Flag == eo.Flag
return
}
if err = s.dao.InitExp(c, mid); err != nil {
log.Error("s.dao.InitExp(%d) error(%v)", mid, err)
return
}
exp.FlagDailyReset(now)
if err = s.dao.UpdateExpFlag(c, mid, exp.Flag, exp.Addtime); err != nil {
log.Error("s.dao.UpdateExpFlag(%d) flag(%d) addtime(%v)", mid, exp.Flag, exp.Addtime)
return
}
added = exp.Flag&eo.Flag == eo.Flag
return
}
// CheckExpInit check init user exp if exp not exist.
func (s *Service) CheckExpInit(c context.Context, mid int64) (opers []*model.ExpOper, err error) {
var aso *model.MemberAso
if aso, err = s.dao.AsoStatus(c, mid); err != nil {
log.Error("s.dao.AsoStatus(%d) error(%v)", mid, err)
return
}
if aso.Spacesta >= 0 && len(aso.Email) != 0 {
opers = append(opers, model.ExpFlagOper["email"])
}
if len(aso.Telphone) != 0 {
opers = append(opers, model.ExpFlagOper["phone"])
}
if aso.SafeQs != 0 {
opers = append(opers, model.ExpFlagOper["safe"])
}
var ri *model.RealnameInfo
if ri, err = s.dao.RealnameInfo(c, mid); err != nil {
log.Error("s.dao.RealnameInfo(%d) error(%+v)", mid, err)
return
}
if ri != nil && ri.Status.IsPass() {
opers = append(opers, model.ExpFlagOper["identify"])
}
log.Info("exp init opers with mid: %d: %+v", mid, opers)
return
}
func sameAccInfo(base *model.BaseInfo, res *model.AccountInfo) (same bool) {
return sameName(base, res)
}
func sameName(base *model.BaseInfo, res *model.AccountInfo) bool {
return base.Name == res.Name
}

170
app/job/main/member/service/exp.go Normal file
View File

@@ -0,0 +1,170 @@
package service
import (
"context"
"encoding/json"
"time"
"go-common/app/job/main/member/model"
"go-common/library/log"
)
const (
expMulti = 100
level1 = 1
level2 = 200
level3 = 1500
level4 = 4500
level5 = 10800
level6 = 28800
)
func (s *Service) initExp(c context.Context, mid int64) (err error) {
var opers []*model.ExpOper
if opers, err = s.CheckExpInit(c, mid); err != nil {
log.Error("s.CheckExpInit(%d) error(%v)", mid, err)
return
}
if len(opers) == 0 {
log.Info("s.CheckExpInit(%d) opers eq(0) continue", mid)
return
}
var exp *model.NewExp
if exp, err = s.dao.SelExp(c, mid); err != nil {
log.Error("s.dao.SelExp(%d) error(%v)", mid, err)
return
}
if exp.Mid == 0 {
if err = s.dao.InitExp(c, mid); err != nil {
log.Error("s.dao.InitExp(%d) init user exp completed error(%v)", mid, err)
return
}
}
var (
rows int64
now = time.Now().Unix()
)
for _, oper := range opers {
if rows, err = s.dao.UpdateExpAped(c, mid, oper.Count*100, oper.Flag); err != nil {
log.Error("s.dao.UpdateExpAped(%d) error(%v)", mid, err)
return
}
if rows == 0 {
log.Info("s.dao.UpdateExpAped(%d) exp(%d) flag(%d) rows affected eq(0) continue", mid, oper.Count*100, oper.Flag)
continue
}
if err = s.dao.DatabusAddLog(c, mid, exp.Exp/100, (exp.Exp+oper.Count*100)/100, now, oper.Oper, oper.Reason, ""); err != nil {
log.Error("s.dao.DatabusAddLog(%d) fromExp(%d) toExp(%d) ts(%d) oper(%s) reason(%s) error(%v)", mid, exp.Exp/100, (exp.Exp+oper.Count*100)/100, now, oper.Oper, oper.Reason, err)
err = nil
continue
} else {
log.Info("s.dao.DatabusAddLog(%d) fromExp(%d) toExp(%d) ts(%d) oper(%s) reason(%s) msg published", mid, exp.Exp/100, (exp.Exp+oper.Count*100)/100, now, oper.Oper, oper.Reason)
exp.Exp = exp.Exp + oper.Count*100
now++
}
}
return
}
func (s *Service) delayUpdateExp() {
s.limiter.UpdateExp.Wait(context.Background())
}
func (s *Service) addExp(c context.Context, e *model.AddExp) (err error) {
if e.Mid <= 0 {
return
}
now := time.Unix(e.Ts, 0)
exp, eo, added, ok, err := s.checkExpAdd(c, e.Mid, e.Event, now)
if err != nil || added || !ok {
log.Info("s.checkExpAdd(%d) event(%s) result added(%v) ok(%v) err(%v)", e.Mid, e.Event, added, ok, err)
return
}
// 写数据库限速，防止写入过大导致主从延迟
s.delayUpdateExp()
var rows int64
if rows, err = s.dao.UpdateExpAped(c, e.Mid, eo.Count*100, eo.Flag); err != nil {
log.Error("s.dao.UpdateExpAped(%d) exp(%d) flag(%d) error(%v) ", e.Mid, eo.Count*100, eo.Flag, err)
return
}
if rows == 0 {
log.Info("s.dao.UpdateExpAped(%d) exp(%d) flag(%d) rows affected eq(0) continue!", e.Mid, eo.Count*100, eo.Flag)
return
}
if _, err = s.dao.SetExpAdded(context.Background(), e.Mid, now.Day(), eo.Oper); err != nil {
log.Error("s.dao.SetExpAdded(%d) oper(%s)", e.Mid, eo.Oper)
err = nil
}
if err = s.dao.DatabusAddLog(context.Background(), e.Mid, (exp.Exp)/100, (exp.Exp+eo.Count*100)/100, e.Ts, eo.Oper, eo.Reason, e.IP); err != nil {
log.Error("s.dao.DatabusAddLog(%d) oper(%s) reason(%s) error(%v)", e.Mid, eo.Oper, eo.Reason, err)
err = nil
} else {
log.Info("s.dao.DatabusAddLog(%d) oper(%s) reason(%s) msg published!", e.Mid, eo.Oper, eo.Reason)
}
return
}
func (s *Service) awardDo(ms []interface{}) {
for _, m := range ms {
l, ok := m.(*model.LoginLogIPString)
if !ok {
continue
}
s.addExp(context.TODO(), &model.AddExp{
Mid: l.Mid,
IP: l.Loginip,
Ts: l.Timestamp,
Event: "login",
})
s.recoverMoral(context.TODO(), l.Mid)
log.Info("consumer mid:%d,ts: %d", l.Mid, l.Timestamp)
}
}
func isExpAndLevelChange(mu *model.Message) (bool, bool) {
if mu.Action == "insert" {
return true, true
}
if len(mu.Old) <= 0 || len(mu.New) <= 0 {
return false, false
}
old := &model.ExpMessage{}
new := &model.ExpMessage{}
if err := json.Unmarshal(mu.New, new); err != nil {
return false, false
}
if err := json.Unmarshal(mu.Old, old); err != nil {
return false, false
}
expChange := false
levelChange := false
if old.Exp != new.Exp {
expChange = true
}
if level(old.Exp) != level(new.Exp) {
levelChange = true
}
return expChange, levelChange
}
func level(exp int64) int8 {
exp = exp / expMulti
switch {
case exp < level1:
return 0
case exp < level2:
return 1
case exp < level3:
return 2
case exp < level4:
return 3
case exp < level5:
return 4
case exp < level6:
return 5
default:
return 6
}
}

117
app/job/main/member/service/fixer.go Normal file
View File

@@ -0,0 +1,117 @@
package service
import (
"context"
"encoding/json"
"sync/atomic"
"time"
"go-common/app/job/main/member/model"
"go-common/library/log"
"github.com/pkg/errors"
)
var (
csclice []chan int64
maxmid int64 = 310000000
scanned int64
errCount int64
)
func (s *Service) makeChan(num int) {
csclice = make([]chan int64, num)
for i := 0; i < num; i++ {
csclice[i] = make(chan int64, 10000)
}
}
// dataCheckMids check mid
func (s *Service) dataCheckMids() {
var (
i int64
)
if s.c.SyncRange.End > maxmid {
s.c.SyncRange.End = maxmid
}
if s.c.SyncRange.Start < 0 {
s.c.SyncRange.Start = 0
}
for i = s.c.SyncRange.Start; i < s.c.SyncRange.End; i++ {
csclice[i%30] <- i
}
}
// dataFixer
func (s *Service) dataFixer(cs chan int64) {
for {
mids := make([]int64, 0, 10)
for mid := range cs {
mids = append(mids, mid)
if len(mids) >= 5 {
break
}
atomic.AddInt64(&scanned, 1)
}
s.fix(mids)
}
}
func (s *Service) fix(mids []int64) {
var (
err error
accs = make(map[int64]*model.AccountInfo)
errs = make(map[int64]map[string]bool)
c = context.TODO()
base *model.BaseInfo
)
func() {
defer func() {
if r := recover(); r != nil {
r = errors.WithStack(r.(error))
log.Error("fixer: wocao jingran recover le error(%+v)", r)
time.Sleep(10 * time.Second)
}
time.Sleep(10 * time.Millisecond)
}()
if accs, errs, err = s.dao.Accounts(c, mids); err != nil {
log.Error("fixer: dao.AccountInfo mid(%v) res(%v) error(%v)", mids, accs, err)
return
}
for mid, res := range accs {
log.Error("fixer: mid(%d) res(%+v)", mid, res)
if base, err = s.dao.BaseInfo(c, mid); err != nil {
log.Error("fixer: s.dao.BaseInfo mid(%d) err(%v)", mid, err)
continue
}
if base == nil {
log.Error("fixer: dataCheckErr mid(%d) res(%v),base(%v),detail(%v)", mid, res, base)
continue
}
// all fields are same
if sameAccInfo(base, res) {
log.Info("fixer: sameAccInfo mid(%d) result true continue", mid)
continue
}
// increase errCount and logging
bs, _ := json.Marshal(base)
jres, _ := json.Marshal(res)
atomic.AddInt64(&errCount, 1)
log.Error("fixer: dataCheckFail mid(%d) base(%s),res(%s),errCount(%d)", mid, bs, jres, atomic.LoadInt64(&errCount))
if _, ok := errs[mid]; !ok {
log.Error("fixer,errs[%v] is not ok", mid)
continue
}
if asoOK := errs[mid]["asoOK"]; asoOK && !sameName(base, res) && len(res.Name) > 0 {
s.dao.SetName(c, mid, res.Name)
}
}
log.Info("fixer: dataCheckRight mids(%v) scanned(%d) errCount(%d)", mids, scanned, atomic.LoadInt64(&errCount))
}()
}

24
app/job/main/member/service/member.go Normal file
View File

@@ -0,0 +1,24 @@
package service
import (
"context"
"go-common/library/log"
)
// setName set user name.
func (s *Service) setName(mid int64) (err error) {
var (
name string
)
if name, err = s.dao.Name(context.TODO(), mid); err != nil {
log.Error("s.dao.Name(%d) error(%v)", mid, err)
return
}
if len(name) > 0 {
if err = s.dao.SetName(context.TODO(), mid, name); err != nil {
return
}
}
return
}

82
app/job/main/member/service/moral.go Normal file
View File

@@ -0,0 +1,82 @@
package service
import (
"context"
"fmt"
"strconv"
"time"
"go-common/app/job/main/member/model"
smodel "go-common/app/service/main/member/model"
"go-common/library/log"
)
func (s *Service) recoverMoral(c context.Context, mid int64) (err error) {
var (
moral *smodel.Moral
rowsAffected int64
)
if moral, err = s.dao.SelMoral(c, mid); err != nil {
return
}
if moral == nil || moral.Moral >= smodel.DefaultMoral {
log.Info("recoverMoral ignore, moral(%v)", moral)
return
}
now := time.Now()
deltaDays := deltaDays(now, moral.LastRecoverDate.Time())
if deltaDays <= 0 {
return
}
deltaMoral := deltaDays * 100
if moral.Moral+deltaMoral > smodel.DefaultMoral {
deltaMoral = smodel.DefaultMoral - moral.Moral
}
if rowsAffected, err = s.dao.RecoverMoral(c, mid, deltaMoral, deltaMoral, now.Format("2006-01-02")); err != nil {
return
}
if rowsAffected == 0 {
return
}
// report log
ul := &model.UserLog{
Mid: mid,
IP: "127.0.0.1",
TS: now.Unix(),
LogID: model.UUID4(),
Content: map[string]string{
"from_moral": strconv.FormatInt(moral.Moral, 10),
"to_moral": strconv.FormatInt(deltaMoral+moral.Moral, 10),
"origin": strconv.FormatInt(smodel.ManualRecoveryType, 10),
"status": strconv.FormatInt(smodel.RevocableMoralStatus, 10),
"remark": fmt.Sprintf("自动恢复(%d)天", deltaDays),
"operater": "系统",
"reason": fmt.Sprintf("时间：%d天", deltaDays),
},
}
s.dao.AddMoralLog(c, ul)
// origin log
// content := make(map[string][]byte, 10)
// content["mid"] = []byte(strconv.FormatInt(mid, 10))
// content["ip"] = []byte("127.0.0.1")
// content["from_moral"] = []byte(strconv.FormatInt(moral.Moral, 10))
// content["to_moral"] = []byte(strconv.FormatInt(deltaMoral+moral.Moral, 10))
// content["origin"] = []byte(strconv.FormatInt(smodel.ManualRecoveryType, 10))
// content["status"] = []byte(strconv.FormatInt(smodel.RevocableMoralStatus, 10))
// content["remark"] = []byte(fmt.Sprintf("自动恢复(%d)天", deltaDays))
// content["operater"] = []byte("系统")
// content["reason"] = []byte(fmt.Sprintf("时间：%d天", deltaDays))
// if err = s.dao.AddLog(c, mid, now.Unix(), content, model.TableMoralLog); err != nil {
// log.Error("recoverMoral mid: %v from_moral:%v,to_moral:%v error(%v)", mid, content["from_moral"], content["to_moral"], err)
// } else {
// log.Info("recoverMoral mid: %v from_moral:%v,to_moral:%v error(%v)", mid, content["from_moral"], content["to_moral"], err)
// }
s.dao.DelMoralCache(c, mid)
return
}
func deltaDays(now, old time.Time) int64 {
return int64(now.Sub(old).Hours() / 24)
}

261
app/job/main/member/service/realname.go Normal file
View File

@@ -0,0 +1,261 @@
package service
import (
"context"
"encoding/json"
"fmt"
"net/url"
"strconv"
"time"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/model"
memmodel "go-common/app/service/main/member/model"
"go-common/library/log"
"go-common/library/net/ip"
"github.com/pkg/errors"
)
// constrs for gender
const (
_genderMale = "male"
_genderFemale = "female"
)
// realname alipay polling
func (s *Service) realnamealipaycheckproc() {
defer func() {
if x := recover(); x != nil {
log.Error("%+v", errors.WithStack(fmt.Errorf("service.realnamealipaycheckproc panic(%v)", x)))
go s.realnamealipaycheckproc()
log.Info("service.realnamealipaycheckproc recover")
}
}()
for {
var (
to = time.Now()
from = to.Add(-2 * time.Duration(conf.Conf.Biz.RealnameAlipayCheckTick))
expiredTime = from
startTime = expiredTime.AddDate(0, -1, 0)
)
log.Info("realname alipay check start from : %s , to : %s", from, to)
s.realnameAlipayCheckHandler(context.Background(), from, to)
// to = from
// from = to.Add(-2 * time.Duration(conf.Conf.Biz.RealnameAlipayCheckTick))
log.Info("realname alipay handle expired end startTime : %s , expiredTime : %s", startTime, expiredTime)
s.realnameAlipayExpiredHandler(context.Background(), startTime, expiredTime)
time.Sleep(time.Duration(conf.Conf.Biz.RealnameAlipayCheckTick))
}
}
// realnameAlipayCheckHandler 轮询时间段 [from,to] 中，未完成阿里实名的实名申请
func (s *Service) realnameAlipayCheckHandler(c context.Context, from, to time.Time) {
if conf.Conf.Biz.RealnameAlipayCheckLimit <= 0 {
log.Error("conf.Conf.Property.realnameAlipayCheckHandler [%d] <= 0", conf.Conf.Biz.RealnameAlipayCheckLimit)
return
}
var (
applys = make([]*model.RealnameAlipayApply, conf.Conf.Biz.RealnameAlipayCheckLimit)
startID int64
err error
)
for len(applys) >= conf.Conf.Biz.RealnameAlipayCheckLimit {
if startID, applys, err = s.dao.RealnameAlipayApplyList(c, startID, model.RealnameApplyStatusPending, from, to, conf.Conf.Biz.RealnameAlipayCheckLimit); err != nil {
log.Error("%+v", err)
return
}
for _, apply := range applys {
log.Info("Start check realname alipay apply mid (%d) bizno (%s)", apply.MID, apply.Bizno)
if err = s.realnameAlipayConfirm(c, apply); err != nil {
log.Error("%+v", err)
continue
}
}
}
for len(applys) >= conf.Conf.Biz.RealnameAlipayCheckLimit {
if startID, applys, err = s.dao.RealnameAlipayApplyList(c, startID, model.RealnameApplyStatusBack, from, to, conf.Conf.Biz.RealnameAlipayCheckLimit); err != nil {
log.Error("%+v", err)
return
}
for _, apply := range applys {
log.Info("Start check realname alipay apply mid (%d) bizno (%s)", apply.MID, apply.Bizno)
if err = s.realnameAlipayConfirm(c, apply); err != nil {
log.Error("%+v", err)
continue
}
}
}
}
func (s *Service) realnameAlipayConfirm(c context.Context, apply *model.RealnameAlipayApply) (err error) {
if apply.Bizno == "" {
return
}
var (
pass bool
reason string
)
if pass, reason, err = s.alipayQuery(c, apply.Bizno); err != nil {
return
}
// rpc call
var (
rpcConfirmArg = &memmodel.ArgRealnameAlipayConfirm{
MID: apply.MID,
Pass: pass,
Reason: reason,
}
)
if err = s.memrpc.RealnameAlipayConfirm(c, rpcConfirmArg); err != nil {
return
}
log.Info("Succeed to confirm realname alipay with arg: %+v", rpcConfirmArg)
if pass {
expArg := &model.AddExp{
Mid: apply.MID,
IP: ip.InternalIP(),
Ts: time.Now().Unix(),
Event: "identify",
}
if expErr := s.addExp(context.TODO(), expArg); expErr != nil {
log.Error("realname exp error(%+v) ", expErr)
return
}
log.Info("realname exp success(%+v)", expArg)
}
return
}
func (s *Service) alipayQuery(c context.Context, bizno string) (pass bool, reason string, err error) {
var (
param url.Values
biz struct {
Bizno string `json:"biz_no"`
}
)
biz.Bizno = bizno
if param, err = s.alipayParam("zhima.customer.certification.query", biz, ""); err != nil {
return
}
if pass, reason, err = s.dao.AlipayQuery(c, param); err != nil {
return
}
return
}
// alipayParam 构造阿里请求param，biz为 biz_content struct
func (s *Service) alipayParam(method string, biz interface{}, returnURL string) (p url.Values, err error) {
var (
sign string
bizBytes []byte
)
if bizBytes, err = json.Marshal(biz); err != nil {
err = errors.WithStack(err)
return
}
p = url.Values{}
p.Set("app_id", conf.Conf.Biz.RealnameAlipayAppID)
p.Set("method", method)
p.Set("charset", "utf-8")
p.Set("sign_type", "RSA2")
p.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
p.Set("version", "1.0")
p.Set("biz_content", string(bizBytes))
if returnURL != "" {
p.Set("return_url", returnURL)
}
if sign, err = s.alipayCryptor.SignParam(p); err != nil {
return
}
p.Set("sign", sign)
return
}
// rejectExpiredRealnameAlipay 自动驳回超过两天还没有通过芝麻认证的实名认证
func (s *Service) realnameAlipayExpiredHandler(c context.Context, startTime, expiredTime time.Time) {
if conf.Conf.Biz.RealnameAlipayCheckLimit <= 0 {
log.Error("conf.Conf.Property.realnameAlipayCheckHandler [%d] <= 0", conf.Conf.Biz.RealnameAlipayCheckLimit)
return
}
var (
applys []*model.RealnameAlipayApply
startID int64
err error
)
// 每次查询（一个月里）100条过期的未处理的位处理的芝麻认证数据，进行驳回
for {
log.Info("realname handle startID (%d)", startID)
startID, applys, err = s.dao.RealnameAlipayApplyList(c, startID, model.RealnameApplyStatusPending, startTime, expiredTime, conf.Conf.Biz.RealnameAlipayCheckLimit)
if err != nil {
log.Error("realnameAlipayExpiredHandler search err(%+v)", err)
return
}
// 没有查询到预期的过期数据，则停止循环，等待下一次检查
if len(applys) == 0 {
log.Error("realnameAlipayExpiredHandler search no row in result")
return
}
// 循环驳回验证超时的芝麻认证
for _, apply := range applys {
log.Info("Start expire realname alipay apply mid (%d) bizno (%s)", apply.MID, apply.Bizno)
var (
rpcConfirmArg = &memmodel.ArgRealnameAlipayConfirm{
MID: apply.MID,
Pass: false,
Reason: "超时自动驳回",
}
)
if err = s.memrpc.RealnameAlipayConfirm(c, rpcConfirmArg); err != nil {
log.Error("realnameAlipayExpiredHandler reject err(%+v)", err)
continue
}
}
time.Sleep(10 * time.Millisecond)
}
}
// ParseIdentity to birthday and gender
func ParseIdentity(id string) (birthday time.Time, gender string, err error) {
var (
ystr, mstr, dstr, gstr string
y, m, d, g int
)
switch len(id) {
case 15:
ystr, mstr, dstr = "19"+id[6:8], id[8:10], id[10:12]
gstr = id[14:15]
case 18:
ystr, mstr, dstr = id[6:10], id[10:12], id[12:14]
gstr = id[16:17]
default:
err = errors.Errorf("identity id invalid : %s", id)
return
}
if y, err = strconv.Atoi(ystr); err != nil {
err = errors.WithStack(err)
return
}
if m, err = strconv.Atoi(mstr); err != nil {
err = errors.WithStack(err)
return
}
if d, err = strconv.Atoi(dstr); err != nil {
err = errors.WithStack(err)
return
}
if g, err = strconv.Atoi(gstr); err != nil {
err = errors.WithStack(err)
return
}
birthday = time.Date(y, time.Month(m), d, 0, 0, 0, 0, time.Local)
if g%2 == 1 {
gender = _genderMale
} else {
gender = _genderFemale
}
return
}

167
app/job/main/member/service/service.go Normal file
View File

@@ -0,0 +1,167 @@
package service
import (
"context"
"encoding/json"
"fmt"
"net"
"go-common/app/interface/main/account/service/realname/crypto"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/dao"
"go-common/app/job/main/member/model"
"go-common/app/job/main/member/model/queue"
"go-common/app/job/main/member/service/block"
memrpc "go-common/app/service/main/member/api/gorpc"
"go-common/library/log"
"go-common/library/log/infoc"
"go-common/library/queue/databus"
"go-common/library/queue/databus/databusutil"
"golang.org/x/time/rate"
)
// Service struct of service.
type Service struct {
c *conf.Config
dao *dao.Dao
block *block.Service
ds *databus.Databus
accDs *databus.Databus
passortDs *databus.Databus
logDatabus *databus.Databus
expDatabus *databus.Databus
realnameDatabus *databus.Databus
shareMidDatabus *databus.Databus
loginGroup *databusutil.Group
awardGroup *databusutil.Group
memrpc *memrpc.Service
cachepq *queue.PriorityQueue
alipayCryptor *crypto.Alipay
limiter *limiter
ParsedRealnameInfoc *infoc.Infoc
}
type limiter struct {
UpdateExp *rate.Limiter
}
// New create service instance and return.
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
ds: databus.New(c.DataBus),
accDs: databus.New(c.AccDataBus),
passortDs: databus.New(c.PassortDataBus),
logDatabus: databus.New(c.LogDatabus),
expDatabus: databus.New(c.ExpDatabus),
realnameDatabus: databus.New(c.RealnameDatabus),
shareMidDatabus: databus.New(c.ShareMidDatabus),
alipayCryptor: crypto.NewAlipay(string(c.RealnameAlipayPub), string(c.RealnameAlipayBiliPriv)),
loginGroup: databusutil.NewGroup(c.Databusutil, databus.New(c.LoginDatabus).Messages()),
awardGroup: databusutil.NewGroup(c.Databusutil, databus.New(c.AwardDatabus).Messages()),
memrpc: memrpc.New(nil),
cachepq: queue.NewPriorityQueue(1024, false),
limiter: &limiter{
UpdateExp: rate.NewLimiter(200, 10),
},
ParsedRealnameInfoc: infoc.New(c.ParsedRealnameInfoc),
}
s.dao = dao.New(c)
s.block = block.New(c, s.dao.BlockImpl(), databus.New(c.BlockCreditDatabus))
s.loginGroup.New = newMsg
s.loginGroup.Split = split
s.loginGroup.Do = s.awardDo
s.awardGroup.New = newMsg
s.awardGroup.Split = split
s.awardGroup.Do = s.awardDo
s.loginGroup.Start()
s.awardGroup.Start()
go s.passportSubproc()
go s.realnameSubproc()
go s.realnamealipaycheckproc()
go s.cachedelayproc(context.Background())
go s.shareMidproc()
accproc := int32(10)
expproc := int32(1)
if c.Biz.AccprocCount > accproc {
accproc = c.Biz.AccprocCount
}
if c.Biz.ExpprocCount > expproc {
expproc = c.Biz.ExpprocCount
}
log.Info("Starting %d account sub proc", accproc)
for i := 0; i < int(accproc); i++ {
go s.subproc()
go s.accSubproc()
go s.logproc()
}
log.Info("Starting %d exp sub proc", expproc)
for i := 0; i < int(expproc); i++ {
go s.expproc()
}
if s.c.FeatureGates.DataFixer && s.dao.LeaderEleciton(context.Background()) {
fmt.Println("Leader elected")
// 数据检查
s.makeChan(30)
go s.dataCheckMids()
for i := 0; i < 60; i++ {
go s.dataFixer(csclice[i%30])
}
}
return
}
func split(msg *databus.Message, data interface{}) int {
t, ok := data.(*model.LoginLogIPString)
if !ok {
return 0
}
return int(t.Mid)
}
func newMsg(msg *databus.Message) (res interface{}, err error) {
llm := new(model.LoginLogIPString)
ll := new(model.LoginLog)
if err = json.Unmarshal(msg.Value, &llm); err != nil {
if err = json.Unmarshal(msg.Value, &ll); err != nil {
msg.Commit()
log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
return
}
llm.Mid = ll.Mid
llm.Timestamp = ll.Timestamp
llm.Loginip = inetNtoA(uint32(ll.Loginip))
}
res = llm
return
}
func inetNtoA(sum uint32) string {
ip := make(net.IP, net.IPv4len)
ip[0] = byte((sum >> 24) & 0xFF)
ip[1] = byte((sum >> 16) & 0xFF)
ip[2] = byte((sum >> 8) & 0xFF)
ip[3] = byte(sum & 0xFF)
return ip.String()
}
// Ping check service health.
func (s *Service) Ping(c context.Context) error {
return s.dao.Ping(c)
}
// Close kafka consumer close.
func (s *Service) Close() (err error) {
if err = s.ds.Close(); err != nil {
log.Error("s.ds.Close(),err(%v)", err)
}
if err = s.accDs.Close(); err != nil {
log.Error("s.accDs.Close(),err(%v)", err)
}
if err = s.passortDs.Close(); err != nil {
log.Error("s.passportDs.Close(),err(%v)", err)
}
s.block.Close()
return
}

46
app/job/main/member/service/service_test.go Normal file
View File

@@ -0,0 +1,46 @@
package service
import (
"context"
"flag"
"testing"
"time"
"go-common/app/job/main/member/conf"
"go-common/app/job/main/member/model"
"go-common/library/log"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *Service
)
func initConf() {
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Xlog)
defer log.Close()
}
func init() {
flag.Set("conf", "../cmd/member-job-dev.toml")
initConf()
s = New(conf.Conf)
}
func TestAddexp(t *testing.T) {
Convey("addexp", t, func() {
err := s.addExp(context.Background(), &model.AddExp{Mid: 1})
So(err, ShouldBeNil)
})
}
func TestRecoverMoral(t *testing.T) {
time.Sleep(time.Second * 2)
Convey("recoverMoral", t, func() {
err := s.recoverMoral(context.Background(), 2)
So(err, ShouldBeNil)
})
}

504
app/job/main/member/service/subproc.go Normal file
View File

@@ -0,0 +1,504 @@
package service
import (
"context"
"encoding/json"
"runtime/debug"
"strings"
"time"
"go-common/app/job/main/member/model"
share "go-common/app/service/main/share/model"
"go-common/library/log"
"go-common/library/net/ip"
"github.com/pkg/errors"
)
const (
_DedeMember = "dede_member"
_AsoAccount = "aso_account"
_DedeMemberPerson = "dede_member_person"
_DedeMemberSpace = "dede_member_space"
_MemberBaseInfo = "user_base_"
_MemberMoral = "user_moral"
_memberExp = "user_exp_"
_DedeMemberTags = "dede_member_tags"
_dedeMemberMoral = "dede_member_moral"
_memberRealnameApply = "realname_apply"
_memberRealnameInfo = "realname_info"
_retry = 3
)
var (
_shareVideoType = map[int]struct{}{
1: {},
3: {},
}
)
// subproc databus sub
func (s *Service) subproc() {
var err error
var c = context.TODO()
for res := range s.ds.Messages() {
mu := &model.Message{}
if err = json.Unmarshal(res.Value, mu); err != nil {
log.Error("member-job,json.Unmarshal (%v) error(%v)", string(res.Value), err)
continue
}
for i := 0; i < _retry; i++ {
if strings.HasPrefix(mu.Table, _MemberBaseInfo) {
var (
rank = &struct {
Mid int64 `json:"mid"`
Rank int64 `json:"rank"`
}{}
)
oldRank := int64(5000)
if mu.Old != nil && len(mu.Old) != 0 {
if err = json.Unmarshal(mu.Old, rank); err != nil {
log.Error("json.Unmarsha(%s) error(%v)", string(mu.Old), err)
break
}
oldRank = rank.Rank
}
if err = json.Unmarshal(mu.New, rank); err != nil {
log.Error("json.Unmarsha(%s) error(%v)", string(mu.New), err)
break
}
newRank := rank.Rank
if oldRank <= 5000 && newRank >= 10000 {
if err = s.initExp(c, rank.Mid); err != nil {
log.Error("s.initExp(%d) error(%v)", rank.Mid, err)
}
}
if err = s.dao.DelBaseInfoCache(c, rank.Mid); err != nil {
continue
}
// sync face to old account .
s.updateAccFace(c, rank.Mid) //todo delete
// TODO: with update face or name to purge cache at the same time
if err = s.dao.NotifyPurgeCache(c, rank.Mid, model.ActUpdateFace); err != nil {
log.Error("s.dao.NotifyPurgeCache(%d, %s) error(%v)", rank.Mid, model.ActUpdateFace, err)
break
}
item := &Item{
Mid: rank.Mid,
Time: time.Now(),
Action: model.ActUpdateUname,
}
if err = s.cachepq.Put(item); err != nil {
log.Error("Failed to put into cachepq with item: %+v: %+v", item, err)
err = nil
}
log.Info("Notify to purge cache with mid(%d) action(%s) message(old: %s, new: %s)", rank.Mid, model.ActUpdateFace, string(mu.Old), string(mu.New))
} else if mu.Table == _MemberMoral {
var p = &model.MemberMid{}
if err = json.Unmarshal(mu.New, p); err != nil {
log.Error("member-job,json.Unmarshal (%v) error(%v)", string(mu.New), err)
break
}
if err = s.dao.DelMoralCache(c, p.Mid); err != nil {
continue
}
if err = s.dao.NotifyPurgeCache(c, p.Mid, model.ActUpdateMoral); err != nil {
log.Error("s.dao.NotifyPurgeCache(%d, %s) error(%v)", p.Mid, model.ActUpdateMoral, err)
break
}
} else if mu.Table == _memberRealnameInfo || mu.Table == _memberRealnameApply {
var p = &struct {
Mid int64 `json:"mid"`
Status model.RealnameApplyStatus `json:"status"`
}{}
if err = json.Unmarshal(mu.New, p); err != nil {
log.Error("member-job,json.Unmarshal (%v) error(%+v)", string(mu.New), err)
break
}
if err = s.dao.DeleteRealnameCache(c, p.Mid); err != nil {
log.Error("Delete RealnameCache cache err : %+v", err)
continue
}
if err = s.dao.NotifyPurgeCache(c, p.Mid, "updateRealname"); err != nil {
log.Error("s.dao.NotifyPurgeCache(%d, %s) error(%+v)", p.Mid, "updateRealname", err)
break
}
log.Info("Notify to purge realname cache with mid(%d) action(%s) message(old: %s, new: %s)", p.Mid, "updateRealname", string(mu.Old), string(mu.New))
if p.Status.IsPass() {
// 尝试补发一次经验
s.addExp(context.TODO(), &model.AddExp{
Mid: p.Mid,
IP: ip.InternalIP(),
Ts: time.Now().Unix(),
Event: "identify",
})
}
if mu.Table == _memberRealnameInfo {
s.syncParsedRealnameInfo(c, p.Mid)
}
} else if strings.HasPrefix(mu.Table, _memberExp) {
var (
p = &model.MemberMid{}
exp *model.NewExp
)
if err = json.Unmarshal(mu.New, p); err != nil {
log.Error("s.subproc() table(%s) json.Unmarshal() error(%v)", mu.Table, err)
break
}
if exp, err = s.dao.SelExp(c, p.Mid); err != nil {
log.Error("s.dao.SelNewExp(%d) error(%v)", p.Mid, err)
break
}
if err = s.dao.SetExpCache(c, exp.Mid, exp.Exp); err != nil {
log.Error("Failed to set exp cache: %+v: %+v", exp, err)
break
}
log.Info("s.dao.SetExpCache(%d) set exp cache complete. exp(%d)", exp.Mid, exp.Exp)
expChange, levelChange := isExpAndLevelChange(mu)
if expChange {
log.Info("Notify to purge cache with mid(%d) action(%s) message(old: %s, new: %s)", p.Mid, model.ActUpdateExp, string(mu.Old), string(mu.New))
if err = s.dao.NotifyPurgeCache(c, p.Mid, model.ActUpdateExp); err != nil {
log.Error("s.dao.NotifyPurgeCache(%d, %s) error(%v)", p.Mid, model.ActUpdateExp, err)
break
}
}
if levelChange {
log.Info("Notify to purge cache with mid(%d) action(%s) message(old: %s, new: %s)", p.Mid, model.ActUpdateLevel, string(mu.Old), string(mu.New))
if err = s.dao.NotifyPurgeCache(c, p.Mid, model.ActUpdateLevel); err != nil {
log.Error("s.dao.NotifyPurgeCache(%d, %s) error(%v)", p.Mid, model.ActUpdateLevel, err)
break
}
}
}
if err == nil {
break
}
}
if err = res.Commit(); err != nil {
log.Error("databus.Commit err(%v)", err)
}
log.Info("subproc key:%v,topic: %v, part:%v offset:%v,message %s,", res.Key, res.Topic, res.Partition, res.Offset, res.Value)
}
}
// subproc databus sub
func (s *Service) accSubproc() {
var (
err error
)
for res := range s.accDs.Messages() {
mu := &model.Message{}
ms := &struct {
Mid int64 `json:"mid"`
}{}
if err = json.Unmarshal(res.Value, mu); err != nil {
log.Error("member-job,json.Unmarshal (%v) error(%v)", string(res.Value), err)
continue
}
if err = json.Unmarshal(mu.New, ms); err != nil {
log.Error("json.Unmarsha(%s) error(%v)", string(mu.New), ms)
continue
}
for num := 0; num < 3; num++ {
switch {
//sex,face,rank
case strings.HasPrefix(mu.Table, _DedeMember) && len(mu.Table) < 14:
//err = s.setFace(c, ms.Mid)
//birthday,dating,place,marital
case mu.Table == _DedeMemberPerson:
//sign
case mu.Table == _DedeMemberSpace:
//err = s.setSign(c, ms.Mid)
//tag
case strings.HasPrefix(mu.Table, _DedeMemberTags):
//moral
case mu.Table == _dedeMemberMoral:
}
if err != nil {
log.Error("accSubproc err(%v)", err)
time.Sleep(1 * time.Second)
continue
}
break
}
if err = res.Commit(); err != nil {
log.Error("databus.Commit err(%v)", err)
}
log.Info("subproc key:%v,topic: %v, part:%v offset:%v,message %s,", res.Key, res.Topic, res.Partition, res.Offset, res.Value)
}
}
// subproc databus sub.
func (s *Service) passportSubproc() {
var err error
for res := range s.passortDs.Messages() {
mu := &model.Message{}
ms := &struct {
Mid int64 `json:"mid"`
}{}
if err = json.Unmarshal(res.Value, mu); err != nil {
log.Error("member-job,json.Unmarshal (%v) error(%v)", string(res.Value), err)
continue
}
if err = json.Unmarshal(mu.New, ms); err != nil {
log.Error("json.Unmarsha(%s) error(%v)", string(mu.New), ms)
continue
}
for num := 0; num < 3; num++ {
//name
if mu.Table == _AsoAccount {
if mu.Action != "delete" {
err = s.setName(ms.Mid)
}
}
if err != nil {
log.Error("passportSubproc err(%v)", err)
time.Sleep(1 * time.Second)
continue
}
break
}
if err = res.Commit(); err != nil {
log.Error("databus.Commit err(%v)", err)
}
log.Info("subproc key:%v,topic: %v, part:%v offset:%v,message %s,", res.Key, res.Topic, res.Partition, res.Offset, res.Value)
}
}
func (s *Service) logproc() {
for {
mu := <-s.logDatabus.Messages()
l := &model.UserLog{}
err := json.Unmarshal(mu.Value, l)
if err != nil {
log.Error("Failed to parse log databus message value: value(%s): err: %+v", string(mu.Value), err)
continue
}
// send log to report
s.dao.AddExpLog(context.TODO(), l)
// send log to origin hbase
// content := make(map[string][]byte, len(l.Content))
// for k, v := range l.Content {
// content[k] = []byte(v)
// }
// content["ip"] = []byte(l.IP)
// for i := 0; i < 3; i++ {
// err := s.dao.AddLog(context.TODO(), l.Mid, l.TS, content, model.TableExpLog)
// if err == nil {
// break
// }
// log.Error("addlog mid %d err %v", l.Mid, err)
// time.Sleep(time.Millisecond * 500)
// }
log.Info("consumer key:%s,message:%s", mu.Key, mu.Value)
mu.Commit()
}
}
func (s *Service) expproc() {
for {
mu := <-s.expDatabus.Messages()
ex := new(model.AddExp)
err := json.Unmarshal(mu.Value, ex)
if err != nil {
log.Error("s.expproc() json.Unmarshal error(%v)", err)
mu.Commit()
continue
}
try := 0
success := false
for {
if err = s.addExp(context.TODO(), ex); err == nil {
success = true
break
}
try++
if try > 3 {
log.Error("Failed to add exp, try 3 times mid: %d error: %+v", ex.Mid, err)
mu.Commit()
break
}
time.Sleep(time.Millisecond * 500)
}
if !success {
continue
}
// 如果是一个观看视频的消息就尝试补发一下登录奖励
if ex.Event == "view" {
s.addExp(context.TODO(), &model.AddExp{
Mid: ex.Mid,
IP: ex.IP,
Ts: ex.Ts,
Event: "login",
})
s.recoverMoral(context.TODO(), ex.Mid)
}
log.Info("expproc consumer key:%s,value: %s", mu.Key, mu.Value)
mu.Commit()
}
}
func isVideoShare(shareType int) bool {
_, ok := _shareVideoType[shareType]
return ok
}
func (s *Service) shareMidproc() {
for {
mu := <-s.shareMidDatabus.Messages()
sh := new(share.MIDShare)
err := json.Unmarshal(mu.Value, sh)
if err != nil {
log.Error("s.shareMidproc() json.Unmarshal error(%v)", err)
mu.Commit()
continue
}
if !isVideoShare(sh.TP) {
log.Warn("Not a video share, skip to add exp: %+v", sh)
mu.Commit()
continue
}
try := 0
success := false
ex := &model.AddExp{
Event: "share",
Mid: sh.MID,
IP: ip.InternalIP(),
Ts: sh.Time,
}
for {
if err = s.addExp(context.TODO(), ex); err == nil {
success = true
break
}
try++
if try > 3 {
log.Error("Failed to add share exp, try 3 times mid: %d error: %+v", ex.Mid, err)
mu.Commit()
break
}
time.Sleep(time.Millisecond * 500)
}
if !success {
continue
}
log.Info("shareMidproc consumer key:%s,value: %s", mu.Key, mu.Value)
mu.Commit()
}
}
func (s *Service) realnameSubproc() {
defer func() {
if x := recover(); x != nil {
log.Error("realnameSubproc panic(%+v) :\n %s", x, debug.Stack())
go s.realnameSubproc()
}
}()
log.Info("realnameSubproc run")
var (
c = context.TODO()
err error
)
for res := range s.realnameDatabus.Messages() {
msg := &model.Message{}
if err = json.Unmarshal(res.Value, msg); err != nil {
log.Error("member-job,json.Unmarshal (%v) error(%v)", string(res.Value), errors.WithStack(err))
continue
}
switch msg.Table {
case "dede_identification_card_apply":
if msg.Action == "delete" {
log.Error("dede_identification_card_apply got delete msg (%s)", msg.New)
continue
}
ms := &model.RealnameApplyMessage{}
if err = json.Unmarshal(msg.New, ms); err != nil {
err = errors.Wrapf(err, "dede_identification_card_apply , %s", msg.New)
log.Error("%+v", err)
continue
}
log.Info("upsert realname apply : (%+v)", ms)
if err = s.dao.UpdateRealnameFromMSG(c, ms); err != nil {
log.Error("%+v", err)
continue
}
if err = s.dao.DeleteRealnameCache(c, ms.MID); err != nil {
log.Error("Delete RealnameApplyStatus cache err : %+v", err)
continue
}
if err = s.dao.NotifyPurgeCache(c, ms.MID, "updateRealname"); err != nil {
log.Error("s.dao.NotifyPurgeCache(%d, %s) error(%+v)", ms.MID, "updateRealname", err)
continue
}
log.Info("Notify to purge realname cache with mid(%d) action(%s) message(old: %s, new: %s)", ms.MID, "updateRealname", string(msg.Old), string(msg.New))
case "dede_identification_card_apply_img":
if msg.Action == "delete" {
log.Error("dede_identification_card_apply_img got delete msg (%s)", msg.New)
continue
}
ms := &model.RealnameApplyImgMessage{}
if err = json.Unmarshal(msg.New, ms); err != nil {
err = errors.Wrapf(err, "dede_identification_card_apply_img , %s", msg.New)
log.Error("%+v", err)
continue
}
log.Info("upsert realname apply img : (%+v)", ms)
if err = s.dao.UpsertRealnameApplyImg(c, ms); err != nil {
log.Error("%+v", err)
continue
}
}
if err = res.Commit(); err != nil {
err = errors.Wrapf(err, "realnameSubproc commit")
log.Error("%+v", err)
}
log.Info("Realname subproc key:%v,topic: %v, part:%v offset:%v,message %s,", res.Key, res.Topic, res.Partition, res.Offset, res.Value)
}
}
func (s *Service) syncParsedRealnameInfo(ctx context.Context, mid int64) {
info, err := s.dao.RealnameInfo(ctx, mid)
if err != nil {
log.Error("Failed to fetch realname info with mid: %d: %+v", mid, err)
return
}
if info.Country != model.RealnameCountryChina ||
info.CardType != model.RealnameCardTypeIdentity {
log.Info("Skip to sync parsed realname info with mid: %d", mid)
return
}
card, err := info.DecryptedCard()
if err != nil {
log.Error("Failed to decrypt realname card with mid: %d: %+v", mid, err)
return
}
birth, gender, err := ParseIdentity(card)
if err != nil {
log.Error("Failed to parse idenitfy with mid: %d: %+v", mid, err)
return
}
// sync to hive
log.Infov(ctx,
log.KV("action", "Syning realname parsed info"),
log.KV("mid", info.MID),
log.KV("birthday", birth.Format("2006-01-02")),
log.KV("status", info.Status),
log.KV("gender", gender),
log.KV("mtime", info.MTime.Format("2006-01-02 15:04:05")),
)
s.ParsedRealnameInfoc.Infov(ctx,
birth.Format("2006-01-02"),
info.MTime.Format("2006-01-02 15:04:05"),
int64(info.Status),
int64(info.MID),
gender,
)
}