Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,40 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"common.go",
"databus.go",
"email.go",
"message.go",
"report.go",
"resource.go",
"task.go",
],
importpath = "go-common/app/job/main/aegis/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["//library/time:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/main/aegis/model/monitor:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,78 @@
package model
import (
"database/sql/driver"
"fmt"
"strconv"
xtime "time"
)
// WaitTime 计算等待时长
func WaitTime(ctime xtime.Time) string {
wt := xtime.Since(ctime)
h := int(wt.Hours())
m := int(wt.Minutes()) % 60
s := int(wt.Seconds()) % 60
return fmt.Sprintf("%.2d:%.2d:%.2d", h, m, s)
}
//IntTime .
type IntTime int64
// Scan scan time.
func (jt *IntTime) Scan(src interface{}) (err error) {
switch sc := src.(type) {
case xtime.Time:
*jt = IntTime(sc.Unix())
case string:
var i int64
i, err = strconv.ParseInt(sc, 10, 64)
*jt = IntTime(i)
}
return
}
// Value get time value.
func (jt IntTime) Value() (driver.Value, error) {
return xtime.Unix(int64(jt), 0), nil
}
// Time get time.
func (jt IntTime) Time() xtime.Time {
return xtime.Unix(int64(jt), 0)
}
// UnmarshalJSON implement Unmarshaler
func (jt *IntTime) UnmarshalJSON(data []byte) error {
if data == nil || len(data) <= 1 {
*jt = 0
return nil
}
if data[0] != '"' {
// 1.直接判断数字
sti, err := strconv.Atoi(string(data))
if err == nil {
*jt = IntTime(sti)
}
return nil
}
str := string(data[1 : len(data)-1])
// 2.标准格式判断
st, err := xtime.ParseInLocation("2006-01-02 15:04:05", str, xtime.Local)
if err == nil {
*jt = IntTime(st.Unix())
return nil
}
*jt = IntTime(0)
return nil
}
//BaseResponse .
type BaseResponse struct {
Code int64 `json:"code"`
Message string `json:"message"`
}

View File

@@ -0,0 +1,9 @@
package model
// CreateTaskMsg databus msg
type CreateTaskMsg struct {
BizID int64 `json:"business_id"`
RID int64 `json:"rid"`
FlowID int64 `json:"flow_id"`
DispatchLimit int64 `json:"dispatch_limit"`
}

View File

@@ -0,0 +1,9 @@
package model
//MoniTemp 监控邮件模板
type MoniTemp struct {
From string `json:"from"`
Members []string `json:"members"`
Subject string `json:"subject"`
Body string `json:"body"`
}

View File

@@ -0,0 +1,27 @@
package model
import (
"encoding/json"
)
const (
BinlogInsert = "insert"
BinlogUpdate = "update"
BinlogDelete = "delete"
)
// BinLog databus binlog message.
type BinLog struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
MTS int64
}
//RscMsg databus resource message
type RscMsg struct {
Action string `json:"action"`
BizID int64 `json:"business_id"`
Raw json.RawMessage `json:"raw"`
}

View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["monitor.go"],
importpath = "go-common/app/job/main/aegis/model/monitor",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,186 @@
package monitor
const (
// RedisPrefix 参数:business。参数1bid参数2监控ID
RedisPrefix = "monitor_stats_%d"
// RedisDelArcInfo 稿件删除监控key
RedisDelArcInfo = "monitor_stats_del_arc"
// BusVideo 视频业务
BusVideo = 1
// BusArc 稿件业务
BusArc = 2
// NotifyTypeEmail 邮件通知
NotifyTypeEmail = 1
// NotityTypeSms 短信通知
NotityTypeSms = 2
// 稿件业务常量
// ArchiveBitPGC 稿件PGC属性位
ArchiveBitPGC = 9
// ArchiveStateDel 稿件删除状态
ArchiveStateDel = -100
// ArchiveOriginal 自制稿件
ArchiveOriginal = 1
// RuleHighUpDelArc 高能联盟UP主大量删除稿件监控
RuleHighUpDelArc = 1
// RuleFamUpDelArc 大UP主大量删除稿件监控
RuleFamUpDelArc = 17
// CompGT 大于
CompGT = ">"
// CompLT 小于
CompLT = "<"
// CompGET 大于等于
CompGET = ">="
// CompLET 小于等于
CompLET = "<="
// CompNE 不等于
CompNE = "!="
// CompE 等于
CompE = "="
)
var (
// SpecialTypeIDs 特殊分区(变更非常不频繁)
SpecialTypeIDs = map[int64]int8{
15: 1, 34: 1, 32: 1, 82: 1, 33: 1, 83: 1, 145: 1, 146: 1,
147: 1, 153: 1, 185: 1, 186: 1, 187: 1, 37: 1, 178: 1, 179: 1,
180: 1, 128: 1, 85: 1, 86: 1, 183: 1,
}
)
// BinlogArchive 稿件 binlog 结构
type BinlogArchive struct {
ID int64 `json:"id"`
State int64 `json:"state"`
Round int64 `json:"round"`
MID int64 `json:"mid"`
Attr int64 `json:"attribute"`
TypeID int64 `json:"typeid"`
IsSpecTID int8 `json:"is_special_tid"`
HumanRank int `json:"humanrank"`
Duration int `json:"duration"`
Desc string `json:"desc"`
Title string `json:"title"`
Cover string `json:"cover"`
Content string `json:"content"`
Tag string `json:"tag"`
Copyright int8 `json:"copyright"`
AreaLimit int8 `json:"arealimit"`
Author string `json:"author"`
Access int `json:"access"`
Forward int `json:"forward"`
PubTime string `json:"pubtime"`
Reason string `json:"reject_reason"`
CTime string `json:"ctime"`
MTime string `json:"mtime"`
PTime string `json:"ptime"`
Addit *ArchiveAddit `json:"_"`
}
// BinlogVideo 视频binlog结构
type BinlogVideo struct {
ID int64 `json:"id"`
Filename string `json:"filename"`
Cid int64 `json:"cid"`
Aid int64 `json:"aid"`
Title string `json:"eptitle"`
Desc string `json:"description"`
SrcType string `json:"src_type"`
Duration int64 `json:"duration"`
Filesize int64 `json:"filesize"`
Resolutions string `json:"resolutions"`
Playurl string `json:"playurl"`
FailCode int8 `json:"failinfo"`
Index int `json:"index_order"`
Attribute int32 `json:"attribute"`
XcodeState int8 `json:"xcode_state"`
State int8 `json:"state"`
Status int16 `json:"status"`
CTime string `json:"ctime"`
MTime string `json:"mtime"`
}
// ArchiveAddit 稿件附加属性
type ArchiveAddit struct {
Aid int64 `json:"aid"`
MissionID int64 `json:"mission_id"`
UpFrom int8 `json:"up_from"`
FromIP int64 `json:"from_ip"`
IPv6 []byte `json:"ipv6"`
Source string `json:"source"`
OrderID int64 `json:"order_id"`
RecheckReason string `json:"recheck_reason"`
RedirectURL string `json:"redirect_url"`
FlowID int64 `json:"flow_id"`
Advertiser string `json:"advertiser"`
FlowRemark string `json:"flow_remark"`
DescFormatID int64 `json:"desc_format_id"`
Desc string `json:"desc"`
Dynamic string `json:"dynamic"`
}
// RuleResultRes 监控结果
type RuleResultRes struct {
Code int `json:"code"`
Data []*RuleResultData `json:"data"`
}
// RuleResultData 监控结果
type RuleResultData struct {
Rule *Rule `json:"rule"`
Stats *Stats `json:"stats"`
}
// Rule 监控规则信息
type Rule struct {
ID int64 `json:"id"`
Type int8 `json:"type"`
BID int8 `json:"bid"`
Name string `json:"name"`
State int8 `json:"state"`
STime string `json:"stime"`
ETime string `json:"etime"`
CTime string `json:"ctime"`
MTime string `json:"mtime"`
UID int64 `json:"uid"`
RuleConf *RuleConf `json:"rule"`
}
// RuleConf 监控方案配置结构体
type RuleConf struct {
Name string `json:"name"`
MoniCdt map[string]struct { //监控方案的监控条件
Comp string `json:"comparison"`
} `json:"moni_cdt"`
NotifyCdt map[string]struct { //达到发送通知的条件
Comp string `json:"comparison"`
Value int64 `json:"value"`
} `json:"notify_cdt"`
Notify struct { //通知类型配置
Way int8 `json:"way"`
Member []string `json:"member"`
} `json:"notify"`
}
// Stats 监控统计
type Stats struct {
TotalCount int `json:"total_count"`
MoniCount int `json:"moni_count"`
MaxTime int `json:"max_time"`
}
// FieldsConf 监控字段配置
type FieldsConf struct {
Comparison string
}
// DelArcInfo UP主删稿信息
type DelArcInfo struct {
AID int64 `json:"aid"`
MID int64 `json:"mid"`
Time string `json:"time"`
Title string `json:"title"`
}

View File

@@ -0,0 +1,88 @@
package model
import (
"fmt"
"strconv"
"strings"
)
//hash fields
const (
Dispatch = "ds"
Delay = "dy"
Submit = "st_%d_%d" // 参数1:提交状态(任务提交,资源提交,任务关闭) 参数2:提交前属于谁
Release = "rl"
RscState = "rs_%d"
UseTime = "ut"
SetKey = "report_set"
//type
TypeMeta = int8(0)
TypeTotal = int8(1)
)
//RIR resource item report
type RIR struct {
BizID int64
FlowID int64
UID int64
RID int64
}
//Report .
type Report struct {
ID int64 `gorm:"AUTO_INCREMENT;primary_key;"`
BusinessID int64 `gorm:"column:business_id"`
FlowID int64 `gorm:"column:flow_id"`
UID int64 `gorm:"column:uid"`
TYPE int8 `gorm:"column:type"`
Content []byte `gorm:"column:content"`
}
//TableName .
func (r Report) TableName() string {
return "task_report"
}
//PersonalHashKey .
func PersonalHashKey(bizid, flowid, uid int64) string {
return fmt.Sprintf("report_hash_%d_%d_%d", bizid, flowid, uid)
}
//TotalHashKey .
func TotalHashKey(bizid, flowid int64) string {
return fmt.Sprintf("total_inout_%d_%d_%d", bizid, flowid, 0)
}
//ParseKey .
func ParseKey(key string) (tp int8, bizid, flowid, uid int, err error) {
arr := strings.Split(key, "_")
if len(arr) != 5 {
err = fmt.Errorf(key)
return
}
prefix := arr[0] + "_" + arr[1]
switch prefix {
case "report_hash":
tp = TypeMeta
case "total_inout":
tp = TypeTotal
default:
err = fmt.Errorf(key)
return
}
if bizid, err = strconv.Atoi(arr[2]); err != nil || bizid == 0 {
err = fmt.Errorf(key)
return
}
if flowid, err = strconv.Atoi(arr[3]); err != nil || flowid == 0 {
err = fmt.Errorf(key)
return
}
if uid, err = strconv.Atoi(arr[4]); err != nil {
err = fmt.Errorf(key)
return
}
return
}

View File

@@ -0,0 +1,110 @@
package model
import (
"encoding/json"
"net/url"
"strconv"
"strings"
xtime "go-common/library/time"
)
// Resource .
type Resource struct {
ID int64 `json:"id" gorm:"primary_key" form:"id"`
BusinessID int64 `json:"business_id" gorm:"column:business_id" form:"business_id"`
OID string `json:"oid" gorm:"column:oid" form:"oid"`
MID int64 `json:"mid" gorm:"column:mid" form:"mid"`
Content string `json:"content" gorm:"column:content" form:"content"`
Extra1 int64 `json:"extra1" gorm:"column:extra1" form:"extra1"`
Extra2 int64 `json:"extra2" gorm:"column:extra2" form:"extra2"`
Extra3 int64 `json:"extra3" gorm:"column:extra3" form:"extra3"`
Extra4 int64 `json:"extra4" gorm:"column:extra4" form:"extra4"`
Extra1s string `json:"extra1s" gorm:"column:extra1s" form:"extra1s"`
Extra2s string `json:"extra2s" gorm:"column:extra2s" form:"extra2s"`
MetaData string `json:"metadata" gorm:"column:metadata" form:"metadata"`
Ctime xtime.Time `json:"ctime" gorm:"column:ctime"`
Mtime xtime.Time `json:"mtime" gorm:"column:mtime"`
Extra5 int64 `json:"extra5" gorm:"column:extra5" form:"extra5"`
Extra6 int64 `json:"extra6" gorm:"column:extra6" form:"extra6"`
Extra3s string `json:"extra3s" gorm:"column:extra3s" form:"extra3s"`
Extra4s string `json:"extra4s" gorm:"column:extra4s" form:"extra4s"`
ExtraTime1 string `json:"extratime1" gorm:"column:extratime1" form:"extratime1"`
OCtime string `json:"octime" gorm:"column:octime" form:"octime"`
Ptime string `json:"ptime" gorm:"column:ptime" form:"ptime"`
}
// TableName .
func (t *Resource) TableName() string {
return "resource"
}
//AddOption add option
type AddOption struct {
Resource
State int `form:"state" json:"state"`
NetID int64 `form:"net_id" json:"net_id"`
}
// ToQueryURI convert field to uri.
func (opt AddOption) ToQueryURI() url.Values {
var params = url.Values{}
params.Add("business_id", strconv.Itoa(int(opt.BusinessID)))
params.Add("net_id", strconv.Itoa(int(opt.NetID)))
params.Add("oid", opt.OID)
params.Add("mid", strconv.Itoa(int(opt.MID)))
params.Add("content", opt.Content)
params.Add("extra1", strconv.Itoa(int(opt.Extra1)))
params.Add("extra2", strconv.Itoa(int(opt.Extra2)))
params.Add("extra3", strconv.Itoa(int(opt.Extra3)))
params.Add("extra4", strconv.Itoa(int(opt.Extra4)))
params.Add("extra5", strconv.Itoa(int(opt.Extra5)))
params.Add("extra5", strconv.Itoa(int(opt.Extra6)))
params.Add("extra1s", opt.Extra1s)
params.Add("extra2s", opt.Extra2s)
params.Add("extra3s", opt.Extra3s)
params.Add("extra4s", opt.Extra4s)
params.Add("extratime1", opt.ExtraTime1)
params.Add("octime", opt.OCtime)
params.Add("ptime", opt.Ptime)
params.Add("metadata", opt.MetaData)
return params
}
//UpdateOption update option
type UpdateOption struct {
BusinessID int64 `json:"business_id"`
NetID int64 `json:"net_id"`
OID string `json:"oid"`
Update map[string]interface{} `json:"update"`
}
//ToQueryURI convert field to uri.
func (opt UpdateOption) ToQueryURI() url.Values {
var params = url.Values{}
params.Add("business_id", strconv.Itoa(int(opt.BusinessID)))
params.Add("net_id", strconv.Itoa(int(opt.NetID)))
params.Add("oid", opt.OID)
if bs, err := json.Marshal(opt.Update); err == nil && len(bs) > 0 {
params.Add("update", string(bs))
}
return params
}
//CancelOption .
type CancelOption struct {
BusinessID int64 `json:"business_id"`
Oids []string `json:"oids"`
Reason string `json:"reason"`
}
// ToQueryURI convert field to uri.
func (opt CancelOption) ToQueryURI() url.Values {
var params = url.Values{}
params.Add("business_id", strconv.Itoa(int(opt.BusinessID)))
params.Add("oids", strings.Join(opt.Oids, ","))
params.Add("reason", opt.Reason)
return params
}

View File

@@ -0,0 +1,184 @@
package model
import (
libtime "go-common/library/time"
)
//..
const (
//初始状态
TaskStateInit = int8(0)
//已派发
TaskStateDispatch = int8(1)
//延迟
TaskStateDelay = int8(2)
//任务提交
TaskStateSubmit = int8(3)
//资源列表提交
TaskStateRscSb = int8(4)
//任务关闭
TaskStateClosed = int8(5)
// ActionCreate 生成任务
ActionCreate = uint8(0)
// ActionSeize 抢占任务
ActionSeize = uint8(1)
// ActionRelease 释放任务
ActionRelease = uint8(2)
// ActionDelay 延迟任务
ActionDelay = uint8(3)
// ActionSubmit 提交任务
ActionSubmit = uint8(4)
// ActionUnknow 其他变更
ActionUnknow = uint8(5)
LogBusinessTask = int(232)
LogTypeTaskDispatch = int(1)
LogTypeTaskConsumer = int(2)
LogTYpeTaskWeight = int(3)
// WeightTypeCycle 周期权重
WeightTypeCycle = int8(0)
// WeightTypeConst 定值权重
WeightTypeConst = int8(1)
)
const (
// ConfigStateOn .
ConfigStateOn = int8(0)
// ConfigStateOff .
ConfigStateOff = int8(1)
// ConsumerStateOn on
ConsumerStateOn = int8(1)
// ConsumerStateOff off
ConsumerStateOff = int8(0)
// ActionConsumerOff .
ActionConsumerOff = int8(0)
// ActionConsumerOn .
ActionConsumerOn = int8(1)
// TaskConfigAssign 指派
TaskConfigAssign = int8(1)
// TaskConfigRangeWeight 权重
TaskConfigRangeWeight = int8(2)
// TaskConfigEqualWeight 权重
TaskConfigEqualWeight = int8(3)
// TaskRoleMember 组员
TaskRoleMember = int8(1)
// TaskRoleLeader 组长
TaskRoleLeader = int8(2)
)
// WeightItem 权重值
type WeightItem struct {
ID int64
Weight int64
}
// Task ..
type Task struct {
ID int64 `form:"id" json:"id" gorm:"AUTO_INCREMENT;primary_key;"`
BusinessID int64 `form:"business_id" json:"business_id" gorm:"column:business_id"`
FlowID int64 `form:"flow_id" json:"flow_id" gorm:"column:flow_id"`
RID int64 `form:"rid" json:"rid" gorm:"column:rid"`
AdminID int64 `form:"admin_id" json:"admin_id" gorm:"column:admin_id"`
UID int64 `form:"uid" json:"uid" gorm:"column:uid"`
State int8 `form:"state" json:"state" gorm:"column:state"`
Weight int64 `form:"weight" json:"weight" gorm:"column:weight"`
Utime int64 `form:"utime" json:"utime" gorm:"column:utime"`
Gtime IntTime `form:"gtime" json:"gtime" gorm:"column:gtime"`
MID int64 `form:"mid" json:"mid" gorm:"column:mid"`
Fans int64 `form:"fans" json:"fans" gorm:"column:fans"`
Group string `form:"group" json:"group" gorm:"column:group"`
Reason string `form:"reason" json:"reason" grom:"column:reason"`
Ctime IntTime `form:"ctime" json:"ctime" gorm:"column:ctime"`
Mtime IntTime `form:"mtime" json:"mtime" gorm:"column:mtime"`
}
// WeightLog task log
type WeightLog struct {
UPtime string `json:"uptime"`
Mid int64 `json:"mid"`
Fans int64 `json:"fans"`
FansWeight int64 `json:"fans_weight"`
Group string `json:"group"`
GroupWeight int64 `json:"group_weight"`
WaitTime string `json:"wait_time"`
WaitWeight int64 `json:"wait_weight"`
EqualWeight int64 `json:"config_weight"`
ConfigItems []*ConfigItem `json:"config_items"`
Weight int64 `json:"weight"`
}
// ConfigItem .
type ConfigItem struct {
Name string `json:"name"`
Desc string `json:"desc"`
Uname string `json:"uname"`
}
// EqualWeightConfig 等值权重
type EqualWeightConfig struct {
Uname string // 配置人
Description string // 描述
Name string `json:"name"` // taskid 或者 mid
IDs string `json:"ids"`
Weight int64 `json:"weight"`
Type int8 `json:"type"` // 周期或者定值
}
// RangeWeightConfig 权重
type RangeWeightConfig struct {
Name string `json:"name"`
Range []*RangeConfig `json:"range"`
}
// RangeConfig 范围配置
type RangeConfig struct {
Threshold int64 `json:"threshold"`
Weight int64 `json:"weight"`
}
// AssignConfig 指派
type AssignConfig struct {
Admin int64 `json:"-"`
Mids []int64 `json:"mids"`
Uids []int64 `json:"uids"`
}
// TaskConfig .
type TaskConfig struct {
ID int64 `form:"id" json:"id" gorm:"AUTO_INCREMENT;primary_key;"`
ConfJSON string `json:"conf_json" gorm:"column:conf_json"`
ConfType int8 `form:"conf_type" json:"conf_type" gorm:"column:conf_type"`
BusinessID int64 `form:"business_id" json:"business_id" gorm:"column:business_id"`
FlowID int64 `form:"flow_id" json:"flow_id" gorm:"column:flow_id"`
Btime libtime.Time `form:"btime" json:"btime" gorm:"column:btime"`
Etime libtime.Time `form:"etime" json:"etime" gorm:"column:etime"`
State int8 `form:"state" json:"state" gorm:"column:state"`
UID int64 `form:"uid" json:"uid" gorm:"column:uid"`
Uname string `form:"uname" json:"uname" gorm:"column:uname"`
Description string `form:"description" json:"description" gorm:"column:description"`
Ctime libtime.Time `form:"ctime" json:"ctime" gorm:"column:ctime"`
Mtime libtime.Time `form:"mtime" json:"mtime" gorm:"column:mtime"`
}
// TableName for orm
func (TaskConfig) TableName() string {
return "task_config"
}
// WeightOPT .
type WeightOPT struct {
BusinessID int64
FlowID int64
TopListLen int64
BatchListLen int64
RedisListLen int64
DbListLen int64
AssignLen int64
Minute int64
}