Create & Init Project...

app/job/main/videoup-report/model/email/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["detector_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = ["//vendor/github.com/smartystreets/goconvey/convey:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "detector.go",
+        "email.go",
+        "redis.go",
+        "template.go",
+    ],
+    importpath = "go-common/app/job/main/videoup-report/model/email",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/videoup-report/model/email/detector.go

@@ -0,0 +1,76 @@
+package email
+
+import (
+	"fmt"
+	"time"
+)
+
+//FastDetector detecte speed and unique analyze
+type FastDetector struct {
+	lastSec            int64
+	sameSecCnt         int
+	sameSecThreshold   int
+	overspeedCnt       int
+	overspeedThreshold int
+	uniqueSpeed        map[int64]int
+	fastUnique         int64
+}
+
+//NewFastDetector new
+func NewFastDetector(speedThreshold, overspeedThreshold int) *FastDetector {
+	return &FastDetector{
+		lastSec:            time.Now().Unix(),
+		sameSecCnt:         0,
+		sameSecThreshold:   speedThreshold,
+		overspeedCnt:       0,
+		overspeedThreshold: overspeedThreshold,
+		uniqueSpeed:        map[int64]int{},
+	}
+}
+
+//String string info
+func (fd *FastDetector) String() string {
+	return fmt.Sprintf("same_sec_cnt=%d,overspeed_cnt=%d,fast_unique=%d,unique_speed=%v",
+		fd.sameSecCnt, fd.overspeedCnt, fd.fastUnique, fd.uniqueSpeed)
+}
+
+//Detect 快慢探查, 超限名单只能被慢速/下一个超速名单/间隔5s后空名单代替，超限名单只有在(overspeedthreshold+1) * samesecthreshold时才确定，此时返回true
+func (fd *FastDetector) Detect(unique int64) (fast bool) {
+	now := time.Now().Unix()
+	//连续n次超限
+	if now == fd.lastSec {
+		fd.sameSecCnt++
+		if fd.sameSecCnt == fd.sameSecThreshold {
+			fd.overspeedCnt++
+		}
+	} else {
+		if fd.sameSecCnt < fd.sameSecThreshold || (now-fd.lastSec > 5) {
+			fd.overspeedCnt = 0
+			fd.fastUnique = 0
+			fd.uniqueSpeed = map[int64]int{}
+		}
+		fd.sameSecCnt = 1
+		fd.lastSec = now
+	}
+
+	//连续超限后，最先超限的unique指定为超限名单
+	if fd.overspeedCnt == fd.overspeedThreshold && fd.sameSecCnt == fd.sameSecThreshold {
+		fd.uniqueSpeed[unique] = 0
+	}
+	if (fd.overspeedCnt == fd.overspeedThreshold && fd.sameSecCnt != fd.sameSecThreshold) || (fd.overspeedCnt > fd.overspeedThreshold) {
+		fd.uniqueSpeed[unique]++
+		if fd.uniqueSpeed[unique] >= fd.sameSecThreshold {
+			fast = true
+			fd.fastUnique = unique //指定超限名单
+			fd.uniqueSpeed = map[int64]int{}
+			fd.overspeedCnt = 0
+		}
+	}
+
+	return
+}
+
+//IsFastUnique 是否为超限名单
+func (fd *FastDetector) IsFastUnique(unique int64) bool {
+	return fd.fastUnique == unique
+}

app/job/main/videoup-report/model/email/detector_test.go

@@ -0,0 +1,125 @@
+package email
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+	"time"
+)
+
+var fd *FastDetector
+var speedThreshold = 3
+var overspeedThreshold = 2
+var overlimit = speedThreshold * overspeedThreshold
+
+func TestEmailnewFastDetector(t *testing.T) {
+	convey.Convey("newFastDetector", t, func(ctx convey.C) {
+		fd = NewFastDetector(speedThreshold, overspeedThreshold)
+		ctx.Convey("Then fd should not be nil.", func(ctx convey.C) {
+			ctx.So(fd, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+//一秒多少个
+func limit(t *testing.T, ids []int64) {
+	now := time.Now().UnixNano()
+	limit := len(ids)
+	for i := 0; i < limit; i++ {
+		unique := ids[i]
+		res := fd.Detect(unique)
+		hit := fd.IsFastUnique(unique)
+		t.Logf("index=%d, unique=%d, res=%v, hit=%v, detector(%+v)", i, unique, res, hit, fd)
+	}
+	if diff := now + 1e9 - time.Now().UnixNano(); diff > 0 {
+		time.Sleep(time.Duration(diff))
+	}
+	return
+}
+
+//分片
+func batch(tk []int64, length int) (path [][]int64) {
+	ll := len(tk) / length
+	if len(tk)%length > 0 {
+		ll++
+	}
+	path = [][]int64{}
+	item := []int64{}
+	for i := 0; i < len(tk); i++ {
+		if i > 0 && i%length == 0 {
+			path = append(path, item)
+			item = []int64{}
+		}
+		item = append(item, tk[i])
+	}
+	if len(item) > 0 {
+		path = append(path, item)
+	}
+
+	return
+}
+
+func TestEmaildetect(t *testing.T) {
+	TestEmailnewFastDetector(t)
+
+	ids := []int64{222, 333}
+
+	//convey.Convey("慢速，无超限名额", t, func() {
+	//	tk := []int64{}
+	//	for i := 1; i <= overlimit*2; i++ {
+	//		tk = append(tk, ids[0])
+	//	}
+	//	path := batch(tk, speedThreshold-1)
+	//	for _, tk := range path {
+	//		limit(t, tk)
+	//	}
+	//})
+	//return
+
+	//convey.Convey("部分超限，但未超次数，无超限名额", t, func() {
+	//	tk := []int64{}
+	//	for i := 1; i < overlimit; i++ {
+	//		tk = append(tk, ids[0])
+	//	}
+	//	path := [][]int64{tk}
+	//	path = append(path, batch(tk, speedThreshold-1)...)
+	//	for _, tk := range path {
+	//		limit(t, tk)
+	//	}
+	//})
+	//return
+
+	convey.Convey("部分超限且超次数，没有回落,有超限名额且被替代, 几秒后再超限，但没有超限名单", t, func() {
+		tk := []int64{}
+		tk2 := []int64{}
+		for i := 1; i < overlimit*2; i++ {
+			tk = append(tk, ids[0])
+			tk2 = append(tk2, ids[1])
+		}
+		path := [][]int64{tk}
+		path = append(path, batch(tk, speedThreshold+1)...)
+		path = append(path, batch(tk2, speedThreshold+1)...)
+		for _, tk := range path {
+			limit(t, tk)
+		}
+		limit(t, path[len(path)-1])
+
+		sl := int64(5)
+		time.Sleep(time.Duration(sl) * time.Second)
+		t.Logf("after %ds sleep", sl)
+		limit(t, path[len(path)-1])
+	})
+
+	convey.Convey("部分超限且超次数，有超限名额，但有回落，没有超限名额", t, func(ctx convey.C) {
+		tk := []int64{}
+		for i := 1; i < overlimit*2; i++ {
+			tk = append(tk, ids[0])
+		}
+		path := batch(tk, speedThreshold+1)
+		path = append(path, batch(tk, speedThreshold-1)...)
+		for _, tk := range path {
+			limit(t, tk)
+		}
+	})
+
+}

app/job/main/videoup-report/model/email/email.go

@@ -0,0 +1,25 @@
+package email
+
+//LogBusEmail 稿件邮件任务business id
+const LogBusEmail = 141
+
+//LogTypeEmailJob 稿件邮件任务type id, 报备邮件任务
+const LogTypeEmailJob = 1
+
+//EmailPrivateVideo 一审私单
+const EmailPrivateVideo = "任务/视频私单"
+
+//EmailPrivateArchive 稿件私单
+const EmailPrivateArchive = "稿件私单"
+
+//EmailUP 十万粉丝\优质\时政\企业up主报备
+const EmailUP = "十万粉报备"
+
+//EmailMonitor 监控报警
+const EmailMonitor = "审核监控报警"
+
+//EmailResOK 邮件发送成功
+const EmailResOK = "成功"
+
+//EmailResFail 邮件发送失败
+const EmailResFail = "失败"

app/job/main/videoup-report/model/email/redis.go

@@ -0,0 +1,21 @@
+package email
+
+const (
+	//MailKey 实时邮件队列
+	MailKey = "f_mail_list"
+	//MailFastKey 超限名单的邮件队列
+	MailFastKey = "f_mail_list_fast"
+	//RetryListKey 重试列表
+	RetryListKey = "f_retry_list"
+	//RetryActionReply 评论重试
+	RetryActionReply = "reply"
+)
+
+//Retry retry
+type Retry struct {
+	AID        int64  `json:"aid"`
+	Action     string `json:"action"`
+	Flag       int64  `json:"flag"`
+	FlagA      int64  `json:"flag_a"`
+	CreateTime int64  `json:"create_time"`
+}

app/job/main/videoup-report/model/email/template.go

@@ -0,0 +1,21 @@
+package email
+
+//邮件的发件人、收件列表、抄送列表、主题
+const (
+	FROM    = "From"
+	TO      = "To"
+	CC      = "Cc"
+	SUBJECT = "Subject"
+)
+
+//Template 邮件模板
+type Template struct {
+	Headers     map[string][]string
+	Body        string
+	ContentType string
+	Type        string
+	AID         int64
+	UID         int64
+	Username    string
+	Department  string
+}