Create & Init Project...

app/interface/main/push-archive/service/BUILD

@@ -0,0 +1,73 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "archive_test.go",
+        "limit_test.go",
+        "service_test.go",
+        "setting_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/push-archive/conf:go_default_library",
+        "//app/interface/main/push-archive/dao:go_default_library",
+        "//app/interface/main/push-archive/model:go_default_library",
+        "//app/service/main/push/api/grpc/v1:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "abtest.go",
+        "archive.go",
+        "limit.go",
+        "relation.go",
+        "service.go",
+        "setting.go",
+        "statistics.go",
+    ],
+    importpath = "go-common/app/interface/main/push-archive/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/push-archive/conf:go_default_library",
+        "//app/interface/main/push-archive/dao:go_default_library",
+        "//app/interface/main/push-archive/model:go_default_library",
+        "//app/service/main/account/model:go_default_library",
+        "//app/service/main/account/rpc/client:go_default_library",
+        "//app/service/main/push/api/grpc/v1:go_default_library",
+        "//library/cache:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/queue/databus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/main/push-archive/service/abtest.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"go-common/app/interface/main/push-archive/dao"
+	"go-common/app/interface/main/push-archive/model"
+)
+
+var (
+	// 存放实验组mid尾号
+	fansTestGroup = make(map[int]struct{})
+	// 存放对照组mid尾号
+	fansComparisonGroup = make(map[int]struct{})
+	// 指定mid放进测试组
+	fansTestMids = make(map[int64]struct{})
+)
+
+func (s *Service) mappingAbtest() {
+	for _, n := range s.c.Abtest.TestGroup {
+		fansTestGroup[n] = struct{}{}
+	}
+	for _, n := range s.c.Abtest.ComparisonGroup {
+		fansComparisonGroup[n] = struct{}{}
+	}
+	for _, n := range s.c.Abtest.TestMids {
+		fansTestMids[n] = struct{}{}
+	}
+}
+
+// 将所有粉丝通过abtest规则，拆分成 (实验流量||对照组流量) && 其余流量
+func (s *Service) fansByAbtest(group *dao.FanGroup, fans []int64) (result, others []int64) {
+	for _, fan := range fans {
+		n := int(fan % 10)
+		if group.Hitby == model.GroupDataTypeAbtest {
+			if _, ok := fansTestMids[fan]; ok {
+				result = append(result, fan)
+				continue
+			}
+			if _, ok := fansTestGroup[n]; ok {
+				result = append(result, fan)
+				continue
+			}
+		} else if group.Hitby == model.GroupDataTypeAbComparison {
+			if _, ok := fansComparisonGroup[n]; ok {
+				result = append(result, fan)
+				continue
+			}
+		}
+		others = append(others, fan)
+	}
+	return
+}

app/interface/main/push-archive/service/archive.go

@@ -0,0 +1,333 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/interface/main/push-archive/dao"
+	"go-common/app/interface/main/push-archive/model"
+	accmdl "go-common/app/service/main/account/model"
+	"go-common/library/log"
+)
+
+const (
+	_insertAct = "insert"
+	_updateAct = "update"
+	_deleteAct = "delete"
+
+	_archiveTable = "archive"
+
+	_pushRetryTimes = 3
+	_pushPartSize   = 500000 // 每次请求最多推50w mid
+
+	_hbaseBatch = 100
+)
+
+func (s *Service) consumeArchiveproc() {
+	defer s.wg.Done()
+	var (
+		err  error
+		msgs = s.archiveSub.Messages()
+	)
+	for {
+		msg, ok := <-msgs
+		if !ok {
+			log.Warn("s.ArchiveSub has been closed.")
+			return
+		}
+		msg.Commit()
+		s.arcMo++
+		log.Info("consume archive key(%s) offset(%d) message(%s)", msg.Key, msg.Offset, msg.Value)
+		m := new(model.Message)
+		if err = json.Unmarshal(msg.Value, &m); err != nil {
+			log.Error("json.Unmarshal(%v) error(%v)", string(msg.Value), err)
+			continue
+		}
+		if m.Table != _archiveTable {
+			continue
+		}
+		n := new(model.Archive)
+		if err = json.Unmarshal(m.New, n); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", m.New, err)
+			continue
+		}
+		// 稿件过审
+		switch m.Action {
+		case _insertAct:
+			if !n.IsNormal() {
+				log.Info("archive (%d) upper(%d) is not normal when insert, no need to push", n.ID, n.Mid)
+				continue
+			}
+		case _updateAct:
+			o := new(model.Archive)
+			if err = json.Unmarshal(m.Old, o); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", m.Old, err)
+				continue
+			}
+			if !n.IsNormal() || o.State == n.State || o.PubTime == n.PubTime {
+				log.Info("archive (%d) upper(%d) is not normal when update, no need to push", n.ID, n.Mid)
+				continue
+			}
+		default:
+			continue
+		}
+		dao.PromInfo("archive_push")
+		// 在指定周期内，up主只推送一次
+		if s.limit(n.Mid) {
+			log.Info("upper's push limit upper(%d) aid(%d)", n.Mid, n.ID)
+			continue
+		}
+		log.Info("noticeFans start, mid(%d) aid(%d)", n.Mid, n.ID)
+		err = s.noticeFans(n) // notice fans
+		log.Info("noticeFans end, mid(%d) aid(%d) error(%v)", n.Mid, n.ID, err)
+	}
+}
+
+// isForbidTime 当前时间是否被禁止推送消息
+func (s *Service) isForbidTime() (forbid bool, err error) {
+	var forbidStart, forbidEnd time.Time
+	now := time.Now()
+	for _, f := range s.ForbidTimes {
+		forbidStart, err = s.getTodayTime(f.PushForbidStartTime)
+		if err != nil {
+			log.Error("isForbidTime getTodayTime(%s) error(%v)", f.PushForbidStartTime, err)
+			return
+		}
+		forbidEnd, err = s.getTodayTime(f.PushForbidEndTime)
+		if err != nil {
+			log.Error("isForbidTime getTodayTime(%s) error(%v)", f.PushForbidEndTime, err)
+			return
+		}
+		if now.After(forbidStart) && now.Before(forbidEnd) {
+			forbid = true
+			break
+		}
+	}
+	return
+}
+
+func (s *Service) noticeFans(arc *model.Archive) (err error) {
+	isPGC := s.isPGC(arc)
+	mids, counter, err := s.fans(arc.Mid, arc.ID, isPGC)
+	if err != nil {
+		return
+	}
+	log.Info("noticeFans get fans mid(%d) aid(%d) fans number(%d)", arc.Mid, arc.ID, counter)
+	arc.Author = s.name(arc.Mid)
+	for gKey, list := range mids {
+		g := s.dao.FanGroups[gKey]
+		params := model.NewBatchParam(map[string]interface{}{
+			"relationType": g.RelationType,
+			"archive":      arc,
+			"group":        s.getGroupParam(g),
+			"msgTemplate":  g.MsgTemplate,
+		}, model.PushParamHandler)
+		dao.Batch(&list, _pushPartSize, _pushRetryTimes, params, s.dao.NoticeFans)
+	}
+	return
+}
+
+func (s *Service) fans(upper int64, aid int64, isPGC bool) (mids map[string][]int64, counter int, err error) {
+	mids = make(map[string][]int64)
+	// 从 HBase 获取upper粉丝数据
+	fans, err := s.dao.Fans(context.TODO(), upper, isPGC)
+	ln := len(fans)
+	if err != nil {
+		log.Error("fans from hbase upper(%d) error(%v)", upper, err)
+		return
+	}
+	log.Info("filter fans -- get all fans of upper(%d), fans num(%d)", upper, ln)
+	if ln == 0 {
+		return
+	}
+	dao.PromInfoAdd("archive_fans", int64(ln))
+	// 筛选粉丝，分配到对应的实验组
+	hitFans := s.group(upper, fans)
+	hour := time.Now().Hour()
+	for gKey, hits := range hitFans {
+		log.Info("before filter by active time, upper(%d) group(%s) fans(%d)", upper, gKey, len(hits))
+		if len(hits) <= 0 {
+			continue
+		}
+		g := s.dao.FanGroups[gKey]
+		var activeFans, excluded []int64
+		if g.RelationType == model.RelationAttention {
+			activeFans, excluded = s.dao.FansByActiveTime(hour, &hits)
+			noLimitFans := s.noPushLimitFans(upper, gKey, &activeFans)
+			for _, mid := range activeFans {
+				if !s.filterUserSetting(mid, g.RelationType) {
+					excluded = append(excluded, mid)
+					log.Info("filter fans: excluded by usersettings, fan(%d), relationtype(%d), upper(%d) table(%s)", mid, g.RelationType, upper, g.HBaseTable)
+					continue
+				}
+				if !s.pushLimit(mid, upper, g, &noLimitFans) {
+					excluded = append(excluded, mid)
+					continue
+				}
+				mids[gKey] = append(mids[gKey], mid)
+				dao.PromInfo(fmt.Sprintf("%s__send", g.Name))
+				log.Info("filter fans: included by pushlimit: upper(%d)'s fan(%d) group.name(%s)", upper, mid, g.Name)
+			}
+		} else {
+			mids[gKey] = hits
+		}
+		counter += len(mids[gKey])
+		log.Info("upper(%d) aid(%d) group(%s) fans(%d)", upper, aid, gKey, len(mids[gKey]))
+		// 统计数据落表
+		s.formStatisticsProc(aid, g.Name, mids[gKey], &excluded)
+	}
+	return
+}
+
+// group 粉丝分组: 分组优先级(优先级 & 分组的可用性)-->分组规则(默认default,hbase过滤走hbase)
+func (s *Service) group(upper int64, fans map[int64]int) (list map[string][]int64) {
+	list = make(map[string][]int64)
+	// 将粉丝分为 特殊关注 和 普通关注 两类
+	attentions, specials := s.dao.FansByProportion(upper, fans)
+	log.Info("group count upper(%d) attentions(%d) specials(%d)", upper, len(attentions), len(specials))
+	// 禁止推送时间判断, 免打扰用户(针对普通关注)
+	isForbid, err := s.isForbidTime()
+	if err != nil {
+		log.Error("isForbidTime upper(%d) error(%v)", upper, err)
+	}
+	// 按顺序筛选出每个分组的数据
+	for _, gkey := range s.dao.GroupOrder {
+		g := s.dao.FanGroups[gkey]
+		if g.RelationType == model.RelationAttention && isForbid {
+			log.Info("forbid by time upper(%d) group(%+v)", upper, g)
+			continue
+		}
+		// 优先处理abtest的流量
+		if g.Hitby == model.GroupDataTypeAbtest {
+			switch g.RelationType {
+			case model.RelationAttention:
+				var pool, exists, ex []int64
+				pool, attentions = s.fansByAbtest(g, attentions)
+				for _, gk := range s.dao.GroupOrder {
+					gg := s.dao.FanGroups[gk]
+					if gg.RelationType != model.RelationAttention ||
+						gg.Hitby != model.GroupDataTypeHBase ||
+						gg.Name == "ai:pushlist_follow_recent" {
+						continue
+					}
+					ex, pool = s.dao.FansByHBase(upper, gk, &pool)
+					exists = append(exists, ex...)
+				}
+				list[gkey] = s.filterByBlacklist(upper, exists)
+			case model.RelationSpecial:
+				list[gkey], specials = s.fansByAbtest(g, specials)
+			}
+			continue
+		}
+		if g.Hitby == model.GroupDataTypeAbComparison {
+			// 对照组保持原来逻辑
+			switch g.RelationType {
+			case model.RelationAttention:
+				var pool, exists []int64
+				pool, attentions = s.fansByAbtest(g, attentions)
+				for _, gk := range s.dao.GroupOrder {
+					gg := s.dao.FanGroups[gk]
+					if gg.RelationType != model.RelationAttention || gg.Hitby != model.GroupDataTypeHBase {
+						continue
+					}
+					exists, pool = s.dao.FansByHBase(upper, gk, &pool)
+					list[gkey] = append(list[gkey], exists...)
+				}
+			case model.RelationSpecial:
+				list[gkey], specials = s.fansByAbtest(g, specials)
+			}
+			continue
+		}
+		switch {
+		case g.RelationType == model.RelationAttention && g.Hitby == model.GroupDataTypeDefault:
+			list[gkey] = attentions
+			attentions = []int64{}
+		case g.RelationType == model.RelationSpecial && g.Hitby == model.GroupDataTypeDefault:
+			list[gkey] = specials
+			specials = []int64{}
+		case g.RelationType == model.RelationAttention && g.Hitby == model.GroupDataTypeHBase:
+			list[gkey], attentions = s.dao.FansByHBase(upper, gkey, &attentions)
+		case g.RelationType == model.RelationSpecial && g.Hitby == model.GroupDataTypeHBase:
+			list[gkey], specials = s.dao.FansByHBase(upper, gkey, &specials)
+		default:
+			log.Error("group failed for grouporder(%s) & group.relationtype(%d) & hitby(%s)", gkey, g.RelationType, g.Hitby)
+		}
+	}
+	return
+}
+
+func (s *Service) filterByBlacklist(upper int64, fans []int64) (result []int64) {
+	for {
+		var mids []int64
+		l := len(fans)
+		if l == 0 {
+			return
+		}
+		if l <= _hbaseBatch {
+			mids = fans[:]
+			fans = []int64{}
+		} else {
+			mids = fans[:_hbaseBatch]
+			fans = fans[_hbaseBatch:]
+		}
+		exists, notExists := s.dao.ExistsInBlacklist(context.Background(), upper, mids)
+		result = append(result, notExists...) // 不在黑名单中的用户直接可推
+		exists, notExists = s.dao.ExistsInWhitelist(context.Background(), upper, exists)
+		result = append(result, exists...) // 存在黑名单中但是后来被恢复到白名单中的用户可推
+		for _, mid := range notExists {
+			// 只出现在黑名单中的用户不推
+			log.Warn("filter by blacklist, mid(%d)", mid)
+		}
+	}
+}
+
+// getGroupParam g.name被_分隔后的第1个/2个值（比如：ai:pushlist_offline_up的是offline, special的就是special）
+func (s *Service) getGroupParam(g *dao.FanGroup) string {
+	p := strings.Split(g.Name, "_")
+	if len(p) == 1 {
+		return p[0]
+	}
+	if g.RelationType == model.RelationSpecial {
+		return "s_" + p[1]
+	}
+	return p[1]
+}
+
+// filterUserSetting 普通关注up主的粉丝中，推送开启配置为全部推送的人很少，因此，只要其开启了特殊关注/全部关注／没有设置的就发送；特殊关注等同
+func (s *Service) filterUserSetting(mid int64, relationType int) bool {
+	if relationType == model.RelationSpecial && s.userSettings[mid] != nil && s.userSettings[mid].Type == model.PushTypeForbid {
+		return false
+	}
+	if relationType == model.RelationAttention && s.userSettings[mid] != nil && s.userSettings[mid].Type == model.PushTypeForbid {
+		return false
+	}
+	return true
+}
+
+func (s *Service) name(mid int64) (name string) {
+	arg := &accmdl.ArgMid{Mid: mid}
+	info, err := s.accRPC.Info3(context.TODO(), arg)
+	if err != nil {
+		dao.PromError("archive:获取作者信息")
+		log.Error("s.accRPC.Info3(%+v) error(%v)", arg, err)
+		return
+	}
+	name = info.Name
+	return
+}
+
+func (s *Service) isPGC(arc *model.Archive) bool {
+	return (arc.Attribute >> model.AttrBitIsPGC & 1) == 1
+}
+
+// Test for test
+// TODO delete, for test
+func (s *Service) Test(arc *model.Archive) {
+	if arc.ID == 0 || arc.Mid == 0 {
+		return
+	}
+	go s.noticeFans(arc)
+} // TODO delete, for test

app/interface/main/push-archive/service/archive_test.go

@@ -0,0 +1,75 @@
+package service
+
+import (
+	"go-common/app/interface/main/push-archive/dao"
+	"go-common/app/interface/main/push-archive/model"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func Test_groupparam(t *testing.T) {
+	initd()
+
+	expect := map[string]string{
+		"1#ai:pushlist_follow_recent": "follow",
+		"1#ai:pushlist_play_recent":   "play",
+		"1#ai:pushlist_offline_up":    "offline",
+		"2#special":                   "special",
+	}
+	convey.Convey("推送的group参数", t, func() {
+		for k, g := range s.dao.FanGroups {
+			group := s.getGroupParam(g)
+			convey.So(group, convey.ShouldEqual, expect[k])
+		}
+	})
+}
+
+func Test_usersettingfilter(t *testing.T) {
+	initd()
+
+	mid := int64(11111111)
+	s.userSettings[mid] = &model.Setting{Type: model.PushTypeForbid}
+	allow := s.filterUserSetting(mid, model.RelationSpecial)
+	convey.Convey("usersettings filter关闭开关，则排除", t, func() {
+		convey.So(allow, convey.ShouldEqual, false)
+	})
+
+	s.userSettings[mid] = nil
+	allow = s.filterUserSetting(mid, model.RelationSpecial)
+	convey.Convey("usersettings filter未设置开关，则允许", t, func() {
+		convey.So(allow, convey.ShouldEqual, true)
+	})
+
+	s.userSettings[mid] = &model.Setting{Type: model.PushTypeAttention}
+	allow = s.filterUserSetting(mid, model.RelationSpecial)
+	convey.Convey("usersettings filter设置未所有关注，则允许", t, func() {
+		convey.So(allow, convey.ShouldEqual, true)
+	})
+}
+
+func Test_ispgc(t *testing.T) {
+	arc := new(model.Archive)
+	convey.Convey("pgc稿件判断", t, func() {
+		arc.Attribute = int32(110336)
+		convey.So(s.isPGC(arc), convey.ShouldEqual, true)
+
+		arc.Attribute = int32(16512)
+		convey.So(s.isPGC(arc), convey.ShouldEqual, false)
+	})
+}
+
+func TestServicefansByAbtest(t *testing.T) {
+	initd()
+	group := &dao.FanGroup{
+		Hitby:       "ab_test",
+		HBaseTable:  "push_archive_ab_test",
+		HBaseFamily: []string{"cf"},
+	}
+	fans := []int64{1, 2, 3, 4, 5, 6}
+	convey.Convey("fansByAbtest", t, func() {
+		exists, notExists := s.fansByAbtest(group, fans)
+		t.Logf("exists(%v)", exists)
+		t.Logf("notExists(%v)", notExists)
+	})
+}

app/interface/main/push-archive/service/limit.go

@@ -0,0 +1,138 @@
+package service
+
+import (
+	"context"
+	"go-common/app/interface/main/push-archive/dao"
+	"go-common/app/interface/main/push-archive/model"
+	"go-common/library/log"
+)
+
+func (s *Service) pushLimit(fan int64, upper int64, g *dao.FanGroup, noLimitFans *map[int64]int) (allow bool) {
+	if _, ok := (*noLimitFans)[fan]; ok {
+		log.Info("included by pushlimit(%d) upper(%d) group.name(%s) without pushlimit)", fan, upper, g.Name)
+		allow = true
+		return
+	}
+	if !s.fanLimit(fan, g) {
+		log.Info("excluded by fanlimit(%d) upper(%d) group.name(%s)", fan, upper, g.Name)
+		return
+	}
+	if !s.perUpperLimit(fan, upper, g) {
+		log.Info("excluded by perupperlimit(%d) upper(%d) group.name(%s)", fan, upper, g.Name)
+		return
+	}
+
+	allow = true
+	return
+}
+
+//perUpperLimit 粉丝的在指定周期内的次数限制
+func (s *Service) perUpperLimit(fan int64, upper int64, g *dao.FanGroup) (allow bool) {
+	limit := g.PerUpperLimit
+	//没有次数限制
+	if limit <= 0 {
+		allow = true
+		return
+	}
+	//有次数限制
+	var (
+		now int
+		err error
+	)
+	if now, err = s.dao.GetPerUpperLimitCache(context.TODO(), fan, upper); err != nil {
+		log.Error("s.dao.GetPerUpperLimitCache err(%v), fan(%d), upper(%d) group.name(%s)", err, fan, upper, g.Name)
+		return
+	}
+
+	now = now + 1
+	if limit < now {
+		return
+	}
+	if err = s.dao.AddPerUpperLimitCache(context.TODO(), fan, upper, now, g.LimitExpire); err != nil {
+		log.Error("s.dao.AddPerUpperLimitCache err(%v), fan(%d), upper(%d), value(%d) group.name(%s)", err, fan, upper, now, g.Name)
+		return
+	}
+	allow = true
+	return
+}
+
+//fanLimit 粉丝的在指定周期内的次数限制
+func (s *Service) fanLimit(fan int64, g *dao.FanGroup) (allow bool) {
+	limit := g.Limit
+	//没有次数限制
+	if limit <= 0 {
+		allow = true
+		return
+	}
+	//有次数限制
+	var (
+		now int
+		err error
+	)
+	if now, err = s.dao.GetFanLimitCache(context.TODO(), fan, g.RelationType); err != nil {
+		log.Error("s.dao.GetFanLimitCache err(%v), fan(%d), group.name(%s)", err, fan, g.Name)
+		return
+	}
+
+	now = now + 1
+	if limit < now {
+		return
+	}
+	if err = s.dao.AddFanLimitCache(context.TODO(), fan, g.RelationType, now, g.LimitExpire); err != nil {
+		log.Error("s.dao.AddFanLimitCache err(%v), fan(%d), value(%d), group.name(%s)", err, fan, now, g.Name)
+		return
+	}
+	allow = true
+	return
+}
+
+// limit limits push frequency.
+func (s *Service) limit(upper int64) (limit bool) {
+	if s.dao.UpperLimitExpire == 0 {
+		return
+	}
+	limit = true
+	exist, err := s.dao.ExistUpperLimitCache(context.TODO(), upper)
+	if err != nil {
+		log.Error("s.dao.ExistUpperLimitCache(%d) error(%v)", upper, err)
+		return
+	}
+	if exist {
+		return
+	}
+
+	if err = s.dao.AddUpperLimitCache(context.TODO(), upper); err != nil {
+		log.Error("s.dao.AddUpperLimitCache(%d) error(%v)", upper, err)
+		return
+	}
+
+	limit = false
+	return
+}
+
+func (s *Service) noPushLimitFans(upper int64, fanGroupKey string, fans *[]int64) (noLimitFans map[int64]int) {
+	noLimitFans = map[int64]int{}
+	g := s.dao.FanGroups[fanGroupKey]
+	// 没有频率限制，没有免限制范围的概念
+	if g.Limit <= 0 {
+		return
+	}
+	// 只有特殊关注，才有免限制范围
+	if g.RelationType != model.RelationSpecial {
+		return
+	}
+	// 没有hbase表，没有免限制的概念
+	if len(g.HBaseTable) == 0 {
+		return
+	}
+	// abtest 不走免限制逻辑
+	if g.Hitby == model.GroupDataTypeAbtest || g.Hitby == model.GroupDataTypeAbComparison {
+		return
+	}
+	f := *fans
+	hit, _ := s.dao.FansByHBase(upper, fanGroupKey, &f)
+	for _, mid := range hit {
+		noLimitFans[mid] = 1
+	}
+	return
+}

app/interface/main/push-archive/service/limit_test.go

@@ -0,0 +1,54 @@
+package service
+
+import (
+	"testing"
+	"time"
+
+	"go-common/app/interface/main/push-archive/dao"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func Test_limit(t *testing.T) {
+	initd2()
+
+	upper := int64(113)
+	convey.Convey("upper主次数限制", t, func() {
+		convey.So(s.limit(upper), convey.ShouldEqual, false)
+		convey.So(s.limit(upper), convey.ShouldEqual, true)
+		convey.So(s.limit(upper), convey.ShouldEqual, true)
+		time.Sleep(time.Second * 2)
+		convey.So(s.limit(upper), convey.ShouldEqual, false)
+		convey.So(s.limit(upper), convey.ShouldEqual, true)
+	})
+
+	fan := int64(121)
+	g := &dao.FanGroup{
+		Limit:         2,
+		PerUpperLimit: 0,
+		LimitExpire:   2,
+	}
+	noLimitFans := &map[int64]int{}
+	convey.Convey("粉丝推送次数限制", t, func() {
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, true)
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, true)
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, false)
+		time.Sleep(time.Second * 2)
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, true)
+	})
+
+	g.PerUpperLimit = 1
+	fan = int64(333)
+	convey.Convey("粉丝推送upper主的次数限制", t, func() {
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, true)
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, false)
+		time.Sleep(time.Second * 2)
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, true)
+	})
+
+	convey.Convey("粉丝不受pushlimit限制", t, func() {
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, false)
+		(*noLimitFans)[fan] = 1
+		convey.So(s.pushLimit(fan, upper, g, noLimitFans), convey.ShouldEqual, true)
+	})
+}

app/interface/main/push-archive/service/relation.go

@@ -0,0 +1,173 @@
+package service
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/interface/main/push-archive/model"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+)
+
+const (
+	_relationMidTable     = "user_relation_mid_"
+	_relationTagUserTable = "user_relation_tag_user_"
+
+	_retry = 3
+
+	_relationStatusDeleted = 1 // 取消关注
+
+	_relationTagSpecial = int64(-10) // 特殊关注的tag
+)
+
+func (s *Service) consumeRelationproc() {
+	defer s.wg.Done()
+	var err error
+	for {
+		msg, ok := <-s.relationSub.Messages()
+		if !ok {
+			log.Warn("s.RelationSub has been closed.")
+			return
+		}
+		msg.Commit()
+		time.Sleep(time.Millisecond)
+		s.relMo++
+		log.Info("consume relation key(%s) offset(%d) message(%s)", msg.Key, msg.Offset, msg.Value)
+		m := new(model.Message)
+		if err = json.Unmarshal(msg.Value, &m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+
+		switch {
+		case strings.HasPrefix(m.Table, _relationMidTable):
+			err = s.relationMid(m.Action, m.New, m.Old)
+		case strings.HasPrefix(m.Table, _relationTagUserTable):
+			err = s.relationTagUser(m.Action, m.New, m.Old)
+		default:
+			continue
+		}
+		if err != nil {
+			log.Error("consumeRelationproc data(%s) error(%+v)", msg.Value, err)
+			if env.DeployEnv == env.DeployEnvProd {
+				s.dao.WechatMessage(fmt.Sprintf("push-archive sync relation fail error(%v)", err))
+			}
+		}
+	}
+}
+
+func (s *Service) addFans(upper, fans int64, tp int) (err error) {
+	for i := 0; i < _retry; i++ {
+		if err = s.dao.AddFans(context.TODO(), upper, fans, tp); err == nil {
+			break
+		}
+		log.Info("retry s.dao.AddFans(%d,%d,%d)", upper, fans, tp)
+	}
+	if err != nil {
+		log.Error("s.dao.AddFans(%d,%d,%d) error(%v)", upper, fans, tp, err)
+	}
+	return
+}
+
+func (s *Service) delFans(upper, fans int64) (err error) {
+	for i := 0; i < _retry; i++ {
+		if err = s.dao.DelFans(context.TODO(), upper, fans); err == nil {
+			break
+		}
+		log.Info("retry s.dao.DelFans(%d,%d)", upper, fans)
+	}
+	if err != nil {
+		log.Error("s.dao.DelFans(%d,%d) error(%v)", upper, fans, err)
+	}
+	return
+}
+
+func (s *Service) delSpecialAttention(upper, fans int64) (err error) {
+	for i := 0; i < _retry; i++ {
+		if err = s.dao.DelSpecialAttention(context.TODO(), upper, fans); err == nil {
+			break
+		}
+		log.Info("retry s.dao.DelSpecialAttention(%d,%d)", upper, fans)
+	}
+	if err != nil {
+		log.Error("s.dao.DelSpecialAttention(%d,%d) error(%v)", upper, fans, err)
+	}
+	return
+}
+
+// relationMid .
+func (s *Service) relationMid(action string, nwMsg, oldMsg []byte) (err error) {
+	n := &model.Relation{}
+	if err = json.Unmarshal(nwMsg, n); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", nwMsg, err)
+		return
+	}
+	switch action {
+	case _insertAct:
+		if !n.Following() {
+			return
+		}
+		err = s.addFans(n.Fid, n.Mid, model.RelationAttention)
+	case _updateAct:
+		o := &model.Relation{}
+		if err = json.Unmarshal(oldMsg, o); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", oldMsg, err)
+			return
+		}
+		if n.Status == o.Status && n.Attribute == o.Attribute {
+			return
+		}
+		if n.Status == _relationStatusDeleted || !n.Following() {
+			err = s.delFans(n.Fid, n.Mid) // 删除粉丝关系
+		} else {
+			err = s.addFans(n.Fid, n.Mid, model.RelationAttention) // 增加、更新关注数据
+		}
+	}
+	if err != nil {
+		log.Error("s.relationMid(%s,%s) error(%v)", nwMsg, oldMsg, err)
+	}
+	return
+}
+
+// relationTagUser .
+func (s *Service) relationTagUser(action string, nwMsg, oldMsg []byte) (err error) {
+	n := &model.RelationTagUser{}
+	if err = json.Unmarshal(nwMsg, n); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", nwMsg, err)
+		return
+	}
+	tagB, _ := base64.StdEncoding.DecodeString(n.Tag)
+	n.Tag = string(tagB)
+	switch action {
+	case _insertAct:
+		if !n.HasTag(_relationTagSpecial) {
+			return
+		}
+		err = s.addFans(n.Fid, n.Mid, model.RelationSpecial)
+	case _updateAct:
+		o := &model.RelationTagUser{}
+		if err = json.Unmarshal(oldMsg, o); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", oldMsg, err)
+			return
+		}
+		tagB, _ = base64.StdEncoding.DecodeString(o.Tag)
+		o.Tag = string(tagB)
+		nt := n.HasTag(_relationTagSpecial)
+		ot := o.HasTag(_relationTagSpecial)
+		if nt && !ot {
+			err = s.addFans(n.Fid, n.Mid, model.RelationSpecial)
+		} else if !nt && ot {
+			err = s.delSpecialAttention(n.Fid, n.Mid)
+		}
+	case _deleteAct:
+		err = s.delSpecialAttention(n.Fid, n.Mid)
+	}
+	if err != nil {
+		log.Error("s.relationTagUser(%s,%s) error(%v)", action, nwMsg, err)
+	}
+	return
+}

app/interface/main/push-archive/service/service.go

@@ -0,0 +1,177 @@
+package service
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"go-common/app/interface/main/push-archive/conf"
+	"go-common/app/interface/main/push-archive/dao"
+	"go-common/app/interface/main/push-archive/model"
+	accrpc "go-common/app/service/main/account/rpc/client"
+	pb "go-common/app/service/main/push/api/grpc/v1"
+	pushrpc "go-common/app/service/main/push/api/grpc/v1"
+	"go-common/library/cache"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/queue/databus"
+)
+
+// Service push service.
+type Service struct {
+	c            *conf.Config
+	dao          *dao.Dao
+	cache        *cache.Cache
+	accRPC       *accrpc.Service3
+	wg           sync.WaitGroup
+	archiveSub   *databus.Databus
+	relationSub  *databus.Databus
+	userSettings map[int64]*model.Setting
+	arcMo, relMo int64
+	CloseCh      chan bool
+	ForbidTimes  []conf.ForbidTime
+	settingCh    chan *pb.SetSettingRequest
+	pushRPC      pushrpc.PushClient
+}
+
+// New creates a push service instance.
+func New(c *conf.Config) *Service {
+	s := &Service{
+		c:            c,
+		dao:          dao.New(c),
+		cache:        cache.New(1, 102400),
+		accRPC:       accrpc.New3(c.AccountRPC),
+		archiveSub:   databus.New(c.ArchiveSub),
+		relationSub:  databus.New(c.RelationSub),
+		userSettings: make(map[int64]*model.Setting),
+		CloseCh:      make(chan bool),
+		ForbidTimes:  c.ArcPush.ForbidTimes,
+		settingCh:    make(chan *pb.SetSettingRequest, 3072),
+	}
+	var err error
+	if s.pushRPC, err = pushrpc.NewClient(c.PushRPC); err != nil {
+		panic(err)
+	}
+	s.mappingAbtest()
+	go s.loadUserSettingsproc()
+	time.Sleep(2 * time.Second) // consumeArchive will notice upper's fans, it depends on user's setting
+	s.wg.Add(1)
+	go s.loadUserSettingsproc()
+	if c.Push.ProdSwitch {
+		s.wg.Add(1)
+		go s.consumeRelationproc()
+		s.wg.Add(1)
+		go s.consumeArchiveproc()
+		go s.monitorConsume()
+	}
+	go s.clearStatisticsProc()
+	s.wg.Add(1)
+	go s.saveStatisticsProc()
+	go s.setSettingProc()
+	return s
+}
+
+// loadUserSettingsproc 若是用户没有设置推送开关，默认会被推送消息；因此，若是新设置load出错/无值，则不替换旧设置
+func (s *Service) loadUserSettingsproc() {
+	defer s.wg.Done()
+	var (
+		ps    int64 = 30000
+		start int64
+		end   int64
+		mxID  int64
+		err   error
+		res   map[int64]*model.Setting
+	)
+
+	for {
+		select {
+		case _, ok := <-s.CloseCh:
+			if !ok {
+				log.Info("CloseCh is closed, close the loadUserSettingsproc")
+				return
+			}
+		default:
+		}
+
+		start = 0
+		end = 0
+		err = nil
+		res = make(map[int64]*model.Setting)
+		mxID, err = s.dao.SettingsMaxID(context.TODO())
+		if err != nil || mxID == 0 {
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+
+		for {
+			start = end
+			end += ps
+			err = s.dao.SettingsAll(context.TODO(), start, end, &res)
+			if err != nil {
+				break
+			}
+			if end >= mxID {
+				break
+			}
+		}
+
+		if err != nil || len(res) == 0 {
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+		s.userSettings = res
+		time.Sleep(time.Duration(s.c.Push.LoadSettingsInterval))
+	}
+}
+
+func (s *Service) monitorConsume() {
+	if env.DeployEnv != env.DeployEnvProd {
+		return
+	}
+	var (
+		arc int64 // archive result count
+		rel int64 // relation count
+	)
+	for {
+		time.Sleep(10 * time.Minute)
+		if s.arcMo-arc == 0 {
+			msg := "databus: push-archive archiveResult did not consume within ten minute"
+			s.dao.WechatMessage(msg)
+			log.Warn(msg)
+		}
+		arc = s.arcMo
+		if s.relMo-rel == 0 {
+			msg := "databus: push-archive relation did not consume within ten minute"
+			s.dao.WechatMessage(msg)
+			log.Warn(msg)
+		}
+		rel = s.relMo
+	}
+}
+
+// Close closes service.
+func (s *Service) Close() {
+	s.archiveSub.Close()
+	s.relationSub.Close()
+	close(s.CloseCh)
+	s.wg.Wait()
+	s.dao.Close()
+}
+
+// Ping checks service.
+func (s *Service) Ping(c *bm.Context) (err error) {
+	err = s.dao.Ping(c)
+	return
+}
+
+// getTodayTime 获取当日的某个时间点的时间
+func (s *Service) getTodayTime(tm string) (todayTime time.Time, err error) {
+	now := time.Now()
+	today := now.Format("2006-01-02")
+	todayTime, err = time.ParseInLocation("2006-01-02 15:04:05", today+" "+tm, time.Local)
+	if err != nil {
+		log.Error("clearStatisticsProc time.ParseInLocation error(%v)", err)
+	}
+	return
+}

app/interface/main/push-archive/service/service_test.go

@@ -0,0 +1,185 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/interface/main/push-archive/conf"
+	"go-common/app/interface/main/push-archive/model"
+	time2 "go-common/library/time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+var s *Service
+
+func initd() {
+	dir, _ := filepath.Abs("../cmd/push-archive-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+//普通关注的3个实验组比列均为5%，尾号分别为：00~04, 05~09, 10~14
+func initd2() {
+	dir, _ := filepath.Abs("../cmd/push-archive-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+
+	conf.Conf.ArcPush.UpperLimitExpire = time2.Duration(2 * time.Second)
+
+	ps := []conf.Proportion{}
+	for i := 0; i < 5; i++ {
+		p := conf.Proportion{
+			Proportion:          "0.05",
+			ProportionStartFrom: fmt.Sprintf("%d", i*5),
+		}
+		ps = append(ps, p)
+	}
+	conf.Conf.ArcPush.Proportions = ps
+	s = New(conf.Conf)
+}
+
+func Test_todaytime(t *testing.T) {
+	initd()
+
+	todayTime, err := s.getTodayTime("06:10:00")
+	convey.Convey("当日定时时间", t, func() {
+		convey.So(err, convey.ShouldBeNil)
+	})
+	t.Logf("todaytime(%s) unix(%d)", todayTime.Format("2006-01-02 15:04:05"), todayTime.Unix())
+}
+
+func Test_forbidTime(t *testing.T) {
+	initd()
+
+	t.Logf("the forbidtimes(%v)\n", s.ForbidTimes)
+	forbid, err := s.isForbidTime()
+	convey.Convey("禁止时间判断", t, func() {
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(forbid, convey.ShouldEqual, false)
+	})
+}
+
+func Test_deadline(t *testing.T) {
+	initd()
+
+	deadline, err := s.getDeadline()
+	convey.Convey("最近第3天的凌晨", t, func() {
+		convey.So(err, convey.ShouldBeNil)
+	})
+	t.Logf("the deadline(%s) unix(%d)", deadline.Format("2006-01-02 15:04:05"), deadline.Unix())
+}
+
+func Test_proportion(t *testing.T) {
+	initd2()
+
+	fans := map[int64]int{
+		100014: model.RelationAttention,
+		100015: model.RelationAttention,
+		100016: model.RelationAttention,
+		100017: model.RelationAttention,
+		100038: model.RelationAttention,
+		100029: model.RelationAttention,
+		100070: model.RelationAttention,
+		100071: model.RelationAttention,
+		100072: model.RelationAttention,
+		100073: model.RelationSpecial,
+		10034:  model.RelationSpecial,
+		10038:  model.RelationSpecial,
+	}
+	expected := map[int]int{
+		1: 4,
+		2: 3,
+	}
+
+	attentions, specials := s.dao.FansByProportion(121231, fans)
+	convey.Convey("根据比列过滤各组", t, func() {
+		convey.So(len(attentions), convey.ShouldEqual, expected[1])
+		convey.So(len(specials), convey.ShouldEqual, expected[2])
+	})
+}
+
+func Test_Group(t *testing.T) {
+	initd2()
+	upper := int64(27515256)
+
+	//upper主所有粉丝
+	fans, err := s.dao.Fans(context.TODO(), upper, false)
+	convey.Convey("获取所有粉丝", t, func() {
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(len(fans), convey.ShouldEqual, 37)
+	})
+	attentions, specials := s.dao.FansByProportion(upper, fans)
+	t.Logf("attentions(%d), specials(%d)", len(attentions), len(specials))
+
+	convey.Convey("没有hitby，则没有命中任何分组", t, func() {
+		for _, g := range s.dao.FanGroups {
+			g.Hitby = ""
+		}
+		list := s.group(upper, fans)
+		convey.So(len(list), convey.ShouldEqual, 0)
+	})
+
+	convey.Convey("hitby=default，只命中2组", t, func() {
+		s.dao.GroupOrder = []string{"1#attention", "1#ai:pushlist_play_recent", "2#special"}
+		for _, g := range s.dao.FanGroups {
+			g.Hitby = model.GroupDataTypeDefault
+		}
+		list := s.group(upper, fans)
+		convey.So(len(list), convey.ShouldEqual, 3)
+		convey.So(len(list["1#attention"]), convey.ShouldEqual, len(attentions))
+		convey.So(len(list["1#ai:pushlist_play_recent"]), convey.ShouldEqual, 0)
+		convey.So(len(list["2#special"]), convey.ShouldEqual, len(specials))
+	})
+
+	convey.Convey("hitby=hbase，根据表命中获取", t, func() {
+		s.dao.GroupOrder = []string{"1#ai:pushlist_play_recent", "1#ai:pushlist_offline_up", "1#ai:pushlist_follow_recent", "1#attention", "2#special"}
+		for _, g := range s.dao.FanGroups {
+			if g.RelationType == 1 {
+				g.Hitby = "hbase"
+			}
+		}
+		list := s.group(upper, fans)
+		t.Logf("list(%+v)", list)
+	})
+
+	//map[1#ai:pushlist_follow_recent:[]
+	// 2#special:[21231134 4235023 1232032 2089809 88889018]
+	// 1#ai:pushlist_play_recent:[27515303 27515311 27515317 27515300 27515401 27515306]
+	// 1#ai:pushlist_offline_up:[]]
+	expect := map[string]int{
+		"1#ai:pushlist_follow_recent": 1,
+		"1#ai:pushlist_play_recent":   5,
+		"1#ai:pushlist_offline_up":    1,
+		"2#special":                   5,
+	}
+	convey.Convey("实验组粉丝优先级分组", t, func() {
+		hit := s.group(upper, fans)
+		t.Logf("the hits(%v)", hit)
+		for gkey, f := range hit {
+			convey.So(len(f), convey.ShouldEqual, expect[gkey])
+		}
+	})
+}
+
+func Test_manytimes(t *testing.T) {
+	initd2()
+
+	upper := int64(27515256)
+	mids1, len1, err1 := s.fans(upper, 1020, false)
+	time.Sleep(time.Second * 2)
+	mids2, len2, err2 := s.fans(upper, 1030, false)
+	convey.Convey(" 同一up主多次获取过滤后的粉丝，第二次比第一次少", t, func() {
+		convey.So(err1, convey.ShouldBeNil)
+		convey.So(err2, convey.ShouldBeNil)
+		convey.So(len1, convey.ShouldBeGreaterThan, 0)
+		convey.So(len2, convey.ShouldBeGreaterThan, 0)
+		convey.So(len2, convey.ShouldBeLessThanOrEqualTo, len1)
+	})
+	t.Logf("the mids1(%v)\n the mids2(%v)\n", mids1, mids2)
+}

app/interface/main/push-archive/service/setting.go

@@ -0,0 +1,77 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/interface/main/push-archive/model"
+	pb "go-common/app/service/main/push/api/grpc/v1"
+	"go-common/library/log"
+)
+
+// Setting gets user's archive-result setting.
+func (s *Service) Setting(c context.Context, mid int64) (st *model.Setting, err error) {
+	st, err = s.dao.Setting(c, mid)
+	if err != nil {
+		return
+	}
+	if st == nil {
+		st = &model.Setting{Type: model.PushTypeSpecial} // 如果用户还未上传配置，则默认为特殊关注
+	}
+	return
+}
+
+// SetSetting saves user's archive-result setting.
+func (s *Service) SetSetting(c context.Context, mid int64, st *model.Setting) (err error) {
+	err = s.dao.SetSetting(c, mid, st)
+	if err == nil {
+		set := &pb.SetSettingRequest{
+			Mid:   mid,
+			Type:  1,
+			Value: int32(st.Type),
+		}
+		s.settingCh <- set
+	}
+	return
+}
+
+func (s *Service) setSettingProc() (err error) {
+	defer func() {
+		if msg := recover(); msg != nil {
+			log.Error("setSettingProc got panic(%+v)", msg)
+		}
+	}()
+
+	for {
+		time.Sleep(time.Millisecond * 200)
+		set, open := <-s.settingCh
+		if !open {
+			log.Error("setSettingProc settingCh is closed")
+			return
+		}
+
+		// before send rpc, check db value with new value, if diff, then rpc is later than another update
+		dbSet, err := s.dao.Setting(context.TODO(), set.Mid)
+		if err != nil {
+			log.Error("setSettingProc s.dao.Setting error(%v) set(%+v)", err, set)
+			s.settingCh <- set
+			continue
+		}
+		if dbSet == nil || dbSet.Type != int(set.Value) {
+			log.Info("setSettingProc push setting value diff, db(%+v) rpc(%+v)", dbSet, set)
+			continue
+		}
+
+		// rpc中0-关闭，1-开启
+		var tp int32
+		if set.Value == model.PushTypeSpecial || set.Value == model.PushTypeAttention {
+			tp = 1
+		}
+		set.Value = tp
+
+		if _, err := s.pushRPC.SetSetting(context.TODO(), set); err != nil {
+			log.Error("s.pushRPC.SetSetting error(%v) set(%+v)", err, set)
+			s.settingCh <- set
+		}
+	}
+}

app/interface/main/push-archive/service/setting_test.go

@@ -0,0 +1,116 @@
+package service
+
+import (
+	"context"
+	ht "net/http"
+	"sync"
+	"testing"
+
+	"go-common/app/interface/main/push-archive/conf"
+	"go-common/app/interface/main/push-archive/model"
+	pb "go-common/app/service/main/push/api/grpc/v1"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func Test_getsetting(t *testing.T) {
+	initd()
+
+	mid := int64(11111111)
+	s.SetSetting(context.TODO(), mid, nil)
+	convey.Convey("获取默认的用户开关设置", t, func() {
+		st, err := s.Setting(context.TODO(), mid)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(st.Type, convey.ShouldEqual, model.PushTypeSpecial)
+	})
+}
+
+func Test_setsetting(t *testing.T) {
+	initd()
+
+	mid := int64(11111111)
+	convey.Convey("存储用户开关设置", t, func() {
+		err := s.SetSetting(context.TODO(), mid, &model.Setting{Type: model.PushTypeAttention})
+		convey.So(err, convey.ShouldBeNil)
+	})
+}
+
+func Test_ps(t *testing.T) {
+	initd()
+
+	url := "http://127.0.0.1:7031/x/push-archive/setting/get?access_key=848657e31639317257ab274741c6ec7d"
+	client := bm.NewClient(conf.Conf.HTTPClient)
+	type _response struct {
+		Code int           `json:"code"`
+		Data model.Setting `json:"data"`
+	}
+
+	wg := sync.WaitGroup{}
+	all := 10
+	busyIn := 0
+	errIn := 0
+	normalIn := 0
+	mx := sync.Mutex{}
+	for i := 0; i < all; i++ {
+		wg.Add(1)
+
+		go func(i int) {
+			defer wg.Done()
+			defer mx.Unlock()
+
+			req, err := ht.NewRequest("GET", url, nil)
+			if err != nil {
+				mx.Lock()
+				errIn++
+				return
+			}
+
+			r := &_response{}
+			err = client.Do(context.TODO(), req, r)
+			mx.Lock()
+			if err != nil {
+				errIn++
+				return
+			}
+
+			if r.Code == ecode.ServiceUnavailable.Code() {
+				busyIn++
+				return
+			}
+			normalIn++
+		}(i)
+	}
+
+	wg.Wait()
+	convey.Convey("并行访问推送开关，有频率限制", t, func() {
+		convey.So(errIn, convey.ShouldEqual, 0)
+		convey.So(busyIn, convey.ShouldBeGreaterThan, 0)
+		convey.So(errIn+busyIn+normalIn, convey.ShouldEqual, all)
+	})
+	t.Logf("the errin(%d), busyin(%d), normalin(%d)", errIn, busyIn, normalIn)
+}
+
+func Test_pushrpc(t *testing.T) {
+	initd()
+
+	convey.Convey("推送平台rpc设置推送开关", t, func() {
+		set := &pb.SetSettingRequest{
+			Mid:   int64(111111111),
+			Type:  1,
+			Value: 3,
+		}
+		res, err := s.pushRPC.Setting(context.TODO(), &pb.SettingRequest{Mid: set.Mid})
+		convey.So(err, convey.ShouldBeNil)
+		t.Logf("setting(%+v)", res)
+
+		_, err = s.pushRPC.SetSetting(context.TODO(), set)
+		convey.So(err, convey.ShouldBeNil)
+
+		res, err = s.pushRPC.Setting(context.TODO(), &pb.SettingRequest{Mid: set.Mid})
+		convey.So(err, convey.ShouldBeNil)
+		t.Logf("setting(%+v)", res)
+	})
+
+}

app/interface/main/push-archive/service/statistics.go

@@ -0,0 +1,142 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"go-common/app/interface/main/push-archive/dao"
+	"go-common/app/interface/main/push-archive/model"
+	"go-common/library/log"
+)
+
+// 切割需要存储的统计数据
+func (s *Service) formStatisticsProc(aid int64, group string, included []int64, excluded *[]int64) {
+	params := model.NewBatchParam(map[string]interface{}{
+		"aid":         aid,
+		"group":       group,
+		"type":        model.StatisticsPush,
+		"createdTime": time.Now(),
+	}, nil)
+	dao.Batch(&included, 1000, 2, params, s.formPushStatistic)
+
+	params.Params["type"] = model.StatisticsUnpush
+	dao.Batch(excluded, 1000, 2, params, s.formPushStatistic)
+}
+
+// 组建统计对象
+func (s *Service) formPushStatistic(fans *[]int64, params map[string]interface{}) (err error) {
+	ln := len(*fans)
+	b, err := json.Marshal(*fans)
+	if err != nil {
+		log.Error("formStatistic json.Marshal error(%v) fans(%v) params(%v)", err, fans, params)
+		return
+	}
+	ps := &model.PushStatistic{
+		Aid:         params["aid"].(int64),
+		Group:       params["group"].(string),
+		Type:        params["type"].(int),
+		Mids:        string(b),
+		MidsCounter: ln,
+		CTime:       params["createdTime"].(time.Time),
+	}
+	if err = s.dao.AddStatisticsCache(context.TODO(), ps); err != nil {
+		log.Error("formPushStatistic s.dao.AddStatisticsCache error(%v), pushstatistic(%v)", err, ps)
+		return
+	}
+	return
+}
+
+// 每日定时清除推送的统计数据,只保留最近几天的数据
+func (s *Service) clearStatisticsProc() {
+	for {
+		// 到指定时间
+		clearTime, err := s.getTodayTime(s.c.ArcPush.PushStatisticsClearTime)
+		if err != nil {
+			log.Error("clearStatisticsProc getTodayTime(%s) error(%v)", s.c.ArcPush.PushStatisticsClearTime, err)
+			continue
+		}
+		dur := clearTime.Unix() - time.Now().Unix()
+		if dur < 0 || dur > 60 {
+			time.Sleep(time.Second * 50)
+			continue
+		}
+		// 需要删除数据的最大时间
+		time.Sleep(time.Second * 60)
+		deadline, err := s.getDeadline()
+		if err != nil {
+			log.Error("clearStatisticsProc getDeadline error(%v)", err)
+			continue
+		}
+		log.Info("start to clear statistics before deadline(%s)", deadline.Format("2006-01-02 15:04:05"))
+		var min, max, mid int64
+		for i := 0; i < 3; i++ {
+			if min == 0 && max == 0 {
+				min, max, err = s.dao.GetStatisticsIDRange(context.TODO(), deadline)
+				mid = min
+			}
+			for err == nil && mid < max {
+				min = mid
+				mid = min + 5000
+				if mid > max {
+					mid = max
+				}
+				_, err = s.dao.DelStatisticsByID(context.TODO(), min, mid)
+				time.Sleep(time.Second)
+			}
+			if err == nil {
+				log.Info("success end clear statistics before deadline(%s)", deadline.Format("2006-01-02 15:04:05"))
+				break
+			}
+		}
+		if err != nil {
+			s.dao.WechatMessage(fmt.Sprintf("clearStatisticsProc: push-archive failed to clear expired(%s) push_statistics, error(%v)", deadline.Format("2006-01-02 15:04:05"), err))
+		}
+	}
+}
+
+// 获取需要删除数据的 最大时间
+func (s *Service) getDeadline() (deadline time.Time, err error) {
+	dd := "00:00:00"
+	today, err := s.getTodayTime(dd)
+	if err != nil {
+		log.Error("clearStatisticsProc getTodayTime(%s) error(%v)", dd, err)
+		return
+	}
+
+	deadline = today.AddDate(0, 0, -1*s.c.ArcPush.PushStatisticsKeepDays+1)
+	return
+}
+
+// 统计数据落库
+func (s *Service) saveStatisticsProc() {
+	defer s.wg.Done()
+	for {
+		select {
+		case _, ok := <-s.CloseCh:
+			if !ok {
+				log.Info("CloseCh is closed, close the saveStatisticsProc")
+				return
+			}
+		default:
+		}
+
+		ps, err := s.dao.GetStatisticsCache(context.TODO())
+		if err != nil {
+			log.Error("saveStatisticsProc s.dao.GetStatisticsCache error(%v)", err)
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+		if ps == nil {
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+
+		if _, err := s.dao.SetStatistics(context.TODO(), ps); err != nil {
+			log.Error("saveStatisticsProc s.dao.SetStatistics error(%v) pushstatistic(%v)", err, ps)
+			s.dao.AddStatisticsCache(context.TODO(), ps)
+		}
+		time.Sleep(time.Millisecond * 100)
+	}
+}