Create & Init Project...

app/interface/live/push-live/dao/BUILD

@@ -0,0 +1,75 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "config_test.go",
+        "d_test.go",
+        "filter_test.go",
+        "hbase_test.go",
+        "push_test.go",
+        "switch_test.go",
+        "task_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/live/push-live/conf:go_default_library",
+        "//app/interface/live/push-live/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "blacklist.go",
+        "config.go",
+        "dao.go",
+        "filter.go",
+        "hbase.go",
+        "push.go",
+        "switch.go",
+        "task.go",
+    ],
+    importpath = "go-common/app/interface/live/push-live/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/live/push-live/conf:go_default_library",
+        "//app/interface/live/push-live/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//library/xstr:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/live/push-live/dao/blacklist.go

@@ -0,0 +1,58 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"github.com/pkg/errors"
+	"github.com/tsuna/gohbase/hrpc"
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/log"
+	"strconv"
+	"strings"
+)
+
+var (
+	_hbaseTable  = "live:push_blacklist"
+	_hbaseFamily = "blacklist"
+
+	errLinkValueSplit = errors.New("link_value split result nil.")
+)
+
+// GetBlackList get blacklist from hbase by target id
+func (d *Dao) GetBlackList(c context.Context, task *model.ApPushTask) (mids map[int64]bool, err error) {
+	var (
+		key         string
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, d.blackListHBaseReadTimeout)
+		emptyByte   = []byte("")
+		fbytes      = []byte(_hbaseFamily)
+	)
+	defer cancel()
+	split := strings.Split(task.LinkValue, ",")
+	if split == nil {
+		err = errLinkValueSplit
+		return
+	}
+	key = split[0]
+	mids = make(map[int64]bool)
+	if result, err = d.blackListHBase.GetStr(ctx, _hbaseTable, key); err != nil {
+		log.Error("[dao.blacklist|GetBlackList] d.blackListHBase.Get error(%v) querytable(%v), roomid(%s), task(%v)",
+			err, _hbaseTable, key, task)
+		return
+	}
+	if result == nil {
+		return
+	}
+
+	for _, c := range result.Cells {
+		if c != nil && bytes.Equal(c.Family, fbytes) && !bytes.Equal(c.Qualifier, emptyByte) {
+			uid, e := strconv.ParseInt(string(c.Qualifier), 10, 64)
+			if e != nil {
+				continue
+			}
+			mids[uid] = true
+		}
+	}
+	log.Info("[dao.blacklist|GetBlackList] get blacklist(%v), roomid(%s), task(%v)", mids, key, task)
+	return
+}

app/interface/live/push-live/dao/config.go

@@ -0,0 +1,43 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/log"
+)
+
+const (
+	_getPushConfig   = "SELECT `type` FROM ap_push_config WHERE value=? ORDER BY `order` ASC" // 获取推送选项配置
+	_getPushInterval = "SELECT `order` FROM ap_push_config WHERE type=?"                      // 获取推送间隔时间
+)
+
+// GetPushConfig 从DB中获取推送配置
+func (d *Dao) GetPushConfig(c context.Context) (types []string, err error) {
+	var t string
+	types = make([]string, 0)
+	rows, err := d.db.Query(c, _getPushConfig, model.LivePushConfigOn)
+	if err != nil {
+		log.Error("[dao.config|GetPushConfig] db.Query() error(%v)", err)
+		return
+	}
+	for rows.Next() {
+		if err = rows.Scan(&t); err != nil {
+			log.Error("[dao.config|GetPushConfig] rows.Scan() error(%v)", err)
+			return
+		}
+		types = append(types, t)
+	}
+	return
+}
+
+// GetPushInterval 获取推送时间间隔
+func (d *Dao) GetPushInterval(c context.Context) (interval int32, err error) {
+	var i int32
+	row := d.db.QueryRow(c, _getPushInterval, model.PushIntervalKey)
+	if err = row.Scan(&i); err != nil {
+		log.Error("[dao.config|GetPushInterval] row.Scan() error(%v)", err)
+		return
+	}
+	interval = i * 60 // min to sec
+	return
+}

app/interface/live/push-live/dao/config_test.go

@@ -0,0 +1,26 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_config(t *testing.T) {
+	initd()
+	Convey("Test config get", t, func() {
+		cf, err := d.GetPushConfig(context.TODO())
+		t.Logf("the result included(%v) err(%v)", cf, err)
+		So(err, ShouldEqual, nil)
+	})
+}
+
+func Test_GetPushInterval(t *testing.T) {
+	initd()
+	Convey("Test GetPushInterval", t, func() {
+		cf, err := d.GetPushInterval(context.TODO())
+		t.Logf("the result included(%v) err(%v)", cf, err)
+		So(err, ShouldEqual, nil)
+	})
+}

app/interface/live/push-live/dao/d_test.go

@@ -0,0 +1,17 @@
+package dao
+
+import (
+	"flag"
+	"go-common/app/interface/live/push-live/conf"
+	"path/filepath"
+)
+
+var d *Dao
+
+func initd() {
+	dir, _ := filepath.Abs("../cmd/push-live-test.toml")
+	flag.Set("conf", dir)
+	flag.Set("conf_env", "uat")
+	conf.Init()
+	d = New(conf.Conf)
+}

app/interface/live/push-live/dao/dao.go

@@ -0,0 +1,96 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/interface/live/push-live/conf"
+	"go-common/library/cache/redis"
+	"go-common/library/database/hbase.v2"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	xhttp "go-common/library/net/http/blademaster"
+	"go-common/library/stat/prom"
+)
+
+// Dao dao
+type Dao struct {
+	c                         *conf.Config
+	db                        *xsql.DB
+	httpClient                *xhttp.Client
+	relationHBase             *hbase.Client
+	relationHBaseReadTimeout  time.Duration
+	blackListHBase            *hbase.Client
+	blackListHBaseReadTimeout time.Duration
+}
+
+// Prometheus
+var (
+	errorsCount = prom.BusinessErrCount
+	infosCount  = prom.BusinessInfoCount
+)
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:                         c,
+		db:                        xsql.NewMySQL(c.MySQL),
+		relationHBase:             hbase.NewClient(c.HBase.Config),
+		relationHBaseReadTimeout:  time.Duration(c.HBase.ReadTimeout),
+		httpClient:                xhttp.NewClient(c.HTTPClient),
+		blackListHBase:            hbase.NewClient(c.BlackListHBase.Config),
+		blackListHBaseReadTimeout: time.Duration(c.BlackListHBase.ReadTimeout),
+	}
+	return
+}
+
+// RedisOption return redis options
+func (d *Dao) RedisOption() []redis.DialOption {
+	cnop := redis.DialConnectTimeout(time.Duration(d.c.Redis.PushInterval.DialTimeout))
+	rdop := redis.DialReadTimeout(time.Duration(d.c.Redis.PushInterval.ReadTimeout))
+	wrop := redis.DialWriteTimeout(time.Duration(d.c.Redis.PushInterval.WriteTimeout))
+	return []redis.DialOption{cnop, rdop, wrop}
+}
+
+// Close close the resource.
+func (d *Dao) Close() (err error) {
+	if err = d.relationHBase.Close(); err != nil {
+		log.Error("[dao.dao|Close] d.relationHBase.Close() error(%v)", err)
+		PromError("hbase:close")
+	}
+	if err = d.db.Close(); err != nil {
+		log.Error("[dao.dao|Close] d.db.Close() error(%v)", err)
+		PromError("db:close")
+	}
+	return
+}
+
+// PromError prom error
+func PromError(name string) {
+	errorsCount.Incr(name)
+}
+
+// PromInfo add prom info
+func PromInfo(name string) {
+	infosCount.Incr(name)
+}
+
+//PromInfoAdd add prom info by value
+func PromInfoAdd(name string, value int64) {
+	infosCount.Add(name, value)
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.db.Ping(c); err != nil {
+		PromError("mysql:Ping")
+		log.Error("[dao.dao|Ping] d.db.Ping error(%v)", err)
+		return
+	}
+	if err = d.relationHBase.Ping(c); err != nil {
+		PromError("hbase:Ping")
+		log.Error("[dao.dao|Ping] d.relationHBase.Ping error(%v)", err)
+		return
+	}
+	return
+}

app/interface/live/push-live/dao/filter.go

@@ -0,0 +1,263 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_intervalUserkey   = "i:%d"     // 用户推送间隔缓存
+	_limitUserDailyKey = "daily:%d" //用户每日推送额度缓存
+	_defaultPushLimit  = 4          // 用户每日默认最大推送额度
+)
+
+// FilterConfig FilterConfig
+type FilterConfig struct {
+	Business        int
+	IntervalExpired int32
+	IntervalValue   string
+	DailyExpired    float64
+	Task            *model.ApPushTask
+}
+
+// Filter Filter
+type Filter struct {
+	conf *FilterConfig
+	conn redis.Conn
+}
+
+// FilterChain FilterChain
+type FilterChain map[string]func(ctx context.Context, mid int64) (bool, error)
+
+// NewFilter NewFilter
+func (d *Dao) NewFilter(conf *FilterConfig) (f *Filter, err error) {
+	var conn redis.Conn
+	// redis conn
+	conn, err = redis.Dial(d.c.Redis.PushInterval.Proto, d.c.Redis.PushInterval.Addr, d.RedisOption()...)
+	if err != nil {
+		log.Error("[dao.filter|NewFilter] redis.Dial error(%v), conf(%v)", err, conf)
+		return
+	}
+	f = &Filter{
+		conf: conf,
+		conn: conn,
+	}
+	return
+}
+
+// NewFilterChain NewFilterChain
+func (d *Dao) NewFilterChain(f *Filter) FilterChain {
+	funcs := make(FilterChain)
+	if d.needLimit(f.conf.Business) {
+		funcs["limit"] = f.dailyLimitFilter
+	}
+	if d.needSmooth(f.conf.Business) && f.conf.IntervalExpired > 0 {
+		if f.conf.Business == model.ActivityBusiness {
+			funcs["smooth"] = f.appointSmoothFilter
+		} else {
+			funcs["smooth"] = f.intervalSmoothFilter
+		}
+	}
+	return funcs
+}
+
+// needSmooth
+func (d *Dao) needSmooth(business int) bool {
+	return !ignoreFilter(business, d.c.Push.PushFilterIgnores.Smooth)
+}
+
+// needLimit
+func (d *Dao) needLimit(business int) bool {
+	return !ignoreFilter(business, d.c.Push.PushFilterIgnores.Limit)
+}
+
+// Done do some close work
+func (f *Filter) Done() {
+	if f.conn != nil {
+		f.conn.Close()
+	}
+}
+
+// dailyLimitFilter 判断是否到达每日推送上限
+func (f *Filter) dailyLimitFilter(ctx context.Context, mid int64) (b bool, err error) {
+	var (
+		left int
+		key  = fmt.Sprintf(_limitUserDailyKey, mid)
+	)
+
+	// fetch daily push count
+	left, err = redis.Int(f.conn.Do("GET", key))
+	if err != nil {
+		if err == redis.ErrNil {
+			// key not exists, should return false & nil, first push today
+			err = nil
+			return
+		}
+		// actually error occurs
+		return
+	}
+
+	// daily push limit
+	if left <= 0 {
+		b = true
+		return
+	}
+	return
+}
+
+// intervalSmoothFilter 判断是否被平滑推送逻辑过滤
+func (f *Filter) intervalSmoothFilter(ctx context.Context, mid int64) (b bool, err error) {
+	var reply interface{}
+	key := fmt.Sprintf(_intervalUserkey, mid)
+	reply, err = f.conn.Do("SET", key, f.conf.IntervalValue, "EX", f.conf.IntervalExpired, "NX")
+	if err != nil {
+		return
+	}
+	// key exists, nil returned
+	// key not exists, will return OK
+	if reply == nil {
+		b = true
+		return
+	}
+	return
+}
+
+// appointSmoothFilter 预约逻辑的平滑
+func (f *Filter) appointSmoothFilter(ctx context.Context, mid int64) (b bool, err error) {
+	var reply interface{}
+	key := fmt.Sprintf(_intervalUserkey, mid)
+	reply, err = f.conn.Do("SET", key, f.conf.IntervalValue, "EX", f.conf.IntervalExpired, "NX")
+	if err != nil {
+		return
+	}
+	// key exists, nil returned
+	// key not exists, will return OK
+	if reply == nil {
+		// 活动预约有特殊判断逻辑
+		reply, err = redis.String(f.conn.Do("GET", key))
+		if err != nil {
+			return
+		}
+		// 相同房间会被过滤
+		if reply == f.conf.IntervalValue {
+			b = true
+		}
+		return
+	}
+	return
+}
+
+// BatchFilter 对输入mid序列执行所有过滤方法，返回过滤结果
+func (f *Filter) BatchFilter(ctx context.Context, filterChain FilterChain, mids []int64) (resMids []int64) {
+	if len(mids) == 0 || len(filterChain) == 0 {
+		return
+	}
+	var (
+		isFiltered bool
+		err        error
+		filterMids = make(map[string][]int64)
+		errMids    = make(map[string][]int64)
+	)
+	defer func() {
+		filterMids = nil
+		errMids = nil
+	}()
+	resMids = make([]int64, 0, len(mids))
+
+	// 记录被过滤掉的mid，过滤发生错误的mid
+	for name := range filterChain {
+		filterMids[name] = make([]int64, 0, len(mids))
+		errMids[name] = make([]int64, 0, len(mids))
+	}
+
+MidLoop:
+	for _, mid := range mids {
+		for name, fc := range filterChain {
+			isFiltered, err = fc(ctx, mid)
+			// error occurs, next mid
+			if err != nil {
+				errMids[name] = append(errMids[name], mid)
+				continue MidLoop
+			}
+			// filtered by any filterChain func, next mid
+			if isFiltered {
+				filterMids[name] = append(filterMids[name], mid)
+				continue MidLoop
+			}
+		}
+		// mid here is filter result, should push
+		resMids = append(resMids, mid)
+	}
+
+	// log
+	for name, ids := range filterMids {
+		if len(ids) == 0 {
+			continue
+		}
+		log.Info("[dao.filter|BatchFilter] BatchFilter filterMids, task(%v), len(%d), name(%s), mids(%d)",
+			f.conf.Task, len(ids), name, len(mids))
+	}
+	for name, ids := range errMids {
+		if len(ids) == 0 {
+			continue
+		}
+		log.Error("[dao.filter|BatchFilter] BatchFilter errMids, task(%v), len(%d), name(%s), err(%v)",
+			f.conf.Task, len(ids), name, err)
+	}
+	return
+}
+
+// BatchDecreaseLimit 批量减少配额
+func (f *Filter) BatchDecreaseLimit(ctx context.Context, mids []int64) (total int, err error) {
+	defer func() {
+		if f != nil {
+			f.Done()
+		}
+		log.Info("[dao.filter|BatchDecreaseLimit] business(%d), input(%d), exec(%d), err(%v)",
+			f.conf.Business, len(mids), total, err)
+	}()
+	if len(mids) == 0 {
+		return
+	}
+	initLeft := _defaultPushLimit - 1
+	for _, mid := range mids {
+		key := fmt.Sprintf(_limitUserDailyKey, mid)
+		left, err := redis.Int(f.conn.Do("GET", key))
+		if err != nil {
+			if err == redis.ErrNil {
+				// key not exists
+				f.conn.Do("SET", key, initLeft, "EX", f.conf.DailyExpired)
+				total++
+			}
+			continue
+		}
+		f.conn.Do("SET", key, left-1, "EX", f.conf.DailyExpired)
+		total++
+	}
+	return
+}
+
+// ignoreFilter 判断business是否能够不需要过滤
+func ignoreFilter(business int, ignores []int) bool {
+	var f = false
+	for _, ignore := range ignores {
+		if business == ignore {
+			f = true
+		}
+	}
+	return f
+}
+
+// GetIntervalKey return interval smooth redis key
+func GetIntervalKey(mid int64) string {
+	return fmt.Sprintf(_intervalUserkey, mid)
+}
+
+// GetDailyLimitKey return daily limit redis key
+func GetDailyLimitKey(mid int64) string {
+	return fmt.Sprintf(_limitUserDailyKey, mid)
+}

app/interface/live/push-live/dao/filter_test.go

@@ -0,0 +1,335 @@
+package dao
+
+import (
+	"context"
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"math/rand"
+	"testing"
+)
+
+func setRedis(conn redis.Conn, key string, value interface{}, ttl int32) error {
+	_, err := conn.Do("SET", key, value, "EX", ttl)
+	return err
+}
+
+// delRedis
+func delRedis(conn redis.Conn, key string) error {
+	_, err := conn.Do("DEL", key)
+	return err
+}
+
+func filterClean(f *Filter, mids []int64) {
+	for _, mid := range mids {
+		keys := []string{
+			GetDailyLimitKey(mid),
+			GetIntervalKey(mid),
+		}
+		for _, key := range keys {
+			delRedis(f.conn, key)
+		}
+	}
+	f.Done()
+	f = nil
+}
+
+func TestDao_needSmooth(t *testing.T) {
+	initd()
+	Convey("test business need smooth", t, func() {
+		var (
+			business int
+			b        bool
+		)
+		Convey("test need smooth", func() {
+			business = rand.Intn(100)
+			b = d.needSmooth(business)
+			So(b, ShouldEqual, true)
+
+			business = 111
+			b = d.needSmooth(business)
+			So(b, ShouldEqual, true)
+		})
+		Convey("test no need smooth", func() {
+			business = 101
+			b = d.needSmooth(business)
+			So(b, ShouldEqual, false)
+		})
+	})
+}
+
+func TestDao_needLimit(t *testing.T) {
+	initd()
+	Convey("test business need limit", t, func() {
+		var (
+			business int
+			b        bool
+		)
+		Convey("test need limit", func() {
+			business = rand.Intn(110)
+			b = d.needSmooth(business)
+			So(b, ShouldEqual, true)
+		})
+		Convey("test no need smooth", func() {
+			business = 111
+			b = d.needSmooth(business)
+			So(b, ShouldEqual, true)
+		})
+	})
+}
+
+func TestDao_NewFilterChain(t *testing.T) {
+	initd()
+	Convey("test new filter chain", t, func() {
+		var (
+			f    *Filter
+			conf *FilterConfig
+			fc   FilterChain
+			err  error
+		)
+
+		Convey("test business no need to filter", func() {
+			conf = &FilterConfig{
+				Business: 111,
+			}
+			f, err = d.NewFilter(conf)
+			So(err, ShouldBeNil)
+			fc = d.NewFilterChain(f)
+			So(len(fc), ShouldEqual, 0)
+		})
+
+		Convey("test business with filter", func() {
+			conf = &FilterConfig{
+				Business: 101,
+			}
+			f, err = d.NewFilter(conf)
+			So(err, ShouldBeNil)
+			fc = d.NewFilterChain(f)
+			So(len(fc), ShouldEqual, 1)
+
+			// both business and IntervalExpired is necessary
+			conf = &FilterConfig{
+				Business:        rand.Intn(100),
+				IntervalExpired: rand.Int31(),
+			}
+			f, err = d.NewFilter(conf)
+			So(err, ShouldBeNil)
+			fc = d.NewFilterChain(f)
+			So(len(fc), ShouldEqual, 2)
+		})
+	})
+}
+
+func TestDao_dailyLimit(t *testing.T) {
+	var (
+		ctx = context.Background()
+		mid int64
+		key string
+		f   *Filter
+		err error
+		b   bool
+	)
+	Convey("test daily limit filter", t, func() {
+		mid = rand.Int63n(999999)
+		key = GetDailyLimitKey(mid)
+		filterConf := &FilterConfig{}
+		f, err = d.NewFilter(filterConf)
+		So(err, ShouldBeNil)
+		log.Info("TestDao_dailyLimit mid(%d), key(%s), filter(%v)", mid, key, f)
+		// del key first
+		err = delRedis(f.conn, key)
+		So(err, ShouldBeNil)
+		// try get with nil return
+		b, err = f.dailyLimitFilter(ctx, mid)
+		So(b, ShouldEqual, false)
+		So(err, ShouldBeNil)
+		// then set a valid value
+		setRedis(f.conn, key, rand.Intn(4)+1, 30)
+		b, err = f.dailyLimitFilter(ctx, mid)
+		So(b, ShouldEqual, false)
+		So(err, ShouldBeNil)
+		// then set value should be filtered
+		setRedis(f.conn, key, -1, 30)
+		b, err = f.dailyLimitFilter(ctx, mid)
+		So(b, ShouldEqual, true)
+		So(err, ShouldBeNil)
+		// then test with conn error
+		delRedis(f.conn, key)
+		f.Done()
+		b, err = f.dailyLimitFilter(ctx, mid)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+func TestDao_intervalSmooth(t *testing.T) {
+	var (
+		ctx = context.Background()
+		mid int64
+		key string
+		f   *Filter
+		err error
+		b   bool
+	)
+	Convey("test interval smooth filter", t, func() {
+		mid = rand.Int63n(999999)
+		key = GetIntervalKey(mid)
+		// new filter
+		task := &model.ApPushTask{
+			LinkValue: "test",
+		}
+		fc := &FilterConfig{
+			IntervalExpired: 300,
+			Task:            task,
+		}
+		f, err = d.NewFilter(fc)
+		So(err, ShouldBeNil)
+		log.Info("TestDao_intervalSmooth mid(%d), key(%s), filter(%v)", mid, key, f)
+		// del key first
+		err = delRedis(f.conn, key)
+		So(err, ShouldBeNil)
+		// first setnx should success
+		b, err = f.intervalSmoothFilter(ctx, mid)
+		So(b, ShouldEqual, false)
+		So(err, ShouldBeNil)
+		// second setnx should fail
+		b, err = f.intervalSmoothFilter(ctx, mid)
+		So(b, ShouldEqual, true)
+		So(err, ShouldBeNil)
+		// test error
+		delRedis(f.conn, key)
+		f.Done()
+		b, err = f.intervalSmoothFilter(ctx, mid)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+func TestDao_BatchFilter(t *testing.T) {
+	initd()
+	Convey("test mids filter by different business", t, func() {
+		var (
+			ctx           = context.Background()
+			business      int
+			mids, resMids []int64
+			conf          *FilterConfig
+			fc            FilterChain
+			f             *Filter
+			err           error
+		)
+
+		Convey("test empty mids or filter chain", func() {
+			// empty mids
+			business = rand.Int()
+			conf = &FilterConfig{Business: business}
+			f, err = d.NewFilter(conf)
+			So(err, ShouldBeNil)
+			fc = d.NewFilterChain(f)
+			resMids = f.BatchFilter(ctx, fc, mids)
+			So(len(resMids), ShouldEqual, 0)
+
+			// empty fc with business 111
+			business = 111
+			conf = &FilterConfig{Business: business}
+			f, err = d.NewFilter(conf)
+			So(err, ShouldBeNil)
+			fc = d.NewFilterChain(f)
+			So(len(fc), ShouldEqual, 0)
+			resMids = f.BatchFilter(ctx, fc, mids)
+			So(len(resMids), ShouldEqual, 0)
+
+			// test business 101 limit filter case
+			business = 101
+			total := 10
+			for i := 0; i < total; i++ {
+				mids = append(mids, rand.Int63())
+			}
+			conf = &FilterConfig{Business: business}
+			f, err = d.NewFilter(conf)
+			So(err, ShouldBeNil)
+			fc = d.NewFilterChain(f)
+			So(len(fc), ShouldEqual, 1)
+			resMids = f.BatchFilter(ctx, fc, mids)
+			So(len(resMids), ShouldEqual, total)
+
+			// clean test mids
+			filterClean(f, mids)
+		})
+
+		Convey("test filter", func() {
+			var b bool
+			business = rand.Intn(99) + 1 // should through all filters
+			total := 10
+			shouldFilterMids := make([]int64, 0, total)
+			for i := 0; i < total; i++ {
+				mid := rand.Int63()
+				if i%3 == 0 {
+					shouldFilterMids = append(shouldFilterMids, mid)
+				}
+				mids = append(mids, mid)
+			}
+			task := &model.ApPushTask{
+				LinkValue: "test",
+			}
+			conf = &FilterConfig{
+				Business:        business,
+				IntervalExpired: 300,
+				DailyExpired:    300,
+				Task:            task}
+			f, err = d.NewFilter(conf)
+			So(err, ShouldBeNil)
+			fc = d.NewFilterChain(f)
+			So(len(fc), ShouldEqual, 2)
+
+			// init should filtered mids, half interval smooth and another daily limit
+			for i, mid := range shouldFilterMids {
+				if i%2 == 0 {
+					b, err = f.intervalSmoothFilter(ctx, mid)
+					So(b, ShouldEqual, false)
+				} else {
+					key := GetDailyLimitKey(mid)
+					err = setRedis(f.conn, key, 0, int32(f.conf.DailyExpired))
+				}
+				So(err, ShouldBeNil)
+			}
+
+			// do filter
+			resMids = f.BatchFilter(ctx, fc, mids)
+			So(len(resMids), ShouldEqual, len(mids)-len(shouldFilterMids))
+
+			// clean
+			filterClean(f, mids)
+		})
+	})
+}
+
+func TestDao_BatchDecreaseLimit(t *testing.T) {
+	initd()
+	Convey("test batch decrease daily limit", t, func() {
+		var (
+			ctx               = context.Background()
+			mids              []int64
+			total, limitTotal int
+			conf              *FilterConfig
+			f                 *Filter
+			err               error
+		)
+		total = 10
+		for i := 0; i < total; i++ {
+			mids = append(mids, rand.Int63())
+		}
+		log.Info("TestDao_BatchDecreaseLimit mids(%v)", mids)
+		conf = &FilterConfig{
+			DailyExpired: 300,
+		}
+		f, err = d.NewFilter(conf)
+		So(err, ShouldBeNil)
+
+		// do limit decrease
+		limitTotal, err = f.BatchDecreaseLimit(ctx, mids)
+		So(err, ShouldBeNil)
+		So(limitTotal, ShouldEqual, total)
+
+		// clean
+		filterClean(f, mids)
+	})
+}

app/interface/live/push-live/dao/hbase.go

@@ -0,0 +1,140 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"github.com/tsuna/gohbase/hrpc"
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/log"
+	"go-common/library/sync/errgroup"
+	"sync"
+)
+
+const _hbaseShard = 200
+
+var (
+	hbaseTable   = "ugc:PushArchive"
+	hbaseFamily  = "relation"
+	hbaseFamilyB = []byte(hbaseFamily)
+)
+
+// Fans gets the upper's fans.
+func (d *Dao) Fans(c context.Context, upper int64, types int) (fans map[int64]bool, fansSP map[int64]bool, err error) {
+	var mutex sync.Mutex
+	fans = make(map[int64]bool)
+	fansSP = make(map[int64]bool)
+	group := errgroup.Group{}
+	for i := 0; i < _hbaseShard; i++ {
+		shard := int64(i)
+		group.Go(func() (e error) {
+			key := _rowKey(upper, shard)
+			relations, e := d.fansByKey(context.TODO(), key)
+			if e != nil {
+				return
+			}
+			mutex.Lock()
+			for fansID, fansType := range relations {
+				switch types {
+				// 返回普通关注
+				case model.RelationAttention:
+					if fansType == types {
+						fans[fansID] = true
+					}
+				// 返回特别关注
+				case model.RelationSpecial:
+					if fansType == types {
+						fansSP[fansID] = true
+					}
+				// 同时返回普通关注与特别关注
+				case model.RelationAll:
+					if fansType == model.RelationSpecial {
+						fansSP[fansID] = true
+					} else if fansType == model.RelationAttention {
+						fans[fansID] = true
+					}
+				default:
+					return
+				}
+			}
+			mutex.Unlock()
+			return
+		})
+	}
+	group.Wait()
+	return
+}
+
+// SeparateFans Separate the upper's fans by 1 or 2.
+func (d *Dao) SeparateFans(c context.Context, upper int64, fansIn map[int64]bool) (fans map[int64]bool, fansSP map[int64]bool, err error) {
+	var mutex sync.Mutex
+	special := make(map[int64]bool)
+	fans = make(map[int64]bool)
+	fansSP = make(map[int64]bool)
+	group := errgroup.Group{}
+	for i := 0; i < _hbaseShard; i++ {
+		shard := int64(i)
+		group.Go(func() (e error) {
+			key := _rowKey(upper, shard)
+			relations, e := d.fansByKey(context.TODO(), key)
+			if e != nil {
+				return
+			}
+			mutex.Lock()
+			// 获取所有特别关注
+			for fansID, fansType := range relations {
+				if fansType == model.RelationSpecial {
+					special[fansID] = true
+				}
+			}
+			mutex.Unlock()
+			return
+		})
+	}
+	group.Wait()
+	for id := range fansIn {
+		if _, ok := special[id]; ok {
+			// 特别关注
+			fansSP[id] = true
+		} else {
+			// 不是特别关注就是普通关注
+			fans[id] = true
+		}
+	}
+	return
+}
+
+func _rowKey(upper, fans int64) string {
+	k := fmt.Sprintf("%d_%d", upper, fans%_hbaseShard)
+	key := fmt.Sprintf("%x", md5.Sum([]byte(k)))
+	return key
+}
+
+func (d *Dao) fansByKey(c context.Context, key string) (relations map[int64]int, err error) {
+	relations = make(map[int64]int)
+	var (
+		query       *hrpc.Get
+		result      *hrpc.Result
+		ctx, cancel = context.WithTimeout(c, d.relationHBaseReadTimeout)
+	)
+	defer cancel()
+	if result, err = d.relationHBase.GetStr(ctx, hbaseTable, key); err != nil {
+		log.Error("d.relationHBase.Get error(%v) querytable(%v)", err, string(query.Table()))
+		// PromError("hbase:Get")
+		return
+	} else if result == nil {
+		return
+	}
+	for _, c := range result.Cells {
+		if c != nil && bytes.Equal(c.Family, hbaseFamilyB) {
+			if err = json.Unmarshal(c.Value, &relations); err != nil {
+				log.Error("json.Unmarshal() error(%v)", err)
+				return
+			}
+			break
+		}
+	}
+	return
+}

app/interface/live/push-live/dao/hbase_test.go

@@ -0,0 +1,43 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/interface/live/push-live/model"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_fans1(t *testing.T) {
+	initd()
+	Convey("Parse Json To Struct", t, func() {
+		fan := int64(27515316)
+		f, fsp1, err := d.Fans(context.TODO(), fan, model.RelationAttention)
+		t.Logf("the included(%v) includedSP(%v) err(%v)", f, fsp1, err)
+		f, fsp2, err := d.Fans(context.TODO(), fan, model.RelationSpecial)
+		t.Logf("the included(%v) includedSP(%v) err(%v)", f, fsp2, err)
+		f, fsp3, err := d.Fans(context.TODO(), fan, model.RelationAll)
+		t.Logf("the included(%v) includedSP(%v) err(%v)", f, fsp3, err)
+
+		So(len(fsp1)+len(fsp2), ShouldEqual, len(fsp3))
+
+	})
+}
+
+func Test_fans2(t *testing.T) {
+	initd()
+	Convey("Parse Json To Struct", t, func() {
+		upper := int64(27515316)
+		fans := make(map[int64]bool)
+		fans[1232032] = true
+		fans[21231134] = true
+		fans[27515398] = true
+		fans[27515275] = true
+
+		f1, f2, err := d.SeparateFans(context.TODO(), upper, fans)
+
+		t.Logf("the included(%v) includedSP(%v) err(%v)", f1, f2, err)
+
+		So(0, ShouldEqual, 0)
+	})
+}

app/interface/live/push-live/dao/push.go

@@ -0,0 +1,179 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/log"
+	"go-common/library/xstr"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"time"
+)
+
+type _response struct {
+	Code int `json:"code"`
+	Data int `json:"data"`
+}
+
+// BatchPush 批量推送，失败重试
+func (d *Dao) BatchPush(fans *[]int64, task *model.ApPushTask) (total int) {
+	limit := d.c.Push.PushOnceLimit
+	retry := d.c.Push.PushRetryTimes
+	var times int
+	for {
+		var (
+			mids []int64
+			err  error
+		)
+		uuid := d.GetUUID(task, times)
+		l := len(*fans)
+		if l == 0 {
+			break
+		} else if l <= limit {
+			mids = (*fans)[:l]
+		} else {
+			mids = (*fans)[:limit]
+			l = limit
+		}
+		*fans = (*fans)[l:]
+
+		for i := 0; i < retry; i++ {
+			// 每次投递成功结束循环
+			if err = d.Push(mids, task, uuid); err == nil {
+				total += len(mids) //单次投递成功数
+				break
+			}
+			time.Sleep(time.Duration(time.Second * 3))
+		}
+		times++
+		if err != nil {
+			// 重试若干次仍然失败，需要记录日志并且配置elk告警
+			log.Error("[dao.push|BatchPush] retry push failed. error(%+v), retry times(%d), task(%+v)", err, retry, task)
+		}
+	}
+	return
+}
+
+// Push 调用推送接口
+func (d *Dao) Push(fans []int64, task *model.ApPushTask, uuid string) (err error) {
+	if len(fans) == 0 {
+		log.Info("[dao.push|Push] empty fans. task(%+v)", task)
+		return
+	}
+
+	// 业务参数
+	businessID, token := d.getPushBusiness(task.Group)
+	buf := new(bytes.Buffer)
+	w := multipart.NewWriter(buf)
+	w.WriteField("app_id", strconv.Itoa(d.c.Push.AppID))
+	w.WriteField("business_id", strconv.Itoa(businessID))
+	w.WriteField("alert_title", d.GetPushTemplate(task.Group, task.AlertTitle))
+	w.WriteField("alert_body", task.AlertBody)
+	w.WriteField("mids", xstr.JoinInts(fans))
+	w.WriteField("link_type", strconv.Itoa(task.LinkType))
+	w.WriteField("link_value", task.LinkValue)
+	w.WriteField("expire_time", strconv.Itoa(task.ExpireTime))
+	w.WriteField("group", task.Group)
+	w.WriteField("uuid", uuid)
+	w.Close()
+	// 签名
+	query := map[string]string{
+		"ts":     strconv.FormatInt(time.Now().Unix(), 10),
+		"appkey": d.c.HTTPClient.Key,
+	}
+	query["sign"] = d.getSign(query, d.c.HTTPClient.Secret)
+	requestURL := fmt.Sprintf("%s?ts=%s&appkey=%s&sign=%s", d.c.Push.MultiAPI, query["ts"], query["appkey"], query["sign"])
+	// request
+	req, err := http.NewRequest(http.MethodPost, requestURL, buf)
+	if err != nil {
+		log.Error("[dao.push|Push] http.NewRequest error(%+v), url(%s), uuid(%s), task(%+v)",
+			err, requestURL, uuid, task)
+		PromError("[dao.push|Push] http:NewRequest")
+		return
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	req.Header.Set("Authorization", "token="+token)
+	res := &_response{}
+	if err = d.httpClient.Do(context.TODO(), req, res); err != nil {
+		log.Error("[dao|push|Push] httpClient.Do error(%+v), url(%s), uuid(%s), task(%+v)",
+			err, requestURL, uuid, task)
+		PromError("[dao.push|Push] http:Do")
+		return
+	}
+	// response
+	if res.Code != 0 || res.Data == 0 {
+		log.Error("[dao.push|Push] push failed. url(%s), uuid(%s), response(%+v), task(%+v)", requestURL, uuid, res, task)
+		err = fmt.Errorf("[dao.push|Push] push failed. url(%s), uuid(%s), response(%+v), task(%+v)", requestURL, uuid, res, task)
+	} else {
+		log.Info("[dao.push|Push] push success. url(%s), uuid(%s), response(%+v), task(%+v)", requestURL, uuid, res, task)
+	}
+	return
+}
+
+// GetPushTemplate 根据类型返回不同的推送文案
+func (d *Dao) GetPushTemplate(group string, part string) (template string) {
+	switch group {
+	case model.SpecialGroup:
+		template = fmt.Sprintf(d.c.Push.SpecialCopyWriting, part)
+	case model.AttentionGroup:
+		template = fmt.Sprintf(d.c.Push.DefaultCopyWriting, part)
+	default:
+		template = part
+	}
+	return
+}
+
+// GetUUID 构造一个每次请求的uuid
+func (d *Dao) GetUUID(task *model.ApPushTask, times int) string {
+	var b bytes.Buffer
+	b.WriteString(strconv.Itoa(times))
+	b.WriteString(task.Group) // Group必须加入uuid计算，区分单次开播提醒关注与特别关注
+	b.WriteString(strconv.Itoa(d.c.Push.BusinessID))
+	b.WriteString(strconv.FormatInt(task.TargetID, 10))
+	b.WriteString(strconv.Itoa(task.ExpireTime))
+	b.WriteString(strconv.FormatInt(time.Now().UnixNano(), 10))
+	mh := md5.Sum(b.Bytes())
+	uuid := hex.EncodeToString(mh[:])
+	return uuid
+}
+
+// getSign 获取签名
+func (d *Dao) getSign(params map[string]string, secret string) (sign string) {
+	keys := []string{}
+	for k := range params {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	buf := bytes.Buffer{}
+	for _, k := range keys {
+		if buf.Len() > 0 {
+			buf.WriteByte('&')
+		}
+		buf.WriteString(url.QueryEscape(k) + "=")
+		buf.WriteString(url.QueryEscape(params[k]))
+	}
+	hash := md5.New()
+	io.WriteString(hash, buf.String()+secret)
+	return fmt.Sprintf("%x", hash.Sum(nil))
+}
+
+// getPushBusiness 获取推送配置
+func (d *Dao) getPushBusiness(group string) (businessID int, token string) {
+	// 预约走单独的白名单通道，business id 和token不一样
+	if group == "activity_appointment" {
+		businessID = 41
+		token = "13aoowdzm0u8pcqdoulvj5vdkihohtcj"
+	} else {
+		businessID = d.c.Push.BusinessID
+		token = d.c.Push.BusinessToken
+	}
+	return
+}

app/interface/live/push-live/dao/push_test.go

@@ -0,0 +1,38 @@
+package dao
+
+import (
+	"go-common/app/interface/live/push-live/model"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_getSign(t *testing.T) {
+	initd()
+	Convey("should return correct sign string by given params and secret", t, func() {
+		params := map[string]string{
+			"aa": "abc",
+			"bb": "xyz",
+			"cc": "opq",
+		}
+		secret := "abc"
+		sign := d.getSign(params, secret)
+
+		So(sign, ShouldEqual, "4571d284b198823bbf62f34cf38c9307")
+	})
+}
+
+func TestService_GetPushTemplate(t *testing.T) {
+	initd()
+	Convey("should return correct template by different type", t, func() {
+		name := "test"
+		t1 := d.GetPushTemplate(model.AttentionGroup, name)
+		t2 := d.GetPushTemplate(model.SpecialGroup, name)
+		t3 := d.GetPushTemplate("test group", name)
+
+		So(t1, ShouldEqual, "你关注的【test】正在直播~")
+		So(t2, ShouldEqual, "你特别关注的【test】正在直播~")
+		// default type template
+		So(t3, ShouldEqual, name)
+	})
+}

app/interface/live/push-live/dao/switch.go

@@ -0,0 +1,44 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_shard             = 10 //分表十张
+	_getMidsByTargetID = "SELECT uid FROM app_switch_config_%s WHERE target_id=? AND type=? AND switch=?"
+)
+
+// tableIndex return index by target_id
+func tableIndex(targetID int64) string {
+	return fmt.Sprintf("%02d", targetID%_shard)
+}
+
+// GetFansBySwitch 获取直播开关数据
+func (d *Dao) GetFansBySwitch(c context.Context, targetID int64) (fans map[int64]bool, err error) {
+	var mid int64
+	fans = make(map[int64]bool)
+	sql := fmt.Sprintf(_getMidsByTargetID, tableIndex(targetID))
+	rows, err := d.db.Query(c, sql, targetID, model.LivePushType, model.LivePushSwitchOn)
+	if err != nil {
+		err = errors.WithStack(err)
+		fmt.Printf("%v", err)
+		log.Error("[dao.switch|GetSwitchMids] db.Query() error(%v)", err)
+		return
+	}
+	for rows.Next() {
+		if err = rows.Scan(&mid); err != nil {
+			err = errors.WithStack(err)
+			fmt.Printf("%v", err)
+			log.Error("[dao.switch|GetSwitchMids] rows.Scan() error(%v)", err)
+			return
+		}
+		fans[mid] = true
+	}
+	return
+}

app/interface/live/push-live/dao/switch_test.go

@@ -0,0 +1,19 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_switch(t *testing.T) {
+	initd()
+	Convey("Parse Json To Struct", t, func() {
+		target := int64(27515316)
+		fs, err := d.GetFansBySwitch(context.TODO(), target)
+		t.Logf("the result included(%v) err(%v)", fs, err)
+
+		So(err, ShouldEqual, nil)
+	})
+}

app/interface/live/push-live/dao/task.go

@@ -0,0 +1,25 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/interface/live/push-live/model"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_createNewTask = "INSERT INTO ap_push_task(type,target_id,alert_title,alert_body,mid_source,link_type,link_value,expire_time,total) VALUES (?,?,?,?,?,?,?,?,?)"
+)
+
+// CreateNewTask 新增推送任务记录
+func (d *Dao) CreateNewTask(c context.Context, task *model.ApPushTask) (affected int64, err error) {
+	res, err := d.db.Exec(c, _createNewTask, model.LivePushType, task.TargetID, task.AlertTitle,
+		task.AlertBody, task.MidSource, task.LinkType, task.LinkValue, task.ExpireTime, task.Total)
+	if err != nil {
+		err = errors.WithStack(err)
+		log.Error("[dao.task|CreateNewTask] db.Exec() error(%v)", err)
+		return
+	}
+	return res.RowsAffected()
+}

app/interface/live/push-live/dao/task_test.go

@@ -0,0 +1,30 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/interface/live/push-live/model"
+	"math/rand"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_task(t *testing.T) {
+	initd()
+	Convey("Parse Json To Struct", t, func() {
+		task := &model.ApPushTask{
+			Type:       rand.Intn(9999) + 1,
+			TargetID:   rand.Int63n(9999) + 1,
+			AlertTitle: "title",
+			AlertBody:  "body",
+			MidSource:  rand.Intn(15),
+			LinkType:   rand.Intn(10),
+			LinkValue:  "link_value",
+			Total:      rand.Intn(9999),
+		}
+		affected, err := d.CreateNewTask(context.TODO(), task)
+		t.Logf("the result included(%v) err(%v)", affected, err)
+
+		So(err, ShouldEqual, nil)
+	})
+}