Create & Init Project...

app/infra/notify/BUILD

@@ -0,0 +1,22 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/infra/notify/cmd:all-srcs",
+        "//app/infra/notify/conf:all-srcs",
+        "//app/infra/notify/dao:all-srcs",
+        "//app/infra/notify/http:all-srcs",
+        "//app/infra/notify/model:all-srcs",
+        "//app/infra/notify/notify:all-srcs",
+        "//app/infra/notify/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/CHANGELOG.md

@@ -0,0 +1,51 @@
+### notify
+#### v1.5.0
+> 1.迁移到infra  
+
+#### v1.4.5
+1.liverpc 消息header兼容
+#### v1.4.4
+> 1.liverpc 消息格式解析不正确不重试，立即告警
+#### V1.4.3
+> 1.置顶kafka版本 
+#### v1.4.2
+> 1.support liverpc group
+
+#### v1.4.1
+> 1.fix liverpc client msg_content format
+
+#### v1.4.0
+> 1.support liverpc notify callback
+
+#### V1.3.5
+> 1.add err log
+
+#### V1.3.4
+> 1.refactor dao test
+#### V1.3.3
+#### V1.3.4
+> 1.修改优先级  
+#### V1.3.3 
+> 1.no close chan 
+#### V1.3.2
+> 1.修复async retry 
+#### V1.3.1
+> 1.修复chan close
+
+#### V1.3.0
+> 1.热加载notify更新  
+#### V1.2.2
+> 1. 修复filters sql 错误
+#### V1.2.1
+> 1.支持action 字段过滤  
+
+#### V1.2.0
+> 1.修改filters结构  
+
+#### V1.1.0
+> 1.sub失败自动重启
+> 2.新增group自动reload  
+> 3.添加prom监控  
+
+#### v1.0.0
+1. notify init. 

app/infra/notify/CONTRIBUTORS.md

@@ -0,0 +1,9 @@
+# Owner
+haoguanwei
+lintanghui 
+
+# Author
+lintanghui
+ 
+# Reviewer 
+haoguanwei

app/infra/notify/OWNERS

@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- haoguanwei
+- lintanghui
+labels:
+- infra
+- infra/notify
+- main
+- service
+- service/main/notify
+options:
+  no_parent_owners: true
+reviewers:
+- haoguanwei
+- lintanghui

app/infra/notify/README.md

@@ -0,0 +1,21 @@
+# go-common/app/service/notify
+
+##### 项目简介
+> 1. 消息中间件，消息注册监听推送事件通知
+
+##### 编译环境
+> 1. 请只用golang v1.7.x以上版本编译执行。
+
+##### 依赖包
+> 1. 公共依赖
+
+##### 编译执行
+> 1. 启动执行
+
+##### 测试
+> 1. 执行当前目录下所有测试文件，测试所有功能
+
+##### 特别说明
+
+
+	

app/infra/notify/cmd/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["notify-test.toml"],
+    importpath = "go-common/app/infra/notify/cmd",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//app/infra/notify/http:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/cmd/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/http"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	// init log
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("notify start")
+	// service init
+	http.Init(conf.Conf)
+	// init pprof conf.Conf.Perf
+	// init signal
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("notify get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("notify exit")
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/infra/notify/cmd/notify-test.toml

@@ -0,0 +1,49 @@
+# This is a TOML document. Boom
+
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/notify.pid"
+dir = "./"
+perf = "0.0.0.0:7630"
+
+[log]
+dir = "/data/log/notify"
+
+[bm]
+addr = "0.0.0.0:7631"
+timeout = "1s"
+
+[clusters]
+[clusters.test]
+cluster = "test_kafka_9092-266"
+brokers = ["172.16.38.154:9192"]
+sync = true
+
+[httpClient]
+key = "0c4b8fe3ff35a4b6"
+secret = "b370880d1aca7d3a289b9b9a7f4d6812"
+dial = "500ms"
+timeout = "1s"
+keepAlive = "60s"
+    [httpClient.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[mysql]
+	addr = "172.16.33.205"
+	dsn = "root@tcp(127.0.0.1:3306)/bilibili_databus_v2?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	active = 20
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "1s"
+	execTimeout = "1s"
+	tranTimeout = "2s"
+	[mysql.breaker]
+	window = "3s"
+	sleep = "100ms"
+	bucket = 10
+	ratio = 0.5
+	request = 100

app/infra/notify/conf/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/infra/notify/conf",
+    tags = ["automanaged"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/conf/conf.go

@@ -0,0 +1,93 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/BurntSushi/toml"
+)
+
+// global var
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config config set
+type Config struct {
+	// base
+	// elk
+	Log *log.Config
+	// http
+	BM *bm.ServerConfig
+	// MySQL
+	MySQL *sql.Config
+	// kafka cluster.
+	Clusters map[string]*Kafka
+	// http
+	HTTPClient *bm.ClientConfig
+}
+
+// Kafka contains cluster, brokers, sync.
+type Kafka struct {
+	Cluster string
+	Brokers []string
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/infra/notify/dao/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "mysql_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+    ],
+    importpath = "go-common/app/infra/notify/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//app/infra/notify/model:go_default_library",
+        "//library/database/sql:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/dao/dao.go

@@ -0,0 +1,33 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/infra/notify/conf"
+	xsql "go-common/library/database/sql"
+)
+
+// Dao dao
+type Dao struct {
+	c  *conf.Config
+	db *xsql.DB
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:  c,
+		db: xsql.NewMySQL(c.MySQL),
+	}
+	return
+}
+
+// Close close the resource.
+func (dao *Dao) Close() {
+	dao.db.Close()
+}
+
+// Ping dao ping
+func (dao *Dao) Ping(c context.Context) error {
+	return dao.db.Ping(c)
+}

app/infra/notify/dao/dao_test.go

@@ -0,0 +1,33 @@
+package dao
+
+import (
+	"flag"
+	"go-common/app/infra/notify/conf"
+	"os"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.common-arch.notify")
+		flag.Set("conf_token", "9919040d8742a2124abbed8368626075")
+		flag.Set("tree_id", "22513")
+		flag.Set("conf_version", "v1.0.0")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/infra/notify/dao/mysql.go

@@ -0,0 +1,100 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+
+	"go-common/app/infra/notify/model"
+)
+
+const (
+	_loadNotify = "SELECT n.id,topic.cluster,topic.topic,a.group,n.callback,n.concurrent,n.filter,n.mtime FROM notify AS n LEFT JOIN auth2 as a ON a.id=n.gid LEFT JOIN topic ON a.topic_id=topic.id WHERE n.state=1 AND n.zone=?"
+	_loadPub    = `SELECT a.group,topic.cluster,topic.topic,a.operation,app2.app_secret FROM auth2 AS a LEFT JOIN app2 ON app2.id=a.app_id LEFT JOIN topic ON topic.id=a.topic_id`
+	_selFilter  = "SELECT `filters` FROM filters WHERE nid=?"
+	_addFailBk  = "INSERT INTO fail_backup(topic,`group`,`cluster`,msg,`index`) VALUE (?,?,?,?,?)"
+	_delFailBk  = "DELETE FROM fail_backup where id=?"
+	_loadFailBk = "SELECT id,topic,`group`,`cluster`,`msg`,`offset`,`index` FROM fail_backup"
+)
+
+// LoadNotify load all notify config.
+func (d *Dao) LoadNotify(c context.Context, zone string) (ns []*model.Watcher, err error) {
+	rows, err := d.db.Query(c, _loadNotify, zone)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		n := new(model.Watcher)
+		if err = rows.Scan(&n.ID, &n.Cluster, &n.Topic, &n.Group, &n.Callback, &n.Concurrent, &n.Filter, &n.Mtime); err != nil {
+			return
+		}
+		ns = append(ns, n)
+	}
+	return
+}
+
+// LoadPub load all pub config.
+func (d *Dao) LoadPub(c context.Context) (ps []*model.Pub, err error) {
+	rows, err := d.db.Query(c, _loadPub)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		n := new(model.Pub)
+		if err = rows.Scan(&n.Group, &n.Cluster, &n.Topic, &n.Operation, &n.AppSecret); err != nil {
+			return
+		}
+		ps = append(ps, n)
+	}
+	return
+}
+
+// Filters get filter condition.
+func (d *Dao) Filters(c context.Context, id int64) (fs []*model.Filter, err error) {
+	rows := d.db.QueryRow(c, _selFilter, id)
+	if err != nil {
+		return
+	}
+	var filters string
+	if err = rows.Scan(&filters); err != nil {
+		return
+	}
+	err = json.Unmarshal([]byte(filters), &fs)
+	return
+}
+
+// AddFailBk add fail msg to fail backup.
+func (d *Dao) AddFailBk(c context.Context, topic, group, cluster, msg string, index int64) (id int64, err error) {
+	res, err := d.db.Exec(c, _addFailBk, topic, group, cluster, msg, index)
+	if err != nil {
+		return
+	}
+	return res.LastInsertId()
+}
+
+// DelFailBk del msg from fail backup.
+func (d *Dao) DelFailBk(c context.Context, id int64) (affected int64, err error) {
+	res, err := d.db.Exec(c, _delFailBk, id)
+	if err != nil {
+		return
+	}
+	return res.RowsAffected()
+}
+
+// LoadFailBk load all fail backup msg.
+func (d *Dao) LoadFailBk(c context.Context) (fbs []*model.FailBackup, err error) {
+	rows, err := d.db.Query(c, _loadFailBk)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		fb := new(model.FailBackup)
+		if err = rows.Scan(&fb.ID, &fb.Topic, &fb.Group, &fb.Cluster, &fb.Msg, &fb.Offset, &fb.Index); err != nil {
+			return
+		}
+		fbs = append(fbs, fb)
+	}
+	return
+}

app/infra/notify/dao/mysql_test.go

@@ -0,0 +1,93 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoLoadNotify(t *testing.T) {
+	var (
+		c = context.TODO()
+	)
+	convey.Convey("LoadNotify", t, func(ctx convey.C) {
+		_, err := d.LoadNotify(c, "sh001")
+		ctx.Convey("Then err should be nil.ns should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			//ctx.So(ns, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoLoadPub(t *testing.T) {
+	var (
+		c = context.TODO()
+	)
+	convey.Convey("LoadPub", t, func(ctx convey.C) {
+		_, err := d.LoadPub(c)
+		ctx.Convey("Then err should be nil.ps should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			//	ctx.So(ps, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoFilters(t *testing.T) {
+	var (
+		c  = context.TODO()
+		id = int64(7)
+	)
+	convey.Convey("Filters", t, func(ctx convey.C) {
+		fs, err := d.Filters(c, id)
+		ctx.Convey("Then err should be nil.fs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(fs, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoAddFailBk(t *testing.T) {
+	var (
+		c       = context.TODO()
+		topic   = ""
+		group   = ""
+		cluster = ""
+		msg     = ""
+		index   = int64(0)
+	)
+	convey.Convey("AddFailBk", t, func(ctx convey.C) {
+		id, err := d.AddFailBk(c, topic, group, cluster, msg, index)
+		ctx.Convey("Then err should be nil.id should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(id, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoDelFailBk(t *testing.T) {
+	var (
+		c  = context.TODO()
+		id = int64(0)
+	)
+	convey.Convey("DelFailBk", t, func(ctx convey.C) {
+		affected, err := d.DelFailBk(c, id)
+		ctx.Convey("Then err should be nil.affected should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(affected, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoLoadFailBk(t *testing.T) {
+	var (
+		c = context.TODO()
+	)
+	convey.Convey("LoadFailBk", t, func(ctx convey.C) {
+		fbs, err := d.LoadFailBk(c)
+		ctx.Convey("Then err should be nil.fbs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(fbs, convey.ShouldNotBeNil)
+		})
+	})
+}

app/infra/notify/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/infra/notify/http",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//app/infra/notify/model:go_default_library",
+        "//app/infra/notify/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/http/http.go

@@ -0,0 +1,51 @@
+package http
+
+import (
+	"go-common/app/infra/notify/conf"
+	mrl "go-common/app/infra/notify/model"
+	"go-common/app/infra/notify/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	svc *service.Service
+)
+
+// Init init
+func Init(c *conf.Config) {
+	initService(c)
+	// init router
+	eng := bm.DefaultServer(c.BM)
+	initRouter(eng)
+	if err := eng.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// initService init services.
+func initService(c *conf.Config) {
+	svc = service.New(c)
+}
+
+// initRouter init outer router api path.
+func initRouter(e *bm.Engine) {
+	e.Ping(ping)
+	group := e.Group("/x/internal/notify")
+	{
+		group.POST("/pub", pub)
+	}
+}
+
+func pub(c *bm.Context) {
+	arg := new(mrl.ArgPub)
+	if err := c.Bind(arg); err != nil {
+		return
+	}
+	c.JSON(nil, svc.Pub(c, arg))
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+}

app/infra/notify/model/BUILD

@@ -0,0 +1,27 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["notify.go"],
+    importpath = "go-common/app/infra/notify/model",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/model/notify.go

@@ -0,0 +1,93 @@
+package model
+
+import (
+	"net/url"
+	"time"
+)
+
+// Watcher define watcher object.
+type Watcher struct {
+	ID         int64
+	Cluster    string
+	Topic      string
+	Group      string
+	Offset     string
+	Callback   string
+	Callbacks  []*Callback
+	Filter     bool
+	Filters    []*Filter
+	Concurrent int // concurrent goroutine for sub.
+	Mtime      time.Time
+}
+
+// Pub define pub.
+type Pub struct {
+	ID        int64
+	Cluster   string
+	Topic     string
+	Group     string
+	Operation int8
+	AppSecret string
+}
+
+// Callback define callback event
+type Callback struct {
+	URL      *NotifyURL
+	Priority int8
+	Finished bool
+}
+
+// NotifyURL callback url with parsed info
+type NotifyURL struct {
+	RawURL string
+	Schema string
+	Host   string
+	Path   string
+	Query  url.Values
+}
+
+// filter condition
+const (
+	ConditionEq  = 0
+	ConditionPre = 1
+)
+
+// Filter define filter object.
+type Filter struct {
+	Field     string
+	Condition int8 // 0 :eq 1:neq
+	Value     string
+}
+
+// Message define canal message.
+type Message struct {
+	Table  string `json:"table,omitempty"`
+	Action string `json:"action,omitempty"`
+}
+
+// ArgPub pub arg.
+type ArgPub struct {
+	AppKey    string `form:"appkey" validate:"min=1"`
+	AppSecret string `form:"appsecret" validate:"min=1"`
+	Group     string `form:"group" validate:"min=1"`
+	Topic     string `form:"topic" validate:"min=1"`
+	Key       string `form:"key" validate:"min=1"`
+	Msg       string `form:"msg" validate:"min=1"`
+}
+
+// FailBackup fail backup msg.
+type FailBackup struct {
+	ID      int64
+	Cluster string
+	Topic   string
+	Group   string
+	Offset  int64
+	Msg     string
+	Index   int64
+}
+
+// Notify callback schema
+const (
+	LiverpcSchema = "liverpc"
+	HTTPSchema    = "http"
+)

app/infra/notify/notify/BUILD

@@ -0,0 +1,66 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "client_test.go",
+        "liverpc_client_test.go",
+        "notify_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//app/infra/notify/dao:go_default_library",
+        "//app/infra/notify/model:go_default_library",
+        "//library/net/rpc/liverpc:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "client.go",
+        "liverpc_client.go",
+        "pub.go",
+        "sub.go",
+    ],
+    importpath = "go-common/app/infra/notify/notify",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//app/infra/notify/dao:go_default_library",
+        "//app/infra/notify/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/netutil:go_default_library",
+        "//library/net/rpc/liverpc:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/bsm/sarama-cluster:go_default_library",
+        "//vendor/github.com/rcrowley/go-metrics:go_default_library",
+        "//vendor/github.com/snluu/uuid:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/notify/client.go

@@ -0,0 +1,48 @@
+package notify
+
+import (
+	"context"
+	"errors"
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/model"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"net/url"
+)
+
+var (
+	errUnknownSchema = errors.New("callback url with unknown schema")
+)
+
+// Clients notify clients
+type Clients struct {
+	httpClient     *bm.Client
+	liverpcClients *LiverpcClients
+}
+
+// NewClients New NotifyClients
+func NewClients(c *conf.Config, w *model.Watcher) *Clients {
+	nc := &Clients{
+		httpClient:     bm.NewClient(c.HTTPClient),
+		liverpcClients: newLiverpcClients(w),
+	}
+	log.Info("Notify.NewClients topic(%s), group(%s), callback len(%d), liverpc clients(%d)",
+		w.Topic, w.Group, len(w.Callbacks), len(nc.liverpcClients.clients))
+	return nc
+}
+
+// Post do callback with different client vary schemas
+func (nc *Clients) Post(c context.Context, notifyURL *model.NotifyURL, msg string) (err error) {
+	switch notifyURL.Schema {
+	case model.LiverpcSchema:
+		err = nc.liverpcClients.Post(c, notifyURL, msg)
+	case model.HTTPSchema:
+		params := url.Values{}
+		params.Set("msg", msg)
+		client := nc.httpClient
+		err = client.Post(c, notifyURL.RawURL, "", params, nil)
+	default:
+		err = errUnknownSchema
+	}
+	return
+}

app/infra/notify/notify/client_test.go

@@ -0,0 +1,90 @@
+package notify
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/model"
+)
+
+func initConf() *conf.Config {
+	var err error
+	flag.Set("conf", "../cmd/notify-test.toml")
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+	return conf.Conf
+}
+
+func TestClients_Post(t *testing.T) {
+	var (
+		ctx       = context.Background()
+		c         = initConf()
+		err       error
+		notifyURL *model.NotifyURL
+	)
+	Convey("test call post with different urls", t, func() {
+		w := &model.Watcher{}
+
+		Convey("error schema", func() {
+			u := "https://live.bilibili.com"
+			notifyURL, err = parseNotifyURL(u)
+			So(err, ShouldBeNil)
+			nc := NewClients(c, w)
+			err = nc.Post(ctx, notifyURL, "test1")
+			So(err, ShouldResemble, errUnknownSchema)
+		})
+
+		Convey("test http url post", func() {
+			u := "http://127.0.0.1:19999/push3"
+			notifyURL, err = parseNotifyURL(u)
+			So(err, ShouldBeNil)
+			nc := NewClients(c, w)
+			mockhttp2()
+			err = nc.Post(ctx, notifyURL, "test2")
+			So(err, ShouldBeNil)
+		})
+
+		Convey("test liverpc url post", func() {
+			u := "liverpc://live.bannedservice?version=0&cmd=Message.synUser"
+			notifyURL, err = parseNotifyURL(u)
+			cb := &model.Callback{
+				URL:      notifyURL,
+				Priority: 1,
+			}
+			w.Callbacks = []*model.Callback{cb}
+			So(err, ShouldBeNil)
+			nc := NewClients(c, w)
+			So(len(nc.liverpcClients.clients), ShouldEqual, 1)
+			msg := []byte(`{"topic":"BannedUserSyn-T","msg_id":"a485ad75609304b920782462ce1c7632","msg_content":"{\"uid\":1734992,\"status\":0,\"begin\":\"2018-09-10 17:51:45\",\"uname\":\"\\u83ca\\u82b1\\u75db\",\"face\":\"http:\\\/\\\/i2.hdslb.com\\\/bfs\\\/face\\\/9eab4e877c83dd77bd010994e35e0d113ec7bf9d.jpg\",\"rank\":\"10000\",\"identification\":0,\"mobile_verify\":1,\"silence\":0,\"official_verify\":{\"type\":-1,\"desc\":\"\",\"role\":0}}","msg_key":1734992,"timestamp":1536573105.4239,"failure_cnt":0,"caller_header":{"platform":"","src":"","version":"","buvid":"AUTO3715365731053968","trace_id":"6172727a27c922e0:61727287a8ae0264:61727285b093e28e:0","uid":0,"caller":"user.user\\common\\logic\\Databus_Service.call-40","user_ip":"172.18.29.22","source_group":"qa01","sessdata2":"access_key=&SESSDATA=","group":"default"},"__ts1":1536573105.3978,"__ts2":1536573105.4032}`)
+			err = nc.Post(ctx, notifyURL, string(msg))
+			So(err, ShouldBeNil)
+		})
+
+		Convey("test liverpc url post with different message format", func() {
+			u := "liverpc://live.relation?version=1&cmd=Notify.on_relation_changed"
+			notifyURL, err = parseNotifyURL(u)
+			cb := &model.Callback{
+				URL:      notifyURL,
+				Priority: 1,
+			}
+			w.Callbacks = []*model.Callback{cb}
+			So(err, ShouldBeNil)
+			nc := NewClients(c, w)
+			So(len(nc.liverpcClients.clients), ShouldEqual, 1)
+			msg := []byte(`{"action":"insert","table":"user_relation_fid_439","new":{"attribute":2,"ctime":"2018-09-10 18:56:26","fid":18021939,"id":4090654,"mid":354291964,"mtime":"2018-09-10 18:56:26","source":0,"status":0}}`)
+			err = nc.Post(ctx, notifyURL, string(msg))
+			So(err, ShouldBeNil)
+		})
+	})
+}
+
+func mockhttp2() {
+	http.HandleFunc("/push3", testpush1)
+	http.HandleFunc("/push4", testpush2)
+	go http.ListenAndServe(":19999", nil)
+}

app/infra/notify/notify/liverpc_client.go

@@ -0,0 +1,218 @@
+package notify
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"github.com/snluu/uuid"
+	"go-common/app/infra/notify/model"
+	"go-common/library/log"
+	"go-common/library/net/rpc/liverpc"
+	"os"
+	"strconv"
+	"sync"
+)
+
+var (
+	errLiverpcInvalidParams = errors.New("liverpc callback without version or cmd params")
+	errStrconvVersion       = errors.New("liverpc post try strconv version param failed")
+	errCallRaw              = errors.New("liverpc callRaw failed")
+	errEmptyClient          = errors.New("liverpc client empty")
+	_liverpcCaller          = "notify-liverpc-client"
+	_defaultGroup           = "default"
+)
+
+// LiverpcClients liverpc clients
+type LiverpcClients struct {
+	rwLock  sync.RWMutex
+	group   string
+	clients map[string]*liverpc.Client
+}
+
+// LiveMsgValue live databus message.value
+type LiveMsgValue struct {
+	Topic        string         `json:"topic"`
+	MsgID        interface{}    `json:"msg_id"`
+	MsgKey       interface{}    `json:"msg_key"`
+	MsgContent   interface{}    `json:"msg_content"`
+	Timestamp    float64        `json:"timestamp"`
+	CallerHeader *LiveMsgHeader `json:"caller_header"`
+}
+
+// LiveMsgHeader live databus message.value.caller_header
+type LiveMsgHeader struct {
+	TraceId     string `json:"trace_id"`
+	Caller      string `json:"caller"`
+	SourceGroup string `json:"source_group"`
+	Group       string `json:"group"`
+}
+
+// NewClients new clients
+func newLiverpcClients(w *model.Watcher) *LiverpcClients {
+	lrcs := &LiverpcClients{
+		group:   getGroup(),
+		clients: make(map[string]*liverpc.Client),
+	}
+	for _, cb := range w.Callbacks {
+		if cb.URL.Schema == model.LiverpcSchema {
+			lrcs.getClient(cb.URL)
+		}
+	}
+	return lrcs
+}
+
+// Post do callback with liverpc client
+func (lrcs *LiverpcClients) Post(ctx context.Context, notifyURL *model.NotifyURL, msg string) (err error) {
+	var (
+		reply  *liverpc.Reply
+		v      int
+		header *liverpc.Header
+		body   = make(map[string]string)
+		client *liverpc.Client
+	)
+	// parse params, alarm is must, need retry
+	version := notifyURL.Query.Get("version")
+	cmd := notifyURL.Query.Get("cmd")
+	if version == "" || cmd == "" {
+		err = errLiverpcInvalidParams
+		log.Error("LiverpcClient.Post callback params error(%v), url(%v), msg(%s)", err, notifyURL, msg)
+		return
+	}
+	v, err = strconv.Atoi(version)
+	if err != nil {
+		log.Error("LiverpcClient.Post callback params error(%v), strconv version error, url(%v), msg(%s)", err, notifyURL, msg)
+		err = errStrconvVersion
+		return
+	}
+
+	// parse live message
+	// if parse error, live message format invalid, fast fail and no need retry, alarm is must
+	header, body, err = lrcs.formatLiveMsg(msg)
+	if err != nil {
+		log.Error("LiverpcClient.Post format message error(%v), url(%v), msg(%s)", err, notifyURL, msg)
+		return nil
+	}
+
+	// prepare client and args, if no client currently, should retry and alarm
+	client = lrcs.getClient(notifyURL)
+	if client == nil {
+		log.Error("LiverpcClient.Post empty client, url(%v), msg(%s)", notifyURL, msg)
+		err = errEmptyClient
+		return
+	}
+	args := &liverpc.Args{
+		Header: header,
+		Body:   body,
+	}
+
+	// do call
+	reply, err = client.CallRaw(ctx, v, cmd, args)
+	if err != nil {
+		log.Error("LiverpcClient.Post CallRaw error(%v), url(%v), msg(%s)", err, notifyURL, msg)
+		err = errCallRaw
+		return
+	}
+	log.Info("LiverpcClient.Post CallRaw reply(%v), url(%v), msg(%s)", reply, notifyURL, msg)
+	return
+}
+
+// getClient get client by appID
+func (lrcs *LiverpcClients) getClient(notifyURL *model.NotifyURL) *liverpc.Client {
+	lrcs.rwLock.RLock()
+	c, ok := lrcs.clients[notifyURL.Host]
+	lrcs.rwLock.RUnlock()
+	if !ok {
+		c = lrcs.newClient(notifyURL)
+		lrcs.setClient(notifyURL.Host, c)
+	}
+	return c
+}
+
+// setClient set client
+func (lrcs *LiverpcClients) setClient(appID string, client *liverpc.Client) {
+	lrcs.rwLock.Lock()
+	defer lrcs.rwLock.Unlock()
+	lrcs.clients[appID] = client
+}
+
+// newClient create new liverpc client
+func (lrcs *LiverpcClients) newClient(notifyURL *model.NotifyURL) *liverpc.Client {
+	conf := &liverpc.ClientConfig{
+		AppID: notifyURL.Host,
+	}
+	// one can just appoint addr by params
+	if notifyURL.Query.Get("addr") != "" {
+		conf.Addr = notifyURL.Query.Get("addr")
+	}
+	return liverpc.NewClient(conf)
+}
+
+// formatLiveMsg format live databus callback params
+func (lrcs *LiverpcClients) formatLiveMsg(msg string) (header *liverpc.Header, body map[string]string, err error) {
+	var bizMsg, msgContent string
+	header = &liverpc.Header{}
+	value := new(LiveMsgValue)
+	err = json.Unmarshal([]byte(msg), value)
+	if err != nil {
+		log.Error("LiverpcClient.formatLiveMsg unmarshal msg error(%v), msg(%s), value(%v)", err, msg, value)
+		return
+	}
+	if value.MsgContent != nil {
+		var m1, m2 []byte
+		m1, err = json.Marshal(value.MsgContent)
+		if err != nil {
+			log.Error("LiverpcClient.formatLiveMsg unmarshal MsgContent error(%v), msg(%s), value(%v)", err, msg, value)
+			return
+		}
+		bm := map[string]interface{}{
+			"topic":     value.Topic,
+			"msg_id":    value.MsgID,
+			"msg_key":   value.MsgKey,
+			"timestamp": value.Timestamp,
+		}
+		m2, err = json.Marshal(bm)
+		if err != nil {
+			log.Error("LiverpcClient.formatLiveMsg unmarshal bizMsg error(%v), msg(%s), value(%v)", err, msg, value)
+			return
+		}
+		msgContent = string(m1)
+		bizMsg = string(m2)
+
+		// caller header
+		callerHeader := value.CallerHeader
+		if callerHeader != nil {
+			header.TraceId = callerHeader.TraceId
+			if callerHeader.SourceGroup != "" {
+				header.SourceGroup = callerHeader.SourceGroup
+			} else {
+				header.SourceGroup = callerHeader.Group
+			}
+		}
+	} else {
+		msgContent = msg
+	}
+
+	// supplement liverpc header
+	header.Caller = _liverpcCaller
+	if header.TraceId == "" {
+		header.TraceId = uuid.Rand().Hex()
+	}
+	if header.SourceGroup == "" {
+		header.SourceGroup = lrcs.group
+	}
+	// body
+	body = map[string]string{
+		"msg":         bizMsg,
+		"msg_content": msgContent,
+	}
+	return
+}
+
+// getGroup get message source group
+func getGroup() (g string) {
+	g = os.Getenv("group")
+	if g == "" {
+		g = _defaultGroup
+	}
+	return
+}

app/infra/notify/notify/liverpc_client_test.go

@@ -0,0 +1,117 @@
+package notify
+
+import (
+	"context"
+	. "github.com/smartystreets/goconvey/convey"
+	"go-common/app/infra/notify/model"
+	"go-common/library/net/rpc/liverpc"
+	"testing"
+)
+
+func initNc(urls []string) *LiverpcClients {
+	w := &model.Watcher{}
+	w.Callbacks = make([]*model.Callback, 0, len(urls))
+	for i, u := range urls {
+		notifyURL, err := parseNotifyURL(u)
+		So(err, ShouldBeNil)
+		cb := &model.Callback{
+			URL:      notifyURL,
+			Priority: int8(i),
+		}
+		w.Callbacks = append(w.Callbacks, cb)
+	}
+	nc := newLiverpcClients(w)
+	return nc
+}
+
+func TestNewLiverpcClients(t *testing.T) {
+	Convey("test new liverpc clients", t, func() {
+		urls := []string{
+			"http://www.bilibili.com",
+			"liverpc://live.bannedservice?version=0&cmd=Message.synUser&addr=172.18.33.82:20822",
+			"liverpc://live.room?version=1&cmd=Consumer/receiveGift&addr=172.18.33.82:20200",
+		}
+		nc := initNc(urls)
+		So(len(nc.clients), ShouldEqual, 2)
+	})
+}
+
+func TestFormatLiveMsg(t *testing.T) {
+	var (
+		msg    []byte
+		header *liverpc.Header
+		body   map[string]string
+		err    error
+		urls   = []string{}
+		nc     = initNc(urls)
+	)
+
+	Convey("test live post message format result", t, func() {
+		msg = []byte(`{"topic":"BannedUserSyn-T","msg_id":"a485ad75609304b920782462ce1c7632","msg_content":"{\"uid\":1734992,\"status\":0,\"begin\":\"2018-09-10 17:51:45\",\"uname\":\"\\u83ca\\u82b1\\u75db\",\"face\":\"http:\\\/\\\/i2.hdslb.com\\\/bfs\\\/face\\\/9eab4e877c83dd77bd010994e35e0d113ec7bf9d.jpg\",\"rank\":\"10000\",\"identification\":0,\"mobile_verify\":1,\"silence\":0,\"official_verify\":{\"type\":-1,\"desc\":\"\",\"role\":0}}","msg_key":1734992,"timestamp":1536573105.4239,"failure_cnt":0,"caller_header":{"platform":"","src":"","version":"","buvid":"AUTO3715365731053968","trace_id":"6172727a27c922e0:61727287a8ae0264:61727285b093e28e:0","uid":0,"caller":"user.user\\common\\logic\\Databus_Service.call-40","user_ip":"172.18.29.22","source_group":"default","sessdata2":"access_key=&SESSDATA=","group":"default"},"__ts1":1536573105.3978,"__ts2":1536573105.4032}`)
+		header, body, err = nc.formatLiveMsg(string(msg))
+		So(err, ShouldBeNil)
+		So(header.TraceId, ShouldEqual, "6172727a27c922e0:61727287a8ae0264:61727285b093e28e:0")
+		So(header.Caller, ShouldEqual, _liverpcCaller)
+		So(body["msg"], ShouldNotEqual, "")
+		So(body["msg_content"], ShouldNotEqual, "")
+	})
+
+	Convey("test non live post message format result", t, func() {
+		msg = []byte(`{"action":"insert","table":"user_relation_fid_439","new":{"attribute":2,"ctime":"2018-09-10 18:56:26","fid":18021939,"id":4090654,"mid":354291964,"mtime":"2018-09-10 18:56:26","source":0,"status":0}}`)
+		header, body, err = nc.formatLiveMsg(string(msg))
+		So(err, ShouldBeNil)
+		So(header.Caller, ShouldEqual, _liverpcCaller)
+		So(body["msg"], ShouldEqual, "")
+		So(body["msg_content"], ShouldNotEqual, "")
+	})
+}
+
+func TestLiverpcClients_Post(t *testing.T) {
+	Convey("test liverpc client post", t, func() {
+		Convey("test post with invalid params", func() {
+			u := "liverpc://live.bannedservice"
+			urls := []string{u}
+			nc := initNc(urls)
+			notifyURL, err := parseNotifyURL(u)
+			So(err, ShouldBeNil)
+			err = nc.Post(context.Background(), notifyURL, "test")
+			So(err, ShouldResemble, errLiverpcInvalidParams)
+
+			u = "liverpc://live.bannedservice?version=error&cmd=Message.synUser"
+			nc = initNc([]string{u})
+			notifyURL, err = parseNotifyURL(u)
+			So(err, ShouldBeNil)
+			err = nc.Post(context.Background(), notifyURL, "test")
+			So(err, ShouldResemble, errStrconvVersion)
+		})
+
+		Convey("test post with invalid msg", func() {
+			u := "liverpc://live.bannedservice?version=0&cmd=Message.synUser"
+			nc := initNc([]string{u})
+			notifyURL, err := parseNotifyURL(u)
+			So(err, ShouldBeNil)
+			err = nc.Post(context.Background(), notifyURL, "not a json format message")
+			So(err, ShouldResemble, nil)
+		})
+
+		Convey("test post call client failed with invalid addr", func() {
+			u := "liverpc://live.bannedservice?version=0&cmd=Message.synUser&addr=1.1.1.1:11111"
+			nc := initNc([]string{u})
+			notifyURL, err := parseNotifyURL(u)
+			So(err, ShouldBeNil)
+			m := []byte(`{"topic":"BannedUserSyn-T","msg_id":"123456","msg_content":"{\"test\":123456}","msg_key":1734992,"timestamp":1536573105.4239}`)
+			err = nc.Post(context.Background(), notifyURL, string(m))
+			So(err, ShouldResemble, errCallRaw)
+		})
+
+		Convey("test success", func() {
+			u := "liverpc://live.bannedservice?version=0&cmd=Message.synUser"
+			nc := initNc([]string{u})
+			notifyURL, err := parseNotifyURL(u)
+			So(err, ShouldBeNil)
+			m := []byte(`{"topic":"BannedUserSyn-T","msg_id":"a485ad75609304b920782462ce1c7632","msg_content":"{\"uid\":1734992,\"status\":0,\"begin\":\"2018-09-10 17:51:45\",\"uname\":\"\\u83ca\\u82b1\\u75db\",\"face\":\"http:\\\/\\\/i2.hdslb.com\\\/bfs\\\/face\\\/9eab4e877c83dd77bd010994e35e0d113ec7bf9d.jpg\",\"rank\":\"10000\",\"identification\":0,\"mobile_verify\":1,\"silence\":0,\"official_verify\":{\"type\":-1,\"desc\":\"\",\"role\":0}}","msg_key":1734992,"timestamp":1536573105.4239}`)
+			err = nc.Post(context.Background(), notifyURL, string(m))
+			So(err, ShouldBeNil)
+		})
+	})
+}

app/infra/notify/notify/notify_test.go

@@ -0,0 +1,138 @@
+package notify
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	"net/http"
+	"os"
+	"testing"
+
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/dao"
+	"go-common/app/infra/notify/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	d *dao.Dao
+)
+
+func TestMain(m *testing.M) {
+	var err error
+	flag.Set("conf", "../cmd/notify-test.toml")
+	if err = conf.Init(); err != nil {
+		log.Println(err)
+		return
+	}
+	d = dao.New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}
+
+func TestNotify(t *testing.T) {
+	var (
+		nt  *Sub
+		err error
+		pub *Pub
+	)
+	Convey("test notify", t, func() {
+		pub, err = NewPub(&model.Pub{
+			Cluster: "test",
+			Group:   "pub",
+			Topic:   "test1",
+		}, conf.Conf)
+		So(err, ShouldBeNil)
+		err = pub.Send([]byte("test"), []byte(`{"test":"123"}`))
+		So(err, ShouldBeNil)
+		nt, err = NewSub(&model.Watcher{
+			Cluster:  "test",
+			Group:    "test",
+			Topic:    "test1",
+			Callback: string([]byte(`{"http://127.0.0.1:18888/push1": 1}`)),
+		}, d, conf.Conf)
+		So(err, ShouldBeNil)
+		So(nt, ShouldNotBeNil)
+		err = nt.dial()
+		So(err, ShouldBeNil)
+		//	go nt.serve()
+		//fmt.Println(nt.consumer.))
+	})
+	nt, _ = NewSub(&model.Watcher{
+		Cluster: "test",
+		Group:   "test",
+		Topic:   "test1",
+	}, d, conf.Conf)
+	b, err := json.Marshal(map[string]string{
+		"http://127.0.0.1:18888/push1": "1",
+		"http://127.0.0.1:18888/push2": "2",
+	})
+	nt.w.Callback = string(b)
+	mockhttp()
+	Convey("test push", t, func() {
+		nt.push([]byte("push1"))
+	})
+}
+
+func mockhttp() {
+	http.HandleFunc("/push1", testpush1)
+	http.HandleFunc("/push2", testpush2)
+	go http.ListenAndServe(":18888", nil)
+}
+
+func testpush1(resp http.ResponseWriter, req *http.Request) {
+}
+
+func testpush2(resp http.ResponseWriter, req *http.Request) {
+	resp.WriteHeader(500)
+}
+
+func TestSub(t *testing.T) {
+	nt, _ := NewSub(&model.Watcher{
+		Cluster: "test",
+		Group:   "test",
+		Topic:   "test1",
+		Filters: []*model.Filter{
+			{
+				Field:     "table",
+				Condition: model.ConditionEq,
+				Value:     "yes",
+			},
+		},
+	}, d, conf.Conf)
+
+	Convey("test filter eq ", t, func() {
+		fy := []byte(`{"table":"yes"}`)
+		So(nt.filter(fy), ShouldBeFalse)
+		fy = []byte(`{"table":"no"}`)
+		So(nt.filter(fy), ShouldBeTrue)
+	})
+	Convey("test filter prefix ", t, func() {
+		nt.w.Filters = []*model.Filter{
+			{
+				Field:     "table",
+				Condition: model.ConditionPre,
+				Value:     "yes",
+			},
+		}
+		nt.parseFilter()
+		fy := []byte(`{"table":"yes1"}`)
+		So(nt.filter(fy), ShouldBeFalse)
+		fy = []byte(`{"table":"no"}`)
+		So(nt.filter(fy), ShouldBeTrue)
+	})
+}
+
+func TestParseNotifyURL(t *testing.T) {
+	Convey("test url parse result", t, func() {
+		u := "liverpc://live.bannedservice?version=0&cmd=Message.synUser"
+		notifyURL, err := parseNotifyURL(u)
+		So(err, ShouldBeNil)
+		So(notifyURL.RawURL, ShouldEqual, u)
+		So(notifyURL.Schema, ShouldEqual, "liverpc")
+		So(notifyURL.Host, ShouldEqual, "live.bannedservice")
+		So(notifyURL.Query.Get("version"), ShouldEqual, "0")
+		So(notifyURL.Query.Get("cmd"), ShouldEqual, "Message.synUser")
+	})
+}

app/infra/notify/notify/pub.go

@@ -0,0 +1,92 @@
+package notify
+
+import (
+	"sync"
+
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/model"
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+)
+
+// Pub define producer.
+type Pub struct {
+	group     string
+	topic     string
+	cluster   string
+	appsecret string
+	producer  sarama.SyncProducer
+}
+
+var (
+	// producer snapshot, key:group+topic
+	producers = make(map[string]sarama.SyncProducer)
+	pLock     sync.RWMutex
+)
+
+// NewPub new kafka producer.
+func NewPub(w *model.Pub, c *conf.Config) (p *Pub, err error) {
+	cluster, ok := c.Clusters[w.Cluster]
+	if !ok {
+		err = errClusterNotSupport
+		return
+	}
+	producer, err := producer(w.Group, w.Topic, cluster)
+	if err != nil {
+		return
+	}
+	p = &Pub{
+		group:     w.Group,
+		topic:     w.Topic,
+		cluster:   w.Cluster,
+		appsecret: w.AppSecret,
+		producer:  producer,
+	}
+	return
+}
+
+func producer(group, topic string, pCfg *conf.Kafka) (p sarama.SyncProducer, err error) {
+	var (
+		ok  bool
+		key = key(group, topic)
+	)
+	pLock.RLock()
+	if p, ok = producers[key]; ok {
+		pLock.RUnlock()
+		return
+	}
+	pLock.RUnlock()
+	// new
+	conf := sarama.NewConfig()
+	conf.Producer.Return.Successes = true
+	conf.Version = sarama.V1_0_0_0
+	if p, err = sarama.NewSyncProducer(pCfg.Brokers, conf); err != nil {
+		log.Error("group(%s) topic(%s) cluster(%s) NewSyncProducer error(%v)", group, topic, pCfg.Cluster, err)
+		return
+	}
+	pLock.Lock()
+	producers[key] = p
+	pLock.Unlock()
+	return
+}
+
+// Send publish kafka message.
+func (p *Pub) Send(key, value []byte) (err error) {
+	countProm.Incr(_opProducerMsgSpeed, p.group, p.topic)
+	message := &sarama.ProducerMessage{
+		Topic: p.topic,
+		Key:   sarama.ByteEncoder(key),
+		Value: sarama.ByteEncoder(value),
+	}
+	_, _, err = p.producer.SendMessage(message)
+	return
+}
+func key(group, topic string) string {
+	return group + ":" + topic
+}
+
+// Auth check user pub permission.
+func (p *Pub) Auth(secret string) bool {
+	return p.appsecret == secret
+}

app/infra/notify/notify/sub.go

@@ -0,0 +1,400 @@
+package notify
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/dao"
+	"go-common/app/infra/notify/model"
+	"go-common/library/log"
+	"go-common/library/net/netutil"
+	"go-common/library/stat/prom"
+
+	"github.com/Shopify/sarama"
+	cluster "github.com/bsm/sarama-cluster"
+	"github.com/rcrowley/go-metrics"
+)
+
+func init() {
+	// sarama metrics disable
+	metrics.UseNilMetrics = true
+}
+
+var (
+	errClusterNotSupport   = errors.New("cluster not support")
+	errClosedNotifyChannel = errors.New("notification channel is closed")
+	errConsumerOver        = errors.New("too many consumers")
+	errCallbackParse       = errors.New("parse callback error")
+
+	statProm  = prom.New().WithState("go_notify_state", []string{"role", "group", "topic", "partition"})
+	countProm = prom.New().WithState("go_notify_counter", []string{"operation", "group", "topic"})
+	// prom operation
+	_opCurrentConsumer   = "current_consumer"
+	_opProducerMsgSpeed  = "producer_msg_speed"
+	_opConsumerMsgSpeed  = "consumer_msg_speed"
+	_opConsumerPartition = "consumer_partition_speed"
+	_opPartitionOffset   = "consumer_partition_offset"
+	_opConsumerFail      = "consumer_fail"
+)
+
+const (
+	_defRoutine = 1
+	_retry      = 5
+	_syncCall   = 1
+	_asyncCall  = 2
+)
+
+// Sub notify instance
+type Sub struct {
+	c        *conf.Config
+	ctx      context.Context
+	cancel   context.CancelFunc
+	w        *model.Watcher
+	cluster  *conf.Kafka
+	clients  *Clients
+	dao      *dao.Dao
+	consumer *cluster.Consumer
+	filter   func(msg []byte) bool
+	routine  int
+	backoff  netutil.BackoffConfig
+	asyncRty chan *rtyMsg
+	ticker   *time.Ticker
+	stop     bool
+	closed   bool
+	once     sync.Once
+}
+
+type rtyMsg struct {
+	id    int64
+	msg   string
+	index int
+}
+
+// NewSub create notify instance and return it.
+func NewSub(w *model.Watcher, d *dao.Dao, c *conf.Config) (n *Sub, err error) {
+	n = &Sub{
+		c:        c,
+		w:        w,
+		routine:  _defRoutine,
+		backoff:  netutil.DefaultBackoffConfig,
+		asyncRty: make(chan *rtyMsg, 100),
+		dao:      d,
+		ticker:   time.NewTicker(time.Minute),
+	}
+	n.ctx, n.cancel = context.WithCancel(context.Background())
+	if clu, ok := c.Clusters[w.Cluster]; ok {
+		n.cluster = clu
+	} else {
+		err = errClusterNotSupport
+		return
+	}
+	if len(w.Filters) != 0 {
+		n.parseFilter()
+	}
+	err = n.parseCallback()
+	if err != nil {
+		err = errCallbackParse
+		return
+	}
+	// init clients
+	n.clients = NewClients(c, w)
+	err = n.dial()
+	if err != nil {
+		return
+	}
+	if w.Concurrent != 0 {
+		n.routine = w.Concurrent
+	}
+	go n.asyncRtyproc()
+	for i := 0; i < n.routine; i++ {
+		go n.serve()
+	}
+	countProm.Incr(_opCurrentConsumer, w.Group, w.Topic)
+	return
+}
+
+func (n *Sub) parseFilter() {
+	n.filter = func(b []byte) bool {
+		nmsg := new(model.Message)
+		err := json.Unmarshal(b, nmsg)
+		if err != nil {
+			log.Error("json err %v", err)
+			return true
+		}
+		for _, f := range n.w.Filters {
+			switch {
+			case f.Field == "table":
+				if f.Condition == model.ConditionEq && f.Value == nmsg.Table {
+					return false
+				}
+				if f.Condition == model.ConditionPre && strings.HasPrefix(nmsg.Table, f.Value) {
+					return false
+				}
+			case f.Field == "action":
+				if f.Condition == model.ConditionEq && f.Value == nmsg.Action {
+					return false
+				}
+			}
+		}
+		return true
+	}
+}
+
+// parseCallback parse each watcher's callback urls
+func (n *Sub) parseCallback() (err error) {
+	var notifyURL *model.NotifyURL
+	cbm := make(map[string]int8)
+	log.Info("callback(%v), topic(%s), group(%s)", n.w.Callback, n.w.Topic, n.w.Group)
+	err = json.Unmarshal([]byte(n.w.Callback), &cbm)
+	if err != nil {
+		log.Error(" Notify.parseCallback sub parse callback err %v, topic(%s), group(%s), callback(%s)",
+			err, n.w.Topic, n.w.Group, n.w.Callback)
+		return
+	}
+	cbs := make([]*model.Callback, 0, len(cbm))
+	for u, p := range cbm {
+		notifyURL, err = parseNotifyURL(u)
+		if err != nil {
+			log.Error("Notify.parseCallback url parse error(%v), url(%s), topic(%s), group(%s)",
+				err, u, n.w.Topic, n.w.Group)
+			return
+		}
+		cbs = append(cbs, &model.Callback{URL: notifyURL, Priority: p})
+	}
+	sort.Slice(cbs, func(i, j int) bool { return cbs[i].Priority > cbs[j].Priority })
+	n.w.Callbacks = cbs
+	return
+}
+
+func (n *Sub) dial() (err error) {
+	cfg := cluster.NewConfig()
+	cfg.ClientID = fmt.Sprintf("%s-%s", n.w.Topic, n.w.Group)
+	cfg.Net.KeepAlive = time.Second
+	cfg.Consumer.Offsets.CommitInterval = time.Second
+	cfg.Consumer.MaxWaitTime = time.Millisecond * 250
+	cfg.Consumer.MaxProcessingTime = time.Millisecond * 50
+	cfg.Consumer.Return.Errors = true
+	cfg.Version = sarama.V1_0_0_0
+	cfg.Group.Return.Notifications = true
+	cfg.Consumer.Offsets.Initial = sarama.OffsetNewest
+	if n.consumer, err = cluster.NewConsumer(n.cluster.Brokers, n.w.Group, []string{n.w.Topic}, cfg); err != nil {
+		log.Error("group(%s) topic(%s) cluster(%s)  cluster.NewConsumer() error(%v)", n.w.Group, n.w.Topic, n.cluster.Cluster, err)
+	} else {
+		log.Info("group(%s) topic(%s) cluster(%s) cluster.NewConsumer() ok", n.w.Group, n.w.Topic, n.cluster.Cluster)
+	}
+	return
+}
+
+func (n *Sub) serve() {
+	var (
+		msg    *sarama.ConsumerMessage
+		err    error
+		ok     bool
+		notify *cluster.Notification
+	)
+	defer n.once.Do(func() {
+		n.cancel()
+		n.Close()
+	})
+	for {
+		select {
+		case <-n.ctx.Done():
+			log.Error("sub cancel")
+			return
+		case err = <-n.consumer.Errors():
+			log.Error("group(%s) topic(%s) cluster(%s)  catch error(%v)", n.w.Group, n.w.Topic, n.cluster.Cluster, err)
+			return
+		case notify, ok = <-n.consumer.Notifications():
+			if !ok {
+				err = errClosedNotifyChannel
+				log.Info("notification notOk group(%s) topic(%s) cluster(%s) catch error(%v)", n.w.Group, n.w.Topic, n.cluster.Cluster, err)
+				return
+			}
+			switch notify.Type {
+			case cluster.UnknownNotification, cluster.RebalanceError:
+				err = errClosedNotifyChannel
+				log.Error("notification(%s) group(%s) topic(%s) cluster(%s)  catch error(%v)", notify.Type, n.w.Group, n.w.Topic, n.cluster.Cluster, err)
+				return
+			case cluster.RebalanceStart:
+				log.Info("notification(%s) group(%s) topic(%s) cluster(%s)  catch error(%v)", notify.Type, n.w.Group, n.w.Topic, n.cluster.Cluster, err)
+				continue
+			case cluster.RebalanceOK:
+				log.Info("notification(%s) group(%s) topic(%s) cluster(%s) catch error(%v)", notify.Type, n.w.Group, n.w.Topic, n.cluster.Cluster, err)
+			}
+			if len(notify.Current[n.w.Topic]) == 0 {
+				err = errConsumerOver
+				log.Warn("notification(%s) no topic group(%s) topic(%s) cluster(%s)  catch error(%v)", notify.Type, n.w.Group, n.w.Topic, n.cluster.Cluster, err)
+				return
+			}
+		case msg, ok = <-n.consumer.Messages():
+			if !ok {
+				log.Error("group(%s) topic(%s) cluster(%s)  message channel closed", n.w.Group, n.w.Topic, n.cluster.Cluster)
+				return
+			}
+			n.push(msg.Value)
+			n.consumer.MarkPartitionOffset(msg.Topic, msg.Partition, msg.Offset, "")
+			statProm.State(_opPartitionOffset, msg.Offset, n.w.Group, n.w.Topic, strconv.Itoa(int(msg.Partition)))
+			countProm.Incr(_opConsumerMsgSpeed, n.w.Group, n.w.Topic)
+			statProm.Incr(_opConsumerPartition, n.w.Group, n.w.Topic, strconv.Itoa(int(msg.Partition)))
+		}
+	}
+}
+
+// push call retry for each consumer group callback
+// will push to retry channel if failed.
+func (n *Sub) push(nmsg []byte) {
+	if n.filter != nil && n.filter(nmsg) {
+		return
+	}
+	for n.stop {
+		time.Sleep(time.Minute)
+	}
+	msg := string(nmsg)
+	for i := 0; i < len(n.w.Callbacks); i++ {
+		cb := n.w.Callbacks[i]
+		if err := n.retry(cb.URL, string(nmsg), _syncCall); err != nil {
+			id, err := n.backupMsg(msg, i)
+			if err != nil {
+				log.Error("group(%s) topic(%s) add msg(%s) backup fail err %v", n.w.Group, n.w.Topic, string(nmsg), err)
+			}
+			n.addAsyncRty(id, msg, i)
+			return
+		}
+	}
+}
+
+// asyncRtyproc async retry proc
+func (n *Sub) asyncRtyproc() {
+	var err error
+	for {
+		if n.Closed() {
+			return
+		}
+		rty, ok := <-n.asyncRty
+		countProm.Decr(_opConsumerFail, n.w.Group, n.w.Topic)
+		if !ok {
+			log.Error("async chan close ")
+			return
+		}
+		for i := rty.index; i < len(n.w.Callbacks); i++ {
+			err = n.retry(n.w.Callbacks[i].URL, rty.msg, _asyncCall)
+			if err != nil {
+				n.addAsyncRty(rty.id, rty.msg, i)
+				break
+			}
+		}
+		if err == nil {
+			// if ok,restart consumer.
+			n.stop = false
+			n.delBackup(rty.id)
+		}
+	}
+}
+
+// retry Sub do callback with retry
+func (n *Sub) retry(uri *model.NotifyURL, msg string, source int) (err error) {
+	log.Info("Notify.retry do callback url(%v), msg(%s), source(%d)", uri, msg, source)
+	for i := 0; i < _retry; i++ {
+		err = n.clients.Post(context.TODO(), uri, msg)
+		if err != nil {
+			time.Sleep(n.backoff.Backoff(i))
+			continue
+		} else {
+			log.Info("Notify.retry callback success group(%s), topic(%s), retry(%d), msg(%s), source(%d)",
+				n.w.Group, n.w.Topic, i, msg, source)
+			return
+		}
+	}
+	if err != nil {
+		log.Error("Notify.retry callback error(%v), uri(%s), msg(%s), source(%d)",
+			err, uri, msg, source)
+	}
+	return
+}
+
+// addAsyncRty asycn retry from last fail callback index.
+func (n *Sub) addAsyncRty(id int64, nmsg string, cbi int) {
+	if n.Closed() {
+		return
+	}
+	select {
+	case n.asyncRty <- &rtyMsg{id: id, msg: nmsg, index: cbi}:
+		countProm.Incr(_opConsumerFail, n.w.Group, n.w.Topic)
+	case <-n.ticker.C:
+		// async chan full,stop consumer until retry sucess.
+		n.stop = true
+	}
+}
+
+// AddRty add retry msg to asyncretry chan by global service.
+func (n *Sub) AddRty(nmsg string, id, cbi int64) {
+	select {
+	case n.asyncRty <- &rtyMsg{id: id, msg: nmsg, index: int(cbi)}:
+		countProm.Incr(_opConsumerFail, n.w.Group, n.w.Topic)
+	default:
+		log.Error("sub topic %s group %s,async chan full ", n.w.Topic, n.w.Group)
+	}
+}
+
+// backupMsg add failed message into db
+func (n *Sub) backupMsg(msg string, cbi int) (id int64, err error) {
+	id, err = n.dao.AddFailBk(context.Background(), n.w.Topic, n.w.Group, n.w.Cluster, msg, int64(cbi))
+	if err != nil {
+		log.Error("group(%s) topic(%s) add backup msg fail (%s)", n.w.Group, n.w.Topic, msg)
+	}
+	return
+}
+
+// delBackup delete failed message from db after async retry success
+func (n *Sub) delBackup(id int64) {
+	_, err := n.dao.DelFailBk(context.TODO(), id)
+	if err != nil {
+		log.Error("group(%s) topic(%s) del backup msg err(%v)", n.w.Group, n.w.Topic, err)
+	}
+}
+
+// Closed return if sub had been closed.
+func (n *Sub) Closed() bool {
+	return n.closed
+}
+
+// Close close sub.
+func (n *Sub) Close() {
+	if !n.closed {
+		n.closed = true
+		n.consumer.Close()
+		//	close(n.asyncRty)
+		countProm.Decr(_opCurrentConsumer, n.w.Group, n.w.Topic)
+	}
+}
+
+// IsUpdate check watch metadata if update.
+func (n *Sub) IsUpdate(w *model.Watcher) bool {
+	return n.w.Mtime.Unix() != w.Mtime.Unix()
+}
+
+// ParseNotifyURL Parse callback url struct by url string
+func parseNotifyURL(u string) (notifyURL *model.NotifyURL, err error) {
+	var parsedURL *url.URL
+	parsedURL, err = url.Parse(u)
+	if err != nil {
+		return
+	}
+	notifyURL = &model.NotifyURL{
+		RawURL: u,
+		Schema: parsedURL.Scheme,
+		Host:   parsedURL.Host,
+		Path:   parsedURL.Path,
+		Query:  parsedURL.Query(),
+	}
+	return
+}

app/infra/notify/service/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//app/infra/notify/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "pub.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/infra/notify/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/notify/conf:go_default_library",
+        "//app/infra/notify/dao:go_default_library",
+        "//app/infra/notify/model:go_default_library",
+        "//app/infra/notify/notify:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/notify/service/pub.go

@@ -0,0 +1,36 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/infra/notify/model"
+	"go-common/app/infra/notify/notify"
+	"go-common/library/ecode"
+)
+
+// Pub pub message.
+func (s *Service) Pub(c context.Context, arg *model.ArgPub) (err error) {
+	pc, ok := s.pubConfs[key(arg.Group, arg.Topic)]
+	if !ok {
+		err = ecode.AccessDenied
+		return
+	}
+	s.plock.RLock()
+	pub, ok := s.pubs[key(arg.Group, arg.Topic)]
+	s.plock.RUnlock()
+	if !ok {
+		pub, err = notify.NewPub(pc, s.c)
+		if err != nil {
+			return
+		}
+		s.plock.Lock()
+		s.pubs[key(arg.Group, arg.Topic)] = pub
+		s.plock.Unlock()
+	}
+	if !pub.Auth(arg.AppSecret) {
+		err = ecode.AccessDenied
+		return
+	}
+	err = pub.Send([]byte(arg.Key), []byte(arg.Msg))
+	return
+}

app/infra/notify/service/service.go

@@ -0,0 +1,142 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/dao"
+	"go-common/app/infra/notify/model"
+	"go-common/app/infra/notify/notify"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+)
+
+// Service struct
+type Service struct {
+	c        *conf.Config
+	dao      *dao.Dao
+	plock    sync.RWMutex
+	pubConfs map[string]*model.Pub
+	subs     map[string]*notify.Sub
+	pubs     map[string]*notify.Pub
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:        c,
+		dao:      dao.New(c),
+		pubConfs: make(map[string]*model.Pub),
+		subs:     make(map[string]*notify.Sub),
+		pubs:     make(map[string]*notify.Pub),
+	}
+	err := s.loadNotify()
+	if err != nil {
+		return
+	}
+	go s.notifyproc()
+	go s.loadPub()
+	go s.retryproc()
+	return s
+}
+
+func (s *Service) loadPub() {
+	for {
+		pubs, err := s.dao.LoadPub(context.TODO())
+		if err != nil {
+			log.Error("load pub info err %v", err)
+			time.Sleep(time.Minute)
+			continue
+		}
+		ps := make(map[string]*model.Pub, len(pubs))
+		for _, p := range pubs {
+			ps[key(p.Group, p.Topic)] = p
+		}
+		s.pubConfs = ps
+		time.Sleep(time.Minute)
+	}
+}
+
+// TODO():auto reload and update.
+func (s *Service) loadNotify() (err error) {
+	watcher, err := s.dao.LoadNotify(context.TODO(), env.Zone)
+	if err != nil {
+		log.Error("load notify err %v", err)
+		return
+	}
+	subs := make(map[string]*notify.Sub, len(watcher))
+	for _, w := range watcher {
+		if sub, ok := s.subs[key(w.Group, w.Topic)]; ok && !sub.Closed() && !sub.IsUpdate(w) {
+			subs[key(w.Group, w.Topic)] = sub
+		} else {
+			n, err := s.newSub(w)
+			if err != nil {
+				log.Error("create notify topic(%s) group(%s) err(%v)", w.Topic, w.Group, err)
+				continue
+			}
+			subs[key(w.Group, w.Topic)] = n
+			log.Info("new sub %s %s", w.Group, w.Topic)
+		}
+	}
+	// close subs not subscribe any more.
+	for k, sub := range s.subs {
+		if _, ok := subs[k]; !ok {
+			sub.Close()
+			log.Info("close sub not subscribe any %s", k)
+		}
+	}
+	s.subs = subs
+	return
+}
+
+func (s *Service) newSub(w *model.Watcher) (*notify.Sub, error) {
+	var err error
+	if w.Filter {
+		w.Filters, err = s.dao.Filters(context.TODO(), w.ID)
+		if err != nil {
+			log.Error("s.dao.Filters err(%v)", err)
+		}
+	}
+	return notify.NewSub(w, s.dao, s.c)
+}
+
+func (s *Service) notifyproc() {
+	for {
+		time.Sleep(time.Minute)
+		s.loadNotify()
+	}
+}
+
+func key(group, topic string) string {
+	return fmt.Sprintf("%s_%s", group, topic)
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}
+
+func (s *Service) retryproc() {
+	for {
+		fs, err := s.dao.LoadFailBk(context.TODO())
+		if err != nil {
+			log.Error("s.loadFailBk err (%v)", err)
+			time.Sleep(time.Minute)
+			continue
+		}
+		for _, f := range fs {
+			if n, ok := s.subs[key(f.Group, f.Topic)]; ok && !n.Closed() {
+				n.AddRty(f.Msg, f.ID, f.Index)
+			}
+		}
+		time.Sleep(time.Minute * 10)
+	}
+}

app/infra/notify/service/service_test.go

@@ -0,0 +1,41 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"log"
+	"testing"
+
+	"go-common/app/infra/notify/conf"
+	"go-common/app/infra/notify/model"
+	"go-common/library/ecode"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func TestMain(m *testing.M) {
+	var err error
+	flag.Set("conf", "../cmd/notify-test.toml")
+	if err = conf.Init(); err != nil {
+		log.Println(err)
+		return
+	}
+	s = New(conf.Conf)
+	m.Run()
+}
+func TestPub(t *testing.T) {
+	s.pubConfs = map[string]*model.Pub{
+		"test-test": &model.Pub{
+			Topic: "test",
+			Group: "test",
+		},
+	}
+	Convey("test pub", t, func() {
+		err := s.Pub(context.TODO(), &model.ArgPub{Topic: "test", Group: "test", AppSecret: "test"})
+		So(err, ShouldEqual, ecode.AccessDenied)
+	})
+}