Create & Init Project...

app/infra/canal/BUILD

@@ -0,0 +1,22 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/infra/canal/cmd:all-srcs",
+        "//app/infra/canal/conf:all-srcs",
+        "//app/infra/canal/dao:all-srcs",
+        "//app/infra/canal/http:all-srcs",
+        "//app/infra/canal/infoc:all-srcs",
+        "//app/infra/canal/model:all-srcs",
+        "//app/infra/canal/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/CHANGELOG.md

@@ -0,0 +1,78 @@
+### Canal
+#### Version 3.3.4
+> 1.修复tidb start reload阻塞
+
+#### Version 3.3.3
+> 1. 修复tidb sync问题
+
+#### Version 3.3.2
+> 1. 修复tidb重复关闭的问题
+
+#### Version 3.3.1
+> 1.更新企业微信报警
+
+#### Vesion 3.3.0
+1. 增加tidb的支持
+
+#### Version 3.3.2
+> 1.修复canal  reload阻塞
+
+#### Version 3.2.2
+> 1.backoff 策略报警  
+
+#### Version 3.2.1
+> 1.添加告警开关  
+
+#### Vesion 3.2.0
+1. 迁移infra
+
+#### Version 3.1.5
+1. 增加错误检查并发送企业微信告警
+
+#### Version 3.1.4
+1. 修复重置instance泄露target连接问题
+
+#### Version 3.1.3
+1. 检查master权限改为post接口rsion
+
+#### Version 3.1.2
+1. 添加error日志
+
+#### Version 3.1.1
+1. 增加告警时间周期配置
+2. 增加hbase同步延迟告警
+
+#### Version 3.1.0
+1. 增加同步Hbase功能
+2. 增加企业微信告警
+
+#### Version 3.0.9
+1. 修复不报警问题
+
+#### Version 3.0.8
+1. 升级基础库
+
+#### Version 3.0.7
+1. 升级基础库
+
+#### Version 3.0.6
+1. 增加测试环境强制同步pos接口
+
+#### Version 3.0.5 
+1. 增加本地canal测试配置  
+2. 增加同步最新bin_pos接口
+
+#### Version 3.0.4
+1. 修复异常close导致的panic
+
+#### Version 3.0.3
+1. discovery统一
+
+#### Version 3.0.2
+1. 修复err为nil panic
+
+#### Version 3.0.1
+1. 检测实例返回binname和binpos
+
+#### Version 3.0.0
+1. 合并大仓库,完全引用siddontang的go-mysql作为binlog replication

app/infra/canal/CONTRIBUTORS.md

@@ -0,0 +1,12 @@
+# Owner
+haoguanwei
+lintanghui
+
+# Author
+lintanghui
+guhao
+chenshangqiang
+
+# Reviewer
+maojian
+haoguanwei

app/infra/canal/OWNERS

@@ -0,0 +1,18 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- chenshangqiang
+- guhao
+- haoguanwei
+- lintanghui
+labels:
+- infra
+- infra/canal
+options:
+  no_parent_owners: true
+reviewers:
+- chenshangqiang
+- guhao
+- haoguanwei
+- lintanghui
+- maojian

app/infra/canal/README.md

@@ -0,0 +1,16 @@
+# go-common/app/infra/canal
+
+##### 项目简介
+> 1. 实现MySQL slave协议，充当伪从库
+> 2. 一个canal节点可同时同步一个或多个MySQL database,可指定开始binlog位置
+
+##### 编译环境
+> 1. 请只用golang v1.7.x以上版本编译执行。
+
+##### 依赖包
+> 1. 公共依赖
+
+##### 编译执行
+> 1. 启动执行
+> 2. 项目文档http://info.bilibili.co/pages/viewpage.action?pageId=4547253
+

app/infra/canal/cmd/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = [
+        "canal.tidb.toml",
+        "canal-test.toml",
+    ],
+    importpath = "go-common/app/infra/canal/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/http:go_default_library",
+        "//app/infra/canal/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/cmd/canal-test.toml

@@ -0,0 +1,25 @@
+[monitor]
+user = ""
+token = ""
+secret = ""
+
+[httpClient]
+key    = "654af11b5df0c9d3"
+secret = "a7512b8b243b82f4bdb72cf2824b3f8e"
+dial      = "1s"
+timeout   = "1s"
+keepAlive = "60s"
+
+[log]
+dir = "/data/log/canal/"
+Stdout = true
+
+[db]
+#    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_canal?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    dsn = "test:test@tcp(127.0.0.1:3306)/bilibili_canal?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4&allowNativePasswords=True"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "1s"
+    execTimeout = "1s"
+    tranTimeout = "1s"

app/infra/canal/cmd/canal.tidb.toml

@@ -0,0 +1,11 @@
+[instance]
+  name = "thumbup"
+ 	ClusterID     = "1"
+ 	Addrs         = [""]
+ 	Offset        = 0
+ 	CommitTS      = 0
+ 	monitor_period = "1s"
+  [[instance.db]]
+    schema = "bilibili_likes"
+    [[instance.db.table]]
+      name = "counts"

app/infra/canal/cmd/database.sql

@@ -0,0 +1,61 @@
+create database bilibili_canal;
+
+ CREATE TABLE `master_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `addr` varchar(64) NOT NULL COMMENT 'db addr hostname:port',
+  `bin_name` varchar(20) NOT NULL DEFAULT '' COMMENT 'binlog name',
+  `bin_pos` int(11) NOT NULL DEFAULT '0' COMMENT 'binlog position',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `cluster` varchar(50) NOT NULL DEFAULT '' COMMENT 'cluster',
+  `leader` varchar(20) NOT NULL DEFAULT '' COMMENT 'leader',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  `is_delete` tinyint(4) NOT NULL DEFAULT '0' COMMENT '是否删除 0-否 1-是',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_addr` (`addr`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=46585559 DEFAULT CHARSET=utf8 COMMENT='canal位置信息记录'
+
+CREATE TABLE `canal_apply` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增ID',
+  `addr` varchar(64) NOT NULL COMMENT 'db addr hostname:port',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT 'remark',
+  `cluster` varchar(50) NOT NULL DEFAULT '' COMMENT '集群',
+  `leader` varchar(20) NOT NULL DEFAULT '' COMMENT 'leader',
+  `comment` text NOT NULL COMMENT 'comment',
+  `state` tinyint(4) NOT NULL DEFAULT '1' COMMENT 'state',
+  `operator` varchar(32) NOT NULL DEFAULT '' COMMENT 'operator',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  `conf_id` int(11) NOT NULL DEFAULT '0' COMMENT '配置id',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_addr` (`addr`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8 COMMENT='canal申请信息'
+
+CREATE TABLE `hbase_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `cluster_name` varchar(20) NOT NULL DEFAULT '' COMMENT '集群名称',
+  `table_name` varchar(60) NOT NULL DEFAULT '' COMMENT '表名',
+  `latest_ts` int(11) unsigned NOT NULL DEFAULT '0' COMMENT 'lastest ts',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name_table` (`cluster_name`,`table_name`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COMMENT='hbase latest_ts表'
+
+CREATE TABLE `tidb_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `name` varchar(20) NOT NULL DEFAULT '' COMMENT 'name',
+  `cluster_id` varchar(40) NOT NULL DEFAULT '' COMMENT 'cluster id',
+  `offset` bigint(20) NOT NULL DEFAULT 0 COMMENT 'offset',
+  `tso` bigint(20) NOT NULL DEFAULT '0' COMMENT '全局时间戳',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name` (`name`),
+  KEY `ix_mtime` (`mtime`)
+) COMMENT='tidb info';

app/infra/canal/cmd/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/http"
+	"go-common/app/infra/canal/service"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("canal start")
+	canal := service.NewCanal(conf.Conf)
+	http.Init(conf.Conf, canal)
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-ch
+		log.Info("canal get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			canal.Close()
+			log.Info("canal exit")
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/infra/canal/conf/BUILD

@@ -0,0 +1,46 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal_conf.go",
+        "conf.go",
+        "tidb_conf.go",
+    ],
+    importpath = "go-common/app/infra/canal/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/infoc:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/canal:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/client:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/conf/canal_conf.go

@@ -0,0 +1,310 @@
+package conf
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"regexp"
+	"strings"
+
+	"go-common/app/infra/canal/infoc"
+	"go-common/library/conf"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/siddontang/go-mysql/canal"
+	"github.com/siddontang/go-mysql/client"
+	"github.com/siddontang/go-mysql/mysql"
+)
+
+var (
+	// config change event
+	event      = make(chan *InsConf, 1)
+	hbaseEvent = make(chan *HBaseInsConf, 1)
+	tidbEvent  = make(chan *TiDBInsConf, 1)
+	canalPath  string
+)
+
+// Addition addition attrbute of canal.
+type Addition struct {
+	PrimaryKey []string `toml:"primarykey"` // kafka msg key
+	OmitField  []string `toml:"omitfield"`  // field will be ignored in table
+}
+
+// CTable canal table.
+type CTable struct {
+	PrimaryKey []string `toml:"primarykey"` // kafka msg key
+	OmitField  []string `toml:"omitfield"`  // field will be ignored in table
+	OmitAction []string `toml:"omitaction"` // action will be ignored in table
+	Name       string   `toml:"name"`       // table name support regular expression
+	Tables     []string
+}
+
+// Database represent mysql db
+type Database struct {
+	Schema   string          `toml:"schema"`
+	Databus  *databus.Config `toml:"databus"`
+	Infoc    *infoc.Config   `toml:"infoc"`
+	CTables  []*CTable       `toml:"table"`
+	TableMap map[string]*Addition
+}
+
+// CheckTable check database tables.
+func (db *Database) CheckTable(addr, user, passwd string) (err error) {
+	var (
+		conn  *client.Conn
+		res   *mysql.Result
+		regex *regexp.Regexp
+		table string
+	)
+	db.TableMap = make(map[string]*Addition)
+	if conn, err = client.Connect(addr, user, passwd, db.Schema); err != nil {
+		return
+	}
+	defer conn.Close()
+	if res, err = conn.Execute(fmt.Sprintf("SHOW TABLES FROM `%s`", db.Schema)); err != nil {
+		log.Error("conn.Execute() error(%v)", err)
+		return
+	}
+	for _, ctable := range db.CTables {
+		if regex, err = regexp.Compile(ctable.Name); err != nil {
+			log.Error("regexp.Compile(%s) error(%v)", ctable.Name, err)
+			return
+		}
+		for _, value := range res.Values {
+			table = fmt.Sprintf("%s", value[0])
+			if regex.MatchString(table) {
+				db.TableMap[table] = &Addition{
+					PrimaryKey: ctable.PrimaryKey,
+					OmitField:  ctable.OmitField,
+				}
+				ctable.Tables = append(ctable.Tables, table)
+			}
+		}
+		if len(ctable.Tables) == 0 {
+			return fmt.Errorf("addr(%s) db(%s) subscribles nothing,table(%s) is empty", addr, db.Schema, ctable.Name)
+		}
+	}
+	return
+}
+
+// InsConf instance config
+type InsConf struct {
+	*canal.Config
+
+	MonitorPeriod xtime.Duration    `toml:"monitor_period"`
+	MonitorOff    bool              `toml:"monitor_off"`
+	Databases     []*Database       `toml:"db"`
+	MasterInfo    *MasterInfoConfig `toml:"masterinfo"`
+}
+
+// HBaseTable hbase canal table.
+type HBaseTable struct {
+	Name      string   `toml:"name"`      // table name
+	OmitField []string `toml:"omitfield"` // field will be ignored in table
+}
+
+// HBaseDatabase hbase database.
+type HBaseDatabase struct {
+	Tables  []*HBaseTable   `toml:"table"`
+	Databus *databus.Config `toml:"databus"`
+}
+
+// HBaseInsConf hbase instance config.
+type HBaseInsConf struct {
+	Cluster string
+	Root    string
+	Addrs   []string
+
+	MonitorPeriod xtime.Duration    `toml:"monitor_period"`
+	MonitorOff    bool              `toml:"monitor_off"`
+	Databases     []*HBaseDatabase  `toml:"db"`
+	MasterInfo    *MasterInfoConfig `toml:"masterinfo"`
+}
+
+// CanalConfig config struct
+type CanalConfig struct {
+	Instances      []*InsConf      `toml:"instance"`
+	HBaseInstances []*HBaseInsConf `toml:"hbase_instance"`
+	TiDBInstances  []*TiDBInsConf  `toml:"tidb_instance"`
+}
+
+func newInsConf(fn, fc string) (c *InsConf, err error) {
+	var ic struct {
+		InsConf *InsConf `toml:"instance"`
+	}
+	ipPort := strings.TrimSuffix(fn, ".toml")
+	if _, _, err = net.SplitHostPort(ipPort); err != nil {
+		return
+	}
+	if _, err = toml.Decode(fc, &ic); err != nil {
+		return
+	}
+	if ic.InsConf == nil {
+		err = fmt.Errorf("file(%s) cannot decode toml", fn)
+		return
+	}
+	if ic.InsConf.Addr != ipPort {
+		err = fmt.Errorf("file(%s) name not equal addr(%s)", fn, ic.InsConf.Addr)
+		return
+	}
+	if ic.InsConf.MasterInfo == nil {
+		ic.InsConf.MasterInfo = Conf.MasterInfo
+	}
+	return ic.InsConf, nil
+}
+
+func newHBaseConf(fn, fc string) (c *HBaseInsConf, err error) {
+	var ic struct {
+		InsConf *HBaseInsConf `toml:"instance"`
+	}
+	if _, err = toml.Decode(fc, &ic); err != nil {
+		return
+	}
+	if ic.InsConf == nil {
+		err = fmt.Errorf("file(%s) cannot decode toml", fn)
+		return
+	}
+	cluster := strings.TrimSuffix(fn, ".hbase.toml")
+	if ic.InsConf.Cluster != cluster {
+		err = fmt.Errorf("file(%s) name not equal name(%s)", cluster, ic.InsConf.Cluster)
+		return
+	}
+	if ic.InsConf.MasterInfo == nil {
+		ic.InsConf.MasterInfo = Conf.MasterInfo
+	}
+	return ic.InsConf, nil
+}
+
+// LoadCanalConf load canal config.
+func LoadCanalConf() (c *CanalConfig, err error) {
+	var (
+		result []*conf.Value
+		ok     bool
+	)
+	if canalPath != "" {
+		result, err = localCanal()
+	} else {
+		result, ok = ConfClient.Configs()
+		if !ok {
+			panic("no canal-config")
+		}
+	}
+	c = new(CanalConfig)
+	im := map[string]struct{}{}
+	for _, ns := range result {
+		if ns.Name == "canal.toml" || ns.Name == "common.toml" {
+			continue
+		}
+		if strings.HasSuffix(ns.Name, ".hbase.toml") {
+			var ic *HBaseInsConf
+			if ic, err = newHBaseConf(ns.Name, ns.Config); err != nil {
+				err = fmt.Errorf("file(%s) decode error(%v)", ns.Name, err)
+				return
+			}
+			c.HBaseInstances = append(c.HBaseInstances, ic)
+		} else if strings.HasSuffix(ns.Name, ".tidb.toml") {
+			var ic *TiDBInsConf
+			if ic, err = newTiDBConf(ns.Name, ns.Config); err != nil {
+				err = fmt.Errorf("file(%s) decode error(%v)", ns.Name, err)
+				return
+			}
+			c.TiDBInstances = append(c.TiDBInstances, ic)
+		} else {
+			var ic *InsConf
+			if !strings.HasSuffix(ns.Name, ".toml") {
+				err = fmt.Errorf("file(%s) name is not a toml", ns.Name)
+				continue
+			}
+			if ic, err = newInsConf(ns.Name, ns.Config); err != nil {
+				err = fmt.Errorf("file(%s) decode error(%v)", ns.Name, err)
+				return
+			}
+			if _, ok := im[ic.Addr]; ok {
+				err = fmt.Errorf("file(%s) repeat with other toml", ns.Name)
+				return
+			}
+			im[ic.Addr] = struct{}{}
+			c.Instances = append(c.Instances, ic)
+		}
+	}
+	if canalPath == "" {
+		go func() {
+			for name := range ConfClient.Event() {
+				log.Info("config(%s) reload", name)
+				reloadConfig(name)
+			}
+		}()
+	}
+	return
+}
+
+func localCanal() (vs []*conf.Value, ok error) {
+	fs, err := ioutil.ReadDir(canalPath)
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range fs {
+		if !strings.HasSuffix(f.Name(), ".toml") {
+			continue
+		}
+		ct, err := ioutil.ReadFile(canalPath + f.Name())
+		if err != nil {
+			continue
+		}
+		vs = append(vs, &conf.Value{
+			Name:   f.Name(),
+			Config: string(ct),
+		})
+	}
+	return
+}
+
+// Event returns config change event chan,
+func Event() chan *InsConf {
+	return event
+}
+
+// HBaseEvent returns config change event chan,
+func HBaseEvent() chan *HBaseInsConf {
+	return hbaseEvent
+}
+
+func reloadConfig(name string) {
+	var (
+		cf string
+		ok bool
+	)
+	if name == "canal.toml" || name == "common.toml" {
+		LoadCanal()
+		return
+	}
+	if !strings.HasSuffix(name, ".toml") {
+		return
+	}
+	if cf, ok = ConfClient.Value(name); !ok {
+		// TODO(felix): auto reload? or restart hard?
+		return
+	}
+	if strings.HasSuffix(name, ".hbase.toml") {
+		ic, err := newHBaseConf(name, cf)
+		if err != nil {
+			return
+		}
+		hbaseEvent <- ic
+	} else if strings.HasSuffix(name, ".tidb.toml") {
+		ic, err := newTiDBConf(name, cf)
+		if err != nil {
+			return
+		}
+		tidbEvent <- ic
+	} else {
+		ic, err := newInsConf(name, cf)
+		if err != nil {
+			return
+		}
+		event <- ic
+	}
+}

app/infra/canal/conf/conf.go

@@ -0,0 +1,96 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+	"time"
+
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/naming/discovery"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	// ConfClient get config client
+	ConfClient *conf.Client
+	// Conf canal config variable
+	Conf = &Config{}
+)
+
+// Config canal config struct
+type Config struct {
+	Monitor *Monitor
+	// xlog
+	Log *log.Config
+	// http client
+	HTTPClient *bm.ClientConfig
+	// http server
+	BM *bm.ServerConfig
+	// master info
+	MasterInfo *MasterInfoConfig
+	// discovery
+	Discovery *discovery.Config
+	// db
+	DB *sql.Config
+}
+
+// Monitor wechat monitor
+type Monitor struct {
+	User   string
+	Token  string
+	Secret string
+}
+
+// MasterInfoConfig save pos of binlog in file or db
+type MasterInfoConfig struct {
+	Addr     string        `toml:"addr"`
+	DBName   string        `toml:"dbName"`
+	User     string        `toml:"user"`
+	Password string        `toml:"password"`
+	Timeout  time.Duration `toml:"timeout"`
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+	flag.StringVar(&canalPath, "canal", "", "canal instance path")
+}
+
+//Init int config
+func Init() (err error) {
+	if confPath != "" {
+		_, err = toml.DecodeFile(confPath, &Conf)
+		return
+	}
+	return remote()
+}
+
+func remote() (err error) {
+	if ConfClient, err = conf.New(); err != nil {
+		return
+	}
+	ConfClient.WatchAll()
+	err = LoadCanal()
+	return
+}
+
+// LoadCanal canal config
+func LoadCanal() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = ConfClient.Value("canal.toml"); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/infra/canal/conf/tidb_conf.go

@@ -0,0 +1,39 @@
+package conf
+
+import (
+	"fmt"
+
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+// TiDBInsConf tidb instance config
+type TiDBInsConf struct {
+	Name          string
+	ClusterID     string
+	Addrs         []string
+	Offset        int64
+	CommitTS      int64
+	MonitorPeriod xtime.Duration `toml:"monitor_period"`
+	Databases     []*Database    `toml:"db"`
+}
+
+func newTiDBConf(fn, fc string) (c *TiDBInsConf, err error) {
+	var ic struct {
+		InsConf *TiDBInsConf `toml:"instance"`
+	}
+	if _, err = toml.Decode(fc, &ic); err != nil {
+		return
+	}
+	if ic.InsConf == nil {
+		err = fmt.Errorf("file(%s) cannot decode toml", fn)
+		return
+	}
+	return ic.InsConf, nil
+}
+
+// TiDBEvent .
+func TiDBEvent() chan *TiDBInsConf {
+	return tidbEvent
+}

app/infra/canal/dao/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+    ],
+    importpath = "go-common/app/infra/canal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "mysql_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/infra/canal/dao/dao.go

@@ -0,0 +1,23 @@
+package dao
+
+import (
+	"go-common/app/infra/canal/conf"
+	"go-common/library/database/sql"
+)
+
+// Dao dao
+type Dao struct {
+	// config
+	c *conf.Config
+	// db
+	db *sql.DB
+}
+
+// New dao new
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:  c,
+		db: sql.NewMySQL(c.DB),
+	}
+	return
+}

app/infra/canal/dao/dao_test.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/infra/canal/conf"
+)
+
+var d *Dao
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "")
+		flag.Set("conf_token", "")
+		flag.Set("tree_id", "")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/canal-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/infra/canal/dao/mysql.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/infra/canal/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_tidbPositonSQL       = "SELECT name, cluster_id, offset, tso FROM  tidb_info WHERE name = ?"
+	_updateTidbPositonSQL = "INSERT INTO tidb_info(name, cluster_id, offset, tso) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE offset = ?, tso = ?"
+)
+
+// TiDBPosition get tidb positon
+func (d *Dao) TiDBPosition(c context.Context, name string) (res *model.TiDBInfo, err error) {
+	res = &model.TiDBInfo{}
+	if err = d.db.QueryRow(c, _tidbPositonSQL, name).Scan(&res.Name, &res.ClusterID, &res.Offset, &res.CommitTS); err != nil {
+		if err == sql.ErrNoRows {
+			res = nil
+			err = nil
+			return
+		}
+		log.Error("db.TidbPosition.Query error(%v,%v,%v)", _tidbPositonSQL, name, err)
+		return
+	}
+	return
+}
+
+// UpdateTiDBPosition update tidb position
+func (d *Dao) UpdateTiDBPosition(c context.Context, info *model.TiDBInfo) (err error) {
+	if info == nil {
+		return
+	}
+	if _, err = d.db.Exec(c, _updateTidbPositonSQL, info.Name, info.ClusterID, info.Offset, info.CommitTS, info.Offset, info.CommitTS); err != nil {
+		log.Error("db.UpdateTiDBPosition.Exec error(%v,%+v,%v)", _updateTidbPositonSQL, info, err)
+	}
+	return
+}

app/infra/canal/dao/mysql_test.go

@@ -0,0 +1,28 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/infra/canal/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_TiDBPosition(t *testing.T) {
+	info := &model.TiDBInfo{
+		Name:      "test",
+		ClusterID: "1",
+		Offset:    2,
+		CommitTS:  403845808070328359,
+	}
+	convey.Convey("add position", t, func(ctx convey.C) {
+		err := d.UpdateTiDBPosition(context.Background(), info)
+		ctx.So(err, convey.ShouldBeNil)
+		ctx.Convey("get position", func(ctx convey.C) {
+			gotRes, err := d.TiDBPosition(context.Background(), info.Name)
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(gotRes, convey.ShouldResemble, info)
+		})
+	})
+}

app/infra/canal/http/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal.go",
+        "http.go",
+        "infoc.go",
+    ],
+    importpath = "go-common/app/infra/canal/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/infoc:go_default_library",
+        "//app/infra/canal/service:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/canal:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/http/canal.go

@@ -0,0 +1,48 @@
+package http
+
+import (
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+func errors(c *bm.Context) {
+	type result struct {
+		Error         string            `json:"error"`
+		InstanceError map[string]string `json:"instance_error"`
+	}
+	res := result{
+		Error:         cs.Error(),
+		InstanceError: cs.Errors(),
+	}
+	c.JSON(res, nil)
+}
+
+func checkMaster(c *bm.Context) {
+	arg := new(struct {
+		Addr     string `form:"addr" validate:"required"`
+		User     string `form:"user" validate:"required"`
+		Password string `form:"password" validate:"required"`
+	})
+	if err := c.Bind(arg); err != nil {
+		return
+	}
+	name, pos, err := cs.CheckMaster(arg.Addr, arg.User, arg.Password)
+	if err != nil {
+		c.JSON(nil, ecode.AccessDenied)
+		return
+	}
+	res := map[string]interface{}{"name": name, "pos:": pos}
+	c.JSON(res, nil)
+}
+
+func syncPos(c *bm.Context) {
+	arg := new(struct {
+		Addr string `form:"addr" validate:"required"`
+	})
+	if err := c.Bind(arg); err != nil {
+		log.Error("syncpos params err %v", err)
+		return
+	}
+	c.JSON(nil, cs.PosSync(arg.Addr))
+}

app/infra/canal/http/http.go

@@ -0,0 +1,46 @@
+package http
+
+import (
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	cs *service.Canal
+)
+
+// Init int http service
+func Init(c *conf.Config, cs *service.Canal) {
+	initService(cs)
+	// init router
+	eg := bm.DefaultServer(c.BM)
+	initRouter(eg)
+	// init Outer serve
+	if err := eg.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+func initService(canal *service.Canal) {
+	cs = canal
+}
+
+// initRouter init outer router api path.
+func initRouter(e *bm.Engine) {
+	// init api
+	e.Ping(ping)
+	group := e.Group("/x/internal/canal")
+	{
+		group.GET("/infoc/post", infocPost)
+		group.GET("/infoc/current", infocCurrent)
+		group.GET("/errors", errors)
+		group.POST("/master/check", checkMaster)
+		group.POST("/test/sync", syncPos)
+	}
+}
+
+func ping(c *bm.Context) {
+}

app/infra/canal/http/infoc.go

@@ -0,0 +1,178 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"hash/crc32"
+	"io/ioutil"
+	"net/http"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	config "go-common/library/conf"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/BurntSushi/toml"
+	"github.com/siddontang/go-mysql/canal"
+)
+
+const (
+	_heartHeat   = 60
+	_readTimeout = 90
+	_flavor      = "mysql"
+	_updateUser  = "canal"
+	_updateMark  = "infoc"
+)
+
+// InfocConf .
+type infocConf struct {
+	Addr     string     `json:"db_addr"`
+	User     string     `json:"user"`
+	Pass     string     `json:"pass"`
+	InfocDBs []*infocDB `json:"databases"`
+}
+
+// InfocDB .
+type infocDB struct {
+	Schema           string       `json:"schema"`
+	Tables           []*infoTable `json:"tables"`
+	LancerAddr       string       `json:"lancer_addr"`
+	LancerTaskID     string       `json:"lancer_task_id"`
+	LancerReportAddr string       `json:"lancer_report_addr"`
+	Proto            string       `json:"proto"`
+}
+
+// InfoTable .
+type infoTable struct {
+	Name       string   `json:"name"`
+	OmitFlied  []string `json:"omit_field"`
+	OmitAction []string `json:"omit_action"`
+}
+
+func infocPost(c *bm.Context) {
+	var (
+		ics []*infocConf
+		bs  []byte
+		err error
+		buf *bytes.Buffer
+	)
+	content := make(map[string]string)
+	if bs, err = ioutil.ReadAll(c.Request.Body); err != nil {
+		c.AbortWithStatus(http.StatusInternalServerError)
+		return
+	}
+	if err = json.Unmarshal(bs, &ics); err != nil {
+		c.AbortWithStatus(http.StatusInternalServerError)
+		return
+	}
+	for _, ifc := range ics {
+		databases := make([]*conf.Database, len(ifc.InfocDBs))
+		for idx, infocDB := range ifc.InfocDBs {
+			tables := make([]*conf.CTable, len(infocDB.Tables))
+			for ix, table := range infocDB.Tables {
+				tables[ix] = &conf.CTable{
+					Name:       table.Name,
+					OmitAction: table.OmitAction,
+					OmitField:  table.OmitFlied,
+				}
+			}
+			databases[idx] = &conf.Database{
+				Schema: infocDB.Schema,
+				Infoc: &infoc.Config{
+					TaskID:       infocDB.LancerTaskID,
+					Addr:         infocDB.LancerAddr,
+					ReporterAddr: infocDB.LancerReportAddr,
+					Proto:        infocDB.Proto,
+				},
+				CTables: tables,
+			}
+		}
+		ic := &conf.InsConf{
+			Databases: databases,
+			Config: &canal.Config{
+				Addr:            ifc.Addr,
+				User:            ifc.User,
+				Password:        ifc.Pass,
+				ServerID:        crc32.ChecksumIEEE([]byte(ifc.Addr)),
+				Flavor:          _flavor,
+				HeartbeatPeriod: _heartHeat,
+				ReadTimeout:     _readTimeout,
+			},
+		}
+		var isc = &struct {
+			InsConf *conf.InsConf `toml:"instance"`
+		}{
+			InsConf: ic,
+		}
+		buf = new(bytes.Buffer)
+		if err = toml.NewEncoder(buf).Encode(isc); err != nil {
+			c.AbortWithStatus(http.StatusInternalServerError)
+			return
+		}
+		content[fmt.Sprintf("%v.toml", ifc.Addr)] = buf.String()
+	}
+	for cn, cv := range content {
+		value, err := conf.ConfClient.ConfIng(cn)
+		if err == nil {
+			err = conf.ConfClient.Update(value.CID, cv, _updateUser, _updateMark)
+		} else if err == ecode.NothingFound {
+			err = conf.ConfClient.Create(cn, cv, _updateUser, _updateMark)
+		}
+		if err != nil {
+			c.AbortWithStatus(http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func infocCurrent(c *bm.Context) {
+	var (
+		ok     bool
+		result []*config.Value
+	)
+	if result, ok = conf.ConfClient.Configs(); !ok {
+		c.Status(http.StatusInternalServerError)
+		return
+	}
+	ics := make([]*infocConf, 0, len(result))
+	for _, ns := range result {
+		var ic struct {
+			InsConf *conf.InsConf `toml:"instance"`
+		}
+		if _, err := toml.Decode(ns.Config, &ic); err != nil {
+			c.AbortWithStatus(http.StatusInternalServerError)
+			return
+		}
+		if ic.InsConf == nil {
+			continue
+		}
+		icf := &infocConf{
+			Addr: ic.InsConf.Addr,
+			User: ic.InsConf.User,
+			Pass: ic.InsConf.Password,
+		}
+		for _, icdb := range ic.InsConf.Databases {
+			if icdb.Infoc == nil {
+				continue
+			}
+			tables := make([]*infoTable, len(icdb.CTables))
+			for idx, ctable := range icdb.CTables {
+				tables[idx] = &infoTable{
+					Name:       ctable.Name,
+					OmitFlied:  ctable.OmitField,
+					OmitAction: ctable.OmitAction,
+				}
+			}
+			icf.InfocDBs = append(icf.InfocDBs, &infocDB{
+				Schema:       icdb.Schema,
+				Tables:       tables,
+				LancerAddr:   icdb.Infoc.Addr,
+				LancerTaskID: icdb.Infoc.TaskID,
+			})
+		}
+		ics = append(ics, icf)
+	}
+	c.JSON(ics, nil)
+}

app/infra/canal/infoc/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "infoc.go",
+        "reporter.go",
+    ],
+    importpath = "go-common/app/infra/canal/infoc",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/log:go_default_library",
+        "//library/net/ip:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/infoc/infoc.go

@@ -0,0 +1,144 @@
+package infoc
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"go-common/library/log"
+)
+
+var (
+	_infoc2Magic = []byte{172, 190} // NOTE: magic 0xAC0xBE
+	_infoc2Type  = []byte{0, 0}     // NOTE: type 0
+
+	_infocTimeout = 500 * time.Millisecond
+)
+
+// Config is infoc config.
+type Config struct {
+	TaskID string
+	// udp or tcp
+	Proto string
+	Addr  string
+	// reporter
+	ReporterAddr string
+}
+
+// Infoc infoc struct.
+type Infoc struct {
+	c      *Config
+	header []byte
+	// udp or tcp
+	conn net.Conn
+	lock sync.Mutex
+	// reporter
+	reporter *reporter
+}
+
+// New new infoc2 logger.
+func New(c *Config) (i *Infoc) {
+	i = &Infoc{
+		c:      c,
+		header: []byte(c.TaskID),
+	}
+	var err error
+	if i.conn, err = net.Dial(i.c.Proto, i.c.Addr); err != nil {
+		log.Error("infoc net dial error(%v)", err)
+	}
+	if c.ReporterAddr != "" {
+		i.reporter = newReporter(c.TaskID, c.ReporterAddr)
+		go i.reporter.reportproc()
+	}
+	return
+}
+
+// Rows the affected by binlog enent.
+func (i *Infoc) Rows(rows int64) {
+	if i.reporter != nil {
+		i.reporter.receiveIncr(rows)
+	}
+}
+
+// Send send message.
+func (i *Infoc) Send(ctx context.Context, key string, v interface{}) (err error) {
+	var b []byte
+	if b, err = json.Marshal(v); err != nil {
+		log.Error("json.Marshal(%v) error(%v)", v, err)
+		return
+	}
+	var (
+		res bytes.Buffer
+		buf bytes.Buffer
+	)
+	res.Write(_infoc2Magic)
+	// type and body buf, for calc length.
+	buf.Write(_infoc2Type)
+	buf.Write(i.header)
+	buf.WriteString(strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
+	// // append first arg
+	if _, err = buf.WriteString(string(b)); err != nil {
+		return
+	}
+	// put length
+	var ls [4]byte
+	binary.BigEndian.PutUint32(ls[:], uint32(buf.Len()))
+	res.Write(ls[:])       // NOTE: write length
+	res.Write(buf.Bytes()) // NOTE：write type and body
+	// write
+	if err = i.write(res.Bytes()); err != nil {
+		log.Error("infoc write error(%v)", err)
+		return
+	}
+	if i.reporter != nil {
+		i.reporter.sendIncr(1)
+	}
+	return
+}
+
+// write write data into connection.
+func (i *Infoc) write(bs []byte) (err error) {
+	defer func() {
+		if err != nil {
+			if i.conn != nil {
+				i.conn.Close()
+			}
+			i.conn = nil
+		}
+		i.lock.Unlock()
+	}()
+	i.lock.Lock()
+	// connection and write
+	if i.conn == nil {
+		if i.conn, err = net.DialTimeout(i.c.Proto, i.c.Addr, _infocTimeout); err != nil {
+			log.Error("infoc net dial error(%v)", err)
+			return
+		}
+	}
+	if i.c.Proto == "tcp" {
+		i.conn.SetDeadline(time.Now().Add(_infocTimeout))
+	}
+	if _, err = i.conn.Write(bs); err != nil {
+		log.Error("infoc net write error(%v)", err)
+	}
+	return
+}
+
+// Flush flush reporter count.
+func (i *Infoc) Flush() {
+	if i.reporter != nil {
+		i.reporter.flush()
+	}
+}
+
+// Close close resource.
+func (i *Infoc) Close() {
+	if i.conn != nil {
+		i.conn.Close()
+	}
+}

app/infra/canal/infoc/reporter.go

@@ -0,0 +1,84 @@
+package infoc
+
+import (
+	"fmt"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"go-common/library/log"
+	"go-common/library/net/ip"
+)
+
+type reporter struct {
+	taskID string
+	addr   string
+	iip    string
+
+	receiveCount int64
+	sendCount    int64
+
+	fails []string
+}
+
+func newReporter(taskID, addr string) (r *reporter) {
+	r = &reporter{
+		taskID: taskID,
+		addr:   addr,
+		iip:    ip.InternalIP(),
+	}
+	return
+}
+
+func (r *reporter) receiveIncr(delta int64) {
+	atomic.AddInt64(&r.receiveCount, delta)
+}
+
+func (r *reporter) sendIncr(delta int64) {
+	atomic.AddInt64(&r.sendCount, delta)
+}
+
+func (r *reporter) reportproc() {
+	tick := time.NewTicker(1 * time.Minute)
+	for {
+		<-tick.C
+		r.reporter()
+	}
+}
+
+func (r *reporter) flush() {
+	r.reporter()
+}
+
+func (r *reporter) reporter() {
+	const _timeout = time.Second
+	conn, err := net.DialTimeout("tcp", r.addr, _timeout)
+	if err != nil {
+		log.Error("infoc reporter flush dial error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(_timeout))
+	var fails []string
+	for _, fail := range r.fails {
+		if _, err = conn.Write([]byte(fail)); err != nil {
+			log.Error("infoc reporter write fail error(%v)", err)
+			fails = append(fails, fail)
+		}
+	}
+	for _, rc := range r.record(time.Now()) {
+		if _, err = conn.Write([]byte(rc)); err != nil {
+			log.Error("infoc reporter write error(%v)", err)
+			fails = append(fails, rc)
+		}
+	}
+	r.fails = fails
+}
+
+func (r *reporter) record(now time.Time) []string {
+	rc := atomic.SwapInt64(&r.receiveCount, 0)
+	sc := atomic.SwapInt64(&r.sendCount, 0)
+	rcW := fmt.Sprintf("agent.receive.count\001%d\001%s\001%d\001%s\001\001", rc, r.iip, now.UnixNano()/int64(time.Millisecond), r.taskID)
+	scW := fmt.Sprintf("agent.send.success.count\001%d\001%s\001%d\001%s\001\001", sc, r.iip, now.UnixNano()/int64(time.Millisecond), r.taskID)
+	return []string{rcW, scW}
+}

app/infra/canal/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/infra/canal/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/model/model.go

@@ -0,0 +1,19 @@
+package model
+
+// Data msg will be push to databus
+type Data struct {
+	Action string `json:"action"`
+	Table  string `json:"table"`
+	// kafka key
+	Key string                 `json:"-"`
+	Old map[string]interface{} `json:"old,omitempty"`
+	New map[string]interface{} `json:"new,omitempty"`
+}
+
+// TiDBInfo tidb db model
+type TiDBInfo struct {
+	Name      string
+	ClusterID string
+	Offset    int64
+	CommitTS  int64
+}

app/infra/canal/service/BUILD

@@ -0,0 +1,76 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal.go",
+        "hbase.go",
+        "instance.go",
+        "master.go",
+        "target.go",
+        "tidb_check.go",
+        "tidb_data.go",
+        "tidb_instance.go",
+        "tidb_proc.go",
+    ],
+    importpath = "go-common/app/infra/canal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/dao:go_default_library",
+        "//app/infra/canal/infoc:go_default_library",
+        "//app/infra/canal/model:go_default_library",
+        "//app/infra/canal/service/reader:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/netutil:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//vendor/github.com/juju/errors:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/canal:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/client:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/replication:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/infra/canal/service/reader:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["tidb_data_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/canal/model:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+    ],
+)

app/infra/canal/service/canal.go

@@ -0,0 +1,364 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/dao"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	xhttp "go-common/library/net/http/blademaster"
+	"go-common/library/net/netutil"
+	"go-common/library/stat/prom"
+
+	"github.com/siddontang/go-mysql/client"
+)
+
+var (
+	stats       = prom.New().WithState("go_canal_counter", []string{"type", "addr", "scheme", "table", "action"})
+	tblReplacer = regexp.MustCompile("[0-9]+") // NOTE: replace number of sub-table name to space
+)
+
+// Canal is canal.
+type Canal struct {
+	dao            *dao.Dao
+	instances      map[string]*Instance
+	hbaseInstances map[string]*HBaseInstance
+	tidbInstances  map[string]*tidbInstance
+	insl           sync.RWMutex
+	tidbInsl       sync.RWMutex
+	err            error
+	backoff        netutil.Backoff
+	errCount       int64
+	lastErr        time.Time
+}
+
+// NewCanal load config and start canal instance.
+func NewCanal(config *conf.Config) (c *Canal) {
+	c = &Canal{
+		dao: dao.New(config),
+	}
+	cfg, err := conf.LoadCanalConf()
+	if err != nil {
+		panic(err)
+	}
+	c.instances = make(map[string]*Instance, len(cfg.Instances))
+	c.hbaseInstances = make(map[string]*HBaseInstance, len(cfg.HBaseInstances))
+	c.tidbInstances = make(map[string]*tidbInstance, len(cfg.TiDBInstances))
+	for _, insc := range cfg.Instances {
+		ins, err := NewInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%+v)", err)
+		}
+		c.insl.Lock()
+		c.instances[insc.Addr] = ins
+		c.insl.Unlock()
+		if err == nil {
+			go ins.Start()
+			log.Info("start canal instance(%s) success", ins.String())
+		}
+	}
+	c.backoff = &netutil.BackoffConfig{
+		MaxDelay:  120 * time.Second,
+		BaseDelay: 1.0 * time.Second,
+		Factor:    1.6,
+		Jitter:    0.2,
+	}
+	c.lastErr = time.Now()
+	go c.eventproc()
+	// hbase
+	for _, insc := range cfg.HBaseInstances {
+		ins, err := NewHBaseInstance(insc)
+		if err != nil {
+			log.Error("new hbase instance error(%+v)", err)
+		}
+		c.insl.Lock()
+		c.hbaseInstances[insc.Cluster] = ins
+		c.insl.Unlock()
+		if err == nil {
+			ins.Start()
+			log.Info("start hbase instance(%s) success", ins.String())
+		}
+	}
+	go c.hbaseeventproc()
+
+	// tidb
+	for _, insc := range cfg.TiDBInstances {
+		ins, err := newTiDBInstance(c, insc)
+		if err != nil {
+			log.Error("new instance error(%+v)", err)
+			c.sendWx(fmt.Sprintf("new tidb canal instance(%s) failed error(%v)", ins.String(), err))
+			continue
+		}
+		c.tidbInsl.Lock()
+		c.tidbInstances[insc.Name] = ins
+		c.tidbInsl.Unlock()
+		if err == nil {
+			go ins.start()
+			log.Info("start tidb canal instance(%s) success", ins.String())
+		}
+	}
+	go c.tidbEventproc()
+	go c.monitorproc()
+	// errproc
+	go c.errproc()
+	return
+}
+
+// CheckMaster check master status.
+func (c *Canal) CheckMaster(addr, user, pwd string) (name string, pos int64, err error) {
+	conn, err := client.Connect(addr, user, pwd, "")
+	if err != nil {
+		return
+	}
+	rr, err := conn.Execute("SHOW MASTER STATUS")
+	if err != nil {
+		return
+	}
+	name, _ = rr.GetString(0, 0)
+	pos, _ = rr.GetInt(0, 1)
+	if name != "" && pos > 0 {
+		return
+	}
+	return "", 0, fmt.Errorf("check master no name|pos")
+}
+
+// PosSync sync newewst bin_pos.
+func (c *Canal) PosSync(addr string) (err error) {
+	c.insl.Lock()
+	old, ok := c.instances[addr]
+	c.insl.Unlock()
+	if !ok {
+		return
+	}
+	pos, err := old.GetMasterPos()
+	if err != nil {
+		return
+	}
+	old.Close()
+	old.OnPosSynced(pos, true)
+
+	ins, _ := NewInstance(old.config)
+	c.insl.Lock()
+	c.instances[addr] = ins
+	c.insl.Unlock()
+	ins.Start()
+	return
+}
+
+// Errors returns instance errors.
+func (c *Canal) Errors() (ies map[string]string) {
+	ies = map[string]string{}
+	c.insl.RLock()
+	for _, i := range c.instances {
+		ies[i.String()] = i.Error()
+	}
+	for _, i := range c.hbaseInstances {
+		ies[i.String()] = i.Error()
+	}
+	c.insl.RUnlock()
+	return
+}
+
+// Error returns canal error.
+func (c *Canal) Error() string {
+	if c.err == nil {
+		return ""
+	}
+	return c.err.Error()
+}
+
+// errproc check errors.
+func (c *Canal) errproc() {
+	for {
+		time.Sleep(10 * time.Second)
+		es := c.Error()
+		if es != "" {
+			c.sendWx(fmt.Sprintf("canal occur error(%s)", es))
+		}
+		ies := c.Errors()
+		for k, v := range ies {
+			if v != "" {
+				c.sendWx(fmt.Sprintf("canal instance(%s) occur error(%s)", k, v))
+			}
+		}
+	}
+}
+
+// Close close canal instance
+func (c *Canal) Close() {
+	c.insl.RLock()
+	defer c.insl.RUnlock()
+	for _, ins := range c.instances {
+		ins.Close()
+		log.Info("close canal instance(%s) success", ins.String())
+	}
+	for _, ins := range c.hbaseInstances {
+		ins.Close()
+		log.Info("close hbase instance(%s) success", ins.String())
+	}
+	for _, ins := range c.tidbInstances {
+		ins.close()
+		log.Info("close tidb instance(%s) success", ins.String())
+	}
+}
+
+func (c *Canal) eventproc() {
+	ech := conf.Event()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		ins, err := NewInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.sendWx(fmt.Sprintf("reload canal instance(%s) failed error(%v)", ins.String(), err))
+			c.insl.Lock()
+			if old, ok := c.instances[insc.Addr]; ok {
+				old.Close()
+			}
+			c.instances[insc.Addr] = ins
+			c.insl.Unlock()
+			continue
+		}
+		c.insl.Lock()
+		if old, ok := c.instances[insc.Addr]; ok {
+			old.Close()
+		}
+		c.instances[insc.Addr] = ins
+		c.insl.Unlock()
+		go ins.Start()
+		log.Info("reload canal instance(%s) success", ins.String())
+		//	c.sendWx(fmt.Sprintf("reload canal instance(%s) success", ins.String()))
+	}
+}
+
+func (c *Canal) hbaseeventproc() {
+	ech := conf.HBaseEvent()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		ins, err := NewHBaseInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.insl.Lock()
+			if old, ok := c.hbaseInstances[insc.Cluster]; ok {
+				old.Close()
+			}
+			c.hbaseInstances[insc.Cluster] = ins
+			c.insl.Unlock()
+			continue
+		}
+		c.insl.Lock()
+		if old, ok := c.hbaseInstances[insc.Cluster]; ok {
+			old.Close()
+		}
+		c.hbaseInstances[insc.Cluster] = ins
+		c.insl.Unlock()
+		ins.Start()
+		log.Info("reload hbase instance(%s) success", ins.String())
+	}
+}
+
+// monitorproc monitor instance delay.
+func (c *Canal) monitorproc() {
+	if env.DeployEnv != env.DeployEnvProd {
+		return
+	}
+	const delay = 2 * time.Minute
+	for {
+		time.Sleep(delay)
+		c.insl.RLock()
+		for _, ins := range c.instances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && !ins.config.MonitorOff && dt > threshold {
+				for _, db := range ins.config.Databases {
+					c.sendWx(fmt.Sprintf("canal env(%s) 数据库(%s)地址(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, db.Schema, ins.config.Addr, threshold, dt))
+				}
+			}
+		}
+		for _, ins := range c.hbaseInstances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && !ins.config.MonitorOff && dt > threshold {
+				c.sendWx(fmt.Sprintf("canal env(%s) hbase集群(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, ins.config.Cluster, threshold, dt))
+			}
+		}
+		c.insl.RUnlock()
+		c.tidbInsl.RLock()
+		for _, ins := range c.tidbInstances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && dt > threshold {
+				for _, db := range ins.config.Databases {
+					c.sendWx(fmt.Sprintf("tidb canal env(%s) 数据库(%s)地址(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, db.Schema, ins.config.Name, threshold, dt))
+				}
+			}
+		}
+		c.tidbInsl.RUnlock()
+	}
+}
+
+func (c *Canal) sendWx(msg string) {
+	count := atomic.LoadInt64(&c.errCount)
+	atomic.AddInt64(&c.errCount, 1)
+	duration := c.backoff.Backoff(int(count))
+	if time.Since(c.lastErr) < duration {
+		return
+	}
+	c.lastErr = time.Now()
+	sendWechat(msg, conf.Conf.Monitor.User)
+}
+
+func sendWechat(msg string, user string) {
+	params := url.Values{}
+	params.Set("Action", "CreateWechatMessage")
+	params.Set("PublicKey", "9c178e51a7d4dc8aa1dbef0c790b06e7574c4d0etracehubtuhui@bilibili.com")
+	params.Set("UserName", user)
+	params.Set("Content", msg)
+	params.Set("TreeId", "bilibili.main.common-arch.canal")
+	params.Set("Title", "canal 监控报警")
+	params.Set("Signature", "1")
+	req, _ := http.NewRequest("POST", "http://merak.bilibili.co", strings.NewReader(params.Encode()))
+	req.Header.Add("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
+	var v struct {
+		ReqID    string `json:"reqId"`
+		Status   int64  `json:"status"`
+		Response struct {
+			status int
+		} `json:"Response"`
+	}
+	if err := xhttp.NewClient(conf.Conf.HTTPClient).Do(context.TODO(), req, &v); err != nil {
+		log.Error("send wechat monitor status(%d) msg(%v,%v) error(%v)", v.Status, v.Response, v.Response.status, err)
+	}
+}

app/infra/canal/service/hbase.go

@@ -0,0 +1,153 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	hbase "go-common/library/database/hbase.v2"
+
+	"github.com/pkg/errors"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+// HBaseInstance hbase canal instance
+type HBaseInstance struct {
+	config *conf.HBaseInsConf
+	client *hbase.Client
+	// one instance can have lots of target different by schema and table
+	targets []*databus.Databus
+	hinfo   *hbaseInfo
+
+	// scan latest timestamp, be used check delay
+	latestTimestamp int64
+
+	err    error
+	closed bool
+}
+
+// NewHBaseInstance new canal instance
+func NewHBaseInstance(c *conf.HBaseInsConf) (ins *HBaseInstance, err error) {
+	// new instance
+	ins = &HBaseInstance{config: c}
+	// check and modify config
+	if c.MasterInfo == nil {
+		err = errors.New("no masterinfo config")
+		ins.err = err
+		return
+	}
+	// new hbase info
+	if ins.hinfo, err = newHBaseInfo(ins.config.Cluster, ins.config.MasterInfo); err != nil {
+		log.Error("init hbase info error(%v)", err)
+		ins.err = errors.Wrap(err, "init hbase info")
+		return
+	}
+	// new hbase client
+	ins.client = hbase.NewClient(&hbase.Config{Zookeeper: &hbase.ZKConfig{
+		Root:  c.Root,
+		Addrs: c.Addrs,
+	}})
+	return
+}
+
+// Start start binlog receive
+func (ins *HBaseInstance) Start() {
+	for _, db := range ins.config.Databases {
+		databus := databus.New(db.Databus)
+		ins.targets = append(ins.targets, databus)
+		for _, tb := range db.Tables {
+			go ins.scanproc(db, databus, tb)
+		}
+	}
+}
+
+func (ins *HBaseInstance) scanproc(c *conf.HBaseDatabase, databus *databus.Databus, table *conf.HBaseTable) {
+	latestTs := ins.hinfo.LatestTs(table.Name)
+	if latestTs == 0 {
+		latestTs = uint64(time.Now().UnixNano() / 1e6)
+	}
+AGAIN:
+	for {
+		time.Sleep(300 * time.Millisecond)
+		nowTs := uint64(time.Now().UnixNano() / 1e6)
+		// scan
+		scanner, err := ins.client.Scan(context.TODO(), []byte(table.Name), hrpc.TimeRangeUint64(latestTs, nowTs))
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		ins.latestTimestamp = time.Now().Unix()
+		for {
+			res, err := scanner.Next()
+			if err == io.EOF {
+				latestTs = nowTs
+				ins.hinfo.Save(table.Name, latestTs)
+				continue AGAIN
+			} else if err != nil {
+				time.Sleep(time.Second)
+				continue AGAIN
+			}
+			var key string
+			data := map[string]interface{}{}
+			for _, cell := range res.Cells {
+				row := string(cell.Row)
+				if key == "" {
+					key = row
+				}
+				if _, ok := data[row]; !ok {
+					data[row] = map[string]string{}
+				}
+				column := string(cell.Qualifier)
+				omit := false
+				for _, field := range table.OmitField {
+					if field == column {
+						omit = true
+						break
+					}
+				}
+				if !omit {
+					cm := data[row].(map[string]string)
+					cm[column] = string(cell.Value)
+				}
+			}
+			real := &model.Data{Table: table.Name, New: data}
+			databus.Send(context.TODO(), key, real)
+			log.Info("hbase databus:group(%s)topic(%s) pub(key:%s, value:%+v) succeed", c.Databus.Group, c.Databus.Topic, key, real)
+		}
+	}
+}
+
+// Close close instance
+func (ins *HBaseInstance) Close() {
+	if ins.err != nil {
+		return
+	}
+	ins.client.Close()
+	for _, t := range ins.targets {
+		t.Close()
+	}
+	ins.closed = true
+	ins.err = errors.New("canal hbase instance closed")
+}
+
+func (ins *HBaseInstance) String() string {
+	return ins.config.Cluster
+}
+
+// Error returns instance error.
+func (ins *HBaseInstance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *HBaseInstance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}

app/infra/canal/service/instance.go

@@ -0,0 +1,172 @@
+package service
+
+import (
+	"fmt"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+	"github.com/siddontang/go-mysql/canal"
+	"github.com/siddontang/go-mysql/mysql"
+	"github.com/siddontang/go-mysql/replication"
+)
+
+// Instance canal instance
+type Instance struct {
+	*canal.Canal
+	config *conf.InsConf
+	// one instance can have lots of target different by schema and table
+	targets []*Target
+	master  *dbMasterInfo
+
+	// binlog latest timestamp, be used check delay
+	latestTimestamp int64
+	err             error
+	closed          bool
+}
+
+// NewInstance new canal instance
+func NewInstance(c *conf.InsConf) (ins *Instance, err error) {
+	// new instance
+	ins = &Instance{config: c}
+	// check and modify config
+	if c.MasterInfo == nil {
+		err = errors.New("no masterinfo config")
+		ins.err = err
+		return
+	}
+	if c.ReadTimeout == 0 {
+		c.ReadTimeout = 90
+	}
+	c.ReadTimeout = c.ReadTimeout * time.Second
+	c.HeartbeatPeriod = c.HeartbeatPeriod * time.Second
+
+	if ins.master, err = newDBMasterInfo(ins.config.Addr, ins.config.MasterInfo); err != nil {
+		log.Error("init master info error(%v)", err)
+		ins.err = errors.Wrap(err, "init master info")
+		return
+	}
+	if ins.targets, err = newTargets(c); err != nil {
+		log.Error("db.init() error(%v)", err)
+		ins.err = errors.Wrap(err, "db init")
+		return
+	}
+	ins.latestTimestamp = time.Now().Unix()
+	// new canal
+	if ins.Canal, err = canal.NewCanal(c.Config); err != nil {
+		log.Error("canal.NewCanal(%v) error(%v)", c.Config, err)
+		ins.err = errors.Wrapf(err, "canal NewCanal(%v)", c.Config)
+		return
+	}
+	// implement self as canal's event handler
+	ins.Canal.SetEventHandler(ins)
+
+	return
+}
+
+func newTargets(c *conf.InsConf) (targets []*Target, err error) {
+	targets = make([]*Target, 0, len(c.Databases))
+	for _, db := range c.Databases {
+		if err = db.CheckTable(c.Addr, c.User, c.Password); err != nil {
+			log.Error("db.CheckTable() error(%v)", err)
+			return
+		}
+		targets = append(targets, NewTarget(db))
+	}
+	return
+}
+
+// Start start binlog receive
+func (ins *Instance) Start() {
+	pos := ins.master.Pos()
+	if pos.Name == "" || pos.Pos == 0 {
+		var err error
+		if pos, err = ins.Canal.GetMasterPos(); err != nil {
+			log.Error("c.MasterPos error(%v)", err)
+			ins.err = errors.Wrap(err, "canal get master pos when start")
+			return
+		}
+	}
+	ins.err = ins.Canal.RunFrom(pos)
+}
+
+// Close close instance
+func (ins *Instance) Close() {
+	if ins.err != nil {
+		return
+	}
+	ins.Canal.Close()
+	for _, t := range ins.targets {
+		t.close()
+	}
+	ins.closed = true
+	ins.err = errors.New("canal closed")
+}
+
+// Check filter row event
+func (ins *Instance) Check(ev *canal.RowsEvent) (ts []*Target) {
+	for _, t := range ins.targets {
+		if t.compare(ev.Table.Schema, ev.Table.Name, ev.Action) {
+			ts = append(ts, t)
+		}
+	}
+	return
+}
+
+func (ins *Instance) String() string {
+	return ins.config.Addr
+}
+
+// OnRotate OnRotate
+func (ins *Instance) OnRotate(re *replication.RotateEvent) error {
+	log.Info("OnRotate binlog addr(%s) rotate binname(%s) pos(%d)", ins.config.Addr, re.NextLogName, re.Position)
+	return nil
+}
+
+// OnDDL OnDDL
+func (ins *Instance) OnDDL(pos mysql.Position, qe *replication.QueryEvent) error {
+	log.Info("OnDDL binlog addr(%s) ddl binname(%s) pos(%d)", ins.config.Addr, pos.Name, pos.Pos)
+	return nil
+}
+
+// OnXID OnXID
+func (ins *Instance) OnXID(mysql.Position) error {
+	return nil
+}
+
+//OnGTID OnGTID
+func (ins *Instance) OnGTID(mysql.GTIDSet) error {
+	return nil
+}
+
+// OnPosSynced OnPosSynced
+func (ins *Instance) OnPosSynced(pos mysql.Position, force bool) error {
+	return ins.master.Save(pos, force)
+}
+
+// OnRow send the envent to table
+func (ins *Instance) OnRow(ev *canal.RowsEvent) error {
+	for _, t := range ins.Check(ev) {
+		t.send(ev)
+	}
+	if stats != nil {
+		stats.Incr("syncer_counter", ins.String(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+		stats.State("delay_syncer", ins.delay(), ins.String(), ev.Table.Schema, "", "")
+	}
+	ins.latestTimestamp = time.Now().Unix()
+	return nil
+}
+
+// Error returns instance error.
+func (ins *Instance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *Instance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}

app/infra/canal/service/master.go

@@ -0,0 +1,145 @@
+package service
+
+import (
+	"sync"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/client"
+	"github.com/siddontang/go-mysql/mysql"
+)
+
+type dbMasterInfo struct {
+	c *conf.MasterInfoConfig
+
+	addr    string
+	binName string
+	binPos  uint32
+
+	l sync.RWMutex
+
+	lastSaveTime time.Time
+}
+
+func newDBMasterInfo(addr string, c *conf.MasterInfoConfig) (*dbMasterInfo, error) {
+	m := &dbMasterInfo{c: c, addr: addr}
+	conn, err := client.Connect(c.Addr, c.User, c.Password, c.DBName)
+	if err != nil {
+		log.Error("db master info client error(%v)", err)
+		return nil, errors.Trace(err)
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	r, err := conn.Execute("SELECT addr,bin_name,bin_pos FROM master_info WHERE addr=?", addr)
+	if err != nil {
+		log.Error("new db load master.info error(%v)", err)
+		return nil, errors.Trace(err)
+	}
+	if r.RowNumber() == 0 {
+		if m.c.Timeout > 0 {
+			conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+		}
+		if _, err = conn.Execute("INSERT INTO master_info (addr,bin_name,bin_pos) VALUE (?,'',0)", addr); err != nil {
+			log.Error("insert master.info error(%v)", err)
+			return nil, errors.Trace(err)
+		}
+	} else {
+		m.addr, _ = r.GetStringByName(0, "addr")
+		m.binName, _ = r.GetStringByName(0, "bin_name")
+		bpos, _ := r.GetIntByName(0, "bin_pos")
+		m.binPos = uint32(bpos)
+	}
+	return m, nil
+}
+
+func (m *dbMasterInfo) Save(pos mysql.Position, force bool) error {
+	n := time.Now()
+	if !force && n.Sub(m.lastSaveTime) < 2*time.Second {
+		return nil
+	}
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("db master info client error(%v)", err)
+		return errors.Trace(err)
+	}
+	defer conn.Close()
+
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	if _, err = conn.Execute("UPDATE master_info SET bin_name=?,bin_pos=? WHERE addr=?", pos.Name, pos.Pos, m.addr); err != nil {
+		log.Error("db save master info error(%v)", err)
+		return errors.Trace(err)
+	}
+	m.lastSaveTime = n
+	return nil
+}
+
+func (m *dbMasterInfo) Pos() mysql.Position {
+	var pos mysql.Position
+	m.l.RLock()
+	pos.Name = m.binName
+	pos.Pos = m.binPos
+	m.l.RUnlock()
+	return pos
+}
+
+type hbaseInfo struct {
+	c *conf.MasterInfoConfig
+
+	name string
+}
+
+func newHBaseInfo(name string, c *conf.MasterInfoConfig) (*hbaseInfo, error) {
+	m := &hbaseInfo{c: c, name: name}
+	return m, nil
+}
+
+func (m *hbaseInfo) LatestTs(table string) (lts uint64) {
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("db hbase info client error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	r, err := conn.Execute("SELECT latest_ts FROM hbase_info WHERE cluster_name=? AND table_name=?", m.name, table)
+	if err != nil {
+		log.Error("new db load hbase.info error(%v)", err)
+		return
+	}
+	if r.RowNumber() == 0 {
+		if m.c.Timeout > 0 {
+			conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+		}
+		if _, err = conn.Execute("INSERT INTO hbase_info (cluster_name,table_name,latest_ts) VALUE (?,?,0)", m.name, table); err != nil {
+			log.Error("insert hbase.info error(%v)", err)
+			return
+		}
+	} else {
+		lts, _ = r.GetUintByName(0, "latest_ts")
+	}
+	return 0
+}
+
+func (m *hbaseInfo) Save(table string, latestTs uint64) {
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("save hbase info client error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	if _, err = conn.Execute("UPDATE hbase_info SET latest_ts=? WHERE cluster_name=? AND table_name=?", latestTs, m.name, table); err != nil {
+		log.Error("save hbase info error(%v)", err)
+	}
+}

app/infra/canal/service/reader/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "offset.go",
+        "reader.go",
+    ],
+    importpath = "go-common/app/infra/canal/service/reader",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/log:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/service/reader/offset.go

@@ -0,0 +1,173 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reader
+
+import (
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+	pkgerr "github.com/pkg/errors"
+)
+
+// KafkaSeeker seeks offset in kafka topics by given condition
+type KafkaSeeker struct {
+	consumer sarama.Consumer
+	client   sarama.Client
+}
+
+// NewKafkaSeeker creates an instance of KafkaSeeker
+func NewKafkaSeeker(addr []string, config *sarama.Config) (*KafkaSeeker, error) {
+	client, err := sarama.NewClient(addr, config)
+	if err != nil {
+		return nil, pkgerr.WithStack(err)
+	}
+
+	consumer, err := sarama.NewConsumerFromClient(client)
+	if err != nil {
+		return nil, pkgerr.WithStack(err)
+	}
+
+	s := &KafkaSeeker{
+		client:   client,
+		consumer: consumer,
+	}
+
+	return s, nil
+}
+
+// Close releases resources of KafkaSeeker
+func (ks *KafkaSeeker) Close() {
+	ks.consumer.Close()
+	ks.client.Close()
+}
+
+// Seek seeks the first offset which binlog CommitTs bigger than ts
+func (ks *KafkaSeeker) Seek(topic string, ts int64, partitions []int32) (offsets []int64, err error) {
+	if len(partitions) == 0 {
+		partitions, err = ks.consumer.Partitions(topic)
+		if err != nil {
+			log.Error("tidb get partitions from topic %s error %v", topic, err)
+			return nil, pkgerr.WithStack(err)
+		}
+	}
+
+	offsets, err = ks.seekOffsets(topic, partitions, ts)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		log.Error("tidb seek offsets error %v", err)
+	}
+	return
+}
+
+func (ks *KafkaSeeker) getTSFromMSG(msg *sarama.ConsumerMessage) (ts int64, err error) {
+	binlog := new(pb.Binlog)
+	err = binlog.Unmarshal(msg.Value)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	return binlog.CommitTs, nil
+}
+
+// seekOffsets returns all valid offsets in partitions
+func (ks *KafkaSeeker) seekOffsets(topic string, partitions []int32, pos int64) ([]int64, error) {
+	offsets := make([]int64, len(partitions))
+	for _, partition := range partitions {
+		start, err := ks.client.GetOffset(topic, partition, sarama.OffsetOldest)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+
+		end, err := ks.client.GetOffset(topic, partition, sarama.OffsetNewest)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+
+		offset, err := ks.seekOffset(topic, partition, start, end-1, pos)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+		offsets[partition] = offset
+	}
+
+	return offsets, nil
+}
+
+func (ks *KafkaSeeker) seekOffset(topic string, partition int32, start int64, end int64, ts int64) (offset int64, err error) {
+	startTS, err := ks.getTSAtOffset(topic, partition, start)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	if ts < startTS {
+		log.Warn("given ts %v is smaller than oldest message's ts %v, some binlogs may lose", ts, startTS)
+		offset = start
+		return
+	} else if ts == startTS {
+		offset = start + 1
+		return
+	}
+
+	for start < end {
+		mid := (end-start)/2 + start
+		var midTS int64
+		midTS, err = ks.getTSAtOffset(topic, partition, mid)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return
+		}
+
+		if midTS <= ts {
+			start = mid + 1
+		} else {
+			end = mid
+		}
+	}
+
+	var endTS int64
+	endTS, err = ks.getTSAtOffset(topic, partition, end)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	if endTS <= ts {
+		return sarama.OffsetNewest, nil
+	}
+
+	return end, nil
+}
+
+func (ks *KafkaSeeker) getTSAtOffset(topic string, partition int32, offset int64) (ts int64, err error) {
+	pc, err := ks.consumer.ConsumePartition(topic, partition, offset)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+	defer pc.Close()
+
+	for msg := range pc.Messages() {
+		ts, err = ks.getTSFromMSG(msg)
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	panic("unreachable")
+}

app/infra/canal/service/reader/reader.go

@@ -0,0 +1,174 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reader
+
+import (
+	"fmt"
+
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+	pkgerr "github.com/pkg/errors"
+)
+
+func init() {
+	// log.SetLevel(log.LOG_LEVEL_NONE)
+	sarama.MaxResponseSize = 1 << 30
+}
+
+// Config for Reader
+type Config struct {
+	KafkaAddr []string
+	// the CommitTs of binlog return by reader will bigger than the config CommitTs
+	CommitTS  int64
+	Offset    int64 // start at kafka offset
+	ClusterID string
+	Name      string
+}
+
+// Message read from reader
+type Message struct {
+	Binlog *pb.Binlog
+	Offset int64 // kafka offset
+}
+
+// Reader to read binlog from kafka
+type Reader struct {
+	cfg    *Config
+	client sarama.Client
+
+	msgs      chan *Message
+	stop      chan struct{}
+	clusterID string
+}
+
+func (r *Reader) getTopic() (string, int32) {
+	return r.cfg.ClusterID + "_obinlog", 0
+}
+
+func (r *Reader) name() string {
+	return fmt.Sprintf("%s-%s", r.cfg.Name, r.cfg.ClusterID)
+}
+
+// NewReader creates an instance of Reader
+func NewReader(cfg *Config) (r *Reader, err error) {
+	r = &Reader{
+		cfg:       cfg,
+		stop:      make(chan struct{}),
+		msgs:      make(chan *Message, 1024),
+		clusterID: cfg.ClusterID,
+	}
+
+	r.client, err = sarama.NewClient(r.cfg.KafkaAddr, nil)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		r = nil
+		return
+	}
+	if (r.cfg.Offset == 0) && (r.cfg.CommitTS > 0) {
+		r.cfg.Offset, err = r.getOffsetByTS(r.cfg.CommitTS)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			r = nil
+			return
+		}
+		log.Info("tidb %s: set offset to: %v", r.name(), r.cfg.Offset)
+	}
+	return
+}
+
+// Close shuts down the reader
+func (r *Reader) Close() {
+	close(r.stop)
+
+	r.client.Close()
+}
+
+// Messages returns a chan that contains unread buffered message
+func (r *Reader) Messages() (msgs <-chan *Message) {
+	return r.msgs
+}
+
+func (r *Reader) getOffsetByTS(ts int64) (offset int64, err error) {
+	seeker, err := NewKafkaSeeker(r.cfg.KafkaAddr, nil)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	topic, partition := r.getTopic()
+	offsets, err := seeker.Seek(topic, ts, []int32{partition})
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	offset = offsets[0]
+
+	return
+}
+
+// Run start consume msg
+func (r *Reader) Run() {
+	offset := r.cfg.Offset
+	log.Info("tidb %s start at offset: %v", r.name(), offset)
+
+	consumer, err := sarama.NewConsumerFromClient(r.client)
+	if err != nil {
+		log.Error("tidb %s NewConsumerFromClient err: %v", r.name(), err)
+		return
+	}
+	defer consumer.Close()
+	topic, partition := r.getTopic()
+	partitionConsumer, err := consumer.ConsumePartition(topic, partition, offset)
+	if err != nil {
+		log.Error("tidb %s ConsumePartition err: %v", r.name(), err)
+		return
+	}
+	defer partitionConsumer.Close()
+	for {
+		select {
+		case <-r.stop:
+			partitionConsumer.Close()
+			close(r.msgs)
+			log.Info("tidb %s reader stop to run", r.name())
+			return
+		case kmsg, ok := <-partitionConsumer.Messages():
+			if !ok {
+				close(r.msgs)
+				log.Info("tidb %s reader stop to run because partitionConsumer close", r.name())
+				return
+			}
+			if kmsg == nil {
+				continue
+			}
+			log.Info("tidb %s get kmsg offset: %v", r.name(), kmsg.Offset)
+			binlog := new(pb.Binlog)
+			err := binlog.Unmarshal(kmsg.Value)
+			if err != nil {
+				log.Warn("%s unmarshal err %+v", r.name(), err)
+				continue
+			}
+			if r.cfg.CommitTS > 0 && binlog.CommitTs <= r.cfg.CommitTS {
+				log.Warn("%s skip binlog CommitTs: ", r.name(), binlog.CommitTs)
+				continue
+			}
+			r.msgs <- &Message{
+				Binlog: binlog,
+				Offset: kmsg.Offset,
+			}
+		}
+	}
+}

app/infra/canal/service/target.go

@@ -0,0 +1,329 @@
+package service
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"hash/crc32"
+	"strings"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	"go-common/app/infra/canal/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	"github.com/pkg/errors"
+	"github.com/siddontang/go-mysql/canal"
+)
+
+var (
+	errInvalidAction = errors.New("invalid rows action")
+	errInvalidUpdate = errors.New("invalid update rows event")
+	errBinlogFormat  = errors.New("binlog format failed")
+)
+
+type producer interface {
+	Rows(int64)
+	Send(context.Context, string, interface{}) error
+	Close()
+	Name() string
+}
+
+type databusP struct {
+	group, topic string
+	*databus.Databus
+}
+
+func (d *databusP) Rows(b int64) {
+	// ignore
+}
+
+func (d *databusP) Send(c context.Context, key string, data interface{}) error {
+	return d.Databus.Send(c, key, data)
+}
+
+func (d *databusP) Name() string {
+	return fmt.Sprintf("databus:group(%s)topic(%s)", d.group, d.topic)
+}
+
+func (d *databusP) Close() {
+	d.Databus.Close()
+}
+
+// infocP infoc producer
+type infocP struct {
+	taskID string
+	*infoc.Infoc
+}
+
+// Rows rows
+func (i *infocP) Rows(b int64) {
+	i.Infoc.Rows(b)
+}
+
+// Send send msg
+func (i *infocP) Send(c context.Context, key string, data interface{}) error {
+	return i.Infoc.Send(c, key, data)
+}
+
+// Name infoc name
+func (i *infocP) Name() string {
+	return fmt.Sprintf("infoc(%s)", i.taskID)
+}
+
+// Close close infoc
+func (i *infocP) Close() {
+	i.Infoc.Flush()
+	i.Infoc.Close()
+}
+
+// Target databus target
+type Target struct {
+	producers []producer
+	eventLen  uint32
+	events    []chan *canal.RowsEvent
+	db        *conf.Database
+
+	closed bool
+}
+
+// NewTarget new databus target
+func NewTarget(db *conf.Database) (t *Target) {
+	t = &Target{
+		db:       db,
+		eventLen: uint32(len(db.CTables)),
+	}
+	t.events = make([]chan *canal.RowsEvent, t.eventLen)
+	if db.Databus != nil {
+		t.producers = append(t.producers, &databusP{group: db.Databus.Group, topic: db.Databus.Topic, Databus: databus.New(db.Databus)})
+	}
+	if db.Infoc != nil {
+		t.producers = append(t.producers, &infocP{taskID: db.Infoc.TaskID, Infoc: infoc.New(db.Infoc)})
+	}
+	for i := 0; i < int(t.eventLen); i++ {
+		ch := make(chan *canal.RowsEvent, 1024)
+		t.events[i] = ch
+		go t.proc(ch)
+	}
+	return
+}
+
+// compare check if the binlog event is needed
+// check the table name and schame
+func (t *Target) compare(schame, table, action string) bool {
+	if t.db.Schema == schame {
+		for _, ctb := range t.db.CTables {
+			for _, tb := range ctb.Tables {
+				if table == tb {
+					for _, act := range ctb.OmitAction {
+						if act == action { // NOTE: omit action
+							return false
+						}
+					}
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// send send rows event into event chans
+// and hash by table%concurrency.
+func (t *Target) send(ev *canal.RowsEvent) {
+	yu := crc32.ChecksumIEEE([]byte(ev.Table.Name))
+	t.events[yu%t.eventLen] <- ev
+}
+
+func (t *Target) close() {
+	for _, p := range t.producers {
+		p.Close()
+	}
+	t.closed = true
+}
+
+// proc aync method for transfer the binlog data
+// when connection is bad, just refresh it with retry
+func (t *Target) proc(ch chan *canal.RowsEvent) {
+	type pData struct {
+		datas    []*model.Data
+		producer producer
+	}
+	var (
+		err         error
+		normalDatas []*pData
+		errorDatas  []*pData
+		ev          *canal.RowsEvent
+	)
+	for {
+		if t.closed {
+			return
+		}
+		if len(errorDatas) != 0 {
+			normalDatas = errorDatas
+			errorDatas = errorDatas[0:0]
+			time.Sleep(time.Second)
+		} else {
+			ev = <-ch
+			var datas []*model.Data
+			if datas, err = makeDatas(ev, t.db.TableMap); err != nil {
+				log.Error("makeData(%v) error(%v)", ev, err)
+				continue
+			}
+			normalDatas = normalDatas[0:0]
+			for _, p := range t.producers {
+				p.Rows(int64(len(datas)))
+				normalDatas = append(normalDatas, &pData{datas: datas, producer: p})
+				if stats != nil {
+					stats.Incr("send_counter", p.Name(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+				}
+			}
+		}
+		for _, pd := range normalDatas {
+			var eDatas []*model.Data
+			for _, data := range pd.datas {
+				if err = pd.producer.Send(context.TODO(), data.Key, data); err != nil {
+					// retry pub error data
+					eDatas = append(eDatas, data)
+					continue
+				}
+				log.Info("%s pub(key:%s, value:%+v) succeed", pd.producer.Name(), data.Key, data)
+			}
+			if len(eDatas) > 0 {
+				errorDatas = append(errorDatas, &pData{datas: eDatas, producer: pd.producer})
+				if stats != nil && ev != nil {
+					stats.Incr("retry_counter", pd.producer.Name(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+				}
+				log.Error("%s scheme(%s) pub fail,add to retry", pd.producer.Name(), ev.Table.Schema)
+			}
+		}
+	}
+}
+
+// makeDatas parse the binlog event and return the model.Data struct
+// a little bit cautious about the binlog type
+// if the type is update:
+//     the old value and new value will alternate appearing in the event.Rows
+func makeDatas(e *canal.RowsEvent, tbMap map[string]*conf.Addition) (datas []*model.Data, err error) {
+	var (
+		rowsLen     = len(e.Rows)
+		firstRowLen = len(e.Rows[0])
+		lenCol      = len(e.Table.Columns)
+	)
+	if rowsLen == 0 || firstRowLen == 0 || firstRowLen != lenCol {
+		log.Error("rows length(%d) first row length(%d) columns length(%d)", rowsLen, firstRowLen, lenCol)
+		err = errBinlogFormat
+		return
+	}
+	datas = make([]*model.Data, 0, rowsLen)
+	switch e.Action {
+	case canal.InsertAction, canal.DeleteAction:
+		for _, values := range e.Rows {
+			var keys []string
+			data := &model.Data{
+				Action: e.Action,
+				Table:  e.Table.Name,
+				// the first primary key as the kafka key
+				Key: fmt.Sprint(values[0]),
+				New: make(map[string]interface{}, lenCol),
+			}
+			for i, c := range e.Table.Columns {
+				if c.IsUnsigned {
+					values[i] = unsignIntCase(values[i])
+				}
+				if strings.Contains(c.RawType, "binary") {
+					if bs, ok := values[i].(string); ok {
+						values[i] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+				}
+				data.New[c.Name] = values[i]
+			}
+			// set kafka key and remove omit columns data
+			addition, ok := tbMap[e.Table.Name]
+			if ok {
+				for _, omit := range addition.OmitField {
+					delete(data.New, omit)
+				}
+				for _, primary := range addition.PrimaryKey {
+					if _, ok := data.New[primary]; ok {
+						keys = append(keys, fmt.Sprint(data.New[primary]))
+					}
+				}
+			}
+			if len(keys) != 0 {
+				data.Key = strings.Join(keys, ",")
+			}
+			datas = append(datas, data)
+		}
+	case canal.UpdateAction:
+		if rowsLen%2 != 0 {
+			err = errInvalidUpdate
+			return
+		}
+		for i := 0; i < rowsLen; i += 2 {
+			var keys []string
+			data := &model.Data{
+				Action: e.Action,
+				Table:  e.Table.Name,
+				// the first primary key as the kafka key
+				Key: fmt.Sprint(e.Rows[i][0]),
+				Old: make(map[string]interface{}, lenCol),
+				New: make(map[string]interface{}, lenCol),
+			}
+			for j, c := range e.Table.Columns {
+				if c.IsUnsigned {
+					e.Rows[i][j] = unsignIntCase(e.Rows[i][j])
+					e.Rows[i+1][j] = unsignIntCase(e.Rows[i+1][j])
+				}
+				if strings.Contains(c.RawType, "binary") {
+					if bs, ok := e.Rows[i][j].(string); ok {
+						e.Rows[i][j] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+					if bs, ok := e.Rows[i+1][j].(string); ok {
+						e.Rows[i+1][j] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+				}
+				data.Old[c.Name] = e.Rows[i][j]
+				data.New[c.Name] = e.Rows[i+1][j]
+			}
+			// set kafka key and remove omit columns data
+			addition, ok := tbMap[e.Table.Name]
+			if ok {
+				for _, omit := range addition.OmitField {
+					delete(data.New, omit)
+					delete(data.Old, omit)
+				}
+				for _, primary := range addition.PrimaryKey {
+					if _, ok := data.New[primary]; ok {
+						keys = append(keys, fmt.Sprint(data.New[primary]))
+					}
+				}
+			}
+			if len(keys) != 0 {
+				data.Key = strings.Join(keys, ",")
+			}
+			datas = append(datas, data)
+		}
+	default:
+		err = errInvalidAction
+	}
+	return
+}
+
+func unsignIntCase(i interface{}) (v interface{}) {
+	switch si := i.(type) {
+	case int8:
+		v = uint8(si)
+	case int16:
+		v = uint16(si)
+	case int32:
+		v = uint32(si)
+	case int64:
+		v = uint64(si)
+	default:
+		v = i
+	}
+	return
+}

app/infra/canal/service/tidb_check.go

@@ -0,0 +1,65 @@
+package service
+
+import (
+	"regexp"
+
+	"go-common/library/log"
+)
+
+func (ins *tidbInstance) check() (err error) {
+	for _, db := range ins.config.Databases {
+		for _, ctable := range db.CTables {
+			if _, err = regexp.Compile(ctable.Name); err != nil {
+				log.Error("regexp.Compile(%s) error(%v)", ctable.Name, err)
+				return
+			}
+		}
+	}
+	return
+}
+
+func (ins *tidbInstance) getTable(dbName, table string) *Table {
+	if ins.ignoreTables[dbName] != nil && ins.ignoreTables[dbName][table] {
+		return nil
+	}
+	if ins.tables[dbName] != nil && ins.tables[dbName][table] != nil {
+		return ins.tables[dbName][table]
+	}
+	var regex *regexp.Regexp
+	for _, db := range ins.config.Databases {
+		if db.Schema != dbName {
+			continue
+		}
+		for _, ctable := range db.CTables {
+			regex, _ = regexp.Compile(ctable.Name)
+			if !regex.MatchString(table) {
+				continue
+			}
+			if ins.tables[dbName] == nil {
+				ins.tables[dbName] = make(map[string]*Table)
+			}
+			t := &Table{
+				PrimaryKey: ctable.PrimaryKey,
+				OmitField:  make(map[string]bool),
+				OmitAction: make(map[string]bool),
+				name:       ctable.Name,
+				ch:         make(chan *msg, 1024),
+			}
+			for _, action := range ctable.OmitAction {
+				t.OmitAction[action] = true
+			}
+			for _, field := range ctable.OmitField {
+				t.OmitField[field] = true
+			}
+			ins.waitTable.Add(1)
+			go ins.proc(t.ch)
+			ins.tables[dbName][table] = t
+			return t
+		}
+	}
+	if ins.ignoreTables[dbName] == nil {
+		ins.ignoreTables[dbName] = make(map[string]bool)
+	}
+	ins.ignoreTables[dbName][table] = true
+	return nil
+}

app/infra/canal/service/tidb_data.go

@@ -0,0 +1,123 @@
+package service
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"go-common/app/infra/canal/model"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+// lower case column field type in mysql
+// https://dev.mysql.com/doc/refman/8.0/en/data-types.html
+// for numeric type: int bigint smallint tinyint float double decimal bit
+// for string type: text longtext mediumtext char tinytext varchar
+// blob longblog mediumblog binary tinyblob varbinary
+// enum set
+// for json type: json
+
+// for text and char type, string_value is set
+// for blob and binary type, bytes_value is set
+// for enum, set, uint64_value is set
+// for json, bytes_value is set
+
+func tidbMakeData(m *msg) (data *model.Data, err error) {
+	action := m.mu.GetType()
+	if (action != pb.MutationType_Insert) && (action != pb.MutationType_Delete) && (action != pb.MutationType_Update) {
+		err = errInvalidAction
+		return
+	}
+	data = &model.Data{
+		Action: strings.ToLower(action.String()),
+		Table:  m.table,
+	}
+	var keys []string
+	switch action {
+	case pb.MutationType_Insert, pb.MutationType_Delete:
+		var values = m.mu.GetRow().GetColumns()
+		for i, c := range m.columns {
+			for _, key := range m.keys {
+				if c.Name == key {
+					keys = append(keys, columnToString(values[i]))
+					break
+				}
+			}
+			if m.ignore[c.Name] {
+				continue
+			}
+			if data.New == nil {
+				data.New = make(map[string]interface{}, len(m.columns))
+			}
+			if strings.Contains(c.GetMysqlType(), "binary") {
+				data.New[c.Name] = base64.StdEncoding.EncodeToString(values[i].GetBytesValue())
+				continue
+			}
+			data.New[c.Name] = columnToValue(values[i])
+		}
+	case pb.MutationType_Update:
+		if m.mu.Row == nil || m.mu.ChangeRow == nil {
+			err = errInvalidUpdate
+			return
+		}
+		var oldValues = m.mu.GetChangeRow().GetColumns()
+		var newValues = m.mu.GetRow().GetColumns()
+		for i, c := range m.columns {
+			for _, key := range m.keys {
+				if c.Name == key {
+					keys = append(keys, columnToString(newValues[i]))
+					break
+				}
+			}
+			if m.ignore[c.Name] {
+				continue
+			}
+			if data.New == nil {
+				data.New = make(map[string]interface{}, len(m.columns))
+			}
+			if data.Old == nil {
+				data.Old = make(map[string]interface{}, len(m.columns))
+			}
+			if strings.Contains(c.GetMysqlType(), "binary") {
+				data.Old[c.Name] = base64.StdEncoding.EncodeToString(oldValues[i].GetBytesValue())
+				data.New[c.Name] = base64.StdEncoding.EncodeToString(newValues[i].GetBytesValue())
+				continue
+			}
+			data.Old[c.Name] = columnToValue(oldValues[i])
+			data.New[c.Name] = columnToValue(newValues[i])
+		}
+	}
+	if len(keys) == 0 {
+		data.Key = columnToString(m.mu.GetRow().GetColumns()[0])
+	} else {
+		data.Key = strings.Join(keys, ",")
+	}
+	if data.New == nil && data.Old == nil {
+		data = nil
+	}
+	return
+}
+
+func columnToValue(c *pb.Column) interface{} {
+	if c.GetIsNull() {
+		return nil
+	}
+	if c.Int64Value != nil {
+		return c.GetInt64Value()
+	}
+	if c.Uint64Value != nil {
+		return c.GetUint64Value()
+	}
+	if c.DoubleValue != nil {
+		return c.GetDoubleValue()
+	}
+	if c.StringValue != nil {
+		return c.GetStringValue()
+	}
+	return c.GetBytesValue()
+}
+
+func columnToString(c *pb.Column) string {
+	return fmt.Sprint(columnToValue(c))
+}

app/infra/canal/service/tidb_data_test.go

@@ -0,0 +1,188 @@
+package service
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"go-common/app/infra/canal/model"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+func Test_tidbMakeData(t *testing.T) {
+	insertMsg, insertData := prepareInsertData()
+	delMsg, delData := prepareDeleteData()
+	updateMsg, updateData := prepareUpdateData()
+	updateMsg2, updateData2 := prepareUpdateData2()
+	type args struct {
+		m *msg
+	}
+	tests := []struct {
+		name     string
+		args     args
+		wantData *model.Data
+		wantErr  bool
+	}{
+		{name: "insert", args: args{m: insertMsg}, wantData: insertData, wantErr: false},
+		{name: "delete", args: args{m: delMsg}, wantData: delData, wantErr: false},
+		{name: "update", args: args{m: updateMsg}, wantData: updateData, wantErr: false},
+		{name: "update2", args: args{m: updateMsg2}, wantData: updateData2, wantErr: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotData, err := tidbMakeData(tt.args.m)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("tidbMakeData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			gotjson, _ := json.Marshal(gotData)
+			wantjson, _ := json.Marshal(tt.wantData)
+			if !reflect.DeepEqual(gotjson, wantjson) {
+				t.Errorf("tidbMakeData() = %v, want %v", gotData, tt.wantData)
+			}
+		})
+	}
+}
+
+func prepareInsertData() (*msg, *model.Data) {
+	insertPb := &pb.Binlog{}
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846216359608325,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":0,"row":{"columns":[{"uint64_value":1},{"string_value":"2018-10-26 18:50:57"},{"string_value":"2018-10-26 18:50:57"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":1},{"uint64_value":8167601},{"uint64_value":1}]}}]}]}}`), insertPb)
+	insertMsg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          insertPb.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"id", "mid"},
+		columns:     insertPb.DmlData.Tables[0].ColumnInfo,
+	}
+	insertData := &model.Data{
+		Action: "insert",
+		Table:  "counts",
+		Key:    "1,8167601",
+		New: map[string]interface{}{
+			"id":          1,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  1,
+			"mid":         8167601,
+			"type":        1,
+			"mtime":       "2018-10-26 18:50:57",
+		},
+	}
+	return insertMsg, insertData
+}
+
+func prepareDeleteData() (*msg, *model.Data) {
+	pbData := &pb.Binlog{}
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846189135953921,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":2,"row":{"columns":[{"uint64_value":7},{"string_value":"2018-01-11 12:19:10"},{"string_value":"2018-01-11 12:19:10"},{"uint64_value":2},{"uint64_value":0},{"uint64_value":897},{"uint64_value":27515233},{"uint64_value":1}]}}]}]}}`), pbData)
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          pbData.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"message_id"},
+		columns:     pbData.DmlData.Tables[0].ColumnInfo,
+	}
+	data := &model.Data{
+		Action: "delete",
+		Table:  "counts",
+		Key:    "897",
+		New: map[string]interface{}{
+			"id":          7,
+			"business_id": 2,
+			"origin_id":   0,
+			"message_id":  897,
+			"mid":         27515233,
+			"type":        1,
+			"mtime":       "2018-01-11 12:19:10",
+		},
+	}
+	return msg, data
+}
+
+func prepareUpdateData() (*msg, *model.Data) {
+	pbData := &pb.Binlog{}
+	// update likes type from 1 to 0
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846165844459523,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":1,"row":{"columns":[{"uint64_value":4},{"string_value":"2018-10-26 18:47:44"},{"string_value":"2017-12-22 15:05:29"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":46997},{"uint64_value":88895031},{"uint64_value":0}]},"change_row":{"columns":[{"uint64_value":4},{"string_value":"2017-12-22 15:55:52"},{"string_value":"2017-12-22 15:05:29"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":46997},{"uint64_value":88895031},{"uint64_value":1}]}}]}]}}`), pbData)
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          pbData.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"mid"},
+		columns:     pbData.DmlData.Tables[0].ColumnInfo,
+	}
+	data := &model.Data{
+		Action: "update",
+		Table:  "counts",
+		Key:    "88895031",
+		Old: map[string]interface{}{
+			"id":          4,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  46997,
+			"mid":         88895031,
+			"type":        1,
+			"mtime":       "2017-12-22 15:55:52",
+		},
+		New: map[string]interface{}{
+			"id":          4,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  46997,
+			"mid":         88895031,
+			"type":        0,
+			"mtime":       "2018-10-26 18:47:44",
+		},
+	}
+	return msg, data
+}
+
+func prepareUpdateData2() (*msg, *model.Data) {
+	muJson := `{"type":1,"row":{"columns":[{"uint64_value":0},{"string_value":"2018-11-03 17:07:44"},{"string_value":"2018-11-03 14:55:38"},{"uint64_value":3},{"uint64_value":0},{"uint64_value":88889},{"uint64_value":3},{"uint64_value":0},{"int64_value":0},{"int64_value":0},{"uint64_value":8167601}]},"change_row":{"columns":[{"uint64_value":0},{"string_value":"2018-11-03 16:36:39"},{"string_value":"2018-11-03 14:55:38"},{"uint64_value":3},{"uint64_value":0},{"uint64_value":88889},{"uint64_value":2},{"uint64_value":0},{"int64_value":0},{"int64_value":0},{"uint64_value":8167601}]}}`
+	columnJson := `[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"likes_count","mysql_type":"int","is_primary_key":false},{"name":"dislikes_count","mysql_type":"int","is_primary_key":false},{"name":"likes_change","mysql_type":"bigint","is_primary_key":false},{"name":"dislikes_change","mysql_type":"bigint","is_primary_key":false},{"name":"up_mid","mysql_type":"int","is_primary_key":false}]`
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		keys:        []string{"message_id"},
+	}
+	json.Unmarshal([]byte(columnJson), &msg.columns)
+	json.Unmarshal([]byte(muJson), &msg.mu)
+	data := &model.Data{
+		Action: "update",
+		Table:  "counts",
+		Key:    "88889",
+		Old: map[string]interface{}{
+			"ctime":           "2018-11-03 14:55:38",
+			"origin_id":       0,
+			"dislikes_count":  0,
+			"up_mid":          8167601,
+			"id":              0,
+			"mtime":           "2018-11-03 16:36:39",
+			"likes_count":     2,
+			"likes_change":    0,
+			"dislikes_change": 0,
+			"business_id":     3,
+			"message_id":      88889,
+		},
+		New: map[string]interface{}{
+			"likes_count":     3,
+			"dislikes_count":  0,
+			"likes_change":    0,
+			"id":              0,
+			"mtime":           "2018-11-03 17:07:44",
+			"ctime":           "2018-11-03 14:55:38",
+			"origin_id":       0,
+			"message_id":      88889,
+			"business_id":     3,
+			"dislikes_change": 0,
+			"up_mid":          8167601,
+		},
+	}
+	return msg, data
+}

app/infra/canal/service/tidb_instance.go

@@ -0,0 +1,186 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	"go-common/app/infra/canal/model"
+	"go-common/app/infra/canal/service/reader"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+type tidbInstance struct {
+	canal  *Canal
+	config *conf.TiDBInsConf
+	// one instance can have lots of target different by schema and table
+	targets      map[string][]producer
+	latestOffset int64
+	// scan latest timestamp, be used check delay
+	latestTimestamp int64
+	reader          *reader.Reader
+	err             error
+	closed          bool
+	tables          map[string]map[string]*Table
+	ignoreTables    map[string]map[string]bool
+	waitConsume     sync.WaitGroup
+	waitTable       sync.WaitGroup
+}
+
+// Table db table
+type Table struct {
+	PrimaryKey []string        // kafka msg key
+	OmitField  map[string]bool // field will be ignored in table
+	OmitAction map[string]bool // action will be ignored in table
+	name       string
+	ch         chan *msg
+}
+
+type msg struct {
+	db          string
+	table       string
+	tableRegexp string
+	mu          *pb.TableMutation
+	ignore      map[string]bool
+	keys        []string
+	columns     []*pb.ColumnInfo
+}
+
+// newTiDBInstance new canal instance
+func newTiDBInstance(cl *Canal, c *conf.TiDBInsConf) (ins *tidbInstance, err error) {
+	// new instance
+	ins = &tidbInstance{
+		config:       c,
+		canal:        cl,
+		targets:      make(map[string][]producer, len(c.Databases)),
+		tables:       make(map[string]map[string]*Table),
+		ignoreTables: make(map[string]map[string]bool),
+	}
+	cfg := &reader.Config{
+		Name:      c.Name,
+		KafkaAddr: c.Addrs,
+		Offset:    c.Offset,
+		CommitTS:  c.CommitTS,
+		ClusterID: c.ClusterID,
+	}
+	if err = ins.check(); err != nil {
+		return
+	}
+	position, err := cl.dao.TiDBPosition(context.Background(), c.Name)
+	if err == nil && position != nil {
+		cfg.Offset = position.Offset
+		cfg.CommitTS = position.CommitTS
+	}
+	ins.latestOffset = cfg.Offset
+	ins.latestTimestamp = cfg.CommitTS
+	for _, db := range c.Databases {
+		if db.Databus != nil {
+			if ins.targets == nil {
+				ins.targets = make(map[string][]producer)
+			}
+			ins.targets[db.Schema] = append(ins.targets[db.Schema], &databusP{group: db.Databus.Group, topic: db.Databus.Topic, Databus: databus.New(db.Databus)})
+		}
+		if db.Infoc != nil {
+			ins.targets[db.Schema] = append(ins.targets[db.Schema], &infocP{taskID: db.Infoc.TaskID, Infoc: infoc.New(db.Infoc)})
+		}
+	}
+	ins.reader, ins.err = reader.NewReader(cfg)
+	return ins, ins.err
+}
+
+// start start binlog receive
+func (ins *tidbInstance) start() {
+	defer ins.waitConsume.Done()
+	ins.waitConsume.Add(2)
+	go ins.reader.Run()
+	go ins.syncproc()
+	for msg := range ins.reader.Messages() {
+		ins.process(msg)
+	}
+}
+
+// close close instance
+func (ins *tidbInstance) close() {
+	if ins.closed {
+		return
+	}
+	ins.closed = true
+	ins.reader.Close()
+	ins.waitConsume.Wait()
+	for _, tables := range ins.tables {
+		for _, table := range tables {
+			close(table.ch)
+		}
+	}
+	ins.waitTable.Wait()
+	ins.sync()
+}
+
+// String .
+func (ins *tidbInstance) String() string {
+	return fmt.Sprintf("%s-%s", ins.config.Name, ins.config.ClusterID)
+}
+
+func (ins *tidbInstance) process(m *reader.Message) (err error) {
+	if m.Binlog.Type == pb.BinlogType_DDL {
+		log.Info("tidb %s got ddl: %s", ins.String(), m.Binlog.DdlData.String())
+		return
+	}
+	for _, table := range m.Binlog.DmlData.Tables {
+		tb := ins.getTable(table.GetSchemaName(), table.GetTableName())
+		if tb == nil {
+			continue
+		}
+		for _, mu := range table.Mutations {
+			action := strings.ToLower(mu.GetType().String())
+			if tb.OmitAction[action] {
+				continue
+			}
+			tb.ch <- &msg{
+				db:          table.GetSchemaName(),
+				table:       table.GetTableName(),
+				mu:          mu,
+				ignore:      tb.OmitField,
+				keys:        tb.PrimaryKey,
+				columns:     table.ColumnInfo,
+				tableRegexp: tb.name,
+			}
+			if stats != nil {
+				stats.Incr("syncer_counter", ins.String(), table.GetSchemaName(), tb.name, action)
+				stats.State("delay_syncer", ins.delay(), ins.String(), tb.name, "", "")
+			}
+		}
+	}
+	ins.latestOffset = m.Offset
+	ins.latestTimestamp = m.Binlog.CommitTs
+	return nil
+}
+
+// Error returns instance error.
+func (ins *tidbInstance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *tidbInstance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}
+
+func (ins *tidbInstance) sync() {
+	info := &model.TiDBInfo{
+		Name:      ins.config.Name,
+		ClusterID: ins.config.ClusterID,
+		Offset:    ins.latestOffset,
+		CommitTS:  ins.latestTimestamp,
+	}
+	ins.canal.dao.UpdateTiDBPosition(context.Background(), info)
+}

app/infra/canal/service/tidb_proc.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+)
+
+func (ins *tidbInstance) proc(ch chan *msg) {
+	defer ins.waitTable.Done()
+	for {
+		msg, ok := <-ch
+		if !ok {
+			return
+		}
+		data, err := tidbMakeData(msg)
+		if err != nil {
+			log.Error("tidb MakeData(%+v) err: %+v", msg, err)
+			continue
+		}
+		if data == nil {
+			continue
+		}
+		for _, target := range ins.targets[msg.db] {
+			for {
+				if err = target.Send(context.TODO(), data.Key, data); err == nil {
+					stats.Incr("send_counter", target.Name(), msg.db, msg.tableRegexp, data.Action)
+					break
+				}
+				stats.Incr("retry_counter", target.Name(), msg.db, msg.tableRegexp, data.Action)
+				log.Error("tidb %s scheme(%s) pub fail,add to retry", target.Name(), msg.db)
+				time.Sleep(time.Second)
+			}
+			log.Info("tidb %s pub(key:%s, value:%+v) succeed", target.Name(), data.Key, data)
+		}
+	}
+}
+
+func (ins *tidbInstance) syncproc() {
+	defer ins.waitConsume.Done()
+	for {
+		if ins.closed {
+			return
+		}
+		time.Sleep(time.Second * 5)
+		ins.sync()
+	}
+}
+
+func (c *Canal) tidbEventproc() {
+	ech := conf.TiDBEvent()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		c.tidbInsl.Lock()
+		if old, ok := c.tidbInstances[insc.Name]; ok {
+			old.close()
+		}
+		c.tidbInsl.Unlock()
+		ins, err := newTiDBInstance(c, insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.sendWx(fmt.Sprintf("reload tidb canal instance(%s) failed error(%v)", ins.String(), err))
+			continue
+		}
+		c.tidbInsl.Lock()
+		c.tidbInstances[insc.Name] = ins
+		c.tidbInsl.Unlock()
+		go ins.start()
+		log.Info("reload tidb canal instance(%s) success", ins.String())
+		c.sendWx(fmt.Sprintf("reload tidb canal instance(%s) success", ins.String()))
+	}
+}