Create & Init Project...

app/infra/canal/service/BUILD

@@ -0,0 +1,76 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal.go",
+        "hbase.go",
+        "instance.go",
+        "master.go",
+        "target.go",
+        "tidb_check.go",
+        "tidb_data.go",
+        "tidb_instance.go",
+        "tidb_proc.go",
+    ],
+    importpath = "go-common/app/infra/canal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/dao:go_default_library",
+        "//app/infra/canal/infoc:go_default_library",
+        "//app/infra/canal/model:go_default_library",
+        "//app/infra/canal/service/reader:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/netutil:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//vendor/github.com/juju/errors:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/canal:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/client:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/replication:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/infra/canal/service/reader:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["tidb_data_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/canal/model:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+    ],
+)

app/infra/canal/service/canal.go

@@ -0,0 +1,364 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/dao"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	xhttp "go-common/library/net/http/blademaster"
+	"go-common/library/net/netutil"
+	"go-common/library/stat/prom"
+
+	"github.com/siddontang/go-mysql/client"
+)
+
+var (
+	stats       = prom.New().WithState("go_canal_counter", []string{"type", "addr", "scheme", "table", "action"})
+	tblReplacer = regexp.MustCompile("[0-9]+") // NOTE: replace number of sub-table name to space
+)
+
+// Canal is canal.
+type Canal struct {
+	dao            *dao.Dao
+	instances      map[string]*Instance
+	hbaseInstances map[string]*HBaseInstance
+	tidbInstances  map[string]*tidbInstance
+	insl           sync.RWMutex
+	tidbInsl       sync.RWMutex
+	err            error
+	backoff        netutil.Backoff
+	errCount       int64
+	lastErr        time.Time
+}
+
+// NewCanal load config and start canal instance.
+func NewCanal(config *conf.Config) (c *Canal) {
+	c = &Canal{
+		dao: dao.New(config),
+	}
+	cfg, err := conf.LoadCanalConf()
+	if err != nil {
+		panic(err)
+	}
+	c.instances = make(map[string]*Instance, len(cfg.Instances))
+	c.hbaseInstances = make(map[string]*HBaseInstance, len(cfg.HBaseInstances))
+	c.tidbInstances = make(map[string]*tidbInstance, len(cfg.TiDBInstances))
+	for _, insc := range cfg.Instances {
+		ins, err := NewInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%+v)", err)
+		}
+		c.insl.Lock()
+		c.instances[insc.Addr] = ins
+		c.insl.Unlock()
+		if err == nil {
+			go ins.Start()
+			log.Info("start canal instance(%s) success", ins.String())
+		}
+	}
+	c.backoff = &netutil.BackoffConfig{
+		MaxDelay:  120 * time.Second,
+		BaseDelay: 1.0 * time.Second,
+		Factor:    1.6,
+		Jitter:    0.2,
+	}
+	c.lastErr = time.Now()
+	go c.eventproc()
+	// hbase
+	for _, insc := range cfg.HBaseInstances {
+		ins, err := NewHBaseInstance(insc)
+		if err != nil {
+			log.Error("new hbase instance error(%+v)", err)
+		}
+		c.insl.Lock()
+		c.hbaseInstances[insc.Cluster] = ins
+		c.insl.Unlock()
+		if err == nil {
+			ins.Start()
+			log.Info("start hbase instance(%s) success", ins.String())
+		}
+	}
+	go c.hbaseeventproc()
+
+	// tidb
+	for _, insc := range cfg.TiDBInstances {
+		ins, err := newTiDBInstance(c, insc)
+		if err != nil {
+			log.Error("new instance error(%+v)", err)
+			c.sendWx(fmt.Sprintf("new tidb canal instance(%s) failed error(%v)", ins.String(), err))
+			continue
+		}
+		c.tidbInsl.Lock()
+		c.tidbInstances[insc.Name] = ins
+		c.tidbInsl.Unlock()
+		if err == nil {
+			go ins.start()
+			log.Info("start tidb canal instance(%s) success", ins.String())
+		}
+	}
+	go c.tidbEventproc()
+	go c.monitorproc()
+	// errproc
+	go c.errproc()
+	return
+}
+
+// CheckMaster check master status.
+func (c *Canal) CheckMaster(addr, user, pwd string) (name string, pos int64, err error) {
+	conn, err := client.Connect(addr, user, pwd, "")
+	if err != nil {
+		return
+	}
+	rr, err := conn.Execute("SHOW MASTER STATUS")
+	if err != nil {
+		return
+	}
+	name, _ = rr.GetString(0, 0)
+	pos, _ = rr.GetInt(0, 1)
+	if name != "" && pos > 0 {
+		return
+	}
+	return "", 0, fmt.Errorf("check master no name|pos")
+}
+
+// PosSync sync newewst bin_pos.
+func (c *Canal) PosSync(addr string) (err error) {
+	c.insl.Lock()
+	old, ok := c.instances[addr]
+	c.insl.Unlock()
+	if !ok {
+		return
+	}
+	pos, err := old.GetMasterPos()
+	if err != nil {
+		return
+	}
+	old.Close()
+	old.OnPosSynced(pos, true)
+
+	ins, _ := NewInstance(old.config)
+	c.insl.Lock()
+	c.instances[addr] = ins
+	c.insl.Unlock()
+	ins.Start()
+	return
+}
+
+// Errors returns instance errors.
+func (c *Canal) Errors() (ies map[string]string) {
+	ies = map[string]string{}
+	c.insl.RLock()
+	for _, i := range c.instances {
+		ies[i.String()] = i.Error()
+	}
+	for _, i := range c.hbaseInstances {
+		ies[i.String()] = i.Error()
+	}
+	c.insl.RUnlock()
+	return
+}
+
+// Error returns canal error.
+func (c *Canal) Error() string {
+	if c.err == nil {
+		return ""
+	}
+	return c.err.Error()
+}
+
+// errproc check errors.
+func (c *Canal) errproc() {
+	for {
+		time.Sleep(10 * time.Second)
+		es := c.Error()
+		if es != "" {
+			c.sendWx(fmt.Sprintf("canal occur error(%s)", es))
+		}
+		ies := c.Errors()
+		for k, v := range ies {
+			if v != "" {
+				c.sendWx(fmt.Sprintf("canal instance(%s) occur error(%s)", k, v))
+			}
+		}
+	}
+}
+
+// Close close canal instance
+func (c *Canal) Close() {
+	c.insl.RLock()
+	defer c.insl.RUnlock()
+	for _, ins := range c.instances {
+		ins.Close()
+		log.Info("close canal instance(%s) success", ins.String())
+	}
+	for _, ins := range c.hbaseInstances {
+		ins.Close()
+		log.Info("close hbase instance(%s) success", ins.String())
+	}
+	for _, ins := range c.tidbInstances {
+		ins.close()
+		log.Info("close tidb instance(%s) success", ins.String())
+	}
+}
+
+func (c *Canal) eventproc() {
+	ech := conf.Event()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		ins, err := NewInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.sendWx(fmt.Sprintf("reload canal instance(%s) failed error(%v)", ins.String(), err))
+			c.insl.Lock()
+			if old, ok := c.instances[insc.Addr]; ok {
+				old.Close()
+			}
+			c.instances[insc.Addr] = ins
+			c.insl.Unlock()
+			continue
+		}
+		c.insl.Lock()
+		if old, ok := c.instances[insc.Addr]; ok {
+			old.Close()
+		}
+		c.instances[insc.Addr] = ins
+		c.insl.Unlock()
+		go ins.Start()
+		log.Info("reload canal instance(%s) success", ins.String())
+		//	c.sendWx(fmt.Sprintf("reload canal instance(%s) success", ins.String()))
+	}
+}
+
+func (c *Canal) hbaseeventproc() {
+	ech := conf.HBaseEvent()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		ins, err := NewHBaseInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.insl.Lock()
+			if old, ok := c.hbaseInstances[insc.Cluster]; ok {
+				old.Close()
+			}
+			c.hbaseInstances[insc.Cluster] = ins
+			c.insl.Unlock()
+			continue
+		}
+		c.insl.Lock()
+		if old, ok := c.hbaseInstances[insc.Cluster]; ok {
+			old.Close()
+		}
+		c.hbaseInstances[insc.Cluster] = ins
+		c.insl.Unlock()
+		ins.Start()
+		log.Info("reload hbase instance(%s) success", ins.String())
+	}
+}
+
+// monitorproc monitor instance delay.
+func (c *Canal) monitorproc() {
+	if env.DeployEnv != env.DeployEnvProd {
+		return
+	}
+	const delay = 2 * time.Minute
+	for {
+		time.Sleep(delay)
+		c.insl.RLock()
+		for _, ins := range c.instances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && !ins.config.MonitorOff && dt > threshold {
+				for _, db := range ins.config.Databases {
+					c.sendWx(fmt.Sprintf("canal env(%s) 数据库(%s)地址(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, db.Schema, ins.config.Addr, threshold, dt))
+				}
+			}
+		}
+		for _, ins := range c.hbaseInstances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && !ins.config.MonitorOff && dt > threshold {
+				c.sendWx(fmt.Sprintf("canal env(%s) hbase集群(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, ins.config.Cluster, threshold, dt))
+			}
+		}
+		c.insl.RUnlock()
+		c.tidbInsl.RLock()
+		for _, ins := range c.tidbInstances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && dt > threshold {
+				for _, db := range ins.config.Databases {
+					c.sendWx(fmt.Sprintf("tidb canal env(%s) 数据库(%s)地址(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, db.Schema, ins.config.Name, threshold, dt))
+				}
+			}
+		}
+		c.tidbInsl.RUnlock()
+	}
+}
+
+func (c *Canal) sendWx(msg string) {
+	count := atomic.LoadInt64(&c.errCount)
+	atomic.AddInt64(&c.errCount, 1)
+	duration := c.backoff.Backoff(int(count))
+	if time.Since(c.lastErr) < duration {
+		return
+	}
+	c.lastErr = time.Now()
+	sendWechat(msg, conf.Conf.Monitor.User)
+}
+
+func sendWechat(msg string, user string) {
+	params := url.Values{}
+	params.Set("Action", "CreateWechatMessage")
+	params.Set("PublicKey", "9c178e51a7d4dc8aa1dbef0c790b06e7574c4d0etracehubtuhui@bilibili.com")
+	params.Set("UserName", user)
+	params.Set("Content", msg)
+	params.Set("TreeId", "bilibili.main.common-arch.canal")
+	params.Set("Title", "canal 监控报警")
+	params.Set("Signature", "1")
+	req, _ := http.NewRequest("POST", "http://merak.bilibili.co", strings.NewReader(params.Encode()))
+	req.Header.Add("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
+	var v struct {
+		ReqID    string `json:"reqId"`
+		Status   int64  `json:"status"`
+		Response struct {
+			status int
+		} `json:"Response"`
+	}
+	if err := xhttp.NewClient(conf.Conf.HTTPClient).Do(context.TODO(), req, &v); err != nil {
+		log.Error("send wechat monitor status(%d) msg(%v,%v) error(%v)", v.Status, v.Response, v.Response.status, err)
+	}
+}

app/infra/canal/service/hbase.go

@@ -0,0 +1,153 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	hbase "go-common/library/database/hbase.v2"
+
+	"github.com/pkg/errors"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+// HBaseInstance hbase canal instance
+type HBaseInstance struct {
+	config *conf.HBaseInsConf
+	client *hbase.Client
+	// one instance can have lots of target different by schema and table
+	targets []*databus.Databus
+	hinfo   *hbaseInfo
+
+	// scan latest timestamp, be used check delay
+	latestTimestamp int64
+
+	err    error
+	closed bool
+}
+
+// NewHBaseInstance new canal instance
+func NewHBaseInstance(c *conf.HBaseInsConf) (ins *HBaseInstance, err error) {
+	// new instance
+	ins = &HBaseInstance{config: c}
+	// check and modify config
+	if c.MasterInfo == nil {
+		err = errors.New("no masterinfo config")
+		ins.err = err
+		return
+	}
+	// new hbase info
+	if ins.hinfo, err = newHBaseInfo(ins.config.Cluster, ins.config.MasterInfo); err != nil {
+		log.Error("init hbase info error(%v)", err)
+		ins.err = errors.Wrap(err, "init hbase info")
+		return
+	}
+	// new hbase client
+	ins.client = hbase.NewClient(&hbase.Config{Zookeeper: &hbase.ZKConfig{
+		Root:  c.Root,
+		Addrs: c.Addrs,
+	}})
+	return
+}
+
+// Start start binlog receive
+func (ins *HBaseInstance) Start() {
+	for _, db := range ins.config.Databases {
+		databus := databus.New(db.Databus)
+		ins.targets = append(ins.targets, databus)
+		for _, tb := range db.Tables {
+			go ins.scanproc(db, databus, tb)
+		}
+	}
+}
+
+func (ins *HBaseInstance) scanproc(c *conf.HBaseDatabase, databus *databus.Databus, table *conf.HBaseTable) {
+	latestTs := ins.hinfo.LatestTs(table.Name)
+	if latestTs == 0 {
+		latestTs = uint64(time.Now().UnixNano() / 1e6)
+	}
+AGAIN:
+	for {
+		time.Sleep(300 * time.Millisecond)
+		nowTs := uint64(time.Now().UnixNano() / 1e6)
+		// scan
+		scanner, err := ins.client.Scan(context.TODO(), []byte(table.Name), hrpc.TimeRangeUint64(latestTs, nowTs))
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		ins.latestTimestamp = time.Now().Unix()
+		for {
+			res, err := scanner.Next()
+			if err == io.EOF {
+				latestTs = nowTs
+				ins.hinfo.Save(table.Name, latestTs)
+				continue AGAIN
+			} else if err != nil {
+				time.Sleep(time.Second)
+				continue AGAIN
+			}
+			var key string
+			data := map[string]interface{}{}
+			for _, cell := range res.Cells {
+				row := string(cell.Row)
+				if key == "" {
+					key = row
+				}
+				if _, ok := data[row]; !ok {
+					data[row] = map[string]string{}
+				}
+				column := string(cell.Qualifier)
+				omit := false
+				for _, field := range table.OmitField {
+					if field == column {
+						omit = true
+						break
+					}
+				}
+				if !omit {
+					cm := data[row].(map[string]string)
+					cm[column] = string(cell.Value)
+				}
+			}
+			real := &model.Data{Table: table.Name, New: data}
+			databus.Send(context.TODO(), key, real)
+			log.Info("hbase databus:group(%s)topic(%s) pub(key:%s, value:%+v) succeed", c.Databus.Group, c.Databus.Topic, key, real)
+		}
+	}
+}
+
+// Close close instance
+func (ins *HBaseInstance) Close() {
+	if ins.err != nil {
+		return
+	}
+	ins.client.Close()
+	for _, t := range ins.targets {
+		t.Close()
+	}
+	ins.closed = true
+	ins.err = errors.New("canal hbase instance closed")
+}
+
+func (ins *HBaseInstance) String() string {
+	return ins.config.Cluster
+}
+
+// Error returns instance error.
+func (ins *HBaseInstance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *HBaseInstance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}

app/infra/canal/service/instance.go

@@ -0,0 +1,172 @@
+package service
+
+import (
+	"fmt"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+	"github.com/siddontang/go-mysql/canal"
+	"github.com/siddontang/go-mysql/mysql"
+	"github.com/siddontang/go-mysql/replication"
+)
+
+// Instance canal instance
+type Instance struct {
+	*canal.Canal
+	config *conf.InsConf
+	// one instance can have lots of target different by schema and table
+	targets []*Target
+	master  *dbMasterInfo
+
+	// binlog latest timestamp, be used check delay
+	latestTimestamp int64
+	err             error
+	closed          bool
+}
+
+// NewInstance new canal instance
+func NewInstance(c *conf.InsConf) (ins *Instance, err error) {
+	// new instance
+	ins = &Instance{config: c}
+	// check and modify config
+	if c.MasterInfo == nil {
+		err = errors.New("no masterinfo config")
+		ins.err = err
+		return
+	}
+	if c.ReadTimeout == 0 {
+		c.ReadTimeout = 90
+	}
+	c.ReadTimeout = c.ReadTimeout * time.Second
+	c.HeartbeatPeriod = c.HeartbeatPeriod * time.Second
+
+	if ins.master, err = newDBMasterInfo(ins.config.Addr, ins.config.MasterInfo); err != nil {
+		log.Error("init master info error(%v)", err)
+		ins.err = errors.Wrap(err, "init master info")
+		return
+	}
+	if ins.targets, err = newTargets(c); err != nil {
+		log.Error("db.init() error(%v)", err)
+		ins.err = errors.Wrap(err, "db init")
+		return
+	}
+	ins.latestTimestamp = time.Now().Unix()
+	// new canal
+	if ins.Canal, err = canal.NewCanal(c.Config); err != nil {
+		log.Error("canal.NewCanal(%v) error(%v)", c.Config, err)
+		ins.err = errors.Wrapf(err, "canal NewCanal(%v)", c.Config)
+		return
+	}
+	// implement self as canal's event handler
+	ins.Canal.SetEventHandler(ins)
+
+	return
+}
+
+func newTargets(c *conf.InsConf) (targets []*Target, err error) {
+	targets = make([]*Target, 0, len(c.Databases))
+	for _, db := range c.Databases {
+		if err = db.CheckTable(c.Addr, c.User, c.Password); err != nil {
+			log.Error("db.CheckTable() error(%v)", err)
+			return
+		}
+		targets = append(targets, NewTarget(db))
+	}
+	return
+}
+
+// Start start binlog receive
+func (ins *Instance) Start() {
+	pos := ins.master.Pos()
+	if pos.Name == "" || pos.Pos == 0 {
+		var err error
+		if pos, err = ins.Canal.GetMasterPos(); err != nil {
+			log.Error("c.MasterPos error(%v)", err)
+			ins.err = errors.Wrap(err, "canal get master pos when start")
+			return
+		}
+	}
+	ins.err = ins.Canal.RunFrom(pos)
+}
+
+// Close close instance
+func (ins *Instance) Close() {
+	if ins.err != nil {
+		return
+	}
+	ins.Canal.Close()
+	for _, t := range ins.targets {
+		t.close()
+	}
+	ins.closed = true
+	ins.err = errors.New("canal closed")
+}
+
+// Check filter row event
+func (ins *Instance) Check(ev *canal.RowsEvent) (ts []*Target) {
+	for _, t := range ins.targets {
+		if t.compare(ev.Table.Schema, ev.Table.Name, ev.Action) {
+			ts = append(ts, t)
+		}
+	}
+	return
+}
+
+func (ins *Instance) String() string {
+	return ins.config.Addr
+}
+
+// OnRotate OnRotate
+func (ins *Instance) OnRotate(re *replication.RotateEvent) error {
+	log.Info("OnRotate binlog addr(%s) rotate binname(%s) pos(%d)", ins.config.Addr, re.NextLogName, re.Position)
+	return nil
+}
+
+// OnDDL OnDDL
+func (ins *Instance) OnDDL(pos mysql.Position, qe *replication.QueryEvent) error {
+	log.Info("OnDDL binlog addr(%s) ddl binname(%s) pos(%d)", ins.config.Addr, pos.Name, pos.Pos)
+	return nil
+}
+
+// OnXID OnXID
+func (ins *Instance) OnXID(mysql.Position) error {
+	return nil
+}
+
+//OnGTID OnGTID
+func (ins *Instance) OnGTID(mysql.GTIDSet) error {
+	return nil
+}
+
+// OnPosSynced OnPosSynced
+func (ins *Instance) OnPosSynced(pos mysql.Position, force bool) error {
+	return ins.master.Save(pos, force)
+}
+
+// OnRow send the envent to table
+func (ins *Instance) OnRow(ev *canal.RowsEvent) error {
+	for _, t := range ins.Check(ev) {
+		t.send(ev)
+	}
+	if stats != nil {
+		stats.Incr("syncer_counter", ins.String(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+		stats.State("delay_syncer", ins.delay(), ins.String(), ev.Table.Schema, "", "")
+	}
+	ins.latestTimestamp = time.Now().Unix()
+	return nil
+}
+
+// Error returns instance error.
+func (ins *Instance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *Instance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}

app/infra/canal/service/master.go

@@ -0,0 +1,145 @@
+package service
+
+import (
+	"sync"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/client"
+	"github.com/siddontang/go-mysql/mysql"
+)
+
+type dbMasterInfo struct {
+	c *conf.MasterInfoConfig
+
+	addr    string
+	binName string
+	binPos  uint32
+
+	l sync.RWMutex
+
+	lastSaveTime time.Time
+}
+
+func newDBMasterInfo(addr string, c *conf.MasterInfoConfig) (*dbMasterInfo, error) {
+	m := &dbMasterInfo{c: c, addr: addr}
+	conn, err := client.Connect(c.Addr, c.User, c.Password, c.DBName)
+	if err != nil {
+		log.Error("db master info client error(%v)", err)
+		return nil, errors.Trace(err)
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	r, err := conn.Execute("SELECT addr,bin_name,bin_pos FROM master_info WHERE addr=?", addr)
+	if err != nil {
+		log.Error("new db load master.info error(%v)", err)
+		return nil, errors.Trace(err)
+	}
+	if r.RowNumber() == 0 {
+		if m.c.Timeout > 0 {
+			conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+		}
+		if _, err = conn.Execute("INSERT INTO master_info (addr,bin_name,bin_pos) VALUE (?,'',0)", addr); err != nil {
+			log.Error("insert master.info error(%v)", err)
+			return nil, errors.Trace(err)
+		}
+	} else {
+		m.addr, _ = r.GetStringByName(0, "addr")
+		m.binName, _ = r.GetStringByName(0, "bin_name")
+		bpos, _ := r.GetIntByName(0, "bin_pos")
+		m.binPos = uint32(bpos)
+	}
+	return m, nil
+}
+
+func (m *dbMasterInfo) Save(pos mysql.Position, force bool) error {
+	n := time.Now()
+	if !force && n.Sub(m.lastSaveTime) < 2*time.Second {
+		return nil
+	}
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("db master info client error(%v)", err)
+		return errors.Trace(err)
+	}
+	defer conn.Close()
+
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	if _, err = conn.Execute("UPDATE master_info SET bin_name=?,bin_pos=? WHERE addr=?", pos.Name, pos.Pos, m.addr); err != nil {
+		log.Error("db save master info error(%v)", err)
+		return errors.Trace(err)
+	}
+	m.lastSaveTime = n
+	return nil
+}
+
+func (m *dbMasterInfo) Pos() mysql.Position {
+	var pos mysql.Position
+	m.l.RLock()
+	pos.Name = m.binName
+	pos.Pos = m.binPos
+	m.l.RUnlock()
+	return pos
+}
+
+type hbaseInfo struct {
+	c *conf.MasterInfoConfig
+
+	name string
+}
+
+func newHBaseInfo(name string, c *conf.MasterInfoConfig) (*hbaseInfo, error) {
+	m := &hbaseInfo{c: c, name: name}
+	return m, nil
+}
+
+func (m *hbaseInfo) LatestTs(table string) (lts uint64) {
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("db hbase info client error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	r, err := conn.Execute("SELECT latest_ts FROM hbase_info WHERE cluster_name=? AND table_name=?", m.name, table)
+	if err != nil {
+		log.Error("new db load hbase.info error(%v)", err)
+		return
+	}
+	if r.RowNumber() == 0 {
+		if m.c.Timeout > 0 {
+			conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+		}
+		if _, err = conn.Execute("INSERT INTO hbase_info (cluster_name,table_name,latest_ts) VALUE (?,?,0)", m.name, table); err != nil {
+			log.Error("insert hbase.info error(%v)", err)
+			return
+		}
+	} else {
+		lts, _ = r.GetUintByName(0, "latest_ts")
+	}
+	return 0
+}
+
+func (m *hbaseInfo) Save(table string, latestTs uint64) {
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("save hbase info client error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	if _, err = conn.Execute("UPDATE hbase_info SET latest_ts=? WHERE cluster_name=? AND table_name=?", latestTs, m.name, table); err != nil {
+		log.Error("save hbase info error(%v)", err)
+	}
+}

app/infra/canal/service/reader/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "offset.go",
+        "reader.go",
+    ],
+    importpath = "go-common/app/infra/canal/service/reader",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/log:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/service/reader/offset.go

@@ -0,0 +1,173 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reader
+
+import (
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+	pkgerr "github.com/pkg/errors"
+)
+
+// KafkaSeeker seeks offset in kafka topics by given condition
+type KafkaSeeker struct {
+	consumer sarama.Consumer
+	client   sarama.Client
+}
+
+// NewKafkaSeeker creates an instance of KafkaSeeker
+func NewKafkaSeeker(addr []string, config *sarama.Config) (*KafkaSeeker, error) {
+	client, err := sarama.NewClient(addr, config)
+	if err != nil {
+		return nil, pkgerr.WithStack(err)
+	}
+
+	consumer, err := sarama.NewConsumerFromClient(client)
+	if err != nil {
+		return nil, pkgerr.WithStack(err)
+	}
+
+	s := &KafkaSeeker{
+		client:   client,
+		consumer: consumer,
+	}
+
+	return s, nil
+}
+
+// Close releases resources of KafkaSeeker
+func (ks *KafkaSeeker) Close() {
+	ks.consumer.Close()
+	ks.client.Close()
+}
+
+// Seek seeks the first offset which binlog CommitTs bigger than ts
+func (ks *KafkaSeeker) Seek(topic string, ts int64, partitions []int32) (offsets []int64, err error) {
+	if len(partitions) == 0 {
+		partitions, err = ks.consumer.Partitions(topic)
+		if err != nil {
+			log.Error("tidb get partitions from topic %s error %v", topic, err)
+			return nil, pkgerr.WithStack(err)
+		}
+	}
+
+	offsets, err = ks.seekOffsets(topic, partitions, ts)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		log.Error("tidb seek offsets error %v", err)
+	}
+	return
+}
+
+func (ks *KafkaSeeker) getTSFromMSG(msg *sarama.ConsumerMessage) (ts int64, err error) {
+	binlog := new(pb.Binlog)
+	err = binlog.Unmarshal(msg.Value)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	return binlog.CommitTs, nil
+}
+
+// seekOffsets returns all valid offsets in partitions
+func (ks *KafkaSeeker) seekOffsets(topic string, partitions []int32, pos int64) ([]int64, error) {
+	offsets := make([]int64, len(partitions))
+	for _, partition := range partitions {
+		start, err := ks.client.GetOffset(topic, partition, sarama.OffsetOldest)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+
+		end, err := ks.client.GetOffset(topic, partition, sarama.OffsetNewest)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+
+		offset, err := ks.seekOffset(topic, partition, start, end-1, pos)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+		offsets[partition] = offset
+	}
+
+	return offsets, nil
+}
+
+func (ks *KafkaSeeker) seekOffset(topic string, partition int32, start int64, end int64, ts int64) (offset int64, err error) {
+	startTS, err := ks.getTSAtOffset(topic, partition, start)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	if ts < startTS {
+		log.Warn("given ts %v is smaller than oldest message's ts %v, some binlogs may lose", ts, startTS)
+		offset = start
+		return
+	} else if ts == startTS {
+		offset = start + 1
+		return
+	}
+
+	for start < end {
+		mid := (end-start)/2 + start
+		var midTS int64
+		midTS, err = ks.getTSAtOffset(topic, partition, mid)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return
+		}
+
+		if midTS <= ts {
+			start = mid + 1
+		} else {
+			end = mid
+		}
+	}
+
+	var endTS int64
+	endTS, err = ks.getTSAtOffset(topic, partition, end)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	if endTS <= ts {
+		return sarama.OffsetNewest, nil
+	}
+
+	return end, nil
+}
+
+func (ks *KafkaSeeker) getTSAtOffset(topic string, partition int32, offset int64) (ts int64, err error) {
+	pc, err := ks.consumer.ConsumePartition(topic, partition, offset)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+	defer pc.Close()
+
+	for msg := range pc.Messages() {
+		ts, err = ks.getTSFromMSG(msg)
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	panic("unreachable")
+}

app/infra/canal/service/reader/reader.go

@@ -0,0 +1,174 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reader
+
+import (
+	"fmt"
+
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+	pkgerr "github.com/pkg/errors"
+)
+
+func init() {
+	// log.SetLevel(log.LOG_LEVEL_NONE)
+	sarama.MaxResponseSize = 1 << 30
+}
+
+// Config for Reader
+type Config struct {
+	KafkaAddr []string
+	// the CommitTs of binlog return by reader will bigger than the config CommitTs
+	CommitTS  int64
+	Offset    int64 // start at kafka offset
+	ClusterID string
+	Name      string
+}
+
+// Message read from reader
+type Message struct {
+	Binlog *pb.Binlog
+	Offset int64 // kafka offset
+}
+
+// Reader to read binlog from kafka
+type Reader struct {
+	cfg    *Config
+	client sarama.Client
+
+	msgs      chan *Message
+	stop      chan struct{}
+	clusterID string
+}
+
+func (r *Reader) getTopic() (string, int32) {
+	return r.cfg.ClusterID + "_obinlog", 0
+}
+
+func (r *Reader) name() string {
+	return fmt.Sprintf("%s-%s", r.cfg.Name, r.cfg.ClusterID)
+}
+
+// NewReader creates an instance of Reader
+func NewReader(cfg *Config) (r *Reader, err error) {
+	r = &Reader{
+		cfg:       cfg,
+		stop:      make(chan struct{}),
+		msgs:      make(chan *Message, 1024),
+		clusterID: cfg.ClusterID,
+	}
+
+	r.client, err = sarama.NewClient(r.cfg.KafkaAddr, nil)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		r = nil
+		return
+	}
+	if (r.cfg.Offset == 0) && (r.cfg.CommitTS > 0) {
+		r.cfg.Offset, err = r.getOffsetByTS(r.cfg.CommitTS)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			r = nil
+			return
+		}
+		log.Info("tidb %s: set offset to: %v", r.name(), r.cfg.Offset)
+	}
+	return
+}
+
+// Close shuts down the reader
+func (r *Reader) Close() {
+	close(r.stop)
+
+	r.client.Close()
+}
+
+// Messages returns a chan that contains unread buffered message
+func (r *Reader) Messages() (msgs <-chan *Message) {
+	return r.msgs
+}
+
+func (r *Reader) getOffsetByTS(ts int64) (offset int64, err error) {
+	seeker, err := NewKafkaSeeker(r.cfg.KafkaAddr, nil)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	topic, partition := r.getTopic()
+	offsets, err := seeker.Seek(topic, ts, []int32{partition})
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	offset = offsets[0]
+
+	return
+}
+
+// Run start consume msg
+func (r *Reader) Run() {
+	offset := r.cfg.Offset
+	log.Info("tidb %s start at offset: %v", r.name(), offset)
+
+	consumer, err := sarama.NewConsumerFromClient(r.client)
+	if err != nil {
+		log.Error("tidb %s NewConsumerFromClient err: %v", r.name(), err)
+		return
+	}
+	defer consumer.Close()
+	topic, partition := r.getTopic()
+	partitionConsumer, err := consumer.ConsumePartition(topic, partition, offset)
+	if err != nil {
+		log.Error("tidb %s ConsumePartition err: %v", r.name(), err)
+		return
+	}
+	defer partitionConsumer.Close()
+	for {
+		select {
+		case <-r.stop:
+			partitionConsumer.Close()
+			close(r.msgs)
+			log.Info("tidb %s reader stop to run", r.name())
+			return
+		case kmsg, ok := <-partitionConsumer.Messages():
+			if !ok {
+				close(r.msgs)
+				log.Info("tidb %s reader stop to run because partitionConsumer close", r.name())
+				return
+			}
+			if kmsg == nil {
+				continue
+			}
+			log.Info("tidb %s get kmsg offset: %v", r.name(), kmsg.Offset)
+			binlog := new(pb.Binlog)
+			err := binlog.Unmarshal(kmsg.Value)
+			if err != nil {
+				log.Warn("%s unmarshal err %+v", r.name(), err)
+				continue
+			}
+			if r.cfg.CommitTS > 0 && binlog.CommitTs <= r.cfg.CommitTS {
+				log.Warn("%s skip binlog CommitTs: ", r.name(), binlog.CommitTs)
+				continue
+			}
+			r.msgs <- &Message{
+				Binlog: binlog,
+				Offset: kmsg.Offset,
+			}
+		}
+	}
+}

app/infra/canal/service/target.go

@@ -0,0 +1,329 @@
+package service
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"hash/crc32"
+	"strings"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	"go-common/app/infra/canal/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	"github.com/pkg/errors"
+	"github.com/siddontang/go-mysql/canal"
+)
+
+var (
+	errInvalidAction = errors.New("invalid rows action")
+	errInvalidUpdate = errors.New("invalid update rows event")
+	errBinlogFormat  = errors.New("binlog format failed")
+)
+
+type producer interface {
+	Rows(int64)
+	Send(context.Context, string, interface{}) error
+	Close()
+	Name() string
+}
+
+type databusP struct {
+	group, topic string
+	*databus.Databus
+}
+
+func (d *databusP) Rows(b int64) {
+	// ignore
+}
+
+func (d *databusP) Send(c context.Context, key string, data interface{}) error {
+	return d.Databus.Send(c, key, data)
+}
+
+func (d *databusP) Name() string {
+	return fmt.Sprintf("databus:group(%s)topic(%s)", d.group, d.topic)
+}
+
+func (d *databusP) Close() {
+	d.Databus.Close()
+}
+
+// infocP infoc producer
+type infocP struct {
+	taskID string
+	*infoc.Infoc
+}
+
+// Rows rows
+func (i *infocP) Rows(b int64) {
+	i.Infoc.Rows(b)
+}
+
+// Send send msg
+func (i *infocP) Send(c context.Context, key string, data interface{}) error {
+	return i.Infoc.Send(c, key, data)
+}
+
+// Name infoc name
+func (i *infocP) Name() string {
+	return fmt.Sprintf("infoc(%s)", i.taskID)
+}
+
+// Close close infoc
+func (i *infocP) Close() {
+	i.Infoc.Flush()
+	i.Infoc.Close()
+}
+
+// Target databus target
+type Target struct {
+	producers []producer
+	eventLen  uint32
+	events    []chan *canal.RowsEvent
+	db        *conf.Database
+
+	closed bool
+}
+
+// NewTarget new databus target
+func NewTarget(db *conf.Database) (t *Target) {
+	t = &Target{
+		db:       db,
+		eventLen: uint32(len(db.CTables)),
+	}
+	t.events = make([]chan *canal.RowsEvent, t.eventLen)
+	if db.Databus != nil {
+		t.producers = append(t.producers, &databusP{group: db.Databus.Group, topic: db.Databus.Topic, Databus: databus.New(db.Databus)})
+	}
+	if db.Infoc != nil {
+		t.producers = append(t.producers, &infocP{taskID: db.Infoc.TaskID, Infoc: infoc.New(db.Infoc)})
+	}
+	for i := 0; i < int(t.eventLen); i++ {
+		ch := make(chan *canal.RowsEvent, 1024)
+		t.events[i] = ch
+		go t.proc(ch)
+	}
+	return
+}
+
+// compare check if the binlog event is needed
+// check the table name and schame
+func (t *Target) compare(schame, table, action string) bool {
+	if t.db.Schema == schame {
+		for _, ctb := range t.db.CTables {
+			for _, tb := range ctb.Tables {
+				if table == tb {
+					for _, act := range ctb.OmitAction {
+						if act == action { // NOTE: omit action
+							return false
+						}
+					}
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// send send rows event into event chans
+// and hash by table%concurrency.
+func (t *Target) send(ev *canal.RowsEvent) {
+	yu := crc32.ChecksumIEEE([]byte(ev.Table.Name))
+	t.events[yu%t.eventLen] <- ev
+}
+
+func (t *Target) close() {
+	for _, p := range t.producers {
+		p.Close()
+	}
+	t.closed = true
+}
+
+// proc aync method for transfer the binlog data
+// when connection is bad, just refresh it with retry
+func (t *Target) proc(ch chan *canal.RowsEvent) {
+	type pData struct {
+		datas    []*model.Data
+		producer producer
+	}
+	var (
+		err         error
+		normalDatas []*pData
+		errorDatas  []*pData
+		ev          *canal.RowsEvent
+	)
+	for {
+		if t.closed {
+			return
+		}
+		if len(errorDatas) != 0 {
+			normalDatas = errorDatas
+			errorDatas = errorDatas[0:0]
+			time.Sleep(time.Second)
+		} else {
+			ev = <-ch
+			var datas []*model.Data
+			if datas, err = makeDatas(ev, t.db.TableMap); err != nil {
+				log.Error("makeData(%v) error(%v)", ev, err)
+				continue
+			}
+			normalDatas = normalDatas[0:0]
+			for _, p := range t.producers {
+				p.Rows(int64(len(datas)))
+				normalDatas = append(normalDatas, &pData{datas: datas, producer: p})
+				if stats != nil {
+					stats.Incr("send_counter", p.Name(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+				}
+			}
+		}
+		for _, pd := range normalDatas {
+			var eDatas []*model.Data
+			for _, data := range pd.datas {
+				if err = pd.producer.Send(context.TODO(), data.Key, data); err != nil {
+					// retry pub error data
+					eDatas = append(eDatas, data)
+					continue
+				}
+				log.Info("%s pub(key:%s, value:%+v) succeed", pd.producer.Name(), data.Key, data)
+			}
+			if len(eDatas) > 0 {
+				errorDatas = append(errorDatas, &pData{datas: eDatas, producer: pd.producer})
+				if stats != nil && ev != nil {
+					stats.Incr("retry_counter", pd.producer.Name(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+				}
+				log.Error("%s scheme(%s) pub fail,add to retry", pd.producer.Name(), ev.Table.Schema)
+			}
+		}
+	}
+}
+
+// makeDatas parse the binlog event and return the model.Data struct
+// a little bit cautious about the binlog type
+// if the type is update:
+//     the old value and new value will alternate appearing in the event.Rows
+func makeDatas(e *canal.RowsEvent, tbMap map[string]*conf.Addition) (datas []*model.Data, err error) {
+	var (
+		rowsLen     = len(e.Rows)
+		firstRowLen = len(e.Rows[0])
+		lenCol      = len(e.Table.Columns)
+	)
+	if rowsLen == 0 || firstRowLen == 0 || firstRowLen != lenCol {
+		log.Error("rows length(%d) first row length(%d) columns length(%d)", rowsLen, firstRowLen, lenCol)
+		err = errBinlogFormat
+		return
+	}
+	datas = make([]*model.Data, 0, rowsLen)
+	switch e.Action {
+	case canal.InsertAction, canal.DeleteAction:
+		for _, values := range e.Rows {
+			var keys []string
+			data := &model.Data{
+				Action: e.Action,
+				Table:  e.Table.Name,
+				// the first primary key as the kafka key
+				Key: fmt.Sprint(values[0]),
+				New: make(map[string]interface{}, lenCol),
+			}
+			for i, c := range e.Table.Columns {
+				if c.IsUnsigned {
+					values[i] = unsignIntCase(values[i])
+				}
+				if strings.Contains(c.RawType, "binary") {
+					if bs, ok := values[i].(string); ok {
+						values[i] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+				}
+				data.New[c.Name] = values[i]
+			}
+			// set kafka key and remove omit columns data
+			addition, ok := tbMap[e.Table.Name]
+			if ok {
+				for _, omit := range addition.OmitField {
+					delete(data.New, omit)
+				}
+				for _, primary := range addition.PrimaryKey {
+					if _, ok := data.New[primary]; ok {
+						keys = append(keys, fmt.Sprint(data.New[primary]))
+					}
+				}
+			}
+			if len(keys) != 0 {
+				data.Key = strings.Join(keys, ",")
+			}
+			datas = append(datas, data)
+		}
+	case canal.UpdateAction:
+		if rowsLen%2 != 0 {
+			err = errInvalidUpdate
+			return
+		}
+		for i := 0; i < rowsLen; i += 2 {
+			var keys []string
+			data := &model.Data{
+				Action: e.Action,
+				Table:  e.Table.Name,
+				// the first primary key as the kafka key
+				Key: fmt.Sprint(e.Rows[i][0]),
+				Old: make(map[string]interface{}, lenCol),
+				New: make(map[string]interface{}, lenCol),
+			}
+			for j, c := range e.Table.Columns {
+				if c.IsUnsigned {
+					e.Rows[i][j] = unsignIntCase(e.Rows[i][j])
+					e.Rows[i+1][j] = unsignIntCase(e.Rows[i+1][j])
+				}
+				if strings.Contains(c.RawType, "binary") {
+					if bs, ok := e.Rows[i][j].(string); ok {
+						e.Rows[i][j] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+					if bs, ok := e.Rows[i+1][j].(string); ok {
+						e.Rows[i+1][j] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+				}
+				data.Old[c.Name] = e.Rows[i][j]
+				data.New[c.Name] = e.Rows[i+1][j]
+			}
+			// set kafka key and remove omit columns data
+			addition, ok := tbMap[e.Table.Name]
+			if ok {
+				for _, omit := range addition.OmitField {
+					delete(data.New, omit)
+					delete(data.Old, omit)
+				}
+				for _, primary := range addition.PrimaryKey {
+					if _, ok := data.New[primary]; ok {
+						keys = append(keys, fmt.Sprint(data.New[primary]))
+					}
+				}
+			}
+			if len(keys) != 0 {
+				data.Key = strings.Join(keys, ",")
+			}
+			datas = append(datas, data)
+		}
+	default:
+		err = errInvalidAction
+	}
+	return
+}
+
+func unsignIntCase(i interface{}) (v interface{}) {
+	switch si := i.(type) {
+	case int8:
+		v = uint8(si)
+	case int16:
+		v = uint16(si)
+	case int32:
+		v = uint32(si)
+	case int64:
+		v = uint64(si)
+	default:
+		v = i
+	}
+	return
+}

app/infra/canal/service/tidb_check.go

@@ -0,0 +1,65 @@
+package service
+
+import (
+	"regexp"
+
+	"go-common/library/log"
+)
+
+func (ins *tidbInstance) check() (err error) {
+	for _, db := range ins.config.Databases {
+		for _, ctable := range db.CTables {
+			if _, err = regexp.Compile(ctable.Name); err != nil {
+				log.Error("regexp.Compile(%s) error(%v)", ctable.Name, err)
+				return
+			}
+		}
+	}
+	return
+}
+
+func (ins *tidbInstance) getTable(dbName, table string) *Table {
+	if ins.ignoreTables[dbName] != nil && ins.ignoreTables[dbName][table] {
+		return nil
+	}
+	if ins.tables[dbName] != nil && ins.tables[dbName][table] != nil {
+		return ins.tables[dbName][table]
+	}
+	var regex *regexp.Regexp
+	for _, db := range ins.config.Databases {
+		if db.Schema != dbName {
+			continue
+		}
+		for _, ctable := range db.CTables {
+			regex, _ = regexp.Compile(ctable.Name)
+			if !regex.MatchString(table) {
+				continue
+			}
+			if ins.tables[dbName] == nil {
+				ins.tables[dbName] = make(map[string]*Table)
+			}
+			t := &Table{
+				PrimaryKey: ctable.PrimaryKey,
+				OmitField:  make(map[string]bool),
+				OmitAction: make(map[string]bool),
+				name:       ctable.Name,
+				ch:         make(chan *msg, 1024),
+			}
+			for _, action := range ctable.OmitAction {
+				t.OmitAction[action] = true
+			}
+			for _, field := range ctable.OmitField {
+				t.OmitField[field] = true
+			}
+			ins.waitTable.Add(1)
+			go ins.proc(t.ch)
+			ins.tables[dbName][table] = t
+			return t
+		}
+	}
+	if ins.ignoreTables[dbName] == nil {
+		ins.ignoreTables[dbName] = make(map[string]bool)
+	}
+	ins.ignoreTables[dbName][table] = true
+	return nil
+}

app/infra/canal/service/tidb_data.go

@@ -0,0 +1,123 @@
+package service
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"go-common/app/infra/canal/model"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+// lower case column field type in mysql
+// https://dev.mysql.com/doc/refman/8.0/en/data-types.html
+// for numeric type: int bigint smallint tinyint float double decimal bit
+// for string type: text longtext mediumtext char tinytext varchar
+// blob longblog mediumblog binary tinyblob varbinary
+// enum set
+// for json type: json
+
+// for text and char type, string_value is set
+// for blob and binary type, bytes_value is set
+// for enum, set, uint64_value is set
+// for json, bytes_value is set
+
+func tidbMakeData(m *msg) (data *model.Data, err error) {
+	action := m.mu.GetType()
+	if (action != pb.MutationType_Insert) && (action != pb.MutationType_Delete) && (action != pb.MutationType_Update) {
+		err = errInvalidAction
+		return
+	}
+	data = &model.Data{
+		Action: strings.ToLower(action.String()),
+		Table:  m.table,
+	}
+	var keys []string
+	switch action {
+	case pb.MutationType_Insert, pb.MutationType_Delete:
+		var values = m.mu.GetRow().GetColumns()
+		for i, c := range m.columns {
+			for _, key := range m.keys {
+				if c.Name == key {
+					keys = append(keys, columnToString(values[i]))
+					break
+				}
+			}
+			if m.ignore[c.Name] {
+				continue
+			}
+			if data.New == nil {
+				data.New = make(map[string]interface{}, len(m.columns))
+			}
+			if strings.Contains(c.GetMysqlType(), "binary") {
+				data.New[c.Name] = base64.StdEncoding.EncodeToString(values[i].GetBytesValue())
+				continue
+			}
+			data.New[c.Name] = columnToValue(values[i])
+		}
+	case pb.MutationType_Update:
+		if m.mu.Row == nil || m.mu.ChangeRow == nil {
+			err = errInvalidUpdate
+			return
+		}
+		var oldValues = m.mu.GetChangeRow().GetColumns()
+		var newValues = m.mu.GetRow().GetColumns()
+		for i, c := range m.columns {
+			for _, key := range m.keys {
+				if c.Name == key {
+					keys = append(keys, columnToString(newValues[i]))
+					break
+				}
+			}
+			if m.ignore[c.Name] {
+				continue
+			}
+			if data.New == nil {
+				data.New = make(map[string]interface{}, len(m.columns))
+			}
+			if data.Old == nil {
+				data.Old = make(map[string]interface{}, len(m.columns))
+			}
+			if strings.Contains(c.GetMysqlType(), "binary") {
+				data.Old[c.Name] = base64.StdEncoding.EncodeToString(oldValues[i].GetBytesValue())
+				data.New[c.Name] = base64.StdEncoding.EncodeToString(newValues[i].GetBytesValue())
+				continue
+			}
+			data.Old[c.Name] = columnToValue(oldValues[i])
+			data.New[c.Name] = columnToValue(newValues[i])
+		}
+	}
+	if len(keys) == 0 {
+		data.Key = columnToString(m.mu.GetRow().GetColumns()[0])
+	} else {
+		data.Key = strings.Join(keys, ",")
+	}
+	if data.New == nil && data.Old == nil {
+		data = nil
+	}
+	return
+}
+
+func columnToValue(c *pb.Column) interface{} {
+	if c.GetIsNull() {
+		return nil
+	}
+	if c.Int64Value != nil {
+		return c.GetInt64Value()
+	}
+	if c.Uint64Value != nil {
+		return c.GetUint64Value()
+	}
+	if c.DoubleValue != nil {
+		return c.GetDoubleValue()
+	}
+	if c.StringValue != nil {
+		return c.GetStringValue()
+	}
+	return c.GetBytesValue()
+}
+
+func columnToString(c *pb.Column) string {
+	return fmt.Sprint(columnToValue(c))
+}

app/infra/canal/service/tidb_data_test.go

@@ -0,0 +1,188 @@
+package service
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"go-common/app/infra/canal/model"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+func Test_tidbMakeData(t *testing.T) {
+	insertMsg, insertData := prepareInsertData()
+	delMsg, delData := prepareDeleteData()
+	updateMsg, updateData := prepareUpdateData()
+	updateMsg2, updateData2 := prepareUpdateData2()
+	type args struct {
+		m *msg
+	}
+	tests := []struct {
+		name     string
+		args     args
+		wantData *model.Data
+		wantErr  bool
+	}{
+		{name: "insert", args: args{m: insertMsg}, wantData: insertData, wantErr: false},
+		{name: "delete", args: args{m: delMsg}, wantData: delData, wantErr: false},
+		{name: "update", args: args{m: updateMsg}, wantData: updateData, wantErr: false},
+		{name: "update2", args: args{m: updateMsg2}, wantData: updateData2, wantErr: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotData, err := tidbMakeData(tt.args.m)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("tidbMakeData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			gotjson, _ := json.Marshal(gotData)
+			wantjson, _ := json.Marshal(tt.wantData)
+			if !reflect.DeepEqual(gotjson, wantjson) {
+				t.Errorf("tidbMakeData() = %v, want %v", gotData, tt.wantData)
+			}
+		})
+	}
+}
+
+func prepareInsertData() (*msg, *model.Data) {
+	insertPb := &pb.Binlog{}
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846216359608325,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":0,"row":{"columns":[{"uint64_value":1},{"string_value":"2018-10-26 18:50:57"},{"string_value":"2018-10-26 18:50:57"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":1},{"uint64_value":8167601},{"uint64_value":1}]}}]}]}}`), insertPb)
+	insertMsg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          insertPb.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"id", "mid"},
+		columns:     insertPb.DmlData.Tables[0].ColumnInfo,
+	}
+	insertData := &model.Data{
+		Action: "insert",
+		Table:  "counts",
+		Key:    "1,8167601",
+		New: map[string]interface{}{
+			"id":          1,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  1,
+			"mid":         8167601,
+			"type":        1,
+			"mtime":       "2018-10-26 18:50:57",
+		},
+	}
+	return insertMsg, insertData
+}
+
+func prepareDeleteData() (*msg, *model.Data) {
+	pbData := &pb.Binlog{}
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846189135953921,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":2,"row":{"columns":[{"uint64_value":7},{"string_value":"2018-01-11 12:19:10"},{"string_value":"2018-01-11 12:19:10"},{"uint64_value":2},{"uint64_value":0},{"uint64_value":897},{"uint64_value":27515233},{"uint64_value":1}]}}]}]}}`), pbData)
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          pbData.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"message_id"},
+		columns:     pbData.DmlData.Tables[0].ColumnInfo,
+	}
+	data := &model.Data{
+		Action: "delete",
+		Table:  "counts",
+		Key:    "897",
+		New: map[string]interface{}{
+			"id":          7,
+			"business_id": 2,
+			"origin_id":   0,
+			"message_id":  897,
+			"mid":         27515233,
+			"type":        1,
+			"mtime":       "2018-01-11 12:19:10",
+		},
+	}
+	return msg, data
+}
+
+func prepareUpdateData() (*msg, *model.Data) {
+	pbData := &pb.Binlog{}
+	// update likes type from 1 to 0
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846165844459523,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":1,"row":{"columns":[{"uint64_value":4},{"string_value":"2018-10-26 18:47:44"},{"string_value":"2017-12-22 15:05:29"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":46997},{"uint64_value":88895031},{"uint64_value":0}]},"change_row":{"columns":[{"uint64_value":4},{"string_value":"2017-12-22 15:55:52"},{"string_value":"2017-12-22 15:05:29"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":46997},{"uint64_value":88895031},{"uint64_value":1}]}}]}]}}`), pbData)
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          pbData.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"mid"},
+		columns:     pbData.DmlData.Tables[0].ColumnInfo,
+	}
+	data := &model.Data{
+		Action: "update",
+		Table:  "counts",
+		Key:    "88895031",
+		Old: map[string]interface{}{
+			"id":          4,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  46997,
+			"mid":         88895031,
+			"type":        1,
+			"mtime":       "2017-12-22 15:55:52",
+		},
+		New: map[string]interface{}{
+			"id":          4,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  46997,
+			"mid":         88895031,
+			"type":        0,
+			"mtime":       "2018-10-26 18:47:44",
+		},
+	}
+	return msg, data
+}
+
+func prepareUpdateData2() (*msg, *model.Data) {
+	muJson := `{"type":1,"row":{"columns":[{"uint64_value":0},{"string_value":"2018-11-03 17:07:44"},{"string_value":"2018-11-03 14:55:38"},{"uint64_value":3},{"uint64_value":0},{"uint64_value":88889},{"uint64_value":3},{"uint64_value":0},{"int64_value":0},{"int64_value":0},{"uint64_value":8167601}]},"change_row":{"columns":[{"uint64_value":0},{"string_value":"2018-11-03 16:36:39"},{"string_value":"2018-11-03 14:55:38"},{"uint64_value":3},{"uint64_value":0},{"uint64_value":88889},{"uint64_value":2},{"uint64_value":0},{"int64_value":0},{"int64_value":0},{"uint64_value":8167601}]}}`
+	columnJson := `[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"likes_count","mysql_type":"int","is_primary_key":false},{"name":"dislikes_count","mysql_type":"int","is_primary_key":false},{"name":"likes_change","mysql_type":"bigint","is_primary_key":false},{"name":"dislikes_change","mysql_type":"bigint","is_primary_key":false},{"name":"up_mid","mysql_type":"int","is_primary_key":false}]`
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		keys:        []string{"message_id"},
+	}
+	json.Unmarshal([]byte(columnJson), &msg.columns)
+	json.Unmarshal([]byte(muJson), &msg.mu)
+	data := &model.Data{
+		Action: "update",
+		Table:  "counts",
+		Key:    "88889",
+		Old: map[string]interface{}{
+			"ctime":           "2018-11-03 14:55:38",
+			"origin_id":       0,
+			"dislikes_count":  0,
+			"up_mid":          8167601,
+			"id":              0,
+			"mtime":           "2018-11-03 16:36:39",
+			"likes_count":     2,
+			"likes_change":    0,
+			"dislikes_change": 0,
+			"business_id":     3,
+			"message_id":      88889,
+		},
+		New: map[string]interface{}{
+			"likes_count":     3,
+			"dislikes_count":  0,
+			"likes_change":    0,
+			"id":              0,
+			"mtime":           "2018-11-03 17:07:44",
+			"ctime":           "2018-11-03 14:55:38",
+			"origin_id":       0,
+			"message_id":      88889,
+			"business_id":     3,
+			"dislikes_change": 0,
+			"up_mid":          8167601,
+		},
+	}
+	return msg, data
+}

app/infra/canal/service/tidb_instance.go

@@ -0,0 +1,186 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	"go-common/app/infra/canal/model"
+	"go-common/app/infra/canal/service/reader"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+type tidbInstance struct {
+	canal  *Canal
+	config *conf.TiDBInsConf
+	// one instance can have lots of target different by schema and table
+	targets      map[string][]producer
+	latestOffset int64
+	// scan latest timestamp, be used check delay
+	latestTimestamp int64
+	reader          *reader.Reader
+	err             error
+	closed          bool
+	tables          map[string]map[string]*Table
+	ignoreTables    map[string]map[string]bool
+	waitConsume     sync.WaitGroup
+	waitTable       sync.WaitGroup
+}
+
+// Table db table
+type Table struct {
+	PrimaryKey []string        // kafka msg key
+	OmitField  map[string]bool // field will be ignored in table
+	OmitAction map[string]bool // action will be ignored in table
+	name       string
+	ch         chan *msg
+}
+
+type msg struct {
+	db          string
+	table       string
+	tableRegexp string
+	mu          *pb.TableMutation
+	ignore      map[string]bool
+	keys        []string
+	columns     []*pb.ColumnInfo
+}
+
+// newTiDBInstance new canal instance
+func newTiDBInstance(cl *Canal, c *conf.TiDBInsConf) (ins *tidbInstance, err error) {
+	// new instance
+	ins = &tidbInstance{
+		config:       c,
+		canal:        cl,
+		targets:      make(map[string][]producer, len(c.Databases)),
+		tables:       make(map[string]map[string]*Table),
+		ignoreTables: make(map[string]map[string]bool),
+	}
+	cfg := &reader.Config{
+		Name:      c.Name,
+		KafkaAddr: c.Addrs,
+		Offset:    c.Offset,
+		CommitTS:  c.CommitTS,
+		ClusterID: c.ClusterID,
+	}
+	if err = ins.check(); err != nil {
+		return
+	}
+	position, err := cl.dao.TiDBPosition(context.Background(), c.Name)
+	if err == nil && position != nil {
+		cfg.Offset = position.Offset
+		cfg.CommitTS = position.CommitTS
+	}
+	ins.latestOffset = cfg.Offset
+	ins.latestTimestamp = cfg.CommitTS
+	for _, db := range c.Databases {
+		if db.Databus != nil {
+			if ins.targets == nil {
+				ins.targets = make(map[string][]producer)
+			}
+			ins.targets[db.Schema] = append(ins.targets[db.Schema], &databusP{group: db.Databus.Group, topic: db.Databus.Topic, Databus: databus.New(db.Databus)})
+		}
+		if db.Infoc != nil {
+			ins.targets[db.Schema] = append(ins.targets[db.Schema], &infocP{taskID: db.Infoc.TaskID, Infoc: infoc.New(db.Infoc)})
+		}
+	}
+	ins.reader, ins.err = reader.NewReader(cfg)
+	return ins, ins.err
+}
+
+// start start binlog receive
+func (ins *tidbInstance) start() {
+	defer ins.waitConsume.Done()
+	ins.waitConsume.Add(2)
+	go ins.reader.Run()
+	go ins.syncproc()
+	for msg := range ins.reader.Messages() {
+		ins.process(msg)
+	}
+}
+
+// close close instance
+func (ins *tidbInstance) close() {
+	if ins.closed {
+		return
+	}
+	ins.closed = true
+	ins.reader.Close()
+	ins.waitConsume.Wait()
+	for _, tables := range ins.tables {
+		for _, table := range tables {
+			close(table.ch)
+		}
+	}
+	ins.waitTable.Wait()
+	ins.sync()
+}
+
+// String .
+func (ins *tidbInstance) String() string {
+	return fmt.Sprintf("%s-%s", ins.config.Name, ins.config.ClusterID)
+}
+
+func (ins *tidbInstance) process(m *reader.Message) (err error) {
+	if m.Binlog.Type == pb.BinlogType_DDL {
+		log.Info("tidb %s got ddl: %s", ins.String(), m.Binlog.DdlData.String())
+		return
+	}
+	for _, table := range m.Binlog.DmlData.Tables {
+		tb := ins.getTable(table.GetSchemaName(), table.GetTableName())
+		if tb == nil {
+			continue
+		}
+		for _, mu := range table.Mutations {
+			action := strings.ToLower(mu.GetType().String())
+			if tb.OmitAction[action] {
+				continue
+			}
+			tb.ch <- &msg{
+				db:          table.GetSchemaName(),
+				table:       table.GetTableName(),
+				mu:          mu,
+				ignore:      tb.OmitField,
+				keys:        tb.PrimaryKey,
+				columns:     table.ColumnInfo,
+				tableRegexp: tb.name,
+			}
+			if stats != nil {
+				stats.Incr("syncer_counter", ins.String(), table.GetSchemaName(), tb.name, action)
+				stats.State("delay_syncer", ins.delay(), ins.String(), tb.name, "", "")
+			}
+		}
+	}
+	ins.latestOffset = m.Offset
+	ins.latestTimestamp = m.Binlog.CommitTs
+	return nil
+}
+
+// Error returns instance error.
+func (ins *tidbInstance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *tidbInstance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}
+
+func (ins *tidbInstance) sync() {
+	info := &model.TiDBInfo{
+		Name:      ins.config.Name,
+		ClusterID: ins.config.ClusterID,
+		Offset:    ins.latestOffset,
+		CommitTS:  ins.latestTimestamp,
+	}
+	ins.canal.dao.UpdateTiDBPosition(context.Background(), info)
+}

app/infra/canal/service/tidb_proc.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+)
+
+func (ins *tidbInstance) proc(ch chan *msg) {
+	defer ins.waitTable.Done()
+	for {
+		msg, ok := <-ch
+		if !ok {
+			return
+		}
+		data, err := tidbMakeData(msg)
+		if err != nil {
+			log.Error("tidb MakeData(%+v) err: %+v", msg, err)
+			continue
+		}
+		if data == nil {
+			continue
+		}
+		for _, target := range ins.targets[msg.db] {
+			for {
+				if err = target.Send(context.TODO(), data.Key, data); err == nil {
+					stats.Incr("send_counter", target.Name(), msg.db, msg.tableRegexp, data.Action)
+					break
+				}
+				stats.Incr("retry_counter", target.Name(), msg.db, msg.tableRegexp, data.Action)
+				log.Error("tidb %s scheme(%s) pub fail,add to retry", target.Name(), msg.db)
+				time.Sleep(time.Second)
+			}
+			log.Info("tidb %s pub(key:%s, value:%+v) succeed", target.Name(), data.Key, data)
+		}
+	}
+}
+
+func (ins *tidbInstance) syncproc() {
+	defer ins.waitConsume.Done()
+	for {
+		if ins.closed {
+			return
+		}
+		time.Sleep(time.Second * 5)
+		ins.sync()
+	}
+}
+
+func (c *Canal) tidbEventproc() {
+	ech := conf.TiDBEvent()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		c.tidbInsl.Lock()
+		if old, ok := c.tidbInstances[insc.Name]; ok {
+			old.close()
+		}
+		c.tidbInsl.Unlock()
+		ins, err := newTiDBInstance(c, insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.sendWx(fmt.Sprintf("reload tidb canal instance(%s) failed error(%v)", ins.String(), err))
+			continue
+		}
+		c.tidbInsl.Lock()
+		c.tidbInstances[insc.Name] = ins
+		c.tidbInsl.Unlock()
+		go ins.start()
+		log.Info("reload tidb canal instance(%s) success", ins.String())
+		c.sendWx(fmt.Sprintf("reload tidb canal instance(%s) success", ins.String()))
+	}
+}